| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| import logging |
| import os |
| import subprocess |
| import csv |
| import sys |
| import ConfigParser |
| |
| |
| class Util(object): |
| @classmethod |
| def get_logger(cls, logger_name, create_file=False): |
| |
| # create logger for prd_ci |
| log = logging.getLogger(logger_name) |
| log.setLevel(level=logging.INFO) |
| |
| # create formatter and add it to the handlers |
| formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') |
| |
| if create_file: |
| # create file handler for logger. |
| fh = logging.FileHandler('oa.log') |
| fh.setLevel(level=logging.DEBUG) |
| fh.setFormatter(formatter) |
| # reate console handler for logger. |
| ch = logging.StreamHandler() |
| ch.setLevel(level=logging.DEBUG) |
| ch.setFormatter(formatter) |
| |
| # add handlers to logger. |
| if create_file: |
| log.addHandler(fh) |
| |
| log.addHandler(ch) |
| return log |
| |
| @classmethod |
| def get_spot_conf(cls): |
| |
| conf_file = "/etc/spot.conf" |
| config = ConfigParser.ConfigParser() |
| config.readfp(SecHead(open(conf_file))) |
| |
| return config |
| |
| @classmethod |
| def create_oa_folders(cls, type, date): |
| |
| # create date and ingest summary folder structure if they don't' exist. |
| root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| data_type_folder = "{0}/data/{1}/{2}" |
| if not os.path.isdir(data_type_folder.format(root_path, type, date)): os.makedirs( |
| data_type_folder.format(root_path, type, date)) |
| if not os.path.isdir(data_type_folder.format(root_path, type, "ingest_summary")): os.makedirs( |
| data_type_folder.format(root_path, type, "ingest_summary")) |
| |
| # create ipynb folders. |
| ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path, type, date) |
| if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder) |
| |
| # retun path to folders. |
| data_path = data_type_folder.format(root_path, type, date) |
| ingest_path = data_type_folder.format(root_path, type, "ingest_summary") |
| return data_path, ingest_path, ipynb_folder |
| |
| @classmethod |
| def get_ml_results_form_hdfs(cls, hdfs_file_path, local_path): |
| |
| # get results from hdfs. |
| get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path, local_path) |
| subprocess.call(get_results_cmd, shell=True) |
| return get_results_cmd |
| |
| @classmethod |
| def read_results(cls, file, limit, delimiter=','): |
| |
| # read csv results. |
| result_rows = [] |
| with open(file, 'rb') as results_file: |
| csv_reader = csv.reader(results_file, delimiter=delimiter) |
| for i in range(0, int(limit)): |
| try: |
| row = csv_reader.next() |
| except StopIteration: |
| return result_rows |
| result_rows.append(row) |
| return result_rows |
| |
| @classmethod |
| def ip_to_int(self, ip): |
| |
| try: |
| o = map(int, ip.split('.')) |
| res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3] |
| return res |
| |
| except ValueError: |
| return None |
| |
| @classmethod |
| def create_csv_file(cls, full_path_file, content, delimiter=','): |
| with open(full_path_file, 'w+') as u_file: |
| writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter) |
| writer.writerows(content) |
| |
| @classmethod |
| def cast_val(self, value): |
| try: |
| val = int(value) |
| except: |
| try: |
| val = float(value) |
| except: |
| val = str(value) |
| return val |
| |
| |
| class SecHead(object): |
| def __init__(self, fp): |
| self.fp = fp |
| self.sechead = '[conf]\n' |
| |
| def readline(self): |
| if self.sechead: |
| try: |
| return self.sechead |
| finally: |
| self.sechead = None |
| else: |
| return self.fp.readline() |
| |
| |
| class ProgressBar(object): |
| def __init__(self, total, prefix='', sufix='', decimals=2, barlength=60): |
| self._total = total |
| self._prefix = prefix |
| self._sufix = sufix |
| self._decimals = decimals |
| self._bar_length = barlength |
| self._auto_iteration_status = 0 |
| |
| def start(self): |
| self._move_progress_bar(0) |
| |
| def update(self, iterator): |
| self._move_progress_bar(iterator) |
| |
| def auto_update(self): |
| self._auto_iteration_status += 1 |
| self._move_progress_bar(self._auto_iteration_status) |
| |
| def _move_progress_bar(self, iteration): |
| filledLength = int(round(self._bar_length * iteration / float(self._total))) |
| percents = round(100.00 * (iteration / float(self._total)), self._decimals) |
| bar = '#' * filledLength + '-' * (self._bar_length - filledLength) |
| sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix)) |
| sys.stdout.flush() |
| |
| if iteration == self._total: print("\n") |