[OA][PEP 8] reformat utils.py
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 2bed10e..8ac6555 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -22,115 +22,114 @@
import sys
import ConfigParser
+
class Util(object):
-
- @classmethod
- def get_logger(cls,logger_name,create_file=False):
-
+ @classmethod
+ def get_logger(cls, logger_name, create_file=False):
- # create logger for prd_ci
- log = logging.getLogger(logger_name)
- log.setLevel(level=logging.INFO)
-
- # create formatter and add it to the handlers
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
- if create_file:
- # create file handler for logger.
- fh = logging.FileHandler('oa.log')
- fh.setLevel(level=logging.DEBUG)
- fh.setFormatter(formatter)
- # reate console handler for logger.
- ch = logging.StreamHandler()
- ch.setLevel(level=logging.DEBUG)
- ch.setFormatter(formatter)
+ # create logger for prd_ci
+ log = logging.getLogger(logger_name)
+ log.setLevel(level=logging.INFO)
- # add handlers to logger.
- if create_file:
- log.addHandler(fh)
+ # create formatter and add it to the handlers
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- log.addHandler(ch)
- return log
+ if create_file:
+ # create file handler for logger.
+ fh = logging.FileHandler('oa.log')
+ fh.setLevel(level=logging.DEBUG)
+ fh.setFormatter(formatter)
+ # reate console handler for logger.
+ ch = logging.StreamHandler()
+ ch.setLevel(level=logging.DEBUG)
+ ch.setFormatter(formatter)
- @classmethod
- def get_spot_conf(cls):
-
- conf_file = "/etc/spot.conf"
- config = ConfigParser.ConfigParser()
- config.readfp(SecHead(open(conf_file)))
+ # add handlers to logger.
+ if create_file:
+ log.addHandler(fh)
- return config
-
- @classmethod
- def create_oa_folders(cls,type,date):
+ log.addHandler(ch)
+ return log
- # create date and ingest summary folder structure if they don't' exist.
- root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- data_type_folder = "{0}/data/{1}/{2}"
- if not os.path.isdir(data_type_folder.format(root_path,type,date)): os.makedirs(data_type_folder.format(root_path,type,date))
- if not os.path.isdir(data_type_folder.format(root_path,type,"ingest_summary")): os.makedirs(data_type_folder.format(root_path,type,"ingest_summary"))
+ @classmethod
+ def get_spot_conf(cls):
- # create ipynb folders.
- ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path,type,date)
- if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)
+ conf_file = "/etc/spot.conf"
+ config = ConfigParser.ConfigParser()
+ config.readfp(SecHead(open(conf_file)))
- # retun path to folders.
- data_path = data_type_folder.format(root_path,type,date)
- ingest_path = data_type_folder.format(root_path,type,"ingest_summary")
- return data_path,ingest_path,ipynb_folder
-
- @classmethod
- def get_ml_results_form_hdfs(cls,hdfs_file_path,local_path):
+ return config
- # get results from hdfs.
- get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path,local_path)
- subprocess.call(get_results_cmd,shell=True)
- return get_results_cmd
+ @classmethod
+ def create_oa_folders(cls, type, date):
- @classmethod
- def read_results(cls,file,limit, delimiter=','):
-
- # read csv results.
- result_rows = []
- with open(file, 'rb') as results_file:
- csv_reader = csv.reader(results_file, delimiter = delimiter)
- for i in range(0, int(limit)):
- try:
- row = csv_reader.next()
- except StopIteration:
- return result_rows
- result_rows.append(row)
- return result_rows
+ # create date and ingest summary folder structure if they don't' exist.
+ root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ data_type_folder = "{0}/data/{1}/{2}"
+ if not os.path.isdir(data_type_folder.format(root_path, type, date)): os.makedirs(
+ data_type_folder.format(root_path, type, date))
+ if not os.path.isdir(data_type_folder.format(root_path, type, "ingest_summary")): os.makedirs(
+ data_type_folder.format(root_path, type, "ingest_summary"))
- @classmethod
- def ip_to_int(self,ip):
-
- try:
- o = map(int, ip.split('.'))
- res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
- return res
+ # create ipynb folders.
+ ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path, type, date)
+ if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)
- except ValueError:
- return None
-
-
- @classmethod
- def create_csv_file(cls,full_path_file,content,delimiter=','):
- with open(full_path_file, 'w+') as u_file:
- writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
- writer.writerows(content)
+ # retun path to folders.
+ data_path = data_type_folder.format(root_path, type, date)
+ ingest_path = data_type_folder.format(root_path, type, "ingest_summary")
+ return data_path, ingest_path, ipynb_folder
+ @classmethod
+ def get_ml_results_form_hdfs(cls, hdfs_file_path, local_path):
- @classmethod
- def cast_val(self,value):
- try:
- val = int(value)
+ # get results from hdfs.
+ get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path, local_path)
+ subprocess.call(get_results_cmd, shell=True)
+ return get_results_cmd
+
+ @classmethod
+ def read_results(cls, file, limit, delimiter=','):
+
+ # read csv results.
+ result_rows = []
+ with open(file, 'rb') as results_file:
+ csv_reader = csv.reader(results_file, delimiter=delimiter)
+ for i in range(0, int(limit)):
+ try:
+ row = csv_reader.next()
+ except StopIteration:
+ return result_rows
+ result_rows.append(row)
+ return result_rows
+
+ @classmethod
+ def ip_to_int(self, ip):
+
+ try:
+ o = map(int, ip.split('.'))
+ res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
+ return res
+
+ except ValueError:
+ return None
+
+ @classmethod
+ def create_csv_file(cls, full_path_file, content, delimiter=','):
+ with open(full_path_file, 'w+') as u_file:
+ writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
+ writer.writerows(content)
+
+ @classmethod
+ def cast_val(self, value):
+ try:
+ val = int(value)
+ except:
+ try:
+ val = float(value)
except:
- try:
- val = float(value)
- except:
- val = str(value)
- return val
+ val = str(value)
+ return val
class SecHead(object):
@@ -140,47 +139,38 @@
def readline(self):
if self.sechead:
- try:
+ try:
return self.sechead
- finally:
+ finally:
self.sechead = None
- else:
+ else:
return self.fp.readline()
+
class ProgressBar(object):
+ def __init__(self, total, prefix='', sufix='', decimals=2, barlength=60):
+ self._total = total
+ self._prefix = prefix
+ self._sufix = sufix
+ self._decimals = decimals
+ self._bar_length = barlength
+ self._auto_iteration_status = 0
- def __init__(self,total,prefix='',sufix='',decimals=2,barlength=60):
+ def start(self):
+ self._move_progress_bar(0)
- self._total = total
- self._prefix = prefix
- self._sufix = sufix
- self._decimals = decimals
- self._bar_length = barlength
- self._auto_iteration_status = 0
+ def update(self, iterator):
+ self._move_progress_bar(iterator)
- def start(self):
+ def auto_update(self):
+ self._auto_iteration_status += 1
+ self._move_progress_bar(self._auto_iteration_status)
- self._move_progress_bar(0)
-
- def update(self,iterator):
-
- self._move_progress_bar(iterator)
+ def _move_progress_bar(self, iteration):
+ filledLength = int(round(self._bar_length * iteration / float(self._total)))
+ percents = round(100.00 * (iteration / float(self._total)), self._decimals)
+ bar = '#' * filledLength + '-' * (self._bar_length - filledLength)
+ sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix))
+ sys.stdout.flush()
- def auto_update(self):
-
- self._auto_iteration_status += 1
- self._move_progress_bar(self._auto_iteration_status)
-
- def _move_progress_bar(self,iteration):
-
- filledLength = int(round(self._bar_length * iteration / float(self._total)))
- percents = round(100.00 * (iteration / float(self._total)), self._decimals)
- bar = '#' * filledLength + '-' * (self._bar_length - filledLength)
- sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix))
- sys.stdout.flush()
-
- if iteration == self._total:print("\n")
-
-
-
-
+ if iteration == self._total: print("\n")