| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| from __future__ import absolute_import, division, print_function |
| |
| import os |
| import numpy |
| import json |
| import sys |
| import re |
| import scipy.signal |
| import logging |
| import ast |
| import inspect |
| import collections |
| import numbers |
| try: |
| import cPickle as pickle |
| except: |
| import pickle |
| from collections import namedtuple, OrderedDict |
| import time |
| import mxnet as mx |
| import mxnet.ndarray as nd |
| |
| |
| _ctx = mx.cpu() |
| _numpy_rng = numpy.random.RandomState(123456) |
| |
| |
| def get_default_ctx(): |
| return _ctx |
| |
| |
| def get_numpy_rng(): |
| return _numpy_rng |
| |
| |
| def get_saving_path(prefix="", epoch=None): |
| sym_saving_path = os.path.join('%s-symbol.json' % prefix) |
| if epoch is not None: |
| param_saving_path = os.path.join('%s-%05d.params' % (prefix, epoch)) |
| else: |
| param_saving_path = os.path.join('%s.params' % prefix) |
| misc_saving_path = os.path.join('%s-misc.json' % prefix) |
| return sym_saving_path, param_saving_path, misc_saving_path |
| |
| |
| def logging_config(name=None, level=logging.DEBUG, console_level=logging.DEBUG): |
| if name is None: |
| name = inspect.stack()[1][1].split('.')[0] |
| folder = os.path.join(os.getcwd(), name) |
| if not os.path.exists(folder): |
| os.makedirs(folder) |
| logpath = os.path.join(folder, name + ".log") |
| print("All Logs will be saved to %s" %logpath) |
| logging.root.setLevel(level) |
| formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') |
| logfile = logging.FileHandler(logpath) |
| logfile.setLevel(level) |
| logfile.setFormatter(formatter) |
| logging.root.addHandler(logfile) |
| #TODO Update logging patterns in other files |
| logconsole = logging.StreamHandler() |
| logconsole.setLevel(console_level) |
| logconsole.setFormatter(formatter) |
| logging.root.addHandler(logconsole) |
| return folder |
| |
| |
| def save_params(dir_path=os.curdir, epoch=None, name="", params=None, aux_states=None, |
| ctx=mx.cpu()): |
| prefix = os.path.join(dir_path, name) |
| _, param_saving_path, _ = get_saving_path(prefix, epoch) |
| if not os.path.isdir(dir_path) and not (dir_path == ""): |
| os.makedirs(dir_path) |
| save_dict = {('arg:%s' % k): v.copyto(ctx) for k, v in params.items()} |
| save_dict.update({('aux:%s' % k): v.copyto(ctx) for k, v in aux_states.items()}) |
| nd.save(param_saving_path, save_dict) |
| return param_saving_path |
| |
| |
| def save_misc(dir_path=os.curdir, epoch=None, name="", content=None): |
| prefix = os.path.join(dir_path, name) |
| _, _, misc_saving_path = get_saving_path(prefix, epoch) |
| with open(misc_saving_path, 'w') as fp: |
| json.dump(content, fp) |
| return misc_saving_path |
| |
| |
| def quick_save_json(dir_path=os.curdir, file_name="", content=None): |
| file_path = os.path.join(dir_path, file_name) |
| if not os.path.isdir(dir_path): |
| os.makedirs(dir_path) |
| with open(file_path, 'w') as fp: |
| json.dump(content, fp) |
| logging.info('Save json into %s' % file_path) |
| |
| |
| def safe_eval(expr): |
| if type(expr) is str: |
| return ast.literal_eval(expr) |
| else: |
| return expr |
| |
| |
| def norm_clipping(params_grad, threshold): |
| assert isinstance(params_grad, dict) |
| norm_val = numpy.sqrt(sum([nd.norm(grad).asnumpy()[0]**2 for grad in params_grad.values()])) |
| # print('grad norm: %g' % norm_val) |
| ratio = 1.0 |
| if norm_val > threshold: |
| ratio = threshold / norm_val |
| for grad in params_grad.values(): |
| grad *= ratio |
| return norm_val |
| |
| |
| def sample_categorical(prob, rng): |
| """Sample from independent categorical distributions |
| |
| Each batch is an independent categorical distribution. |
| |
| Parameters |
| ---------- |
| prob : numpy.ndarray |
| Probability of the categorical distribution. Shape --> (batch_num, category_num) |
| rng : numpy.random.RandomState |
| |
| Returns |
| ------- |
| ret : numpy.ndarray |
| Sampling result. Shape --> (batch_num,) |
| """ |
| ret = numpy.empty(prob.shape[0], dtype=numpy.float32) |
| for ind in range(prob.shape[0]): |
| ret[ind] = numpy.searchsorted(numpy.cumsum(prob[ind]), rng.rand()).clip(min=0.0, |
| max=prob.shape[ |
| 1] - 0.5) |
| return ret |
| |
| |
| def sample_normal(mean, var, rng): |
| """Sample from independent normal distributions |
| |
| Each element is an independent normal distribution. |
| |
| Parameters |
| ---------- |
| mean : numpy.ndarray |
| Means of the normal distribution. Shape --> (batch_num, sample_dim) |
| var : numpy.ndarray |
| Variance of the normal distribution. Shape --> (batch_num, sample_dim) |
| rng : numpy.random.RandomState |
| |
| Returns |
| ------- |
| ret : numpy.ndarray |
| The sampling result. Shape --> (batch_num, sample_dim) |
| """ |
| ret = numpy.sqrt(var) * rng.randn(*mean.shape) + mean |
| return ret |
| |
| |
| def sample_mog(prob, mean, var, rng): |
| """Sample from independent mixture of gaussian (MoG) distributions |
| |
| Each batch is an independent MoG distribution. |
| |
| Parameters |
| ---------- |
| prob : numpy.ndarray |
| mixture probability of each gaussian. Shape --> (batch_num, center_num) |
| mean : numpy.ndarray |
| mean of each gaussian. Shape --> (batch_num, center_num, sample_dim) |
| var : numpy.ndarray |
| variance of each gaussian. Shape --> (batch_num, center_num, sample_dim) |
| rng : numpy.random.RandomState |
| |
| Returns |
| ------- |
| ret : numpy.ndarray |
| sampling result. Shape --> (batch_num, sample_dim) |
| """ |
| gaussian_inds = sample_categorical(prob, rng).astype(numpy.int32) |
| mean = mean[numpy.arange(mean.shape[0]), gaussian_inds, :] |
| var = var[numpy.arange(mean.shape[0]), gaussian_inds, :] |
| ret = sample_normal(mean=mean, var=var, rng=rng) |
| return ret |
| |
| |
| def npy_softmax(x, axis=1): |
| e_x = numpy.exp(x - numpy.max(x, axis=axis, keepdims=True)) |
| out = e_x / e_x.sum(axis=axis, keepdims=True) |
| return out |
| |
| |
| def npy_sigmoid(x): |
| return 1/(1 + numpy.exp(-x)) |
| |
| |
| def npy_onehot(x, num): |
| ret = numpy.zeros(shape=(x.size, num)) |
| ret[numpy.arange(x.size), x.ravel()] = 1 |
| ret = ret.reshape(x.shape + (num,)) |
| return ret |
| |
| def npy_binary_entropy(prediction, target): |
| assert prediction.shape == target.shape |
| return - (numpy.log(prediction + 1E-9) * target + |
| numpy.log(1 - prediction + 1E-9) * (1 - target)).sum() |
| |
| |
| def block_all(sym_list): |
| return [mx.symbol.BlockGrad(sym) for sym in sym_list] |
| |
| |
| def load_params(dir_path="", epoch=None, name=""): |
| prefix = os.path.join(dir_path, name) |
| _, param_loading_path, _ = get_saving_path(prefix, epoch) |
| while not os.path.isfile(param_loading_path): |
| logging.info("in load_param, %s Not Found!" % param_loading_path) |
| time.sleep(60) |
| save_dict = nd.load(param_loading_path) |
| arg_params = {} |
| aux_params = {} |
| for k, v in save_dict.items(): |
| tp, name = k.split(':', 1) |
| if tp == 'arg': |
| arg_params[name] = v |
| if tp == 'aux': |
| aux_params[name] = v |
| return arg_params, aux_params, param_loading_path |
| |
| |
| def load_misc(dir_path="", epoch=None, name=""): |
| prefix = os.path.join(dir_path, name) |
| _, _, misc_saving_path = get_saving_path(prefix, epoch) |
| with open(misc_saving_path, 'r') as fp: |
| misc = json.load(fp) |
| return misc |
| |
| |
| def load_npz(path): |
| with numpy.load(path) as data: |
| ret = {k: data[k] for k in data.keys()} |
| return ret |
| |
| |
| def discount_cumsum(x, discount): |
| # See https://docs.scipy.org/doc/scipy/reference/tutorial/signal.html#difference-equation-filtering |
| # Here, we have y[t] - discount*y[t+1] = x[t] |
| # or rev(y)[t] - discount*rev(y)[t-1] = rev(x)[t] |
| return scipy.signal.lfilter([1], [1, -discount], x[::-1], axis=0)[::-1] |
| |
| |
| def discount_return(x, discount): |
| return numpy.sum(x * (discount ** numpy.arange(len(x)))) |
| |
| |
| def update_on_kvstore(kv, params, params_grad): |
| for ind, k in enumerate(params.keys()): |
| kv.push(ind, params_grad[k], priority=-ind) |
| kv.pull(ind, params[k], priority=-ind) |
| |
| |
| def parse_ctx(ctx_args): |
| ctx = re.findall('([a-z]+)(\d*)', ctx_args) |
| ctx = [(device, int(num)) if len(num) > 0 else (device, 0) for device, num in ctx] |
| return ctx |
| |
| |
| def get_npy_list(ndarray_list): |
| """Get a numpy-array list from a ndarray list |
| Parameters |
| ---------- |
| ndarray_list : list of NDArray |
| |
| Returns |
| ------- |
| ret : list of numpy.ndarray |
| """ |
| ret = [v.asnumpy() for v in ndarray_list] |
| return ret |
| |
| |
| def get_sym_list(syms, default_names=None, default_shapes=None): |
| if syms is None and default_names is not None: |
| if default_shapes is not None: |
| return [mx.sym.Variable(name=name, shape=shape) for (name, shape) |
| in zip(default_names, default_shapes)] |
| else: |
| return [mx.sym.Variable(name=name) for name in default_names] |
| assert isinstance(syms, (list, tuple, mx.symbol.Symbol)) |
| if isinstance(syms, (list, tuple)): |
| if default_names is not None and len(syms) != len(default_names): |
| raise ValueError("Size of symbols do not match expectation. Received %d, Expected %d. " |
| "syms=%s, names=%s" %(len(syms), len(default_names), |
| str(list(sym.name for sym in syms)), |
| str(default_names))) |
| return list(syms) |
| else: |
| if default_names is not None and len(default_names) != 1: |
| raise ValueError("Size of symbols do not match expectation. Received 1, Expected %d. " |
| "syms=%s, names=%s" |
| % (len(default_names), str([syms.name]), str(default_names))) |
| return [syms] |
| |
| |
| def get_numeric_list(values, typ, expected_len=None): |
| if isinstance(values, numbers.Number): |
| if expected_len is not None: |
| return [typ(values)] * expected_len |
| else: |
| return [typ(values)] |
| elif isinstance(values, (list, tuple)): |
| if expected_len is not None: |
| assert len(values) == expected_len |
| try: |
| ret = [typ(value) for value in values] |
| return ret |
| except(ValueError): |
| print("Need iterable with numeric elements, received: %s" %str(values)) |
| sys.exit(1) |
| else: |
| raise ValueError("Unaccepted value type, values=%s" %str(values)) |
| |
| |
| def get_int_list(values, expected_len=None): |
| return get_numeric_list(values, numpy.int32, expected_len) |
| |
| |
| def get_float_list(values, expected_len=None): |
| return get_numeric_list(values, numpy.float32, expected_len) |
| |
| |
| def get_bucket_key(bucket_kwargs): |
| assert isinstance(bucket_kwargs, dict) |
| return tuple(bucket_kwargs.items()) |
