blob: 8b6e0436c22b3229535e24975ef479611ed13aab [file] [log] [blame]
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import numpy as np
import os
import sys
import logging
from StringIO import StringIO
import json
from datetime import datetime
from kaldi_parser import *
import utils.utils as utils
# nicer interface for file2nnet, nnet2file
def load(model, filename, gradients, num_hidden_layers=-1, with_final=True, factors=None):
_file2nnet(model.sigmoid_layers, set_layer_num = num_hidden_layers,
filename=filename, activation="sigmoid", withfinal=with_final, factor=1.0, gradients=gradients, factors=factors)
def save(model, filename):
_nnet2file(model.sigmoid_layers, set_layer_num = -1, filename=filename,
activation="sigmoid", start_layer = 0, withfinal=True)
# convert an array to a string
def array_2_string(array):
return array.astype('float32')
# convert a string to an array
def string_2_array(string):
if isinstance(string, str) or isinstance(string, unicode):
str_in = StringIO(string)
return np.loadtxt(str_in)
else:
return string
def _nnet2file(layers, set_layer_num = -1, filename='nnet.out', activation='sigmoid', start_layer = 0, withfinal=True, input_factor = 0.0, factor=[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]):
logger = logging.getLogger(__name__)
logger.info("Saving network "+filename)
n_layers = len(layers)
nnet_dict = {}
if set_layer_num == -1:
set_layer_num = n_layers - 1
for i in range(start_layer, set_layer_num):
logger.info("Saving hidden layer "+str(i))
dict_a = str(i) + ' ' + activation + ' W'
if i == 0:
nnet_dict[dict_a] = array_2_string((1.0 - input_factor) * layers[i].params[0].get_value())
else:
nnet_dict[dict_a] = array_2_string((1.0 - factor[i-1]) * layers[i].params[0].get_value())
dict_a = str(i) + ' ' + activation + ' b'
nnet_dict[dict_a] = array_2_string(layers[i].params[1].get_value())
# gradients
dict_a = str(i) + ' ' + activation + ' dW'
nnet_dict[dict_a] = array_2_string(layers[i].delta_params[0].get_value())
dict_a = str(i) + ' ' + activation + ' db'
nnet_dict[dict_a] = array_2_string(layers[i].delta_params[1].get_value())
if layers[i].kahan:
logger.info("Loading hidden kahan")
dict_a = str(i) + ' ' + activation + ' W_carry'
nnet_dict[dict_a] = array_2_string(layers[i].params_carry[0].get_value())
dict_a = str(i) + ' ' + activation + ' b_carry'
nnet_dict[dict_a] = array_2_string(layers[i].params_carry[1].get_value())
#dict_a = str(i) + ' ' + activation + ' dW_carry'
#nnet_dict[dict_a] = array_2_string(layers[i].delta_params_carry[0].get_value())
#dict_a = str(i) + ' ' + activation + ' db_carry'
#nnet_dict[dict_a] = array_2_string(layers[i].delta_params_carry[1].get_value())
if withfinal:
logger.info("Saving final layer ")
dict_a = 'logreg W'
nnet_dict[dict_a] = array_2_string((1.0 - factor[-1]) * layers[-1].params[0].get_value())
dict_a = 'logreg b'
nnet_dict[dict_a] = array_2_string(layers[-1].params[1].get_value())
#gradients
dict_a = 'logreg dW'
nnet_dict[dict_a] = array_2_string(layers[-1].delta_params[0].get_value())
dict_a = 'logreg db'
nnet_dict[dict_a] = array_2_string(layers[-1].delta_params[1].get_value())
if layers[-1].kahan:
logger.info("Loading softmax kahan")
dict_a = 'logreg W_carry'
nnet_dict[dict_a] = array_2_string(layers[-1].params_carry[0].get_value())
dict_a = 'logreg b_carry'
nnet_dict[dict_a] = array_2_string(layers[-1].params_carry[1].get_value())
#dict_a = 'logreg dW_carry'
#nnet_dict[dict_a] = array_2_string(layers[-1].delta_params_carry[0].get_value())
#dict_a = 'logreg db_carry'
#nnet_dict[dict_a] = array_2_string(layers[-1].delta_params_carry[1].get_value())
utils.pickle_save(nnet_dict, filename)
def zero(x):
x.set_value(np.zeros_like(x.get_value(borrow=True), dtype=theano.config.floatX))
def _file2nnet(layers, set_layer_num = -1, filename='nnet.in', activation='sigmoid', withfinal=True, factor=1.0, gradients=False, factors=None):
logger = logging.getLogger(__name__)
logger.info("Loading "+filename)
# if is KALDI binary
if fileIsBinary(filename):
print("Warning dropout factors ignored here")
nnet = file2nnet_binary(filename)
n_layers = len(nnet)
if set_layer_num == -1:
set_layer_num = n_layers - 1
for i in xrange(set_layer_num):
layers[i].params[0].set_value(factor * nnet[i]["weights"].astype(dtype=theano.config.floatX))
layers[i].params[1].set_value(nnet[i]["bias"].astype(dtype=theano.config.floatX))
if withfinal:
#print(nnet[-1]["weights"][0][0:10])
layers[-1].params[0].set_value(nnet[-1]["weights"].astype(dtype=theano.config.floatX))
layers[-1].params[1].set_value(nnet[-1]["bias"].astype(dtype=theano.config.floatX))
return
# else, it's pdnn format
n_layers = len(layers)
if factors is None:
factors = [1.0 for l in layers]
if len(factors) != n_layers:
raise Exception("number of factors does not equal number of hidden + softmax")
nnet_dict = {}
if set_layer_num == -1:
set_layer_num = n_layers - 1
nnet_dict = utils.pickle_load(filename)
for i in xrange(set_layer_num):
logger.info("Loading hidden layer "+str(i))
dict_key = str(i) + ' ' + activation + ' W'
layers[i].params[0].set_value(factors[i] * factor * np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = str(i) + ' ' + activation + ' b'
layers[i].params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
if gradients:
dict_key = str(i) + ' ' + activation + ' dW'
layers[i].delta_params[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = str(i) + ' ' + activation + ' db'
layers[i].delta_params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
else:
zero(layers[i].delta_params[0])
zero(layers[i].delta_params[1])
dict_key = str(i) + ' ' + activation + ' W_carry'
if layers[i].kahan and dict_key in nnet_dict:
logger.info("Loading hidden kahan")
dict_key = str(i) + ' ' + activation + ' W_carry'
layers[i].params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = str(i) + ' ' + activation + ' b_carry'
layers[i].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
#dict_key = str(i) + ' ' + activation + ' dW_carry'
#layers[i].delta_params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
#dict_key = str(i) + ' ' + activation + ' db_carry'
#layers[i].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
if layers[i].sync:
layers[i].params_sync[0].set_value(layers[i].params[0].get_value().astype('float32'))
layers[i].params_sync[1].set_value(layers[i].params[1].get_value().astype('float32'))
logger.info("Copy params to sync")
if withfinal:
logger.info("Loading final layer ")
dict_key = 'logreg W'
layers[-1].params[0].set_value(factors[-1] * np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = 'logreg b'
layers[-1].params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
if gradients:
dict_key = 'logreg dW'
layers[-1].delta_params[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = 'logreg db'
layers[-1].delta_params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
else:
zero(layers[-1].delta_params[0])
zero(layers[-1].delta_params[1])
dict_key = 'logreg W_carry'
if layers[-1].kahan and dict_key in nnet_dict:
logger.info("Loading softmax kahan")
dict_key = 'logreg W_carry'
layers[-1].params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
dict_key = 'logreg b_carry'
layers[-1].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
#dict_key = 'logreg dW_carry'
#layers[-1].delta_params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
#dict_key = 'logreg db_carry'
#layers[-1].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX))
if layers[-1].sync:
layers[-1].params_sync[0].set_value(layers[-1].params[0].get_value().astype('float32'))
layers[-1].params_sync[1].set_value(layers[-1].params[1].get_value().astype('float32'))
logger.info("Copy softmax params to sync")
if gradients:
logger.info("Loading gradients")
else:
logger.info("Zero-ing gradients")
def _cnn2file(conv_layers, filename='nnet.out', activation='sigmoid', withfinal=True, input_factor = 1.0, factor=1.0):
n_layers = len(conv_layers)
nnet_dict = {}
for i in xrange(n_layers):
conv_layer = conv_layers[i]
filter_shape = conv_layer.filter_shape
for next_X in xrange(filter_shape[0]):
for this_X in xrange(filter_shape[1]):
dict_a = 'W ' + str(i) + ' ' + str(next_X) + ' ' + str(this_X)
if i == 0:
nnet_dict[dict_a] = array_2_string(input_factor * (conv_layer.W.get_value())[next_X, this_X])
else:
nnet_dict[dict_a] = array_2_string(factor * (conv_layer.W.get_value())[next_X, this_X])
dict_a = 'b ' + str(i)
nnet_dict[dict_a] = array_2_string(conv_layer.b.get_value())
with open(filename, 'wb') as fp:
json.dump(nnet_dict, fp, indent=2, sort_keys = True)
fp.flush()
def _file2cnn(conv_layers, filename='nnet.in', activation='sigmoid', withfinal=True, factor=1.0):
n_layers = len(conv_layers)
nnet_dict = {}
with open(filename, 'rb') as fp:
nnet_dict = json.load(fp)
for i in xrange(n_layers):
conv_layer = conv_layers[i]
filter_shape = conv_layer.filter_shape
W_array = conv_layer.W.get_value()
for next_X in xrange(filter_shape[0]):
for this_X in xrange(filter_shape[1]):
dict_a = 'W ' + str(i) + ' ' + str(next_X) + ' ' + str(this_X)
W_array[next_X, this_X, :, :] = factor * np.asarray(string_2_array(nnet_dict[dict_a]))
conv_layer.W.set_value(W_array)
dict_a = 'b ' + str(i)
conv_layer.b.set_value(np.asarray(string_2_array(nnet_dict[dict_a]), dtype=theano.config.floatX))