| # coding=utf-8 |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| m4_changequote(`<!', `!>') |
| |
| import sys |
| import numpy as np |
| import os |
| import re |
| from os import path |
| # Add convex module to the pythonpath. |
| sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))) |
| sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))) |
| |
| import unittest |
| from mock import * |
| import plpy_mock as plpy |
| |
| try: |
| from pgsanity.pgsanity import check_string as pglint |
| pglint("SELECT 1;") |
| except: |
| # Just skip this test if "pip install pgsanity" hasn't been run |
| # or isn't functioning (eg. ecpg isn't in path) |
| def pglint(q, s=False): |
| return (True, '') |
| |
| # helper for multiplying array by int |
| def mult(k,arr): |
| return [ k*a for a in arr ] |
| |
| class MadlibKerasFitEvalTransitionTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras |
| self.subject = madlib_keras |
| |
| self.model = Sequential() |
| self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', |
| input_shape=(1,1,1,), padding='same')) |
| self.model.add(Flatten()) |
| |
| self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']" |
| self.fit_params = "batch_size=1, epochs=1" |
| self.model_weights = [3,4,5,6] |
| self.serialized_weights = np.array(self.model_weights, dtype=np.float32 |
| ).tostring() |
| self.loss = 0.5947071313858032 |
| self.accuracy = 1.0 |
| self.dist_key_mapping = [0,1,2] |
| self.accessible_gpus_for_seg = [0] |
| |
| self.independent_var_real = [[[[0.5]]]] * 10 |
| self.dependent_var_int = [[0,1]] * 10 |
| |
| # Params as bytea |
| self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes() |
| self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes() |
| |
| self.independent_var_shape = [10,1,1,1] |
| self.dependent_var_shape = [10,2] |
| # We test on segment 0, which has 3 buffers filled with 10 identical |
| # images each, or 30 images total |
| self.total_images_per_seg = [3*len(self.dependent_var_int),20,40] |
| |
| self.dummy_prev_weights = 'dummy weights' |
| |
| # Mock calls to tf.keras fit |
| model_class = self.subject.tf.keras.Model |
| model_class.fit = MagicMock() |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| self.subject.K.clear_session() |
| |
| def _test_fit_transition_first_buffer_pass(self, **kwargs): |
| ending_image_count = len(self.dependent_var_int) |
| |
| previous_state = np.array(self.model_weights, dtype=np.float32) |
| |
| new_state = self.subject.fit_transition( |
| None, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_state.tostring(), **kwargs) |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| |
| def _test_fit_transition_multiple_model_no_cache_first_buffer_pass(self, |
| **kwargs): |
| ending_image_count = len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| |
| new_state = self.subject.fit_transition( |
| None, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), |
| True, **kwargs) |
| |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| |
| def test_fit_transition_multiple_model_cache_first_buffer_pass(self): |
| ending_image_count = len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| |
| k = {'GD': {}} |
| new_state = self.subject.fit_multiple_transition_caching( |
| None, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), True, **k) |
| |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| self.assertTrue('sess' not in k['GD']) |
| self.assertTrue('segment_model' not in k['GD']) |
| self.assertTrue('cache_set' not in k['GD']) |
| self.assertTrue(k['GD']['x_train']) |
| self.assertTrue(k['GD']['y_train']) |
| |
| def _test_fit_transition_middle_buffer_pass(self, **kwargs): |
| |
| starting_image_count = len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| state = starting_image_count |
| new_state = self.subject.fit_transition( |
| state, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), None, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, self.dummy_prev_weights, True, **kwargs) |
| |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| |
| def _test_fit_transition_multiple_model_no_cache_middle_buffer_pass(self, |
| **kwargs): |
| starting_image_count = len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| state = starting_image_count |
| new_state = self.subject.fit_transition( |
| state, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), None, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True, |
| **kwargs) |
| |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| |
| def test_fit_transition_multiple_model_cache_middle_buffer_pass(self): |
| starting_image_count = len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| x_train = list() |
| y_train = list() |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| |
| k = {'GD': {'x_train': x_train, 'y_train': y_train}} |
| |
| state = starting_image_count |
| new_state = self.subject.fit_multiple_transition_caching( |
| state, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), True, **k) |
| |
| image_count = new_state |
| self.assertEqual(ending_image_count, image_count) |
| self.assertTrue('sess' not in k['GD']) |
| self.assertTrue('segment_model' not in k['GD']) |
| self.assertTrue('cache_set' not in k['GD']) |
| self.assertTrue(k['GD']['x_train']) |
| self.assertTrue(k['GD']['y_train']) |
| |
| def _test_fit_transition_last_buffer_pass(self, **kwargs): |
| |
| starting_image_count = 2*len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| state = starting_image_count |
| previous_state = np.array(self.model_weights, dtype=np.float32) |
| new_state = self.subject.fit_transition( |
| state, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), None, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_state.tostring(), |
| **kwargs) |
| state = np.fromstring(new_state, dtype=np.float32) |
| image_count = state[0] |
| # We need to assert that the weights should be multiplied by final image count. |
| weights = state[1:] |
| multiplied_weights = mult(self.total_images_per_seg[0], self.model_weights) |
| self.assertTrue((weights == multiplied_weights).all()) |
| self.assertEqual(ending_image_count, image_count) |
| |
| def _test_internal_keras_eval_transition_first_buffer(self, |
| last_iteration = False, |
| **kwargs): |
| ending_image_count = len(self.dependent_var_int) |
| |
| state = [0,0,0] |
| new_state = self.subject.internal_keras_eval_transition( |
| state, self.dependent_var , self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), |
| self.serialized_weights, self.compile_params, 0, |
| self.dist_key_mapping, 0, 4, |
| self.total_images_per_seg, self.accessible_gpus_for_seg, |
| last_iteration, None, **kwargs) |
| |
| agg_loss, agg_accuracy, image_count = new_state |
| |
| self.assertEqual(ending_image_count, image_count) |
| # loss and accuracy should be unchanged |
| self.assertAlmostEqual(self.loss * image_count, agg_loss, 4) |
| self.assertAlmostEqual(self.accuracy * image_count, agg_accuracy, 4) |
| |
| def _test_internal_keras_eval_transition_last_buffer(self, |
| last_iteration = False, |
| **kwargs): |
| starting_image_count = 2*len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| state = [self.loss * starting_image_count, |
| self.accuracy * starting_image_count, |
| starting_image_count] |
| |
| new_state = self.subject.internal_keras_eval_transition( |
| state, self.dependent_var , self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), |
| 'dummy_model_weights', None, 0, |
| self.dist_key_mapping, 0, 4, |
| self.total_images_per_seg, self.accessible_gpus_for_seg, |
| last_iteration, **kwargs) |
| agg_loss, agg_accuracy, image_count = new_state |
| |
| self.assertEqual(ending_image_count, image_count) |
| # loss and accuracy should be unchanged |
| self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) |
| self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) |
| |
| def _test_internal_keras_eval_transition_middle_buffer(self, |
| last_iteration = False, |
| **kwargs): |
| starting_image_count = len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| |
| state = [self.loss * starting_image_count, |
| self.accuracy * starting_image_count, starting_image_count] |
| |
| new_state = self.subject.internal_keras_eval_transition( |
| state, self.dependent_var , self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), |
| 'dummy_model_weights', None, 0, |
| self.dist_key_mapping, 0, 4, |
| self.total_images_per_seg, self.accessible_gpus_for_seg, |
| last_iteration, **kwargs) |
| |
| agg_loss, agg_accuracy, image_count = new_state |
| |
| self.assertEqual(ending_image_count, image_count) |
| self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) |
| self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) |
| |
| def _test_fit_transition_multiple_model_no_cache_last_buffer_pass(self, |
| **kwargs): |
| starting_image_count = 2*len(self.dependent_var_int) |
| |
| state = starting_image_count |
| new_state = self.subject.fit_transition( |
| state , self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), None, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, self.dummy_prev_weights, |
| True, **kwargs) |
| |
| state = np.fromstring(new_state, dtype=np.float32) |
| weights = np.rint(state[0:]).astype(np.int) |
| |
| ## image count should not be added to the final state of |
| # fit multiple |
| self.assertEqual(len(self.model_weights), len(weights)) |
| |
| def test_fit_transition_multiple_model_cache_last_buffer_pass(self): |
| starting_image_count = 2*len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| x_train = list() |
| y_train = list() |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| |
| k = {'GD': {'x_train': x_train, 'y_train': y_train}} |
| |
| state = starting_image_count |
| graph1 = self.subject.tf.get_default_graph() |
| new_state = self.subject.fit_multiple_transition_caching( |
| state, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), False, **k) |
| graph2 = self.subject.tf.get_default_graph() |
| self.assertNotEquals(graph1, graph2) |
| state = np.fromstring(new_state, dtype=np.float32) |
| weights = np.rint(state[0:]).astype(np.int) |
| |
| ## image count should not be added to the final state of |
| # fit multiple |
| self.assertEqual(len(self.model_weights), len(weights)) |
| |
| self.assertTrue('sess' not in k['GD']) |
| self.assertTrue('segment_model' not in k['GD']) |
| self.assertTrue(k['GD']['cache_set']) |
| self.assertTrue(k['GD']['x_train']) |
| self.assertTrue(k['GD']['y_train']) |
| |
| def test_fit_transition_multiple_model_cache_filled_pass(self): |
| starting_image_count = 2*len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| x_train = list() |
| y_train = list() |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| |
| self.subject.compile_and_set_weights(self.model, self.compile_params, |
| '/cpu:0', self.serialized_weights) |
| s1 = self.subject.K.get_session() |
| k = {'GD': {'x_train': x_train, 'y_train': y_train, 'cache_set': True, |
| 'sess': s1, 'segment_model': self.model}} |
| graph1 = self.subject.tf.get_default_graph() |
| new_state = self.subject.fit_multiple_transition_caching( |
| None, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), False, **k) |
| graph2 = self.subject.tf.get_default_graph() |
| self.assertNotEquals(graph1, graph2) |
| state = np.fromstring(new_state, dtype=np.float32) |
| weights = np.rint(state[0:]).astype(np.int) |
| |
| ## image count should not be added to the final state of |
| # fit multiple |
| self.assertEqual(len(self.model_weights), len(weights)) |
| |
| self.assertTrue('sess' not in k['GD']) |
| self.assertTrue('segment_model' not in k['GD']) |
| self.assertTrue(k['GD']['cache_set']) |
| self.assertTrue(k['GD']['x_train']) |
| self.assertTrue(k['GD']['y_train']) |
| |
| def test_fit_transition_multiple_model_cache_filled_final_training_pass(self): |
| starting_image_count = 2*len(self.dependent_var_int) |
| ending_image_count = starting_image_count + len(self.dependent_var_int) |
| |
| previous_weights = np.array(self.model_weights, dtype=np.float32) |
| x_train = list() |
| y_train = list() |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| x_train.append(self.subject.np_array_float32(self.independent_var, self.independent_var_shape)) |
| y_train.append(self.subject.np_array_int16(self.dependent_var, self.dependent_var_shape)) |
| |
| k = {'GD': {'x_train': x_train, 'y_train': y_train, 'cache_set': True}} |
| graph1 = self.subject.tf.get_default_graph() |
| new_state = self.subject.fit_multiple_transition_caching( |
| None, self.dependent_var, self.independent_var, |
| self.dependent_var_shape, self.independent_var_shape, |
| self.model.to_json(), self.compile_params, self.fit_params, 0, |
| self.dist_key_mapping, 0, 4, self.total_images_per_seg, |
| self.accessible_gpus_for_seg, previous_weights.tostring(), True, **k) |
| graph2 = self.subject.tf.get_default_graph() |
| self.assertNotEquals(graph1, graph2) |
| |
| state = np.fromstring(new_state, dtype=np.float32) |
| weights = np.rint(state[0:]).astype(np.int) |
| |
| ## image count should not be added to the final state of |
| # fit multiple |
| self.assertEqual(len(self.model_weights), len(weights)) |
| |
| self.assertTrue('sess' not in k['GD']) |
| self.assertTrue('segment_model' not in k['GD']) |
| self.assertTrue('cache_set' not in k['GD']) |
| self.assertTrue('x_train' not in k['GD']) |
| self.assertTrue('y_train' not in k['GD']) |
| |
| ############### GRAPH AND SESSION TESTS ################################ |
| def test_fit_eval_2_iterations_mcf_null_gpdb(self): |
| kwargs = {'GD': {}} |
| GD = kwargs['GD'] |
| |
| ######################### fit for 2 iterations ########## |
| # iteration 1 |
| first_iter_keras_sess = self._run_fit_iteration(**kwargs) |
| self._assert_keras_session_same_as_gd_session(GD) |
| |
| first_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| # iteration 2 (last iteration) |
| last_iter_keras_sess = self._run_fit_iteration(**kwargs) |
| self._assert_keras_session_same_as_gd_session(GD) |
| |
| last_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| self.assertEquals(first_iter_keras_sess, last_iter_keras_sess) |
| self.assertEquals(first_iter_tf_graph, last_iter_tf_graph) |
| |
| ###################### eval transition for last iteration ########### |
| self._run_eval_iteration(True, last_iter_keras_sess, last_iter_tf_graph, **kwargs) |
| eval_last_iter_keras_sess = self.subject.K.get_session() |
| eval_last_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| self.assertNotEquals(eval_last_iter_keras_sess, last_iter_keras_sess) |
| self.assertNotEquals(eval_last_iter_tf_graph, last_iter_tf_graph) |
| self._assert_gd_cleared(GD) |
| |
| def test_fit_eval_2_iterations_mcf_1_gpdb(self): |
| kwargs = {'GD': {}} |
| GD = kwargs['GD'] |
| |
| ######################### fit + eval for 2 iterations ########## |
| # iteration 1 fit |
| first_iter_keras_sess = self._run_fit_iteration(**kwargs) |
| self._assert_keras_session_same_as_gd_session(GD) |
| |
| first_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| # iteration 1 eval |
| self._run_eval_iteration(False, first_iter_keras_sess, first_iter_tf_graph, **kwargs) |
| self._assert_keras_session_same_as_gd_session(GD) |
| |
| eval_first_iter_keras_sess = self.subject.K.get_session() |
| eval_first_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| self.assertEquals(eval_first_iter_keras_sess, first_iter_keras_sess) |
| self.assertEquals(eval_first_iter_tf_graph, first_iter_tf_graph) |
| |
| # iteration 2 fit (last iteration) |
| last_iter_keras_sess = self._run_fit_iteration(**kwargs) |
| self._assert_keras_session_same_as_gd_session(GD) |
| |
| last_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| self.assertEquals(first_iter_keras_sess, last_iter_keras_sess) |
| self.assertEquals(first_iter_tf_graph, last_iter_tf_graph) |
| |
| # iteration 2 eval (last iteration) |
| # eval and fit use the same session & graph for all iterations. |
| # After the last call to eval(last iteration), we want to assert |
| # the session was closed and the graph cleared out. |
| # To assert this we call get_session() and get_default_graph() |
| # that will give a new session and a new graph and assert its not |
| # equal to the prev iteration session and graph. |
| self._run_eval_iteration(True, last_iter_keras_sess, last_iter_tf_graph, **kwargs) |
| |
| eval_last_iter_keras_sess = self.subject.K.get_session() |
| eval_last_iter_tf_graph = self.subject.tf.get_default_graph() |
| |
| self.assertNotEquals(eval_last_iter_keras_sess, last_iter_keras_sess) |
| self.assertNotEquals(eval_last_iter_tf_graph, last_iter_tf_graph) |
| self._assert_gd_cleared(GD) |
| |
| def test_fit_multiple_2_iterations(self): |
| kwargs = {'GD': {}} |
| GD = kwargs['GD'] |
| |
| ############ fit multiple for 2 iterations ########## |
| # iteration 1 |
| # first_iter_tf_graph is used to assert that calling fit_multiple closes the tf session |
| # and graph at the last buffer. |
| # It is fetched prior to calling the fit_transition(from fit_multiple) as when we create |
| # a session inside fit_transition, instead of creating a new graph it will use first_iter_tf_graph. |
| # This enables us to do the not equals assert. |
| first_iter_tf_graph = self.subject.tf.get_default_graph() |
| first_iter_keras_sess = self._run_fit_multiple_iteration(**kwargs) |
| self._assert_gd_cleared(GD) |
| |
| # iteration 2 (last iteration) |
| last_iter_tf_graph = self.subject.tf.get_default_graph() |
| last_iter_keras_sess = self._run_fit_multiple_iteration(**kwargs) |
| self._assert_gd_cleared(GD) |
| |
| self.assertNotEquals(first_iter_keras_sess, last_iter_keras_sess) |
| self.assertNotEquals(first_iter_tf_graph, last_iter_tf_graph) |
| |
| def test_eval_multiple_any_iteration(self): |
| # This test tests 2 things: |
| # 1. Calling eval_transition from fit_multiple |
| # 2. Calling eval_transition from evaluate directly |
| kwargs = {'GD': {}} |
| GD = kwargs['GD'] |
| |
| # eval_iter_tf_graph1 is used to assert that calling eval clears the tf session and graph |
| # It is fetched prior to calling the eval_transition as when we create a session inside |
| # eval_transition, instead of creating a new graph it will use eval_iter_tf_graph1. |
| # This enables us to do the not equals assert. |
| eval_iter_tf_graph1 = self.subject.tf.get_default_graph() |
| eval_iter_keras_sess1 = self._run_eval_iteration(True, None, None, True, **kwargs) |
| eval_iter_keras_sess2 = self.subject.K.get_session() |
| eval_iter_tf_graph2 = self.subject.tf.get_default_graph() |
| |
| self.assertNotEquals(eval_iter_keras_sess1, eval_iter_keras_sess2) |
| self.assertNotEquals(eval_iter_tf_graph1, eval_iter_tf_graph2) |
| self._assert_gd_cleared(GD) |
| |
| def _run_eval_iteration(self, final_iteration, prev_keras_sess, prev_tf_graph, called_from_fit_multiple=False, **kwargs): |
| self._test_internal_keras_eval_transition_first_buffer(final_iteration, |
| **kwargs) |
| self._assert_gd_is_valid(kwargs['GD']) |
| self._assert_keras_session_same_as_gd_session(kwargs['GD']) |
| |
| eval_first_buffer_keras_sess = kwargs['GD']['sess'] |
| self.assertFalse(eval_first_buffer_keras_sess._closed) |
| eval_first_buffer_tf_graph = self.subject.tf.get_default_graph() |
| |
| if not called_from_fit_multiple: |
| self.assertEquals(eval_first_buffer_keras_sess, prev_keras_sess) |
| self.assertEquals(eval_first_buffer_tf_graph, prev_tf_graph) |
| |
| self._test_internal_keras_eval_transition_middle_buffer(final_iteration, |
| **kwargs ) |
| self._assert_gd_is_valid(kwargs['GD']) |
| self._assert_keras_session_same_as_gd_session(kwargs['GD']) |
| self.assertFalse(eval_first_buffer_keras_sess._closed) |
| |
| self._test_internal_keras_eval_transition_last_buffer(final_iteration, |
| **kwargs) |
| if final_iteration: |
| self._assert_gd_cleared(kwargs['GD']) |
| self.assertTrue(eval_first_buffer_keras_sess._closed) |
| else: |
| self._assert_gd_is_valid(kwargs['GD']) |
| self.assertFalse(eval_first_buffer_keras_sess._closed) |
| return eval_first_buffer_keras_sess |
| |
| def _run_fit_iteration(self, **kwargs): |
| self._test_fit_transition_first_buffer_pass(**kwargs) |
| gd_first_buffer = kwargs['GD'] |
| self._assert_gd_is_valid(gd_first_buffer) |
| iter_sess = gd_first_buffer['sess'] |
| self.assertFalse(iter_sess._closed) |
| self._assert_keras_session_same_as_gd_session(gd_first_buffer) |
| |
| self._test_fit_transition_middle_buffer_pass(**kwargs) |
| gd_middle_buffer = kwargs['GD'] |
| self._assert_gd_is_valid(gd_middle_buffer) |
| self.assertFalse(iter_sess._closed) |
| |
| self._test_fit_transition_last_buffer_pass(**kwargs) |
| gd_last_buffer = kwargs['GD'] |
| self._assert_gd_is_valid(gd_last_buffer) |
| self.assertFalse(iter_sess._closed) |
| return iter_sess |
| |
| def _run_fit_multiple_iteration(self, **kwargs): |
| self._test_fit_transition_multiple_model_no_cache_first_buffer_pass(**kwargs) |
| self._assert_gd_is_valid(kwargs['GD']) |
| self._assert_keras_session_same_as_gd_session(kwargs['GD']) |
| iter_sess = kwargs['GD']['sess'] |
| self.assertFalse(iter_sess._closed) |
| |
| self._test_fit_transition_multiple_model_no_cache_middle_buffer_pass(**kwargs) |
| self._assert_gd_is_valid(kwargs['GD']) |
| self._assert_keras_session_same_as_gd_session(kwargs['GD']) |
| self.assertFalse(iter_sess._closed) |
| |
| self._test_fit_transition_multiple_model_no_cache_last_buffer_pass(**kwargs) |
| self._assert_gd_cleared(kwargs['GD']) |
| self.assertTrue(iter_sess._closed) |
| return iter_sess |
| |
| def _assert_keras_session_same_as_gd_session(self, gd): |
| sess = self.subject.K.get_session() |
| self.assertEquals(sess, gd['sess']) |
| |
| def _assert_gd_cleared(self, gd): |
| self.assertEquals(0, len(gd.keys())) |
| |
| def _assert_gd_is_valid(self, gd): |
| self.assertTrue(gd['sess']) |
| self.assertTrue(gd['segment_model']) |
| |
| ################################################################ |
| |
| def test_fit_transition_first_tuple_none_ind_var_dep_var(self): |
| k = {} |
| self.assertEqual('dummy_state', |
| self.subject.fit_transition('dummy_state', [0], None, |
| 'noshape', 'noshape', |
| 'dummy_model_json', "foo", "bar", |
| 1, [0,1,2], 0, 4, [3,3,3], False, |
| [0], 'dummy_prev_state', **k)) |
| self.assertEqual('dummy_state', |
| self.subject.fit_transition('dummy_state', None, [[0.5]], |
| 'noshape', 'noshape', |
| 'dummy_model_json', "foo", "bar", |
| 1, [0,1,2], 0, 4, [3,3,3], False, |
| [0], 'dummy_prev_state', **k)) |
| self.assertEqual('dummy_state', |
| self.subject.fit_transition('dummy_state', None, None, |
| 'noshape', 'noshape', |
| 'dummy_model_json', "foo", "bar", |
| 1, [0,1,2], 0, 4, [3,3,3], False, |
| [0], 'dummy_prev_state', **k)) |
| |
| def test_fit_merge(self): |
| image_count = self.total_images_per_seg[0] |
| state1 = [image_count] |
| state1.extend(mult(3,self.model_weights)) |
| state1 = np.array(state1, dtype=np.float32) |
| state2 = [image_count+30] |
| state2.extend(mult(2,self.model_weights)) |
| state2 = np.array(state2, dtype=np.float32) |
| merged_state = self.subject.fit_merge(state1.tostring(),state2.tostring()) |
| state = np.fromstring(merged_state, dtype=np.float32) |
| image_count_total = state[0] |
| weights = np.rint(state[1:]).astype(np.int) |
| |
| self.assertEqual( 2*image_count+30 , image_count_total ) |
| self.assertTrue( (mult(5,self.model_weights) == weights).all()) |
| |
| def test_fit_merge_none_first(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [image_count] |
| input_state.extend(self.model_weights) |
| input_state = np.array(input_state, dtype=np.float32) |
| merged_state = self.subject.fit_merge(None, input_state.tostring()) |
| state = np.fromstring(merged_state, dtype=np.float32) |
| image_count_total = state[0] |
| weights = np.rint(state[1:]).astype(np.int) |
| |
| self.assertEqual(image_count, image_count_total) |
| self.assertTrue((self.model_weights == weights).all()) |
| |
| def test_fit_merge_none_second(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [image_count] |
| input_state.extend(self.model_weights) |
| input_state = np.array(input_state, dtype=np.float32) |
| merged_state = self.subject.fit_merge(input_state.tostring(), None) |
| state = np.fromstring(merged_state, dtype=np.float32) |
| image_count_total = state[0] |
| weights = np.rint(state[1:]).astype(np.int) |
| |
| self.assertEqual(image_count, image_count_total) |
| self.assertTrue((self.model_weights == weights).all()) |
| |
| def test_fit_merge_both_none(self): |
| result = self.subject.fit_merge(None,None) |
| self.assertEqual(None, result) |
| |
| def test_fit_final(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [image_count] |
| input_state.extend(mult(image_count,self.model_weights)) |
| input_state = np.array(input_state, dtype=np.float32) |
| |
| output_state = self.subject.fit_final(input_state.tostring()) |
| output_state = np.fromstring(output_state, dtype=np.float32) |
| weights = np.rint(output_state).astype(np.int) |
| |
| self.assertTrue((self.model_weights == weights).all()) |
| |
| def test_fit_final_none(self): |
| result = self.subject.fit_final(None) |
| self.assertEqual(result, None) |
| |
| def test_fit_final_image_count_zero(self): |
| input_state = [0] |
| input_state.extend(self.model_weights) |
| input_state = np.array(input_state, dtype=np.float32) |
| |
| with self.assertRaises(plpy.PLPYException): |
| result = self.subject.fit_final(input_state.tostring()) |
| |
| def test_should_compute_metrics_this_iter_last_iter_case1(self): |
| res = self.subject.should_compute_metrics_this_iter(5, 5, 5) |
| self.assertEqual(True, res) |
| def test_should_compute_metrics_this_iter_last_iter_case2(self): |
| res = self.subject.should_compute_metrics_this_iter(5, 3, 5) |
| self.assertEqual(True, res) |
| def test_should_compute_metrics_this_iter_before_interval(self): |
| res = self.subject.should_compute_metrics_this_iter(1, 3, 5) |
| self.assertEqual(False, res) |
| def test_should_compute_metrics_this_iter_after_interval(self): |
| res = self.subject.should_compute_metrics_this_iter(4, 3, 5) |
| self.assertEqual(False, res) |
| def test_should_compute_metrics_this_iter_at_interval(self): |
| res = self.subject.should_compute_metrics_this_iter(3, 3, 5) |
| self.assertEqual(True, res) |
| def test_should_compute_metrics_this_iter_every_iter(self): |
| res = self.subject.should_compute_metrics_this_iter(2, 1, 5) |
| self.assertEqual(True, res) |
| |
| |
| class InternalKerasPredictTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras_predict |
| self.subject = madlib_keras_predict |
| |
| self.model = Sequential() |
| self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', |
| input_shape=(1,1,1,), padding='same')) |
| self.model.add(Flatten()) |
| |
| self.all_seg_ids = [0,1,2] |
| |
| self.independent_var = [[[240]]] |
| self.total_images_per_seg = [3,3,4] |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_predict_first_image_pass_gpdb(self): |
| self.subject.is_platform_pg = Mock(return_value = False) |
| model_weights = [1, 2, 3, 4] |
| serialized_weights = np.array(model_weights, dtype=np.float32).tostring() |
| |
| k = {'SD': {}} |
| result = self.subject.internal_keras_predict( |
| self.independent_var, self.model.to_json(), |
| serialized_weights, 255, 0, self.all_seg_ids, |
| self.total_images_per_seg, 0, 4, **k) |
| self.assertEqual(2, len(result)) |
| self.assertEqual(1, k['SD']['row_count']) |
| self.assertEqual(True, 'segment_model_predict' in k['SD']) |
| |
| def test_predict_middle_image_pass_gpdb(self): |
| self.subject.is_platform_pg = Mock(return_value = False) |
| |
| k = {'SD': { 'row_count': 1}} |
| k['SD']['segment_model_predict'] = self.model |
| result = self.subject.internal_keras_predict( |
| self.independent_var, None, None, 255, 0, |
| self.all_seg_ids, self.total_images_per_seg, 0, 4, **k) |
| self.assertEqual(2, len(result)) |
| self.assertEqual(2, k['SD']['row_count']) |
| self.assertEqual(True, 'segment_model_predict' in k['SD']) |
| |
| def test_predict_last_image_pass_gpdb(self): |
| self.subject.is_platform_pg = Mock(return_value = False) |
| self.model.add(Dense(3)) |
| |
| k = {'SD': { 'row_count': 2}} |
| k['SD']['segment_model_predict'] = self.model |
| result = self.subject.internal_keras_predict( |
| self.independent_var, None, None, 255, 0, |
| self.all_seg_ids, self.total_images_per_seg, 0, 4, **k) |
| |
| # we except len(result) to be 3 because we have 3 dense layers in the |
| # architecture |
| self.assertEqual(3, len(result)) |
| self.assertEqual(False, 'row_count' in k['SD']) |
| self.assertEqual(False, 'segment_model_predict' in k['SD']) |
| |
| def test_predict_error_should_clear_sd(self): |
| self.subject.is_platform_pg = Mock(return_value = False) |
| # self.model.add(Dense(3)) |
| model_weights = [1, 2, 3, 4] |
| serialized_weights = np.array(model_weights, dtype=np.float32).tostring() |
| |
| # inject error by passing current_seg_id as -1 |
| current_seg_id = -1 |
| k = {'SD':{}} |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.internal_keras_predict( |
| self.independent_var, self.model.to_json(), serialized_weights, |
| 255, current_seg_id, self.all_seg_ids, |
| self.total_images_per_seg, 0, 4, **k) |
| self.assertEqual("ValueError('-1 is not in list',)", str(error.exception)) |
| self.assertEqual(False, 'row_count' in k['SD']) |
| self.assertEqual(False, 'segment_model_predict' in k['SD']) |
| |
| |
| class MadlibKerasPredictBYOMTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| self.num_classes = 5 |
| self.model = Sequential() |
| self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', |
| input_shape=(1,1,1,), padding='same')) |
| self.model.add(Dense(self.num_classes)) |
| |
| self.pred_type = 'prob' |
| self.use_gpus = False |
| self.class_values = ['foo', 'bar', 'baaz', 'foo2', 'bar2'] |
| self.normalizing_const = 255.0 |
| |
| import madlib_keras_predict |
| self.module = madlib_keras_predict |
| self.module.get_model_arch_weights = Mock(return_value=( |
| self.model.to_json(), 'weights')) |
| self.module.InputValidator.validate_predict_byom_tables = Mock() |
| self.module.InputValidator.validate_input_shape = Mock() |
| self.module.BasePredict.call_internal_keras = Mock() |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_predictbyom_defaults_1(self): |
| self.module.get_accessible_gpus_for_seg = Mock(return_value = [2,2,2]) |
| res = self.module.PredictBYOM('schema_madlib', 'model_arch_table', |
| 'model_id', 'test_table', 'id_col', |
| 'independent_varname', 'output_table', None, |
| True, None, None) |
| self.assertEqual('prob', res.pred_type) |
| self.assertEqual(2, res.gpus_per_host) |
| self.assertEqual([0,1,2,3,4], res.class_values) |
| self.assertEqual(1.0, res.normalizing_const) |
| |
| def test_predictbyom_defaults_2(self): |
| res = self.module.PredictBYOM('schema_madlib', 'model_arch_table', |
| 'model_id', 'test_table', 'id_col', |
| 'independent_varname', 'output_table', |
| self.pred_type, self.use_gpus, |
| self.class_values, self.normalizing_const) |
| self.assertEqual('prob', res.pred_type) |
| self.assertEqual(0, res.gpus_per_host) |
| self.assertEqual(['foo', 'bar', 'baaz', 'foo2', 'bar2'], res.class_values) |
| self.assertEqual(255.0, res.normalizing_const) |
| |
| def test_predictbyom_exception_invalid_params(self): |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.module.PredictBYOM('schema_madlib', 'model_arch_table', |
| 'model_id', 'test_table', 'id_col', |
| 'independent_varname', 'output_table', |
| 'invalid_pred_type', self.use_gpus, |
| self.class_values, self.normalizing_const) |
| self.assertIn('invalid_pred_type', str(error.exception)) |
| |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.module.PredictBYOM('schema_madlib', 'model_arch_table', |
| 'model_id', 'test_table', 'id_col', |
| 'independent_varname', 'output_table', |
| self.pred_type, self.use_gpus, |
| ["foo", "bar", "baaz"], self.normalizing_const) |
| self.assertIn('class values', str(error.exception).lower()) |
| |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.module.PredictBYOM('schema_madlib', 'model_arch_table', |
| 'model_id', 'test_table', 'id_col', |
| 'independent_varname', 'output_table', |
| self.pred_type, self.use_gpus, |
| self.class_values, 0) |
| self.assertIn('normalizing const', str(error.exception).lower()) |
| |
| |
| class MadlibKerasWrapperTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| |
| import madlib_keras_wrapper |
| self.subject = madlib_keras_wrapper |
| |
| self.use_gpus = False |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_get_gpu_memory_fraction(self): |
| |
| gpus_per_host = 4 |
| segments_per_host = 4 |
| result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) |
| self.assertEqual(result, 0.9) |
| |
| gpus_per_host = 10 |
| segments_per_host = 4 |
| result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) |
| self.assertEqual(result, 0.9) |
| |
| gpus_per_host = 2 |
| segments_per_host = 6 |
| result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) |
| self.assertEqual(result, 0.3) |
| |
| gpus_per_host = 1 |
| segments_per_host = 4 |
| result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) |
| self.assertEqual(result, 0.225) |
| |
| def test_get_device_name_and_set_cuda_env_postgres(self): |
| self.subject.is_platform_pg = Mock(return_value = True) |
| |
| seg_id = -1 |
| gpus_per_host = 3 |
| |
| self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env( |
| gpus_per_host, seg_id )) |
| self.assertEqual('0,1,2', os.environ['CUDA_VISIBLE_DEVICES']) |
| |
| gpus_per_host = 0 |
| self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env( |
| gpus_per_host, seg_id )) |
| self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES']) |
| |
| def test_get_device_name_and_set_cuda_env_gpdb(self): |
| self.subject.is_platform_pg = Mock(return_value = False) |
| |
| seg_id=3 |
| gpus_per_host=2 |
| self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env( |
| gpus_per_host, seg_id)) |
| self.assertEqual('1', os.environ['CUDA_VISIBLE_DEVICES']) |
| |
| gpus_per_host=0 |
| self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env( |
| gpus_per_host, seg_id)) |
| self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES']) |
| |
| |
| def test_split_and_strip(self): |
| self.assertEqual(('a','b'), self.subject.split_and_strip(' a = b ')) |
| |
| def test_parse_optimizer(self): |
| opt_name = 'SGD' |
| final_args = {'lr':0.01, 'decay':1e-6, 'nesterov':True} |
| compile_dict = {} |
| compile_dict['optimizer']='SGD(lr=0.01, decay=1e-6, nesterov=True)' |
| result_name, result_params = self.subject.parse_optimizer(compile_dict) |
| |
| self.assertEqual(result_name, opt_name) |
| self.assertDictEqual(result_params, final_args) |
| |
| def test_validate_and_literal_eval_keys(self): |
| test_dict = {'batch_size':'2', 'epochs':'1', 'verbose':'0'} |
| target_dict = {'batch_size':2, 'epochs':1, 'verbose':0} |
| literal_eval_fit_params = ['batch_size','epochs','verbose','shuffle', |
| 'class_weight','initial_epoch','steps_per_epoch'] |
| accepted_fit_params = literal_eval_fit_params + ['shuffle'] |
| result_params = self.subject.validate_and_literal_eval_keys( |
| test_dict, |
| literal_eval_fit_params, |
| accepted_fit_params) |
| self.assertDictEqual(result_params, target_dict) |
| |
| def test_parse_and_validate_fit_params(self): |
| result = {'batch_size':2, 'epochs':1, 'verbose':0} |
| self.assertDictEqual(result, self.subject.parse_and_validate_fit_params('batch_size=2, epochs=1, verbose=0')) |
| |
| def test_parse_and_validate_fit_params(self): |
| test_str = "batch_size=2, epochs=1, verbose=0" |
| fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0} |
| result_params = self.subject.parse_and_validate_fit_params(test_str) |
| self.assertDictEqual(result_params, fit_dict) |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, shuffle = 'batch'" |
| fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':'batch'} |
| result_params = self.subject.parse_and_validate_fit_params(test_str) |
| self.assertDictEqual(result_params, fit_dict) |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, shuffle = True" |
| fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':True} |
| result_params = self.subject.parse_and_validate_fit_params(test_str) |
| self.assertDictEqual(result_params, fit_dict) |
| |
| test_str = "" |
| fit_dict = {} |
| result_params = self.subject.parse_and_validate_fit_params(test_str) |
| self.assertDictEqual(result_params, fit_dict) |
| |
| def test_parse_and_validate_fit_params_duplicate_params(self): |
| test_str = "batch_size=1, batch_size=10, verbose=0" |
| fit_dict = {'batch_size':10, 'verbose':0} |
| result_params = self.subject.parse_and_validate_fit_params(test_str) |
| self.assertDictEqual(result_params, fit_dict) |
| |
| def test_parse_and_validate_fit_params_invalid_fit_param_fail(self): |
| test_str = "does_not_exist=2, epochs=1, verbose=0" |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| self.assertIn('not currently accepted', str(error.exception)) |
| |
| test_str = "batch_size=not_lit_eval(1), epochs=1, verbose=0" |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| self.assertIn('invalid input value', str(error.exception)) |
| |
| def test_parse_and_validate_compile_params(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ |
| "loss='categorical_crossentropy', metrics=['accuracy']" |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['accuracy'], 'loss':'categorical_crossentropy'} |
| opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str) |
| self.assertDictEqual(result_params, compile_dict) |
| |
| # Test without the metrics |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ |
| "loss='categorical_crossentropy'" |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'loss':'categorical_crossentropy'} |
| opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str) |
| self.assertDictEqual(result_params, compile_dict) |
| |
| def test_validate_metrics_None_pass(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['accuracy'], 'loss':'categorical_crossentropy'} |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_validate_metrics_empty_pass(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':[], 'loss':'categorical_crossentropy'} |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_validate_metrics_two_params_fail(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['accuracy','mae'], 'loss':'categorical_crossentropy'} |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_validate_metrics_one_unsupported_fail(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['sparse_categorical_accuracy'], 'loss':'categorical_crossentropy'} |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_validate_metrics_two_unsupported_fail(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['sparse_categorical_accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'} |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_validate_metrics_one_supported_one_unsupported_fail(self): |
| compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', |
| 'metrics':['accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'} |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject._validate_metrics(compile_dict) |
| |
| def test_parse_and_validate_compile_params_default_optimizer_pass(self): |
| test_str = "optimizer='SGD', loss='categorical_crossentropy'" |
| _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) |
| self.assertEqual('SGD', result_dict['optimizer']) |
| |
| test_str = "optimizer='sgd()', loss='categorical_crossentropy'" |
| _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) |
| self.assertEqual('sgd()', result_dict['optimizer']) |
| |
| def test_parse_and_validate_compile_params_duplicate_loss(self): |
| test_str = "optimizer='SGD', loss='loss1', metrics=['accuracy'], loss='loss2'" |
| _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) |
| self.assertEqual('loss2', result_dict['loss']) |
| |
| def test_parse_and_validate_compile_params_no_optimizer_fail(self): |
| test_str = "loss='categorical_crossentropy', metrics=['accuracy']" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_compile_params_no_loss_fail(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ |
| "metrics=['accuracy']" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_compile_params_unsupported_loss_fail(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ |
| "metrics=['accuracy'], loss='sparse_categorical_crossentropy'" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_compile_params_dict_metrics_fail(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ |
| "loss='categorical_crossentropy', metrics={'0':'accuracy'}" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_compile_params_tensor_loss_weights_fail(self): |
| |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy', metrics=['accuracy']," \ |
| " loss_weights = keras.layers.Input(shape=(32,))" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_compile_params_list_dict_sample_weight_mode_fail(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy', metrics=['accuracy']," \ |
| " sample_weight_mode = [0,1]" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy', metrics=['accuracy']," \ |
| " sample_weight_mode = {'0':'1'}" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| |
| def test_parse_and_validate_compile_params_target_tensors_fail(self): |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy', metrics=['accuracy']," \ |
| " target_tensors = keras.layers.Input(shape=(32,))" |
| |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_compile_params(test_str) |
| |
| def test_parse_and_validate_fit_params_callbacks_fail(self): |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, callbacks=keras.callbacks.Callback()" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_fit_params_validation_split_fail(self): |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, validation_split=0.1" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_fit_params_validation_data_fail(self): |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, validation_data=(1,2,3)" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_fit_params_sample_weight_fail(self): |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, sample_weight=np.array([1,2,3])" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_fit_params_validation_steps_fail(self): |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, validation_steps=1" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_fit_params_validation_freq_fail(self): |
| test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=1" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=[1]" |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.parse_and_validate_fit_params(test_str) |
| |
| def test_parse_and_validate_compile_params_syntax_error_fail(self): |
| #missing end parentheses |
| test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True" |
| with self.assertRaises(ValueError) as error: |
| self.subject.parse_and_validate_compile_params(test_str) |
| self.assertIn('could not convert string to float', str(error.exception)) |
| |
| #missing beginning parentheses |
| test_str = "optimizer=SGDlr=0.01, decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy'" |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.parse_and_validate_compile_params(test_str) |
| self.assertIn('not currently accepted', str(error.exception)) |
| |
| #missing comma |
| test_str = "optimizer=SGD(lr=0.01 decay=1e-6, nesterov=True)," \ |
| " loss='categorical_crossentropy'" |
| with self.assertRaises(ValueError) as error: |
| self.subject.parse_and_validate_compile_params(test_str) |
| self.assertIn('invalid literal for float', str(error.exception)) |
| |
| def test_parse_and_validate_fit_params_invalid_optimizer_fail(self): |
| test_str = "optimizer='SGD1', loss='categorical_crossentropy'" |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.parse_and_validate_compile_params(test_str) |
| self.assertIn('invalid optimizer', str(error.exception)) |
| |
| |
| class MadlibKerasFitCommonValidatorTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras_validator |
| self.subject = madlib_keras_validator |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| |
| def test_is_valid_metrics_compute_frequency_True_None(self): |
| self.subject.FitCommonValidator._validate_common_args = Mock() |
| obj = self.subject.FitCommonValidator( |
| 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, |
| 'dep_varname', 'independent_varname', 5, None, False, False, [0], |
| 'module_name', None) |
| self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) |
| |
| def test_is_valid_metrics_compute_frequency_True_num(self): |
| self.subject.FitCommonValidator._validate_common_args = Mock() |
| obj = self.subject.FitCommonValidator( |
| 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, |
| 'dep_varname', 'independent_varname', 5, 3, False, False, [0], |
| 'module_name', None) |
| self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) |
| |
| def test_is_valid_metrics_compute_frequency_False_zero(self): |
| self.subject.FitCommonValidator._validate_common_args = Mock() |
| obj = self.subject.FitCommonValidator( |
| 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, |
| 'dep_varname', 'independent_varname', 5, 0, False, False, [0], |
| 'module_name', None) |
| self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) |
| |
| def test_is_valid_metrics_compute_frequency_False_greater(self): |
| self.subject.FitCommonValidator._validate_common_args = Mock() |
| obj = self.subject.FitCommonValidator( |
| 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, |
| 'dep_varname', 'independent_varname', 5, 6, False, False, [0], |
| 'module_name', None) |
| self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) |
| |
| |
| class InputValidatorTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras_validator |
| self.module = madlib_keras_validator |
| self.subject = self.module.InputValidator |
| |
| self.module_name = 'module' |
| self.test_table = 'test_table' |
| self.model_table = 'model_table' |
| self.id_col = 'id_col' |
| self.ind_var = 'ind_var' |
| self.model_arch_table = 'model_arch_table' |
| self.model_id = 2 |
| self.num_classes = 1598 |
| self.model = Sequential() |
| self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', |
| input_shape=(1,1,1,), padding='same')) |
| self.model.add(Dense(self.num_classes)) |
| self.classes = ['train', 'boat', 'car', 'airplane'] |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_validate_pred_type_invalid_pred_type(self): |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.validate_pred_type( |
| self.module_name, 'invalid_pred_type', ['cat', 'dog']) |
| self.assertIn('type', str(error.exception).lower()) |
| |
| def test_validate_class_values_greater_than_1600_class_values(self): |
| self.model.add(Dense(1599)) |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.validate_class_values( |
| self.module_name, range(1599), 'prob', self.model.to_json()) |
| self.assertIn('1600', str(error.exception)) |
| |
| def test_validate_class_values_valid_class_values_prob(self): |
| self.subject.validate_class_values( |
| self.module_name, range(self.num_classes), 'prob', self.model.to_json()) |
| self.subject.validate_class_values( |
| self.module_name, None, 'prob', self.model.to_json()) |
| |
| def test_validate_class_values_valid_pred_type_valid_class_values_response(self): |
| self.subject.validate_class_values( |
| self.module_name, range(self.num_classes), 'response', self.model.to_json()) |
| self.subject.validate_class_values( |
| self.module_name, None, 'response', self.model.to_json()) |
| |
| def test_validate_input_shape_shapes_do_not_match(self): |
| # minibatched data |
| self.plpy_mock_execute.return_value = [{'shape': [1,3,32,32]}] |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.validate_input_shape( |
| self.test_table, self.ind_var, [32,32,3], 2, True) |
| # non-minibatched data |
| self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 32,'n_2': 32,'n_3': 3}] |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.validate_input_shape( |
| self.test_table, self.ind_var, [32,32,3], 1) |
| self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 3}] |
| with self.assertRaises(plpy.PLPYException): |
| self.subject.validate_input_shape( |
| self.test_table, self.ind_var, [3,32], 1) |
| |
| def test_validate_input_shape_shapes_match(self): |
| # minibatched data |
| self.plpy_mock_execute.return_value = [{'shape': [1,32,32,3]}] |
| self.subject.validate_input_shape( |
| self.test_table, self.ind_var, [32,32,3], 2, True) |
| # non-minibatched data |
| self.plpy_mock_execute.return_value = [{'n_0': 32,'n_1': 32,'n_2': 3}] |
| self.subject.validate_input_shape( |
| self.test_table, self.ind_var, [32,32,3], 1) |
| |
| def test_validate_model_arch_table_none_values(self): |
| with self.assertRaises(plpy.PLPYException) as error: |
| obj = self.subject.validate_model_arch_table( |
| self.module_name, None, self.model_id) |
| self.assertIn('null', str(error.exception).lower()) |
| |
| self.module.input_tbl_valid = Mock() |
| with self.assertRaises(plpy.PLPYException) as error: |
| obj = self.subject.validate_model_arch_table( |
| self.module_name, self.model_arch_table, None) |
| self.assertIn('id', str(error.exception).lower()) |
| |
| def test_validate_gpu_config_with_gpu_all_segments(self): |
| |
| self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}] |
| obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1]) |
| |
| def test_validate_gpu_config_no_gpu_all_segments(self): |
| |
| self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}] |
| with self.assertRaises(plpy.PLPYException) as error: |
| obj = self.subject._validate_gpu_config(self.module_name, 'foo', [0]) |
| self.assertIn('missing gpus', str(error.exception).lower()) |
| |
| def test_validate_gpu_config_with_gpu_valid_seg_list(self): |
| |
| self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}] |
| obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,1,0,1]) |
| |
| def test_validate_gpu_config_with_gpu_invalid_seg_list(self): |
| |
| self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}] |
| with self.assertRaises(plpy.PLPYException) as error: |
| obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,0,0,1]) |
| self.assertIn('does not have gpu', str(error.exception).lower()) |
| |
| |
| class MadlibSerializerTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras_serializer |
| self.subject = madlib_keras_serializer |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_deserialize_image_1d_weights_null_state_returns_none(self): |
| self.assertEqual(None, self.subject.deserialize_as_image_1d_weights(None)) |
| |
| def test_deserialize_image_1d_weights_returns_not_none(self): |
| dummy_state = np.array([2,3,4,5,6], dtype=np.float32) |
| res = self.subject.deserialize_as_image_1d_weights(dummy_state.tostring()) |
| self.assertEqual(2, res[0]) |
| self.assertEqual([3,4,5,6], res[1].tolist()) |
| |
| def test_deserialize_nd_weights_null_input_returns_none(self): |
| dummy_state = np.array([0,1,2,3,4,5,6], dtype=np.float32) |
| self.assertEqual(None, self.subject.deserialize_as_nd_weights(dummy_state.tostring(), None)) |
| self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, [1, 2, 3])) |
| self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, None)) |
| |
| def test_deserialize_nd_weights_valid_input_returns_not_none(self): |
| dummy_model_weights = np.array([3,4,5], dtype=np.float32) |
| dummy_model_shape = [(2, 1, 1, 1), (1,)] |
| res = self.subject.deserialize_as_nd_weights(dummy_model_weights.tostring(), |
| dummy_model_shape) |
| self.assertEqual([[[[3.0]]], [[[4.0]]]], res[0].tolist()) |
| self.assertEqual([5], res[1].tolist()) |
| |
| def test_deserialize_nd_weights_invalid_input_fails(self): |
| # pass an invalid state with missing model weights |
| invalid_model_weights = np.array([1,2], dtype=np.float32) |
| dummy_model_shape = [(2, 1, 1, 1), (1,)] |
| |
| # we expect raise exception because we cannot reshape |
| # model weights of size 0 into shape (2,2,3,1) |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), |
| dummy_model_shape) |
| |
| invalid_model_weights = np.array([1,2,3,4], dtype=np.float32) |
| dummy_model_shape = [(2, 2, 3, 1), (1,)] |
| # we expect raise exception because we cannot reshape |
| # model weights of size 2 into shape (2,2,3,1) |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), |
| dummy_model_shape) |
| |
| invalid_model_weights = np.array([0,1,2,3,4], dtype=np.float32) |
| dummy_model_shape = [(2, 1), (1,)] |
| # we expect to raise exception because we cannot reshape |
| # model weights of size 2 into shape (1,) |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), |
| dummy_model_shape) |
| |
| def test_serialize_image_nd_weights_none_weights_returns_none(self): |
| res = self.subject.serialize_state_with_nd_weights(0, None) |
| self.assertEqual(None , res) |
| |
| def test_serialize_image_nd_weights_valid_output(self): |
| res = self.subject.serialize_state_with_nd_weights(0, [np.array([1, 3]), |
| np.array([4,5])]) |
| self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(), |
| res) |
| |
| def test_serialize_image_1d_weights_none_weights_returns_none(self): |
| res = self.subject.serialize_state_with_1d_weights(0, None) |
| self.assertEqual(None , res) |
| |
| def test_serialize_image_1d_weights_valid_output(self): |
| res = self.subject.serialize_state_with_1d_weights(0, np.array([1, 3, 4, 5])) |
| self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(), |
| res) |
| |
| |
| class MadlibKerasHelperTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras_helper |
| self.subject = madlib_keras_helper |
| self.input_data = [32, 32, 3] |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_expand_input_dims(self): |
| self.assertEqual(np.array(self.input_data).shape, (3,)) |
| res = self.subject.expand_input_dims(self.input_data) |
| self.assertEqual(res.shape, (1, 3)) |
| |
| def test_strip_trailing_nulls_from_class_values(self): |
| self.assertEqual(['cat', 'dog'], |
| self.subject.strip_trailing_nulls_from_class_values( |
| ['cat', 'dog'])) |
| self.assertEqual([None, 'cat', 'dog'], |
| self.subject.strip_trailing_nulls_from_class_values( |
| [None, 'cat', 'dog'])) |
| self.assertEqual([None, 'cat', 'dog'], |
| self.subject.strip_trailing_nulls_from_class_values( |
| [None, 'cat', 'dog', None, None])) |
| self.assertEqual(['cat', 'dog'], |
| self.subject.strip_trailing_nulls_from_class_values( |
| ['cat', 'dog', None, None])) |
| self.assertEqual([None], |
| self.subject.strip_trailing_nulls_from_class_values( |
| [None, None])) |
| |
| def test_get_gpus_per_one_seg_gpu_gpdb(self): |
| |
| self.subject.is_platform_pg = Mock(return_value = False) |
| |
| self.plpy_mock_execute.side_effect = \ |
| [ [], |
| [ {'hostname': 'mdw0', 'count' : 1}], |
| [], |
| [ {'hostname': 'mdw0', 'segment_id' : 0}, |
| {'hostname': 'mdw1', 'segment_id' : 1}, |
| {'hostname': 'mdw2', 'segment_id' : 2} |
| ]] |
| |
| self.assertEqual([1,0,0], self.subject.get_accessible_gpus_for_seg( |
| 'schema_madlib', 2, 'foo')) |
| |
| def test_get_gpus_per_mult_seg_gpu_gpdb(self): |
| |
| self.subject.is_platform_pg = Mock(return_value = False) |
| |
| self.plpy_mock_execute.side_effect = \ |
| [[], |
| [ {'hostname': 'mdw0', 'count' : 1}], |
| [], |
| [ {'hostname': 'mdw0', 'segment_id' : 0}, |
| {'hostname': 'mdw0', 'segment_id' : 1}, |
| {'hostname': 'mdw1', 'segment_id' : 2}, |
| {'hostname': 'mdw1', 'segment_id' : 3} |
| ]] |
| |
| self.assertEqual([1,1,0,0], self.subject.get_accessible_gpus_for_seg( |
| 'schema_madlib', 2, 'foo')) |
| |
| def test_get_gpus_per_no_gpu_gpdb(self): |
| |
| self.subject.is_platform_pg = Mock(return_value = False) |
| |
| self.plpy_mock_execute.side_effect = [[],[],[]] |
| with self.assertRaises(plpy.PLPYException) as error: |
| self.subject.get_accessible_gpus_for_seg('schema_madlib', 2, 'foo') |
| self.assertIn('no gpus configured on hosts', str(error.exception).lower()) |
| |
| def test_get_metrics_sql_string(self): |
| NaN = float('nan') |
| test_metrics = [ 3.2, NaN, 0.0, None, 8.94, -6.8, 1.2, NaN, NaN ] |
| |
| py2sql = { x : str(x) for x in test_metrics if type(x) == float } |
| py2sql[NaN] = "'NaN'::DOUBLE PRECISION" |
| py2sql[None] = 'NULL' |
| correct = [ py2sql[x] for x in test_metrics ] |
| |
| final, metrics = self.subject.get_metrics_sql_string(test_metrics) |
| |
| m = re.match("\(ARRAY\[(.*)\]\)", metrics) |
| answers = m.group(1).split(', ') |
| self.assertListEqual(answers, correct) |
| self.assertEqual(final, '(' + correct[-1] + ')') |
| |
| # Check that postgresql parser sees this as a valid sql data type |
| res = pglint('SELECT ' + final, True) |
| self.assertEqual( |
| res, |
| (True, ''), |
| "This is not valid PostgresSQL: {}".format('SELECT' + final) |
| ) |
| |
| # Check that postgresql parser sees this as a valid array |
| res = pglint('SELECT {}[1]'.format(metrics), True) |
| self.assertEqual( |
| res, |
| (True, ''), |
| "This is not valid PostgresSQL: SELECT {}[1]".format(metrics) |
| ) |
| |
| class MadlibKerasEvaluationMergeFinalTestCase(unittest.TestCase): |
| def setUp(self): |
| self.plpy_mock = Mock(spec='error') |
| patches = { |
| 'plpy': plpy, |
| 'utilities.mean_std_dev_calculator': Mock() |
| } |
| |
| self.plpy_mock_execute = MagicMock() |
| plpy.execute = self.plpy_mock_execute |
| |
| self.module_patcher = patch.dict('sys.modules', patches) |
| self.module_patcher.start() |
| import madlib_keras |
| self.subject = madlib_keras |
| |
| self.model = Sequential() |
| self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', |
| input_shape=(1,1,1,), padding='same')) |
| self.model.add(Flatten()) |
| |
| self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']" |
| self.model_weights = [3,4,5,6] |
| self.serialized_weights = np.array(self.model_weights, dtype='float32' |
| ).tostring() |
| self.loss = 0.5947071313858032 |
| self.accuracy = 1.0 |
| self.dist_key_mapping = [0,1,2] |
| self.accessible_gpus_for_seg = [0] |
| |
| #self.model.evaluate = Mock(return_value = [self.loss, self.accuracy]) |
| |
| self.independent_var_real = [[[[0.5]]]] * 10 |
| self.dependent_var_int = [[0,1]] * 10 |
| |
| # Params as bytea |
| self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes() |
| self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes() |
| |
| self.independent_var_shape = [10,1,1,1] |
| self.dependent_var_shape = [10,2] |
| # We test on segment 0, which has 3 buffers filled with 10 identical |
| # images each, or 30 images total |
| self.total_images_per_seg = [3*len(self.dependent_var_int),20,40] |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| def test_internal_keras_eval_merge(self): |
| image_count = self.total_images_per_seg[0] |
| state1 = [3.0*self.loss, 3.0*self.accuracy, image_count] |
| state1 = state1 |
| state2 = [2.0*self.loss, 2.0*self.accuracy, image_count+30] |
| state2 = state2 |
| merged_state = self.subject.internal_keras_eval_merge(state1,state2) |
| agg_loss = merged_state[0] |
| agg_accuracy = merged_state[1] |
| image_count_total = merged_state[2] |
| |
| self.assertEqual( 2*image_count+30 , image_count_total ) |
| self.assertAlmostEqual( 5.0*self.loss, agg_loss, 2) |
| self.assertAlmostEqual( 5.0*self.accuracy, agg_accuracy, 2) |
| |
| def test_internal_keras_eval_merge_none_first(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [self.loss, self.accuracy, image_count] |
| merged_state = self.subject.internal_keras_eval_merge(None, input_state) |
| agg_loss = merged_state[0] |
| agg_accuracy = merged_state[1] |
| image_count_total = merged_state[2] |
| |
| self.assertEqual(image_count, image_count_total) |
| self.assertAlmostEqual(self.loss, agg_loss, 2) |
| self.assertAlmostEqual(self.accuracy, agg_accuracy, 2) |
| |
| def test_internal_keras_eval_merge_none_second(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [self.loss, self.accuracy, image_count] |
| merged_state = self.subject.internal_keras_eval_merge(input_state, None) |
| agg_loss = merged_state[0] |
| agg_accuracy = merged_state[1] |
| image_count_total = merged_state[2] |
| |
| self.assertEqual(image_count, image_count_total) |
| self.assertAlmostEqual(self.loss, agg_loss, 2) |
| self.assertAlmostEqual(self.accuracy, agg_accuracy, 2) |
| |
| def test_internal_keras_eval_merge_both_none(self): |
| result = self.subject.internal_keras_eval_merge(None,None) |
| self.assertEqual(None, result) |
| |
| def test_internal_keras_eval_final(self): |
| image_count = self.total_images_per_seg[0] |
| input_state = [image_count*self.loss, image_count*self.accuracy, image_count] |
| |
| output_state = self.subject.internal_keras_eval_final(input_state) |
| self.assertEqual(len(output_state), 2) |
| agg_loss = output_state[0] |
| agg_accuracy = output_state[1] |
| |
| self.assertAlmostEqual(self.loss, agg_loss,2) |
| self.assertAlmostEqual(self.accuracy, agg_accuracy,2) |
| |
| def internal_keras_eval_final_none(self): |
| result = self.subject.internal_keras_eval_final(None) |
| self.assertEqual(result, None) |
| |
| def test_internal_keras_eval_final_image_count_zero(self): |
| input_state = [0, 0, 0] |
| |
| with self.assertRaises(plpy.PLPYException): |
| result = self.subject.internal_keras_eval_final(input_state) |
| |
| def tearDown(self): |
| self.module_patcher.stop() |
| |
| if __name__ == '__main__': |
| # Do not move any of the tensorflow imports outside of this block. This is |
| # because importing tensorflow.keras.models/layers changes the `__name__` |
| # variable because of which the if condition fails and the unit tests don't |
| # get run |
| |
| # turn off verbose output |
| os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' |
| import tensorflow as tf |
| tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) |
| |
| from tensorflow.keras.models import * |
| from tensorflow.keras.layers import * |
| unittest.main() |
| # --------------------------------------------------------------------- |
