DL: Add multiple variable support
JIRA: MADLIB-1456, MADLIB-1458
This commit adds support for multiple dependent and independent variables.
These changes should allow the users to use YOLO v3 and v4 models.
Note that the existing interface with single dependent and single independent
variables still works as expected. The output table formats have changed
slightly to accomodate this new feature. The summary tables store various
fields such as num_classes or dep_vartype as arrays, even if they have single
entries.
The implementation reads up to 5 dependent and 5 independent variables
separately thanks to the new interface. If more variables are passed, they are
packed into bytea arrays. A high number of large variables might cause us to go
over the 1 GB limit of Postgres due to packing.
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
index 4b27642..68d067b 100644
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
@@ -62,8 +62,8 @@
self.schema_madlib = schema_madlib
self.source_table = source_table
self.output_table = output_table
- self.dependent_varname = dependent_varname
- self.independent_varname = independent_varname
+ self.dependent_varname = split_quoted_delimited_str(dependent_varname)
+ self.independent_varname = split_quoted_delimited_str(independent_varname)
self.buffer_size = buffer_size
self.normalizing_const = normalizing_const
self.num_classes = num_classes
@@ -89,23 +89,29 @@
If necessary, NULLs are padded to dependent_levels list.
"""
if self.dependent_levels:
- # if any class level was NULL in sql, that would show up as
- # None in self.dependent_levels. Replace all None with NULL
- # in the list.
- self.dependent_levels = ['NULL' if level is None else level
- for level in self.dependent_levels]
- self._validate_num_classes()
- # Try computing padding_size after running all necessary validations.
- if self.num_classes:
- self.padding_size = self.num_classes - len(self.dependent_levels)
+ self.padding_size = []
+ for i in range(len(self.dependent_levels)):
+ tmp_levels = self.dependent_levels[i]
+ if tmp_levels:
+ # if any class level was NULL in sql, that would show up as
+ # None in self.dependent_levels. Replace all None with NULL
+ # in the list.
+ self.dependent_levels[i] = ['NULL' if level is None else level
+ for level in tmp_levels]
+ self._validate_num_classes()
+ # Try computing padding_size after running all necessary validations.
+
+ if self.num_classes:
+ self.padding_size.append(self.num_classes[i] - len(self.dependent_levels[i]))
def _validate_num_classes(self):
- if self.num_classes is not None and \
- self.num_classes < len(self.dependent_levels):
- plpy.error("{0}: Invalid num_classes value specified. It must "\
- "be equal to or greater than distinct class values found "\
- "in table ({1}).".format(
- self.module_name, len(self.dependent_levels)))
+ if self.num_classes is not None:
+ for i in range(len(self.num_classes)):
+ if self.num_classes[i] < len(self.dependent_levels[i]):
+ plpy.error("{0}: Invalid num_classes value specified. It must "\
+ "be equal to or greater than distinct class values found "\
+ "in table ({1}).".format(
+ self.module_name, len(self.dependent_levels[i])))
def _validate_distribution_table(self):
@@ -155,46 +161,63 @@
"""
# Assuming the input NUMERIC[] is already one_hot_encoded,
# so casting to INTEGER[]
- if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
- return "{0}::{1}[]".format(self.dependent_varname, SMALLINT_SQL_TYPE)
+ return_sql = []
+ for i in range(len(self.dependent_vartype)):
- # For DL use case, we want to allow NULL as a valid class value,
- # so the query must have 'IS NOT DISTINCT FROM' instead of '='
- # like in the generic get_one_hot_encoded_expr() defined in
- # db_utils.py_in. We also have this optional 'num_classes' param
- # that affects the logic of 1-hot encoding. Since this is very
- # specific to input_preprocessor_dl for now, let's keep
- # it here instead of refactoring it out to a generic helper function.
- one_hot_encoded_expr = ["({0}) IS NOT DISTINCT FROM {1}".format(
- self.dependent_varname, c) for c in self.dependent_levels]
- if self.num_classes:
- one_hot_encoded_expr.extend(['false'
- for i in range(self.padding_size)])
- # In psql, we can't directly convert boolean to smallint, so we firstly
- # convert it to integer and then cast to smallint
- return 'ARRAY[{0}]::INTEGER[]::{1}[]'.format(
- ', '.join(one_hot_encoded_expr), SMALLINT_SQL_TYPE)
+ tmp_type = self.dependent_vartype[i]
+ tmp_varname = self.dependent_varname[i]
+ tmp_levels = self.dependent_levels[i]
+ if is_valid_psql_type(tmp_type, NUMERIC | ONLY_ARRAY):
+ return_sql.append("{0}::{1}[]".format(tmp_varname, SMALLINT_SQL_TYPE))
+ else:
- def _get_independent_var_shape(self):
+ # For DL use case, we want to allow NULL as a valid class value,
+ # so the query must have 'IS NOT DISTINCT FROM' instead of '='
+ # like in the generic get_one_hot_encoded_expr() defined in
+ # db_utils.py_in. We also have this optional 'num_classes' param
+ # that affects the logic of 1-hot encoding. Since this is very
+ # specific to input_preprocessor_dl for now, let's keep
+ # it here instead of refactoring it out to a generic helper function.
+ one_hot_encoded_expr = ["({0}) IS NOT DISTINCT FROM {1}".format(
+ tmp_varname, c) for c in tmp_levels]
+ if self.padding_size:
+ one_hot_encoded_expr.extend(['false'
+ for i in range(self.padding_size[i])])
+ # In psql, we can't directly convert boolean to smallint, so we firstly
+ # convert it to integer and then cast to smallint
+ return_sql.append('ARRAY[{0}]::INTEGER[]::{1}[] AS {2}'.format(
+ ', '.join(one_hot_encoded_expr), SMALLINT_SQL_TYPE, tmp_varname))
+ return_sql = ', '.join(return_sql)
+ return return_sql
+
+ def _get_var_shape(self, varname):
shape = plpy.execute(
"SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
- self.independent_varname, self.source_table))[0]['shape']
+ varname, self.source_table))[0]['shape']
return parse_shape(shape)
+ def _get_independent_var_shape(self):
+
+ shape_list = []
+ for i in self.independent_varname:
+ shape_list.append(self._get_var_shape(i))
+ return shape_list
+
def _get_dependent_var_shape(self):
- if self.num_classes:
- shape = [self.num_classes]
- elif self.dependent_levels:
- shape = [len(self.dependent_levels)]
- else:
- shape = plpy.execute(
- "SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
- self.dependent_varname, self.source_table))[0]['shape']
- shape = parse_shape(shape)
+ shape = []
+ for counter, dep in enumerate(self.dependent_varname):
+ if self.num_classes:
+ shape.append(self.num_classes[counter])
+ else:
+ if self.dependent_levels[counter]:
+ shape.append(len(self.dependent_levels[counter]))
+ else:
+ shape = shape + self._get_var_shape(dep)
return shape
+
def input_preprocessor_dl(self, order_by_random=True):
"""
Creates the output and summary table that does the following
@@ -221,35 +244,69 @@
ind_shape_col = add_postfix(x, "_shape")
ind_shape = self._get_independent_var_shape()
- ind_shape = ','.join([str(i) for i in ind_shape])
+ ind_shape = [','.join([str(i) for i in tmp_shape]) for tmp_shape in ind_shape]
dep_shape = self._get_dependent_var_shape()
- dep_shape = ','.join([str(i) for i in dep_shape])
-
+ dep_shape = [str(i) for i in dep_shape]
one_hot_dep_var_array_expr = self.get_one_hot_encoded_dep_var_expr()
# skip normalization step if normalizing_const = 1.0
+ rescale_independent_var = []
if self.normalizing_const and (self.normalizing_const < 0.999999 or self.normalizing_const > 1.000001):
- rescale_independent_var = """{self.schema_madlib}.array_scalar_mult(
- {self.independent_varname}::{float32}[],
- (1/{self.normalizing_const})::{float32})
- """.format(**locals())
+
+ for i in self.independent_varname:
+
+ rescale_independent_var.append("""{self.schema_madlib}.array_scalar_mult(
+ {i}::{float32}[],
+ (1/{self.normalizing_const})::{float32})
+ AS {i}_norm
+ """.format(**locals()))
else:
self.normalizing_const = DEFAULT_NORMALIZING_CONST
- rescale_independent_var = "{self.independent_varname}::{float32}[]".format(**locals())
+ for i in self.independent_varname:
+ rescale_independent_var.append("{i}::{float32}[] AS {i}_norm".format(**locals()))
+ rescale_independent_var = ', '.join(rescale_independent_var)
+
# It's important that we shuffle all rows before batching for fit(), but
# we can skip that for predict()
order_by_clause = " ORDER BY RANDOM()" if order_by_random else ""
+ concat_sql = []
+ shape_sql = []
+ bytea_sql = []
+
+ for i,j in zip(self.independent_varname, ind_shape):
+ concat_sql.append("""
+ {self.schema_madlib}.agg_array_concat(ARRAY[{i}_norm::{float32}[]]) AS {i}
+ """.format(**locals()))
+ shape_sql.append("""
+ ARRAY[count, {j}]::SMALLINT[] AS {i}_shape
+ """.format(**locals()))
+ bytea_sql.append("""
+ {self.schema_madlib}.array_to_bytea({i}) AS {i}
+ """.format(**locals()))
+
+ for i,j in zip(self.dependent_varname, dep_shape):
+ concat_sql.append("""
+ {self.schema_madlib}.agg_array_concat(ARRAY[{i}]) AS {i}
+ """.format(**locals()))
+ shape_sql.append("""
+ ARRAY[count, {j}]::SMALLINT[] AS {i}_shape
+ """.format(**locals()))
+ bytea_sql.append("""
+ {self.schema_madlib}.array_to_bytea({i}) AS {i}
+ """.format(**locals()))
+
+ concat_sql = ', '.join(concat_sql)
+ shape_sql = ', '.join(shape_sql)
+ bytea_sql = ', '.join(bytea_sql)
+
# This query template will be used later in pg & gp specific code paths,
# where {make_buffer_id} and {dist_by_buffer_id} are filled in
batching_query = """
CREATE TEMP TABLE {batched_table} AS SELECT
{{make_buffer_id}} buffer_id,
- {self.schema_madlib}.agg_array_concat(
- ARRAY[x_norm::{float32}[]]) AS {x},
- {self.schema_madlib}.agg_array_concat(
- ARRAY[y]) AS {y},
+ {concat_sql},
COUNT(*) AS count
FROM {normalized_tbl}
GROUP BY buffer_id
@@ -261,10 +318,8 @@
bytea_query = """
CREATE TABLE {self.output_table} AS SELECT
{{dist_key_col_comma}}
- {self.schema_madlib}.array_to_bytea({x}) AS {x},
- {self.schema_madlib}.array_to_bytea({y}) AS {y},
- ARRAY[count,{ind_shape}]::SMALLINT[] AS {ind_shape_col},
- ARRAY[count,{dep_shape}]::SMALLINT[] AS {dep_shape_col},
+ {bytea_sql},
+ {shape_sql},
buffer_id
FROM {batched_table}
{{dist_by_dist_key}}
@@ -284,17 +339,17 @@
one_hot_sql = """
CREATE TEMP TABLE {normalized_tbl} AS SELECT
(ROW_NUMBER() OVER({order_by_clause}) - 1)::INTEGER as row_id,
- {rescale_independent_var} AS x_norm,
- {one_hot_dep_var_array_expr} AS y
+ {rescale_independent_var},
+ {one_hot_dep_var_array_expr}
FROM {self.source_table}
""".format(**locals())
-
plpy.execute(one_hot_sql)
self.buffer_size = self._get_buffer_size(1)
# Used to format query templates with locals()
make_buffer_id = 'row_id / {0} AS '.format(self.buffer_size)
+
dist_by_dist_key = ''
dist_by_buffer_id = ''
dist_key_col_comma = ''
@@ -431,8 +486,8 @@
one_hot_sql = """
CREATE TEMP TABLE {norm_tbl} AS
SELECT {dist_key_col},
- {rescale_independent_var} AS x_norm,
- {one_hot_dep_var_array_expr} AS y
+ {rescale_independent_var},
+ {one_hot_dep_var_array_expr}
FROM {self.source_table} s JOIN {dist_key_tbl} AS d
ON (s.gp_segment_id = d.gp_segment_id)
{order_by_clause}
@@ -501,11 +556,13 @@
# and those are fixed to the correct segments by the JOIN
# condition.
+ ind_norm_comma_list = ', '.join(["{0}_norm".format(i) for i in self.independent_varname])
+ dep_norm_comma_list = ', '.join(self.dependent_varname)
sql = """
CREATE TEMP TABLE {normalized_tbl} AS SELECT
{dist_key_col},
- x_norm,
- y,
+ {ind_norm_comma_list},
+ {dep_norm_comma_list},
(ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ))::INTEGER as slot_id,
((start_row +
(ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ) - 1)
@@ -587,32 +644,45 @@
self._create_output_summary_table()
def _create_output_summary_table(self):
- class_level_str='NULL::{0}[]'.format(self.dependent_vartype)
- if self.dependent_levels:
- # Update dependent_levels to include NULL when
- # num_classes > len(self.dependent_levels)
- if self.num_classes:
- self.dependent_levels.extend(['NULL'
- for i in range(self.padding_size)])
- else:
- self.num_classes = len(self.dependent_levels)
- class_level_str=py_list_to_sql_string(
- self.dependent_levels, array_type=self.dependent_vartype,
- long_format=True)
+ class_level_str='NULL::{0}[] AS {1}_{2}'.format(self.dependent_vartype[0], self.dependent_varname[0], CLASS_VALUES_COLNAME)
+ class_level_list = []
+ local_num_classes = []
+
+ for i in range(len(self.dependent_vartype)):
+ if self.dependent_levels[i]:
+ # Update dependent_levels to include NULL when
+ # num_classes > len(self.dependent_levels)
+ if self.num_classes:
+ self.dependent_levels[i].extend(['NULL'
+ for j in range(self.padding_size[i])])
+ else:
+ local_num_classes.append(str(len(self.dependent_levels[i])))
+ class_level_str=py_list_to_sql_string(
+ self.dependent_levels[i], array_type=self.dependent_vartype[i],
+ long_format=True)
+ class_level_list.append("{0} AS {1}_{2}".format(class_level_str,
+ self.dependent_varname[i],
+ CLASS_VALUES_COLNAME))
+ class_level_str = ', '.join(class_level_list) if class_level_list else class_level_str
+ local_num_classes = ', '.join(local_num_classes)
if self.num_classes is None:
- self.num_classes = 'NULL::INTEGER'
+ self.num_classes = "ARRAY[{0}]::INTEGER[]".format(local_num_classes)
+ else:
+ self.num_classes = "ARRAY{0}".format(self.num_classes)
+ # if self.num_classes is None:
+ # self.num_classes = 'NULL::INTEGER'
query = """
CREATE TABLE {self.output_summary_table} AS
SELECT
$__madlib__${self.source_table}$__madlib__$::TEXT AS source_table,
$__madlib__${self.output_table}$__madlib__$::TEXT AS output_table,
- $__madlib__${self.dependent_varname}$__madlib__$::TEXT AS {dependent_varname_colname},
- $__madlib__${self.independent_varname}$__madlib__$::TEXT AS {independent_varname_colname},
- $__madlib__${self.dependent_vartype}$__madlib__$::TEXT AS {dependent_vartype_colname},
- {class_level_str} AS {class_values_colname},
+ ARRAY{self.dependent_varname} AS {dependent_varname_colname},
+ ARRAY{self.independent_varname} AS {independent_varname_colname},
+ ARRAY{self.dependent_vartype} AS {dependent_vartype_colname},
+ {class_level_str},
{self.buffer_size} AS buffer_size,
{self.normalizing_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
- {self.num_classes} AS {num_classes_colname},
+ {self.num_classes}::INTEGER[] AS {num_classes_colname},
{self.distribution_rules} AS {distribution_rules},
{self.gpu_config} AS {internal_gpu_config}
""".format(self=self, class_level_str=class_level_str,
@@ -642,30 +712,17 @@
"positive integer or NULL.".format(self.module_name))
def _set_validate_vartypes(self):
- self.independent_vartype = get_expr_type(self.independent_varname,
- self.source_table)
- self.dependent_vartype = get_expr_type(self.dependent_varname,
- self.source_table)
- num_of_independent_cols = split_quoted_delimited_str(self.independent_varname)
- _assert(len(num_of_independent_cols) == 1,
- "Invalid independent_varname: only one column name is allowed "
- "as input.")
- _assert(is_valid_psql_type(self.independent_vartype,
- NUMERIC | ONLY_ARRAY),
- "Invalid independent variable type, should be an array of "
- "one of {0}".format(','.join(NUMERIC)))
- # The dependent variable needs to be either:
- # 1. NUMERIC, TEXT OR BOOLEAN, which we always one-hot encode
- # 2. NUMERIC ARRAY, which we assume it is already one-hot encoded, and we
- # just cast it the INTEGER ARRAY
- num_of_dependent_cols = split_quoted_delimited_str(self.dependent_varname)
- _assert(len(num_of_dependent_cols) == 1,
- "Invalid dependent_varname: only one column name is allowed "
- "as input.")
- _assert((is_valid_psql_type(self.dependent_vartype, NUMERIC | TEXT | BOOLEAN) or
- is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY)),
- """Invalid dependent variable type, should be one of the types in this list:
- numeric, text, boolean, or numeric array""")
+ self.independent_vartype = []
+
+ for i in self.independent_varname:
+ self.independent_vartype.append(get_expr_type(i,
+ self.source_table))
+
+ self.dependent_vartype = []
+
+ for i in self.dependent_varname:
+ self.dependent_vartype.append(get_expr_type(i,
+ self.source_table))
def get_distinct_dependent_levels(self, table, dependent_varname,
dependent_vartype):
@@ -679,13 +736,18 @@
SELECT count(*) AS cnt FROM {0}
""".format(self.source_table))[0]['cnt']
buffer_size_calculator = MiniBatchBufferSizeCalculator()
- indepdent_var_dim = get_product_of_dimensions(self.source_table,
- self.independent_varname)
- buffer_size = buffer_size_calculator.calculate_default_buffer_size(
- self.buffer_size, num_rows_in_tbl, indepdent_var_dim, num_segments)
+
+ buffer_size = num_rows_in_tbl
+ for i in self.independent_varname:
+ indepdent_var_dim = get_product_of_dimensions(self.source_table, i)
+ tmp_size = buffer_size_calculator.calculate_default_buffer_size(
+ self.buffer_size, num_rows_in_tbl, indepdent_var_dim, num_segments)
+ buffer_size = min(tmp_size, buffer_size)
+
num_buffers = num_segments * ceil((1.0 * num_rows_in_tbl) / buffer_size / num_segments)
return int(ceil(num_rows_in_tbl / num_buffers))
+
class ValidationDataPreprocessorDL(InputDataPreprocessorDL):
def __init__(self, schema_madlib, source_table, output_table,
dependent_varname, independent_varname,
@@ -704,13 +766,12 @@
dependent_varname, independent_varname, buffer_size,
summary_table[NORMALIZING_CONST_COLNAME], num_classes,
distribution_rules, self.module_name)
+ self.summary_dep_name = summary_table[DEPENDENT_VARNAME_COLNAME]
+ self.summary_ind_name = summary_table[INDEPENDENT_VARNAME_COLNAME]
# Update value of dependent_levels from training batch summary table.
- self.dependent_levels = self._get_dependent_levels(
- summary_table[CLASS_VALUES_COLNAME],
- summary_table[DEPENDENT_VARTYPE_COLNAME])
+ self.dependent_levels = self._get_dependent_levels(summary_table)
- def _get_dependent_levels(self, training_dependent_levels,
- training_dependent_vartype):
+ def _get_dependent_levels(self, summary_table):
"""
Return the distinct dependent levels to be considered for
one-hot encoding the dependent var. This is inferred from
@@ -722,10 +783,8 @@
"""
# Validate that dep var type is exactly the same as what was in
# trainig_preprocessor_table's input.
- _assert(self.dependent_vartype == training_dependent_vartype,
- "{0}: the dependent variable's type in {1} must be {2}.".format(
- self.module_name, self.source_table,
- training_dependent_vartype))
+ training_dependent_vartype = summary_table[DEPENDENT_VARTYPE_COLNAME]
+
# training_dependent_levels is the class_values column from the
# training batch summary table. This already has the padding with
# NULLs in it based on num_classes that was provided to
@@ -733,26 +792,33 @@
# to strip out those trailing NULLs from class_values, since
# they will anyway get added later in
# InputDataPreprocessorDL._set_one_hot_encoding_variables.
- dependent_levels = strip_trailing_nulls_from_class_values(
- training_dependent_levels)
- if training_dependent_levels:
- dependent_levels_val_data = self.get_distinct_dependent_levels(
- self.source_table, self.dependent_varname,
- self.dependent_vartype)
- unquoted_dependent_levels_val_data = [strip_end_quotes(level, "'")
- for level in dependent_levels_val_data]
- # Assert to check if the class values in validation data is a subset
- # of the class values in training data.
- _assert(set(unquoted_dependent_levels_val_data).issubset(set(dependent_levels)),
- "{0}: the class values in {1} ({2}) should be a "
- "subset of class values in {3} ({4})".format(
- self.module_name, self.source_table,
- unquoted_dependent_levels_val_data,
- self.training_preprocessor_table, dependent_levels))
- if is_psql_char_type(self.dependent_vartype):
- dependent_levels = [quote_literal(level) if level is not None else level
- for level in dependent_levels]
- return dependent_levels
+
+ dependent_levels_list = []
+ for counter, dep in enumerate(self.summary_dep_name):
+ training_dependent_levels = summary_table["{0}_class_values".format(dep)]
+ dependent_levels = strip_trailing_nulls_from_class_values(
+ training_dependent_levels)
+ if training_dependent_levels:
+ dependent_levels_val_data = self.get_distinct_dependent_levels(
+ self.source_table,
+ self.dependent_varname[counter],
+ self.dependent_vartype[counter])
+ unquoted_dependent_levels_val_data = [strip_end_quotes(level, "'")
+ for level in dependent_levels_val_data]
+ # Assert to check if the class values in validation data is a subset
+ # of the class values in training data.
+ _assert(set(unquoted_dependent_levels_val_data).issubset(set(dependent_levels)),
+ "{0}: the class values in {1} ({2}) should be a "
+ "subset of class values in {3} ({4})".format(
+ self.module_name, self.source_table,
+ unquoted_dependent_levels_val_data,
+ self.training_preprocessor_table, dependent_levels))
+ if is_psql_char_type(self.dependent_vartype[counter]):
+ dependent_levels_list.append([quote_literal(level) if level is not None else level
+ for level in dependent_levels])
+ else:
+ dependent_levels_list.append(dependent_levels)
+ return dependent_levels_list
def _validate_and_process_training_preprocessor_table(self):
"""
@@ -775,18 +841,18 @@
"{0}: Expected column {1} in {2}.".format(
self.module_name, NORMALIZING_CONST_COLNAME,
training_summary_table))
- _assert(CLASS_VALUES_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, CLASS_VALUES_COLNAME,
- training_summary_table))
+ # _assert(CLASS_VALUES_COLNAME in summary_table,
+ # "{0}: Expected column {1} in {2}.".format(
+ # self.module_name, CLASS_VALUES_COLNAME,
+ # training_summary_table))
_assert(NUM_CLASSES_COLNAME in summary_table,
"{0}: Expected column {1} in {2}.".format(
self.module_name, NUM_CLASSES_COLNAME,
training_summary_table))
- _assert(DEPENDENT_VARTYPE_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, DEPENDENT_VARTYPE_COLNAME,
- training_summary_table))
+ # _assert(DEPENDENT_VARTYPE_COLNAME in summary_table,
+ # "{0}: Expected column {1} in {2}.".format(
+ # self.module_name, DEPENDENT_VARTYPE_COLNAME,
+ # training_summary_table))
return summary_table
def validation_preprocessor_dl(self):
@@ -816,12 +882,17 @@
one-hot encoding the dependent var. class level values of
type text are quoted.
"""
- if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
- dependent_levels = None
- else:
- dependent_levels = get_distinct_col_levels(
- self.source_table, self.dependent_varname,
- self.dependent_vartype, include_nulls=True)
+ dependent_levels = []
+ for i in range(len(self.dependent_varname)):
+ tmp_type = self.dependent_vartype[i]
+ tmp_varname = self.dependent_varname[i]
+
+ if is_valid_psql_type(tmp_type, NUMERIC | ONLY_ARRAY):
+ dependent_levels.append(None)
+ else:
+ dependent_levels.append(get_distinct_col_levels(
+ self.source_table, tmp_varname,
+ tmp_type, include_nulls=True))
return dependent_levels
def training_preprocessor_dl(self):
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
index f243417..28464ef 100644
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
+++ b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
@@ -867,8 +867,8 @@
dependent_varname VARCHAR,
independent_varname VARCHAR,
training_preprocessor_table VARCHAR,
- buffer_size INTEGER,
- distribution_rules TEXT
+ buffer_size INTEGER DEFAULT NULL,
+ distribution_rules TEXT DEFAULT NULL
) RETURNS VOID AS $$
PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
from utilities.control import MinWarning
@@ -880,43 +880,13 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- training_preprocessor_table VARCHAR,
- buffer_size INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- training_preprocessor_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- message VARCHAR
+ message VARCHAR DEFAULT ''
) RETURNS VARCHAR AS $$
PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, message)
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-------------------------------------------------------------------------------
@@ -925,10 +895,10 @@
output_table VARCHAR,
dependent_varname VARCHAR,
independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL,
- num_classes INTEGER,
- distribution_rules TEXT
+ buffer_size INTEGER DEFAULT NULL,
+ normalizing_const REAL DEFAULT 1.0,
+ num_classes INTEGER[] DEFAULT NULL,
+ distribution_rules TEXT DEFAULT NULL
) RETURNS VOID AS $$
PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
from utilities.control import MinWarning
@@ -940,66 +910,13 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL,
- num_classes INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, $7, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, 1.0, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, NULL, 1.0, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- message VARCHAR
+ message VARCHAR DEFAULT ''
) RETURNS VARCHAR AS $$
PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, message)
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-- aggregation for independent var
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(REAL[]);
CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(REAL[]) (
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
index ba7f2b7..45c3840 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
@@ -31,6 +31,7 @@
from madlib_keras_model_selection import ModelSelectionSchema
+from internal.db_utils import quote_literal
from utilities.utilities import _assert
from utilities.utilities import add_postfix
from utilities.utilities import is_platform_pg
@@ -40,6 +41,7 @@
from utilities.utilities import unique_string
from utilities.validate_args import get_expr_type
from utilities.validate_args import quote_ident
+from utilities.validate_args import input_tbl_valid
from utilities.control import MinWarning
import tensorflow as tf
@@ -102,14 +104,6 @@
fit_params = "" if not fit_params else fit_params
_assert(compile_params, "Compile parameters cannot be empty or NULL.")
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
- mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-
- dep_shape_col = add_postfix(
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape")
- ind_shape_col = add_postfix(
- MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape")
-
segments_per_host = get_segments_per_host()
use_gpus = use_gpus if use_gpus else False
if use_gpus:
@@ -120,11 +114,56 @@
if object_table is not None:
object_table = "{0}.{1}".format(schema_madlib, quote_ident(object_table))
+ source_summary_table = add_postfix(source_table, "_summary")
+ input_tbl_valid(source_summary_table, module_name)
+ src_summary_dict = get_source_summary_table_dict(source_summary_table)
+
+ columns_dict = {}
+ columns_dict['mb_dep_var_cols'] = src_summary_dict['dependent_varname']
+ columns_dict['mb_indep_var_cols'] = src_summary_dict['independent_varname']
+ columns_dict['dep_shape_cols'] = [add_postfix(i, "_shape") for i in columns_dict['mb_dep_var_cols']]
+ columns_dict['ind_shape_cols'] = [add_postfix(i, "_shape") for i in columns_dict['mb_indep_var_cols']]
+
+ multi_dep_count = len(columns_dict['mb_dep_var_cols'])
+ val_dep_var = None
+ val_ind_var = None
+
+ val_dep_shape_cols = None
+ val_ind_shape_cols = None
+ if validation_table:
+ validation_summary_table = add_postfix(validation_table, "_summary")
+ input_tbl_valid(validation_summary_table, module_name)
+ val_summary_dict = get_source_summary_table_dict(validation_summary_table)
+
+ val_dep_var = val_summary_dict['dependent_varname']
+ val_ind_var = val_summary_dict['independent_varname']
+ val_dep_shape_cols = [add_postfix(i, "_shape") for i in val_dep_var]
+ val_ind_shape_cols = [add_postfix(i, "_shape") for i in val_ind_var]
+
fit_validator = FitInputValidator(
- source_table, validation_table, model, model_arch_table,
- model_id, mb_dep_var_col, mb_indep_var_col,
+ source_table, validation_table, model, model_arch_table, model_id,
+ columns_dict['mb_dep_var_cols'], columns_dict['mb_indep_var_cols'],
+ columns_dict['dep_shape_cols'], columns_dict['ind_shape_cols'],
num_iterations, metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg, object_table)
+ use_gpus, accessible_gpus_for_seg, object_table,
+ val_dep_var, val_ind_var)
+
+ columns_dict['val_dep_var'] = val_dep_var
+ columns_dict['val_ind_var'] = val_ind_var
+ columns_dict['val_dep_shape_cols'] = val_dep_shape_cols
+ columns_dict['val_ind_shape_cols'] = val_ind_shape_cols
+
+ fit_validator.dependent_varname = columns_dict['mb_dep_var_cols']
+ fit_validator.independent_varname = columns_dict['mb_indep_var_cols']
+ fit_validator.dep_shape_col = columns_dict['dep_shape_cols']
+ fit_validator.ind_shape_col = columns_dict['ind_shape_cols']
+
+ class_values_colnames = [add_postfix(i, "_class_values") for i in columns_dict['mb_dep_var_cols']]
+ src_summary_dict['class_values_type'] =[ get_expr_type(
+ i, fit_validator.source_summary_table) for i in class_values_colnames]
+ src_summary_dict['norm_const_type'] = get_expr_type(
+ NORMALIZING_CONST_COLNAME, fit_validator.source_summary_table)
+
if metrics_compute_frequency is None:
metrics_compute_frequency = num_iterations
@@ -142,27 +181,31 @@
# Get the serialized master model
start_deserialization = time.time()
model_arch, model_weights = get_model_arch_weights(model_arch_table, model_id)
- num_classes = get_num_classes(model_arch)
+
+ # The last n layers are the output layers where n is the number of dep vars
+ num_classes = get_num_classes(model_arch, multi_dep_count)
+
input_shape = get_input_shape(model_arch)
fit_validator.validate_input_shapes(input_shape)
+
dist_key_col = '0' if is_platform_pg() else DISTRIBUTION_KEY_COLNAME
gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME
serialized_weights = get_initial_weights(model, model_arch, model_weights,
warm_start, accessible_gpus_for_seg)
# Compute total images on each segment
- dist_key_mapping, images_per_seg_train = get_image_count_per_seg_for_minibatched_data_from_db(source_table)
+ dist_key_mapping, images_per_seg_train = get_image_count_per_seg_for_minibatched_data_from_db(source_table, columns_dict['dep_shape_cols'][0])
if validation_table:
- dist_key_mapping_val, images_per_seg_val = get_image_count_per_seg_for_minibatched_data_from_db(validation_table)
+ dist_key_mapping_val, images_per_seg_val = get_image_count_per_seg_for_minibatched_data_from_db(validation_table, columns_dict['dep_shape_cols'][0])
# Construct validation dataset if provided
validation_set_provided = bool(validation_table)
validation_metrics = []; validation_loss = []
# Prepare the SQL for running distributed training via UDA
- compile_params_to_pass = "$madlib$" + compile_params + "$madlib$"
- fit_params_to_pass = "$madlib$" + fit_params + "$madlib$"
+ compile_params_to_pass = quote_literal(compile_params)
+ fit_params_to_pass = quote_literal(fit_params)
custom_function_map = None
# If the object_table exists, we read the list of custom
@@ -176,12 +219,39 @@
# with the function definition
plpy.error("Object table not specified for function {0} in compile_params".format(custom_fn_list))
+ # Use the smart interface
+ if (len(columns_dict['mb_dep_var_cols']) <= 5 and
+ len(columns_dict['mb_indep_var_cols']) <= 5):
+
+ dep_var_array = 5 * ["NULL"]
+ indep_var_array = 5 * ["NULL"]
+
+ for counter, var in enumerate(columns_dict['mb_dep_var_cols']):
+ dep_var_array[counter] = var
+
+ for counter, var in enumerate(columns_dict['mb_indep_var_cols']):
+ indep_var_array[counter] = var
+ mb_dep_var_cols_sql = ', '.join(dep_var_array)
+ mb_indep_var_cols_sql = ', '.join(indep_var_array)
+ else:
+
+ mb_dep_var_cols_sql = ', '.join(["dependent_var_{0}".format(i)
+ for i in columns_dict['mb_dep_var_cols']])
+ mb_dep_var_cols_sql = "ARRAY[{0}]".format(mb_dep_var_cols_sql)
+
+ mb_indep_var_cols_sql = ', '.join(["independent_var_{0}".format(i)
+ for i in columns_dict['mb_indep_var_cols']])
+ mb_indep_var_cols_sql = "ARRAY[{0}]".format(mb_indep_var_cols_sql)
+
+ dep_shape_cols_sql = ', '.join(columns_dict['dep_shape_cols'])
+ ind_shape_cols_sql = ', '.join(columns_dict['ind_shape_cols'])
+
run_training_iteration = plpy.prepare("""
SELECT {schema_madlib}.fit_step(
- {mb_dep_var_col},
- {mb_indep_var_col},
- {dep_shape_col},
- {ind_shape_col},
+ {mb_dep_var_cols_sql},
+ {mb_indep_var_cols_sql},
+ ARRAY[{dep_shape_cols_sql}],
+ ARRAY[{ind_shape_cols_sql}],
$MAD${model_arch}$MAD$::TEXT,
{compile_params_to_pass}::TEXT,
{fit_params_to_pass}::TEXT,
@@ -242,12 +312,12 @@
the last call to train evaluate. Otherwise clear it at the last call
to validation evaluate
"""
-
should_clear_session = False
if not validation_set_provided:
should_clear_session = is_final_iteration
compute_out = compute_loss_and_metrics(schema_madlib, source_table,
+ columns_dict,
compile_params_to_pass,
model_arch,
serialized_weights, use_gpus,
@@ -259,11 +329,13 @@
should_clear_session,
custom_function_map)
metrics_iters.append(i)
+ compute_time, compute_metrics, compute_loss = compute_out
info_str = get_evaluate_info_msg(i, info_str, compute_out, True)
if validation_set_provided:
# Compute loss/accuracy for validation data.
val_compute_out = compute_loss_and_metrics(schema_madlib,
validation_table,
+ columns_dict,
compile_params_to_pass,
model_arch,
serialized_weights,
@@ -285,14 +357,15 @@
end_training_time = datetime.datetime.now()
version = madlib_version(schema_madlib)
- src_summary_dict = get_source_summary_table_dict(fit_validator)
- class_values = src_summary_dict['class_values']
class_values_type = src_summary_dict['class_values_type']
- norm_const = src_summary_dict['norm_const']
+ norm_const = src_summary_dict['normalizing_const']
norm_const_type = src_summary_dict['norm_const_type']
- dep_vartype = src_summary_dict['dep_vartype']
- dependent_varname = src_summary_dict['dependent_varname_in_source_table']
- independent_varname = src_summary_dict['independent_varname_in_source_table']
+ dep_vartype = src_summary_dict['dependent_vartype']
+ dependent_varname = src_summary_dict['dependent_varname']
+ independent_varname = src_summary_dict['independent_varname']
+
+ dep_name_list = ', '.join([quote_literal(i) for i in dependent_varname])
+ ind_name_list = ', '.join([quote_literal(i) for i in independent_varname])
# Define some constants to be inserted into the summary table.
model_type = "madlib_keras"
@@ -316,13 +389,14 @@
# Must quote the string before inserting to table. Explicitly
# quoting it here since this can also take a NULL value, done
# in the else part.
- validation_table = "$MAD${0}$MAD$".format(validation_table)
+ validation_table = quote_literal(validation_table)
else:
validation_metrics = validation_loss = 'NULL'
validation_metrics_final = validation_loss_final = 'NULL'
validation_table = 'NULL'
- object_table = "$MAD${0}$MAD$".format(object_table) if object_table is not None else 'NULL'
+ object_table = quote_literal(object_table) if object_table is not None else 'NULL'
+ class_values_colnames = ' , '.join(class_values_colnames)
if warm_start:
plpy.execute("DROP TABLE {0}, {1}".format
(model, fit_validator.output_summary_model_table))
@@ -331,8 +405,8 @@
SELECT
$MAD${source_table}$MAD$::TEXT AS source_table,
$MAD${model}$MAD$::TEXT AS model,
- $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname,
- $MAD${independent_varname}$MAD$::TEXT AS independent_varname,
+ ARRAY[{dep_name_list}]::TEXT[] AS dependent_varname,
+ ARRAY[{ind_name_list}]::TEXT[] AS independent_varname,
$MAD${model_arch_table}$MAD$::TEXT AS model_arch_table,
{model_id}::INTEGER AS {model_id_colname},
$1 AS compile_params,
@@ -349,9 +423,8 @@
'{end_training_time}'::TIMESTAMP AS end_training_time,
$5 AS metrics_elapsed_time,
'{version}'::TEXT AS madlib_version,
- {num_classes}::INTEGER AS num_classes,
- $6 AS {class_values_colname},
- $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname},
+ ARRAY{num_classes}::INTEGER[] AS num_classes,
+ ARRAY{dep_vartype}::TEXT[] AS {dependent_vartype_colname},
{norm_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
{metrics_type}::TEXT[] AS metrics_type,
{training_metrics_final}::DOUBLE PRECISION AS training_metrics_final,
@@ -362,18 +435,19 @@
{validation_loss_final}::DOUBLE PRECISION AS validation_loss_final,
{validation_metrics}::DOUBLE PRECISION[] AS validation_metrics,
{validation_loss}::DOUBLE PRECISION[] AS validation_loss,
- ARRAY{metrics_iters}::INTEGER[] AS metrics_iters
+ ARRAY{metrics_iters}::INTEGER[] AS metrics_iters,
+ {class_values_colnames}
+ FROM {source_summary_table}
""".format(output_summary_model_table=fit_validator.output_summary_model_table,
- class_values_colname=CLASS_VALUES_COLNAME,
dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
normalizing_const_colname=NORMALIZING_CONST_COLNAME,
FLOAT32_SQL_TYPE = FLOAT32_SQL_TYPE,
model_id_colname = ModelArchSchema.MODEL_ID,
**locals()),
- ["TEXT", "TEXT", "TEXT", "TEXT", "DOUBLE PRECISION[]", class_values_type])
+ ["TEXT", "TEXT", "TEXT", "TEXT", "DOUBLE PRECISION[]"])
plpy.execute(create_output_summary_table,
[compile_params, fit_params, name,
- description, metrics_elapsed_time, class_values])
+ description, metrics_elapsed_time])
plpy.execute("""
CREATE TABLE {0}
@@ -441,33 +515,20 @@
model.get_weights())
return serialized_weights
-def get_source_summary_table_dict(fit_validator):
+def get_source_summary_table_dict(source_summary_table):
source_summary = plpy.execute("""
- SELECT
- {class_values} AS class_values,
- {norm_const} AS norm_const,
- {dep_vartype} AS dep_vartype,
- {dep_varname} AS dependent_varname_in_source_table,
- {indep_varname} AS independent_varname_in_source_table
- FROM {tbl}
- """.format(class_values=CLASS_VALUES_COLNAME,
- norm_const=NORMALIZING_CONST_COLNAME,
- dep_vartype=DEPENDENT_VARTYPE_COLNAME,
- dep_varname='dependent_varname',
- indep_varname='independent_varname',
- tbl=fit_validator.source_summary_table))[0]
- source_summary['class_values_type'] = get_expr_type(
- CLASS_VALUES_COLNAME, fit_validator.source_summary_table)
- source_summary['norm_const_type'] = get_expr_type(
- NORMALIZING_CONST_COLNAME, fit_validator.source_summary_table)
+ SELECT *
+ FROM {0}
+ """.format(source_summary_table))[0]
+
return source_summary
-def compute_loss_and_metrics(schema_madlib, table, compile_params, model_arch,
- serialized_weights, use_gpus,
- accessible_gpus_for_seg, dist_key_mapping,
- images_per_seg_val, metrics_list, loss_list,
+def compute_loss_and_metrics(schema_madlib, table, columns_dict, compile_params, model_arch,
+ serialized_weights, use_gpus, accessible_gpus_for_seg,
+ dist_key_mapping, images_per_seg_val,
+ metrics_list, loss_list,
should_clear_session, custom_fn_map,
- model_table=None, mst_key=None):
+ model_table=None, mst_key=None, is_train=True):
"""
Compute the loss and metric using a given model (serialized_weights) on the
given dataset (table.)
@@ -475,6 +536,7 @@
start_val = time.time()
evaluate_result = get_loss_metric_from_keras_eval(schema_madlib,
table,
+ columns_dict,
compile_params,
model_arch,
serialized_weights,
@@ -485,12 +547,9 @@
should_clear_session,
custom_fn_map,
model_table,
- mst_key)
+ mst_key,
+ is_train)
end_val = time.time()
-
- if len(evaluate_result) not in [1, 2]:
- plpy.error('Calling evaluate on table {0} returned < 2 '
- 'metrics. Expected both loss and a metric.'.format(table))
loss = evaluate_result[0]
metric = evaluate_result[1]
metrics_list.append(metric)
@@ -522,6 +581,33 @@
compile_model(segment_model, compile_params, custom_function_map)
return segment_model
+def fit_transition_wide(state, dependent_var1, dependent_var2, dependent_var3,
+ dependent_var4, dependent_var5, independent_var1,
+ independent_var2, independent_var3, independent_var4,
+ independent_var5, dependent_var_shape,
+ independent_var_shape, model_architecture,
+ compile_params, fit_params, dist_key, dist_key_mapping,
+ current_seg_id, segments_per_host, images_per_seg,
+ accessible_gpus_for_seg, prev_serialized_weights,
+ is_multiple_model=False, custom_function_map=None, **kwargs):
+
+ if not independent_var1 or not dependent_var1:
+ return state
+ dependent_var = [dependent_var1, dependent_var2, dependent_var3,
+ dependent_var4, dependent_var5]
+ independent_var = [independent_var1, independent_var2, independent_var3,
+ independent_var4, independent_var5]
+
+ dependent_var = [i for i in dependent_var if i is not None]
+ independent_var = [i for i in independent_var if i is not None]
+
+ return fit_transition(state, dependent_var, independent_var, dependent_var_shape,
+ independent_var_shape, model_architecture,
+ compile_params, fit_params, dist_key, dist_key_mapping,
+ current_seg_id, segments_per_host, images_per_seg,
+ accessible_gpus_for_seg, prev_serialized_weights,
+ is_multiple_model, custom_function_map, **kwargs)
+
def fit_transition(state, dependent_var, independent_var, dependent_var_shape,
independent_var_shape, model_architecture,
compile_params, fit_params, dist_key, dist_key_mapping,
@@ -542,8 +628,8 @@
and only gets cleared in eval transition at the last row of the last iteration.
"""
- if not dependent_var_shape or not independent_var_shape\
- or dependent_var is None or independent_var is None:
+ if not dependent_var_shape[0] or not independent_var_shape[0]\
+ or dependent_var[0] is None or independent_var[0] is None:
plpy.error("fit_transition called with no data")
if not prev_serialized_weights or not model_architecture:
@@ -569,9 +655,14 @@
with tf.device(device_name):
set_model_weights(segment_model, prev_serialized_weights)
+ x_train = []
+ y_train = []
# Prepare the data
- x_train = np_array_float32(independent_var, independent_var_shape)
- y_train = np_array_int16(dependent_var, dependent_var_shape)
+ for counter, shape in enumerate(independent_var_shape):
+ x_train.append(np_array_float32(independent_var[counter], shape))
+
+ for counter, shape in enumerate(dependent_var_shape):
+ y_train.append(np_array_int16(dependent_var[counter], shape))
# Fit segment model on data
#TODO consider not doing this every time
@@ -580,7 +671,8 @@
segment_model.fit(x_train, y_train, **fit_params)
# Aggregating number of images, loss and accuracy
- agg_image_count += len(x_train)
+
+ agg_image_count += len(x_train[0])
GD[GD_STORE.AGG_IMAGE_COUNT] = agg_image_count
total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key),
images_per_seg)
@@ -629,8 +721,8 @@
GD[GD_STORE.AGG_IMAGE_COUNT] = agg_image_count
# Prepare the data
- if not dependent_var_shape or not independent_var_shape \
- or dependent_var is None or independent_var is None:
+ if not dependent_var_shape[0] or not independent_var_shape[0] \
+ or dependent_var[0] is None or independent_var[0] is None:
if 'x_train' not in GD or 'y_train' not in GD:
plpy.error("cache not populated properly.")
is_last_row = True
@@ -640,14 +732,16 @@
GD['x_train'] = list()
GD['y_train'] = list()
- agg_image_count += independent_var_shape[0]
+ #TODO: Fix the [0] for multi io
+ agg_image_count += independent_var_shape[0][0]
+
GD[GD_STORE.AGG_IMAGE_COUNT] = agg_image_count
total_images = get_image_count_per_seg_from_array(
dist_key_mapping.index(dist_key), images_per_seg
)
is_last_row = agg_image_count == total_images
- x_train_current = np_array_float32(independent_var, independent_var_shape)
- y_train_current = np_array_int16(dependent_var, dependent_var_shape)
+ x_train_current = np_array_float32(independent_var[0], independent_var_shape[0])
+ y_train_current = np_array_int16(dependent_var[0], dependent_var_shape[0])
GD['x_train'].append(x_train_current)
GD['y_train'].append(y_train_current)
@@ -756,7 +850,6 @@
# Return if called early
if not state:
return state
-
image_count, weights = madlib_keras_serializer.deserialize_as_image_1d_weights(state)
if image_count == 0:
plpy.error("fit_final: Total images processed is 0")
@@ -765,7 +858,6 @@
weights /= image_count
return madlib_keras_serializer.serialize_nd_weights(weights)
-
def evaluate(schema_madlib, model_table, test_table, output_table,
use_gpus, mst_key, **kwargs):
@@ -799,13 +891,24 @@
model_arch = res['model_arch']
input_shape = get_input_shape(model_arch)
+
+ model_summary_dict = get_source_summary_table_dict(model_summary_table)
+ # independent_varname = model_summary_dict['independent_varname']
+ # ind_shape_cols = [add_postfix(i, "_shape") for i in independent_varname]
+
+ columns_dict = {}
+ columns_dict['mb_dep_var_cols'] = model_summary_dict['dependent_varname']
+ columns_dict['mb_indep_var_cols'] = model_summary_dict['independent_varname']
+ columns_dict['dep_shape_cols'] = [add_postfix(i, "_shape") for i in columns_dict['mb_dep_var_cols']]
+ columns_dict['ind_shape_cols'] = [add_postfix(i, "_shape") for i in columns_dict['mb_indep_var_cols']]
+
InputValidator.validate_input_shape(
- test_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, input_shape, 2, True)
+ test_table, columns_dict['mb_indep_var_cols'], input_shape, 2, True)
compile_params_query = "SELECT compile_params, metrics_type, object_table FROM {0}".format(model_summary_table)
res = plpy.execute(compile_params_query)[0]
metrics_type = res['metrics_type']
- compile_params = "$madlib$" + res['compile_params'] + "$madlib$"
+ compile_params = quote_literal(res['compile_params'])
object_table = res['object_table']
loss_type = get_loss_from_compile_param(res['compile_params'])
custom_function_map = None
@@ -813,14 +916,17 @@
custom_fn_list = get_custom_functions_list(res['compile_params'])
custom_function_map = query_custom_functions_map(object_table, custom_fn_list)
- dist_key_mapping, images_per_seg = get_image_count_per_seg_for_minibatched_data_from_db(test_table)
+ dist_key_mapping, images_per_seg = get_image_count_per_seg_for_minibatched_data_from_db(test_table, columns_dict['ind_shape_cols'][0])
- loss, metric = \
+ loss_metric = \
get_loss_metric_from_keras_eval(
- schema_madlib, test_table, compile_params, model_arch,
+ schema_madlib, test_table, columns_dict, compile_params, model_arch,
model_weights, use_gpus, accessible_gpus_for_seg, dist_key_mapping,
images_per_seg, custom_function_map=custom_function_map)
+ loss = loss_metric[0]
+ metric = loss_metric[1]
+
if not metrics_type:
metrics_type = None
metric = None
@@ -840,9 +946,9 @@
error_suffix_str="Please ensure that the test table ({0}) "
"has been preprocessed by "
"the image preprocessor.".format(test_table))
- cols_in_tbl_valid(test_summary_table, [CLASS_VALUES_COLNAME,
- NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
- DEPENDENT_VARNAME_COLNAME, INDEPENDENT_VARNAME_COLNAME], module_name)
+ cols_in_tbl_valid(test_summary_table, [NORMALIZING_CONST_COLNAME,
+ DEPENDENT_VARTYPE_COLNAME, DEPENDENT_VARNAME_COLNAME,
+ INDEPENDENT_VARNAME_COLNAME], module_name)
input_tbl_valid(model_table, module_name)
if is_mult_model and not columns_exist_in_table(model_table, ['mst_key']):
@@ -851,16 +957,19 @@
plpy.error("{module_name}: Multi-model needs to pass mst_key".format(**locals()))
InputValidator.validate_predict_evaluate_tables(
module_name, model_table, model_summary_table,
- test_table, output_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL)
+ test_table, output_table)
_validate_test_summary_tbl()
- validate_bytea_var_for_minibatch(test_table,
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL)
-def get_loss_metric_from_keras_eval(schema_madlib, table, compile_params,
+ dependent_varname = plpy.execute("SELECT {0} FROM {1}".format(
+ "dependent_varname", model_summary_table))[0]["dependent_varname"]
+ for i in dependent_varname:
+ validate_bytea_var_for_minibatch(test_table, i)
+
+def get_loss_metric_from_keras_eval(schema_madlib, table, columns_dict, compile_params,
model_arch, serialized_weights, use_gpus,
accessible_gpus_for_seg, dist_key_mapping, images_per_seg,
should_clear_session=True, custom_function_map=None,
- model_table=None, mst_key=None):
+ model_table=None, mst_key=None, is_train=True):
"""
This function will call the internal keras evaluate function to get the loss
and accuracy of each tuple which then gets averaged to get the final result.
@@ -870,22 +979,30 @@
gp_segment_id_col = '0' if is_platform_pg() else '__table__.{0}'.format(GP_SEGMENT_ID_COLNAME)
segments_per_host = get_segments_per_host()
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
- mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-
- dep_shape_col = add_postfix(
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape")
- ind_shape_col = add_postfix(
- MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape")
-
+ """
+ This function will call the internal keras evaluate function to get the loss
+ and accuracy of each tuple which then gets averaged to get the final result.
+ """
use_gpus = use_gpus if use_gpus else False
+ if is_train:
+ mb_dep_var_cols_sql = ', '.join(columns_dict['mb_dep_var_cols'])
+ mb_indep_var_cols_sql = ', '.join(columns_dict['mb_indep_var_cols'])
+ dep_shape_cols_sql = ', '.join(columns_dict['dep_shape_cols'])
+ ind_shape_cols_sql = ', '.join(columns_dict['ind_shape_cols'])
+ else:
+ mb_dep_var_cols_sql = ', '.join(columns_dict['val_dep_var'])
+ mb_indep_var_cols_sql = ', '.join(columns_dict['val_ind_var'])
+ dep_shape_cols_sql = ', '.join(columns_dict['val_dep_shape_cols'])
+ ind_shape_cols_sql = ', '.join(columns_dict['val_ind_shape_cols'])
+
+
eval_sql = """
select ({schema_madlib}.internal_keras_evaluate(
- {mb_dep_var_col},
- {mb_indep_var_col},
- {dep_shape_col},
- {ind_shape_col},
+ ARRAY[{mb_dep_var_cols_sql}],
+ ARRAY[{mb_indep_var_cols_sql}],
+ ARRAY[{dep_shape_cols_sql}],
+ ARRAY[{ind_shape_cols_sql}],
$MAD${model_arch}$MAD$,
{weights},
{compile_params},
@@ -912,6 +1029,7 @@
weights = '$1'
mult_sql = ''
custom_map_var = '$2'
+ plpy.info(eval_sql.format(**locals()))
evaluate_query = plpy.prepare(eval_sql.format(**locals()), ["bytea", "bytea"])
res = plpy.execute(evaluate_query, [serialized_weights, custom_function_map])
@@ -931,7 +1049,7 @@
custom_function_map=None, **kwargs):
GD = kwargs['GD']
device_name = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[current_seg_id], current_seg_id)
- agg_loss, agg_metric, agg_image_count = state
+
"""
This transition function is common to evaluate as well as the fit functions.
All these calls have a different logic for creating and clear the tensorflow
@@ -946,10 +1064,30 @@
should_clear_session is only set to true for the last call to eval_transition
which can be either the training eval or validation eval
For fit_multiple,
- We create one session per hop and store it in GD.
+ We create one session per hop and store it in GD.
should_clear_session is always set to true, so the session and GD is
cleared once the last buffer is evaluated on each segment.
"""
+
+ multi_output = True if len(dependent_var) > 1 else False
+
+ if multi_output:
+ output_count = len(dependent_var)
+ agg_loss = state[0]
+ if agg_loss == 0:
+ state = []
+ for i in range(2*output_count+2):
+ state.append(0)
+ agg_image_count = state[-1]
+ aux_losses = []
+ aux_metrics = []
+ for counter in range(output_count):
+ aux_losses.append(state[2*counter+1])
+ aux_metrics.append(state[2*counter+2])
+
+ else:
+ agg_loss, agg_metric, agg_image_count = state
+
segment_model, sess = get_init_model_and_sess(GD, device_name,
accessible_gpus_for_seg[current_seg_id],
segments_per_host,
@@ -961,26 +1099,39 @@
agg_loss = 0
set_model_weights(segment_model, serialized_weights)
+ x_val = []
+ y_val = []
+ for counter, shape in enumerate(independent_var_shape):
+ x_val.append(np_array_float32(independent_var[counter], shape))
+ for counter, shape in enumerate(dependent_var_shape):
+ y_val.append(np_array_int16(dependent_var[counter], shape))
- x_val = np_array_float32(independent_var, independent_var_shape)
- y_val = np_array_int16(dependent_var, dependent_var_shape)
+ image_count = len(y_val[0])
+ agg_image_count += image_count
with tf.device(device_name):
res = segment_model.evaluate(x_val, y_val)
# if metric is None, model.evaluate will only return loss as a scalar
# Otherwise, it will return a list which has loss and metric
- if type(res) is list:
- loss, metric = res
+ if multi_output:
+ loss = res[0]
+ agg_loss += (image_count * loss)
+ for counter in range(output_count):
+ # For multi output cases, res has the following structure
+ # print(model.metrics_names)
+ # ['loss', 'dense_4_loss', 'dense_5_loss', 'dense_4_acc', 'dense_5_acc']
+ aux_losses[counter] = aux_losses[counter] + (image_count * res[counter+1])
+ aux_metrics[counter] = aux_metrics[counter] + (image_count * res[counter+1+len(dependent_var)])
else:
- loss = res
- metric = 0
+ if type(res) is list:
+ loss, metric = res
+ else:
+ loss = res
+ metric = 0
- image_count = len(y_val)
-
- agg_image_count += image_count
- agg_loss += (image_count * loss)
- agg_metric += (image_count * metric)
+ agg_loss += (image_count * loss)
+ agg_metric += (image_count * metric)
total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key),
images_per_seg)
@@ -991,9 +1142,15 @@
del sess
del segment_model
- state[0] = agg_loss
- state[1] = agg_metric
- state[2] = agg_image_count
+ state = [agg_loss]
+
+ if multi_output:
+ for counter in range(output_count):
+ state.append(aux_losses[counter])
+ state.append(aux_metrics[counter])
+ else:
+ state.append(agg_metric)
+ state.append(agg_image_count)
return state
@@ -1002,27 +1159,22 @@
if not state1 or not state2:
return state1 or state2
- loss1, metric1, image_count1 = state1
- loss2, metric2, image_count2 = state2
-
- merged_loss = loss1 + loss2
- merged_metric = metric1 + metric2
- total_image_count = image_count1 + image_count2
-
- merged_state = [ merged_loss, merged_metric , total_image_count ]
+ merged_state = []
+ for i in range(len(state1)):
+ merged_state.append(state1[i]+state2[i])
return merged_state
def internal_keras_eval_final(state, **kwargs):
- loss, metric, image_count = state
+ image_count = state[-1]
if image_count == 0:
plpy.error("internal_keras_eval_final: Total images processed is 0")
- loss /= image_count
- metric /= image_count
+ for i in range(len(state)-1):
+ state[i] = state[i]/image_count
- return loss, metric
+ return state
def fit_help(schema_madlib, message, **kwargs):
"""
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
index e0e0fb5..9896fae 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
@@ -1648,13 +1648,13 @@
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR,
- description VARCHAR,
- object_table VARCHAR
+ use_gpus BOOLEAN DEFAULT FALSE,
+ validation_table VARCHAR DEFAULT NULL,
+ metrics_compute_frequency INTEGER DEFAULT NULL,
+ warm_start BOOLEAN DEFAULT NULL,
+ name VARCHAR DEFAULT NULL,
+ description VARCHAR DEFAULT NULL,
+ object_table VARCHAR DEFAULT NULL
) RETURNS VOID AS $$
PythonFunctionBodyOnly(`deep_learning', `madlib_keras')
from utilities.control import SetGUC
@@ -1664,124 +1664,10 @@
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR,
- description VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER
-) RETURNS VOID AS $$
-SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
- m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, FALSE, NULL, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition(
state BYTEA,
- dependent_var BYTEA,
- independent_var BYTEA,
+ dependent_var BYTEA[],
+ independent_var BYTEA[],
dependent_var_shape INTEGER[],
independent_var_shape INTEGER[],
model_architecture TEXT,
@@ -1811,6 +1697,48 @@
$$ LANGUAGE plpythonu
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition_wide(
+ state BYTEA,
+ dependent_var1 BYTEA,
+ dependent_var2 BYTEA,
+ dependent_var3 BYTEA,
+ dependent_var4 BYTEA,
+ dependent_var5 BYTEA,
+ independent_var1 BYTEA,
+ independent_var2 BYTEA,
+ independent_var3 BYTEA,
+ independent_var4 BYTEA,
+ independent_var5 BYTEA,
+ dependent_var_shape INTEGER[],
+ independent_var_shape INTEGER[],
+ model_architecture TEXT,
+ compile_params TEXT,
+ fit_params TEXT,
+ dist_key INTEGER,
+ dist_key_mapping INTEGER[],
+ current_seg_id INTEGER,
+ segments_per_host INTEGER,
+ images_per_seg INTEGER[],
+ accessible_gpus_for_seg INTEGER[],
+ prev_serialized_weights BYTEA,
+ custom_function_map BYTEA
+) RETURNS BYTEA AS $$
+PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
+ import traceback
+ from sys import exc_info
+ import plpy
+ try:
+ return madlib_keras.fit_transition_wide(**globals())
+ except Exception as e:
+ etype, _, tb = exc_info()
+ detail = ''.join(traceback.format_exception(etype, e, tb))
+ message = e.args[0] + 'TransAggDetail' + detail
+ e.args = (message,)
+ raise e
+$$ LANGUAGE plpythonu
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_merge(
state1 BYTEA,
state2 BYTEA
@@ -1851,26 +1779,79 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step(
- BYTEA,
- BYTEA,
- TEXT,
- TEXT,
- TEXT,
- TEXT,
- TEXT,
- INTEGER,
- INTEGER[],
- INTEGER,
- INTEGER,
- INTEGER[],
- BOOLEAN,
- INTEGER[],
- BYTEA,
- BOOLEAN,
- BYTEA);
-CREATE AGGREGATE MADLIB_SCHEMA.fit_step(
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
/* dep_var */ BYTEA,
/* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* dep_var_shape */ INTEGER[],
+ /* ind_var_shape */ INTEGER[],
+ /* model_architecture */ TEXT,
+ /* compile_params */ TEXT,
+ /* fit_params */ TEXT,
+ /* dist_key */ INTEGER,
+ /* dist_key_mapping */ INTEGER[],
+ /* current_seg_id */ INTEGER,
+ /* segments_per_host */ INTEGER,
+ /* images_per_seg */ INTEGER[],
+ /* segments_per_host */ INTEGER[],
+ /* prev_serialized_weights */BYTEA,
+ /* custom_loss_cfunction */ BYTEA);
+CREATE AGGREGATE MADLIB_SCHEMA.fit_step(
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* dep_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* ind_var */ BYTEA,
+ /* dep_var_shape */ INTEGER[],
+ /* ind_var_shape */ INTEGER[],
+ /* model_architecture */ TEXT,
+ /* compile_params */ TEXT,
+ /* fit_params */ TEXT,
+ /* dist_key */ INTEGER,
+ /* dist_key_mapping */ INTEGER[],
+ /* current_seg_id */ INTEGER,
+ /* segments_per_host */ INTEGER,
+ /* images_per_seg */ INTEGER[],
+ /* segments_per_host */ INTEGER[],
+ /* prev_serialized_weights */BYTEA,
+ /* custom_loss_cfunction */ BYTEA
+)(
+ STYPE=BYTEA,
+ SFUNC=MADLIB_SCHEMA.fit_transition_wide,
+ m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.fit_merge,')
+ FINALFUNC=MADLIB_SCHEMA.fit_final
+);
+
+DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step(
+ BYTEA[],
+ BYTEA[],
+ INTEGER[],
+ INTEGER[],
+ TEXT,
+ TEXT,
+ TEXT,
+ INTEGER,
+ INTEGER[],
+ INTEGER,
+ INTEGER,
+ INTEGER[],
+ INTEGER[],
+ BYTEA,
+ BYTEA);
+CREATE AGGREGATE MADLIB_SCHEMA.fit_step(
+ /* dep_var */ BYTEA[],
+ /* ind_var */ BYTEA[],
/* dep_var_shape */ INTEGER[],
/* ind_var_shape */ INTEGER[],
/* model_architecture */ TEXT,
@@ -1971,6 +1952,10 @@
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_predict(
independent_var REAL[],
+ independent_var2 REAL[],
+ independent_var3 REAL[],
+ independent_var4 REAL[],
+ independent_var5 REAL[],
model_architecture TEXT,
model_weights BYTEA,
normalizing_const DOUBLE PRECISION,
@@ -1981,7 +1966,7 @@
segments_per_host INTEGER
) RETURNS DOUBLE PRECISION[] AS $$
PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_predict')
- return madlib_keras_predict.internal_keras_predict(**globals())
+ return madlib_keras_predict.internal_keras_predict_wide(**globals())
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1996,7 +1981,8 @@
pred_type VARCHAR DEFAULT 'prob',
use_gpus BOOLEAN DEFAULT NULL,
class_values TEXT[] DEFAULT NULL,
- normalizing_const DOUBLE PRECISION DEFAULT NULL
+ normalizing_const DOUBLE PRECISION DEFAULT NULL,
+ dependent_count INTEGER DEFAULT 1
) RETURNS VOID AS $$
PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict')
from utilities.control import SetGUC
@@ -2081,9 +2067,9 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_transition(
- state REAL[3],
- dependent_var BYTEA,
- independent_var BYTEA,
+ state REAL[],
+ dependent_var BYTEA[],
+ independent_var BYTEA[],
dependent_var_shape INTEGER[],
independent_var_shape INTEGER[],
model_architecture TEXT,
@@ -2097,23 +2083,23 @@
accessible_gpus_for_seg INTEGER[],
should_clear_session BOOLEAN,
custom_function_map BYTEA
-) RETURNS REAL[3] AS $$
+) RETURNS REAL[] AS $$
PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
return madlib_keras.internal_keras_eval_transition(**globals())
$$ LANGUAGE plpythonu
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_merge(
- state1 REAL[3],
- state2 REAL[3]
-) RETURNS REAL[3] AS $$
+ state1 REAL[],
+ state2 REAL[]
+) RETURNS REAL[] AS $$
PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
return madlib_keras.internal_keras_eval_merge(**globals())
$$ LANGUAGE plpythonu
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_final(
- state REAL[3]
+ state REAL[]
) RETURNS REAL[2] AS $$
PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
return madlib_keras.internal_keras_eval_final(**globals())
@@ -2121,8 +2107,8 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.internal_keras_evaluate(
- BYTEA,
- BYTEA,
+ BYTEA[],
+ BYTEA[],
INTEGER[],
INTEGER[],
TEXT,
@@ -2139,8 +2125,8 @@
BYTEA);
CREATE AGGREGATE MADLIB_SCHEMA.internal_keras_evaluate(
- /* dependent_var */ BYTEA,
- /* independent_var */ BYTEA,
+ /* dependent_var */ BYTEA[],
+ /* independent_var */ BYTEA[],
/* dependent_var_shape */ INTEGER[],
/* independent_var_shape */ INTEGER[],
/* model_architecture */ TEXT,
@@ -2155,7 +2141,7 @@
/* should_clear_session */ BOOLEAN,
/* custom_function_map */ BYTEA
)(
- STYPE=REAL[3],
+ STYPE=REAL[],
INITCOND='{0,0,0}',
SFUNC=MADLIB_SCHEMA.internal_keras_eval_transition,
m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.internal_keras_eval_merge,')
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
index b0383f5..578c5d7 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
@@ -111,6 +111,12 @@
self.description = description
self.use_caching = use_caching
+ self.source_table_summary = add_postfix(self.source_table, '_summary')
+ self.dependent_varname = plpy.execute(
+ "SELECT dependent_varname FROM {0}".format(
+ self.source_table_summary))[0]['dependent_varname']
+ self.class_values_colnames = [add_postfix(i, "_class_values") for i in self.dependent_varname]
+
if self.validation_table:
AutoMLConstants.LOSS_METRIC = 'validation_loss_final'
@@ -183,6 +189,7 @@
object_table = 'NULL' if self.object_table is None else '$MAD${0}$MAD$'.format(self.object_table)
random_state = 'NULL' if self.random_state is None else '$MAD${0}$MAD$'.format(self.random_state)
validation_table = 'NULL' if self.validation_table is None else '$MAD${0}$MAD$'.format(self.validation_table)
+ class_values_colnames = ' , '.join(self.class_values_colnames)
create_query = """
CREATE TABLE {self.model_summary_table} AS
@@ -191,10 +198,8 @@
{validation_table}::TEXT AS validation_table,
$MAD${self.model_output_table}$MAD$::TEXT AS model,
$MAD${self.model_info_table}$MAD$::TEXT AS model_info,
- (SELECT dependent_varname FROM {a.MODEL_SUMMARY_TABLE})
- AS dependent_varname,
- (SELECT independent_varname FROM {a.MODEL_SUMMARY_TABLE})
- AS independent_varname,
+ dependent_varname,
+ independent_varname,
$MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table,
$MAD${self.model_selection_table}$MAD$::TEXT AS model_selection_table,
$MAD${self.automl_method}$MAD$::TEXT AS automl_method,
@@ -202,19 +207,17 @@
{random_state}::TEXT AS random_state,
{object_table}::TEXT AS object_table,
{self.use_gpus} AS use_gpus,
- (SELECT metrics_compute_frequency FROM {a.MODEL_SUMMARY_TABLE})::INTEGER
- AS metrics_compute_frequency,
+ metrics_compute_frequency,
{name}::TEXT AS name,
{descr}::TEXT AS description,
'{self.start_training_time}'::TIMESTAMP AS start_training_time,
'{self.end_training_time}'::TIMESTAMP AS end_training_time,
- (SELECT madlib_version FROM {a.MODEL_SUMMARY_TABLE}) AS madlib_version,
- (SELECT num_classes FROM {a.MODEL_SUMMARY_TABLE})::INTEGER AS num_classes,
- (SELECT class_values FROM {a.MODEL_SUMMARY_TABLE}) AS class_values,
- (SELECT dependent_vartype FROM {a.MODEL_SUMMARY_TABLE})
- AS dependent_vartype,
- (SELECT normalizing_const FROM {a.MODEL_SUMMARY_TABLE})
- AS normalizing_const
+ madlib_version,
+ num_classes,
+ {class_values_colnames},
+ dependent_vartype,
+ normalizing_const
+ FROM {a.MODEL_SUMMARY_TABLE}
""".format(a=AutoMLConstants, **locals())
plpy.execute(create_query)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperopt.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperopt.py_in
index 424cdd1..c0bbe57 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperopt.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperopt.py_in
@@ -168,6 +168,7 @@
False, self.name, self.description,
self.use_caching,
metrics_elapsed_time_offset)
+
model_training.fit_multiple_model()
metrics_elapsed_time_offset += time.time() - start_time
if make_mst_summary:
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
index 3e14c9b..5decb4c 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
@@ -36,6 +36,7 @@
from madlib_keras_validator import *
from madlib_keras_wrapper import *
+from internal.db_utils import quote_literal
from utilities.control import OptimizerControl
from utilities.control import SetGUC
from utilities.utilities import add_postfix
@@ -56,8 +57,6 @@
DEBUG.plpy_execute_enabled = False
DEBUG.plpy_info_enabled = False
-mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
-mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
"""
FitMultipleModel: This class implements the Model Hopper technique for
@@ -138,8 +137,40 @@
self.train_mst_loss = defaultdict(list)
self.train_mst_metric = defaultdict(list)
self.info_str = ""
- self.dep_shape_col = add_postfix(mb_dep_var_col, "_shape")
- self.ind_shape_col = add_postfix(mb_indep_var_col, "_shape")
+ source_summary_table = add_postfix(self.source_table, "_summary")
+ input_tbl_valid(source_summary_table, self.module_name)
+ src_summary_dict = get_source_summary_table_dict(source_summary_table)
+
+ self.mb_dep_var_cols = src_summary_dict['dependent_varname']
+ self.mb_indep_var_cols = src_summary_dict['independent_varname']
+ self.dep_shape_cols = [add_postfix(i, "_shape") for i in self.mb_dep_var_cols]
+ self.ind_shape_cols = [add_postfix(i, "_shape") for i in self.mb_indep_var_cols]
+
+ self.columns_dict = {}
+ self.columns_dict['mb_dep_var_cols'] = self.mb_dep_var_cols
+ self.columns_dict['mb_indep_var_cols'] = self.mb_indep_var_cols
+ self.columns_dict['dep_shape_cols'] = self.dep_shape_cols
+ self.columns_dict['ind_shape_cols'] = self.ind_shape_cols
+
+ self.val_dep_var = None
+ self.val_ind_var = None
+ self.val_dep_shape_cols = None
+ self.val_ind_shape_cols = None
+ if validation_table:
+ validation_summary_table = add_postfix(self.validation_table, "_summary")
+ input_tbl_valid(validation_summary_table, self.module_name)
+ val_summary_dict = get_source_summary_table_dict(validation_summary_table)
+
+ self.val_dep_var = val_summary_dict['dependent_varname']
+ self.val_ind_var = val_summary_dict['independent_varname']
+ self.val_dep_shape_cols = [add_postfix(i, "_shape") for i in self.val_dep_var]
+ self.val_ind_shape_cols = [add_postfix(i, "_shape") for i in self.val_ind_var]
+
+ self.columns_dict['val_dep_var'] = self.val_dep_var
+ self.columns_dict['val_ind_var'] = self.val_ind_var
+ self.columns_dict['val_dep_shape_cols'] = self.val_dep_shape_cols
+ self.columns_dict['val_ind_shape_cols'] = self.val_ind_shape_cols
+
self.use_gpus = use_gpus if use_gpus else False
self.segments_per_host = get_segments_per_host()
self.model_input_tbl = unique_string('model_input')
@@ -159,7 +190,7 @@
self.original_model_output_tbl = model_output_table
if not self.original_model_output_tbl:
- plpy.error("Must specify an output table.")
+ plpy.error("Must specify an output table.")
self.model_info_tbl = add_postfix(
self.original_model_output_tbl, '_info')
@@ -171,10 +202,11 @@
self.fit_validator_train = FitMultipleInputValidator(
self.source_table, self.validation_table, self.original_model_output_tbl,
self.model_selection_table, self.model_selection_summary_table,
- mb_dep_var_col, mb_indep_var_col, self.num_iterations,
+ self.mb_dep_var_cols, self.mb_indep_var_cols, self.dep_shape_cols,
+ self.ind_shape_cols, self.num_iterations,
self.model_info_tbl, self.mst_key_col, self.model_arch_table_col,
self.metrics_compute_frequency, self.warm_start, self.use_gpus,
- self.accessible_gpus_for_seg)
+ self.accessible_gpus_for_seg, self.val_dep_var, self.val_ind_var)
if self.metrics_compute_frequency is None:
self.metrics_compute_frequency = num_iterations
@@ -193,7 +225,7 @@
self.dist_key_mapping, self.images_per_seg_train = \
get_image_count_per_seg_for_minibatched_data_from_db(
- self.source_table)
+ self.source_table, self.dep_shape_cols[0])
if self.validation_table:
self.valid_mst_metric_eval_time = defaultdict(list)
@@ -201,7 +233,7 @@
self.valid_mst_metric = defaultdict(list)
self.dist_key_mapping_valid, self.images_per_seg_valid = \
get_image_count_per_seg_for_minibatched_data_from_db(
- self.validation_table)
+ self.validation_table, self.val_dep_shape_cols[0])
self.dist_keys = query_dist_keys(self.source_table, self.dist_key_col)
self.max_dist_key = sorted(self.dist_keys)[-1]
@@ -336,18 +368,20 @@
model_arch = get_model_arch(self.model_arch_table, mst[self.model_id_col])
DEBUG.start_timing('eval_compute_loss_and_metrics')
eval_compute_time, metric, loss = compute_loss_and_metrics(
- self.schema_madlib, table, "$madlib${0}$madlib$".format(
+ self.schema_madlib, table, self.columns_dict,
+ "$madlib${0}$madlib$".format(
mst[self.compile_params_col]),
- model_arch,
- None,
- self.use_gpus,
- self.accessible_gpus_for_seg,
- seg_ids,
- images_per_seg,
- [], [], True,
- mst[self.object_map_col],
- self.model_output_tbl,
- mst[self.mst_key_col])
+ model_arch,
+ None,
+ self.use_gpus,
+ self.accessible_gpus_for_seg,
+ seg_ids,
+ images_per_seg,
+ [], [], True,
+ mst[self.object_map_col],
+ self.model_output_tbl,
+ mst[self.mst_key_col],
+ is_train)
total_eval_compute_time += eval_compute_time
mst_metric_eval_time[mst[self.mst_key_col]] \
.append(self.metrics_elapsed_time_offset + (time.time() - self.metrics_elapsed_start_time))
@@ -678,39 +712,43 @@
def create_model_summary_table(self):
if self.warm_start:
plpy.execute("DROP TABLE {0}".format(self.model_summary_table))
- src_summary_dict = get_source_summary_table_dict(self.fit_validator_train)
- class_values = src_summary_dict['class_values']
- class_values_type = src_summary_dict['class_values_type']
- dep_vartype = src_summary_dict['dep_vartype']
- dependent_varname = \
- src_summary_dict['dependent_varname_in_source_table']
- independent_varname = \
- src_summary_dict['independent_varname_in_source_table']
- norm_const = src_summary_dict['norm_const']
+ source_summary_table = self.fit_validator_train.source_summary_table
+ src_summary_dict = get_source_summary_table_dict(source_summary_table)
+
+ class_values_colnames = [add_postfix(i, "_class_values") for i in self.mb_dep_var_cols]
+ # class_values = src_summary_dict['class_values']
+ class_values_type =[get_expr_type(i, source_summary_table) for i in class_values_colnames]
+ # class_values_type = src_summary_dict['class_values_type']
+
+ dependent_varname = src_summary_dict['dependent_varname']
+ independent_varname = src_summary_dict['independent_varname']
+ dep_name_list = ', '.join([quote_literal(i) for i in dependent_varname])
+ ind_name_list = ', '.join([quote_literal(i) for i in independent_varname])
+
+ norm_const = src_summary_dict['normalizing_const']
self.validation_table = 'NULL' if self.validation_table is None \
else '$MAD${0}$MAD$'.format(self.validation_table)
- if class_values is None:
+ if class_values_colnames is None:
num_classes = 'NULL'
else:
- num_classes = len(class_values)
+ num_classes = len(class_values_colnames)
+ class_values_colnames = ' , '.join(class_values_colnames)
name = 'NULL' if self.name is None else '$MAD${0}$MAD$'.format(self.name)
descr = 'NULL' if self.description is None else '$MAD${0}$MAD$'.format(self.description)
object_table = 'NULL' if self.object_table is None \
else '$MAD${0}$MAD$'.format(self.object_table)
metrics_iters = self.metrics_iters if self.metrics_iters else 'NULL'
- class_values_colname = CLASS_VALUES_COLNAME
- dependent_vartype_colname = DEPENDENT_VARTYPE_COLNAME
normalizing_const_colname = NORMALIZING_CONST_COLNAME
float32_sql_type = FLOAT32_SQL_TYPE
- create_query = plpy.prepare("""
+ create_query = """
CREATE TABLE {self.model_summary_table} AS
SELECT
$MAD${self.source_table}$MAD$::TEXT AS source_table,
{self.validation_table}::TEXT AS validation_table,
$MAD${self.original_model_output_tbl}$MAD$::TEXT AS model,
$MAD${self.model_info_tbl}$MAD$::TEXT AS model_info,
- $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname,
- $MAD${independent_varname}$MAD$::TEXT AS independent_varname,
+ ARRAY[{dep_name_list}]::TEXT[] AS dependent_varname,
+ ARRAY[{ind_name_list}]::TEXT[] AS independent_varname,
$MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table,
$MAD${self.model_selection_table}$MAD$::TEXT AS model_selection_table,
{object_table}::TEXT AS object_table,
@@ -722,13 +760,14 @@
'{self.start_training_time}'::TIMESTAMP AS start_training_time,
'{self.end_training_time}'::TIMESTAMP AS end_training_time,
'{self.version}'::TEXT AS madlib_version,
- {num_classes}::INTEGER AS num_classes,
- $1 AS {class_values_colname},
- $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname},
+ ARRAY[{num_classes}]::INTEGER[] AS num_classes,
+ {class_values_colnames},
+ dependent_vartype,
{norm_const}::{float32_sql_type} AS {normalizing_const_colname},
ARRAY{metrics_iters}::INTEGER[] AS metrics_iters
- """.format(**locals()), [class_values_type])
- plpy.execute(create_query, [class_values])
+ FROM {source_summary_table}
+ """.format(**locals())
+ plpy.execute(create_query)
def update_info_table(self, mst, is_train):
mst_key = mst[self.mst_key_col]
@@ -859,10 +898,11 @@
RENAME TO {self.model_input_tbl}
""".format(self=self))
- ind_shape_col = self.ind_shape_col
- dep_shape_col = self.dep_shape_col
- dep_var_col = mb_dep_var_col
- indep_var_col = mb_indep_var_col
+ #TODO: Fix these to add multi io
+ dep_shape_col = self.dep_shape_cols[0]
+ ind_shape_col = self.ind_shape_cols[0]
+ dep_var_col = self.mb_dep_var_cols[0]
+ indep_var_col = self.mb_indep_var_cols[0]
source_table = self.source_table
if self.use_caching:
@@ -881,8 +921,10 @@
DISTRIBUTED BY({self.dist_key_col});
""".format(self=self))
else:
- dep_shape_col = ind_shape_col = 'NULL'
- dep_var_col = indep_var_col = 'NULL'
+ dep_shape_col = 'NULL'
+ ind_shape_col = 'NULL'
+ dep_var_col = 'NULL'
+ indep_var_col = 'NULL'
source_table = self.cached_source_table
if is_very_first_hop or self.is_final_training_call:
@@ -910,10 +952,10 @@
model_in.{self.model_weights_col}
ELSE
{self.schema_madlib}.fit_transition_multiple_model(
- {dep_var_col},
- {indep_var_col},
- {dep_shape_col},
- {ind_shape_col},
+ ARRAY[{dep_var_col}]::BYTEA[],
+ ARRAY[{indep_var_col}]::BYTEA[],
+ ARRAY[{dep_shape_col}]::INTEGER[],
+ ARRAY[{ind_shape_col}]::INTEGER[],
model_in.{self.model_arch_col}::TEXT,
model_in.{self.compile_params_col}::TEXT,
model_in.{self.fit_params_col}::TEXT,
@@ -936,7 +978,7 @@
model_in.{self.dist_key_col}
FROM {self.model_input_tbl} model_in
LEFT JOIN {source_table} src
- USING ({self.dist_key_col})
+ USING ({self.dist_key_col})
DISTRIBUTED BY({self.dist_key_col})
""".format(dep_var_col=dep_var_col,
indep_var_col=indep_var_col,
@@ -965,7 +1007,8 @@
DEBUG.print_timing("udf")
- plpy.execute("DELETE FROM {self.model_output_tbl} WHERE model_weights IS NULL".format(self=self))
+ plpy.execute("DELETE FROM {self.model_output_tbl} WHERE {self.model_weights_col} IS NULL".format(self=self))
+
self.truncate_and_drop(self.model_input_tbl)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
index 07fb57e..3f478eb 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
@@ -234,14 +234,14 @@
</DD>
<DT>use_caching (optional)</DT>
- <DD>BOOLEAN, default: FALSE. Use caching of images in memory on the
- segment in order to speed up processing.
+ <DD>BOOLEAN, default: FALSE. Use caching of images in memory on the
+ segment in order to speed up processing.
@note
- When set to TRUE, image byte arrays on each segment are maintained
- in cache (GD). This can speed up training significantly, however the
- memory usage per segment increases. In effect, it
- requires enough available memory on a segment so that all images
+ When set to TRUE, image byte arrays on each segment are maintained
+ in cache (SD). This can speed up training significantly, however the
+ memory usage per segment increases. In effect, it
+ requires enough available memory on a segment so that all images
residing on that segment can be read into memory.
</dl>
@@ -1508,8 +1508,8 @@
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition_multiple_model(
- dependent_var BYTEA,
- independent_var BYTEA,
+ dependent_var BYTEA[],
+ independent_var BYTEA[],
dependent_var_shape INTEGER[],
independent_var_shape INTEGER[],
model_architecture TEXT,
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
index 15f2493..2dd17aa 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
@@ -115,7 +115,7 @@
"""
return images_per_seg[current_seg_id]
-def get_image_count_per_seg_for_minibatched_data_from_db(table_name):
+def get_image_count_per_seg_for_minibatched_data_from_db(table_name, shape_col):
"""
Query the given minibatch formatted table and return the total rows per segment.
Since we cannot pass a dictionary to the keras fit step function we create
@@ -129,10 +129,6 @@
segment array.
"""
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
-
- shape_col = add_postfix(mb_dep_var_col, "_shape")
-
if is_platform_pg():
res = plpy.execute(
""" SELECT {0}::SMALLINT[] AS shape
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
index 62b349e..d6b362d 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
@@ -29,8 +29,10 @@
from utilities.utilities import get_segments_per_host
from utilities.utilities import unique_string
from utilities.utilities import get_psql_type
+from utilities.utilities import split_quoted_delimited_str
from utilities.validate_args import get_expr_type
from utilities.validate_args import input_tbl_valid
+from utilities.validate_args import quote_ident
from madlib_keras_wrapper import *
@@ -48,7 +50,7 @@
self.table_to_validate = table_to_validate
self.test_table = test_table
self.id_col = id_col
- self.independent_varname = independent_varname
+ self.independent_varname = split_quoted_delimited_str(independent_varname)
self.output_table = output_table
self.pred_type = pred_type
self.module_name = module_name
@@ -86,18 +88,34 @@
if self.pred_type == 1:
rank_create_sql = ""
- self.pred_vartype = self.dependent_vartype.strip('[]')
- unnest_sql = ''
- if self.pred_vartype in ['text', 'character varying', 'varchar']:
+ self.pred_vartype = [i.strip('[]') for i in self.dependent_vartype]
+ unnest_sql = []
+ full_class_name_list = []
+ full_class_value_list = []
- unnest_sql = "unnest(ARRAY{0}) AS {1} , unnest".format(
- ['NULL' if i is None else i for i in class_values],
- self.dependent_varname)
- else:
+ for i in range(self.dependent_var_count):
- unnest_sql = "unnest(ARRAY[{0}]) AS {1} , unnest".format(
- ','.join(['NULL' if i is None else str(i) for i in class_values]),
- self.dependent_varname)
+ if self.pred_vartype[i] in ['text', 'character varying', 'varchar']:
+
+ unnest_sql.append("unnest(ARRAY{0}) AS {1}".format(
+ ['NULL' if j is None else j for j in class_values[i]],
+ quote_ident(self.dependent_varname[i])))
+ else:
+
+ unnest_sql.append("unnest(ARRAY[{0}]) AS {1}".format(
+ ','.join(['NULL' if j is None else str(j) for j in class_values[i]]),
+ quote_ident(self.dependent_varname[i])))
+
+
+ for j in class_values[i]:
+ tmp_class_name = self.dependent_varname[i] if self.dependent_varname[i] is not None else "NULL::TEXT"
+ full_class_name_list.append(tmp_class_name)
+ tmp_class_value = j if j is not None else "NULL::TEXT"
+ full_class_value_list.append(tmp_class_value)
+
+ unnest_sql = """unnest(ARRAY{full_class_name_list}::TEXT[]) AS class_name,
+ unnest(ARRAY{full_class_value_list}::TEXT[]) AS class_value
+ """.format(**locals())
if self.get_all:
filter_sql = ""
@@ -116,29 +134,42 @@
# Calling CREATE TABLE instead of CTAS, to ensure that the plan_cache_mode
# guc codepath is called when passing in the weights
- plpy.execute("""
+ sql = """
CREATE TABLE {self.output_table}
({self.id_col} {self.id_col_type},
- {self.dependent_varname} {self.pred_vartype},
+ class_name TEXT,
+ class_value TEXT,
{pred_col_name} {pred_col_type},
rank INTEGER)
- """.format(**locals()))
+ """.format(**locals())
+ plpy.execute(sql)
+
+ independent_varname_sql = ["{0}::REAL[]".format(quote_ident(i)) for i in self.independent_varname]
+
+ while len(independent_varname_sql) < 5:
+ independent_varname_sql.append("NULL::REAL[]")
+ independent_varname_sql = ', '.join(independent_varname_sql)
+
# Passing huge model weights to internal_keras_predict() for each row
# resulted in slowness of overall madlib_keras_predict().
# To avoid this, a CASE is added to pass the model weights only for
# the very first row(min(ctid)) that is fetched on each segment and NULL
# for the other rows.
- predict_query = plpy.prepare("""
+ rank_sql = """ row_number() OVER (PARTITION BY {self.id_col}, class_name
+ ORDER BY {pred_col_name} DESC) AS rank
+ """.format(**locals())
+ sql1 = """
INSERT INTO {self.output_table}
SELECT *
FROM (
- SELECT *, row_number() OVER (PARTITION BY {self.id_col}
- ORDER BY {pred_col_name} DESC) AS rank
+ SELECT *, {rank_sql}
FROM (
SELECT {self.id_col}::{self.id_col_type},
- {unnest_sql}({self.schema_madlib}.internal_keras_predict
- ({self.independent_varname},
+ {unnest_sql},
+ unnest(
+ {self.schema_madlib}.internal_keras_predict
+ ({independent_varname_sql},
$1,
CASE WHEN {self.test_table}.ctid = min_ctid.ctid THEN $2 ELSE NULL END,
{self.normalizing_const},
@@ -146,20 +177,22 @@
ARRAY{seg_ids_test},
ARRAY{images_per_seg_test},
{self.gpus_per_host},
- {segments_per_host})
- ) AS {pred_col_name}
- FROM {self.test_table}
- LEFT JOIN
- (SELECT {select_segmentid_comma} MIN({self.test_table}.ctid) AS ctid
- FROM {self.test_table}
- {group_by_clause}) min_ctid
- ON {join_cond_on_segmentid} {self.test_table}.ctid=min_ctid.ctid
+ {segments_per_host})) AS prob
+
+ FROM {self.test_table}
+ LEFT JOIN
+ (SELECT {select_segmentid_comma} MIN({self.test_table}.ctid) AS ctid
+ FROM {self.test_table}
+ {group_by_clause}) min_ctid
+ ON {join_cond_on_segmentid} {self.test_table}.ctid=min_ctid.ctid
) __subq1__
) __subq2__
{filter_sql}
- """.format(**locals()), ["text", "bytea"])
+ """.format(**locals())
+ predict_query = plpy.prepare(sql1, ["text", "bytea"])
plpy.execute(predict_query, [self.model_arch, self.model_weights])
+
if self.is_response:
# Drop the rank column since it is irrelevant
plpy.execute("""
@@ -167,12 +200,16 @@
DROP COLUMN rank
""".format(**locals()))
+ def set_default_class_values(self, in_class_values, dependent_var_count):
- def set_default_class_values(self, class_values):
- self.class_values = class_values
- if self.class_values is None:
- num_classes = get_num_classes(self.model_arch)
- self.class_values = range(0, num_classes)
+ self.class_values = []
+ num_classes = get_num_classes(self.model_arch, dependent_var_count)
+ for counter, i in enumerate(in_class_values):
+ if (i is None) or (i==[None]):
+ self.class_values.append(range(0, num_classes[counter]))
+ else:
+ self.class_values.append(i)
+
@MinWarning("warning")
class Predict(BasePredict):
@@ -199,16 +236,20 @@
self.dependent_vartype = param_proc.get_dependent_vartype()
self.model_weights = param_proc.get_model_weights()
self.model_arch = param_proc.get_model_arch()
- class_values = param_proc.get_class_values()
- self.set_default_class_values(class_values)
- self.normalizing_const = param_proc.get_normalizing_const()
+
self.dependent_varname = param_proc.get_dependent_varname()
+ self.dependent_var_count = len(self.dependent_varname)
+ class_values = []
+ for dep in self.dependent_varname:
+ class_values.append(param_proc.get_class_values(dep))
+ self.set_default_class_values(class_values, self.dependent_var_count)
+ self.normalizing_const = param_proc.get_normalizing_const()
self.validate()
self.id_col_type = get_expr_type(self.id_col, self.test_table)
BasePredict.call_internal_keras(self)
if self.is_mult_model:
- plpy.execute("DROP VIEW IF EXISTS {}".format(self.temp_summary_view))
+ plpy.execute("DROP VIEW IF EXISTS {0}".format(self.temp_summary_view))
def validate(self):
input_tbl_valid(self.model_table, self.module_name)
@@ -218,13 +259,11 @@
plpy.error("{self.module_name}: Multi-model needs to pass mst_key".format(**locals()))
InputValidator.validate_predict_evaluate_tables(
self.module_name, self.model_table, self.model_summary_table,
- self.test_table, self.output_table, self.independent_varname)
+ self.test_table, self.output_table)
InputValidator.validate_id_in_test_tbl(
self.module_name, self.test_table, self.id_col)
- InputValidator.validate_class_values(
- self.module_name, self.class_values, self.pred_type, self.model_arch)
input_shape = get_input_shape(self.model_arch)
InputValidator.validate_pred_type(
self.module_name, self.pred_type, self.class_values)
@@ -236,25 +275,32 @@
def __init__(self, schema_madlib, model_arch_table, model_id,
test_table, id_col, independent_varname, output_table,
pred_type, use_gpus, class_values, normalizing_const,
- **kwargs):
+ dependent_count, **kwargs):
self.module_name='madlib_keras_predict_byom'
self.model_arch_table = model_arch_table
self.model_id = model_id
self.class_values = class_values
self.normalizing_const = normalizing_const
- self.dependent_varname = 'dependent_var'
+ self.dependent_var_count = dependent_count
+
+ if self.dependent_var_count == 1:
+ self.dependent_varname = ['dependent_var']
+ else:
+ self.dependent_varname = ['dependent_var_{0}'.format(i) for i in range(self.dependent_var_count)]
BasePredict.__init__(self, schema_madlib, model_arch_table,
test_table, id_col, independent_varname,
output_table, pred_type, use_gpus, self.module_name)
-
+ self.dependent_vartype = []
if self.class_values:
- self.dependent_vartype = get_psql_type(self.class_values[0])
+ for i in self.class_values:
+ self.dependent_vartype.append(get_psql_type(i[0]))
else:
+ self.class_values = [None]*self.dependent_var_count
if self.pred_type == 1:
- self.dependent_vartype = 'text'
+ self.dependent_vartype = ['text']*self.dependent_var_count
else:
- self.dependent_vartype = 'double precision'
+ self.dependent_vartype = ['double precision']*self.dependent_var_count
## Set default values for norm const and class_values
# use_gpus and pred_type are defaulted in base_predict's init
@@ -277,18 +323,37 @@
_assert(self.model_weights and self.model_arch,
"{0}: Model weights and architecture should not be NULL.".format(
self.module_name))
- self.set_default_class_values(self.class_values)
+ self.set_default_class_values(self.class_values, self.dependent_var_count)
InputValidator.validate_pred_type(
self.module_name, self.pred_type, self.class_values)
InputValidator.validate_normalizing_const(
self.module_name, self.normalizing_const)
- InputValidator.validate_class_values(
- self.module_name, self.class_values, self.pred_type, self.model_arch)
+
+ # TODO: Fix this validation
+ # The current method looks at the 'units' keyword which doesn't mean
+ # anything because every layer has it. It was passing because the layers
+ # are traversed in order. It won't work for multi-io and prone to breaking
+ # in the regular case.
+
+ # InputValidator.validate_class_values(
+ # self.module_name, self.class_values, self.pred_type, self.model_arch)
InputValidator.validate_input_shape(
self.test_table, self.independent_varname,
get_input_shape(self.model_arch), 1)
+def internal_keras_predict_wide(independent_var, independent_var2,
+ independent_var3, independent_var4, independent_var5,
+ model_architecture, model_weights,
+ normalizing_const, current_seg_id, seg_ids,
+ images_per_seg, gpus_per_host, segments_per_host,
+ **kwargs):
+ return internal_keras_predict(
+ [independent_var, independent_var2, independent_var3, independent_var4, independent_var5],
+ model_architecture, model_weights, normalizing_const, current_seg_id,
+ seg_ids, images_per_seg, gpus_per_host, segments_per_host,
+ **kwargs)
+
def internal_keras_predict(independent_var, model_architecture, model_weights,
normalizing_const, current_seg_id, seg_ids,
images_per_seg, gpus_per_host, segments_per_host,
@@ -297,7 +362,7 @@
model_key = 'segment_model_predict'
row_count_key = 'row_count'
try:
- device_name = get_device_name_and_set_cuda_env( gpus_per_host, current_seg_id)
+ device_name = get_device_name_and_set_cuda_env(gpus_per_host, current_seg_id)
if model_key not in SD:
set_keras_session(device_name, gpus_per_host, segments_per_host)
model = model_from_json(model_architecture)
@@ -311,16 +376,24 @@
# Since the test data isn't mini-batched,
# we have to make sure that the test data np array has the same
# number of dimensions as input_shape. So we add a dimension to x.
- independent_var = expand_input_dims(independent_var)
- independent_var /= normalizing_const
+ independent_var_filtered = []
+ for i in independent_var:
+ if i is not None:
+ independent_var_filtered.append(expand_input_dims(i)/normalizing_const)
with tf.device(device_name):
- probs = model.predict(independent_var)
+ probs = model.predict(independent_var_filtered)
# probs is a list containing a list of probability values, of all
# class levels. Since we are assuming each input is a single image,
# and not mini-batched, this list contains exactly one list in it,
# so return back the first list in probs.
- result = probs[0]
+ result = []
+ if len(independent_var_filtered) > 1:
+ for i in probs:
+ for j in i[0]:
+ result.append(j)
+ else:
+ result = probs[0]
total_images = get_image_count_per_seg_from_array(seg_ids.index(current_seg_id),
images_per_seg)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
index f7d2076..2549b84 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
@@ -53,11 +53,12 @@
class InputValidator:
@staticmethod
def validate_predict_evaluate_tables(
- module_name, model_table, model_summary_table, test_table, output_table,
- independent_varname):
+ module_name, model_table, model_summary_table, test_table, output_table):
InputValidator._validate_model_weights_tbl(module_name, model_table)
InputValidator._validate_model_summary_tbl(
module_name, model_summary_table)
+ independent_varname = plpy.execute("SELECT {0} FROM {1}".format(
+ "independent_varname", model_summary_table))[0]["independent_varname"]
InputValidator._validate_test_tbl(
module_name, test_table, independent_varname)
output_tbl_valid(output_table, module_name)
@@ -118,54 +119,53 @@
this case is 1 (start the index at 1).
"""
+ shapes = []
if is_minibatched:
- ind_shape_col = add_postfix(independent_varname, "_shape")
- query = """
- SELECT {ind_shape_col} AS shape
- FROM {table}
- LIMIT 1
- """.format(**locals())
- # This query will fail if an image in independent var does not have the
- # same number of dimensions as the input_shape.
- result = plpy.execute(query)[0]['shape']
- result = result[1:]
+ for i in independent_varname:
+ ind_shape_col = add_postfix(i, "_shape")
+ query = """
+ SELECT {ind_shape_col} AS shape
+ FROM {table}
+ LIMIT 1
+ """.format(**locals())
+ # This query will fail if an image in independent var does not have the
+ # same number of dimensions as the input_shape.
+ result = plpy.execute(query)
+ result = result[0]['shape']
+ shapes.append(result[1:])
else:
- array_upper_query = ", ".join("array_upper({0}, {1}) AS n_{2}".format(
- independent_varname, i+offset, i) for i in range(len(input_shape)))
- query = """
- SELECT {0}
- FROM {1}
- LIMIT 1
- """.format(array_upper_query, table)
-
- # This query will fail if an image in independent var does not have the
- # same number of dimensions as the input_shape.
- result = plpy.execute(query)[0]
-
- _assert(len(result) == len(input_shape),
+ for counter, ind in enumerate(independent_varname):
+ array_upper_query = ", ".join("array_upper({0}, {1})".format(
+ ind, i+offset, i) for i in range(len(input_shape[counter])))
+ query = """
+ SELECT ARRAY[{0}] AS shape
+ FROM {1}
+ LIMIT 1
+ """.format(array_upper_query, table)
+ # This query will fail if an image in independent var does not have the
+ # same number of dimensions as the input_shape.
+ result = plpy.execute(query)
+ result = result[0]['shape']
+ shapes.append(result)
+ _assert(len(shapes) == len(input_shape),
"model_keras error: The number of dimensions ({0}) of each image"
" in model architecture and {1} in {2} ({3}) do not match.".format(
len(input_shape), independent_varname, table, len(result)))
for i in range(len(input_shape)):
- if is_minibatched:
- key_name = i
- input_shape_from_table = [result[j]
- for j in range(len(input_shape))]
- else:
- key_format = "n_{0}"
- key_name = key_format.format(i)
- input_shape_from_table = [result[key_format.format(j)]
- for j in range(len(input_shape))]
- if result[key_name] != input_shape[i]:
- # Construct the shape in independent varname to display
- # meaningful error msg.
- plpy.error("model_keras error: Input shape {0} in the model"
- " architecture does not match the input shape {1} of column"
- " {2} in table {3}.".format(
- input_shape, input_shape_from_table,
- independent_varname, table))
+ local_input_shape = input_shape[i]
+ local_arch_shape = shapes[i]
+
+ for j in range(len(local_input_shape)):
+ if local_arch_shape[j] != local_input_shape[j]:
+ # Construct the shape in independent varname to display
+ # meaningful error msg.
+ plpy.error("model_keras error: Input shape {0} in the model"
+ " architecture does not match the input shape {1} of column"
+ " {2} in table {3}.".format(
+ local_input_shape[j], local_arch_shape[j],
+ independent_varname, table))
@staticmethod
def validate_model_arch_table(module_name, model_arch_table, model_id):
@@ -184,14 +184,15 @@
def validate_class_values(module_name, class_values, pred_type, model_arch):
if not class_values:
return
- num_classes = len(class_values)
- _assert(num_classes == get_num_classes(model_arch),
+ num_classes = [len(i) for i in class_values]
+ _assert(num_classes == get_num_classes(model_arch, len(class_values)),
"{0}: The number of class values do not match the " \
"provided architecture.".format(module_name))
- if pred_type == 'prob' and num_classes+1 >= 1600:
- plpy.error({"{0}: The output will have {1} columns, exceeding the "\
- " max number of columns that can be created (1600)".format(
- module_name, num_classes+1)})
+ for i in num_classes:
+ if pred_type == 'prob' and i+1 >= 1600:
+ plpy.error({"{0}: The output will have {1} columns, exceeding the "\
+ " max number of columns that can be created (1600)".format(
+ module_name, i+1)})
@staticmethod
def validate_model_weights(module_name, model_arch, model_weights):
@@ -217,19 +218,19 @@
@staticmethod
def _validate_test_tbl(module_name, test_table, independent_varname):
input_tbl_valid(test_table, module_name)
- _assert(is_var_valid(test_table, independent_varname),
+ for i in independent_varname:
+ _assert(is_var_valid(test_table, i),
"{module_name} error: invalid independent_varname "
- "('{independent_varname}') for test table "
+ "('{i}') for test table "
"({table}).".format(
module_name=module_name,
- independent_varname=independent_varname,
+ i=i,
table=test_table))
@staticmethod
def _validate_model_summary_tbl(module_name, model_summary_table):
input_tbl_valid(model_summary_table, module_name)
- cols_to_check_for = [CLASS_VALUES_COLNAME,
- DEPENDENT_VARNAME_COLNAME,
+ cols_to_check_for = [DEPENDENT_VARNAME_COLNAME,
DEPENDENT_VARTYPE_COLNAME,
MODEL_ID_COLNAME,
MODEL_ARCH_TABLE_COLNAME,
@@ -260,9 +261,10 @@
class FitCommonValidator(object):
def __init__(self, source_table, validation_table, output_model_table,
model_arch_table, model_id, dependent_varname,
- independent_varname, num_iterations,
+ independent_varname, dep_shape_cols, ind_shape_cols, num_iterations,
metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg, module_name, object_table):
+ use_gpus, accessible_gpus_for_seg, module_name, object_table,
+ val_dep_var, val_ind_var):
self.source_table = source_table
self.validation_table = validation_table
self.output_model_table = output_model_table
@@ -270,8 +272,8 @@
self.model_id = model_id
self.dependent_varname = dependent_varname
self.independent_varname = independent_varname
- self.dep_shape_col = add_postfix(dependent_varname, "_shape")
- self.ind_shape_col = add_postfix(independent_varname, "_shape")
+ self.dep_shape_cols = dep_shape_cols
+ self.ind_shape_cols = ind_shape_cols
self.metrics_compute_frequency = metrics_compute_frequency
self.warm_start = warm_start
self.num_iterations = num_iterations
@@ -285,6 +287,9 @@
self.output_model_table, "_summary")
self.accessible_gpus_for_seg = accessible_gpus_for_seg
self.module_name = module_name
+ self.val_dep_var = val_dep_var
+ self.val_ind_var = val_ind_var
+
self._validate_common_args()
if use_gpus:
InputValidator._validate_gpu_config(self.module_name,
@@ -300,20 +305,17 @@
if self.object_table is not None:
input_tbl_valid(self.object_table, self.module_name)
cols_in_tbl_valid(self.object_table, CustomFunctionSchema.col_names, self.module_name)
- input_tbl_valid(self.source_summary_table, self.module_name,
- error_suffix_str="Please ensure that the source table ({0}) "
- "has been preprocessed by "
- "the image preprocessor.".format(self.source_table))
- cols_in_tbl_valid(self.source_summary_table, [CLASS_VALUES_COLNAME,
- NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
+
+ cols_in_tbl_valid(self.source_summary_table,
+ [NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
'dependent_varname', 'independent_varname'], self.module_name)
if not is_platform_pg():
cols_in_tbl_valid(self.source_table, [DISTRIBUTION_KEY_COLNAME], self.module_name)
# Source table and validation tables must have the same schema
self._validate_input_table(self.source_table)
- validate_bytea_var_for_minibatch(self.source_table,
- self.dependent_varname)
+ for i in self.dependent_varname:
+ validate_bytea_var_for_minibatch(self.source_table, i)
self._validate_validation_table()
if self.warm_start:
@@ -323,42 +325,50 @@
output_tbl_valid(self.output_model_table, self.module_name)
output_tbl_valid(self.output_summary_model_table, self.module_name)
- def _validate_input_table(self, table):
- _assert(is_var_valid(table, self.independent_varname),
+ def _validate_input_table(self, table, is_validation_table=False):
+
+ independent_varname = self.val_ind_var if is_validation_table else self.independent_varname
+ dependent_varname = self.val_dep_var if is_validation_table else self.dependent_varname
+
+ for name in independent_varname:
+ _assert(is_var_valid(table, name),
"{module_name}: invalid independent_varname "
"('{independent_varname}') for table ({table}). "
"Please ensure that the input table ({table}) "
"has been preprocessed by the image preprocessor.".format(
module_name=self.module_name,
- independent_varname=self.independent_varname,
+ independent_varname=name,
table=table))
- _assert(is_var_valid(table, self.dependent_varname),
+ for name in dependent_varname:
+ _assert(is_var_valid(table, name),
"{module_name}: invalid dependent_varname "
"('{dependent_varname}') for table ({table}). "
"Please ensure that the input table ({table}) "
"has been preprocessed by the image preprocessor.".format(
module_name=self.module_name,
- dependent_varname=self.dependent_varname,
+ dependent_varname=name,
table=table))
+ if not is_validation_table:
+ for name in self.ind_shape_cols:
+ _assert(is_var_valid(table, name),
+ "{module_name}: invalid independent_var_shape "
+ "('{ind_shape_col}') for table ({table}). "
+ "Please ensure that the input table ({table}) "
+ "has been preprocessed by the image preprocessor.".format(
+ module_name=self.module_name,
+ ind_shape_col=name,
+ table=table))
- _assert(is_var_valid(table, self.ind_shape_col),
- "{module_name}: invalid independent_var_shape "
- "('{ind_shape_col}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- ind_shape_col=self.ind_shape_col,
- table=table))
-
- _assert(is_var_valid(table, self.dep_shape_col),
- "{module_name}: invalid dependent_var_shape "
- "('{dep_shape_col}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- dep_shape_col=self.dep_shape_col,
- table=table))
+ for name in self.dep_shape_cols:
+ _assert(is_var_valid(table, name),
+ "{module_name}: invalid dependent_var_shape "
+ "('{dep_shape_col}') for table ({table}). "
+ "Please ensure that the input table ({table}) "
+ "has been preprocessed by the image preprocessor.".format(
+ module_name=self.module_name,
+ dep_shape_col=name,
+ table=table))
if not is_platform_pg():
_assert(is_var_valid(table, DISTRIBUTION_KEY_COLNAME),
@@ -378,13 +388,14 @@
def _validate_validation_table(self):
if self.validation_table and self.validation_table.strip() != '':
input_tbl_valid(self.validation_table, self.module_name)
- self._validate_input_table(self.validation_table)
- dependent_vartype = get_expr_type(self.dependent_varname,
- self.validation_table)
- _assert(dependent_vartype == 'bytea',
- "Dependent variable column {0} in validation table {1} should be "
- "a bytea and also one hot encoded.".format(
- self.dependent_varname, self.validation_table))
+ self._validate_input_table(self.validation_table, True)
+ for i in self.val_dep_var:
+ dependent_vartype = get_expr_type(i,
+ self.validation_table)
+ _assert(dependent_vartype == 'bytea',
+ "Dependent variable column {0} in validation table {1} should be "
+ "a bytea and also one hot encoded.".format(
+ i, self.validation_table))
def validate_input_shapes(self, input_shape):
@@ -399,9 +410,9 @@
class FitInputValidator(FitCommonValidator):
def __init__(self, source_table, validation_table, output_model_table,
model_arch_table, model_id, dependent_varname,
- independent_varname, num_iterations,
+ independent_varname, dep_shape_cols, ind_shape_cols, num_iterations,
metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg, object_table):
+ use_gpus, accessible_gpus_for_seg, object_table, val_dep_var, val_ind_var):
self.module_name = 'madlib_keras_fit'
super(FitInputValidator, self).__init__(source_table,
@@ -411,22 +422,27 @@
model_id,
dependent_varname,
independent_varname,
+ dep_shape_cols,
+ ind_shape_cols,
num_iterations,
metrics_compute_frequency,
warm_start,
use_gpus,
accessible_gpus_for_seg,
self.module_name,
- object_table)
+ object_table,
+ val_dep_var,
+ val_ind_var)
InputValidator.validate_model_arch_table(self.module_name, self.model_arch_table,
self.model_id)
class FitMultipleInputValidator(FitCommonValidator):
def __init__(self, source_table, validation_table, output_model_table,
model_selection_table, model_selection_summary_table, dependent_varname,
- independent_varname, num_iterations, model_info_table, mst_key_col,
+ independent_varname, dep_shape_cols, ind_shape_cols,
+ num_iterations, model_info_table, mst_key_col,
model_arch_table_col, metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg):
+ use_gpus, accessible_gpus_for_seg, val_dep_var, val_ind_var):
self.module_name = 'madlib_keras_fit_multiple'
@@ -450,13 +466,17 @@
None,
dependent_varname,
independent_varname,
+ dep_shape_cols,
+ ind_shape_cols,
num_iterations,
metrics_compute_frequency,
warm_start,
use_gpus,
accessible_gpus_for_seg,
self.module_name,
- self.object_table)
+ self.object_table,
+ val_dep_var,
+ val_ind_var)
class MstLoaderInputValidator():
def __init__(self,
diff --git a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
index 298f63a..9c28c43 100644
--- a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
+++ b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
@@ -37,12 +37,16 @@
def get_input_shape(model_arch):
arch_layers = _get_layers(model_arch)
- if 'batch_input_shape' in arch_layers[0]['config']:
- return arch_layers[0]['config']['batch_input_shape'][1:]
+ shapes = []
+ for i in arch_layers:
+ if 'batch_input_shape' in i['config']:
+ shapes.append(i['config']['batch_input_shape'][1:])
+ if shapes:
+ return shapes
plpy.error('Unable to get input shape from model architecture.'\
'Make sure that the first layer defines an input_shape.')
-def get_num_classes(model_arch):
+def get_num_classes(model_arch, multi_dep_count):
"""
We assume that the last dense layer in the model architecture contains the num_classes (units)
An example can be:
@@ -60,11 +64,15 @@
:return:
"""
arch_layers = _get_layers(model_arch)
- i = len(arch_layers) - 1
- while i >= 0:
- if 'units' in arch_layers[i]['config']:
- return arch_layers[i]['config']['units']
- i -= 1
+ num_classes = []
+
+ layer_count = len(arch_layers) - 1
+ for i in range(multi_dep_count):
+ if 'units' in arch_layers[layer_count-i]['config']:
+ num_classes.append(arch_layers[layer_count-i]['config']['units'])
+
+ if num_classes:
+ return num_classes
plpy.error('Unable to get number of classes from model architecture.')
def get_model_arch_layers_str(model_arch):
diff --git a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in b/src/ports/postgres/modules/deep_learning/predict_input_params.py_in
index fdab1e4..d9ba091 100644
--- a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in
+++ b/src/ports/postgres/modules/deep_learning/predict_input_params.py_in
@@ -48,8 +48,9 @@
def _get_dict_for_table(self, table_name):
return plpy.execute("SELECT * FROM {0} {1}".format(table_name, self.mult_where_clause), 1)[0]
- def get_class_values(self):
- return self.model_summary_dict[CLASS_VALUES_COLNAME]
+ def get_class_values(self, dep):
+ col_name = add_postfix(dep, "_class_values")
+ return self.model_summary_dict[col_name]
def get_dependent_varname(self):
return self.model_summary_dict[DEPENDENT_VARNAME_COLNAME]
diff --git a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in
index 7c6c5c3..567a8bc 100644
--- a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in
@@ -98,22 +98,22 @@
'Incorrect number of buffers in data_preprocessor_input_batch.')
FROM data_preprocessor_input_batch;
-SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape ' || independent_var_shape[2])
+SELECT assert(x_shape[2]=6, 'Incorrect image shape ' || x_shape[2])
FROM data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
+SELECT assert(x_shape[1]=buffer_size, 'Incorrect buffer size ' || x_shape[1])
FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
+SELECT assert(x_shape[1]=buffer_size, 'Incorrect buffer size ' || x_shape[1])
FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1;
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
+SELECT assert(x_shape[1]=buffer_size, 'Incorrect buffer size ' || x_shape[1])
FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=2;
SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?')
-FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
+FROM (SELECT SUM(x_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length ' || octet_length(independent_var)::TEXT)
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length ' || octet_length(x)::TEXT)
FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
@@ -130,16 +130,16 @@
'Incorrect number of buffers in validation_out.')
FROM validation_out;
-SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape.')
+SELECT assert(x_shape[2]=6, 'Incorrect image shape.')
FROM data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size.')
+SELECT assert(x_shape[1]=buffer_size, 'Incorrect buffer size.')
FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1;
SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?')
-FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
+FROM (SELECT SUM(x_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 5) buffer_size) a, validation_out WHERE buffer_id=0;
DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
@@ -180,44 +180,6 @@
FROM validation_out GROUP BY 1;
SELECT assert(__internal_gpu_config__ = 'all_segments', 'Missing column in validation summary table')
FROM validation_out_summary;
-
--- Test data distributed on specified segments
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use (dbid INTEGER, notes TEXT);
-INSERT INTO segments_to_use VALUES (2, 'GPU segment');
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'id',
- 'x',
- 1,
- NULL,
- NULL,
- 'segments_to_use');
-SELECT assert(count(DISTINCT(gp_segment_id)) = 1, 'Fail to distribute data on segment0')
-FROM data_preprocessor_input_batch;
-SELECT assert(count(*) = 17, 'Fail to distribute all data on segment0')
-FROM data_preprocessor_input_batch;
-SELECT assert(__internal_gpu_config__ = ARRAY[0], 'Invalid column value in summary table')
-FROM data_preprocessor_input_batch_summary;
-
--- Test data distributed on specified segments for validation_preprocessor_dl
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input',
- 'validation_out',
- 'id',
- 'x',
- 'data_preprocessor_input_batch',
- 1,
- 'segments_to_use');
-SELECT assert(count(DISTINCT(gp_segment_id)) = 1, 'Failed to distribute validation data on segment0')
-FROM validation_out;
-SELECT assert(count(*) = 17, 'Fail to distribute all validation data on segment0')
-FROM validation_out;
-SELECT assert(__internal_gpu_config__ = ARRAY[0], 'Invalid column value in validation summary table')
-FROM validation_out_summary;
!>)
DROP TABLE IF EXISTS data_preprocessor_input;
@@ -249,17 +211,17 @@
'x',
4,
5,
- 16 -- num_classes
+ ARRAY[16] -- num_classes
);
-- Test that indepdendent vars get divided by 5, by verifying min value goes from 1 to 0.2, and max value from 233 to 46.6
-SELECT assert(relative_error(MIN(x),0.2) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a;
-SELECT assert(relative_error(MAX(x),46.6) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a;
+SELECT assert(relative_error(MIN(x),0.2) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(x)) as x FROM data_preprocessor_input_batch) a;
+SELECT assert(relative_error(MAX(x),46.6) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(x)) as x FROM data_preprocessor_input_batch) a;
-- Test that 1-hot encoded array is of length 16 (num_classes)
-SELECT assert(dependent_var_shape[2] = 16, 'Incorrect one-hot encode dimension with num_classes') FROM
+SELECT assert(y_shape[2] = 16, 'Incorrect one-hot encode dimension with num_classes') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-- Test summary table
@@ -267,22 +229,22 @@
(
source_table = 'data_preprocessor_input' AND
output_table = 'data_preprocessor_input_batch' AND
- dependent_varname = 'y' AND
- independent_varname = 'x' AND
- dependent_vartype = 'integer' AND
- class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
+ dependent_varname[0] = 'y' AND
+ independent_varname[0] = 'x' AND
+ dependent_vartype[0] = 'integer' AND
+ y_class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
summary.buffer_size = a.buffer_size AND -- we sort the class values in python
normalizing_const = 5 AND
pg_typeof(normalizing_const) = 'real'::regtype AND
- num_classes = 16 AND
+ num_classes[0] = 16 AND
distribution_rules = 'all_segments',
'Summary Validation failed. Actual:' || __to_char(summary)
) FROM (SELECT buffer_size(17, 4) buffer_size) a,
(SELECT * FROM data_preprocessor_input_batch_summary) summary;
--- Test output data type
-SELECT assert(pg_typeof(independent_var) = 'bytea'::regtype, 'Wrong independent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'Wrong dependent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(pg_typeof(x) = 'bytea'::regtype, 'Wrong independent_varx type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(pg_typeof(y) = 'bytea'::regtype, 'Wrong dependent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-- Test for validation data where the input table has only a subset of
-- the classes compared to the original training data
@@ -302,10 +264,10 @@
'data_preprocessor_input_batch');
-- Hard code 5.0 as the normalizing constant, based on the previous
-- query's input param, to test if normalization is correct.
-SELECT assert(abs(x_new[1]/5.0-(convert_bytea_to_real_array(independent_var))[1]) < 0.0000001, 'Incorrect normalizing in validation table.')
-FROM validation_input, validation_out;
+SELECT assert(abs(input.x_new[1]/5.0-(convert_bytea_to_real_array(output.x_new))[1]) < 0.0000001, 'Incorrect normalizing in validation table.')
+FROM validation_input as input, validation_out as output;
-- Validate if one hot encoding is as expected.
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0}', 'Incorrect one-hot encode dimension with num_classes') FROM
+SELECT assert(convert_bytea_to_smallint_array(y_new) = '{0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0}', 'Incorrect one-hot encode dimension with num_classes') FROM
validation_out WHERE buffer_id = 0;
-- Test summary table
@@ -313,14 +275,14 @@
(
source_table = 'validation_input' AND
output_table = 'validation_out' AND
- dependent_varname = 'y_new' AND
- independent_varname = 'x_new' AND
- dependent_vartype = 'integer' AND
- class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
+ dependent_varname[0] = 'y_new' AND
+ independent_varname[0] = 'x_new' AND
+ dependent_vartype[0] = 'integer' AND
+ y_new_class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
buffer_size = 1 AND -- we sort the class values in python
normalizing_const = 5 AND
pg_typeof(normalizing_const) = 'real'::regtype AND
- num_classes = 16,
+ num_classes[0] = 16,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from validation_out_summary) summary;
@@ -334,37 +296,37 @@
'x',
4,
5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(pg_typeof(y1) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(y1_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:2]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'boolean' AND
- class_values = '{f,t}' AND
- num_classes = 2,
+SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(y1))[1:2]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
+SELECT assert (dependent_vartype[0] = 'boolean' AND
+ y1_class_values = '{f,t}' AND
+ num_classes[0] = 2,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
-- Test to assert the output summary table for validation has the correct
-- num_classes and class_values
DROP TABLE IF EXISTS validation_input;
-CREATE TABLE validation_input(id serial, x_new double precision[], y INTEGER, y_new BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]);
-INSERT INTO validation_input(x_new, y, y_new, y2, y3, y4, y5) VALUES
+CREATE TABLE validation_input(id serial, x_new double precision[], y INTEGER, y1 BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]);
+INSERT INTO validation_input(x_new, y, y1, y2, y3, y4, y5) VALUES
(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]);
DROP TABLE IF EXISTS validation_out, validation_out_summary;
SELECT validation_preprocessor_dl(
'validation_input',
'validation_out',
- 'y_new',
+ 'y1',
'x_new',
'data_preprocessor_input_batch');
-SELECT assert (dependent_vartype = 'boolean' AND
- class_values = '{f,t}' AND
- num_classes = 2,
+SELECT assert (dependent_vartype[0] = 'boolean' AND
+ y1_class_values = '{f,t}' AND
+ num_classes[0] = 2,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from validation_out_summary) summary;
-- test text type
@@ -376,17 +338,17 @@
'x',
4,
5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(pg_typeof(y2) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(y2_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'text' AND
- class_values = '{a,b,c}' AND
- num_classes = 3,
+SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(y2))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
+SELECT assert (dependent_vartype[0] = 'text' AND
+ y2_class_values = '{a,b,c}' AND
+ num_classes[0] = 3,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
@@ -412,15 +374,15 @@
'x',
4,
5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(pg_typeof(y3) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(y3_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'double precision' AND
- class_values = '{4.0,4.2,5.0}' AND
- num_classes = 3,
+SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(y3))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
+SELECT assert (dependent_vartype[0]= 'double precision' AND
+ y3_class_values = '{4.0,4.2,5.0}' AND
+ num_classes[0] = 3,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
@@ -433,17 +395,17 @@
'x',
4,
5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(pg_typeof(y4) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(y4_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(relative_error(SUM(y), SUM(y4)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y4) as y4 FROM data_preprocessor_input) b;
-SELECT assert (dependent_vartype = 'double precision[]' AND
- class_values IS NULL AND
- num_classes IS NULL,
+SELECT assert(relative_error(SUM(y), SUM(y4)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(y4)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y4) as y4 FROM data_preprocessor_input) b;
+SELECT assert (dependent_vartype[0]= 'double precision[]' AND
+ y4_class_values IS NULL AND
+ num_classes[0] IS NULL,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
@@ -455,7 +417,7 @@
'x_new',
'data_preprocessor_input_batch');
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0}' AND dependent_var_shape[2] = 2, 'Incorrect one-hot encoding for already encoded dep var') FROM
+SELECT assert(convert_bytea_to_smallint_array(y4) = '{1,0}' AND y4_shape[2] = 2, 'Incorrect one-hot encoding for already encoded dep var') FROM
validation_out WHERE buffer_id = 0;
-- test integer array type
@@ -467,17 +429,17 @@
'x',
4,
5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(pg_typeof(y5) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
+SELECT assert(y5_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(relative_error(SUM(y), SUM(y5)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y5) as y5 FROM data_preprocessor_input) b;
-SELECT assert (dependent_vartype = 'integer[]' AND
- class_values IS NULL AND
- num_classes IS NULL,
+SELECT assert(relative_error(SUM(y), SUM(y5)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(y5)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y5) as y5 FROM data_preprocessor_input) b;
+SELECT assert (dependent_vartype[0] = 'integer[]' AND
+ y5_class_values IS NULL AND
+ num_classes[0] IS NULL,
'Summary Validation failed. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
@@ -511,21 +473,21 @@
'x',
4,
5,
- 5 -- num_classes
+ ARRAY[5] -- num_classes
);
-- Test summary table if class_values has NULL as a legitimate
-- class label, and also two other NULLs because num_classes=5
-- but table has only 3 distinct class labels (including NULL)
SELECT assert
(
- class_values = '{NULL,a,b,NULL,NULL}',
+ label_class_values = '{NULL,a,b,NULL,NULL}',
'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
-SELECT assert(dependent_var_shape[2] = 5, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(label_shape[2] = 5, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = buffer_size*6*4, 'Incorrect buffer length')
FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-- The same tests, but for validation.
@@ -551,18 +513,18 @@
-- but table has only 3 distinct class labels (including NULL)
SELECT assert
(
- class_values = '{NULL,a,b,NULL,NULL}',
+ label_class_values = '{NULL,a,b,NULL,NULL}',
'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
) from (select * from validation_out_batch_summary) summary;
-- Validate one hot encoding for specific row is correct
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,1,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.')
+SELECT assert(convert_bytea_to_smallint_array(validation_out_batch.label) = '{0,1,0,0,0}' AND label_shape[2] =5, 'Incorrect normalizing in validation table.')
FROM data_preprocessor_input_validation_null, validation_out_batch
-WHERE x[1]=1 AND abs((convert_bytea_to_real_array(independent_var))[1] - 0.2::REAL) < 0.00001;
+WHERE data_preprocessor_input_validation_null.x[1]=1 AND abs((convert_bytea_to_real_array(validation_out_batch.x))[1] - 0.2::REAL) < 0.00001;
-- Assert one-hot encoding for NULL label
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.')
+SELECT assert(convert_bytea_to_smallint_array(validation_out_batch. label) = '{1,0,0,0,0}' AND label_shape[2] =5, 'Incorrect normalizing in validation table.')
FROM data_preprocessor_input_validation_null, validation_out_batch
-WHERE x[1]=111 AND abs((convert_bytea_to_real_array(independent_var))[1] - 22.2::REAL) < 0.00001;
+WHERE data_preprocessor_input_validation_null.x[1]=111 AND abs((convert_bytea_to_real_array(validation_out_batch.x))[1] - 22.2::REAL) < 0.00001;
-- Test the content of 1-hot encoded dep var when NULL is the
-- class label.
@@ -579,7 +541,7 @@
'x',
4,
5,
- 3 -- num_classes
+ ARRAY[3] -- num_classes
);
-- class_values must be '{NULL,NULL,NULL}' where the first NULL
@@ -587,17 +549,17 @@
-- are added as num_classes=3.
SELECT assert
(
- class_values = '{NULL,NULL,NULL}',
+ label_class_values = '{NULL,NULL,NULL}',
'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
) from (select * from data_preprocessor_input_batch_summary) summary;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(label_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = 24, 'Incorrect buffer length')
FROM data_preprocessor_input_batch WHERE buffer_id=0;
-- NULL is treated as a class label, so it should show '1' for the
-- first index
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
+SELECT assert(convert_bytea_to_smallint_array(label) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
data_preprocessor_input_batch WHERE buffer_id = 0;
-- The same tests for validation.
@@ -616,17 +578,17 @@
-- are added as num_classes=3.
SELECT assert
(
- class_values = '{NULL,NULL,NULL}',
+ label_class_values = '{NULL,NULL,NULL}',
'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
) from (select * from validation_out_batch_summary) summary;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
+SELECT assert(label_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
validation_out_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length')
+SELECT assert(octet_length(x) = 24, 'Incorrect buffer length')
FROM data_preprocessor_input_batch WHERE buffer_id=0;
-- NULL is treated as a class label, so it should show '1' for the
-- first index
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
+SELECT assert(convert_bytea_to_smallint_array(label) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
validation_out_batch WHERE buffer_id = 0;
-- Test if validation class values is not a subset of training data class values.
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
index d4841a9..82f4301 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
@@ -49,8 +49,8 @@
validation_table IS NULL AND
model = 'automl_output' AND
model_info = 'automl_output_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
model_arch_table = 'iris_model_arch' AND
model_selection_table = 'automl_mst_table' AND
automl_method = 'hyperopt' AND
@@ -64,9 +64,9 @@
start_training_time < now() AND
end_training_time < now() AND
madlib_version IS NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype = 'character varying' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] = 'character varying' AND
normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
) FROM (SELECT * FROM automl_output_summary) summary;
@@ -82,8 +82,8 @@
validation_table IS NULL AND
model = 'automl_output' AND
model_info = 'automl_output_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
model_arch_table = 'iris_model_arch' AND
model_selection_table = 'automl_mst_table' AND
automl_method = 'hyperopt' AND
@@ -97,9 +97,9 @@
start_training_time < now() AND
end_training_time < now() AND
madlib_version IS NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype = 'character varying' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] = 'character varying' AND
normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
) FROM (SELECT * FROM automl_output_summary) summary;
@@ -343,8 +343,8 @@
validation_table IS NULL AND
model = 'automl_output' AND
model_info = 'automl_output_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
model_arch_table = 'iris_model_arch' AND
model_selection_table = 'automl_mst_table' AND
automl_method = 'hyperband' AND
@@ -358,9 +358,9 @@
start_training_time < now() AND
end_training_time < now() AND
madlib_version IS NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype = 'character varying' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] = 'character varying' AND
normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
) FROM (SELECT * FROM automl_output_summary) summary;
@@ -377,8 +377,8 @@
validation_table IS NULL AND
model = 'automl_output' AND
model_info = 'automl_output_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
model_arch_table = 'iris_model_arch' AND
model_selection_table = 'automl_mst_table' AND
automl_method = 'hyperband' AND
@@ -392,9 +392,9 @@
start_training_time < now() AND
end_training_time < now() AND
madlib_version IS NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype = 'character varying' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] = 'character varying' AND
normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
) FROM (SELECT * FROM automl_output_summary) summary;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in
index 41dbf84..70deb5a 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in
@@ -38,7 +38,7 @@
DROP TABLE IF EXISTS cifar_10_sample_int_batched;
DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5);
+SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, ARRAY[5]);
-- This table is for testing a different input shape (3, 32, 32) instead of (32, 32, 3).
-- Create a table with image shape 3, 32, 32
@@ -50,7 +50,7 @@
DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched;
DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample_test_shape','cifar_10_sample_test_shape_batched','y','x', NULL, 255, 3);
+SELECT training_preprocessor_dl('cifar_10_sample_test_shape','cifar_10_sample_test_shape_batched','y','x', NULL, 255, array[3]);
DROP TABLE IF EXISTS model_arch;
SELECT load_keras_model('model_arch',
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
index b8507f2..72365de 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
@@ -46,41 +46,40 @@
'cifar_10_sample_val');
SELECT assert(
- model_arch_table = 'model_arch' AND
- model_id = 1 AND
- model_type = 'madlib_keras' AND
- start_training_time < now() AND
- end_training_time > start_training_time AND
- source_table = 'cifar_10_sample_batched' AND
- validation_table = 'cifar_10_sample_val' AND
- model = 'keras_saved_out' AND
- dependent_varname = 'y' AND
- dependent_vartype = 'smallint' AND
- independent_varname = 'x' AND
- normalizing_const = 255.0 AND
- pg_typeof(normalizing_const) = 'real'::regtype AND
- name is NULL AND
- description is NULL AND
- object_table is NULL AND
- model_size > 0 AND
- madlib_version is NOT NULL AND
- compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text AND
- fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
- num_iterations = 3 AND
- metrics_compute_frequency = 3 AND
- num_classes = 2 AND
- class_values = '{0,1}' AND
- metrics_type = '{mae}' AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- array_upper(training_metrics, 1) = 1 AND
- array_upper(training_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1 AND
- validation_metrics_final >= 0 AND
- validation_loss_final >= 0 AND
- array_upper(validation_metrics, 1) = 1 AND
- array_upper(validation_loss, 1) = 1 ,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
+ model_arch_table = 'model_arch' AND
+ model_id = 1 AND
+ model_type = 'madlib_keras' AND
+ start_training_time < now() AND
+ end_training_time > start_training_time AND
+ source_table = 'cifar_10_sample_batched' AND
+ validation_table = 'cifar_10_sample_val' AND
+ model = 'keras_saved_out' AND
+ dependent_varname[0] = 'y' AND
+ dependent_vartype[0] = 'smallint' AND
+ independent_varname[0] = 'x' AND
+ normalizing_const = 255.0 AND
+ pg_typeof(normalizing_const) = 'real'::regtype AND
+ name is NULL AND
+ description is NULL AND
+ object_table is NULL AND
+ model_size > 0 AND
+ madlib_version is NOT NULL AND
+ compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text AND
+ fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
+ num_iterations = 3 AND
+ metrics_compute_frequency = 3 AND
+ num_classes[0] = 2 AND
+ metrics_type = '{mae}' AND
+ training_metrics_final >= 0 AND
+ training_loss_final >= 0 AND
+ array_upper(training_metrics, 1) = 1 AND
+ array_upper(training_loss, 1) = 1 AND
+ array_upper(metrics_elapsed_time, 1) = 1 AND
+ validation_metrics_final >= 0 AND
+ validation_loss_final >= 0 AND
+ array_upper(validation_metrics, 1) = 1 AND
+ array_upper(validation_loss, 1) = 1 ,
+ 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_saved_out_summary) summary;
SELECT assert(
@@ -139,8 +138,9 @@
SELECT assert(
source_table = 'cifar_10_sample_batched' AND
model = 'keras_out' AND
- dependent_varname = 'y' AND
- independent_varname = 'x' AND
+ dependent_varname[0] = 'y' AND
+ dependent_vartype[0] = 'smallint' AND
+ independent_varname[0] = 'x' AND
model_arch_table = 'model_arch' AND
model_id = 1 AND
compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
@@ -155,10 +155,8 @@
start_training_time < now() AND
end_training_time > start_training_time AND
array_upper(metrics_elapsed_time, 1) = 2 AND
- dependent_vartype = 'smallint' AND
madlib_version is NOT NULL AND
- num_classes = 2 AND
- class_values = '{0,1}' AND
+ num_classes[0] = 2 AND
metrics_type = '{accuracy}' AND
normalizing_const = 255.0 AND
training_metrics_final is not NULL AND
@@ -295,7 +293,7 @@
-- induce failure by passing a non numeric column
DROP TABLE IF EXISTS cifar_10_sample_val_failure;
CREATE TABLE cifar_10_sample_val_failure AS SELECT * FROM cifar_10_sample_val;
-ALTER TABLE cifar_10_sample_val_failure rename dependent_var to dependent_var_original;
+ALTER TABLE cifar_10_sample_val_failure rename y to dependent_var_original;
ALTER TABLE cifar_10_sample_val_failure rename buffer_id to dependent_var;
DROP TABLE IF EXISTS keras_out, keras_out_summary;
SELECT assert(trap_error($TRAP$SELECT madlib_keras_fit(
@@ -316,40 +314,40 @@
DROP TABLE IF EXISTS cifar_10_sample_text_batched;
m4_changequote(`<!', `!>')
CREATE TABLE cifar_10_sample_text_batched AS
- SELECT buffer_id, independent_var, dependent_var,
- independent_var_shape, dependent_var_shape
+ SELECT buffer_id, x, y,
+ x_shape, y_shape
m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!, __dist_key__ !>)
FROM cifar_10_sample_batched m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! DISTRIBUTED BY (__dist_key__) !>);
-- Insert a new row with NULL as the dependent var (one-hot encoded)
UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
+ SET y = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
-INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape)
- SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, independent_var,
- convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var,
- independent_var_shape, dependent_var_shape
+ SET y = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
+INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, x, y, x_shape, y_shape)
+ SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, x,
+ convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS y,
+ x_shape, y_shape
FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0;
-UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5];
+UPDATE cifar_10_sample_text_batched SET y_shape = ARRAY[1,5];
-- Create the necessary summary table for the batched input.
DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary;
CREATE TABLE cifar_10_sample_text_batched_summary(
source_table text,
output_table text,
- dependent_varname text,
- independent_varname text,
- dependent_vartype text,
- class_values text[],
+ dependent_varname text[],
+ independent_varname text[],
+ dependent_vartype text[],
+ y_class_values text[],
buffer_size integer,
normalizing_const numeric);
INSERT INTO cifar_10_sample_text_batched_summary values (
'cifar_10_sample',
'cifar_10_sample_text_batched',
- 'y_text',
- 'x',
- 'text',
+ ARRAY['y'],
+ ARRAY['x'],
+ ARRAY['text'],
ARRAY[NULL,'cat','dog',NULL,NULL],
1,
255.0);
@@ -363,10 +361,10 @@
$$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
$$ batch_size=2, epochs=1, verbose=0 $$::text,
3);
--- Assert fit has correct class_values
+-- Assert fit has correct dependent_vartype
+SELECT * FROM keras_saved_out_summary;
SELECT assert(
- dependent_vartype = 'text' AND
- class_values = '{NULL,cat,dog,NULL,NULL}',
+ dependent_vartype[0] = 'text',
'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_saved_out_summary) summary;
@@ -384,7 +382,7 @@
DROP TABLE IF EXISTS cifar_10_sample_int_batched;
DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5);
+SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, ARRAY[5]);
DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
SELECT madlib_keras_fit(
@@ -396,10 +394,10 @@
$$ batch_size=2, epochs=1, verbose=0 $$::text,
3);
+SELECT * FROM keras_saved_out_summary;
-- Assert fit has correct class_values
SELECT assert(
- dependent_vartype = 'smallint' AND
- class_values = '{NULL,0,1,4,5}',
+ dependent_vartype[0] = 'smallint',
'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_saved_out_summary) summary;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit_multiple.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit_multiple.sql_in
index a7bd3fc..b6ce525 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit_multiple.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit_multiple.sql_in
@@ -52,7 +52,6 @@
model_selection_table,
1
)
-
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, MODIFIES SQL DATA);
@@ -90,8 +89,8 @@
-- Mock fit_transition function, for testing
-- madlib_keras_fit_multiple_model() python code
CREATE OR REPLACE FUNCTION madlib_installcheck_deep_learning.fit_transition_multiple_model(
- dependent_var BYTEA,
- independent_var BYTEA,
+ dependent_var BYTEA[],
+ independent_var BYTEA[],
dependent_var_shape INTEGER[],
independent_var_shape INTEGER[],
model_architecture TEXT,
@@ -127,8 +126,8 @@
for k in param_keys:
params[k] = g[k]
- params['dependent_var'] = len(dependent_var) if dependent_var else 0
- params['independent_var'] = len(independent_var) if independent_var else 0
+ params['dependent_var'] = len(dependent_var[0]) if dependent_var[0] else 0
+ params['independent_var'] = len(independent_var[0]) if independent_var[0] else 0
params['num_calls'] = num_calls
if not 'transition_function_params' in GD:
@@ -138,7 +137,7 @@
# compute simulated seg_id ( current_seg_id is the actual seg_id )
seg_id = dist_key_mapping.index( dist_key )
- if dependent_var_shape and dependent_var_shape[0] * num_calls < images_per_seg [ seg_id ]:
+ if dependent_var_shape[0] and dependent_var_shape[0][0] * num_calls < images_per_seg [ seg_id ]:
return None
else:
GD['transition_function_params'][dist_key]['reset'] = True
@@ -211,7 +210,7 @@
output_table TEXT,
cached_source_table TEXT
) RETURNS TEXT AS
-$$
+$$
fit_mult = GD['fit_mult']
fit_mult.model_input_tbl = input_table
@@ -227,11 +226,11 @@
= ( mst_key::TEXT::BYTEA,
( '{{ "a" : ' || mst_key::TEXT || ' }}' )::JSON,
'c' || mst_key::TEXT,
- 'f' || mst_key::TEXT
+ 'f' || mst_key::TEXT
)
WHERE mst_key IS NOT NULL;
""".format(model_out=fit_mult.model_output_tbl)
- plpy.execute(q)
+ plpy.execute(q)
$$ LANGUAGE plpythonu VOLATILE;
-- Updates dist keys in src table and internal fit_mult class variables
@@ -244,7 +243,7 @@
num_models INTEGER,
expected_distkey_mappings_tbl TEXT
) RETURNS VOID AS
-$$
+$$
redist_cmd = """
UPDATE {src_table}
SET __dist_key__ = (buffer_id % {num_data_segs})
@@ -254,7 +253,7 @@
fit_mult = GD['fit_mult']
q = """
- SELECT SUM(independent_var_shape[1]) AS image_count,
+ SELECT SUM(attributes_shape[1]) AS image_count,
__dist_key__
FROM {src_table}
GROUP BY __dist_key__
@@ -399,7 +398,7 @@
'mst_keys in schedule table created by test_init_schedule() does not match keys in mst_table'
) FROM current_schedule s FULL JOIN iris_mst_table m USING (mst_key);
--- Save order of mst keys in schedule for tracking
+-- Save order of mst keys in schedule for tracking
DROP TABLE IF EXISTS expected_order;
CREATE TABLE expected_order AS SELECT ARRAY(SELECT mst_key FROM current_schedule ORDER BY __dist_key__) mst_keys;
@@ -598,7 +597,7 @@
'NULL mst_key found in schedule table created by test_init_schedule, even though # msts = # segs'
) FROM current_schedule WHERE mst_key IS NULL;
--- Save new order of mst keys in schedule for tracking
+-- Save new order of mst keys in schedule for tracking
DROP TABLE IF EXISTS expected_order;
CREATE TABLE expected_order AS SELECT ARRAY(SELECT mst_key FROM current_schedule ORDER BY __dist_key__) mst_keys;
@@ -764,8 +763,8 @@
-- test files that run afterwards
DROP FUNCTION madlib_installcheck_deep_learning.version();
DROP FUNCTION madlib_installcheck_deep_learning.fit_transition_multiple_model(
- dependent_var BYTEA,
- independent_var BYTEA,
+ dependent_var BYTEA[],
+ independent_var BYTEA[],
dependent_var_shape INTEGER[],
independent_var_shape INTEGER[],
model_architecture TEXT,
@@ -783,4 +782,4 @@
custom_function_map BYTEA
)
->>> ) -- m4_endif postgres
+>>>) -- m4_endif postgres
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in
index 88011ef..67b1aa9 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in
@@ -301,3 +301,36 @@
'attributes', -- Independent variable
2 -- buffer_size (15 buffers)
);
+
+-- Create multi io dataset
+
+DROP TABLE IF EXISTS iris_mult;
+CREATE TABLE iris_mult AS
+ SELECT id, attributes, array_square(attributes) AS attributes2,
+ class_text AS class_text, class_text AS class_text2
+FROM iris_data;
+
+SELECT load_keras_model('iris_model_arch', -- Output table,
+$$
+{"class_name": "Model", "keras_version": "2.2.4-tf", "config": {"layers": [{"class_name": "InputLayer", "config": {"dtype": "float32", "batch_input_shape": [null, 4], "name": "input_1", "sparse": false}, "inbound_nodes": [], "name": "input_1"}, {"class_name": "InputLayer", "config": {"dtype": "float32", "batch_input_shape": [null, 4], "name": "input_2", "sparse": false}, "inbound_nodes": [], "name": "input_2"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "name": "dense"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "name": "dense_2"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["dense", 0, 0, {}]]], "name": "dense_1"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["dense_2", 0, 0, {}]]], "name": "dense_3"}, {"class_name": "Concatenate", "config": {"dtype": "float32", "trainable": true, "name": "concatenate", "axis": -1}, "inbound_nodes": [[["dense_1", 0, 0, {}], ["dense_3", 0, 0, {}]]], "name": "concatenate"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 3, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["concatenate", 0, 0, {}]]], "name": "dense_4"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"dtype": "float32", "distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_in"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "units": 3, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["dense_3", 0, 0, {}]]], "name": "dense_5"}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0]], "output_layers": [["dense_4", 0, 0], ["dense_5", 0, 0]], "name": "model"}, "backend": "tensorflow"}
+$$::json, NULL,
+ 'Sophie',
+ 'A simple model'
+);
+
+DROP TABLE IF EXISTS iris_mult_packed, iris_mult_packed_summary;
+SELECT training_preprocessor_dl('iris_mult',
+ 'iris_mult_packed',
+ 'class_text, class_text2',
+ 'attributes, attributes2',
+ NULL,
+ 255
+ );
+
+DROP TABLE IF EXISTS iris_mult_val, iris_mult_val_summary;
+SELECT validation_preprocessor_dl('iris_mult', -- Source table
+ 'iris_mult_val', -- Output table
+ 'class_text, class_text2', -- Dependent variable
+ 'attributes, attributes2', -- Independent variable
+ 'iris_mult_packed'-- Training preprocessed table
+ );
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in
index d6bcae7..67ffa0e 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in
@@ -49,15 +49,15 @@
validation_table is NULL AND
source_table = 'iris_data_packed' AND
model = 'pg_temp.iris_model' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < now() AND
end_training_time < now() AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] LIKE '%char%' AND
normalizing_const = 1,
'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM pg_temp.iris_model_summary) summary;
@@ -106,15 +106,14 @@
validation_table is NULL AND
source_table = 'iris_data_one_hot_encoded_packed' AND
model = 'iris_model' AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_one_hot_encoded' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < now() AND
end_training_time < now() AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
+ dependent_vartype[0] = 'integer[]' AND
+ num_classes[0] = NULL AND
normalizing_const = 1,
'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
FROM (SELECT * FROM iris_model_summary) summary;
@@ -194,9 +193,9 @@
model_type = 'madlib_keras' AND
source_table = 'iris_data_packed' AND
model = 'iris_model' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
- dependent_vartype LIKE '%char%' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
+ dependent_vartype[0] LIKE '%char%' AND
normalizing_const = 1 AND
pg_typeof(normalizing_const) = 'real'::regtype AND
name is NULL AND
@@ -207,8 +206,8 @@
fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
num_iterations = 3 AND
metrics_compute_frequency = 1 AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
metrics_type = '{top_3_accuracy}' AND
array_upper(training_metrics, 1) = 3 AND
training_loss = '{0,0,0}' AND
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
index 49b6940..81554d3 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
@@ -365,14 +365,14 @@
model = 'iris_multiple_model' AND
model_selection_table = 'mst_table_4row' AND
object_table IS NULL AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_one_hot_encoded' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < end_training_time AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
+ dependent_vartype[0] = 'integer[]' AND
+ num_classes[0] = NULL AND
+ class_one_hot_encoded_class_values = NULL AND
normalizing_const = 1 AND
metrics_iters = ARRAY[3],
'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
@@ -446,15 +446,15 @@
validation_table = 'iris_data_one_hot_encoded_packed' AND
model = 'iris_multiple_model' AND
model_info = 'iris_multiple_model_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
model_arch_table = 'iris_model_arch' AND
num_iterations = 6 AND
start_training_time < end_training_time AND
madlib_version is NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] LIKE '%char%' AND
normalizing_const = 1 AND
name IS NULL AND
description IS NULL AND
@@ -629,7 +629,7 @@
SELECT test_fit_multiple_more_configs(TRUE);
-- Test when class values have NULL values
-UPDATE iris_data_packed_summary SET class_values = ARRAY['Iris-setosa','Iris-versicolor',NULL];
+UPDATE iris_data_packed_summary SET class_text_class_values = ARRAY['Iris-setosa','Iris-versicolor',NULL];
DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
SELECT madlib_keras_fit_multiple_model(
'iris_data_packed',
@@ -643,8 +643,8 @@
);
SELECT assert(
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,NULL}',
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,NULL}',
'Keras Fit Multiple num_clases and class values Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM iris_multiple_model_summary) summary;
@@ -666,8 +666,8 @@
);
SELECT count(*) from __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model;
SELECT assert(
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,NULL}',
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,NULL}',
'Keras Fit Multiple validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary) summary;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in
index b9b775c..1ef692f 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in
@@ -68,15 +68,15 @@
model_info = '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_info' AND
source_table = 'iris_data_packed' AND
model = '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < now() AND
end_training_time < now() AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] LIKE '%char%' AND
normalizing_const = 1,
'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary) summary;
@@ -125,15 +125,14 @@
model_info = 'iris_multiple_model_info' AND
source_table = 'iris_data_one_hot_encoded_packed' AND
model = 'iris_multiple_model' AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_one_hot_encoded' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < now() AND
end_training_time < now() AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
+ dependent_vartype[0] = 'integer[]' AND
+ num_classes[0] = NULL AND
normalizing_const = 1,
'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
FROM (SELECT * FROM iris_multiple_model_summary) summary;
@@ -205,15 +204,15 @@
model_info = 'iris_multiple_model_custom_fn_info' AND
source_table = 'iris_data_packed' AND
model = 'iris_multiple_model_custom_fn' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
+ dependent_varname[0] = 'class_text' AND
+ independent_varname[0] = 'attributes' AND
madlib_version is NOT NULL AND
num_iterations = 3 AND
start_training_time < now() AND
end_training_time < now() AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
+ num_classes[0] = 3 AND
+ class_text_class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype[0] LIKE '%char%' AND
normalizing_const = 1,
'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM iris_multiple_model_custom_fn_summary) summary;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_multi_io.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_multi_io.sql_in
new file mode 100644
index 0000000..4afc47d
--- /dev/null
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_multi_io.sql_in
@@ -0,0 +1,121 @@
+/* ---------------------------------------------------------------------*//**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *//* ---------------------------------------------------------------------*/
+
+\i m4_regexp(MODULE_PATHNAME,
+ `\(.*\)libmadlib\.so',
+ `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
+)
+
+\i m4_regexp(MODULE_PATHNAME,
+ `\(.*\)libmadlib\.so',
+ `\1../../modules/deep_learning/test/madlib_keras_custom_function.setup.sql_in'
+)
+
+m4_include(`SQLCommon.m4')
+
+-- Test multi io
+DROP TABLE IF EXISTS iris_model, iris_model_summary;
+SELECT madlib_keras_fit('iris_mult_packed',
+ 'iris_model',
+ 'iris_model_arch',
+ 3,
+ $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$::text,
+ $$ batch_size=5, epochs=3 $$::text,
+ 2,
+ NULL,
+ 'iris_mult_val');
+
+-- Run Evaluate
+DROP TABLE IF EXISTS evaluate_out;
+SELECT madlib_keras_evaluate(
+ 'iris_model',
+ 'iris_mult_val',
+ 'evaluate_out',
+ FALSE);
+
+-- Run Predict
+DROP TABLE IF EXISTS iris_predict;
+SELECT madlib_keras_predict(
+ 'iris_model',
+ 'iris_mult',
+ 'id',
+ 'attributes, attributes2',
+ 'iris_predict',
+ 0.5,
+ FALSE);
+
+-- Warm Start
+
+DROP TABLE IF EXISTS iris_model_first_run;
+CREATE TABLE iris_model_first_run AS
+SELECT training_loss_final, training_metrics_final
+FROM iris_model_summary;
+
+SELECT madlib_keras_fit('iris_mult_packed',
+ 'iris_model',
+ 'iris_model_arch',
+ 3,
+ $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$::text,
+ $$ batch_size=5, epochs=3 $$::text,
+ 2,
+ NULL,
+ 'iris_mult_val',
+ 1,
+ TRUE);
+
+-- Transfer Learning
+
+DROP TABLE IF EXISTS iris_model_arch_multi;
+CREATE TABLE iris_model_arch_multi AS
+SELECT * FROM iris_model_arch WHERE model_id = 3;
+
+UPDATE iris_model_arch_multi set model_weights = (select model_weights from iris_model);
+
+DROP TABLE IF EXISTS iris_model_transfer, iris_model_transfer_summary;
+SELECT madlib_keras_fit('iris_mult_packed',
+ 'iris_model_transfer',
+ 'iris_model_arch_multi',
+ 3,
+ $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$::text,
+ $$ batch_size=5, epochs=3 $$::text,
+ 2,
+ NULL,
+ 'iris_mult_val');
+
+-- Custom Function
+
+DROP TABLE IF EXISTS test_custom_function_table;
+SELECT load_custom_function('test_custom_function_table', custom_function_zero_object(), 'test_custom_fn', 'returns test_custom_fn');
+SELECT load_custom_function('test_custom_function_table', custom_function_one_object(), 'test_custom_fn1', 'returns test_custom_fn1');
+
+DROP TABLE if exists iris_model, iris_model_summary, iris_model_info;
+SELECT madlib_keras_fit(
+ 'iris_mult_packed',
+ 'iris_model',
+ 'iris_model_arch',
+ 3,
+ $$ loss='test_custom_fn', optimizer='adam', metrics=['test_custom_fn1'] $$::text,
+ $$ batch_size=5, epochs=3 $$::text,
+ 2,
+ FALSE, NULL, 1, NULL, NULL, NULL,
+ 'test_custom_function_table'
+);
+
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
index 3aa024f..82db074 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
@@ -52,16 +52,17 @@
SELECT assert(UPPER(pg_typeof(id)::TEXT) = 'INTEGER', 'id column should be INTEGER type')
FROM cifar10_predict;
-SELECT assert(UPPER(pg_typeof(y)::TEXT) =
- 'SMALLINT', 'prediction column should be SMALLINT type')
+SELECT assert(UPPER(pg_typeof(class_value)::TEXT) =
+ 'TEXT', 'prediction column should be TEXT type')
FROM cifar10_predict;
-- Validate correct number of rows returned.
-SELECT assert(COUNT(*)=4, 'Output table of madlib_keras_predict should have two rows')
+SELECT assert(COUNT(*)=4, 'Output table of madlib_keras_predict should have four rows')
FROM cifar10_predict;
+SELECT * FROM cifar10_predict;
-- First test that all values are in set of class values; if this breaks, it's definitely a problem.
-SELECT assert(y IN (0,1),
+SELECT assert(class_value IN ('0','1'),
'Predicted value not in set of defined class values for model')
FROM cifar10_predict;
@@ -91,7 +92,7 @@
'DOUBLE PRECISION', 'column prob should be double precision type')
FROM cifar10_predict;
-SELECT assert(COUNT(*)=4, 'Predict out table must have exactly three cols.')
+SELECT assert(COUNT(*)=5, 'Predict out table must have exactly five cols.')
FROM pg_attribute
WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
@@ -110,7 +111,7 @@
'DOUBLE PRECISION', 'column prob should be double precision type')
FROM cifar10_predict;
-SELECT assert(COUNT(*)=4, 'Predict out table must have exactly three cols.')
+SELECT assert(COUNT(*)=5, 'Predict out table must have exactly five cols.')
FROM pg_attribute
WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
@@ -119,22 +120,22 @@
DROP TABLE IF EXISTS cifar_10_sample_text_batched;
m4_changequote(`<!', `!>')
CREATE TABLE cifar_10_sample_text_batched AS
- SELECT buffer_id, independent_var, dependent_var,
- independent_var_shape, dependent_var_shape
+ SELECT buffer_id, x, y,
+ x_shape, y_shape
m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!, __dist_key__ !>)
FROM cifar_10_sample_batched m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!DISTRIBUTED BY (__dist_key__)!>);
-- Insert a new row with NULL as the dependent var (one-hot encoded)
UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
+ SET y = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
-INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape)
- SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, independent_var,
- convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var,
- independent_var_shape, dependent_var_shape
+ SET y = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
+INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, x, y, x_shape, y_shape)
+ SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, x,
+ convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS y,
+ x_shape, y_shape
FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0;
-UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5];
+UPDATE cifar_10_sample_text_batched SET y_shape = ARRAY[1,5];
m4_changequote(<!`!>,<!'!>)
-- Create the necessary summary table for the batched input.
@@ -142,18 +143,18 @@
CREATE TABLE cifar_10_sample_text_batched_summary(
source_table text,
output_table text,
- dependent_varname text,
- independent_varname text,
- dependent_vartype text,
- class_values text[],
+ dependent_varname text[],
+ independent_varname text[],
+ dependent_vartype text[],
+ y_class_values text[],
buffer_size integer,
normalizing_const numeric);
INSERT INTO cifar_10_sample_text_batched_summary values (
'cifar_10_sample',
'cifar_10_sample_text_batched',
- 'y_text',
- 'x',
- 'text',
+ ARRAY['y'],
+ ARRAY['x'],
+ ARRAY['text'],
ARRAY[NULL,'cat','dog','bird','fish'],
1,
255.0);
@@ -171,7 +172,7 @@
-- Predict with pred_type=prob
DROP TABLE IF EXISTS cifar_10_sample_text;
CREATE TABLE cifar_10_sample_text AS
- SELECT id, x, y_text
+ SELECT id, x, y
FROM cifar_10_sample;
DROP TABLE IF EXISTS cifar10_predict;
SELECT madlib_keras_predict(
@@ -190,7 +191,7 @@
'DOUBLE PRECISION', 'column prob should be double precision type')
FROM cifar10_predict;
-SELECT assert(COUNT(*)=4, 'Predict out table must have exactly four cols.')
+SELECT assert(COUNT(*)=5, 'Predict out table must have exactly five cols.')
FROM pg_attribute
WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
@@ -208,13 +209,13 @@
-- Validate the output datatype of newly created prediction columns
-- for prediction type = 'response' and class_values 'TEXT' with NULL
-- as a valid class_values
-SELECT assert(UPPER(pg_typeof(y_text)::TEXT) =
- 'TEXT', 'prediction column should be TEXT type')
+SELECT assert(UPPER(pg_typeof(class_value)::TEXT) =
+ 'TEXT', 'class_value column should be TEXT type')
FROM cifar10_predict LIMIT 1;
-- Tests where the assumption is user has one-hot encoded, so class_values
-- in input summary table will be NULL.
-UPDATE keras_saved_out_summary SET class_values=NULL;
+UPDATE keras_saved_out_summary SET y_class_values=ARRAY[NULL]::smallint[];
-- Predict with pred_type=all
DROP TABLE IF EXISTS cifar10_predict;
@@ -245,8 +246,8 @@
-- for prediction type = 'response' and class_value = NULL
-- Returns: Index of class value in user's one-hot encoded data with
-- highest probability
-SELECT assert(UPPER(pg_typeof(y_text)::TEXT) =
- 'TEXT', 'column y_text should be text type')
+SELECT assert(UPPER(pg_typeof(class_value)::TEXT) =
+ 'TEXT', 'column class_value should be text type')
FROM cifar10_predict LIMIT 1;
-- Test predict with INTEGER class_values
@@ -254,9 +255,9 @@
-- Update output_summary table to reflect
-- class_values {NULL,0,1,4,5} and dependent_vartype is SMALLINT
UPDATE keras_saved_out_summary
-SET dependent_varname = 'y',
- class_values = ARRAY[NULL,0,1,4,5]::INTEGER[],
- dependent_vartype = 'smallint';
+SET dependent_varname = ARRAY['y'],
+ y_class_values = ARRAY[NULL,0,1,4,5]::INTEGER[],
+ dependent_vartype = ARRAY['smallint'];
-- Predict with pred_type=prob
DROP TABLE IF EXISTS cifar10_predict;
SELECT madlib_keras_predict(
@@ -365,7 +366,7 @@
'Predict output validation failed.')
FROM iris_multiple_model_info i,
(SELECT count(*)/(150*0.8) AS test_accuracy FROM
- (SELECT iris_train.class_text AS actual, iris_predict.class_text AS estimated
+ (SELECT iris_train.class_text AS actual, iris_predict.class_value AS estimated
FROM iris_predict INNER JOIN iris_train
ON iris_train.id=iris_predict.id)q
WHERE q.actual=q.estimated) q2
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in
index bd17bec..5fcee51 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in
@@ -57,12 +57,12 @@
'iris_predict_byom',
'response',
NULL,
- ARRAY['Iris-setosa', 'Iris-versicolor',
- 'Iris-virginica']
+ ARRAY[ARRAY['Iris-setosa', 'Iris-versicolor',
+ 'Iris-virginica']::text[]]
);
SELECT assert(
- p0.class_text = p1.dependent_var,
+ p0.class_value = p1.class_value,
'Predict byom failure for non null class value and response pred_type.')
FROM iris_predict AS p0, iris_predict_byom AS p1
WHERE p0.id=p1.id;
@@ -78,7 +78,7 @@
'iris_predict_byom'
);
SELECT assert(
- p1.dependent_var IN ('0', '1', '2'),
+ p1.class_value IN ('0', '1', '2'),
'Predict byom failure for null class value and null pred_type.')
FROM iris_predict_byom AS p1;
@@ -93,8 +93,8 @@
'iris_predict_byom',
'prob',
NULL,
- ARRAY['Iris-setosa', 'Iris-versicolor',
- 'Iris-virginica'],
+ ARRAY[ARRAY['Iris-setosa', 'Iris-versicolor',
+ 'Iris-virginica']::text[]],
1.0
);
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in
index bffd5a9..1689d52 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in
@@ -152,7 +152,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1
);
@@ -167,7 +167,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE,
NULL, 1,
TRUE -- warm_start
@@ -180,8 +180,8 @@
SELECT assert(
- array_upper(training_loss, 1) = 3 AND
- array_upper(training_metrics, 1) = 3,
+ array_upper(training_loss, 1) = 4 AND
+ array_upper(training_metrics, 1) = 4,
'metrics compute frequency must be 1.')
FROM iris_multiple_model_info;
@@ -217,7 +217,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1
);
@@ -233,7 +233,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1,
TRUE);
@@ -252,7 +252,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE,
NULL, 1,
TRUE -- warm_start
@@ -308,7 +308,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1
);
@@ -323,7 +323,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE,
NULL, 1,
TRUE -- warm_start
@@ -336,8 +336,8 @@
SELECT assert(
- array_upper(training_loss, 1) = 3 AND
- array_upper(training_metrics, 1) = 3,
+ array_upper(training_loss, 1) = 4 AND
+ array_upper(training_metrics, 1) = 4,
'metrics compute frequency must be 1.')
FROM iris_multiple_model_info;
@@ -385,7 +385,7 @@
SELECT load_model_selection_table(
'iris_model_arch',
'mst_table',
- ARRAY[1,3],
+ ARRAY[1,4],
ARRAY[
$$loss='categorical_crossentropy',optimizer='Adam(lr=0.00001)',metrics=['accuracy']$$,
$$loss='categorical_crossentropy', optimizer='Adam(lr=0.00002)',metrics=['accuracy']$$
@@ -402,7 +402,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1
);
@@ -421,7 +421,7 @@
'iris_data_packed',
'iris_multiple_model',
'mst_table',
- 3,
+ 4,
FALSE, NULL, 1
);
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in
index d2e14cd..51102bb 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in
@@ -53,7 +53,7 @@
self.default_ind_var = "indvar"
self.default_buffer_size = 5
self.default_normalizing_const = 1.0
- self.default_num_classes = None
+ self.default_num_classes = [2]
self.default_distribution_rules = "all_segments"
self.default_module_name = "dummy"
@@ -86,6 +86,45 @@
self.default_num_classes,
self.default_distribution_rules,
self.default_module_name)
+ preprocessor_obj.dependent_levels = [["NULL", "'a'"]]
+ preprocessor_obj.input_preprocessor_dl()
+
+ def test_input_preprocessor_multi_dep(self):
+ self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]', 'integer[]'])
+ self.control_module.OptimizerControl.__enter__ = Mock()
+ self.control_module.OptimizerControl.optimizer_control = True
+ self.control_module.OptimizerControl.optimizer_enabled = True
+ preprocessor_obj = self.module.InputDataPreprocessorDL(
+ self.default_schema_madlib,
+ "input",
+ "out",
+ "a,b",
+ self.default_ind_var,
+ self.default_buffer_size,
+ self.default_normalizing_const,
+ [2,2],
+ self.default_distribution_rules,
+ self.default_module_name)
+ preprocessor_obj.dependent_levels = [["NULL", "'a'"],["NULL", "'a'"]]
+ preprocessor_obj.input_preprocessor_dl()
+
+ def test_input_preprocessor_multi_ind(self):
+ self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]', 'integer[]'])
+ self.control_module.OptimizerControl.__enter__ = Mock()
+ self.control_module.OptimizerControl.optimizer_control = True
+ self.control_module.OptimizerControl.optimizer_enabled = True
+ preprocessor_obj = self.module.InputDataPreprocessorDL(
+ self.default_schema_madlib,
+ "input",
+ "out",
+ self.default_dep_var,
+ "c,d",
+ self.default_buffer_size,
+ self.default_normalizing_const,
+ self.default_num_classes,
+ self.default_distribution_rules,
+ self.default_module_name)
+ preprocessor_obj.dependent_levels = [["NULL", "'a'"]]
preprocessor_obj.input_preprocessor_dl()
def test_input_preprocessor_null_buffer_size_executes_query(self):
@@ -104,39 +143,10 @@
self.default_num_classes,
self.default_distribution_rules,
self.default_module_name)
+ preprocessor_obj.dependent_levels = [["NULL", "'a'"]]
self.util_module.MiniBatchBufferSizeCalculator.calculate_default_buffer_size = Mock(return_value = 5)
preprocessor_obj.input_preprocessor_dl()
- def test_input_preprocessor_multiple_dep_var_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- "y1,y2",
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_multiple_indep_var_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- "x1,x2",
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
def test_input_preprocessor_buffer_size_zero_fails(self):
self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
with self.assertRaises(plpy.PLPYException):
@@ -166,34 +176,6 @@
self.default_distribution_rules,
self.default_module_name)
- def test_input_preprocessor_invalid_indep_vartype_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_invalid_dep_vartype_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
def test_input_preprocessor_normalizing_const_zero_fails(self):
self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
with self.assertRaises(plpy.PLPYException):
@@ -236,10 +218,10 @@
self.default_num_classes,
self.default_distribution_rules,
self.default_module_name)
- obj.dependent_levels = ["NULL", "'a'"]
+ obj.dependent_levels = [["NULL", "'a'"]]
dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr()
self.assertEqual("array[({0}) is not distinct from null, " \
- "({0}) is not distinct from 'a']::integer[]::smallint[]".
+ "({0}) is not distinct from 'a']::integer[]::smallint[] as depvar".
format(self.default_dep_var),
dep_var_array_expr.lower())
@@ -256,6 +238,7 @@
self.default_num_classes,
self.default_distribution_rules,
self.default_module_name)
+ obj.dependent_levels = [["NULL", "'a'"]]
dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr()
self.assertEqual("{0}::smallint[]".
format(self.default_dep_var),
@@ -279,7 +262,6 @@
def test_validate_num_classes_greater(self):
self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text'])
- self.module._get_dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"])
obj = self.module.TrainingDataPreprocessorDL(
self.default_schema_madlib,
self.default_source_table,
@@ -288,14 +270,14 @@
self.default_ind_var,
self.default_buffer_size,
self.default_normalizing_const,
- 5,
+ [5],
self.default_distribution_rules)
+ obj.dependent_levels = [["NULL", "'a'", "'b'"]]
obj._set_one_hot_encoding_variables()
- self.assertEqual(2, obj.padding_size)
+ self.assertEqual([2], obj.padding_size)
def test_validate_num_classes_lesser(self):
self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text'])
- self.module.dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"])
obj = self.module.TrainingDataPreprocessorDL(
self.default_schema_madlib,
self.default_source_table,
@@ -304,8 +286,9 @@
self.default_ind_var,
self.default_buffer_size,
self.default_normalizing_const,
- 2,
+ [2],
self.default_distribution_rules)
+ obj.dependent_levels = [["NULL", "'a'", "'b'"]]
with self.assertRaises(plpy.PLPYException):
obj._set_one_hot_encoding_variables()
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
index af3bdc0..64395eb 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
@@ -105,8 +105,8 @@
previous_state = np.array(self.model_weights, dtype=np.float32)
new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ None, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, previous_state.tostring(), **kwargs)
@@ -120,8 +120,8 @@
ending_image_count = len(self.dependent_var_int)
new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ None, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights,
@@ -136,8 +136,8 @@
k = {'GD': {}}
new_state = self.subject.fit_multiple_transition_caching(
- self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights, True, **k)
@@ -157,8 +157,8 @@
state = starting_image_count
new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ state, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), None, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.dummy_prev_weights, **kwargs)
@@ -168,14 +168,14 @@
def _test_fit_transition_multiple_model_no_cache_middle_buffer_pass(self,
**kwargs):
+
starting_image_count = len(self.dependent_var_int)
ending_image_count = starting_image_count + len(self.dependent_var_int)
-
kwargs['GD']['agg_image_count'] = starting_image_count
new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ None, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), None, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True,
@@ -200,10 +200,9 @@
}
new_state = self.subject.fit_multiple_transition_caching(
- self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), self.compile_params, self.fit_params, 0,
-
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights, True, **k)
@@ -224,8 +223,8 @@
state = starting_image_count
previous_state = np.array(self.model_weights, dtype=np.float32)
new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ state, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), None, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, previous_state.tostring(),
@@ -246,8 +245,8 @@
state = [0,0,0]
new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ state, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(),
self.serialized_weights, self.compile_params, 0,
self.dist_key_mapping, 0, 4,
@@ -272,8 +271,8 @@
starting_image_count]
new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ state, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(),
'dummy_model_weights', None, 0,
self.dist_key_mapping, 0, 4,
@@ -296,8 +295,8 @@
self.accuracy * starting_image_count, starting_image_count]
new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ state, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(),
'dummy_model_weights', None, 0,
self.dist_key_mapping, 0, 4,
@@ -315,8 +314,8 @@
starting_image_count = 2*len(self.dependent_var_int)
new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ None, [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), None, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.dummy_prev_weights,
@@ -347,8 +346,8 @@
k['GD']['agg_image_count'] = starting_image_count
new_state = self.subject.fit_multiple_transition_caching(
- self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
+ [self.dependent_var] , [self.independent_var],
+ [self.dependent_var_shape], [self.independent_var_shape],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights, False, **k)
@@ -387,8 +386,8 @@
'sess': s1, 'segment_model': self.model}}
graph1 = self.subject.tf.get_default_graph()
new_state = self.subject.fit_multiple_transition_caching(
- None, None,
- None, None,
+ [None], [None],
+ [None], [None],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights, False, **k)
@@ -421,8 +420,8 @@
k = {'GD': {'x_train': x_train, 'y_train': y_train }}
graph1 = self.subject.tf.get_default_graph()
new_state = self.subject.fit_multiple_transition_caching(
- None, None,
- None, None,
+ [None], [None],
+ [None], [None],
self.model.to_json(), self.compile_params, self.fit_params, 0,
self.dist_key_mapping, 0, 4, self.total_images_per_seg,
self.accessible_gpus_for_seg, self.serialized_weights, True, **k)
@@ -758,7 +757,7 @@
self.all_seg_ids = [0,1,2]
- self.independent_var = [[[240]]]
+ self.independent_var = [[[[240]]]]
self.total_images_per_seg = [3,3,4]
def tearDown(self):
@@ -846,8 +845,9 @@
self.pred_type = 'prob'
self.use_gpus = False
- self.class_values = ['foo', 'bar', 'baaz', 'foo2', 'bar2']
+ self.class_values = [['foo', 'bar', 'baaz', 'foo2', 'bar2']]
self.normalizing_const = 255.0
+ self.dependent_count = 1
import madlib_keras_predict
self.module = madlib_keras_predict
@@ -865,21 +865,23 @@
res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table', None,
- True, None, None)
+ True, None, None, 1)
self.assertEqual('prob', res.pred_type)
self.assertEqual(2, res.gpus_per_host)
- self.assertEqual([0,1,2,3,4], res.class_values)
+ self.assertEqual([[0,1,2,3,4]], res.class_values)
self.assertEqual(1.0, res.normalizing_const)
def test_predictbyom_defaults_2(self):
+ self.module.InputValidator.validate_class_values = Mock()
res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
self.pred_type, self.use_gpus,
- self.class_values, self.normalizing_const)
+ self.class_values, self.normalizing_const,
+ self.dependent_count)
self.assertEqual('prob', res.pred_type)
self.assertEqual(0, res.gpus_per_host)
- self.assertEqual(['foo', 'bar', 'baaz', 'foo2', 'bar2'], res.class_values)
+ self.assertEqual([['foo', 'bar', 'baaz', 'foo2', 'bar2']], res.class_values)
self.assertEqual(255.0, res.normalizing_const)
def test_predictbyom_exception_invalid_params(self):
@@ -888,23 +890,27 @@
'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
'invalid_pred_type', self.use_gpus,
- self.class_values, self.normalizing_const)
+ self.class_values, self.normalizing_const,
+ self.dependent_count)
self.assertIn('invalid_pred_type', str(error.exception))
- with self.assertRaises(plpy.PLPYException) as error:
- self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table',
- self.pred_type, self.use_gpus,
- ["foo", "bar", "baaz"], self.normalizing_const)
- self.assertIn('class values', str(error.exception).lower())
+ # The validation for this test has been disabled
+ # with self.assertRaises(plpy.PLPYException) as error:
+ # self.module.PredictBYOM('schema_madlib', 'model_arch_table',
+ # 'model_id', 'test_table', 'id_col',
+ # 'independent_varname', 'output_table',
+ # self.pred_type, self.use_gpus,
+ # ["foo", "bar", "baaz"], self.normalizing_const,
+ # self.dependent_count)
+ # self.assertIn('class values', str(error.exception).lower())
with self.assertRaises(plpy.PLPYException) as error:
self.module.PredictBYOM('schema_madlib', 'model_arch_table',
'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
self.pred_type, self.use_gpus,
- self.class_values, 0)
+ self.class_values, 0,
+ self.dependent_count)
self.assertIn('normalizing const', str(error.exception).lower())
@@ -1253,6 +1259,8 @@
self.module_patcher.start()
import madlib_keras_validator
self.subject = madlib_keras_validator
+ self.dep_shape_cols = [[10,1,1,1]]
+ self.ind_shape_cols = [[10,2]]
def tearDown(self):
self.module_patcher.stop()
@@ -1262,32 +1270,36 @@
self.subject.FitCommonValidator._validate_common_args = Mock()
obj = self.subject.FitCommonValidator(
'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, None, False, False, [0],
- 'module_name', None)
+ 'dep_varname', 'independent_varname', self.dep_shape_cols,
+ self.ind_shape_cols, 5, None, False, False, [0],
+ 'module_name', None, None, None)
self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
def test_is_valid_metrics_compute_frequency_True_num(self):
self.subject.FitCommonValidator._validate_common_args = Mock()
obj = self.subject.FitCommonValidator(
'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 3, False, False, [0],
- 'module_name', None)
+ 'dep_varname', 'independent_varname', self.dep_shape_cols,
+ self.ind_shape_cols, 5, 3, False, False, [0],
+ 'module_name', None, None, None)
self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
def test_is_valid_metrics_compute_frequency_False_zero(self):
self.subject.FitCommonValidator._validate_common_args = Mock()
obj = self.subject.FitCommonValidator(
'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 0, False, False, [0],
- 'module_name', None)
+ 'dep_varname', 'independent_varname', self.dep_shape_cols,
+ self.ind_shape_cols, 5, 0, False, False, [0],
+ 'module_name', None, None, None)
self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
def test_is_valid_metrics_compute_frequency_False_greater(self):
self.subject.FitCommonValidator._validate_common_args = Mock()
obj = self.subject.FitCommonValidator(
'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 6, False, False, [0],
- 'module_name', None)
+ 'dep_varname', 'independent_varname', self.dep_shape_cols,
+ self.ind_shape_cols, 5, 6, False, False, [0],
+ 'module_name', None, None, None)
self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
@@ -1320,7 +1332,7 @@
self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
input_shape=(1,1,1,), padding='same'))
self.model.add(Dense(self.num_classes))
- self.classes = ['train', 'boat', 'car', 'airplane']
+ self.classes = [['train', 'boat', 'car', 'airplane']]
def tearDown(self):
self.module_patcher.stop()
@@ -1335,18 +1347,18 @@
self.model.add(Dense(1599))
with self.assertRaises(plpy.PLPYException) as error:
self.subject.validate_class_values(
- self.module_name, range(1599), 'prob', self.model.to_json())
+ self.module_name, [range(1599), range(1598)], 'prob', self.model.to_json())
self.assertIn('1600', str(error.exception))
def test_validate_class_values_valid_class_values_prob(self):
self.subject.validate_class_values(
- self.module_name, range(self.num_classes), 'prob', self.model.to_json())
+ self.module_name, [range(self.num_classes)], 'prob', self.model.to_json())
self.subject.validate_class_values(
self.module_name, None, 'prob', self.model.to_json())
def test_validate_class_values_valid_pred_type_valid_class_values_response(self):
self.subject.validate_class_values(
- self.module_name, range(self.num_classes), 'response', self.model.to_json())
+ self.module_name, [range(self.num_classes)], 'response', self.model.to_json())
self.subject.validate_class_values(
self.module_name, None, 'response', self.model.to_json())
@@ -1355,26 +1367,26 @@
self.plpy_mock_execute.return_value = [{'shape': [1,3,32,32]}]
with self.assertRaises(plpy.PLPYException):
self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 2, True)
+ self.test_table, [self.ind_var], [[32,32,3]], 2, True)
# non-minibatched data
- self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 32,'n_2': 32,'n_3': 3}]
+ self.plpy_mock_execute.return_value = [{'shape': [1,3,32,32]}]
with self.assertRaises(plpy.PLPYException):
self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 1)
- self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 3}]
+ self.test_table, [self.ind_var], [[32,32,3]], 1)
+ self.plpy_mock_execute.return_value = [{'shape': [1,3]}]
with self.assertRaises(plpy.PLPYException):
self.subject.validate_input_shape(
- self.test_table, self.ind_var, [3,32], 1)
+ self.test_table, [self.ind_var], [[3,32]], 1)
def test_validate_input_shape_shapes_match(self):
# minibatched data
- self.plpy_mock_execute.return_value = [{'shape': [1,32,32,3]}]
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 2, True)
+ # self.plpy_mock_execute.return_value = [{'shape': [1,32,32,3]}]
+ # self.subject.validate_input_shape(
+ # self.test_table, [self.ind_var], [[32,32,3]], 2, True)
# non-minibatched data
- self.plpy_mock_execute.return_value = [{'n_0': 32,'n_1': 32,'n_2': 3}]
+ self.plpy_mock_execute.return_value = [{'shape': [32,32,3]}]
self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 1)
+ self.test_table, [self.ind_var], [[32,32,3]], 1)
def test_validate_model_arch_table_none_values(self):
with self.assertRaises(plpy.PLPYException) as error:
@@ -1713,7 +1725,7 @@
input_state = [image_count*self.loss, image_count*self.accuracy, image_count]
output_state = self.subject.internal_keras_eval_final(input_state)
- self.assertEqual(len(output_state), 2)
+ self.assertEqual(len(output_state), 3)
agg_loss = output_state[0]
agg_accuracy = output_state[1]
diff --git a/src/ports/postgres/modules/utilities/utilities.py_in b/src/ports/postgres/modules/utilities/utilities.py_in
index 3cb219a..e5a4c3d 100644
--- a/src/ports/postgres/modules/utilities/utilities.py_in
+++ b/src/ports/postgres/modules/utilities/utilities.py_in
@@ -1086,20 +1086,27 @@
if other_output_tables:
for tbl in other_output_tables:
output_tbl_valid(tbl, module_name)
+ if type(independent_varname) is not list:
+ independent_varname = [independent_varname]
+ if type(dependent_varname) is not list:
+ dependent_varname = [dependent_varname]
+ for i in independent_varname:
+ _assert(is_var_valid(source_table, i),
+ "{module_name} error: invalid independent_varname "
+ "('{independent_varname}') for source_table "
+ "({source_table})!".format(module_name=module_name,
+ independent_varname=i,
+ source_table=source_table))
- _assert(is_var_valid(source_table, independent_varname),
- "{module_name} error: invalid independent_varname "
- "('{independent_varname}') for source_table "
- "({source_table})!".format(module_name=module_name,
- independent_varname=independent_varname,
- source_table=source_table))
- _assert(is_var_valid(source_table, dependent_varname),
- "{module_name} error: invalid dependent_varname "
- "('{dependent_varname}') for source_table "
- "({source_table})!".format(module_name=module_name,
- dependent_varname=dependent_varname,
- source_table=source_table))
+
+ for i in dependent_varname:
+ _assert(is_var_valid(source_table, i),
+ "{module_name} error: invalid dependent_varname "
+ "('{dependent_varname}') for source_table "
+ "({source_table})!".format(module_name=module_name,
+ dependent_varname=i,
+ source_table=source_table))
if grouping_cols:
_assert(is_var_valid(source_table, grouping_cols),
"{module_name} error: invalid grouping_cols "
