commit c87ddbaee8b2fb3af5c1d86bc0b51134a0e503a4
author Orhan Kislal <okislal@apache.org> Mon Feb 28 21:30:17 2022 -0500
committer Orhan Kislal <okislal@apache.org> Thu Mar 03 17:00:23 2022 -0500
tree 40b06f52924a3c83f5605730e9fd599a5f21baef
parent 44ca0ef433c0153880833ec789753210488d93b6

GLM-multinom: Use non-temp tables in GroupIterationController

There is a potential issue with pg_temp not cleaning up correctly in
the case of a failure in GLM and multinom. This commit changes the
default value to create the temp table used for state aggregation to
avoid the temporary tables.

Co-authored-by: Bhuvnesh Chaudhary <bchaudhary@pivotal.io>

src/ports/postgres/modules/glm/glm.py_in

diff --git a/src/ports/postgres/modules/glm/glm.py_in b/src/ports/postgres/modules/glm/glm.py_in
index bdf7318..742cb57 100644
--- a/src/ports/postgres/modules/glm/glm.py_in
+++ b/src/ports/postgres/modules/glm/glm.py_in
@@ -195,7 +195,8 @@
             'rel_state': unique_string(),
             'col_grp_iteration': unique_string(),
             'col_grp_state': unique_string(),
-            'state_type': schema_madlib + ".bytea8"
+            'state_type': schema_madlib + ".bytea8",
+            'temporaryTables': False
             }
     args.update(optim_params)
     args.update(family_params)
@@ -350,7 +351,7 @@
                    **args))
 
     # clean up
-    plpy.execute("""DROP TABLE IF EXISTS pg_temp.{rel_state} """.format(**args))
+    plpy.execute("""DROP TABLE IF EXISTS {rel_state} """.format(**args))
     plpy.execute("SET client_min_messages TO " + old_msg_level)
     return None
 

src/ports/postgres/modules/glm/multinom.py_in

diff --git a/src/ports/postgres/modules/glm/multinom.py_in b/src/ports/postgres/modules/glm/multinom.py_in
index 1293f57..7adfcb0 100644
--- a/src/ports/postgres/modules/glm/multinom.py_in
+++ b/src/ports/postgres/modules/glm/multinom.py_in
@@ -186,7 +186,8 @@
             'ref_category': ref_category,
             'n_categories': len(category_list),
             'link': link_func,
-            'state_type': schema_madlib + ".bytea8"
+            'state_type': schema_madlib + ".bytea8",
+            'temporaryTables': False
             }
     args.update(optim_params_dict)
 
@@ -341,7 +342,7 @@
 
     # clean up
     plpy.execute("""
-        DROP TABLE IF EXISTS pg_temp.{rel_state}
+        DROP TABLE IF EXISTS {rel_state}
         """.format(**args))
     plpy.execute("SET client_min_messages TO " + old_msg_level)
 

src/ports/postgres/modules/utilities/in_mem_group_control.py_in

diff --git a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
index 86f0c17..68834c0 100644
--- a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
+++ b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
@@ -222,6 +222,7 @@
                           if arg_dict["grouping_col"] is None
                           else arg_dict["grouping_col"]),
         )
+        self.kwargs['temporaryTables'] = self.kwargs.get('temporaryTables', True)
         self.grp_to_n_tuples = {}
         self.failed_grp_keys = []
 
@@ -287,9 +288,10 @@
                          else "{grouping_col}").format(**self.kwargs)
             groupby_str = ("{col_grp_null}" if self.is_group_null
                            else "{grouping_col}").format(**self.kwargs)
+            temp = 'TEMPORARY' if self.kwargs['temporaryTables'] else ''
             plpy.execute("""
                 DROP TABLE IF EXISTS {rel_state};
-                CREATE TEMPORARY TABLE {rel_state} AS (
+                CREATE {temp} TABLE {rel_state} AS (
                     SELECT
                         array_to_string(ARRAY[{grouping_str}], ',') AS {col_grp_key},
                         0::integer                                  AS {col_grp_iteration},
@@ -303,6 +305,7 @@
                 );
                 """.format(group_col=group_col,
                            groupby_str=groupby_str,
+                           temp=temp,
                            **self.kwargs))
 
             ############################