commit b28db24b15ada0c5ac6f8273dd90af09d9a1f0e7
author Deon Griessel <dgriessel@searchtechnologies.com> Fri Aug 04 19:27:45 2017 +0000
committer Deon Griessel <dgriessel@searchtechnologies.com> Fri Aug 04 19:27:45 2017 +0000
tree f0851cecc18b06660643d5339cb597bdc246a680
parent c90a2b5a669852f1b87785e67174a2a8eed0d3b4

SPOT-212: [Ingest] [Proxy] bluecoat.py parser not able to save into Hive table
Replaced saveAsTable syntax (deprecated for Hive tables in Spark 2.0) with insertInto

spot-ingest/pipelines/proxy/bluecoat.py

diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index de9ca6c..31d89ca 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -121,7 +121,7 @@
         sqc.setConf("hive.exec.dynamic.partition", "true")
         sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
         hive_table = "{0}.{1}".format(db,db_table)
-        df.write.saveAsTable(hive_table,format="parquet",mode="append",partitionBy=('y','m','d','h'))
+        df.write.format("parquet").mode("append").insertInto(hive_table)
 
     else:
         print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))