commit f722127ca410a88355cc3f6d952739845c7499ce
author tpltnt <tpltnt.github@dropcut.net> Mon Feb 05 19:05:13 2018 +0100
committer tpltnt <tpltnt.github@dropcut.net> Mon Feb 05 19:05:13 2018 +0100
tree 6873709a4bc9b1934d6184a97a0d12750a236842
parent a166efc306ca041f668c9af7dfd65a94f006e454

corrected data insertion

spot-ingest/pipelines/proxy/bluecoat.py

diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index 01b9922..d476733 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -168,7 +168,7 @@
         sqc.setConf("hive.exec.dynamic.partition", "true")
         sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
         hive_table = "{0}.{1}".format(db, db_table)
-        df.write.format("parquet").mode("append").insertInto(hive_table)
+        df.write.format("parquet").mode("append").partitionBy('y', 'm', 'd', 'h').insertInto(hive_table)
 
     else:
         print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))