[SPOT-213][SPOT-216] [setup] moved script files to support additional engines such as beeline, impala
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/beeline/create_dns_parquet.hql
similarity index 74%
copy from spot-setup/create_dns_parquet.hql
copy to spot-setup/beeline/create_dns_parquet.hql
index 38025c6..b9be108 100755
--- a/spot-setup/create_dns_parquet.hql
+++ b/spot-setup/beeline/create_dns_parquet.hql
@@ -14,8 +14,7 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns (
frame_time STRING,
unix_tstamp BIGINT,
frame_len INT,
@@ -34,10 +33,10 @@
h TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive';
+LOCATION '${huser}/dns/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_dendro (
unix_tstamp BIGINT,
dns_a STRING,
dns_qry_name STRING,
@@ -49,10 +48,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/dendro';
+LOCATION '${huser}/dns/hive/oa/dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_edge (
unix_tstamp BIGINT,
frame_len BIGINT,
ip_dst STRING,
@@ -74,10 +73,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/edge';
+LOCATION '${huser}/dns/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -87,10 +86,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/summary';
+LOCATION '${huser}/dns/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_scores (
frame_time STRING,
unix_tstamp BIGINT,
frame_len BIGINT,
@@ -114,10 +113,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/suspicious';
+LOCATION '${huser}/dns/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_storyboard (
ip_threat STRING,
dns_threat STRING,
title STRING,
@@ -129,10 +128,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/storyboard';
+LOCATION '${huser}/dns/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_threat_dendro (
anchor STRING,
total BIGINT,
dns_qry_name STRING,
@@ -144,10 +143,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
+LOCATION '${huser}/dns/hive/oa/threat_dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.dns_threat_investigation (
unix_tstamp BIGINT,
ip_dst STRING,
dns_qry_name STRING,
@@ -160,4 +159,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/threat_investigation';
+LOCATION '${huser}/dns/hive/oa/threat_investigation';
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/beeline/create_flow_parquet.hql
similarity index 75%
copy from spot-setup/create_flow_parquet.hql
copy to spot-setup/beeline/create_flow_parquet.hql
index 41c4819..25e860a 100755
--- a/spot-setup/create_flow_parquet.hql
+++ b/spot-setup/beeline/create_flow_parquet.hql
@@ -14,8 +14,7 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow (
treceived STRING,
unix_tstamp BIGINT,
tryear INT,
@@ -52,10 +51,10 @@
h TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive';
+LOCATION '${huser}/flow/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_chords (
ip_threat STRING,
srcip STRING,
dstip STRING,
@@ -68,10 +67,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/chords';
+LOCATION '${huser}/flow/hive/oa/chords';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_edge (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -96,10 +95,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/edge';
+LOCATION '${huser}/flow/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -109,10 +108,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/summary';
+LOCATION '${huser}/flow/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_scores (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -140,10 +139,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/suspicious';
+LOCATION '${huser}/flow/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_storyboard (
ip_threat STRING,
title STRING,
text STRING
@@ -154,10 +153,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/storyboard';
+LOCATION '${huser}/flow/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_threat_investigation (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -171,10 +170,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
+LOCATION '${huser}/flow/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.flow_timeline (
ip_threat STRING,
tstart STRING,
tend STRING,
@@ -192,4 +191,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/timeline';
+LOCATION '${huser}/flow/hive/oa/timeline';
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/beeline/create_proxy_parquet.hql
similarity index 77%
copy from spot-setup/create_proxy_parquet.hql
copy to spot-setup/beeline/create_proxy_parquet.hql
index f665dc2..d9cd79f 100755
--- a/spot-setup/create_proxy_parquet.hql
+++ b/spot-setup/beeline/create_proxy_parquet.hql
@@ -14,8 +14,10 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
+SET huser;
+SET dbname;
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy (
p_date STRING,
p_time STRING,
clientip STRING,
@@ -52,10 +54,10 @@
h STRING
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive';
+LOCATION '${huser}/proxy/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_edge (
tdate STRING,
time STRING,
clientip STRING,
@@ -80,10 +82,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/edge';
+LOCATION '${huser}/proxy/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -93,10 +95,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/summary';
+LOCATION '${huser}/proxy/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_scores (
tdate STRING,
time STRING,
clientip STRING,
@@ -128,10 +130,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/suspicious';
+LOCATION '${huser}/proxy/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_storyboard (
p_threat STRING,
title STRING,
text STRING
@@ -142,10 +144,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/storyboard';
+LOCATION '${huser}/proxy/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_threat_investigation (
tdate STRING,
fulluri STRING,
uri_sev INT
@@ -156,10 +158,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
+LOCATION '${huser}/proxy/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${dbname}.proxy_timeline (
p_threat STRING,
tstart STRING,
tend STRING,
@@ -174,4 +176,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/timeline';
+LOCATION '${huser}/proxy/hive/oa/timeline';
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/hive/create_dns_parquet.hql
similarity index 70%
copy from spot-setup/create_dns_parquet.hql
copy to spot-setup/hive/create_dns_parquet.hql
index 38025c6..8e31ed3 100755
--- a/spot-setup/create_dns_parquet.hql
+++ b/spot-setup/hive/create_dns_parquet.hql
@@ -14,8 +14,10 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
+SET hiveconf:huser;
+SET hiveconf:dbname;
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns (
frame_time STRING,
unix_tstamp BIGINT,
frame_len INT,
@@ -34,10 +36,10 @@
h TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive';
+LOCATION '${hiveconf:huser}/dns/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_dendro (
unix_tstamp BIGINT,
dns_a STRING,
dns_qry_name STRING,
@@ -49,10 +51,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/dendro';
+LOCATION '${hiveconf:huser}/dns/hive/oa/dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_edge (
unix_tstamp BIGINT,
frame_len BIGINT,
ip_dst STRING,
@@ -74,10 +76,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/edge';
+LOCATION '${hiveconf:huser}/dns/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -87,10 +89,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/summary';
+LOCATION '${hiveconf:huser}/dns/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_scores (
frame_time STRING,
unix_tstamp BIGINT,
frame_len BIGINT,
@@ -114,10 +116,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/suspicious';
+LOCATION '${hiveconf:huser}/dns/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_storyboard (
ip_threat STRING,
dns_threat STRING,
title STRING,
@@ -129,10 +131,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/storyboard';
+LOCATION '${hiveconf:huser}/dns/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_threat_dendro (
anchor STRING,
total BIGINT,
dns_qry_name STRING,
@@ -144,10 +146,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
+LOCATION '${hiveconf:huser}/dns/hive/oa/threat_dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns_threat_investigation (
unix_tstamp BIGINT,
ip_dst STRING,
dns_qry_name STRING,
@@ -160,4 +162,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/dns/hive/oa/threat_investigation';
+LOCATION '${hiveconf:huser}/dns/hive/oa/threat_investigation';
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/hive/create_flow_parquet.hql
similarity index 71%
copy from spot-setup/create_flow_parquet.hql
copy to spot-setup/hive/create_flow_parquet.hql
index 41c4819..034e194 100755
--- a/spot-setup/create_flow_parquet.hql
+++ b/spot-setup/hive/create_flow_parquet.hql
@@ -14,8 +14,10 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
+SET hiveconf:huser;
+SET hiveconf:dbname;
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow (
treceived STRING,
unix_tstamp BIGINT,
tryear INT,
@@ -52,10 +54,10 @@
h TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive';
+LOCATION '${hiveconf:huser}/flow/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_chords (
ip_threat STRING,
srcip STRING,
dstip STRING,
@@ -68,10 +70,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/chords';
+LOCATION '${hiveconf:huser}/flow/hive/oa/chords';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_edge (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -96,10 +98,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/edge';
+LOCATION '${hiveconf:huser}/flow/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -109,10 +111,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/summary';
+LOCATION '${hiveconf:huser}/flow/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_scores (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -140,10 +142,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/suspicious';
+LOCATION '${hiveconf:huser}/flow/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_storyboard (
ip_threat STRING,
title STRING,
text STRING
@@ -154,10 +156,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/storyboard';
+LOCATION '${hiveconf:huser}/flow/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_threat_investigation (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -171,10 +173,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
+LOCATION '${hiveconf:huser}/flow/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow_timeline (
ip_threat STRING,
tstart STRING,
tend STRING,
@@ -192,4 +194,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/flow/hive/oa/timeline';
+LOCATION '${hiveconf:huser}/flow/hive/oa/timeline';
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/hive/create_proxy_parquet.hql
similarity index 74%
copy from spot-setup/create_proxy_parquet.hql
copy to spot-setup/hive/create_proxy_parquet.hql
index f665dc2..16d90c0 100755
--- a/spot-setup/create_proxy_parquet.hql
+++ b/spot-setup/hive/create_proxy_parquet.hql
@@ -14,8 +14,10 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
+SET hiveconf:huser;
+SET hiveconf:dbname;
-CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy (
p_date STRING,
p_time STRING,
clientip STRING,
@@ -52,10 +54,10 @@
h STRING
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive';
+LOCATION '${hiveconf:huser}/proxy/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_edge (
tdate STRING,
time STRING,
clientip STRING,
@@ -80,10 +82,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/edge';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -93,10 +95,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/summary';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_scores (
tdate STRING,
time STRING,
clientip STRING,
@@ -128,10 +130,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/suspicious';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_storyboard (
p_threat STRING,
title STRING,
text STRING
@@ -142,10 +144,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/storyboard';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_threat_investigation (
tdate STRING,
fulluri STRING,
uri_sev INT
@@ -156,10 +158,10 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy_timeline (
p_threat STRING,
tstart STRING,
tend STRING,
@@ -174,4 +176,4 @@
d TINYINT
)
STORED AS PARQUET
-LOCATION '${var:huser}/proxy/hive/oa/timeline';
+LOCATION '${hiveconf:huser}/proxy/hive/oa/timeline';
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/impala/create_dns_parquet.hql
similarity index 85%
rename from spot-setup/create_dns_parquet.hql
rename to spot-setup/impala/create_dns_parquet.hql
index 38025c6..274ea9d 100755
--- a/spot-setup/create_dns_parquet.hql
+++ b/spot-setup/impala/create_dns_parquet.hql
@@ -37,7 +37,7 @@
LOCATION '${var:huser}/dns/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_dendro (
unix_tstamp BIGINT,
dns_a STRING,
dns_qry_name STRING,
@@ -52,7 +52,7 @@
LOCATION '${var:huser}/dns/hive/oa/dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_edge (
unix_tstamp BIGINT,
frame_len BIGINT,
ip_dst STRING,
@@ -77,7 +77,7 @@
LOCATION '${var:huser}/dns/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -90,7 +90,7 @@
LOCATION '${var:huser}/dns/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_scores (
frame_time STRING,
unix_tstamp BIGINT,
frame_len BIGINT,
@@ -117,7 +117,7 @@
LOCATION '${var:huser}/dns/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_storyboard (
ip_threat STRING,
dns_threat STRING,
title STRING,
@@ -132,7 +132,7 @@
LOCATION '${var:huser}/dns/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_threat_dendro (
anchor STRING,
total BIGINT,
dns_qry_name STRING,
@@ -147,7 +147,7 @@
LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
-CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns_threat_investigation (
unix_tstamp BIGINT,
ip_dst STRING,
dns_qry_name STRING,
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/impala/create_flow_parquet.hql
similarity index 86%
rename from spot-setup/create_flow_parquet.hql
rename to spot-setup/impala/create_flow_parquet.hql
index 41c4819..c8d3481 100755
--- a/spot-setup/create_flow_parquet.hql
+++ b/spot-setup/impala/create_flow_parquet.hql
@@ -55,7 +55,7 @@
LOCATION '${var:huser}/flow/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_chords (
ip_threat STRING,
srcip STRING,
dstip STRING,
@@ -71,7 +71,7 @@
LOCATION '${var:huser}/flow/hive/oa/chords';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_edge (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -99,7 +99,7 @@
LOCATION '${var:huser}/flow/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -112,7 +112,7 @@
LOCATION '${var:huser}/flow/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_scores (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -143,7 +143,7 @@
LOCATION '${var:huser}/flow/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_storyboard (
ip_threat STRING,
title STRING,
text STRING
@@ -157,7 +157,7 @@
LOCATION '${var:huser}/flow/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_threat_investigation (
tstart STRING,
srcip STRING,
dstip STRING,
@@ -174,7 +174,7 @@
LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow_timeline (
ip_threat STRING,
tstart STRING,
tend STRING,
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/impala/create_proxy_parquet.hql
similarity index 87%
rename from spot-setup/create_proxy_parquet.hql
rename to spot-setup/impala/create_proxy_parquet.hql
index f665dc2..ddf3283 100755
--- a/spot-setup/create_proxy_parquet.hql
+++ b/spot-setup/impala/create_proxy_parquet.hql
@@ -55,7 +55,7 @@
LOCATION '${var:huser}/proxy/hive';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_edge (
tdate STRING,
time STRING,
clientip STRING,
@@ -83,7 +83,7 @@
LOCATION '${var:huser}/proxy/hive/oa/edge';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_ingest_summary (
tdate STRING,
total BIGINT
)
@@ -96,7 +96,7 @@
LOCATION '${var:huser}/proxy/hive/oa/summary';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_scores (
tdate STRING,
time STRING,
clientip STRING,
@@ -131,7 +131,7 @@
LOCATION '${var:huser}/proxy/hive/oa/suspicious';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_storyboard (
p_threat STRING,
title STRING,
text STRING
@@ -145,7 +145,7 @@
LOCATION '${var:huser}/proxy/hive/oa/storyboard';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_threat_investigation (
tdate STRING,
fulluri STRING,
uri_sev INT
@@ -159,7 +159,7 @@
LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
-CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy_timeline (
p_threat STRING,
tstart STRING,
tend STRING,
