BIGTOP-3767. Add Spark Thrift Server Puppet Deploy Code (#973)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 76e3eed..76715dc 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -148,6 +148,8 @@
# spark
spark::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
+spark::common::spark_sql_warehouse_dir: "/user/spark/spark-warehouse"
+spark::common::spark_hive_server2_thrift_port: 12000
# to enable spark HA, ensure zookeeper is available and uncomment the line below
#spark::common::zookeeper_connection_string: "%{hiera('hadoop::zk')}"
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index b016d6f..c432036 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -69,6 +69,7 @@
worker => ["spark-on-yarn"],
client => ["spark-client"],
library => ["spark-yarn-slave"],
+ gateway_server => ["spark-thriftserver"],
},
spark-standalone => {
master => ["spark-master"],
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index 2d4a3da..d2e403c 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -39,6 +39,31 @@
if ('spark-history-server' in $roles) {
include spark::history_server
}
+
+ if ('spark-thriftserver' in $roles) {
+ include spark::spark_thriftserver
+ }
+
+ }
+
+ class spark_thriftserver {
+ include spark::common
+
+ package { 'spark-thriftserver':
+ ensure => latest,
+ }
+
+ service { 'spark-thriftserver':
+ ensure => running,
+ subscribe => [
+ Package['spark-thriftserver'],
+ File['/etc/spark/conf/spark-env.sh'],
+ File['/etc/spark/conf/spark-defaults.conf'],
+ ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+
}
class client {
@@ -182,6 +207,8 @@
}
class common(
+ $spark_hive_server2_thrift_port = undef,
+ $spark_sql_warehouse_dir = undef,
$master_url = undef,
$master_host = $fqdn,
$zookeeper_connection_string = undef,
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
index 2ef4a04..ff1cff7 100644
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
@@ -16,7 +16,7 @@
<% if @master_url -%>
spark.master <%= @master_url %>
<% else -%>
-<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+<% if (scope['spark::deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
spark.master spark://<%= @master_host %>:<%= @master_port %>
<% else -%>
spark.master yarn
@@ -36,3 +36,9 @@
<% end -%>
spark.driver.memory <%= @driver_mem %>
spark.executor.memory <%= @executor_mem %>
+<% if @spark_sql_warehouse_dir -%>
+spark.sql.warehouse.dir <%= @spark_sql_warehouse_dir %>
+<% end -%>
+<% if @spark_hive_server2_thrift_port -%>
+spark.hive.server2.thrift.port <%= @spark_hive_server2_thrift_port %>
+<% end -%>
\ No newline at end of file
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
index e1dd40b..c104141 100755
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
@@ -27,7 +27,7 @@
<% if @master_url -%>
export SPARK_MASTER_URL=<%= @master_url %>
<% else -%>
-<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+<% if (scope['spark::deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
export SPARK_MASTER_URL=spark://<%= @master_host %>:<%= @master_port %>
<% else -%>
export SPARK_MASTER_URL=yarn
