Fix issues with setting up Spark (#322)
The Ansible task to copy the Spark YARN shuffle JAR is now sequenced
to run as part of the Spark role. In addition, Spark 2.4.5 is the
current latest production release so defaulting to that.
diff --git a/ansible/roles/hadoop/tasks/main.yml b/ansible/roles/hadoop/tasks/main.yml
index 45d72ec..2126c8c 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -41,9 +41,6 @@
synchronize: src={{ user_home }}/mvn_dep/ dest={{ hadoop_home }}/share/hadoop/common/lib/
when: hadoop_major_version == '3' and java_product_version == 11
-- name: "copy spark yarn shuffle jar to hadoop lib"
- command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar {{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home }}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
- when: "'spark' in groups"
- name: "setup hadoop short circuit socket dir"
file: path=/var/lib/hadoop-hdfs state=directory owner={{ cluster_user }} group={{ cluster_group }} mode=0755
become: yes
diff --git a/ansible/roles/spark/tasks/main.yml b/ansible/roles/spark/tasks/main.yml
index 4310754..9ccfe81 100644
--- a/ansible/roles/spark/tasks/main.yml
+++ b/ansible/roles/spark/tasks/main.yml
@@ -22,3 +22,6 @@
with_items:
- spark-defaults.conf
- spark-env.sh
+- name: "copy spark yarn shuffle jar to hadoop lib"
+ command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar {{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home }}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
+ when: "'spark' in groups"
diff --git a/conf/checksums b/conf/checksums
index 3f4c3d4..68c6eb4 100644
--- a/conf/checksums
+++ b/conf/checksums
@@ -22,6 +22,7 @@
hadoop:2.7.6:f2327ea93f4bc5a5d7150dee8e0ede196d3a77ff8526a7dd05a48a09aae25669
hadoop:2.7.5:0bfc4d9b04be919be2fdf36f67fa3b4526cdbd406c512a7a1f5f1b715661f831
hadoop:2.6.5:001ad18d4b6d0fe542b15ddadba2d092bc97df1c4d2d797381c8d12887691898
+spark:2.4.5:17A8B3BD0151E506E2B3C09FFE6F1010FFE1C6B7B906F3A19D357B37098EBD0C3AD91E6501545AE87834B81D2EF07556E5FECCFB7275D1C74533777BC7A5DF3B
spark:2.4.4:A3E4FDEAD2409C8D1E0F213145ACC3B43788285715EB100DD29B171E6B24012673DDD94BDC612D946764B71CBC8F78BCD93889A4BC60B3136A33CD7E6C4DBC26
spark:2.4.3:4db62e110c8080f28ac8f1d701b7477cede23e60b51231ab63687b2cc6150faa
spark:2.3.4:0241cbe73d3ad4fec1c655d839facc16e647ff9a3825f51c78d9fb270a753da4
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index 63898be..84e42e0 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -38,7 +38,7 @@
# Software versions (make sure you have a corresponding entry for the checksum in conf/checksums)
hadoop_version = 3.2.1
zookeeper_version = 3.5.6
-spark_version = 2.3.2
+spark_version = 2.4.5
fluo_version = 1.2.0
fluo_yarn_version = 1.0.0
accumulo_version = 2.0.0
