Improve support for Fedora AMIs (#429)
* Make default/example AMI for EC2 use a recent Fedora 35 image (and update default user to match)
* Conditionally use epel-release when using CentOS, but not when using non-CentOS
* Enable socks proxy port in example config by default (uncomment) for convenience
* Change filesystem formatting to ext4 instead of ext3 (just to use the most recent ext version)
* Make python tests less sensitive to config file defaults changes without losing coverage
diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml
index c0eb250..b844f63 100644
--- a/ansible/roles/common/tasks/main.yml
+++ b/ansible/roles/common/tasks/main.yml
@@ -21,10 +21,12 @@
retries: 10
delay: 15
until: epelresult is not failed
+ when: ansible_facts['distribution'] == "CentOS"
- name: "install packages"
yum:
name:
- vim
+ - bash-completion
- git
- wget
- gcc-c++
@@ -47,7 +49,7 @@
retries: 10
delay: 15
until: yumresult_centos7 is not failed
- when: ansible_facts['distribution_major_version'] == "7"
+ when: (ansible_facts['distribution'] == "CentOS") and (ansible_facts['distribution_major_version'] == "7")
- name: "Install packages specific to CentOS 8"
yum:
name:
@@ -60,7 +62,20 @@
retries: 10
delay: 15
until: yumresult_centos8 is not failed
- when: ansible_facts['distribution_major_version'] == "8"
+ when: (ansible_facts['distribution'] == "CentOS") and (ansible_facts['distribution_major_version'] == "8")
+- name: "Install packages specific to Fedora"
+ yum:
+ name:
+ - python3-policycoreutils
+ - collectd-disk
+ - collectd-write_http
+ - make
+ state: present
+ register: yumresult_fedora
+ retries: 10
+ delay: 15
+ until: yumresult_fedora is not failed
+ when: ansible_facts['distribution'] == "Fedora"
- name: "get exact jdk folder path"
find:
file_type: directory
diff --git a/ansible/scripts/install_ansible.sh b/ansible/scripts/install_ansible.sh
index 5a4eb35..2286dfc 100755
--- a/ansible/scripts/install_ansible.sh
+++ b/ansible/scripts/install_ansible.sh
@@ -23,9 +23,12 @@
set -e
# enable yum epel repo
-is_installed_epel_release="rpm -q --quiet epel-release"
-install_epel_release="sudo yum install -q -y epel-release"
-for i in {1..10}; do ($is_installed_epel_release || $install_epel_release) && break || sleep 15; done
+os_id=$(grep '^ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
+if [[ $os_id = "centos" ]]; then
+ is_installed_epel_release="rpm -q --quiet epel-release"
+ install_epel_release="sudo yum install -q -y epel-release"
+ for i in {1..10}; do ($is_installed_epel_release || $install_epel_release) && break || sleep 15; done
+fi
# install ansible
is_installed_ansible="rpm -q --quiet ansible"
@@ -33,25 +36,29 @@
for i in {1..10}; do ($is_installed_ansible || $install_ansible) && break || sleep 15; done
# setup user-specific ansible configuration
-if [ ! -h ~/.ansible.cfg ]; then
+if [[ ! -h ~/.ansible.cfg ]]; then
cd ~/
rm -f .ansible.cfg
ln -s $base_dir/conf/ansible.cfg .ansible.cfg
fi
# setup ansible hosts
-if [ ! -h /etc/ansible/hosts ]; then
+if [[ ! -h /etc/ansible/hosts ]]; then
cd /etc/ansible
sudo rm -f hosts
sudo ln -s $base_dir/conf/hosts hosts
fi
# install lxml as it is a dependency for the maven_artifact Ansible module
-centos_version=`cat /etc/os-release | grep '^VERSION_ID' | cut -d'"' -f2`
-if [ $centos_version -eq 7 ]; then
- sudo yum install -q -y python-lxml
-elif [ $centos_version -eq 8 ]; then
- sudo yum install -q -y python3-lxml
+centos_version=$(grep '^VERSION_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"')
+if [[ $os_id = "centos" ]]; then
+ if [[ $centos_version -eq 7 ]]; then
+ sudo yum install -q -y python-lxml
+ elif [[ $centos_version -eq 8 ]]; then
+ sudo yum install -q -y python3-lxml
+ fi
+elif [[ $os_id = "fedora" ]]; then
+ sudo yum install -q -y python3-lxml
fi
# install jq to ease JSON parsing on the proxy
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index ccdbbd9..5f8f5be 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -18,7 +18,7 @@
cluster_type = ec2
# Cluster user name (install command will SSH to cluster using this user)
# Leave default below if launching cluster in AWS
-cluster_user = centos
+cluster_user = fedora
# Cluster user group
cluster_group = %(cluster_user)s
# Cluster user home directory
@@ -30,7 +30,7 @@
# from your machine. Hostname can be chosen from "nodes" section below.
proxy_hostname = leader1
# If set, a SOCKS proxy will be created on the specified port when connecting to proxy using 'muchos ssh <cluster>'
-#proxy_socks_port = 38585
+proxy_socks_port = 38585
# Accumulo Instance name
accumulo_instance = muchos
# Accumulo Password
@@ -66,11 +66,9 @@
logstash_version = 7.10.2
[ec2]
-# AWS machine image to use. The default below is for a CentOS 7 image (in us-east-1).
+# AWS machine image to use. The default below is for a Fedora image (in us-east-1).
# You may need to change this value if a new image has been released or you are running in a different region.
-# Before using this AMI, subscribe to it on the CentOS product page below or launching will fail:
-# https://aws.amazon.com/marketplace/pp/B00O7WM7QW
-aws_ami = ami-0affd4508a5d2481b
+aws_ami = ami-08b4ee602f76bff79
# Type of AWS instance launched by default
default_instance_type = m5d.large
# Type of AWS instance launched for any node running 'worker' service
@@ -91,7 +89,7 @@
# Name below should be your 'Key pair name' in EC2 and not name of your public key file.
key_name = my_aws_key
# Type of filesystem to format instance storage as.
-fstype = ext3
+fstype = ext4
# Force formatting of instance devices, even when it has an existing filesystem.
force_format = no
# Tags to add instances
diff --git a/lib/tests/azure/test_config.py b/lib/tests/azure/test_config.py
index 0ffa340..cb4ee10 100644
--- a/lib/tests/azure/test_config.py
+++ b/lib/tests/azure/test_config.py
@@ -44,7 +44,7 @@
)
assert c.get("azure", "vm_sku") == "Standard_D8s_v3"
assert c.get("azure", "data_disk_sku") == "Standard_LRS"
- assert c.user_home() == "/home/centos"
+ assert c.user_home() == "/home/" + c.get("general", "cluster_user")
assert c.mount_root() == "/var/data"
assert c.use_multiple_vmss() is False
assert c.worker_data_dirs() == ["/var/data1", "/var/data2", "/var/data3"]
@@ -106,8 +106,9 @@
assert c.get("general", "proxy_hostname") == "leader1"
assert c.proxy_public_ip() == "23.0.0.0"
assert c.proxy_private_ip() == "10.0.0.0"
- assert c.get("general", "cluster_user") == "centos"
- assert c.get("general", "cluster_group") == "centos"
+ assert c.get("general", "cluster_user") == (
+ c.get("general", "cluster_group")
+ )
assert c.get_non_proxy() == [
("10.0.0.1", "leader2"),
("10.0.0.2", "worker1"),
diff --git a/lib/tests/ec2/test_config.py b/lib/tests/ec2/test_config.py
index b41c9da..95ffbeb 100644
--- a/lib/tests/ec2/test_config.py
+++ b/lib/tests/ec2/test_config.py
@@ -38,11 +38,11 @@
)
assert c.get("ec2", "default_instance_type") == "m5d.large"
assert c.get("ec2", "worker_instance_type") == "m5d.large"
- assert c.get("ec2", "aws_ami") == "ami-0affd4508a5d2481b"
- assert c.user_home() == "/home/centos"
+ assert c.get("ec2", "aws_ami").startswith("ami-")
+ assert c.user_home() == "/home/" + c.get("general", "cluster_user")
assert c.max_ephemeral() == 1
assert c.mount_root() == "/media/ephemeral"
- assert c.fstype() == "ext3"
+ assert c.fstype() == "ext4"
assert c.force_format() == "no"
assert c.worker_data_dirs() == ["/media/ephemeral0"]
assert c.default_data_dirs() == ["/media/ephemeral0"]
@@ -110,8 +110,9 @@
assert c.get("general", "proxy_hostname") == "leader1"
assert c.proxy_public_ip() == "23.0.0.0"
assert c.proxy_private_ip() == "10.0.0.0"
- assert c.get("general", "cluster_user") == "centos"
- assert c.get("general", "cluster_group") == "centos"
+ assert c.get("general", "cluster_user") == (
+ c.get("general", "cluster_group")
+ )
assert c.get_non_proxy() == [
("10.0.0.1", "leader2"),
("10.0.0.2", "worker1"),
