Merge pull request #270 from srajtiwari/master
Support Accumulo installs on Microsoft Azure
diff --git a/.gitignore b/.gitignore
index a2f1c6d..3c468ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
*.pyc
-.idea/
-pom.xml
+/.idea/
+/pom.xml
diff --git a/README.md b/README.md
index e429802..d124b1f 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
The following commands will install Muchos, launch a cluster, and setup/run Accumulo:
-```bash
+```sh
git clone https://github.com/apache/fluo-muchos.git
cd fluo-muchos/
@@ -91,13 +91,13 @@
After your cluster is launched, SSH to it using the following command:
-```bash
+```sh
./bin/muchos ssh
```
Run the following command to terminate your cluster. WARNING: All cluster data will be lost.
-```bash
+```sh
./bin/muchos terminate
```
@@ -248,7 +248,7 @@
with Muchos as it configures your shell with common environment variables. To run an example
application, SSH to a node on cluster where Fluo is installed and clone the example repo:
-```bash
+```sh
./bin/muchos ssh # SSH to cluster proxy node
ssh <node where Fluo is installed> # Nodes with Fluo installed is determined by Muchos config
hub clone apache/fluo-examples # Clone repo of Fluo example applications. Press enter for user/password.
@@ -258,7 +258,7 @@
to run the [WebIndex] application are shown below. Read the [WebIndex] README to learn more
before running these commands.
-```bash
+```sh
cd fluo-examples/webindex
./bin/webindex init # Initialize and start webindex Fluo application
./bin/webindex getpaths 2015-18 # Retrieves CommonCrawl paths file for 2015-18 crawl
@@ -296,7 +296,7 @@
shutdown behavior will be stopping the node. If you would like your cluster to terminate after 8 hours,
set the following configuration in [muchos.props]:
-```
+```ini
shutdown_delay_minutes = 480
shutdown_behavior = terminate
```
@@ -307,7 +307,7 @@
The `config` command allows you to retrieve cluster configuration for your own scripts:
-```bash
+```sh
$ ./bin/muchos config -p leader.public.ip
10.10.10.10
```
diff --git a/ansible/.gitignore b/ansible/.gitignore
index b39a143..e23687b 100644
--- a/ansible/.gitignore
+++ b/ansible/.gitignore
@@ -1 +1 @@
-site.yml
+/site.yml
diff --git a/ansible/conf/.gitignore b/ansible/conf/.gitignore
index 55d232d..cc13667 100644
--- a/ansible/conf/.gitignore
+++ b/ansible/conf/.gitignore
@@ -1,2 +1,2 @@
-hosts
-keys
+/hosts
+/keys
diff --git a/ansible/docker.yml b/ansible/docker.yml
index 0e893ad..43eddf7 100644
--- a/ansible/docker.yml
+++ b/ansible/docker.yml
@@ -35,10 +35,10 @@
shell: docker swarm join-token -q worker
register: swarm_token
changed_when: "'active' not in swarm_status.stdout_lines"
-- hosts: workers
+- hosts: workers:!swarmmanager
become: yes
vars:
- swarm_token: "{{ hostvars[groups['swarmmanager'][0]]['swarm_token']['stdout'] }}"
+ swarm_token_stdout: "{{ hostvars[groups['swarmmanager'][0]]['swarm_token']['stdout'] }}"
manager_ip: "{{ hostvars[groups['swarmmanager'][0]]['ansible_default_ipv4']['address'] }}"
tasks:
- name: get swarm status
@@ -47,6 +47,7 @@
register: swarm_status
changed_when: "'active' not in swarm_status.stdout_lines"
- name: join worker to swarm
- shell: >
- docker swarm join --token={{ swarm_token }} {{ manager_ip }}:2377
+ shell: |
+ docker swarm leave || echo "Not part of a swarm."
+ docker swarm join --token={{ swarm_token_stdout }} {{ manager_ip }}:2377
when: "'active' not in swarm_status.stdout_lines"
diff --git a/ansible/group_vars/.gitignore b/ansible/group_vars/.gitignore
index 0702cb5..244b0c6 100644
--- a/ansible/group_vars/.gitignore
+++ b/ansible/group_vars/.gitignore
@@ -1 +1 @@
-all
+/all
diff --git a/ansible/roles/accumulo/tasks/main.yml b/ansible/roles/accumulo/tasks/main.yml
index 241ab7b..b52851c 100644
--- a/ansible/roles/accumulo/tasks/main.yml
+++ b/ansible/roles/accumulo/tasks/main.yml
@@ -56,8 +56,8 @@
- name: "configure tservers using managed templates"
template: src=tservers dest={{ accumulo_home }}/conf/{{ accumulo_tservers_fn[accumulo_major_version] }}
- name: "build accumulo native libraries"
- shell: "{{ accumulo_build_native_cmd[accumulo_major_version] }}"
- args:
+ shell: "{{ accumulo_build_native_cmd[accumulo_major_version] }}"
+ args:
creates: "{{ accumulo_home }}/lib/native/libaccumulo.so"
- name: "Create accumulo log dir"
file: path={{ worker_data_dirs[0] }}/logs/accumulo state=directory
diff --git a/ansible/roles/accumulo/templates/log4j-service.properties b/ansible/roles/accumulo/templates/log4j-service.properties
index c09d46b..5f255ea 100644
--- a/ansible/roles/accumulo/templates/log4j-service.properties
+++ b/ansible/roles/accumulo/templates/log4j-service.properties
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=${accumulo.log.dir}/${accumulo.application}.log
log4j.appender.file.MaxFileSize=1000MB
diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml
index 93a4be4..185aae5 100644
--- a/ansible/roles/common/tasks/main.yml
+++ b/ansible/roles/common/tasks/main.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-- name: "enable epel yum repo"
+- name: "enable epel yum repo"
yum: name=epel-release state=present
- name: "install packages"
yum:
diff --git a/ansible/roles/common/tasks/ssh.yml b/ansible/roles/common/tasks/ssh.yml
index 0e2e2af..77e8af8 100644
--- a/ansible/roles/common/tasks/ssh.yml
+++ b/ansible/roles/common/tasks/ssh.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-- name: "ensure cluster user exists but don't generate ssh key"
+- name: "ensure cluster user exists but don't generate ssh key"
user: name={{ cluster_user }} generate_ssh_key=no state=present
- name: "copy cluster private key to all nodes"
copy: src=/home/{{ cluster_user }}/.ssh/id_rsa dest=/home/{{ cluster_user }}/.ssh/id_rsa owner={{ cluster_user }} group={{ cluster_group }} mode=0600
diff --git a/ansible/roles/fluo/templates/fluo-app.properties b/ansible/roles/fluo/templates/fluo-app.properties
index dd22cb0..ebeef4a 100644
--- a/ansible/roles/fluo/templates/fluo-app.properties
+++ b/ansible/roles/fluo/templates/fluo-app.properties
@@ -20,7 +20,7 @@
#############################
# NOTE - All properties that have a default are set with it. Uncomment
-# a property if you want to use a value different than the default.
+# a property if you want to use a value different than the default.
# Properties that have no default are uncommented and must be set by
# the user.
@@ -29,7 +29,7 @@
## Specifies an observer provider. This should be the name of a class that
## implements org.apache.fluo.api.observer.ObserverProvider.
#fluo.observer.provider=com.foo.AppObserverProvider
-## Observer jars in this directory are copied to the DFS (specified by fluo.dfs.root) during initialization.
+## Observer jars in this directory are copied to the DFS (specified by fluo.dfs.root) during initialization.
## If this property is set, fluo.observer.jars.url should not be set.
#fluo.observer.init.dir=/path/to/observer/jars/
## Observer jars are retreived from this URL. If set, fluo.observer.init.dir should not be set.
diff --git a/ansible/roles/fluo/templates/fluo-conn.properties b/ansible/roles/fluo/templates/fluo-conn.properties
index 29c0cb3..bcd6725 100644
--- a/ansible/roles/fluo/templates/fluo-conn.properties
+++ b/ansible/roles/fluo/templates/fluo-conn.properties
@@ -20,7 +20,7 @@
############################
# NOTE - All properties that have a default are set with it. Uncomment
-# a property if you want to use a value different than the default.
+# a property if you want to use a value different than the default.
# Properties that have no default are uncommented and must be set by
# the user.
diff --git a/ansible/roles/fluo/templates/fluo.properties b/ansible/roles/fluo/templates/fluo.properties
index 1ff4df3..d3ae473 100644
--- a/ansible/roles/fluo/templates/fluo.properties
+++ b/ansible/roles/fluo/templates/fluo.properties
@@ -20,7 +20,7 @@
#################
# NOTE - All properties that have a default are set with it. Uncomment
-# a property if you want to use a value different than the default.
+# a property if you want to use a value different than the default.
# Properties that have no default are uncommented and must be set by
# the user. Most are unset except for fluo.accumulo.classpath which
# has a suggested value.
diff --git a/ansible/roles/fluo_yarn/templates/fluo-yarn.properties b/ansible/roles/fluo_yarn/templates/fluo-yarn.properties
index 2ae8884..900433b 100644
--- a/ansible/roles/fluo_yarn/templates/fluo-yarn.properties
+++ b/ansible/roles/fluo_yarn/templates/fluo-yarn.properties
@@ -20,7 +20,7 @@
########################
# NOTE - All properties that have a default are set with it. Uncomment
-# a property if you want to use a value different than the default.
+# a property if you want to use a value different than the default.
# Properties that have no default are uncommented and must be set by
# the user.
diff --git a/ansible/roles/grafana/tasks/main.yml b/ansible/roles/grafana/tasks/main.yml
index 35c86d2..e66d219 100644
--- a/ansible/roles/grafana/tasks/main.yml
+++ b/ansible/roles/grafana/tasks/main.yml
@@ -16,8 +16,8 @@
#
- name: "download grafana rpm"
- get_url:
- args:
+ get_url:
+ args:
url: https://grafanarel.s3.amazonaws.com/builds/{{ grafana_rpm }}
dest: /tmp/{{ grafana_rpm }}
sha256sum: "{{ grafana_sha256 }}"
diff --git a/ansible/roles/hadoop/templates/core-site.xml b/ansible/roles/hadoop/templates/core-site.xml
index 4f44f2e..409c103 100644
--- a/ansible/roles/hadoop/templates/core-site.xml
+++ b/ansible/roles/hadoop/templates/core-site.xml
@@ -23,7 +23,7 @@
<configuration>
<property>
- <name>fs.defaultFS</name>
+ <name>fs.defaultFS</name>
<value>{{ hdfs_root }}</value>
</property>
<property>
diff --git a/ansible/roles/hadoop/templates/hdfs-site.xml b/ansible/roles/hadoop/templates/hdfs-site.xml
index f102fea..684d4c4 100644
--- a/ansible/roles/hadoop/templates/hdfs-site.xml
+++ b/ansible/roles/hadoop/templates/hdfs-site.xml
@@ -18,19 +18,6 @@
limitations under the License.
-->
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
<!-- Put site-specific property overrides in this file. -->
@@ -45,8 +32,8 @@
</property>
<property>
<name>dfs.datanode.data.dir</name>
- <value>{% for dir in worker_data_dirs -%}
- {{ dir }}/hadoop/data
+ <value>{% for dir in worker_data_dirs -%}
+ {{ dir }}/hadoop/data
{%- if not loop.last -%} , {%- endif -%}
{%- endfor %}</value>
</property>
diff --git a/ansible/roles/hadoop/templates/mapred-site.xml b/ansible/roles/hadoop/templates/mapred-site.xml
index c6be0ce..7dbb5f6 100644
--- a/ansible/roles/hadoop/templates/mapred-site.xml
+++ b/ansible/roles/hadoop/templates/mapred-site.xml
@@ -28,14 +28,14 @@
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
- <value>{% for dir in worker_data_dirs -%}
+ <value>{% for dir in worker_data_dirs -%}
{{ dir }}/hadoop/mapred/temp
{%- if not loop.last -%} , {%- endif -%}
{%- endfor %}</value>
</property>
<property>
<name>mapreduce.cluster.local.dir</name>
- <value>{% for dir in worker_data_dirs -%}
+ <value>{% for dir in worker_data_dirs -%}
{{ dir }}/hadoop/mapred/local
{%- if not loop.last -%} , {%- endif -%}
{%- endfor %}</value>
diff --git a/ansible/roles/hadoop/templates/yarn-site.xml b/ansible/roles/hadoop/templates/yarn-site.xml
index 85033a6..c0ba8d9 100644
--- a/ansible/roles/hadoop/templates/yarn-site.xml
+++ b/ansible/roles/hadoop/templates/yarn-site.xml
@@ -18,19 +18,6 @@
limitations under the License.
-->
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
<!-- Put site-specific property overrides in this file. -->
@@ -41,7 +28,7 @@
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
- <value>{% for dir in worker_data_dirs -%}
+ <value>{% for dir in worker_data_dirs -%}
{{ dir }}/hadoop/yarn/local
{%- if not loop.last -%} , {%- endif -%}
{%- endfor %}</value>
diff --git a/ansible/roles/influxdb/tasks/main.yml b/ansible/roles/influxdb/tasks/main.yml
index ae83a2e..1f2897d 100644
--- a/ansible/roles/influxdb/tasks/main.yml
+++ b/ansible/roles/influxdb/tasks/main.yml
@@ -17,7 +17,7 @@
- name: "download influxdb rpm"
get_url:
- args:
+ args:
url: https://influxdb.s3.amazonaws.com/{{ influxdb_rpm }}
dest: /tmp/{{ influxdb_rpm }}
sha256sum: "{{ influxdb_sha256 }}"
diff --git a/ansible/roles/proxy/tasks/download.yml b/ansible/roles/proxy/tasks/download.yml
index 5744070..48a3a7c 100644
--- a/ansible/roles/proxy/tasks/download.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -31,4 +31,4 @@
- { urlp: "{{ apache_mirror.stdout }}/zookeeper/zookeeper-{{ zookeeper_version }}", fn: "{{ zookeeper_tarball }}", sum: "{{ zookeeper_sha256 }}" }
- { urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{ hadoop_version }}", fn: "{{ hadoop_tarball }}", sum: "{{ hadoop_sha256 }}" }
- { urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version }}/binaries", fn: "{{ maven_tarball }}", sum: "{{ maven_sha256 }}" }
- - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version }}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" }
+ - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version }}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" }
diff --git a/ansible/roles/spark/tasks/start-spark-history.yml b/ansible/roles/spark/tasks/start-spark-history.yml
index 0b092b9..ad136c8 100644
--- a/ansible/roles/spark/tasks/start-spark-history.yml
+++ b/ansible/roles/spark/tasks/start-spark-history.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-- name: "ensure spark history directory exists in hdfs"
+- name: "ensure spark history directory exists in hdfs"
command: "{{ hadoop_home}}/bin/hdfs dfs -mkdir -p /spark/history"
register: mk_hist_dir
changed_when: mk_hist_dir.rc != 0
diff --git a/ansible/roles/zookeeper/templates/zoo.cfg b/ansible/roles/zookeeper/templates/zoo.cfg
index 1c154a6..2eda8f6 100644
--- a/ansible/roles/zookeeper/templates/zoo.cfg
+++ b/ansible/roles/zookeeper/templates/zoo.cfg
@@ -1,13 +1,13 @@
# The number of milliseconds of each tick
tickTime=2000
-# The number of ticks that the initial
+# The number of ticks that the initial
# synchronization phase can take
initLimit=10
-# The number of ticks that can pass between
+# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
-# do not use /tmp for storage, /tmp here is just
+# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir={{ default_data_dirs[0] }}/zookeeper
# the port at which the clients will connect
diff --git a/conf/.gitignore b/conf/.gitignore
index dc331b4..0e1a2fc 100644
--- a/conf/.gitignore
+++ b/conf/.gitignore
@@ -1,4 +1,4 @@
-muchos.props*
-hosts/*
-keys
-user_data
+/muchos.props*
+/hosts/*
+/keys
+/user_data
diff --git a/conf/templates/README.md b/conf/templates/README.md
index cb15e83..099f082 100644
--- a/conf/templates/README.md
+++ b/conf/templates/README.md
@@ -6,31 +6,31 @@
cases that require distinct, per-host launch configurations, and
for use cases that require hosts to have persistent, EBS-backed data
volumes (rather than ephemeral volumes, the muchos default for EC2
-clusters)
+clusters).
If you are already familiar with muchos and with the basics of EC2
launch requests, then creating your own launch templates will be simple
and straightforward.
Please follow the guidance provided here to ensure compatibility between
-your custom templates and muchos automation
+your custom templates and muchos automation.
## Configuration
### Select a cluster template in *muchos.props*
-```
+
+```ini
[ec2]
...
cluster_template = example
...
```
+
The configured value must match the name of a subdirectory under
`conf/templates`
-```
-~$ ls -1a fluo-muchos/conf/templates/example
-.
-..
+```sh
+~$ ls -1 fluo-muchos/conf/templates/example
devices
metrics.json
namenode.json
@@ -38,21 +38,23 @@
zookeeper.json
worker.json
```
+
The subdirectory will contain one or more user-defined EC2 launch
-templates (*\*.json*) for your various host types, and it will
-include a *devices* file specifying the desired mount points for all
+templates `*.json` for your various host service types, and it will
+include a `devices` file specifying the desired mount points for all
data volumes (excluding root volumes, as they are mounted
-automatically)
+automatically).
### Defining EC2 launch templates and device mounts for your hosts
-#### Launch Templates: *{service-name}.json* files
+#### Launch Templates: `{service-name}.json` files
Each JSON file represents a standard EC2 launch request, and each file
name must match one of the predefined muchos service names, as
defined in the **nodes** section of *muchos.props*. E.g.,
-```
-...
+
+```ini
+...
[nodes]
leader1 = namenode,resourcemanager,accumulomaster
leader2 = metrics,zookeeper
@@ -61,129 +63,131 @@
worker3 = worker
worker4 = worker
```
+
In template mode, the first service listed for a given host denotes the
template to be selected for its launch.
Based on the example given above:
-* **leader1** selects **namenode.json**
-* **leader2** selects **metrics.json**
-* **worker1** selects **worker.json**
+* **leader1** selects `namenode.json`
+* **leader2** selects `metrics.json`
+* **worker1** selects `worker.json`
* and so on...
-
-For example, *namenode.json* might be defined as follows...
-```
+
+For example, `namenode.json` might be defined as follows...
+
+```json
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1", # Here, /dev/sda1 denotes the root volume
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf", # Here, /dev/sdf (a.k.a "/dev/xvdf") denotes
- "Ebs": { # our single EBS-backed data volume
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.4xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1", # Here, /dev/sda1 denotes the root volume
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf", # Here, /dev/sdf (a.k.a "/dev/xvdf") denotes
+ "Ebs": { # our single EBS-backed data volume
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.4xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
```
-*Property Placeholders*
+
+#### Property Placeholders
The `${property name}` placeholders demonstrated above are optional and
are intended to simplify template creation and reduce maintenance burden.
-They allow any matching properties from the **ec2** section of *muchos.props*
+They allow any matching properties from the **ec2** section of `muchos.props`
to be interpolated automatically.
If needed, you may also define your own custom properties and have them be injected
automatically by simply adding them to the **ec2** section of
-*muchos.props* and to your templates
+`muchos.props` and to your templates
-#### Device Mounts: *devices* file
+#### Device Mounts: `devices` file
-The *devices* file contains the user-defined mapping of storage
+The `devices` file contains the user-defined mapping of storage
devices and mount points for all data (i.e., non-root) volumes in your
cluster. Muchos Ansible scripts leverage this information to prepare
all data volumes during the cluster `setup` phase.
-Two (and only two) device mappings should exist within *devices*:
+Two (and only two) device mappings should exist within `devices`:
* One map to represent your **worker** device mounts, and
* One map to represent the device mounts on all other hosts, i.e., the
**default** map
-
-For example, the *devices* file below specifies 4 mount points for all
+
+For example, the `devices` file below specifies 4 mount points for all
`worker` instance types, and specifies 1 mount point for all
other hosts via the `default` map.
-```
+
+```json
{
- "default": {
- "mounts": [
- "/data0" # For non-workers, mount the /data0 directory
- ], # on /dev/xvdf (a.k.a "/dev/sdf")
- "devices": [
- "/dev/xvdf"
- ]
+ "default": {
+ "mounts": [
+ "/data0" # For non-workers, mount the /data0 directory
+ ], # on /dev/xvdf (a.k.a "/dev/sdf")
+ "devices": [
+ "/dev/xvdf"
+ ]
},
"worker": {
- "mounts": [
- "/data0",
- "/data1",
- "/data2",
- "/data3" # For workers, mount the /data0 directory on
- ], # /dev/xvdf (a.k.a "/dev/sdf"), mount /data1 on
- "devices": [ # /dev/xvdg (a.k.a "/dev/sdg"), and so on...
- "/dev/xvdf",
- "/dev/xvdg",
- "/dev/xvdh",
- "/dev/xvdi"
- ]
+ "mounts": [
+ "/data0",
+ "/data1",
+ "/data2",
+ "/data3" # For workers, mount the /data0 directory on
+ ], # /dev/xvdf (a.k.a "/dev/sdf"), mount /data1 on
+ "devices": [ # /dev/xvdg (a.k.a "/dev/sdg"), and so on...
+ "/dev/xvdf",
+ "/dev/xvdg",
+ "/dev/xvdh",
+ "/dev/xvdi"
+ ]
}
}
```
+
Naturally, you should take care to ensure that your **BlockDeviceMappings**
also align to *default* vs *worker* node type semantics. As you
explore the example files, you should observe the implicit link betweeen
-a data volume denoted by *BlockDeviceMappings\[N].DeviceName* and its
-respective device map entry in the *devices* file.
+a data volume denoted by `BlockDeviceMappings[N].DeviceName` and its
+respective device map entry in the `devices` file.
* **Note**: While *DeviceName* mount profiles should not vary among
your default (non-worker) nodes, other attributes within *BlockDeviceMappings*,
such as *DeleteOnTermination*, *VolumeSize*, *etc*, may vary as needed
-
* **Note**: Be aware that the device names used by AWS EC2 launch requests
may differ from the actual names assigned by the EC2 block device driver, which
can be confusing. Please read
[this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html)
and [also this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html)
for additional information and guidance, if needed
-
* **Note**: Root EBS volumes may be configured as desired in your JSON
launch templates, but only non-root, "data" storage devices should be
- specified in *devices*, as root devices are mounted automatically
-
-
+ specified in `devices`, as root devices are mounted automatically
+
## Beyond the Launch Phase: *Setup*, *Terminate*, *Etc*
Aside from the configuration differences described above, which impact
the `launch` phase, all other muchos operations behave the same in
-template mode as in default mode
-
+template mode as in default mode.
+
diff --git a/conf/templates/example/accumulomaster.json b/conf/templates/example/accumulomaster.json
index 263c192..5569994 100644
--- a/conf/templates/example/accumulomaster.json
+++ b/conf/templates/example/accumulomaster.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.2xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.2xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/client.json b/conf/templates/example/client.json
index c5d4de1..5569994 100644
--- a/conf/templates/example/client.json
+++ b/conf/templates/example/client.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.2xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.2xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/devices b/conf/templates/example/devices
index f338f9f..5d49087 100644
--- a/conf/templates/example/devices
+++ b/conf/templates/example/devices
@@ -21,4 +21,4 @@
"/dev/xvdi"
]
}
-}
\ No newline at end of file
+}
diff --git a/conf/templates/example/metrics.json b/conf/templates/example/metrics.json
index c5d4de1..5569994 100644
--- a/conf/templates/example/metrics.json
+++ b/conf/templates/example/metrics.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.2xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.2xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/namenode.json b/conf/templates/example/namenode.json
index b281782..1cf398a 100644
--- a/conf/templates/example/namenode.json
+++ b/conf/templates/example/namenode.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.4xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.4xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/resourcemanager.json b/conf/templates/example/resourcemanager.json
index a406feb..5569994 100644
--- a/conf/templates/example/resourcemanager.json
+++ b/conf/templates/example/resourcemanager.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.2xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.2xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/worker.json b/conf/templates/example/worker.json
index ae693ee..ebd581e 100644
--- a/conf/templates/example/worker.json
+++ b/conf/templates/example/worker.json
@@ -1,58 +1,58 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 250,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdg",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 250,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdh",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 250,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdi",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 250,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "c4.4xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 250,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdg",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 250,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdh",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 250,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdi",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 250,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "c4.4xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/example/zookeeper.json b/conf/templates/example/zookeeper.json
index 16ae67e..7add19e 100644
--- a/conf/templates/example/zookeeper.json
+++ b/conf/templates/example/zookeeper.json
@@ -1,34 +1,34 @@
{
- "KeyName": "${key_name}",
- "BlockDeviceMappings": [
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 40,
- "VolumeType": "gp2"
- }
- },
- {
- "DeviceName": "/dev/sdf",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeSize": 500,
- "VolumeType": "gp2"
- }
- }
- ],
- "ImageId": "${aws_ami}",
- "InstanceType": "m4.xlarge",
- "NetworkInterfaces": [
- {
- "DeviceIndex": 0,
- "AssociatePublicIpAddress": ${associate_public_ip},
- "Groups": [
- "${security_group_id}"
- ]
- }
- ],
- "EbsOptimized": true,
- "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
+ "KeyName": "${key_name}",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 40,
+ "VolumeType": "gp2"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeSize": 500,
+ "VolumeType": "gp2"
+ }
+ }
+ ],
+ "ImageId": "${aws_ami}",
+ "InstanceType": "m4.xlarge",
+ "NetworkInterfaces": [
+ {
+ "DeviceIndex": 0,
+ "AssociatePublicIpAddress": "${associate_public_ip}",
+ "Groups": [
+ "${security_group_id}"
+ ]
+ }
+ ],
+ "EbsOptimized": true,
+ "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}"
}
diff --git a/conf/templates/user/.gitignore b/conf/templates/user/.gitignore
index 72e8ffc..33662f5 100644
--- a/conf/templates/user/.gitignore
+++ b/conf/templates/user/.gitignore
@@ -1 +1 @@
-*
+/*
diff --git a/conf/upload/.gitignore b/conf/upload/.gitignore
index 9db4ee6..fa80b40 100644
--- a/conf/upload/.gitignore
+++ b/conf/upload/.gitignore
@@ -1,2 +1,2 @@
-*.tar.gz
-*.tgz
+/*.tar.gz
+/*.tgz
diff --git a/contrib/add-license-headers.md b/contrib/add-license-headers.md
index 723ab64..bffc698 100644
--- a/contrib/add-license-headers.md
+++ b/contrib/add-license-headers.md
@@ -1,16 +1,15 @@
## Adding licence header to Muchos files
-Using [license-maven-plugin][license-plugin] we can generate a consistent license
+Using [license-maven-plugin][license-plugin] we can generate a consistent license
header for the project with the following command.
-
-
* Temporally convert the project to a maven project. Make sure not to commit the new `pom.xml` file.
* Via command line, run the following command to generate the license header.
-```
+
+```sh
mvn com.mycila:license-maven-plugin:3.0:format -Dlicense.header=contrib/license-header.txt
```
* Check the output for warnings and manually add the appropriate license to those files.
-[license-plugin]: https://code.mycila.com/license-maven-plugin/
\ No newline at end of file
+[license-plugin]: https://code.mycila.com/license-maven-plugin/
diff --git a/contrib/license-header.txt b/contrib/license-header.txt
index 9f4ff31..8853bce 100644
--- a/contrib/license-header.txt
+++ b/contrib/license-header.txt
@@ -11,4 +11,4 @@
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.
diff --git a/contrib/muchos-logo-dark.svg b/contrib/muchos-logo-dark.svg
index 5e978dc..d82dcda 100644
--- a/contrib/muchos-logo-dark.svg
+++ b/contrib/muchos-logo-dark.svg
@@ -15,10 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-
-<!-- This logo was created using:
+<!-- This logo was created using:
- Cloud Icon by Stephen Hutchings (CC BY-SA license)
- https://github.com/stephenhutchings/typicons.font
+ https://github.com/stephenhutchings/typicons.font
- Audiowide Font (SIL Open Font License 1.1)
https://www.google.com/fonts/specimen/Audiowide
-->
diff --git a/contrib/muchos-logo.svg b/contrib/muchos-logo.svg
index c5a4418..320887f 100644
--- a/contrib/muchos-logo.svg
+++ b/contrib/muchos-logo.svg
@@ -15,9 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<!-- This logo was created using:
+<!-- This logo was created using:
- Cloud Icon by Stephen Hutchings (CC BY-SA license)
- https://github.com/stephenhutchings/typicons.font
+ https://github.com/stephenhutchings/typicons.font
- Audiowide Font (SIL Open Font License 1.1)
https://www.google.com/fonts/specimen/Audiowide
-->
diff --git a/contrib/pom.xml b/contrib/pom.xml
new file mode 100644
index 0000000..48f9e2c
--- /dev/null
+++ b/contrib/pom.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.fluo</groupId>
+ <artifactId>muchos</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <name>Apache Fluo Muchos</name>
+ <description>This is a dummy POM for use with contrib/add-license-headers.md</description>
+</project>
diff --git a/lib/.gitignore b/lib/.gitignore
index 70e362f..1f67f4d 100644
--- a/lib/.gitignore
+++ b/lib/.gitignore
@@ -1 +1 @@
-boto*
+/boto*
