recipes/hadoop-yarn-ec2.properties

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Hadoop Cluster on AWS EC2
# 

# Read the Configuration Guide for more info:
# http://whirr.apache.org/docs/latest/configuration-guide.html

# Change the cluster name here
whirr.cluster-name=hadoop-yarn

# Expert: specify the version of Hadoop to install.
whirr.hadoop.version=0.23.5
whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz

# Change the number of machines in the cluster here
whirr.instance-templates=1 hadoop-namenode+yarn-resourcemanager+mapreduce-historyserver,1 hadoop-datanode+yarn-nodemanager

# We need to use modified scripts for the installation since it has changed
# significantly since 0.20.x
whirr.hadoop.install-function=install_hadoop_mr2
whirr.hadoop.configure-function=configure_hadoop_mr2

whirr.java.install-function=install_openjdk
hadoop-env.JAVA_HOME=/usr/lib/jvm/java-6-openjdk

# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
whirr.provider=aws-ec2
whirr.identity=${env:AWS_ACCESS_KEY_ID}
whirr.credential=${env:AWS_SECRET_ACCESS_KEY}

# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
whirr.hardware-id=m1.large
# Ubuntu 10.04 LTS Lucid. See http://alestic.com/
#whirr.image-id=us-east-1/ami-da0cf8b3
# If you choose a different location, make sure whirr.image-id is updated too
whirr.location-id=us-east-1

# 64 bit required, and either Ubuntu 10.04 or CentOS/RHEL 5.x. 
whirr.template=osFamily=UBUNTU,osVersionMatches=10.04,os64Bit=true,minRam=2048

# You can also specify the spot instance price
# http://aws.amazon.com/ec2/spot-instances/
# whirr.aws-ec2-spot-price=0.15

# By default use the user system SSH keys. Override them here.
# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
# whirr.public-key-file=${whirr.private-key-file}.pub

# Expert: override Hadoop properties by setting properties with the prefix
# hadoop-common, hadoop-hdfs, hadoop-mapreduce to set Common, HDFS, MapReduce
# site properties, respectively. The prefix is removed by Whirr, so that for
# example, setting 
# hadoop-common.fs.trash.interval=1440
# will result in fs.trash.interval being set to 1440 in core-site.xml.
hadoop-yarn.yarn.nodemanager.log-dirs=/tmp/nm-logs
hadoop-yarn.yarn.nodemanager.remote-app-log-dir=/tmp/nm-remote-app-logs
hadoop-yarn.yarn.nodemanager.aux-services=mapreduce.shuffle
hadoop-yarn.yarn.nodemanager.aux-services.mapreduce.shuffle.class=org.apache.hadoop.mapred.ShuffleHandler
hadoop-yarn.yarn.nodemanager.delete.debug-delay-sec=6000
hadoop-yarn.yarn.app.mapreduce.am.staging-dir=/user
hadoop-mapreduce.mapreduce.framework.name=yarn
hadoop-common.ipc.client.connect.max.retries=100

# Hadoop environment var overrides
hadoop-env.JAVA_HOME=/usr/lib/jvm/java-6-openjdk
yarn-env.YARN_CONF_DIR=/usr/local/hadoop-\${whirr.hadoop.version}/etc/hadoop

# Must be set for Hadoop-2.0.x or trunk to run MRv2 on YARN
hadoop-common.hadoop.tmp.dir=/data/tmp/hadoop-\${user.name}
hadoop-yarn.yarn.nodemanager.local-dirs=/data/tmp/hadoop-\${user.name}
# Set this to the max you want to allocate per compute node
hadoop-yarn.yarn.nodemanager.resource.memory-mb=4096