hmc/puppet/modules/hdp2-hadoop/templates/mapred-site.xml.erb

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!--
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration xmlns:xi="http://www.w3.org/2001/XInclude">

<!-- MR AM properties -->

  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property> 

  <property>
   <name>yarn.app.mapreduce.am.staging-dir</name>
   <value>/user</value>
  </property>

  <property>
   <name>yarn.app.mapreduce.am.command-opts</name>
   <value>-Xmx1024m</value>
  </property>

  <property>
   <name>mapreduce.job.hdfs-servers</name>
   <value>${fs.defaultFS}</value>
  </property>
       
  <property>
    <name>mapreduce.cluster.local.dir</name>
    <value><%=scope.function_hdp_template_var("mapreduce_cluster_local_dir")%></value>
    <description>No description</description>
    <final>true</final>
  </property>

  <property>
    <name>mapred.map.tasks.speculative.execution</name>
    <value>false</value>
    <description>If true, then multiple instances of some map tasks
               may be executed in parallel.</description>
  </property>

  <property>
    <name>mapred.reduce.tasks.speculative.execution</name>
    <value>false</value>
    <description>If true, then multiple instances of some reduce tasks
               may be executed in parallel.</description>
  </property>

  <property>
    <name>mapred.reduce.slowstart.completed.maps</name>
    <value>0.05</value>
  </property>

  <property>
    <name>mapred.output.compression.type</name>
    <value>BLOCK</value>
    <description>If the job outputs are to compressed as SequenceFiles, how should
                 they be compressed? Should be one of NONE, RECORD or BLOCK.
    </description>
  </property>

  <property>
    <name>mapred.task.timeout</name>
    <value>600000</value>
    <description>The number of milliseconds before a task will be
  terminated if it neither reads an input, writes an output, nor
  updates its status string.
  </description>
  </property>

  <property>
    <name>jetty.connector</name>
    <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
    <description>No description</description>
  </property>

  <property>
    <name>mapred.child.root.logger</name>
    <value>INFO,TLA</value>
  </property>

  <property>
    <name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
    <value>false</value>
  </property>

  <property>
    <name>mapreduce.admin.user.env</name>
    <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native/`$JAVA_HOME/bin/java -d32 -version &amp;&gt; /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value>
  </property>
    
  <property>
    <name>mapreduce.job.acl-view-job</name>
    <value>*</value>
  </property>

<!-- i/o properties -->

  <property>
    <name>io.sort.mb</name>
    <value><%=scope.function_hdp_template_var("mapreduce_io_sort_mb")%></value>
    <description>No description</description>
  </property>

  <property>
    <name>io.sort.spill.percent</name>
    <value><%=scope.function_hdp_template_var("io_sort_spill_percent")%></value>
    <description>No description</description>
  </property>

  <property>
    <name>io.sort.factor</name>
    <value>100</value>
    <description>No description</description>
  </property>

<!-- map tasks' properties -->

  <property>
    <name>mapred.map.child.java.opts</name>
    <value><%=scope.function_hdp_template_var("mapred_map_child_java_opts_sz")%></value>
  </property>

  <property>
    <name>mapreduce.admin.map.child.java.opts</name>
    <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true</value>
    <final>true</final>
  </property>

  <property>
    <name>mapred.job.map.memory.mb</name>
    <value><%=scope.function_hdp_template_var("mapred_job_map_mem_mb")%></value>
  </property>

  <property>
    <name>mapred.map.output.compression.codec</name>
    <value><%=scope.function_hdp_template_var("mapred_map_output_compression_codec")%></value>
    <description>If the map outputs are compressed, how should they be
      compressed
    </description>
  </property>

  <property>
    <name>mapred.compress.map.output</name>
    <value><%=scope.function_hdp_template_var("mapred_compress_map_output")%></value>
  </property>

<!-- reduce tasks' properties -->

  <property>
    <name>mapred.reduce.child.java.opts</name>
    <value><%=scope.function_hdp_template_var("mapred_reduce_child_java_opts_sz")%></value>
  </property>

  <property>
    <name>mapreduce.admin.reduce.child.java.opts</name>
    <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true</value>
    <final>true</final>
  </property>

  <property>
    <name>mapred.job.reduce.memory.mb</name>
    <value><%=scope.function_hdp_template_var("mapred_job_red_mem_mb")%></value>
  </property>

  <property>
    <name>mapred.reduce.parallel.copies</name>
    <value>30</value>
    <description>No description</description>
  </property>

  <property>
    <name>mapred.inmem.merge.threshold</name>
    <value>1000</value>
    <description>The threshold, in terms of the number of files
  for the in-memory merge process. When we accumulate threshold number of files
  we initiate the in-memory merge and spill to disk. A value of 0 or less than
  0 indicates we want to DON'T have any threshold and instead depend only on
  the ramfs's memory consumption to trigger the merge.
  </description>
  </property>

  <property>
    <name>mapred.job.shuffle.merge.percent</name>
    <value>0.66</value>
    <description>The usage threshold at which an in-memory merge will be
  initiated, expressed as a percentage of the total memory allocated to
  storing in-memory map outputs, as defined by
  mapred.job.shuffle.input.buffer.percent.
  </description>
  </property>

  <property>
    <name>mapred.job.shuffle.input.buffer.percent</name>
    <value>0.7</value>
    <description>The percentage of memory to be allocated from the maximum heap
  size to storing map outputs during the shuffle.
  </description>
  </property>

  <property>
    <name>mapreduce.reduce.input.buffer.percent</name>
    <value>0.0</value>
    <description>The percentage of memory- relative to the maximum heap size- to
  retain map outputs during the reduce. When the shuffle is concluded, any
  remaining map outputs in memory must consume less than this threshold before
  the reduce can begin.
  </description>
  </property>

  <property>
    <name>mapreduce.reduce.input.limit</name>
    <value>10737418240</value>
    <description>The limit on the input size of the reduce. (This value
    is 10 Gb.)  If the estimated input size of the reduce is greater than
    this value, job is failed. A value of -1 means that there is no limit
    set. </description>
  </property>

  <!-- JobHistory Server -->

  <property>
   <name>mapreduce.jobhistory.intermediate-done-dir</name>
   <value>/mapred/history/done_intermediate</value>
  </property>

  <property>
   <name>mapreduce.jobhistory.done-dir</name>
   <value>/mapred/history/done</value>
  </property>
    
  <property>
    <name>mapreduce.jobhistory.address</name>
    <value><%=scope.function_hdp_host("yarn_rm_host")%>:10020</value>
  </property>

  <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value><%=scope.function_hdp_host("yarn_rm_host")%>:19888</value>
  </property>

  <property>
    <name>yarn.app.mapreduce.am.create-intermediate-jh-base-dir</name>
    <value>false</value>
  </property>

</configuration>