| <?xml version="1.0" encoding="UTF-8"?> |
| <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> |
| <!-- |
| |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| |
| --> |
| |
| <!-- Put site-specific property overrides in this file. --> |
| |
| <configuration> |
| <property> |
| <name>yarn.nodemanager.local-dirs</name> |
| <value>{% for dir in worker_data_dirs -%} |
| {{ dir }}/hadoop/yarn/local |
| {%- if not loop.last -%} , {%- endif -%} |
| {%- endfor %}</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log-dirs</name> |
| <value>{{ worker_data_dirs[0] }}/hadoop/yarn/logs</value> |
| </property> |
| {% if 'spark' in groups %} |
| <property> |
| <name>yarn.nodemanager.aux-services</name> |
| <value>mapreduce_shuffle,spark_shuffle</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.aux-services.spark_shuffle.class</name> |
| <value>org.apache.spark.network.yarn.YarnShuffleService</value> |
| </property> |
| {% else %} |
| <property> |
| <name>yarn.nodemanager.aux-services</name> |
| <value>mapreduce_shuffle</value> |
| </property> |
| {% endif %} |
| <property> |
| <name>yarn.nodemanager.resource.memory-mb</name> |
| <value>{{ yarn_nm_mem_mb }}</value> |
| </property> |
| <property> |
| <name>yarn.scheduler.minimum-allocation-mb</name> |
| <value>128</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.vmem-check-enabled</name> |
| <value>false</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.vmem-pmem-ratio</name> |
| <value>3.1</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.delete.debug-delay-sec</name> |
| <value>259200</value> |
| </property> |
| <property> |
| <name>yarn.resourcemanager.scheduler.class</name> |
| <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value> |
| </property> |
| <property> |
| <name>twill.java.reserved.memory.mb</name> |
| <value>{{ twill_reserve_mem_mb }}</value> |
| </property> |
| {% if cluster_type == 'azure' and use_adlsg2 %} |
| <property> |
| <name>yarn.application.classpath</name> |
| <value>${HADOOP_HOME}/share/hadoop/tools/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/lib/*,${HADOOP_HOME}/share/hadoop/common/lib/*,${HADOOP_HOME}/share/hadoop/yarn/*,${HADOOP_HOME}/share/hadoop/yarn/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/*,${HADOOP_HOME}/share/hadoop/common/*,${HADOOP_HOME}/share/hadoop/mapreduce/*,${HADOOP_HOME}/share/hadoop/mapreduce/lib/*,${HADOOP_HOME}/share/hadoop/client/*</value> |
| </property> |
| {% endif %} |
| <property> |
| <name>yarn.log.server.url</name> |
| <value>http://{{ groups['resourcemanager'][0] }}:19888/jobhistory/logs</value> |
| </property> |
| <property> |
| <name>yarn.log-aggregation-enable</name> |
| <value>true</value> |
| </property> |
| <property> |
| <name>yarn.log-aggregation.retain-seconds</name> |
| <value>2592000</value> |
| </property> |
| <property> |
| <name>yarn.log.server.web-service.url</name> |
| <value>http://{{ groups['resourcemanager'][0] }}:8188/ws/v1/applicationhistory</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log-aggregation.compression-type</name> |
| <value>gz</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log-aggregation.debug-enabled</name> |
| <value>false</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name> |
| <value>30</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name> |
| <value>3600</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.log.retain-seconds</name> |
| <value>604800</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.remote-app-log-dir</name> |
| <value>/app-logs</value> |
| </property> |
| <property> |
| <name>yarn.nodemanager.remote-app-log-dir-suffix</name> |
| <value>logs</value> |
| </property> |
| |
| {% if hdfs_ha %} |
| <!-- RM HA Configurations --> |
| |
| <property> |
| <name>yarn.resourcemanager.ha.enabled</name> |
| <value>true</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.cluster-id</name> |
| <value>yarn-cluster</value> |
| </property> |
| |
| {% set rm_list = [] %} |
| {% for item in groups['resourcemanager'] %}{{ rm_list.append('rm' + loop.index|string() ) }}{% endfor %} |
| <property> |
| <name>yarn.resourcemanager.ha.rm-ids</name> |
| <value>{{ rm_list | join(',') }}</value> |
| </property> |
| |
| {% for rm_host in groups['resourcemanager'] %}{% set rm_id = 'rm' + loop.index|string() %} |
| <property> |
| <name>yarn.resourcemanager.hostname.{{ rm_id }}</name> |
| <value>{{ rm_host }}</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.webapp.address.{{ rm_id }}</name> |
| <value>{{ rm_host }}:8088</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.webapp.https.address.{{ rm_id }}</name> |
| <value>{{ rm_host }}:8090</value> |
| </property> |
| {% endfor %} |
| |
| {% if hadoop_major_version == '2' %} |
| <property> |
| <name>yarn.resourcemanager.zk-address</name> |
| <value>{{ zookeeper_connect }}</value> |
| </property> |
| {% elif hadoop_major_version == '3' %} |
| <property> |
| <name>hadoop.zk.address</name> |
| <value>{{ zookeeper_connect }}</value> |
| </property> |
| {% endif %} |
| |
| <!-- Below properties required for work-preserving RM restarts --> |
| |
| <property> |
| <name>yarn.resourcemanager.recovery.enabled</name> |
| <value>true</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.zk-state-store.parent-path</name> |
| <value>/rmstore</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.store.class</name> |
| <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.work-preserving-recovery.enabled</name> |
| <value>true</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name> |
| <value>10000</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.zk-num-retries</name> |
| <value>1000</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.zk-retry-interval-ms</name> |
| <value>1000</value> |
| </property> |
| |
| <property> |
| <name>yarn.client.failover-proxy-provider</name> |
| <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name> |
| <value>/yarn-leader-election</value> |
| </property> |
| |
| {% else %} |
| |
| <property> |
| <name>yarn.resourcemanager.webapp.address</name> |
| <value>{{ groups['resourcemanager'][0] }}:8088</value> |
| </property> |
| |
| <property> |
| <name>yarn.resourcemanager.hostname</name> |
| <value>{{ groups['resourcemanager'][0] }}</value> |
| </property> |
| {% endif %} |
| |
| </configuration> |