bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Licensed to the Apache Software Foundation (ASF) under one or more       -->
<!-- contributor license agreements.  See the NOTICE file distributed with    -->
<!-- this work for additional information regarding copyright ownership.      -->
<!-- The ASF licenses this file to You under the Apache License, Version 2.0  -->
<!-- (the "License"); you may not use this file except in compliance with     -->
<!-- the License.  You may obtain a copy of the License at                    -->
<!--                                                                          -->
<!--     http://www.apache.org/licenses/LICENSE-2.0                           -->
<!--                                                                          -->
<!-- Unless required by applicable law or agreed to in writing, software      -->
<!-- distributed under the License is distributed on an "AS IS" BASIS,        -->
<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
<!-- See the License for the specific language governing permissions and      -->
<!-- limitations under the License.                                           -->

<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>

<% if ha == "enabled" -%>
  <!-- HA -->
  <property> 
    <name>dfs.federation.nameservices</name>
    <value><%= nameservice_id %></value>
  </property>

  <property>
    <name>dfs.ha.namenodes.<%= nameservice_id %></name>
    <value><%= (1..namenode_hosts.length).map { |n| "nn#{n}" }.join(",") %></value>
  </property>

<%   namenode_hosts.each_with_index do |host,idx| -%>
  <property>
    <name>dfs.namenode.rpc-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= host %>:<%= hadoop_namenode_port %></value>
  </property>
 
  <property>
    <name>dfs.namenode.http-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
    <value><%= host %>:50070</value>
  </property>

<%   end -%>
<%   if has_variable?("shared_edits_dir") -%>
  <property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value><%= shared_edits_dir %></value>
  </property>

<%   end -%>
  <property>
    <name>dfs.client.failover.proxy.provider.<%= nameservice_id %></name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>

<%   if has_variable?("sshfence_privkey") -%>
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value>sshfence(<%= sshfence_user %>)</value>
  </property>

  <property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value><%= sshfence_keypath %></value>
  </property>

<%   else -%>
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value></value>
  </property>

<%   end -%>
<% elsif hadoop_security_authentication == "kerberos" -%>
  <property>
    <name>dfs.block.access.token.enable</name>
    <value>true</value>
  </property>
  
  <!-- NameNode security config -->
  <property>
    <name>dfs.https.address</name>
    <value><%= namenode_hosts[0] %>:50475</value>
  </property>
  <property>
    <name>dfs.https.port</name>
    <value>50475</value>
  </property>
  <property>
    <name>dfs.namenode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.namenode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.namenode.kerberos.https.principal</name>
    <value>host/_HOST@<%= kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.web.authentication.kerberos.keytab</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.web.authentication.kerberos.principal</name>
    <value>HTTP/_HOST@<%= kerberos_realm %></value>
  </property>

  
  <!-- Secondary NameNode security config -->
  <property>
    <name>dfs.secondary.http.address</name>
    <value><%= namenode_hosts[0] %>:0</value>
  </property>
  <property>
    <name>dfs.secondary.https.address</name>
    <value><%= namenode_hosts[0] %>:50495</value>
  </property>
  <property>
    <name>dfs.secondary.https.port</name>
    <value>50495</value>
  </property>
  <property>
    <name>dfs.secondary.namenode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.secondary.namenode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.secondary.namenode.kerberos.https.principal</name>
    <value>host/_HOST@<%= kerberos_realm %></value>
  </property>
  
  <!-- DataNode security config -->
  <property>
    <name>dfs.datanode.data.dir.perm</name>
    <value>700</value> 
  </property>
  <property>
    <name>dfs.datanode.address</name>
    <value>0.0.0.0:1004</value>
  </property>
  <property>
    <name>dfs.datanode.http.address</name>
    <value>0.0.0.0:1006</value>
  </property>
  <property>
    <name>dfs.datanode.keytab.file</name>
    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
  </property>
  <property>
    <name>dfs.datanode.kerberos.principal</name>
    <value>hdfs/_HOST@<%= kerberos_realm %></value>
  </property>
  <property>
    <name>dfs.datanode.kerberos.https.principal</name>
    <value>host/_HOST@<%= kerberos_realm %></value>
  </property>

<% end -%>
  <!-- name node -->
  <property>
    <name>dfs.data.dir</name>
    <value><%= hdfs_data_dirs.join(",") %></value>
  </property>

<% if has_variable?("hdfs_support_append") %>
  <property>
    <name>dfs.support.append</name>
    <value><%= hdfs_support_append %></value>
  </property>
<% end %>

<% if has_variable?("hdfs_shortcut_reader_user") %>
  <property>
    <name>dfs.client.read.shortcircuit</name>
    <value>true</value>
  </property>

  <property>
    <name>dfs.block.local-path-access.user</name>
    <value><%= hdfs_shortcut_reader_user %></value>
  </property>
<% end %>
 
  <property>
    <name>dfs.name.dir</name>
    <value><%= namenode_data_dirs.join(",") %></value>
  </property>

  <property>
    <name>dfs.permissions.superusergroup</name>
    <value>hadoop</value>
    <description>The name of the group of super-users.</description>
  </property>

  <!-- Enable Hue plugins -->
<% if has_variable?("hadoop_dfs_namenode_plugins") -%>
  <property>
    <name>dfs.namenode.plugins</name>
    <value><%= hadoop_dfs_namenode_plugins %></value>
    <description>Comma-separated list of namenode plug-ins to be activated.
    </description>
  </property>

<% end -%>
<% if has_variable?("hadoop_dfs_datanode_plugins") -%>
  <property>
    <name>dfs.datanode.plugins</name>
    <value><%= hadoop_dfs_datanode_plugins %></value>
    <description>Comma-separated list of datanode plug-ins to be activated.
    </description>
  </property>

<% end -%>
<% if has_variable?("hadoop_namenode_thrift_port") -%>
  <property>
    <name>dfs.thrift.address</name>
    <value>0.0.0.0:<%= hadoop_namenode_thrift_port %></value>
  </property>

<% end -%>
  <!-- increase the number of datanode transceivers way above the default of 256
     - this is for hbase -->
  <property>
    <name>dfs.datanode.max.xcievers</name>
    <value>4096</value>
  </property>

  <!-- Configurations for large cluster -->
<% if has_variable?("hadoop_config_dfs_block_size") -%>
  <property>
    <name>dfs.block.size</name>
    <value><%= hadoop_config_dfs_block_size %></value>
  </property>

<% end -%>
<% if has_variable?("hadoop_config_namenode_handler_count") -%>
  <property>
    <name>dfs.namenode.handler.count</name>
    <value><%= hadoop_config_namenode_handler_count %></value>
  </property>

<% end -%>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
  </property>

</configuration>