| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| $roles_map = { |
| hdfs-non-ha => { |
| master => ["namenode"], |
| worker => ["datanode"], |
| standby => ["secondarynamenode"], |
| }, |
| hdfs-ha => { |
| master => ["namenode"], |
| worker => ["datanode"], |
| standby => ["standby-namenode"], |
| }, |
| yarn => { |
| master => ["resourcemanager"], |
| worker => ["nodemanager"], |
| client => ["hadoop-client"], |
| # mapred is the default app which runs on yarn. |
| library => ["mapred-app"], |
| }, |
| mapred => { |
| library => ["mapred-app"], |
| }, |
| hbase => { |
| master => ["hbase-master"], |
| worker => ["hbase-server"], |
| client => ["hbase-client"], |
| }, |
| ignite_hadoop => { |
| worker => ["ignite-server"], |
| }, |
| solrcloud => { |
| worker => ["solr-server"], |
| }, |
| spark => { |
| master => ["spark-master"], |
| worker => ["spark-worker"], |
| }, |
| tachyon => { |
| master => ["tachyon-master"], |
| worker => ["tachyon-worker"], |
| }, |
| flume => { |
| worker => ["flume-agent"], |
| }, |
| kerberos => { |
| master => ["kerberos-server"], |
| }, |
| oozie => { |
| master => ["oozie-server"], |
| client => ["oozie-client"], |
| }, |
| hcat => { |
| master => ["hcatalog-server"], |
| gateway_server => ["webhcat-server"], |
| }, |
| sqoop => { |
| gateway_server => ["sqoop-server"], |
| client => ["sqoop-client"], |
| }, |
| httpfs => { |
| gateway_server => ["httpfs-server"], |
| }, |
| hue => { |
| gateway_server => ["hue-server"], |
| }, |
| mahout => { |
| client => ["mahout-client"], |
| }, |
| giraph => { |
| client => ["giraph-client"], |
| }, |
| crunch => { |
| client => ["crunch-client"], |
| }, |
| pig => { |
| client => ["pig-client"], |
| }, |
| hive => { |
| client => ["hive-client"], |
| }, |
| tez => { |
| client => ["tez-client"], |
| }, |
| zookeeper => { |
| worker => ["zookeeper-server"], |
| client => ["zookeeper-client"], |
| }, |
| ycsb => { |
| client => ["ycsb-client"], |
| }, |
| zeppelin => { |
| master => ["zeppelin-server"], |
| }, |
| hawq => { |
| master => ["hawq"], |
| }, |
| } |
| |
| class hadoop_cluster_node ( |
| $hadoop_security_authentication = hiera("hadoop::hadoop_security_authentication", "simple"), |
| $bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ], |
| $cluster_components = ["all"] |
| ) { |
| |
| user { $bigtop_real_users: |
| ensure => present, |
| system => false, |
| managehome => true, |
| } |
| |
| if ($hadoop_security_authentication == "kerberos") { |
| kerberos::host_keytab { $bigtop_real_users: } |
| User<||> -> Kerberos::Host_keytab<||> |
| include kerberos::client |
| } |
| |
| $hadoop_head_node = hiera("bigtop::hadoop_head_node") |
| $standby_head_node = hiera("bigtop::standby_head_node", "") |
| $hadoop_gateway_node = hiera("bigtop::hadoop_gateway_node", $hadoop_head_node) |
| |
| $ha_enabled = $standby_head_node ? { |
| "" => false, |
| default => true, |
| } |
| |
| # look into alternate hiera datasources configured using this path in |
| # hiera.yaml |
| $hadoop_hiera_ha_path = $ha_enabled ? { |
| false => "noha", |
| true => "ha", |
| } |
| } |
| |
| class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_node { |
| define deploy_module($roles) { |
| class { "${name}::deploy": |
| roles => $roles, |
| } |
| } |
| |
| $modules = [ |
| "crunch", |
| "giraph", |
| "hadoop", |
| "hadoop_hbase", |
| "ignite_hadoop", |
| "hadoop_flume", |
| "hadoop_hive", |
| "hadoop_oozie", |
| "hadoop_pig", |
| "hawq", |
| "sqoop2", |
| "hadoop_zookeeper", |
| "hcatalog", |
| "hue", |
| "mahout", |
| "solr", |
| "spark", |
| "tachyon", |
| "tez", |
| "ycsb", |
| "kerberos", |
| "zeppelin" |
| ] |
| |
| deploy_module { $modules: |
| roles => $roles, |
| } |
| } |
| |
| class node_with_components inherits hadoop_cluster_node { |
| |
| # Ensure (even if a single value) that the type is an array. |
| if (is_array($cluster_components)) { |
| $components_array = $cluster_components |
| } else { |
| if ($cluster_components == undef) { |
| $components_array = ["all"] |
| } else { |
| $components_array = [$cluster_components] |
| } |
| } |
| |
| $given_components = $components_array[0] ? { |
| "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]), |
| default => $components_array, |
| } |
| $ha_dependent_components = $ha_enabled ? { |
| true => ["hdfs-ha"], |
| default => ["hdfs-non-ha"], |
| } |
| $components = concat($given_components, $ha_dependent_components) |
| |
| $master_role_types = ["master", "worker", "library"] |
| $standby_role_types = ["standby", "library"] |
| $worker_role_types = ["worker", "library"] |
| $gateway_role_types = ["client", "gateway_server"] |
| |
| if ($::fqdn == $hadoop_head_node or $::fqdn == $hadoop_gateway_node) { |
| if ($hadoop_gateway_node == $hadoop_head_node) { |
| $role_types = concat($master_role_types, $gateway_role_types) |
| } elsif ($::fqdn == $hadoop_head_node) { |
| $role_types = $master_role_types |
| } else { |
| $role_types = $gateway_role_types |
| } |
| } elsif ($::fqdn == $standby_head_node) { |
| $role_types = $standby_role_types |
| } else { |
| $role_types = $worker_role_types |
| } |
| |
| $roles = get_roles($components, $role_types, $roles_map) |
| |
| class { 'node_with_roles': |
| roles => $roles, |
| } |
| } |