| # |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| # |
| class hdp-hadoop::params( |
| ) inherits hdp::params |
| { |
| |
| ##TODO: for testing in masterless mode |
| $use_preconditions = false |
| #### |
| $conf_dir = $hdp::params::hadoop_conf_dir |
| |
| ####### users |
| |
| $mapred_user = $hdp::params::mapred_user |
| $hdfs_user = $hdp::params::hdfs_user |
| |
| ##### security related |
| $keytab_path = hdp_default("keytab_path","/etc/security/keytabs") |
| |
| if ($hdp::params::security_enabled == true) { |
| $enable_security_authorization = true |
| $security_type = "kerberos" |
| $task_controller = "org.apache.hadoop.mapred.LinuxTaskController" |
| $dfs_datanode_address = 1019 |
| $dfs_datanode_http_address = 1022 |
| } else { |
| $enable_security_authorization = false |
| $security_type = "simple" |
| $task_controller = "org.apache.hadoop.mapred.DefaultTaskController" |
| $dfs_datanode_address = 50075 |
| $dfs_datanode_http_address = 50075 |
| } |
| |
| ### hadoop-env |
| |
| $dtnode_heapsize = hdp_default("hadoop/hadoop-env/dtnode_heapsize","1024m") |
| $ttnode_heapsize = hdp_default("hadoop/hadoop-env/ttnode_heapsize","1024m") |
| |
| $hadoop_heapsize = hdp_default("hadoop/hadoop-env/hadoop_heapsize","1024m") |
| |
| $hdfs_log_dir_prefix = hdp_default("hadoop/hadoop-env/hdfs_log_dir_prefix","/var/log/hadoop") |
| |
| $hadoop_pid_dir_prefix = hdp_default("hadoop/hadoop-env/hadoop_pid_dir_prefix","/var/run/hadoop") |
| $run_dir = $hadoop_pid_dir_prefix |
| |
| $namenode_formatted_mark_dir = "${run_dir}/hdfs/namenode/formatted/" |
| |
| $jtnode_heapsize = hdp_default("hadoop/hadoop-env/jtnode_heapsize","1024m") |
| |
| $jtnode_opt_maxnewsize = hdp_default("hadoop/hadoop-env/jtnode_opt_maxnewsize","200m") |
| |
| $jtnode_opt_newsize = hdp_default("hadoop/hadoop-env/jtnode_opt_newsize","200m") |
| |
| $namenode_heapsize = hdp_default("hadoop/hadoop-env/namenode_heapsize","1024m") |
| |
| $namenode_opt_maxnewsize = hdp_default("hadoop/hadoop-env/namenode_opt_maxnewsize","640m") |
| |
| $namenode_opt_newsize = hdp_default("hadoop/hadoop-env/namenode_opt_newsize","640m") |
| |
| ### compression related |
| if (($hdp::params::lzo_enabled == true) and ($hdp::params::snappy_enabled == true)) { |
| $mapred_compress_map_output = true |
| $compression_codecs = "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec" |
| $mapred_map_output_compression_codec = "org.apache.hadoop.io.compress.SnappyCodec" |
| } elsif ($hdp::params::snappy_enabled == true) { |
| $mapred_compress_map_output = true |
| $compression_codecs = "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec" |
| $mapred_map_output_compression_codec = "org.apache.hadoop.io.compress.SnappyCodec" |
| } elsif ($hdp::params::lzo_enabled == true) { |
| $mapred_compress_map_output = true |
| $compression_codecs = "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec" |
| $mapred_map_output_compression_codec = "com.hadoop.compression.lzo.LzoCodec" |
| } else { |
| $mapred_compress_map_output = false |
| $compression_codecs = "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec" |
| $mapred_map_output_compression_codec = "org.apache.hadoop.io.compress.DefaultCodec" |
| } |
| |
| ### core-site |
| $fs_checkpoint_dir = hdp_default("hadoop/core-site/fs_checkpoint_dir","/tmp/hadoop-hdfs/dfs/namesecondary") |
| |
| $proxyuser_group = hdp_default("hadoop/core-site/proxyuser_group","users") |
| |
| ### hdfs-site |
| $datanode_du_reserved = hdp_default("hadoop/hdfs-site/datanode_du_reserved",1073741824) |
| |
| $dfs_block_local_path_access_user = hdp_default("hadoop/hdfs-site/dfs_block_local_path_access_user","hbase") |
| |
| $dfs_data_dir = $hdp::params::dfs_data_dir |
| |
| $dfs_datanode_data_dir_perm = hdp_default("hadoop/hdfs-site/dfs_datanode_data_dir_perm",750) |
| |
| $dfs_datanode_failed_volume_tolerated = hdp_default("hadoop/hdfs-site/dfs_datanode_failed_volume_tolerated",0) |
| |
| $dfs_exclude = hdp_default("hadoop/hdfs-site/dfs_exclude","dfs.exclude") |
| |
| $dfs_include = hdp_default("hadoop/hdfs-site/dfs_include","dfs.include") |
| |
| $dfs_name_dir = hdp_default("hadoop/hdfs-site/dfs_name_dir","/tmp/hadoop-hdfs/dfs/name") |
| |
| $dfs_replication = hdp_default("hadoop/hdfs-site/dfs_replication",3) |
| |
| $dfs_support_append = hdp_default("hadoop/hdfs-site/dfs_support_append",true) |
| |
| $dfs_webhdfs_enabled = hdp_default("hadoop/hdfs-site/dfs_webhdfs_enabled",false) |
| |
| |
| ######### mapred ####### |
| ### mapred-site |
| |
| $mapred_system_dir = '/mapred/system' |
| |
| $io_sort_mb = hdp_default("hadoop/mapred-site/io_sort_mb","200") |
| |
| $io_sort_spill_percent = hdp_default("hadoop/mapred-site/io_sort_spill_percent","0.9") |
| |
| $mapred_child_java_opts_sz = hdp_default("hadoop/mapred-site/mapred_child_java_opts_sz","-Xmx768m") |
| |
| $mapred_cluster_map_mem_mb = hdp_default("hadoop/mapred-site/mapred_cluster_map_mem_mb","-1") |
| |
| $mapred_cluster_max_map_mem_mb = hdp_default("hadoop/mapred-site/mapred_cluster_max_map_mem_mb","-1") |
| |
| $mapred_cluster_max_red_mem_mb = hdp_default("hadoop/mapred-site/mapred_cluster_max_red_mem_mb","-1") |
| |
| $mapred_cluster_red_mem_mb = hdp_default("hadoop/mapred-site/mapred_cluster_red_mem_mb","-1") |
| |
| $mapred_hosts_exclude = hdp_default("hadoop/mapred-site/mapred_hosts_exclude","mapred.exclude") |
| |
| $mapred_hosts_include = hdp_default("hadoop/mapred-site/mapred_hosts_include","mapred.include") |
| |
| $mapred_job_map_mem_mb = hdp_default("hadoop/mapred-site/mapred_job_map_mem_mb","-1") |
| |
| $mapred_job_red_mem_mb = hdp_default("hadoop/mapred-site/mapred_job_red_mem_mb","-1") |
| |
| $mapred_jobstatus_dir = hdp_default("hadoop/mapred-site/mapred_jobstatus_dir","file:////mapred/jobstatus") |
| |
| $mapred_local_dir = hdp_default("hadoop/mapred-site/mapred_local_dir","/tmp/hadoop-mapred/mapred/local") |
| |
| $mapred_map_tasks_max = hdp_default("hadoop/mapred-site/mapred_map_tasks_max",4) |
| |
| $mapred_red_tasks_max = hdp_default("hadoop/mapred-site/mapred_red_tasks_max",4) |
| |
| $mapreduce_userlog_retainhours = hdp_default("hadoop/mapred-site/mapreduce_userlog_retainhours",24) |
| |
| $maxtasks_per_job = hdp_default("hadoop/mapred-site/maxtasks_per_job","-1") |
| |
| $scheduler_name = hdp_default("hadoop/mapred-site/scheduler_name","org.apache.hadoop.mapred.CapacityTaskScheduler") |
| |
| #### health_check |
| |
| $security_enabled = $hdp::params::security_enabled |
| |
| $task_bin_exe = hdp_default("hadoop/health_check/task_bin_exe") |
| |
| $rca_enabled = hdp_default("rca_enabled", false) |
| if ($rca_enabled == true) { |
| $rca_prefix = "" |
| } else { |
| $rca_prefix = "###" |
| } |
| $ambari_db_server_host = hdp_default("ambari_db_server_host", "localhost") |
| } |
