| { |
| "configurations" : [ |
| { |
| "capacity-scheduler" : { |
| "yarn.scheduler.capacity.root.default.maximum-am-resource-percent" : "0.5", |
| "yarn.scheduler.capacity.maximum-am-resource-percent" : "0.5" |
| } |
| }, |
| { |
| "cluster-env": { |
| "cluster_name": "PigView", |
| "smokeuser": "ambari-qa", |
| "user_group": "hadoop", |
| "security_enabled": "false" |
| } |
| }, |
| { |
| "core-site" : { |
| "hadoop.proxyuser.root.hosts" : "*", |
| "hadoop.proxyuser.root.groups" : "*", |
| "hadoop.proxyuser.hcat.hosts" : "*", |
| "hadoop.proxyuser.hcat.groups" : "*" |
| } |
| }, |
| { |
| "hadoop-env": { |
| "dtnode_heapsize" : "250", |
| "hadoop_heapsize" : "250", |
| "namenode_heapsize" : "250", |
| "namenode_opt_newsize": "50", |
| "namenode_opt_maxnewsize": "100", |
| "content" : "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}\n\n# Hadoop Configuration Directory\n#TODO: if env var set that can cause problems\nexport HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nHADOOP_DATANODE_OPTS=\"-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf\nfi\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n#Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64\n\n#Hadoop logging options\nexport HADOOP_ROOT_LOGGER={{hadoop_root_logger}}\n " |
| } |
| }, |
| { |
| "hive-env": { |
| "content" : "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\nexport HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra ibraries required for hive compilation/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelif [ -d \"/usr/lib/hive-webhcat/\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/lib/hive-webhcat/share/hcatalog/hive-hcatalog-core-*.jar\nelse\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\nfi\nexport METASTORE_PORT={{hive_metastore_port}}\n " |
| } |
| }, |
| { |
| "hbase-site" : { |
| "hbase.security.authorization": "true", |
| "hbase.rpc.engine": "org.apache.hadoop.hbase.ipc.SecureRpcEngine", |
| "hbase_master_heapsize": "250", |
| "hbase_regionserver_heapsize": "250", |
| "hbase.rpc.protection": "PRIVACY" |
| } |
| }, |
| { |
| "hdfs-site" : { |
| "dfs.block.size" : "34217472", |
| "dfs.replication" : "1", |
| "dfs.namenode.accesstime.precision" : "3600000", |
| "dfs.nfs3.dump.dir" : "/tmp/.hdfs-nfs", |
| "dfs.nfs.exports.allowed.hosts" : "* rw", |
| "dfs.datanode.max.xcievers" : "1024", |
| "dfs.block.access.token.enable" : "false" |
| } |
| }, |
| { |
| "hive-site" : { |
| "javax.jdo.option.ConnectionPassword" : "hive", |
| "hive.tez.container.size" : "250", |
| "hive.tez.java.opts" : "-server -Xmx200m -Djava.net.preferIPv4Stack=true", |
| "hive.heapsize" : "250", |
| "hive.users.in.admin.role" : "hive", |
| "hive_metastore_user_passwd" : "hive", |
| "hive.server2.enable.impersonation": "true", |
| "hive.compactor.check.interval": "300s", |
| "hive.compactor.initiator.on": "true", |
| "hive.compactor.worker.timeout": "86400s", |
| "hive.enforce.bucketing": "true", |
| "hive.support.concurrency": "true", |
| "hive.exec.dynamic.partition.mode": "nonstrict", |
| "hive.server2.enable.doAs": "false", |
| "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager", |
| "hive.txn.max.open.batch": "1000", |
| "hive.txn.timeout": "300", |
| "hive.security.authorization.enabled": "false" |
| } |
| }, |
| { |
| "mapred-env": { |
| "jobhistory_heapsize" : "250" |
| } |
| }, |
| { |
| "mapred-site" : { |
| "mapreduce.map.memory.mb" : "250", |
| "mapreduce.reduce.memory.mb" : "250", |
| "mapreduce.task.io.sort.mb" : "64", |
| "yarn.app.mapreduce.am.resource.mb" : "250", |
| "yarn.app.mapreduce.am.command-opts" : "-Xmx200m", |
| "mapred.job.reduce.memory.mb" : "250", |
| "mapred.child.java.opts" : "-Xmx200m", |
| "mapred.job.map.memory.mb" : "250", |
| "io.sort.mb" : "64", |
| "mapreduce.map.java.opts" : "-Xmx200m", |
| "mapreduce.reduce.java.opts" : "-Xmx200m" |
| } |
| }, |
| { |
| "nagios-env":{ |
| "nagios_contact" : "admin@localhost.localdomain", |
| "nagios_web_login" : "nagiosadmin", |
| "nagios_web_password" : "admin" |
| } |
| }, |
| { |
| "tez-site" : { |
| "tez.am.java.opts" : "-server -Xmx200m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC", |
| "tez.am.resource.memory.mb" : "250", |
| "tez.dag.am.resource.memory.mb" : "250", |
| "yarn.app.mapreduce.am.command-opts" : "-Xmx200m" |
| } |
| }, |
| { |
| "webhcat-site" : { |
| "webhcat.proxyuser.hcat.hosts" : "*", |
| "webhcat.proxyuser.hcat.groups" : "*" |
| } |
| }, |
| { |
| "yarn-env": { |
| "apptimelineserver_heapsize" : "250", |
| "resourcemanager_heapsize" : "250", |
| "nodemanager_heapsize" : "250", |
| "yarn_heapsize" : "250" |
| } |
| |
| }, |
| { |
| "yarn-site" : { |
| "yarn.nodemanager.resource.memory-mb": "2250", |
| "yarn.nodemanager.vmem-pmem-ratio" : "10", |
| "yarn.scheduler.minimum-allocation-mb" : "250", |
| "yarn.scheduler.maximum-allocation-mb" : "2250", |
| "yarn.nodemanager.pmem-check-enabled" : "false", |
| "yarn.acl.enable" : "false", |
| "yarn.resourcemanager.webapp.proxyuser.hcat.groups" : "*", |
| "yarn.resourcemanager.webapp.proxyuser.hcat.hosts" : "*" |
| } |
| } |
| ], |
| "host_groups" : [ |
| { |
| "name" : "single_host", |
| "components" : [ |
| {"name": "APP_TIMELINE_SERVER"}, |
| {"name" : "DATANODE"}, |
| {"name" : "GANGLIA_SERVER"}, |
| {"name" : "HDFS_CLIENT"}, |
| {"name" : "HISTORYSERVER"}, |
| {"name" : "HIVE_CLIENT"}, |
| {"name" : "HIVE_METASTORE"}, |
| {"name" : "HIVE_SERVER"}, |
| {"name" : "MAPREDUCE2_CLIENT"}, |
| {"name" : "MYSQL_SERVER"}, |
| {"name" : "NAGIOS_SERVER"}, |
| {"name" : "NAMENODE"}, |
| {"name" : "NODEMANAGER"}, |
| {"name" : "PIG"}, |
| {"name" : "RESOURCEMANAGER"}, |
| {"name" : "SECONDARY_NAMENODE"}, |
| {"name" : "TEZ_CLIENT"}, |
| {"name" : "WEBHCAT_SERVER"}, |
| {"name" : "YARN_CLIENT"}, |
| {"name" : "ZOOKEEPER_CLIENT"}, |
| {"name" : "ZOOKEEPER_SERVER"} |
| ], |
| "cardinality" : "1" |
| } |
| ], |
| "Blueprints" : { |
| "blueprint_name" : "pig-view", |
| "stack_name" : "HDP", |
| "stack_version" : "2.2" |
| } |
| } |
