| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # Set Hadoop-specific environment variables here. |
| |
| ## |
| ## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS. |
| ## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS. THEREFORE, |
| ## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE |
| ## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh. |
| ## |
| ## Precedence rules: |
| ## |
| ## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults |
| ## |
| ## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults |
| ## |
| |
| # Many of the options here are built from the perspective that users |
| # may want to provide OVERWRITING values on the command line. |
| # For example: |
| # |
| # JAVA_HOME=/usr/java/testing hdfs dfs -ls |
| # |
| # Therefore, the vast majority (BUT NOT ALL!) of these defaults |
| # are configured for substitution and not append. If append |
| # is preferable, modify this file accordingly. |
| |
| ### |
| # Generic settings for HADOOP |
| ### |
| |
| # Technically, the only required environment variable is JAVA_HOME. |
| # All others are optional. However, the defaults are probably not |
| # preferred. Many sites configure these options outside of Hadoop, |
| # such as in /etc/profile.d |
| |
| # The java implementation to use. By default, this environment |
| # variable is REQUIRED on ALL platforms except OS X! |
| # export JAVA_HOME= |
| |
| # Location of Hadoop. By default, Hadoop will attempt to determine |
| # this location based upon its execution path. |
| # export HADOOP_HOME= |
| |
| # Location of Hadoop's configuration information. i.e., where this |
| # file is living. If this is not defined, Hadoop will attempt to |
| # locate it based upon its execution path. |
| # |
| # NOTE: It is recommend that this variable not be set here but in |
| # /etc/profile.d or equivalent. Some options (such as |
| # --config) may react strangely otherwise. |
| # |
| # export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop |
| |
| # The maximum amount of heap to use (Java -Xmx). If no unit |
| # is provided, it will be converted to MB. Daemons will |
| # prefer any Xmx setting in their respective _OPT variable. |
| # There is no default; the JVM will autoscale based upon machine |
| # memory size. |
| # export HADOOP_HEAPSIZE_MAX= |
| |
| # The minimum amount of heap to use (Java -Xms). If no unit |
| # is provided, it will be converted to MB. Daemons will |
| # prefer any Xms setting in their respective _OPT variable. |
| # There is no default; the JVM will autoscale based upon machine |
| # memory size. |
| # export HADOOP_HEAPSIZE_MIN= |
| |
| # Enable extra debugging of Hadoop's JAAS binding, used to set up |
| # Kerberos security. |
| # export HADOOP_JAAS_DEBUG=true |
| |
| # Extra Java runtime options for all Hadoop commands. We don't support |
| # IPv6 yet/still, so by default the preference is set to IPv4. |
| # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true" |
| # For Kerberos debugging, an extended option set logs more invormation |
| # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug" |
| |
| # Some parts of the shell code may do special things dependent upon |
| # the operating system. We have to set this here. See the next |
| # section as to why.... |
| export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} |
| |
| |
| # Under certain conditions, Java on OS X will throw SCDynamicStore errors |
| # in the system logs. |
| # See HADOOP-8719 for more information. If one needs Kerberos |
| # support on OS X, one will want to change/remove this extra bit. |
| case ${HADOOP_OS_TYPE} in |
| Darwin*) |
| export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.realm= " |
| export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.kdc= " |
| export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.conf= " |
| ;; |
| esac |
| |
| # Extra Java runtime options for some Hadoop commands |
| # and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for |
| # such commands. In most cases, # this should be left empty and |
| # let users supply it on the command line. |
| # export HADOOP_CLIENT_OPTS="" |
| |
| # |
| # A note about classpaths. |
| # |
| # By default, Apache Hadoop overrides Java's CLASSPATH |
| # environment variable. It is configured such |
| # that it sarts out blank with new entries added after passing |
| # a series of checks (file/dir exists, not already listed aka |
| # de-deduplication). During de-depulication, wildcards and/or |
| # directories are *NOT* expanded to keep it simple. Therefore, |
| # if the computed classpath has two specific mentions of |
| # awesome-methods-1.0.jar, only the first one added will be seen. |
| # If two directories are in the classpath that both contain |
| # awesome-methods-1.0.jar, then Java will pick up both versions. |
| |
| # An additional, custom CLASSPATH. Site-wide configs should be |
| # handled via the shellprofile functionality, utilizing the |
| # hadoop_add_classpath function for greater control and much |
| # harder for apps/end-users to accidentally override. |
| # Similarly, end users should utilize ${HOME}/.hadooprc . |
| # This variable should ideally only be used as a short-cut, |
| # interactive way for temporary additions on the command line. |
| # export HADOOP_CLASSPATH="/some/cool/path/on/your/machine" |
| |
| # Should HADOOP_CLASSPATH be first in the official CLASSPATH? |
| # export HADOOP_USER_CLASSPATH_FIRST="yes" |
| |
| # If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along |
| # with the main jar are handled by a separate isolated |
| # client classloader when 'hadoop jar', 'yarn jar', or 'mapred job' |
| # is utilized. If it is set, HADOOP_CLASSPATH and |
| # HADOOP_USER_CLASSPATH_FIRST are ignored. |
| # export HADOOP_USE_CLIENT_CLASSLOADER=true |
| |
| # HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of |
| # system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER |
| # is enabled. Names ending in '.' (period) are treated as package names, and |
| # names starting with a '-' are treated as negative matches. For example, |
| # export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop." |
| |
| # Enable optional, bundled Hadoop features |
| # This is a comma delimited list. It may NOT be overridden via .hadooprc |
| # Entries may be added/removed as needed. |
| # export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@" |
| |
| ### |
| # Options for remote shell connectivity |
| ### |
| |
| # There are some optional components of hadoop that allow for |
| # command and control of remote hosts. For example, |
| # start-dfs.sh will attempt to bring up all NNs, DNS, etc. |
| |
| # Options to pass to SSH when one of the "log into a host and |
| # start/stop daemons" scripts is executed |
| # export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s" |
| |
| # The built-in ssh handler will limit itself to 10 simultaneous connections. |
| # For pdsh users, this sets the fanout size ( -f ) |
| # Change this to increase/decrease as necessary. |
| # export HADOOP_SSH_PARALLEL=10 |
| |
| # Filename which contains all of the hosts for any remote execution |
| # helper scripts # such as workers.sh, start-dfs.sh, etc. |
| # export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers" |
| |
| ### |
| # Options for all daemons |
| ### |
| # |
| |
| # |
| # Many options may also be specified as Java properties. It is |
| # very common, and in many cases, desirable, to hard-set these |
| # in daemon _OPTS variables. Where applicable, the appropriate |
| # Java property is also identified. Note that many are re-used |
| # or set differently in certain contexts (e.g., secure vs |
| # non-secure) |
| # |
| |
| # Where (primarily) daemon log files are stored. |
| # ${HADOOP_HOME}/logs by default. |
| # Java property: hadoop.log.dir |
| # export HADOOP_LOG_DIR=${HADOOP_HOME}/logs |
| |
| # A string representing this instance of hadoop. $USER by default. |
| # This is used in writing log and pid files, so keep that in mind! |
| # Java property: hadoop.id.str |
| # export HADOOP_IDENT_STRING=$USER |
| |
| # How many seconds to pause after stopping a daemon |
| # export HADOOP_STOP_TIMEOUT=5 |
| |
| # Where pid files are stored. /tmp by default. |
| # export HADOOP_PID_DIR=/tmp |
| |
| # Default log4j setting for interactive commands |
| # Java property: hadoop.root.logger |
| # export HADOOP_ROOT_LOGGER=INFO,console |
| |
| # Default log4j setting for daemons spawned explicitly by |
| # --daemon option of hadoop, hdfs, mapred and yarn command. |
| # Java property: hadoop.root.logger |
| # export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA |
| |
| # Default log level and output location for security-related messages. |
| # You will almost certainly want to change this on a per-daemon basis via |
| # the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the |
| # defaults for the NN and 2NN override this by default.) |
| # Java property: hadoop.security.logger |
| # export HADOOP_SECURITY_LOGGER=INFO,NullAppender |
| |
| # Default process priority level |
| # Note that sub-processes will also run at this level! |
| # export HADOOP_NICENESS=0 |
| |
| # Default name for the service level authorization file |
| # Java property: hadoop.policy.file |
| # export HADOOP_POLICYFILE="hadoop-policy.xml" |
| |
| # |
| # NOTE: this is not used by default! <----- |
| # You can define variables right here and then re-use them later on. |
| # For example, it is common to use the same garbage collection settings |
| # for all the daemons. So one could define: |
| # |
| # export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" |
| # |
| # .. and then use it as per the b option under the namenode. |
| |
| ### |
| # Secure/privileged execution |
| ### |
| |
| # |
| # Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons |
| # on privileged ports. This functionality can be replaced by providing |
| # custom functions. See hadoop-functions.sh for more information. |
| # |
| |
| # The jsvc implementation to use. Jsvc is required to run secure datanodes |
| # that bind to privileged ports to provide authentication of data transfer |
| # protocol. Jsvc is not required if SASL is configured for authentication of |
| # data transfer protocol using non-privileged ports. |
| # export JSVC_HOME=/usr/bin |
| |
| # |
| # This directory contains pids for secure and privileged processes. |
| #export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR} |
| |
| # |
| # This directory contains the logs for secure and privileged processes. |
| # Java property: hadoop.log.dir |
| # export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR} |
| |
| # |
| # When running a secure daemon, the default value of HADOOP_IDENT_STRING |
| # ends up being a bit bogus. Therefore, by default, the code will |
| # replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER. If one wants |
| # to keep HADOOP_IDENT_STRING untouched, then uncomment this line. |
| # export HADOOP_SECURE_IDENT_PRESERVE="true" |
| |
| ### |
| # NameNode specific parameters |
| ### |
| |
| # Default log level and output location for file system related change |
| # messages. For non-namenode daemons, the Java property must be set in |
| # the appropriate _OPTS if one wants something other than INFO,NullAppender |
| # Java property: hdfs.audit.logger |
| # export HDFS_AUDIT_LOGGER=INFO,NullAppender |
| |
| # Specify the JVM options to be used when starting the NameNode. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # a) Set JMX options |
| # export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" |
| # |
| # b) Set garbage collection logs |
| # export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" |
| # |
| # c) ... or set them directly |
| # export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" |
| |
| # this is the default: |
| # export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" |
| |
| ### |
| # SecondaryNameNode specific parameters |
| ### |
| # Specify the JVM options to be used when starting the SecondaryNameNode. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # This is the default: |
| # export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" |
| |
| ### |
| # DataNode specific parameters |
| ### |
| # Specify the JVM options to be used when starting the DataNode. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # This is the default: |
| # export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS" |
| |
| # On secure datanodes, user to run the datanode as after dropping privileges. |
| # This **MUST** be uncommented to enable secure HDFS if using privileged ports |
| # to provide authentication of data transfer protocol. This **MUST NOT** be |
| # defined if SASL is configured for authentication of data transfer protocol |
| # using non-privileged ports. |
| # This will replace the hadoop.id.str Java property in secure mode. |
| # export HDFS_DATANODE_SECURE_USER=hdfs |
| |
| # Supplemental options for secure datanodes |
| # By default, Hadoop uses jsvc which needs to know to launch a |
| # server jvm. |
| # export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server" |
| |
| ### |
| # NFS3 Gateway specific parameters |
| ### |
| # Specify the JVM options to be used when starting the NFS3 Gateway. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_NFS3_OPTS="" |
| |
| # Specify the JVM options to be used when starting the Hadoop portmapper. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_PORTMAP_OPTS="-Xmx512m" |
| |
| # Supplemental options for priviliged gateways |
| # By default, Hadoop uses jsvc which needs to know to launch a |
| # server jvm. |
| # export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server" |
| |
| # On privileged gateways, user to run the gateway as after dropping privileges |
| # This will replace the hadoop.id.str Java property in secure mode. |
| # export HDFS_NFS3_SECURE_USER=nfsserver |
| |
| ### |
| # ZKFailoverController specific parameters |
| ### |
| # Specify the JVM options to be used when starting the ZKFailoverController. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_ZKFC_OPTS="" |
| |
| ### |
| # QuorumJournalNode specific parameters |
| ### |
| # Specify the JVM options to be used when starting the QuorumJournalNode. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_JOURNALNODE_OPTS="" |
| |
| ### |
| # HDFS Balancer specific parameters |
| ### |
| # Specify the JVM options to be used when starting the HDFS Balancer. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_BALANCER_OPTS="" |
| |
| ### |
| # HDFS Mover specific parameters |
| ### |
| # Specify the JVM options to be used when starting the HDFS Mover. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_MOVER_OPTS="" |
| |
| ### |
| # Router-based HDFS Federation specific parameters |
| # Specify the JVM options to be used when starting the RBF Routers. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_DFSROUTER_OPTS="" |
| |
| ### |
| # HDFS Key Space Manager specific parameters |
| ### |
| # Specify the JVM options to be used when starting the HDFS Key Space Manager. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_KSM_OPTS="" |
| |
| ### |
| # HDFS StorageContainerManager specific parameters |
| ### |
| # Specify the JVM options to be used when starting the HDFS Storage Container Manager. |
| # These options will be appended to the options specified as HADOOP_OPTS |
| # and therefore may override any similar flags set in HADOOP_OPTS |
| # |
| # export HDFS_STORAGECONTAINERMANAGER_OPTS="" |
| |
| ### |
| # Advanced Users Only! |
| ### |
| |
| # |
| # When building Hadoop, one can add the class paths to the commands |
| # via this special env var: |
| # export HADOOP_ENABLE_BUILD_PATHS="true" |
| |
| # |
| # To prevent accidents, shell commands be (superficially) locked |
| # to only allow certain users to execute certain subcommands. |
| # It uses the format of (command)_(subcommand)_USER. |
| # |
| # For example, to limit who can execute the namenode command, |
| # export HDFS_NAMENODE_USER=hdfs |