| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.util; |
| |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.io.InputStreamReader; |
| import java.io.InterruptedIOException; |
| import java.nio.charset.Charset; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.HashSet; |
| import java.util.Map; |
| import java.util.Objects; |
| import java.util.Set; |
| import java.util.Timer; |
| import java.util.TimerTask; |
| import java.util.WeakHashMap; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| |
| import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; |
| import org.apache.hadoop.classification.InterfaceAudience; |
| import org.apache.hadoop.classification.InterfaceStability; |
| import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * A base class for running a Shell command. |
| * |
| * <code>Shell</code> can be used to run shell commands like <code>du</code> or |
| * <code>df</code>. It also offers facilities to gate commands by |
| * time-intervals. |
| */ |
| @InterfaceAudience.Public |
| @InterfaceStability.Evolving |
| public abstract class Shell { |
| private static final Map<Shell, Object> CHILD_SHELLS = |
| Collections.synchronizedMap(new WeakHashMap<Shell, Object>()); |
| public static final Logger LOG = LoggerFactory.getLogger(Shell.class); |
| |
| /** |
| * Text to include when there are windows-specific problems. |
| * {@value} |
| */ |
| private static final String WINDOWS_PROBLEMS = |
| "https://wiki.apache.org/hadoop/WindowsProblems"; |
| |
| /** |
| * Name of the windows utils binary: {@value}. |
| */ |
| static final String WINUTILS_EXE = "winutils.exe"; |
| |
| /** |
| * System property for the Hadoop home directory: {@value}. |
| */ |
| public static final String SYSPROP_HADOOP_HOME_DIR = "hadoop.home.dir"; |
| |
| /** |
| * Environment variable for Hadoop's home dir: {@value}. |
| */ |
| public static final String ENV_HADOOP_HOME = "HADOOP_HOME"; |
| |
| /** |
| * query to see if system is Java 7 or later. |
| * Now that Hadoop requires Java 7 or later, this always returns true. |
| * @deprecated This call isn't needed any more: please remove uses of it. |
| * @return true, always. |
| */ |
| @Deprecated |
| public static boolean isJava7OrAbove() { |
| return true; |
| } |
| |
| // "1.8"->8, "9"->9, "10"->10 |
| private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt( |
| System.getProperty("java.specification.version").split("\\.")[0])); |
| |
| /** |
| * Query to see if major version of Java specification of the system |
| * is equal or greater than the parameter. |
| * |
| * @param version 8, 9, 10 etc. |
| * @return comparison with system property, always true for 8 |
| */ |
| public static boolean isJavaVersionAtLeast(int version) { |
| return JAVA_SPEC_VER >= version; |
| } |
| |
| /** |
| * Maximum command line length in Windows |
| * KB830473 documents this as 8191 |
| */ |
| public static final int WINDOWS_MAX_SHELL_LENGTH = 8191; |
| |
| /** |
| * mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}. |
| * @deprecated use the correctly spelled constant. |
| */ |
| @Deprecated |
| public static final int WINDOWS_MAX_SHELL_LENGHT = WINDOWS_MAX_SHELL_LENGTH; |
| |
| /** |
| * Checks if a given command (String[]) fits in the Windows maximum command |
| * line length Note that the input is expected to already include space |
| * delimiters, no extra count will be added for delimiters. |
| * |
| * @param commands command parts, including any space delimiters |
| */ |
| public static void checkWindowsCommandLineLength(String...commands) |
| throws IOException { |
| int len = 0; |
| for (String s: commands) { |
| len += s.length(); |
| } |
| if (len > WINDOWS_MAX_SHELL_LENGTH) { |
| throw new IOException(String.format( |
| "The command line has a length of %d exceeds maximum allowed length" + |
| " of %d. Command starts with: %s", |
| len, WINDOWS_MAX_SHELL_LENGTH, |
| StringUtils.join("", commands).substring(0, 100))); |
| } |
| } |
| |
| /** |
| * Quote the given arg so that bash will interpret it as a single value. |
| * Note that this quotes it for one level of bash, if you are passing it |
| * into a badly written shell script, you need to fix your shell script. |
| * @param arg the argument to quote |
| * @return the quoted string |
| */ |
| static String bashQuote(String arg) { |
| StringBuilder buffer = new StringBuilder(arg.length() + 2); |
| buffer.append('\'') |
| .append(arg.replace("'", "'\\''")) |
| .append('\''); |
| return buffer.toString(); |
| } |
| |
| /** a Unix command to get the current user's name: {@value}. */ |
| public static final String USER_NAME_COMMAND = "whoami"; |
| |
| /** Windows <code>CreateProcess</code> synchronization object. */ |
| public static final Object WindowsProcessLaunchLock = new Object(); |
| |
| // OSType detection |
| |
| public enum OSType { |
| OS_TYPE_LINUX, |
| OS_TYPE_WIN, |
| OS_TYPE_SOLARIS, |
| OS_TYPE_MAC, |
| OS_TYPE_FREEBSD, |
| OS_TYPE_OTHER |
| } |
| |
| /** |
| * Get the type of the operating system, as determined from parsing |
| * the <code>os.name</code> property. |
| */ |
| public static final OSType osType = getOSType(); |
| |
| private static OSType getOSType() { |
| String osName = System.getProperty("os.name"); |
| if (osName.startsWith("Windows")) { |
| return OSType.OS_TYPE_WIN; |
| } else if (osName.contains("SunOS") || osName.contains("Solaris")) { |
| return OSType.OS_TYPE_SOLARIS; |
| } else if (osName.contains("Mac")) { |
| return OSType.OS_TYPE_MAC; |
| } else if (osName.contains("FreeBSD")) { |
| return OSType.OS_TYPE_FREEBSD; |
| } else if (osName.startsWith("Linux")) { |
| return OSType.OS_TYPE_LINUX; |
| } else { |
| // Some other form of Unix |
| return OSType.OS_TYPE_OTHER; |
| } |
| } |
| |
| // Helper static vars for each platform |
| public static final boolean WINDOWS = (osType == OSType.OS_TYPE_WIN); |
| public static final boolean SOLARIS = (osType == OSType.OS_TYPE_SOLARIS); |
| public static final boolean MAC = (osType == OSType.OS_TYPE_MAC); |
| public static final boolean FREEBSD = (osType == OSType.OS_TYPE_FREEBSD); |
| public static final boolean LINUX = (osType == OSType.OS_TYPE_LINUX); |
| public static final boolean OTHER = (osType == OSType.OS_TYPE_OTHER); |
| |
| public static final boolean PPC_64 |
| = System.getProperties().getProperty("os.arch").contains("ppc64"); |
| |
| /** a Unix command to get the current user's groups list. */ |
| public static String[] getGroupsCommand() { |
| return (WINDOWS)? new String[]{"cmd", "/c", "groups"} |
| : new String[]{"groups"}; |
| } |
| |
| /** |
| * A command to get a given user's groups list. |
| * If the OS is not WINDOWS, the command will get the user's primary group |
| * first and finally get the groups list which includes the primary group. |
| * i.e. the user's primary group will be included twice. |
| */ |
| public static String[] getGroupsForUserCommand(final String user) { |
| //'groups username' command return is inconsistent across different unixes |
| if (WINDOWS) { |
| return new String[] |
| {getWinUtilsPath(), "groups", "-F", "\"" + user + "\""}; |
| } else { |
| String quotedUser = bashQuote(user); |
| return new String[] {"bash", "-c", "id -gn " + quotedUser + |
| "; id -Gn " + quotedUser}; |
| } |
| } |
| |
| /** |
| * A command to get a given user's group id list. |
| * The command will get the user's primary group |
| * first and finally get the groups list which includes the primary group. |
| * i.e. the user's primary group will be included twice. |
| * This command does not support Windows and will only return group names. |
| */ |
| public static String[] getGroupsIDForUserCommand(final String user) { |
| //'groups username' command return is inconsistent across different unixes |
| if (WINDOWS) { |
| return new String[]{getWinUtilsPath(), "groups", "-F", "\"" + user + |
| "\""}; |
| } else { |
| String quotedUser = bashQuote(user); |
| return new String[] {"bash", "-c", "id -g " + quotedUser + "; id -G " + |
| quotedUser}; |
| } |
| } |
| |
| /** A command to get a given netgroup's user list. */ |
| public static String[] getUsersForNetgroupCommand(final String netgroup) { |
| //'groups username' command return is non-consistent across different unixes |
| return new String[] {"getent", "netgroup", netgroup}; |
| } |
| |
| /** Return a command to get permission information. */ |
| public static String[] getGetPermissionCommand() { |
| return (WINDOWS) ? new String[] { getWinUtilsPath(), "ls", "-F" } |
| : new String[] { "ls", "-ld" }; |
| } |
| |
| /** Return a command to set permission. */ |
| public static String[] getSetPermissionCommand(String perm, boolean recursive) { |
| if (recursive) { |
| return (WINDOWS) ? |
| new String[] { getWinUtilsPath(), "chmod", "-R", perm } |
| : new String[] { "chmod", "-R", perm }; |
| } else { |
| return (WINDOWS) ? |
| new String[] { getWinUtilsPath(), "chmod", perm } |
| : new String[] { "chmod", perm }; |
| } |
| } |
| |
| /** |
| * Return a command to set permission for specific file. |
| * |
| * @param perm String permission to set |
| * @param recursive boolean true to apply to all sub-directories recursively |
| * @param file String file to set |
| * @return String[] containing command and arguments |
| */ |
| public static String[] getSetPermissionCommand(String perm, |
| boolean recursive, |
| String file) { |
| String[] baseCmd = getSetPermissionCommand(perm, recursive); |
| String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1); |
| cmdWithFile[cmdWithFile.length - 1] = file; |
| return cmdWithFile; |
| } |
| |
| /** Return a command to set owner. */ |
| public static String[] getSetOwnerCommand(String owner) { |
| return (WINDOWS) ? |
| new String[] { getWinUtilsPath(), "chown", "\"" + owner + "\"" } |
| : new String[] { "chown", owner }; |
| } |
| |
| /** Return a command to create symbolic links. */ |
| public static String[] getSymlinkCommand(String target, String link) { |
| return WINDOWS ? |
| new String[] { getWinUtilsPath(), "symlink", link, target } |
| : new String[] { "ln", "-s", target, link }; |
| } |
| |
| /** Return a command to read the target of the a symbolic link. */ |
| public static String[] getReadlinkCommand(String link) { |
| return WINDOWS ? |
| new String[] { getWinUtilsPath(), "readlink", link } |
| : new String[] { "readlink", link }; |
| } |
| |
| /** |
| * Return a command for determining if process with specified pid is alive. |
| * @param pid process ID |
| * @return a <code>kill -0</code> command or equivalent |
| */ |
| public static String[] getCheckProcessIsAliveCommand(String pid) { |
| return getSignalKillCommand(0, pid); |
| } |
| |
| /** Return a command to send a signal to a given pid. */ |
| public static String[] getSignalKillCommand(int code, String pid) { |
| // Code == 0 means check alive |
| if (Shell.WINDOWS) { |
| if (0 == code) { |
| return new String[] {Shell.getWinUtilsPath(), "task", "isAlive", pid }; |
| } else { |
| return new String[] {Shell.getWinUtilsPath(), "task", "kill", pid }; |
| } |
| } |
| |
| // Use the bash-builtin instead of the Unix kill command (usually |
| // /bin/kill) as the bash-builtin supports "--" in all Hadoop supported |
| // OSes. |
| final String quotedPid = bashQuote(pid); |
| if (isSetsidAvailable) { |
| return new String[] { "bash", "-c", "kill -" + code + " -- -" + |
| quotedPid }; |
| } else { |
| return new String[] { "bash", "-c", "kill -" + code + " " + |
| quotedPid }; |
| } |
| } |
| |
| /** Regular expression for environment variables: {@value}. */ |
| public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*"; |
| |
| /** Return a regular expression string that match environment variables. */ |
| public static String getEnvironmentVariableRegex() { |
| return (WINDOWS) |
| ? "%(" + ENV_NAME_REGEX + "?)%" |
| : "\\$(" + ENV_NAME_REGEX + ")"; |
| } |
| |
| /** |
| * Returns a File referencing a script with the given basename, inside the |
| * given parent directory. The file extension is inferred by platform: |
| * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise. |
| * |
| * @param parent File parent directory |
| * @param basename String script file basename |
| * @return File referencing the script in the directory |
| */ |
| public static File appendScriptExtension(File parent, String basename) { |
| return new File(parent, appendScriptExtension(basename)); |
| } |
| |
| /** |
| * Returns a script file name with the given basename. |
| * |
| * The file extension is inferred by platform: |
| * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise. |
| * |
| * @param basename String script file basename |
| * @return String script file name |
| */ |
| public static String appendScriptExtension(String basename) { |
| return basename + (WINDOWS ? ".cmd" : ".sh"); |
| } |
| |
| /** |
| * Returns a command to run the given script. The script interpreter is |
| * inferred by platform: cmd on Windows or bash otherwise. |
| * |
| * @param script File script to run |
| * @return String[] command to run the script |
| */ |
| public static String[] getRunScriptCommand(File script) { |
| String absolutePath = script.getAbsolutePath(); |
| return WINDOWS ? |
| new String[] {"cmd", "/c", absolutePath } |
| : new String[] {"bash", bashQuote(absolutePath) }; |
| } |
| |
| /** a Unix command to set permission: {@value}. */ |
| public static final String SET_PERMISSION_COMMAND = "chmod"; |
| /** a Unix command to set owner: {@value}. */ |
| public static final String SET_OWNER_COMMAND = "chown"; |
| |
| /** a Unix command to set the change user's groups list: {@value}. */ |
| public static final String SET_GROUP_COMMAND = "chgrp"; |
| /** a Unix command to create a link: {@value}. */ |
| public static final String LINK_COMMAND = "ln"; |
| /** a Unix command to get a link target: {@value}. */ |
| public static final String READ_LINK_COMMAND = "readlink"; |
| |
| /**Time after which the executing script would be timedout. */ |
| protected long timeOutInterval = 0L; |
| /** If or not script timed out*/ |
| private final AtomicBoolean timedOut = new AtomicBoolean(false); |
| |
| /** Indicates if the parent env vars should be inherited or not*/ |
| protected boolean inheritParentEnv = true; |
| |
| /** |
| * Centralized logic to discover and validate the sanity of the Hadoop |
| * home directory. |
| * |
| * This does a lot of work so it should only be called |
| * privately for initialization once per process. |
| * |
| * @return A directory that exists and via was specified on the command line |
| * via <code>-Dhadoop.home.dir</code> or the <code>HADOOP_HOME</code> |
| * environment variable. |
| * @throws FileNotFoundException if the properties are absent or the specified |
| * path is not a reference to a valid directory. |
| */ |
| private static File checkHadoopHome() throws FileNotFoundException { |
| |
| // first check the Dflag hadoop.home.dir with JVM scope |
| String home = System.getProperty(SYSPROP_HADOOP_HOME_DIR); |
| |
| // fall back to the system/user-global env variable |
| if (home == null) { |
| home = System.getenv(ENV_HADOOP_HOME); |
| } |
| return checkHadoopHomeInner(home); |
| } |
| |
| /* |
| A set of exception strings used to construct error messages; |
| these are referred to in tests |
| */ |
| static final String E_DOES_NOT_EXIST = "does not exist"; |
| static final String E_IS_RELATIVE = "is not an absolute path."; |
| static final String E_NOT_DIRECTORY = "is not a directory."; |
| static final String E_NO_EXECUTABLE = "Could not locate Hadoop executable"; |
| static final String E_NOT_EXECUTABLE_FILE = "Not an executable file"; |
| static final String E_HADOOP_PROPS_UNSET = ENV_HADOOP_HOME + " and " |
| + SYSPROP_HADOOP_HOME_DIR + " are unset."; |
| static final String E_HADOOP_PROPS_EMPTY = ENV_HADOOP_HOME + " or " |
| + SYSPROP_HADOOP_HOME_DIR + " set to an empty string"; |
| static final String E_NOT_A_WINDOWS_SYSTEM = "Not a Windows system"; |
| |
| /** |
| * Validate the accessibility of the Hadoop home directory. |
| * |
| * @return A directory that is expected to be the hadoop home directory |
| * @throws FileNotFoundException if the specified |
| * path is not a reference to a valid directory. |
| */ |
| @VisibleForTesting |
| static File checkHadoopHomeInner(String home) throws FileNotFoundException { |
| // couldn't find either setting for hadoop's home directory |
| if (home == null) { |
| throw new FileNotFoundException(E_HADOOP_PROPS_UNSET); |
| } |
| // strip off leading and trailing double quotes |
| while (home.startsWith("\"")) { |
| home = home.substring(1); |
| } |
| while (home.endsWith("\"")) { |
| home = home.substring(0, home.length() - 1); |
| } |
| |
| // after stripping any quotes, check for home dir being non-empty |
| if (home.isEmpty()) { |
| throw new FileNotFoundException(E_HADOOP_PROPS_EMPTY); |
| } |
| |
| // check that the hadoop home dir value |
| // is an absolute reference to a directory |
| File homedir = new File(home); |
| if (!homedir.isAbsolute()) { |
| throw new FileNotFoundException("Hadoop home directory " + homedir |
| + " " + E_IS_RELATIVE); |
| } |
| if (!homedir.exists()) { |
| throw new FileNotFoundException("Hadoop home directory " + homedir |
| + " " + E_DOES_NOT_EXIST); |
| } |
| if (!homedir.isDirectory()) { |
| throw new FileNotFoundException("Hadoop home directory " + homedir |
| + " "+ E_NOT_DIRECTORY); |
| } |
| return homedir; |
| } |
| |
| /** |
| * The Hadoop home directory. |
| */ |
| private static final File HADOOP_HOME_FILE; |
| |
| /** |
| * Rethrowable cause for the failure to determine the hadoop |
| * home directory |
| */ |
| private static final IOException HADOOP_HOME_DIR_FAILURE_CAUSE; |
| |
| static { |
| File home; |
| IOException ex; |
| try { |
| home = checkHadoopHome(); |
| ex = null; |
| } catch (IOException ioe) { |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Failed to detect a valid hadoop home directory", ioe); |
| } |
| ex = ioe; |
| home = null; |
| } |
| HADOOP_HOME_FILE = home; |
| HADOOP_HOME_DIR_FAILURE_CAUSE = ex; |
| } |
| |
| /** |
| * Optionally extend an error message with some OS-specific text. |
| * @param message core error message |
| * @return error message, possibly with some extra text |
| */ |
| private static String addOsText(String message) { |
| return WINDOWS ? (message + " -see " + WINDOWS_PROBLEMS) : message; |
| } |
| |
| /** |
| * Create a {@code FileNotFoundException} with the inner nested cause set |
| * to the given exception. Compensates for the fact that FNFE doesn't |
| * have an initializer that takes an exception. |
| * @param text error text |
| * @param ex inner exception |
| * @return a new exception to throw. |
| */ |
| private static FileNotFoundException fileNotFoundException(String text, |
| Exception ex) { |
| return (FileNotFoundException) new FileNotFoundException(text) |
| .initCause(ex); |
| } |
| |
| /** |
| * Get the Hadoop home directory. Raises an exception if not found |
| * @return the home dir |
| * @throws IOException if the home directory cannot be located. |
| */ |
| public static String getHadoopHome() throws IOException { |
| return getHadoopHomeDir().getCanonicalPath(); |
| } |
| |
| /** |
| * Get the Hadoop home directory. If it is invalid, |
| * throw an exception. |
| * @return a path referring to hadoop home. |
| * @throws FileNotFoundException if the directory doesn't exist. |
| */ |
| private static File getHadoopHomeDir() throws FileNotFoundException { |
| if (HADOOP_HOME_DIR_FAILURE_CAUSE != null) { |
| throw fileNotFoundException( |
| addOsText(HADOOP_HOME_DIR_FAILURE_CAUSE.toString()), |
| HADOOP_HOME_DIR_FAILURE_CAUSE); |
| } |
| return HADOOP_HOME_FILE; |
| } |
| |
| /** |
| * Fully qualify the path to a binary that should be in a known hadoop |
| * bin location. This is primarily useful for disambiguating call-outs |
| * to executable sub-components of Hadoop to avoid clashes with other |
| * executables that may be in the path. Caveat: this call doesn't |
| * just format the path to the bin directory. It also checks for file |
| * existence of the composed path. The output of this call should be |
| * cached by callers. |
| * |
| * @param executable executable |
| * @return executable file reference |
| * @throws FileNotFoundException if the path does not exist |
| */ |
| public static File getQualifiedBin(String executable) |
| throws FileNotFoundException { |
| // construct hadoop bin path to the specified executable |
| return getQualifiedBinInner(getHadoopHomeDir(), executable); |
| } |
| |
| /** |
| * Inner logic of {@link #getQualifiedBin(String)}, accessible |
| * for tests. |
| * @param hadoopHomeDir home directory (assumed to be valid) |
| * @param executable executable |
| * @return path to the binary |
| * @throws FileNotFoundException if the executable was not found/valid |
| */ |
| static File getQualifiedBinInner(File hadoopHomeDir, String executable) |
| throws FileNotFoundException { |
| String binDirText = "Hadoop bin directory "; |
| File bin = new File(hadoopHomeDir, "bin"); |
| if (!bin.exists()) { |
| throw new FileNotFoundException(addOsText(binDirText + E_DOES_NOT_EXIST |
| + ": " + bin)); |
| } |
| if (!bin.isDirectory()) { |
| throw new FileNotFoundException(addOsText(binDirText + E_NOT_DIRECTORY |
| + ": " + bin)); |
| } |
| |
| File exeFile = new File(bin, executable); |
| if (!exeFile.exists()) { |
| throw new FileNotFoundException( |
| addOsText(E_NO_EXECUTABLE + ": " + exeFile)); |
| } |
| if (!exeFile.isFile()) { |
| throw new FileNotFoundException( |
| addOsText(E_NOT_EXECUTABLE_FILE + ": " + exeFile)); |
| } |
| try { |
| return exeFile.getCanonicalFile(); |
| } catch (IOException e) { |
| // this isn't going to happen, because of all the upfront checks. |
| // so if it does, it gets converted to a FNFE and rethrown |
| throw fileNotFoundException(e.toString(), e); |
| } |
| } |
| |
| /** |
| * Fully qualify the path to a binary that should be in a known hadoop |
| * bin location. This is primarily useful for disambiguating call-outs |
| * to executable sub-components of Hadoop to avoid clashes with other |
| * executables that may be in the path. Caveat: this call doesn't |
| * just format the path to the bin directory. It also checks for file |
| * existence of the composed path. The output of this call should be |
| * cached by callers. |
| * |
| * @param executable executable |
| * @return executable file reference |
| * @throws FileNotFoundException if the path does not exist |
| * @throws IOException on path canonicalization failures |
| */ |
| public static String getQualifiedBinPath(String executable) |
| throws IOException { |
| return getQualifiedBin(executable).getCanonicalPath(); |
| } |
| |
| /** |
| * Location of winutils as a string; null if not found. |
| * <p> |
| * <i>Important: caller must check for this value being null</i>. |
| * The lack of such checks has led to many support issues being raised. |
| * <p> |
| * @deprecated use one of the exception-raising getter methods, |
| * specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()} |
| */ |
| @Deprecated |
| public static final String WINUTILS; |
| |
| /** Canonical path to winutils, private to Shell. */ |
| private static final String WINUTILS_PATH; |
| |
| /** file reference to winutils. */ |
| private static final File WINUTILS_FILE; |
| |
| /** the exception raised on a failure to init the WINUTILS fields. */ |
| private static final IOException WINUTILS_FAILURE; |
| |
| /* |
| * Static WINUTILS_* field initializer. |
| * On non-Windows systems sets the paths to null, and |
| * adds a specific exception to the failure cause, so |
| * that on any attempt to resolve the paths will raise |
| * a meaningful exception. |
| */ |
| static { |
| IOException ioe = null; |
| String path = null; |
| File file = null; |
| // invariant: either there's a valid file and path, |
| // or there is a cached IO exception. |
| if (WINDOWS) { |
| try { |
| file = getQualifiedBin(WINUTILS_EXE); |
| path = file.getCanonicalPath(); |
| ioe = null; |
| } catch (IOException e) { |
| LOG.warn("Did not find {}: {}", WINUTILS_EXE, e); |
| // stack trace comes at debug level |
| LOG.debug("Failed to find " + WINUTILS_EXE, e); |
| file = null; |
| path = null; |
| ioe = e; |
| } |
| } else { |
| // on a non-windows system, the invariant is kept |
| // by adding an explicit exception. |
| ioe = new FileNotFoundException(E_NOT_A_WINDOWS_SYSTEM); |
| } |
| WINUTILS_PATH = path; |
| WINUTILS_FILE = file; |
| |
| WINUTILS = path; |
| WINUTILS_FAILURE = ioe; |
| } |
| |
| /** |
| * Predicate to indicate whether or not the path to winutils is known. |
| * |
| * If true, then {@link #WINUTILS} is non-null, and both |
| * {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()} |
| * will successfully return this value. Always false on non-windows systems. |
| * @return true if there is a valid path to the binary |
| */ |
| public static boolean hasWinutilsPath() { |
| return WINUTILS_PATH != null; |
| } |
| |
| /** |
| * Locate the winutils binary, or fail with a meaningful |
| * exception and stack trace as an RTE. |
| * This method is for use in methods which don't explicitly throw |
| * an <code>IOException</code>. |
| * @return the path to {@link #WINUTILS_EXE} |
| * @throws RuntimeException if the path is not resolvable |
| */ |
| public static String getWinUtilsPath() { |
| if (WINUTILS_FAILURE == null) { |
| return WINUTILS_PATH; |
| } else { |
| throw new RuntimeException(WINUTILS_FAILURE.toString(), |
| WINUTILS_FAILURE); |
| } |
| } |
| |
| /** |
| * Get a file reference to winutils. |
| * Always raises an exception if there isn't one |
| * @return the file instance referring to the winutils bin. |
| * @throws FileNotFoundException on any failure to locate that file. |
| */ |
| public static File getWinUtilsFile() throws FileNotFoundException { |
| if (WINUTILS_FAILURE == null) { |
| return WINUTILS_FILE; |
| } else { |
| // raise a new exception to generate a new stack trace |
| throw fileNotFoundException(WINUTILS_FAILURE.toString(), |
| WINUTILS_FAILURE); |
| } |
| } |
| |
| public static boolean checkIsBashSupported() throws InterruptedIOException { |
| if (Shell.WINDOWS) { |
| return false; |
| } |
| |
| ShellCommandExecutor shexec; |
| boolean supported = true; |
| try { |
| String[] args = {"bash", "-c", "echo 1000"}; |
| shexec = new ShellCommandExecutor(args); |
| shexec.execute(); |
| } catch (InterruptedIOException iioe) { |
| LOG.warn("Interrupted, unable to determine if bash is supported", iioe); |
| throw iioe; |
| } catch (IOException ioe) { |
| LOG.warn("Bash is not supported by the OS", ioe); |
| supported = false; |
| } catch (SecurityException se) { |
| LOG.info("Bash execution is not allowed by the JVM " + |
| "security manager.Considering it not supported."); |
| supported = false; |
| } |
| |
| return supported; |
| } |
| |
| /** |
| * Flag which is true if setsid exists. |
| */ |
| public static final boolean isSetsidAvailable = isSetsidSupported(); |
| |
| /** |
| * Look for <code>setsid</code>. |
| * @return true if <code>setsid</code> was present |
| */ |
| private static boolean isSetsidSupported() { |
| if (Shell.WINDOWS) { |
| return false; |
| } |
| ShellCommandExecutor shexec = null; |
| boolean setsidSupported = true; |
| try { |
| String[] args = {"setsid", "bash", "-c", "echo $$"}; |
| shexec = new ShellCommandExecutor(args); |
| shexec.execute(); |
| } catch (IOException ioe) { |
| LOG.debug("setsid is not available on this machine. So not using it."); |
| setsidSupported = false; |
| } catch (SecurityException se) { |
| LOG.debug("setsid is not allowed to run by the JVM "+ |
| "security manager. So not using it."); |
| setsidSupported = false; |
| } catch (Error err) { |
| if (err.getMessage() != null |
| && err.getMessage().contains("posix_spawn is not " + |
| "a supported process launch mechanism") |
| && (Shell.FREEBSD || Shell.MAC)) { |
| // HADOOP-11924: This is a workaround to avoid failure of class init |
| // by JDK issue on TR locale(JDK-8047340). |
| LOG.info("Avoiding JDK-8047340 on BSD-based systems.", err); |
| setsidSupported = false; |
| } |
| } finally { // handle the exit code |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("setsid exited with exit code " |
| + (shexec != null ? shexec.getExitCode() : "(null executor)")); |
| } |
| } |
| return setsidSupported; |
| } |
| |
| /** Token separator regex used to parse Shell tool outputs. */ |
| public static final String TOKEN_SEPARATOR_REGEX |
| = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]"; |
| |
| private long interval; // refresh interval in msec |
| private long lastTime; // last time the command was performed |
| private final boolean redirectErrorStream; // merge stdout and stderr |
| private Map<String, String> environment; // env for the command execution |
| private File dir; |
| private Process process; // sub process used to execute the command |
| private int exitCode; |
| private Thread waitingThread; |
| |
| /** Flag to indicate whether or not the script has finished executing. */ |
| private final AtomicBoolean completed = new AtomicBoolean(false); |
| |
| /** |
| * Create an instance with no minimum interval between runs; stderr is |
| * not merged with stdout. |
| */ |
| protected Shell() { |
| this(0L); |
| } |
| |
| /** |
| * Create an instance with a minimum interval between executions; stderr is |
| * not merged with stdout. |
| * @param interval interval in milliseconds between command executions. |
| */ |
| protected Shell(long interval) { |
| this(interval, false); |
| } |
| |
| /** |
| * Create a shell instance which can be re-executed when the {@link #run()} |
| * method is invoked with a given elapsed time between calls. |
| * |
| * @param interval the minimum duration in milliseconds to wait before |
| * re-executing the command. If set to 0, there is no minimum. |
| * @param redirectErrorStream should the error stream be merged with |
| * the normal output stream? |
| */ |
| protected Shell(long interval, boolean redirectErrorStream) { |
| this.interval = interval; |
| this.lastTime = (interval < 0) ? 0 : -interval; |
| this.redirectErrorStream = redirectErrorStream; |
| this.environment = Collections.emptyMap(); |
| } |
| |
| /** |
| * Set the environment for the command. |
| * @param env Mapping of environment variables |
| */ |
| protected void setEnvironment(Map<String, String> env) { |
| this.environment = Objects.requireNonNull(env); |
| } |
| |
| /** |
| * Set the working directory. |
| * @param dir The directory where the command will be executed |
| */ |
| protected void setWorkingDirectory(File dir) { |
| this.dir = dir; |
| } |
| |
| /** Check to see if a command needs to be executed and execute if needed. */ |
| protected void run() throws IOException { |
| if (lastTime + interval > Time.monotonicNow()) { |
| return; |
| } |
| exitCode = 0; // reset for next run |
| if (Shell.MAC) { |
| System.setProperty("jdk.lang.Process.launchMechanism", "POSIX_SPAWN"); |
| } |
| runCommand(); |
| } |
| |
| /** Run the command. */ |
| private void runCommand() throws IOException { |
| ProcessBuilder builder = new ProcessBuilder(getExecString()); |
| Timer timeOutTimer = null; |
| ShellTimeoutTimerTask timeoutTimerTask = null; |
| timedOut.set(false); |
| completed.set(false); |
| |
| // Remove all env vars from the Builder to prevent leaking of env vars from |
| // the parent process. |
| if (!inheritParentEnv) { |
| builder.environment().clear(); |
| } |
| |
| builder.environment().putAll(this.environment); |
| |
| if (dir != null) { |
| builder.directory(this.dir); |
| } |
| |
| builder.redirectErrorStream(redirectErrorStream); |
| |
| if (Shell.WINDOWS) { |
| synchronized (WindowsProcessLaunchLock) { |
| // To workaround the race condition issue with child processes |
| // inheriting unintended handles during process launch that can |
| // lead to hangs on reading output and error streams, we |
| // serialize process creation. More info available at: |
| // http://support.microsoft.com/kb/315939 |
| process = builder.start(); |
| } |
| } else { |
| process = builder.start(); |
| } |
| |
| waitingThread = Thread.currentThread(); |
| CHILD_SHELLS.put(this, null); |
| |
| if (timeOutInterval > 0) { |
| timeOutTimer = new Timer("Shell command timeout"); |
| timeoutTimerTask = new ShellTimeoutTimerTask( |
| this); |
| //One time scheduling. |
| timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); |
| } |
| final BufferedReader errReader = |
| new BufferedReader(new InputStreamReader( |
| process.getErrorStream(), Charset.defaultCharset())); |
| BufferedReader inReader = |
| new BufferedReader(new InputStreamReader( |
| process.getInputStream(), Charset.defaultCharset())); |
| final StringBuffer errMsg = new StringBuffer(); |
| |
| // read error and input streams as this would free up the buffers |
| // free the error stream buffer |
| Thread errThread = new Thread() { |
| @Override |
| public void run() { |
| try { |
| String line = errReader.readLine(); |
| while((line != null) && !isInterrupted()) { |
| errMsg.append(line) |
| .append(System.getProperty("line.separator")); |
| line = errReader.readLine(); |
| } |
| } catch(IOException ioe) { |
| // Its normal to observe a "Stream closed" I/O error on |
| // command timeouts destroying the underlying process |
| // so only log a WARN if the command didn't time out |
| if (!isTimedOut()) { |
| LOG.warn("Error reading the error stream", ioe); |
| } else { |
| LOG.debug("Error reading the error stream due to shell " |
| + "command timeout", ioe); |
| } |
| } |
| } |
| }; |
| try { |
| errThread.start(); |
| } catch (IllegalStateException ise) { |
| } catch (OutOfMemoryError oe) { |
| LOG.error("Caught " + oe + ". One possible reason is that ulimit" |
| + " setting of 'max user processes' is too low. If so, do" |
| + " 'ulimit -u <largerNum>' and try again."); |
| throw oe; |
| } |
| try { |
| parseExecResult(inReader); // parse the output |
| // clear the input stream buffer |
| String line = inReader.readLine(); |
| while(line != null) { |
| line = inReader.readLine(); |
| } |
| // wait for the process to finish and check the exit code |
| exitCode = process.waitFor(); |
| // make sure that the error thread exits |
| joinThread(errThread); |
| completed.set(true); |
| //the timeout thread handling |
| //taken care in finally block |
| if (exitCode != 0) { |
| throw new ExitCodeException(exitCode, errMsg.toString()); |
| } |
| } catch (InterruptedException ie) { |
| InterruptedIOException iie = new InterruptedIOException(ie.toString()); |
| iie.initCause(ie); |
| throw iie; |
| } finally { |
| if (timeOutTimer != null) { |
| timeOutTimer.cancel(); |
| } |
| // close the input stream |
| try { |
| inReader.close(); |
| } catch (IOException ioe) { |
| LOG.warn("Error while closing the input stream", ioe); |
| } |
| if (!completed.get()) { |
| errThread.interrupt(); |
| joinThread(errThread); |
| } |
| try { |
| errReader.close(); |
| } catch (IOException ioe) { |
| LOG.warn("Error while closing the error stream", ioe); |
| } |
| process.destroy(); |
| waitingThread = null; |
| CHILD_SHELLS.remove(this); |
| lastTime = Time.monotonicNow(); |
| } |
| } |
| |
| private static void joinThread(Thread t) { |
| while (t.isAlive()) { |
| try { |
| t.join(); |
| } catch (InterruptedException ie) { |
| if (LOG.isWarnEnabled()) { |
| LOG.warn("Interrupted while joining on: " + t, ie); |
| } |
| t.interrupt(); // propagate interrupt |
| } |
| } |
| } |
| |
| /** return an array containing the command name and its parameters. */ |
| protected abstract String[] getExecString(); |
| |
| /** Parse the execution result */ |
| protected abstract void parseExecResult(BufferedReader lines) |
| throws IOException; |
| |
| /** |
| * Get an environment variable. |
| * @param env the environment var |
| * @return the value or null if it was unset. |
| */ |
| public String getEnvironment(String env) { |
| return environment.get(env); |
| } |
| |
| /** get the current sub-process executing the given command. |
| * @return process executing the command |
| */ |
| public Process getProcess() { |
| return process; |
| } |
| |
| /** get the exit code. |
| * @return the exit code of the process |
| */ |
| public int getExitCode() { |
| return exitCode; |
| } |
| |
| /** get the thread that is waiting on this instance of <code>Shell</code>. |
| * @return the thread that ran runCommand() that spawned this shell |
| * or null if no thread is waiting for this shell to complete |
| */ |
| public Thread getWaitingThread() { |
| return waitingThread; |
| } |
| |
| |
| /** |
| * This is an IOException with exit code added. |
| */ |
| public static class ExitCodeException extends IOException { |
| private final int exitCode; |
| |
| public ExitCodeException(int exitCode, String message) { |
| super(message); |
| this.exitCode = exitCode; |
| } |
| |
| public int getExitCode() { |
| return exitCode; |
| } |
| |
| @Override |
| public String toString() { |
| final StringBuilder sb = |
| new StringBuilder("ExitCodeException "); |
| sb.append("exitCode=").append(exitCode) |
| .append(": ") |
| .append(super.getMessage()); |
| return sb.toString(); |
| } |
| } |
| |
| public interface CommandExecutor { |
| |
| void execute() throws IOException; |
| |
| int getExitCode() throws IOException; |
| |
| String getOutput() throws IOException; |
| |
| void close(); |
| |
| } |
| |
| /** |
| * A simple shell command executor. |
| * |
| * <code>ShellCommandExecutor</code>should be used in cases where the output |
| * of the command needs no explicit parsing and where the command, working |
| * directory and the environment remains unchanged. The output of the command |
| * is stored as-is and is expected to be small. |
| */ |
| public static class ShellCommandExecutor extends Shell |
| implements CommandExecutor { |
| |
| private String[] command; |
| private StringBuffer output; |
| |
| |
| public ShellCommandExecutor(String[] execString) { |
| this(execString, null); |
| } |
| |
| public ShellCommandExecutor(String[] execString, File dir) { |
| this(execString, dir, null); |
| } |
| |
| public ShellCommandExecutor(String[] execString, File dir, |
| Map<String, String> env) { |
| this(execString, dir, env , 0L); |
| } |
| |
| public ShellCommandExecutor(String[] execString, File dir, |
| Map<String, String> env, long timeout) { |
| this(execString, dir, env , timeout, true); |
| } |
| |
| /** |
| * Create a new instance of the ShellCommandExecutor to execute a command. |
| * |
| * @param execString The command to execute with arguments |
| * @param dir If not-null, specifies the directory which should be set |
| * as the current working directory for the command. |
| * If null, the current working directory is not modified. |
| * @param env If not-null, environment of the command will include the |
| * key-value pairs specified in the map. If null, the current |
| * environment is not modified. |
| * @param timeout Specifies the time in milliseconds, after which the |
| * command will be killed and the status marked as timed-out. |
| * If 0, the command will not be timed out. |
| * @param inheritParentEnv Indicates if the process should inherit the env |
| * vars from the parent process or not. |
| */ |
| public ShellCommandExecutor(String[] execString, File dir, |
| Map<String, String> env, long timeout, boolean inheritParentEnv) { |
| command = execString.clone(); |
| if (dir != null) { |
| setWorkingDirectory(dir); |
| } |
| if (env != null) { |
| setEnvironment(env); |
| } |
| timeOutInterval = timeout; |
| this.inheritParentEnv = inheritParentEnv; |
| } |
| |
| /** |
| * Returns the timeout value set for the executor's sub-commands. |
| * @return The timeout value in milliseconds |
| */ |
| @VisibleForTesting |
| public long getTimeoutInterval() { |
| return timeOutInterval; |
| } |
| |
| /** |
| * Execute the shell command. |
| * @throws IOException if the command fails, or if the command is |
| * not well constructed. |
| */ |
| public void execute() throws IOException { |
| for (String s : command) { |
| if (s == null) { |
| throw new IOException("(null) entry in command string: " |
| + StringUtils.join(" ", command)); |
| } |
| } |
| this.run(); |
| } |
| |
| @Override |
| public String[] getExecString() { |
| return command; |
| } |
| |
| @Override |
| protected void parseExecResult(BufferedReader lines) throws IOException { |
| output = new StringBuffer(); |
| char[] buf = new char[512]; |
| int nRead; |
| while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) { |
| output.append(buf, 0, nRead); |
| } |
| } |
| |
| /** Get the output of the shell command. */ |
| public String getOutput() { |
| return (output == null) ? "" : output.toString(); |
| } |
| |
| /** |
| * Returns the commands of this instance. |
| * Arguments with spaces in are presented with quotes round; other |
| * arguments are presented raw |
| * |
| * @return a string representation of the object. |
| */ |
| @Override |
| public String toString() { |
| StringBuilder builder = new StringBuilder(); |
| String[] args = getExecString(); |
| for (String s : args) { |
| if (s.indexOf(' ') >= 0) { |
| builder.append('"').append(s).append('"'); |
| } else { |
| builder.append(s); |
| } |
| builder.append(' '); |
| } |
| return builder.toString(); |
| } |
| |
| @Override |
| public void close() { |
| } |
| } |
| |
| /** |
| * To check if the passed script to shell command executor timed out or |
| * not. |
| * |
| * @return if the script timed out. |
| */ |
| public boolean isTimedOut() { |
| return timedOut.get(); |
| } |
| |
| /** |
| * Declare that the command has timed out. |
| * |
| */ |
| private void setTimedOut() { |
| this.timedOut.set(true); |
| } |
| |
| /** |
| * Static method to execute a shell command. |
| * Covers most of the simple cases without requiring the user to implement |
| * the <code>Shell</code> interface. |
| * @param cmd shell command to execute. |
| * @return the output of the executed command. |
| */ |
| public static String execCommand(String ... cmd) throws IOException { |
| return execCommand(null, cmd, 0L); |
| } |
| |
| /** |
| * Static method to execute a shell command. |
| * Covers most of the simple cases without requiring the user to implement |
| * the <code>Shell</code> interface. |
| * @param env the map of environment key=value |
| * @param cmd shell command to execute. |
| * @param timeout time in milliseconds after which script should be marked timeout |
| * @return the output of the executed command. |
| * @throws IOException on any problem. |
| */ |
| |
| public static String execCommand(Map<String, String> env, String[] cmd, |
| long timeout) throws IOException { |
| ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env, |
| timeout); |
| exec.execute(); |
| return exec.getOutput(); |
| } |
| |
| /** |
| * Static method to execute a shell command. |
| * Covers most of the simple cases without requiring the user to implement |
| * the <code>Shell</code> interface. |
| * @param env the map of environment key=value |
| * @param cmd shell command to execute. |
| * @return the output of the executed command. |
| * @throws IOException on any problem. |
| */ |
| public static String execCommand(Map<String,String> env, String ... cmd) |
| throws IOException { |
| return execCommand(env, cmd, 0L); |
| } |
| |
| /** |
| * Timer which is used to timeout scripts spawned off by shell. |
| */ |
| private static class ShellTimeoutTimerTask extends TimerTask { |
| |
| private final Shell shell; |
| |
| public ShellTimeoutTimerTask(Shell shell) { |
| this.shell = shell; |
| } |
| |
| @Override |
| public void run() { |
| Process p = shell.getProcess(); |
| try { |
| p.exitValue(); |
| } catch (Exception e) { |
| //Process has not terminated. |
| //So check if it has completed |
| //if not just destroy it. |
| if (p != null && !shell.completed.get()) { |
| shell.setTimedOut(); |
| p.destroy(); |
| } |
| } |
| } |
| } |
| |
| /** |
| * Static method to destroy all running <code>Shell</code> processes. |
| * Iterates through a map of all currently running <code>Shell</code> |
| * processes and destroys them one by one. This method is thread safe |
| */ |
| public static void destroyAllShellProcesses() { |
| synchronized (CHILD_SHELLS) { |
| for (Shell shell : CHILD_SHELLS.keySet()) { |
| if (shell.getProcess() != null) { |
| shell.getProcess().destroy(); |
| } |
| } |
| CHILD_SHELLS.clear(); |
| } |
| } |
| |
| /** |
| * Static method to return a Set of all <code>Shell</code> objects. |
| */ |
| public static Set<Shell> getAllShells() { |
| synchronized (CHILD_SHELLS) { |
| return new HashSet<>(CHILD_SHELLS.keySet()); |
| } |
| } |
| |
| /** |
| * Static method to return the memory lock limit for datanode. |
| * @param ulimit max value at which memory locked should be capped. |
| * @return long value specifying the memory lock limit. |
| */ |
| public static Long getMemlockLimit(Long ulimit) { |
| if (WINDOWS) { |
| // HDFS-13560: if ulimit is too large on Windows, Windows will complain |
| // "1450: Insufficient system resources exist to complete the requested |
| // service". Thus, cap Windows memory lock limit at Integer.MAX_VALUE. |
| return Math.min(Integer.MAX_VALUE, ulimit); |
| } |
| return ulimit; |
| } |
| } |
