hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java - hadoop - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package org.apache.hadoop.util;

 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
 import java.util.regex.Pattern;

 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 /** Run a Hadoop job jar. */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class RunJar {

   private static final Logger LOG = LoggerFactory.getLogger(RunJar.class);

   /** Pattern that matches any string. */
   public static final Pattern MATCH_ANY = Pattern.compile(".*");

   /**
    * Priority of the RunJar shutdown hook.
    */
   public static final int SHUTDOWN_HOOK_PRIORITY = 10;

   /**
    * Environment key for using the client classloader.
    */
   public static final String HADOOP_USE_CLIENT_CLASSLOADER =
       "HADOOP_USE_CLIENT_CLASSLOADER";
   /**
    * Environment key for the (user-provided) hadoop classpath.
    */
   public static final String HADOOP_CLASSPATH = "HADOOP_CLASSPATH";
   /**
    * Environment key for the system classes.
    */
   public static final String HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES =
       "HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES";

   /**
    * Buffer size for copy the content of compressed file to new file.
    */
   private static final int BUFFER_SIZE = 8_192;

   /**
    * Unpack a jar file into a directory.
    *
    * This version unpacks all files inside the jar regardless of filename.
    *
    * @param jarFile the .jar file to unpack
    * @param toDir the destination directory into which to unpack the jar
    *
    * @throws IOException if an I/O error has occurred or toDir
    * cannot be created and does not already exist
    */
   public static void unJar(File jarFile, File toDir) throws IOException {
     unJar(jarFile, toDir, MATCH_ANY);
   }

   /**
    * Unpack matching files from a jar. Entries inside the jar that do
    * not match the given pattern will be skipped.
    *
    * @param jarFile the .jar file to unpack
    * @param toDir the destination directory into which to unpack the jar
    * @param unpackRegex the pattern to match jar entries against
    *
    * @throws IOException if an I/O error has occurred or toDir
    * cannot be created and does not already exist
    */
   public static void unJar(File jarFile, File toDir, Pattern unpackRegex)
       throws IOException {
     try (JarFile jar = new JarFile(jarFile)) {
       int numOfFailedLastModifiedSet = 0;
       Enumeration<JarEntry> entries = jar.entries();
       while (entries.hasMoreElements()) {
         final JarEntry entry = entries.nextElement();
         if (!entry.isDirectory() &&
             unpackRegex.matcher(entry.getName()).matches()) {
           try (InputStream in = jar.getInputStream(entry)) {
             File file = new File(toDir, entry.getName());
             ensureDirectory(file.getParentFile());
             try (OutputStream out = new FileOutputStream(file)) {
               IOUtils.copyBytes(in, out, BUFFER_SIZE);
             }
             if (!file.setLastModified(entry.getTime())) {
               numOfFailedLastModifiedSet++;
             }
           }
         }
       }
       if (numOfFailedLastModifiedSet > 0) {
         LOG.warn("Could not set last modfied time for {} file(s)",
             numOfFailedLastModifiedSet);
       }
     }
   }

   /**
    * Ensure the existence of a given directory.
    *
    * @param dir Directory to check
    *
    * @throws IOException if it cannot be created and does not already exist
    */
   private static void ensureDirectory(File dir) throws IOException {
     if (!dir.mkdirs() && !dir.isDirectory()) {
       throw new IOException("Mkdirs failed to create " +
                             dir.toString());
     }
   }

   /** Run a Hadoop job jar.  If the main class is not in the jar's manifest,
    * then it must be provided on the command line. */
   public static void main(String[] args) throws Throwable {
     new RunJar().run(args);
   }

   public void run(String[] args) throws Throwable {
     String usage = "RunJar jarFile [mainClass] args...";

     if (args.length < 1) {
       System.err.println(usage);
       System.exit(-1);
     }

     int firstArg = 0;
     String fileName = args[firstArg++];
     File file = new File(fileName);
     if (!file.exists() || !file.isFile()) {
       System.err.println("JAR does not exist or is not a normal file: " +
           file.getCanonicalPath());
       System.exit(-1);
     }
     String mainClassName = null;

     JarFile jarFile;
     try {
       jarFile = new JarFile(fileName);
     } catch (IOException io) {
       throw new IOException("Error opening job jar: " + fileName)
         .initCause(io);
     }

     Manifest manifest = jarFile.getManifest();
     if (manifest != null) {
       mainClassName = manifest.getMainAttributes().getValue("Main-Class");
     }
     jarFile.close();

     if (mainClassName == null) {
       if (args.length < 2) {
         System.err.println(usage);
         System.exit(-1);
       }
       mainClassName = args[firstArg++];
     }
     mainClassName = mainClassName.replaceAll("/", ".");

     File tmpDir = new File(System.getProperty("java.io.tmpdir"));
     ensureDirectory(tmpDir);

     final File workDir;
     try {
       workDir = File.createTempFile("hadoop-unjar", "", tmpDir);
     } catch (IOException ioe) {
       // If user has insufficient perms to write to tmpDir, default
       // "Permission denied" message doesn't specify a filename.
       System.err.println("Error creating temp dir in java.io.tmpdir "
                          + tmpDir + " due to " + ioe.getMessage());
       System.exit(-1);
       return;
     }

     if (!workDir.delete()) {
       System.err.println("Delete failed for " + workDir);
       System.exit(-1);
     }
     ensureDirectory(workDir);

     ShutdownHookManager.get().addShutdownHook(
         new Runnable() {
           @Override
           public void run() {
             FileUtil.fullyDelete(workDir);
           }
         }, SHUTDOWN_HOOK_PRIORITY);


     unJar(file, workDir);

     ClassLoader loader = createClassLoader(file, workDir);

     Thread.currentThread().setContextClassLoader(loader);
     Class<?> mainClass = Class.forName(mainClassName, true, loader);
     Method main = mainClass.getMethod("main", String[].class);
     List<String> newArgsSubList = Arrays.asList(args)
         .subList(firstArg, args.length);
     String[] newArgs = newArgsSubList
         .toArray(new String[newArgsSubList.size()]);
     try {
       main.invoke(null, new Object[] {newArgs});
     } catch (InvocationTargetException e) {
       throw e.getTargetException();
     }
   }

   /**
    * Creates a classloader based on the environment that was specified by the
    * user. If HADOOP_USE_CLIENT_CLASSLOADER is specified, it creates an
    * application classloader that provides the isolation of the user class space
    * from the hadoop classes and their dependencies. It forms a class space for
    * the user jar as well as the HADOOP_CLASSPATH. Otherwise, it creates a
    * classloader that simply adds the user jar to the classpath.
    */
   private ClassLoader createClassLoader(File file, final File workDir)
       throws MalformedURLException {
     ClassLoader loader;
     // see if the client classloader is enabled
     if (useClientClassLoader()) {
       StringBuilder sb = new StringBuilder();
       sb.append(workDir).append("/").
           append(File.pathSeparator).append(file).
           append(File.pathSeparator).append(workDir).append("/classes/").
           append(File.pathSeparator).append(workDir).append("/lib/*");
       // HADOOP_CLASSPATH is added to the client classpath
       String hadoopClasspath = getHadoopClasspath();
       if (hadoopClasspath != null && !hadoopClasspath.isEmpty()) {
         sb.append(File.pathSeparator).append(hadoopClasspath);
       }
       String clientClasspath = sb.toString();
       // get the system classes
       String systemClasses = getSystemClasses();
       List<String> systemClassesList = systemClasses == null ?
           null :
           Arrays.asList(StringUtils.getTrimmedStrings(systemClasses));
       // create an application classloader that isolates the user classes
       loader = new ApplicationClassLoader(clientClasspath,
           getClass().getClassLoader(), systemClassesList);
     } else {
       List<URL> classPath = new ArrayList<>();
       classPath.add(new File(workDir + "/").toURI().toURL());
       classPath.add(file.toURI().toURL());
       classPath.add(new File(workDir, "classes/").toURI().toURL());
       File[] libs = new File(workDir, "lib").listFiles();
       if (libs != null) {
         for (File lib : libs) {
           classPath.add(lib.toURI().toURL());
         }
       }
       // create a normal parent-delegating classloader
       loader = new URLClassLoader(classPath.toArray(new URL[classPath.size()]));
     }
     return loader;
   }

   boolean useClientClassLoader() {
     return Boolean.parseBoolean(System.getenv(HADOOP_USE_CLIENT_CLASSLOADER));
   }

   String getHadoopClasspath() {
     return System.getenv(HADOOP_CLASSPATH);
   }

   String getSystemClasses() {
     return System.getenv(HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES);
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package org.apache.hadoop.util;

	import java.io.File;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.OutputStream;
	import java.lang.reflect.InvocationTargetException;
	import java.lang.reflect.Method;
	import java.net.MalformedURLException;
	import java.net.URL;
	import java.net.URLClassLoader;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Enumeration;
	import java.util.List;
	import java.util.jar.JarEntry;
	import java.util.jar.JarFile;
	import java.util.jar.Manifest;
	import java.util.regex.Pattern;

	import org.apache.hadoop.classification.InterfaceAudience;
	import org.apache.hadoop.classification.InterfaceStability;
	import org.apache.hadoop.fs.FileUtil;
	import org.apache.hadoop.io.IOUtils;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	/** Run a Hadoop job jar. */
	@InterfaceAudience.Private
	@InterfaceStability.Unstable
	public class RunJar {

	private static final Logger LOG = LoggerFactory.getLogger(RunJar.class);

	/** Pattern that matches any string. */
	public static final Pattern MATCH_ANY = Pattern.compile(".*");

	/**
	* Priority of the RunJar shutdown hook.
	*/
	public static final int SHUTDOWN_HOOK_PRIORITY = 10;

	/**
	* Environment key for using the client classloader.
	*/
	public static final String HADOOP_USE_CLIENT_CLASSLOADER =
	"HADOOP_USE_CLIENT_CLASSLOADER";
	/**
	* Environment key for the (user-provided) hadoop classpath.
	*/
	public static final String HADOOP_CLASSPATH = "HADOOP_CLASSPATH";
	/**
	* Environment key for the system classes.
	*/
	public static final String HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES =
	"HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES";

	/**
	* Buffer size for copy the content of compressed file to new file.
	*/
	private static final int BUFFER_SIZE = 8_192;

	/**
	* Unpack a jar file into a directory.
	*
	* This version unpacks all files inside the jar regardless of filename.
	*
	* @param jarFile the .jar file to unpack
	* @param toDir the destination directory into which to unpack the jar
	*
	* @throws IOException if an I/O error has occurred or toDir
	* cannot be created and does not already exist
	*/
	public static void unJar(File jarFile, File toDir) throws IOException {
	unJar(jarFile, toDir, MATCH_ANY);
	}

	/**
	* Unpack matching files from a jar. Entries inside the jar that do
	* not match the given pattern will be skipped.
	*
	* @param jarFile the .jar file to unpack
	* @param toDir the destination directory into which to unpack the jar
	* @param unpackRegex the pattern to match jar entries against
	*
	* @throws IOException if an I/O error has occurred or toDir
	* cannot be created and does not already exist
	*/
	public static void unJar(File jarFile, File toDir, Pattern unpackRegex)
	throws IOException {
	try (JarFile jar = new JarFile(jarFile)) {
	int numOfFailedLastModifiedSet = 0;
	Enumeration<JarEntry> entries = jar.entries();
	while (entries.hasMoreElements()) {
	final JarEntry entry = entries.nextElement();
	if (!entry.isDirectory() &&
	unpackRegex.matcher(entry.getName()).matches()) {
	try (InputStream in = jar.getInputStream(entry)) {
	File file = new File(toDir, entry.getName());
	ensureDirectory(file.getParentFile());
	try (OutputStream out = new FileOutputStream(file)) {
	IOUtils.copyBytes(in, out, BUFFER_SIZE);
	}
	if (!file.setLastModified(entry.getTime())) {
	numOfFailedLastModifiedSet++;
	}
	}
	}
	}
	if (numOfFailedLastModifiedSet > 0) {
	LOG.warn("Could not set last modfied time for {} file(s)",
	numOfFailedLastModifiedSet);
	}
	}
	}

	/**
	* Ensure the existence of a given directory.
	*
	* @param dir Directory to check
	*
	* @throws IOException if it cannot be created and does not already exist
	*/
	private static void ensureDirectory(File dir) throws IOException {
	if (!dir.mkdirs() && !dir.isDirectory()) {
	throw new IOException("Mkdirs failed to create " +
	dir.toString());
	}
	}

	/** Run a Hadoop job jar. If the main class is not in the jar's manifest,
	* then it must be provided on the command line. */
	public static void main(String[] args) throws Throwable {
	new RunJar().run(args);
	}

	public void run(String[] args) throws Throwable {
	String usage = "RunJar jarFile [mainClass] args...";

	if (args.length < 1) {
	System.err.println(usage);
	System.exit(-1);
	}

	int firstArg = 0;
	String fileName = args[firstArg++];
	File file = new File(fileName);
	if (!file.exists() \|\| !file.isFile()) {
	System.err.println("JAR does not exist or is not a normal file: " +
	file.getCanonicalPath());
	System.exit(-1);
	}
	String mainClassName = null;

	JarFile jarFile;
	try {
	jarFile = new JarFile(fileName);
	} catch (IOException io) {
	throw new IOException("Error opening job jar: " + fileName)
	.initCause(io);
	}

	Manifest manifest = jarFile.getManifest();
	if (manifest != null) {
	mainClassName = manifest.getMainAttributes().getValue("Main-Class");
	}
	jarFile.close();

	if (mainClassName == null) {
	if (args.length < 2) {
	System.err.println(usage);
	System.exit(-1);
	}
	mainClassName = args[firstArg++];
	}
	mainClassName = mainClassName.replaceAll("/", ".");

	File tmpDir = new File(System.getProperty("java.io.tmpdir"));
	ensureDirectory(tmpDir);

	final File workDir;
	try {
	workDir = File.createTempFile("hadoop-unjar", "", tmpDir);
	} catch (IOException ioe) {
	// If user has insufficient perms to write to tmpDir, default
	// "Permission denied" message doesn't specify a filename.
	System.err.println("Error creating temp dir in java.io.tmpdir "
	+ tmpDir + " due to " + ioe.getMessage());
	System.exit(-1);
	return;
	}

	if (!workDir.delete()) {
	System.err.println("Delete failed for " + workDir);
	System.exit(-1);
	}
	ensureDirectory(workDir);

	ShutdownHookManager.get().addShutdownHook(
	new Runnable() {
	@Override
	public void run() {
	FileUtil.fullyDelete(workDir);
	}
	}, SHUTDOWN_HOOK_PRIORITY);


	unJar(file, workDir);

	ClassLoader loader = createClassLoader(file, workDir);

	Thread.currentThread().setContextClassLoader(loader);
	Class<?> mainClass = Class.forName(mainClassName, true, loader);
	Method main = mainClass.getMethod("main", String[].class);
	List<String> newArgsSubList = Arrays.asList(args)
	.subList(firstArg, args.length);
	String[] newArgs = newArgsSubList
	.toArray(new String[newArgsSubList.size()]);
	try {
	main.invoke(null, new Object[] {newArgs});
	} catch (InvocationTargetException e) {
	throw e.getTargetException();
	}
	}

	/**
	* Creates a classloader based on the environment that was specified by the
	* user. If HADOOP_USE_CLIENT_CLASSLOADER is specified, it creates an
	* application classloader that provides the isolation of the user class space
	* from the hadoop classes and their dependencies. It forms a class space for
	* the user jar as well as the HADOOP_CLASSPATH. Otherwise, it creates a
	* classloader that simply adds the user jar to the classpath.
	*/
	private ClassLoader createClassLoader(File file, final File workDir)
	throws MalformedURLException {
	ClassLoader loader;
	// see if the client classloader is enabled
	if (useClientClassLoader()) {
	StringBuilder sb = new StringBuilder();
	sb.append(workDir).append("/").
	append(File.pathSeparator).append(file).
	append(File.pathSeparator).append(workDir).append("/classes/").
	append(File.pathSeparator).append(workDir).append("/lib/*");
	// HADOOP_CLASSPATH is added to the client classpath
	String hadoopClasspath = getHadoopClasspath();
	if (hadoopClasspath != null && !hadoopClasspath.isEmpty()) {
	sb.append(File.pathSeparator).append(hadoopClasspath);
	}
	String clientClasspath = sb.toString();
	// get the system classes
	String systemClasses = getSystemClasses();
	List<String> systemClassesList = systemClasses == null ?
	null :
	Arrays.asList(StringUtils.getTrimmedStrings(systemClasses));
	// create an application classloader that isolates the user classes
	loader = new ApplicationClassLoader(clientClasspath,
	getClass().getClassLoader(), systemClassesList);
	} else {
	List<URL> classPath = new ArrayList<>();
	classPath.add(new File(workDir + "/").toURI().toURL());
	classPath.add(file.toURI().toURL());
	classPath.add(new File(workDir, "classes/").toURI().toURL());
	File[] libs = new File(workDir, "lib").listFiles();
	if (libs != null) {
	for (File lib : libs) {
	classPath.add(lib.toURI().toURL());
	}
	}
	// create a normal parent-delegating classloader
	loader = new URLClassLoader(classPath.toArray(new URL[classPath.size()]));
	}
	return loader;
	}

	boolean useClientClassLoader() {
	return Boolean.parseBoolean(System.getenv(HADOOP_USE_CLIENT_CLASSLOADER));
	}

	String getHadoopClasspath() {
	return System.getenv(HADOOP_CLASSPATH);
	}

	String getSystemClasses() {
	return System.getenv(HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES);
	}
	}