| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.fs; |
| |
| import java.io.BufferedInputStream; |
| import java.io.BufferedOutputStream; |
| import java.io.BufferedReader; |
| import java.io.BufferedWriter; |
| import java.io.File; |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.InputStreamReader; |
| import java.io.OutputStream; |
| import java.io.OutputStreamWriter; |
| import java.net.InetAddress; |
| import java.net.URI; |
| import java.net.UnknownHostException; |
| import java.nio.charset.Charset; |
| import java.nio.charset.CharsetEncoder; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.AccessDeniedException; |
| import java.nio.file.FileSystems; |
| import java.nio.file.Files; |
| import java.util.ArrayList; |
| import java.util.Enumeration; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Objects; |
| import java.util.concurrent.ExecutionException; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.Future; |
| import java.util.jar.Attributes; |
| import java.util.jar.JarOutputStream; |
| import java.util.jar.Manifest; |
| import java.util.zip.GZIPInputStream; |
| import java.util.zip.ZipEntry; |
| import java.util.zip.ZipFile; |
| import java.util.zip.ZipInputStream; |
| |
| import org.apache.commons.collections.map.CaseInsensitiveMap; |
| import org.apache.commons.compress.archivers.tar.TarArchiveEntry; |
| import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; |
| import org.apache.hadoop.classification.InterfaceAudience; |
| import org.apache.hadoop.classification.InterfaceStability; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.permission.FsAction; |
| import org.apache.hadoop.fs.permission.FsPermission; |
| import org.apache.hadoop.io.IOUtils; |
| import org.apache.hadoop.io.nativeio.NativeIO; |
| import org.apache.hadoop.util.Shell; |
| import org.apache.hadoop.util.Shell.ShellCommandExecutor; |
| import org.apache.hadoop.util.StringUtils; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * A collection of file-processing util methods |
| */ |
| @InterfaceAudience.Public |
| @InterfaceStability.Evolving |
| public class FileUtil { |
| |
| private static final Logger LOG = LoggerFactory.getLogger(FileUtil.class); |
| |
| /* The error code is defined in winutils to indicate insufficient |
| * privilege to create symbolic links. This value need to keep in |
| * sync with the constant of the same name in: |
| * "src\winutils\common.h" |
| * */ |
| public static final int SYMLINK_NO_PRIVILEGE = 2; |
| |
| /** |
| * Buffer size for copy the content of compressed file to new file. |
| */ |
| private static final int BUFFER_SIZE = 8_192; |
| |
| /** |
| * convert an array of FileStatus to an array of Path |
| * |
| * @param stats |
| * an array of FileStatus objects |
| * @return an array of paths corresponding to the input |
| */ |
| public static Path[] stat2Paths(FileStatus[] stats) { |
| if (stats == null) |
| return null; |
| Path[] ret = new Path[stats.length]; |
| for (int i = 0; i < stats.length; ++i) { |
| ret[i] = stats[i].getPath(); |
| } |
| return ret; |
| } |
| |
| /** |
| * convert an array of FileStatus to an array of Path. |
| * If stats if null, return path |
| * @param stats |
| * an array of FileStatus objects |
| * @param path |
| * default path to return in stats is null |
| * @return an array of paths corresponding to the input |
| */ |
| public static Path[] stat2Paths(FileStatus[] stats, Path path) { |
| if (stats == null) |
| return new Path[]{path}; |
| else |
| return stat2Paths(stats); |
| } |
| |
| /** |
| * Register all files recursively to be deleted on exit. |
| * @param file File/directory to be deleted |
| */ |
| public static void fullyDeleteOnExit(final File file) { |
| file.deleteOnExit(); |
| if (file.isDirectory()) { |
| File[] files = file.listFiles(); |
| if (files != null) { |
| for (File child : files) { |
| fullyDeleteOnExit(child); |
| } |
| } |
| } |
| } |
| |
| /** |
| * Delete a directory and all its contents. If |
| * we return false, the directory may be partially-deleted. |
| * (1) If dir is symlink to a file, the symlink is deleted. The file pointed |
| * to by the symlink is not deleted. |
| * (2) If dir is symlink to a directory, symlink is deleted. The directory |
| * pointed to by symlink is not deleted. |
| * (3) If dir is a normal file, it is deleted. |
| * (4) If dir is a normal directory, then dir and all its contents recursively |
| * are deleted. |
| */ |
| public static boolean fullyDelete(final File dir) { |
| return fullyDelete(dir, false); |
| } |
| |
| /** |
| * Delete a directory and all its contents. If |
| * we return false, the directory may be partially-deleted. |
| * (1) If dir is symlink to a file, the symlink is deleted. The file pointed |
| * to by the symlink is not deleted. |
| * (2) If dir is symlink to a directory, symlink is deleted. The directory |
| * pointed to by symlink is not deleted. |
| * (3) If dir is a normal file, it is deleted. |
| * (4) If dir is a normal directory, then dir and all its contents recursively |
| * are deleted. |
| * @param dir the file or directory to be deleted |
| * @param tryGrantPermissions true if permissions should be modified to delete a file. |
| * @return true on success false on failure. |
| */ |
| public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) { |
| if (tryGrantPermissions) { |
| // try to chmod +rwx the parent folder of the 'dir': |
| File parent = dir.getParentFile(); |
| grantPermissions(parent); |
| } |
| if (deleteImpl(dir, false)) { |
| // dir is (a) normal file, (b) symlink to a file, (c) empty directory or |
| // (d) symlink to a directory |
| return true; |
| } |
| // handle nonempty directory deletion |
| if (!fullyDeleteContents(dir, tryGrantPermissions)) { |
| return false; |
| } |
| return deleteImpl(dir, true); |
| } |
| |
| /** |
| * Returns the target of the given symlink. Returns the empty string if |
| * the given path does not refer to a symlink or there is an error |
| * accessing the symlink. |
| * @param f File representing the symbolic link. |
| * @return The target of the symbolic link, empty string on error or if not |
| * a symlink. |
| */ |
| public static String readLink(File f) { |
| /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could |
| * use getCanonicalPath in File to get the target of the symlink but that |
| * does not indicate if the given path refers to a symlink. |
| */ |
| |
| if (f == null) { |
| LOG.warn("Can not read a null symLink"); |
| return ""; |
| } |
| |
| try { |
| return Shell.execCommand( |
| Shell.getReadlinkCommand(f.toString())).trim(); |
| } catch (IOException x) { |
| return ""; |
| } |
| } |
| |
| /* |
| * Pure-Java implementation of "chmod +rwx f". |
| */ |
| private static void grantPermissions(final File f) { |
| FileUtil.setExecutable(f, true); |
| FileUtil.setReadable(f, true); |
| FileUtil.setWritable(f, true); |
| } |
| |
| private static boolean deleteImpl(final File f, final boolean doLog) { |
| if (f == null) { |
| LOG.warn("null file argument."); |
| return false; |
| } |
| final boolean wasDeleted = f.delete(); |
| if (wasDeleted) { |
| return true; |
| } |
| final boolean ex = f.exists(); |
| if (doLog && ex) { |
| LOG.warn("Failed to delete file or dir [" |
| + f.getAbsolutePath() + "]: it still exists."); |
| } |
| return !ex; |
| } |
| |
| /** |
| * Delete the contents of a directory, not the directory itself. If |
| * we return false, the directory may be partially-deleted. |
| * If dir is a symlink to a directory, all the contents of the actual |
| * directory pointed to by dir will be deleted. |
| */ |
| public static boolean fullyDeleteContents(final File dir) { |
| return fullyDeleteContents(dir, false); |
| } |
| |
| /** |
| * Delete the contents of a directory, not the directory itself. If |
| * we return false, the directory may be partially-deleted. |
| * If dir is a symlink to a directory, all the contents of the actual |
| * directory pointed to by dir will be deleted. |
| * @param tryGrantPermissions if 'true', try grant +rwx permissions to this |
| * and all the underlying directories before trying to delete their contents. |
| */ |
| public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) { |
| if (tryGrantPermissions) { |
| // to be able to list the dir and delete files from it |
| // we must grant the dir rwx permissions: |
| grantPermissions(dir); |
| } |
| boolean deletionSucceeded = true; |
| final File[] contents = dir.listFiles(); |
| if (contents != null) { |
| for (int i = 0; i < contents.length; i++) { |
| if (contents[i].isFile()) { |
| if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file |
| deletionSucceeded = false; |
| continue; // continue deletion of other files/dirs under dir |
| } |
| } else { |
| // Either directory or symlink to another directory. |
| // Try deleting the directory as this might be a symlink |
| boolean b = false; |
| b = deleteImpl(contents[i], false); |
| if (b){ |
| //this was indeed a symlink or an empty directory |
| continue; |
| } |
| // if not an empty directory or symlink let |
| // fullydelete handle it. |
| if (!fullyDelete(contents[i], tryGrantPermissions)) { |
| deletionSucceeded = false; |
| // continue deletion of other files/dirs under dir |
| } |
| } |
| } |
| } |
| return deletionSucceeded; |
| } |
| |
| /** |
| * Recursively delete a directory. |
| * |
| * @param fs {@link FileSystem} on which the path is present |
| * @param dir directory to recursively delete |
| * @throws IOException |
| * @deprecated Use {@link FileSystem#delete(Path, boolean)} |
| */ |
| @Deprecated |
| public static void fullyDelete(FileSystem fs, Path dir) |
| throws IOException { |
| fs.delete(dir, true); |
| } |
| |
| // |
| // If the destination is a subdirectory of the source, then |
| // generate exception |
| // |
| private static void checkDependencies(FileSystem srcFS, |
| Path src, |
| FileSystem dstFS, |
| Path dst) |
| throws IOException { |
| if (srcFS == dstFS) { |
| String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR; |
| String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR; |
| if (dstq.startsWith(srcq)) { |
| if (srcq.length() == dstq.length()) { |
| throw new IOException("Cannot copy " + src + " to itself."); |
| } else { |
| throw new IOException("Cannot copy " + src + " to its subdirectory " + |
| dst); |
| } |
| } |
| } |
| } |
| |
| /** Copy files between FileSystems. */ |
| public static boolean copy(FileSystem srcFS, Path src, |
| FileSystem dstFS, Path dst, |
| boolean deleteSource, |
| Configuration conf) throws IOException { |
| return copy(srcFS, src, dstFS, dst, deleteSource, true, conf); |
| } |
| |
| public static boolean copy(FileSystem srcFS, Path[] srcs, |
| FileSystem dstFS, Path dst, |
| boolean deleteSource, |
| boolean overwrite, Configuration conf) |
| throws IOException { |
| boolean gotException = false; |
| boolean returnVal = true; |
| StringBuilder exceptions = new StringBuilder(); |
| |
| if (srcs.length == 1) |
| return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf); |
| |
| // Check if dest is directory |
| try { |
| FileStatus sdst = dstFS.getFileStatus(dst); |
| if (!sdst.isDirectory()) |
| throw new IOException("copying multiple files, but last argument `" + |
| dst + "' is not a directory"); |
| } catch (FileNotFoundException e) { |
| throw new IOException( |
| "`" + dst + "': specified destination directory " + |
| "does not exist", e); |
| } |
| |
| for (Path src : srcs) { |
| try { |
| if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf)) |
| returnVal = false; |
| } catch (IOException e) { |
| gotException = true; |
| exceptions.append(e.getMessage()) |
| .append("\n"); |
| } |
| } |
| if (gotException) { |
| throw new IOException(exceptions.toString()); |
| } |
| return returnVal; |
| } |
| |
| /** Copy files between FileSystems. */ |
| public static boolean copy(FileSystem srcFS, Path src, |
| FileSystem dstFS, Path dst, |
| boolean deleteSource, |
| boolean overwrite, |
| Configuration conf) throws IOException { |
| FileStatus fileStatus = srcFS.getFileStatus(src); |
| return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf); |
| } |
| |
| /** Copy files between FileSystems. */ |
| public static boolean copy(FileSystem srcFS, FileStatus srcStatus, |
| FileSystem dstFS, Path dst, |
| boolean deleteSource, |
| boolean overwrite, |
| Configuration conf) throws IOException { |
| Path src = srcStatus.getPath(); |
| dst = checkDest(src.getName(), dstFS, dst, overwrite); |
| |
| if (srcFS.makeQualified(src).equals(dstFS.makeQualified(dst))) { |
| throw new PathOperationException("Source (" + src + ") and destination " + |
| "(" + dst + ") are equal in the copy command."); |
| } |
| |
| if (srcStatus.isDirectory()) { |
| checkDependencies(srcFS, src, dstFS, dst); |
| if (!dstFS.mkdirs(dst)) { |
| return false; |
| } |
| FileStatus contents[] = srcFS.listStatus(src); |
| for (int i = 0; i < contents.length; i++) { |
| copy(srcFS, contents[i], dstFS, |
| new Path(dst, contents[i].getPath().getName()), |
| deleteSource, overwrite, conf); |
| } |
| } else { |
| InputStream in=null; |
| OutputStream out = null; |
| try { |
| in = srcFS.open(src); |
| out = dstFS.create(dst, overwrite); |
| IOUtils.copyBytes(in, out, conf, true); |
| } catch (IOException e) { |
| IOUtils.closeStream(out); |
| IOUtils.closeStream(in); |
| throw e; |
| } |
| } |
| if (deleteSource) { |
| return srcFS.delete(src, true); |
| } else { |
| return true; |
| } |
| |
| } |
| |
| /** Copy local files to a FileSystem. */ |
| public static boolean copy(File src, |
| FileSystem dstFS, Path dst, |
| boolean deleteSource, |
| Configuration conf) throws IOException { |
| dst = checkDest(src.getName(), dstFS, dst, false); |
| |
| if (src.isDirectory()) { |
| if (!dstFS.mkdirs(dst)) { |
| return false; |
| } |
| File contents[] = listFiles(src); |
| for (int i = 0; i < contents.length; i++) { |
| copy(contents[i], dstFS, new Path(dst, contents[i].getName()), |
| deleteSource, conf); |
| } |
| } else if (src.isFile()) { |
| InputStream in = null; |
| OutputStream out =null; |
| try { |
| in = Files.newInputStream(src.toPath()); |
| out = dstFS.create(dst); |
| IOUtils.copyBytes(in, out, conf); |
| } catch (IOException e) { |
| IOUtils.closeStream( out ); |
| IOUtils.closeStream( in ); |
| throw e; |
| } |
| } else if (!src.canRead()) { |
| throw new IOException(src.toString() + |
| ": Permission denied"); |
| |
| } else { |
| throw new IOException(src.toString() + |
| ": No such file or directory"); |
| } |
| if (deleteSource) { |
| return FileUtil.fullyDelete(src); |
| } else { |
| return true; |
| } |
| } |
| |
| /** Copy FileSystem files to local files. */ |
| public static boolean copy(FileSystem srcFS, Path src, |
| File dst, boolean deleteSource, |
| Configuration conf) throws IOException { |
| FileStatus filestatus = srcFS.getFileStatus(src); |
| return copy(srcFS, filestatus, dst, deleteSource, conf); |
| } |
| |
| /** Copy FileSystem files to local files. */ |
| private static boolean copy(FileSystem srcFS, FileStatus srcStatus, |
| File dst, boolean deleteSource, |
| Configuration conf) throws IOException { |
| Path src = srcStatus.getPath(); |
| if (srcStatus.isDirectory()) { |
| if (!dst.mkdirs()) { |
| return false; |
| } |
| FileStatus contents[] = srcFS.listStatus(src); |
| for (int i = 0; i < contents.length; i++) { |
| copy(srcFS, contents[i], |
| new File(dst, contents[i].getPath().getName()), |
| deleteSource, conf); |
| } |
| } else { |
| InputStream in = srcFS.open(src); |
| IOUtils.copyBytes(in, Files.newOutputStream(dst.toPath()), conf); |
| } |
| if (deleteSource) { |
| return srcFS.delete(src, true); |
| } else { |
| return true; |
| } |
| } |
| |
| private static Path checkDest(String srcName, FileSystem dstFS, Path dst, |
| boolean overwrite) throws IOException { |
| FileStatus sdst; |
| try { |
| sdst = dstFS.getFileStatus(dst); |
| } catch (FileNotFoundException e) { |
| sdst = null; |
| } |
| if (null != sdst) { |
| if (sdst.isDirectory()) { |
| if (null == srcName) { |
| throw new PathIsDirectoryException(dst.toString()); |
| } |
| return checkDest(null, dstFS, new Path(dst, srcName), overwrite); |
| } else if (!overwrite) { |
| throw new PathExistsException(dst.toString(), |
| "Target " + dst + " already exists"); |
| } |
| } |
| return dst; |
| } |
| |
| /** |
| * Convert a os-native filename to a path that works for the shell. |
| * @param filename The filename to convert |
| * @return The unix pathname |
| * @throws IOException on windows, there can be problems with the subprocess |
| */ |
| public static String makeShellPath(String filename) throws IOException { |
| return filename; |
| } |
| |
| /** |
| * Convert a os-native filename to a path that works for the shell. |
| * @param file The filename to convert |
| * @return The unix pathname |
| * @throws IOException on windows, there can be problems with the subprocess |
| */ |
| public static String makeShellPath(File file) throws IOException { |
| return makeShellPath(file, false); |
| } |
| |
| /** |
| * Convert a os-native filename to a path that works for the shell |
| * and avoids script injection attacks. |
| * @param file The filename to convert |
| * @return The unix pathname |
| * @throws IOException on windows, there can be problems with the subprocess |
| */ |
| public static String makeSecureShellPath(File file) throws IOException { |
| if (Shell.WINDOWS) { |
| // Currently it is never called, but it might be helpful in the future. |
| throw new UnsupportedOperationException("Not implemented for Windows"); |
| } else { |
| return makeShellPath(file, false).replace("'", "'\\''"); |
| } |
| } |
| |
| /** |
| * Convert a os-native filename to a path that works for the shell. |
| * @param file The filename to convert |
| * @param makeCanonicalPath |
| * Whether to make canonical path for the file passed |
| * @return The unix pathname |
| * @throws IOException on windows, there can be problems with the subprocess |
| */ |
| public static String makeShellPath(File file, boolean makeCanonicalPath) |
| throws IOException { |
| if (makeCanonicalPath) { |
| return makeShellPath(file.getCanonicalPath()); |
| } else { |
| return makeShellPath(file.toString()); |
| } |
| } |
| |
| /** |
| * Takes an input dir and returns the du on that local directory. Very basic |
| * implementation. |
| * |
| * @param dir |
| * The input dir to get the disk space of this local dir |
| * @return The total disk space of the input local directory |
| */ |
| public static long getDU(File dir) { |
| long size = 0; |
| if (!dir.exists()) |
| return 0; |
| if (!dir.isDirectory()) { |
| return dir.length(); |
| } else { |
| File[] allFiles = dir.listFiles(); |
| if (allFiles != null) { |
| for (File f : allFiles) { |
| if (!org.apache.commons.io.FileUtils.isSymlink(f)) { |
| size += getDU(f); |
| } |
| } |
| } |
| return size; |
| } |
| } |
| |
| /** |
| * Given a stream input it will unzip the it in the unzip directory. |
| * passed as the second parameter |
| * @param inputStream The zip file as input |
| * @param toDir The unzip directory where to unzip the zip file. |
| * @throws IOException an exception occurred |
| */ |
| public static void unZip(InputStream inputStream, File toDir) |
| throws IOException { |
| try (ZipInputStream zip = new ZipInputStream(inputStream)) { |
| int numOfFailedLastModifiedSet = 0; |
| String targetDirPath = toDir.getCanonicalPath() + File.separator; |
| for(ZipEntry entry = zip.getNextEntry(); |
| entry != null; |
| entry = zip.getNextEntry()) { |
| if (!entry.isDirectory()) { |
| File file = new File(toDir, entry.getName()); |
| if (!file.getCanonicalPath().startsWith(targetDirPath)) { |
| throw new IOException("expanding " + entry.getName() |
| + " would create file outside of " + toDir); |
| } |
| File parent = file.getParentFile(); |
| if (!parent.mkdirs() && |
| !parent.isDirectory()) { |
| throw new IOException("Mkdirs failed to create " + |
| parent.getAbsolutePath()); |
| } |
| try (OutputStream out = Files.newOutputStream(file.toPath())) { |
| IOUtils.copyBytes(zip, out, BUFFER_SIZE); |
| } |
| if (!file.setLastModified(entry.getTime())) { |
| numOfFailedLastModifiedSet++; |
| } |
| } |
| } |
| if (numOfFailedLastModifiedSet > 0) { |
| LOG.warn("Could not set last modfied time for {} file(s)", |
| numOfFailedLastModifiedSet); |
| } |
| } |
| } |
| |
| /** |
| * Given a File input it will unzip it in the unzip directory. |
| * passed as the second parameter |
| * @param inFile The zip file as input |
| * @param unzipDir The unzip directory where to unzip the zip file. |
| * @throws IOException An I/O exception has occurred |
| */ |
| public static void unZip(File inFile, File unzipDir) throws IOException { |
| Enumeration<? extends ZipEntry> entries; |
| ZipFile zipFile = new ZipFile(inFile); |
| |
| try { |
| entries = zipFile.entries(); |
| String targetDirPath = unzipDir.getCanonicalPath() + File.separator; |
| while (entries.hasMoreElements()) { |
| ZipEntry entry = entries.nextElement(); |
| if (!entry.isDirectory()) { |
| InputStream in = zipFile.getInputStream(entry); |
| try { |
| File file = new File(unzipDir, entry.getName()); |
| if (!file.getCanonicalPath().startsWith(targetDirPath)) { |
| throw new IOException("expanding " + entry.getName() |
| + " would create file outside of " + unzipDir); |
| } |
| if (!file.getParentFile().mkdirs()) { |
| if (!file.getParentFile().isDirectory()) { |
| throw new IOException("Mkdirs failed to create " + |
| file.getParentFile().toString()); |
| } |
| } |
| OutputStream out = Files.newOutputStream(file.toPath()); |
| try { |
| byte[] buffer = new byte[8192]; |
| int i; |
| while ((i = in.read(buffer)) != -1) { |
| out.write(buffer, 0, i); |
| } |
| } finally { |
| out.close(); |
| } |
| } finally { |
| in.close(); |
| } |
| } |
| } |
| } finally { |
| zipFile.close(); |
| } |
| } |
| |
| /** |
| * Run a command and send the contents of an input stream to it. |
| * @param inputStream Input stream to forward to the shell command |
| * @param command shell command to run |
| * @throws IOException read or write failed |
| * @throws InterruptedException command interrupted |
| * @throws ExecutionException task submit failed |
| */ |
| private static void runCommandOnStream( |
| InputStream inputStream, String command) |
| throws IOException, InterruptedException, ExecutionException { |
| ExecutorService executor = null; |
| ProcessBuilder builder = new ProcessBuilder(); |
| builder.command( |
| Shell.WINDOWS ? "cmd" : "bash", |
| Shell.WINDOWS ? "/c" : "-c", |
| command); |
| Process process = builder.start(); |
| int exitCode; |
| try { |
| // Consume stdout and stderr, to avoid blocking the command |
| executor = Executors.newFixedThreadPool(2); |
| Future output = executor.submit(() -> { |
| try { |
| // Read until the output stream receives an EOF and closed. |
| if (LOG.isDebugEnabled()) { |
| // Log directly to avoid out of memory errors |
| try (BufferedReader reader = |
| new BufferedReader( |
| new InputStreamReader(process.getInputStream(), |
| Charset.forName("UTF-8")))) { |
| String line; |
| while((line = reader.readLine()) != null) { |
| LOG.debug(line); |
| } |
| } |
| } else { |
| org.apache.commons.io.IOUtils.copy( |
| process.getInputStream(), |
| new IOUtils.NullOutputStream()); |
| } |
| } catch (IOException e) { |
| LOG.debug(e.getMessage()); |
| } |
| }); |
| Future error = executor.submit(() -> { |
| try { |
| // Read until the error stream receives an EOF and closed. |
| if (LOG.isDebugEnabled()) { |
| // Log directly to avoid out of memory errors |
| try (BufferedReader reader = |
| new BufferedReader( |
| new InputStreamReader(process.getErrorStream(), |
| Charset.forName("UTF-8")))) { |
| String line; |
| while((line = reader.readLine()) != null) { |
| LOG.debug(line); |
| } |
| } |
| } else { |
| org.apache.commons.io.IOUtils.copy( |
| process.getErrorStream(), |
| new IOUtils.NullOutputStream()); |
| } |
| } catch (IOException e) { |
| LOG.debug(e.getMessage()); |
| } |
| }); |
| |
| // Pass the input stream to the command to process |
| try { |
| org.apache.commons.io.IOUtils.copy( |
| inputStream, process.getOutputStream()); |
| } finally { |
| process.getOutputStream().close(); |
| } |
| |
| // Wait for both stdout and stderr futures to finish |
| error.get(); |
| output.get(); |
| } finally { |
| // Clean up the threads |
| if (executor != null) { |
| executor.shutdown(); |
| } |
| // Wait to avoid leaking the child process |
| exitCode = process.waitFor(); |
| } |
| |
| if (exitCode != 0) { |
| throw new IOException( |
| String.format( |
| "Error executing command. %s " + |
| "Process exited with exit code %d.", |
| command, exitCode)); |
| } |
| } |
| |
| /** |
| * Given a Tar File as input it will untar the file in a the untar directory |
| * passed as the second parameter |
| * |
| * This utility will untar ".tar" files and ".tar.gz","tgz" files. |
| * |
| * @param inputStream The tar file as input. |
| * @param untarDir The untar directory where to untar the tar file. |
| * @param gzipped The input stream is gzipped |
| * TODO Use magic number and PusbackInputStream to identify |
| * @throws IOException an exception occurred |
| * @throws InterruptedException command interrupted |
| * @throws ExecutionException task submit failed |
| */ |
| public static void unTar(InputStream inputStream, File untarDir, |
| boolean gzipped) |
| throws IOException, InterruptedException, ExecutionException { |
| if (!untarDir.mkdirs()) { |
| if (!untarDir.isDirectory()) { |
| throw new IOException("Mkdirs failed to create " + untarDir); |
| } |
| } |
| |
| if(Shell.WINDOWS) { |
| // Tar is not native to Windows. Use simple Java based implementation for |
| // tests and simple tar archives |
| unTarUsingJava(inputStream, untarDir, gzipped); |
| } else { |
| // spawn tar utility to untar archive for full fledged unix behavior such |
| // as resolving symlinks in tar archives |
| unTarUsingTar(inputStream, untarDir, gzipped); |
| } |
| } |
| |
| /** |
| * Given a Tar File as input it will untar the file in a the untar directory |
| * passed as the second parameter |
| * |
| * This utility will untar ".tar" files and ".tar.gz","tgz" files. |
| * |
| * @param inFile The tar file as input. |
| * @param untarDir The untar directory where to untar the tar file. |
| * @throws IOException |
| */ |
| public static void unTar(File inFile, File untarDir) throws IOException { |
| if (!untarDir.mkdirs()) { |
| if (!untarDir.isDirectory()) { |
| throw new IOException("Mkdirs failed to create " + untarDir); |
| } |
| } |
| |
| boolean gzipped = inFile.toString().endsWith("gz"); |
| if(Shell.WINDOWS) { |
| // Tar is not native to Windows. Use simple Java based implementation for |
| // tests and simple tar archives |
| unTarUsingJava(inFile, untarDir, gzipped); |
| } |
| else { |
| // spawn tar utility to untar archive for full fledged unix behavior such |
| // as resolving symlinks in tar archives |
| unTarUsingTar(inFile, untarDir, gzipped); |
| } |
| } |
| |
| private static void unTarUsingTar(InputStream inputStream, File untarDir, |
| boolean gzipped) |
| throws IOException, InterruptedException, ExecutionException { |
| StringBuilder untarCommand = new StringBuilder(); |
| if (gzipped) { |
| untarCommand.append("gzip -dc | ("); |
| } |
| untarCommand.append("cd '") |
| .append(FileUtil.makeSecureShellPath(untarDir)) |
| .append("' && ") |
| .append("tar -x "); |
| |
| if (gzipped) { |
| untarCommand.append(")"); |
| } |
| runCommandOnStream(inputStream, untarCommand.toString()); |
| } |
| |
| private static void unTarUsingTar(File inFile, File untarDir, |
| boolean gzipped) throws IOException { |
| StringBuffer untarCommand = new StringBuffer(); |
| if (gzipped) { |
| untarCommand.append(" gzip -dc '") |
| .append(FileUtil.makeSecureShellPath(inFile)) |
| .append("' | ("); |
| } |
| untarCommand.append("cd '") |
| .append(FileUtil.makeSecureShellPath(untarDir)) |
| .append("' && ") |
| .append("tar -xf "); |
| |
| if (gzipped) { |
| untarCommand.append(" -)"); |
| } else { |
| untarCommand.append(FileUtil.makeSecureShellPath(inFile)); |
| } |
| String[] shellCmd = { "bash", "-c", untarCommand.toString() }; |
| ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); |
| shexec.execute(); |
| int exitcode = shexec.getExitCode(); |
| if (exitcode != 0) { |
| throw new IOException("Error untarring file " + inFile + |
| ". Tar process exited with exit code " + exitcode); |
| } |
| } |
| |
| static void unTarUsingJava(File inFile, File untarDir, |
| boolean gzipped) throws IOException { |
| InputStream inputStream = null; |
| TarArchiveInputStream tis = null; |
| try { |
| if (gzipped) { |
| inputStream = |
| new GZIPInputStream(Files.newInputStream(inFile.toPath())); |
| } else { |
| inputStream = Files.newInputStream(inFile.toPath()); |
| } |
| |
| inputStream = new BufferedInputStream(inputStream); |
| |
| tis = new TarArchiveInputStream(inputStream); |
| |
| for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { |
| unpackEntries(tis, entry, untarDir); |
| entry = tis.getNextTarEntry(); |
| } |
| } finally { |
| IOUtils.cleanupWithLogger(LOG, tis, inputStream); |
| } |
| } |
| |
| private static void unTarUsingJava(InputStream inputStream, File untarDir, |
| boolean gzipped) throws IOException { |
| TarArchiveInputStream tis = null; |
| try { |
| if (gzipped) { |
| inputStream = new GZIPInputStream(inputStream); |
| } |
| inputStream = new BufferedInputStream(inputStream); |
| tis = new TarArchiveInputStream(inputStream); |
| |
| for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { |
| unpackEntries(tis, entry, untarDir); |
| entry = tis.getNextTarEntry(); |
| } |
| } finally { |
| IOUtils.cleanupWithLogger(LOG, tis, inputStream); |
| } |
| } |
| |
| private static void unpackEntries(TarArchiveInputStream tis, |
| TarArchiveEntry entry, File outputDir) throws IOException { |
| String targetDirPath = outputDir.getCanonicalPath() + File.separator; |
| File outputFile = new File(outputDir, entry.getName()); |
| if (!outputFile.getCanonicalPath().startsWith(targetDirPath)) { |
| throw new IOException("expanding " + entry.getName() |
| + " would create entry outside of " + outputDir); |
| } |
| |
| if (entry.isDirectory()) { |
| File subDir = new File(outputDir, entry.getName()); |
| if (!subDir.mkdirs() && !subDir.isDirectory()) { |
| throw new IOException("Mkdirs failed to create tar internal dir " |
| + outputDir); |
| } |
| |
| for (TarArchiveEntry e : entry.getDirectoryEntries()) { |
| unpackEntries(tis, e, subDir); |
| } |
| |
| return; |
| } |
| |
| if (entry.isSymbolicLink()) { |
| // Create symbolic link relative to tar parent dir |
| Files.createSymbolicLink(FileSystems.getDefault() |
| .getPath(outputDir.getPath(), entry.getName()), |
| FileSystems.getDefault().getPath(entry.getLinkName())); |
| return; |
| } |
| |
| if (!outputFile.getParentFile().exists()) { |
| if (!outputFile.getParentFile().mkdirs()) { |
| throw new IOException("Mkdirs failed to create tar internal dir " |
| + outputDir); |
| } |
| } |
| |
| if (entry.isLink()) { |
| File src = new File(outputDir, entry.getLinkName()); |
| HardLink.createHardLink(src, outputFile); |
| return; |
| } |
| |
| org.apache.commons.io.FileUtils.copyToFile(tis, outputFile); |
| } |
| |
| /** |
| * Class for creating hardlinks. |
| * Supports Unix, WindXP. |
| * @deprecated Use {@link org.apache.hadoop.fs.HardLink} |
| */ |
| @Deprecated |
| public static class HardLink extends org.apache.hadoop.fs.HardLink { |
| // This is a stub to assist with coordinated change between |
| // COMMON and HDFS projects. It will be removed after the |
| // corresponding change is committed to HDFS. |
| } |
| |
| /** |
| * Create a soft link between a src and destination |
| * only on a local disk. HDFS does not support this. |
| * On Windows, when symlink creation fails due to security |
| * setting, we will log a warning. The return code in this |
| * case is 2. |
| * |
| * @param target the target for symlink |
| * @param linkname the symlink |
| * @return 0 on success |
| */ |
| public static int symLink(String target, String linkname) throws IOException{ |
| |
| if (target == null || linkname == null) { |
| LOG.warn("Can not create a symLink with a target = " + target |
| + " and link =" + linkname); |
| return 1; |
| } |
| |
| // Run the input paths through Java's File so that they are converted to the |
| // native OS form |
| File targetFile = new File( |
| Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString()); |
| File linkFile = new File( |
| Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString()); |
| |
| String[] cmd = Shell.getSymlinkCommand( |
| targetFile.toString(), |
| linkFile.toString()); |
| |
| ShellCommandExecutor shExec; |
| try { |
| if (Shell.WINDOWS && |
| linkFile.getParentFile() != null && |
| !new Path(target).isAbsolute()) { |
| // Relative links on Windows must be resolvable at the time of |
| // creation. To ensure this we run the shell command in the directory |
| // of the link. |
| // |
| shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile()); |
| } else { |
| shExec = new ShellCommandExecutor(cmd); |
| } |
| shExec.execute(); |
| } catch (Shell.ExitCodeException ec) { |
| int returnVal = ec.getExitCode(); |
| if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) { |
| LOG.warn("Fail to create symbolic links on Windows. " |
| + "The default security settings in Windows disallow non-elevated " |
| + "administrators and all non-administrators from creating symbolic links. " |
| + "This behavior can be changed in the Local Security Policy management console"); |
| } else if (returnVal != 0) { |
| LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed " |
| + returnVal + " with: " + ec.getMessage()); |
| } |
| return returnVal; |
| } catch (IOException e) { |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Error while create symlink " + linkname + " to " + target |
| + "." + " Exception: " + StringUtils.stringifyException(e)); |
| } |
| throw e; |
| } |
| return shExec.getExitCode(); |
| } |
| |
| /** |
| * Change the permissions on a filename. |
| * @param filename the name of the file to change |
| * @param perm the permission string |
| * @return the exit code from the command |
| * @throws IOException |
| * @throws InterruptedException |
| */ |
| public static int chmod(String filename, String perm |
| ) throws IOException, InterruptedException { |
| return chmod(filename, perm, false); |
| } |
| |
| /** |
| * Change the permissions on a file / directory, recursively, if |
| * needed. |
| * @param filename name of the file whose permissions are to change |
| * @param perm permission string |
| * @param recursive true, if permissions should be changed recursively |
| * @return the exit code from the command. |
| * @throws IOException |
| */ |
| public static int chmod(String filename, String perm, boolean recursive) |
| throws IOException { |
| String [] cmd = Shell.getSetPermissionCommand(perm, recursive); |
| String[] args = new String[cmd.length + 1]; |
| System.arraycopy(cmd, 0, args, 0, cmd.length); |
| args[cmd.length] = new File(filename).getPath(); |
| ShellCommandExecutor shExec = new ShellCommandExecutor(args); |
| try { |
| shExec.execute(); |
| }catch(IOException e) { |
| if(LOG.isDebugEnabled()) { |
| LOG.debug("Error while changing permission : " + filename |
| +" Exception: " + StringUtils.stringifyException(e)); |
| } |
| } |
| return shExec.getExitCode(); |
| } |
| |
| /** |
| * Set the ownership on a file / directory. User name and group name |
| * cannot both be null. |
| * @param file the file to change |
| * @param username the new user owner name |
| * @param groupname the new group owner name |
| * @throws IOException |
| */ |
| public static void setOwner(File file, String username, |
| String groupname) throws IOException { |
| if (username == null && groupname == null) { |
| throw new IOException("username == null && groupname == null"); |
| } |
| String arg = (username == null ? "" : username) |
| + (groupname == null ? "" : ":" + groupname); |
| String [] cmd = Shell.getSetOwnerCommand(arg); |
| execCommand(file, cmd); |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#setReadable(boolean)} |
| * File#setReadable does not work as expected on Windows. |
| * @param f input file |
| * @param readable |
| * @return true on success, false otherwise |
| */ |
| public static boolean setReadable(File f, boolean readable) { |
| if (Shell.WINDOWS) { |
| try { |
| String permission = readable ? "u+r" : "u-r"; |
| FileUtil.chmod(f.getCanonicalPath(), permission, false); |
| return true; |
| } catch (IOException ex) { |
| return false; |
| } |
| } else { |
| return f.setReadable(readable); |
| } |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#setWritable(boolean)} |
| * File#setWritable does not work as expected on Windows. |
| * @param f input file |
| * @param writable |
| * @return true on success, false otherwise |
| */ |
| public static boolean setWritable(File f, boolean writable) { |
| if (Shell.WINDOWS) { |
| try { |
| String permission = writable ? "u+w" : "u-w"; |
| FileUtil.chmod(f.getCanonicalPath(), permission, false); |
| return true; |
| } catch (IOException ex) { |
| return false; |
| } |
| } else { |
| return f.setWritable(writable); |
| } |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#setExecutable(boolean)} |
| * File#setExecutable does not work as expected on Windows. |
| * Note: revoking execute permission on folders does not have the same |
| * behavior on Windows as on Unix platforms. Creating, deleting or renaming |
| * a file within that folder will still succeed on Windows. |
| * @param f input file |
| * @param executable |
| * @return true on success, false otherwise |
| */ |
| public static boolean setExecutable(File f, boolean executable) { |
| if (Shell.WINDOWS) { |
| try { |
| String permission = executable ? "u+x" : "u-x"; |
| FileUtil.chmod(f.getCanonicalPath(), permission, false); |
| return true; |
| } catch (IOException ex) { |
| return false; |
| } |
| } else { |
| return f.setExecutable(executable); |
| } |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#canRead()} |
| * @param f input file |
| * @return On Unix, same as {@link File#canRead()} |
| * On Windows, true if process has read access on the path |
| */ |
| public static boolean canRead(File f) { |
| if (Shell.WINDOWS) { |
| try { |
| return NativeIO.Windows.access(f.getCanonicalPath(), |
| NativeIO.Windows.AccessRight.ACCESS_READ); |
| } catch (IOException e) { |
| return false; |
| } |
| } else { |
| return f.canRead(); |
| } |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#canWrite()} |
| * @param f input file |
| * @return On Unix, same as {@link File#canWrite()} |
| * On Windows, true if process has write access on the path |
| */ |
| public static boolean canWrite(File f) { |
| if (Shell.WINDOWS) { |
| try { |
| return NativeIO.Windows.access(f.getCanonicalPath(), |
| NativeIO.Windows.AccessRight.ACCESS_WRITE); |
| } catch (IOException e) { |
| return false; |
| } |
| } else { |
| return f.canWrite(); |
| } |
| } |
| |
| /** |
| * Platform independent implementation for {@link File#canExecute()} |
| * @param f input file |
| * @return On Unix, same as {@link File#canExecute()} |
| * On Windows, true if process has execute access on the path |
| */ |
| public static boolean canExecute(File f) { |
| if (Shell.WINDOWS) { |
| try { |
| return NativeIO.Windows.access(f.getCanonicalPath(), |
| NativeIO.Windows.AccessRight.ACCESS_EXECUTE); |
| } catch (IOException e) { |
| return false; |
| } |
| } else { |
| return f.canExecute(); |
| } |
| } |
| |
| /** |
| * Set permissions to the required value. Uses the java primitives instead |
| * of forking if group == other. |
| * @param f the file to change |
| * @param permission the new permissions |
| * @throws IOException |
| */ |
| public static void setPermission(File f, FsPermission permission |
| ) throws IOException { |
| FsAction user = permission.getUserAction(); |
| FsAction group = permission.getGroupAction(); |
| FsAction other = permission.getOtherAction(); |
| |
| // use the native/fork if the group/other permissions are different |
| // or if the native is available or on Windows |
| if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) { |
| execSetPermission(f, permission); |
| return; |
| } |
| |
| boolean rv = true; |
| |
| // read perms |
| rv = f.setReadable(group.implies(FsAction.READ), false); |
| checkReturnValue(rv, f, permission); |
| if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { |
| rv = f.setReadable(user.implies(FsAction.READ), true); |
| checkReturnValue(rv, f, permission); |
| } |
| |
| // write perms |
| rv = f.setWritable(group.implies(FsAction.WRITE), false); |
| checkReturnValue(rv, f, permission); |
| if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { |
| rv = f.setWritable(user.implies(FsAction.WRITE), true); |
| checkReturnValue(rv, f, permission); |
| } |
| |
| // exec perms |
| rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); |
| checkReturnValue(rv, f, permission); |
| if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { |
| rv = f.setExecutable(user.implies(FsAction.EXECUTE), true); |
| checkReturnValue(rv, f, permission); |
| } |
| } |
| |
| private static void checkReturnValue(boolean rv, File p, |
| FsPermission permission |
| ) throws IOException { |
| if (!rv) { |
| throw new IOException("Failed to set permissions of path: " + p + |
| " to " + |
| String.format("%04o", permission.toShort())); |
| } |
| } |
| |
| private static void execSetPermission(File f, |
| FsPermission permission |
| ) throws IOException { |
| if (NativeIO.isAvailable()) { |
| NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort()); |
| } else { |
| execCommand(f, Shell.getSetPermissionCommand( |
| String.format("%04o", permission.toShort()), false)); |
| } |
| } |
| |
| static String execCommand(File f, String... cmd) throws IOException { |
| String[] args = new String[cmd.length + 1]; |
| System.arraycopy(cmd, 0, args, 0, cmd.length); |
| args[cmd.length] = f.getCanonicalPath(); |
| String output = Shell.execCommand(args); |
| return output; |
| } |
| |
| /** |
| * Create a tmp file for a base file. |
| * @param basefile the base file of the tmp |
| * @param prefix file name prefix of tmp |
| * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits |
| * @return a newly created tmp file |
| * @exception IOException If a tmp file cannot created |
| * @see java.io.File#createTempFile(String, String, File) |
| * @see java.io.File#deleteOnExit() |
| */ |
| public static final File createLocalTempFile(final File basefile, |
| final String prefix, |
| final boolean isDeleteOnExit) |
| throws IOException { |
| File tmp = File.createTempFile(prefix + basefile.getName(), |
| "", basefile.getParentFile()); |
| if (isDeleteOnExit) { |
| tmp.deleteOnExit(); |
| } |
| return tmp; |
| } |
| |
| /** |
| * Move the src file to the name specified by target. |
| * @param src the source file |
| * @param target the target file |
| * @exception IOException If this operation fails |
| */ |
| public static void replaceFile(File src, File target) throws IOException { |
| /* renameTo() has two limitations on Windows platform. |
| * src.renameTo(target) fails if |
| * 1) If target already exists OR |
| * 2) If target is already open for reading/writing. |
| */ |
| if (!src.renameTo(target)) { |
| int retries = 5; |
| while (target.exists() && !target.delete() && retries-- >= 0) { |
| try { |
| Thread.sleep(1000); |
| } catch (InterruptedException e) { |
| throw new IOException("replaceFile interrupted."); |
| } |
| } |
| if (!src.renameTo(target)) { |
| throw new IOException("Unable to rename " + src + |
| " to " + target); |
| } |
| } |
| } |
| |
| /** |
| * A wrapper for {@link File#listFiles()}. This java.io API returns null |
| * when a dir is not a directory or for any I/O error. Instead of having |
| * null check everywhere File#listFiles() is used, we will add utility API |
| * to get around this problem. For the majority of cases where we prefer |
| * an IOException to be thrown. |
| * @param dir directory for which listing should be performed |
| * @return list of files or empty list |
| * @exception IOException for invalid directory or for a bad disk. |
| */ |
| public static File[] listFiles(File dir) throws IOException { |
| File[] files = dir.listFiles(); |
| if(files == null) { |
| throw new IOException("Invalid directory or I/O error occurred for dir: " |
| + dir.toString()); |
| } |
| return files; |
| } |
| |
| /** |
| * A wrapper for {@link File#list()}. This java.io API returns null |
| * when a dir is not a directory or for any I/O error. Instead of having |
| * null check everywhere File#list() is used, we will add utility API |
| * to get around this problem. For the majority of cases where we prefer |
| * an IOException to be thrown. |
| * @param dir directory for which listing should be performed |
| * @return list of file names or empty string list |
| * @exception AccessDeniedException for unreadable directory |
| * @exception IOException for invalid directory or for bad disk |
| */ |
| public static String[] list(File dir) throws IOException { |
| if (!canRead(dir)) { |
| throw new AccessDeniedException(dir.toString(), null, |
| FSExceptionMessages.PERMISSION_DENIED); |
| } |
| String[] fileNames = dir.list(); |
| if(fileNames == null) { |
| throw new IOException("Invalid directory or I/O error occurred for dir: " |
| + dir.toString()); |
| } |
| return fileNames; |
| } |
| |
| public static String[] createJarWithClassPath(String inputClassPath, Path pwd, |
| Map<String, String> callerEnv) throws IOException { |
| return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv); |
| } |
| |
| /** |
| * Create a jar file at the given path, containing a manifest with a classpath |
| * that references all specified entries. |
| * |
| * Some platforms may have an upper limit on command line length. For example, |
| * the maximum command line length on Windows is 8191 characters, but the |
| * length of the classpath may exceed this. To work around this limitation, |
| * use this method to create a small intermediate jar with a manifest that |
| * contains the full classpath. It returns the absolute path to the new jar, |
| * which the caller may set as the classpath for a new process. |
| * |
| * Environment variable evaluation is not supported within a jar manifest, so |
| * this method expands environment variables before inserting classpath entries |
| * to the manifest. The method parses environment variables according to |
| * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, |
| * environment variables are case-insensitive. For example, %VAR% and %var% |
| * evaluate to the same value. |
| * |
| * Specifying the classpath in a jar manifest does not support wildcards, so |
| * this method expands wildcards internally. Any classpath entry that ends |
| * with * is translated to all files at that path with extension .jar or .JAR. |
| * |
| * @param inputClassPath String input classpath to bundle into the jar manifest |
| * @param pwd Path to working directory to save jar |
| * @param targetDir path to where the jar execution will have its working dir |
| * @param callerEnv Map {@literal <}String, String{@literal >} caller's |
| * environment variables to use for expansion |
| * @return String[] with absolute path to new jar in position 0 and |
| * unexpanded wild card entry path in position 1 |
| * @throws IOException if there is an I/O error while writing the jar file |
| */ |
| public static String[] createJarWithClassPath(String inputClassPath, Path pwd, |
| Path targetDir, |
| Map<String, String> callerEnv) throws IOException { |
| // Replace environment variables, case-insensitive on Windows |
| @SuppressWarnings("unchecked") |
| Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) : |
| callerEnv; |
| String[] classPathEntries = inputClassPath.split(File.pathSeparator); |
| for (int i = 0; i < classPathEntries.length; ++i) { |
| classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i], |
| StringUtils.ENV_VAR_PATTERN, env); |
| } |
| File workingDir = new File(pwd.toString()); |
| if (!workingDir.mkdirs()) { |
| // If mkdirs returns false because the working directory already exists, |
| // then this is acceptable. If it returns false due to some other I/O |
| // error, then this method will fail later with an IOException while saving |
| // the jar. |
| LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); |
| } |
| |
| StringBuilder unexpandedWildcardClasspath = new StringBuilder(); |
| // Append all entries |
| List<String> classPathEntryList = new ArrayList<String>( |
| classPathEntries.length); |
| for (String classPathEntry: classPathEntries) { |
| if (classPathEntry.length() == 0) { |
| continue; |
| } |
| if (classPathEntry.endsWith("*")) { |
| // Append all jars that match the wildcard |
| List<Path> jars = getJarsInDirectory(classPathEntry); |
| if (!jars.isEmpty()) { |
| for (Path jar: jars) { |
| classPathEntryList.add(jar.toUri().toURL().toExternalForm()); |
| } |
| } else { |
| unexpandedWildcardClasspath.append(File.pathSeparator) |
| .append(classPathEntry); |
| } |
| } else { |
| // Append just this entry |
| File fileCpEntry = null; |
| if(!new Path(classPathEntry).isAbsolute()) { |
| fileCpEntry = new File(targetDir.toString(), classPathEntry); |
| } |
| else { |
| fileCpEntry = new File(classPathEntry); |
| } |
| String classPathEntryUrl = fileCpEntry.toURI().toURL() |
| .toExternalForm(); |
| |
| // File.toURI only appends trailing '/' if it can determine that it is a |
| // directory that already exists. (See JavaDocs.) If this entry had a |
| // trailing '/' specified by the caller, then guarantee that the |
| // classpath entry in the manifest has a trailing '/', and thus refers to |
| // a directory instead of a file. This can happen if the caller is |
| // creating a classpath jar referencing a directory that hasn't been |
| // created yet, but will definitely be created before running. |
| if (classPathEntry.endsWith(Path.SEPARATOR) && |
| !classPathEntryUrl.endsWith(Path.SEPARATOR)) { |
| classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR; |
| } |
| classPathEntryList.add(classPathEntryUrl); |
| } |
| } |
| String jarClassPath = StringUtils.join(" ", classPathEntryList); |
| |
| // Create the manifest |
| Manifest jarManifest = new Manifest(); |
| jarManifest.getMainAttributes().putValue( |
| Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); |
| jarManifest.getMainAttributes().putValue( |
| Attributes.Name.CLASS_PATH.toString(), jarClassPath); |
| |
| // Write the manifest to output JAR file |
| File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); |
| try (OutputStream fos = Files.newOutputStream(classPathJar.toPath()); |
| BufferedOutputStream bos = new BufferedOutputStream(fos)) { |
| JarOutputStream jos = new JarOutputStream(bos, jarManifest); |
| jos.close(); |
| } |
| String[] jarCp = {classPathJar.getCanonicalPath(), |
| unexpandedWildcardClasspath.toString()}; |
| return jarCp; |
| } |
| |
| /** |
| * Returns all jars that are in the directory. It is useful in expanding a |
| * wildcard path to return all jars from the directory to use in a classpath. |
| * It operates only on local paths. |
| * |
| * @param path the path to the directory. The path may include the wildcard. |
| * @return the list of jars as URLs, or an empty list if there are no jars, or |
| * the directory does not exist locally |
| */ |
| public static List<Path> getJarsInDirectory(String path) { |
| return getJarsInDirectory(path, true); |
| } |
| |
| /** |
| * Returns all jars that are in the directory. It is useful in expanding a |
| * wildcard path to return all jars from the directory to use in a classpath. |
| * |
| * @param path the path to the directory. The path may include the wildcard. |
| * @return the list of jars as URLs, or an empty list if there are no jars, or |
| * the directory does not exist |
| */ |
| public static List<Path> getJarsInDirectory(String path, boolean useLocal) { |
| List<Path> paths = new ArrayList<>(); |
| try { |
| // add the wildcard if it is not provided |
| if (!path.endsWith("*")) { |
| path += File.separator + "*"; |
| } |
| Path globPath = new Path(path).suffix("{.jar,.JAR}"); |
| FileContext context = useLocal ? |
| FileContext.getLocalFSFileContext() : |
| FileContext.getFileContext(globPath.toUri()); |
| FileStatus[] files = context.util().globStatus(globPath); |
| if (files != null) { |
| for (FileStatus file: files) { |
| paths.add(file.getPath()); |
| } |
| } |
| } catch (IOException ignore) {} // return the empty list |
| return paths; |
| } |
| |
| public static boolean compareFs(FileSystem srcFs, FileSystem destFs) { |
| if (srcFs==null || destFs==null) { |
| return false; |
| } |
| URI srcUri = srcFs.getUri(); |
| URI dstUri = destFs.getUri(); |
| if (srcUri.getScheme()==null) { |
| return false; |
| } |
| if (!srcUri.getScheme().equals(dstUri.getScheme())) { |
| return false; |
| } |
| String srcHost = srcUri.getHost(); |
| String dstHost = dstUri.getHost(); |
| if ((srcHost!=null) && (dstHost!=null)) { |
| if (srcHost.equals(dstHost)) { |
| return srcUri.getPort()==dstUri.getPort(); |
| } |
| try { |
| srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); |
| dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); |
| } catch (UnknownHostException ue) { |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Could not compare file-systems. Unknown host: ", ue); |
| } |
| return false; |
| } |
| if (!srcHost.equals(dstHost)) { |
| return false; |
| } |
| } else if (srcHost==null && dstHost!=null) { |
| return false; |
| } else if (srcHost!=null) { |
| return false; |
| } |
| // check for ports |
| return srcUri.getPort()==dstUri.getPort(); |
| } |
| |
| /** |
| * Writes bytes to a file. This utility method opens the file for writing, |
| * creating the file if it does not exist, or overwrites an existing file. All |
| * bytes in the byte array are written to the file. |
| * |
| * @param fs the file system with which to create the file |
| * @param path the path to the file |
| * @param bytes the byte array with the bytes to write |
| * |
| * @return the file system |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileSystem write(final FileSystem fs, final Path path, |
| final byte[] bytes) throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(bytes); |
| |
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) { |
| out.write(bytes); |
| } |
| |
| return fs; |
| } |
| |
| /** |
| * Writes bytes to a file. This utility method opens the file for writing, |
| * creating the file if it does not exist, or overwrites an existing file. All |
| * bytes in the byte array are written to the file. |
| * |
| * @param fileContext the file context with which to create the file |
| * @param path the path to the file |
| * @param bytes the byte array with the bytes to write |
| * |
| * @return the file context |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileContext write(final FileContext fileContext, |
| final Path path, final byte[] bytes) throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(bytes); |
| |
| try (FSDataOutputStream out = |
| fileContext.create(path).overwrite(true).build()) { |
| out.write(bytes); |
| } |
| |
| return fileContext; |
| } |
| |
| /** |
| * Write lines of text to a file. Each line is a char sequence and is written |
| * to the file in sequence with each line terminated by the platform's line |
| * separator, as defined by the system property {@code |
| * line.separator}. Characters are encoded into bytes using the specified |
| * charset. This utility method opens the file for writing, creating the file |
| * if it does not exist, or overwrites an existing file. |
| * |
| * @param fs the file system with which to create the file |
| * @param path the path to the file |
| * @param lines a Collection to iterate over the char sequences |
| * @param cs the charset to use for encoding |
| * |
| * @return the file system |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileSystem write(final FileSystem fs, final Path path, |
| final Iterable<? extends CharSequence> lines, final Charset cs) |
| throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(lines); |
| Objects.requireNonNull(cs); |
| |
| CharsetEncoder encoder = cs.newEncoder(); |
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build(); |
| BufferedWriter writer = |
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { |
| for (CharSequence line : lines) { |
| writer.append(line); |
| writer.newLine(); |
| } |
| } |
| return fs; |
| } |
| |
| /** |
| * Write lines of text to a file. Each line is a char sequence and is written |
| * to the file in sequence with each line terminated by the platform's line |
| * separator, as defined by the system property {@code |
| * line.separator}. Characters are encoded into bytes using the specified |
| * charset. This utility method opens the file for writing, creating the file |
| * if it does not exist, or overwrites an existing file. |
| * |
| * @param fileContext the file context with which to create the file |
| * @param path the path to the file |
| * @param lines a Collection to iterate over the char sequences |
| * @param cs the charset to use for encoding |
| * |
| * @return the file context |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileContext write(final FileContext fileContext, |
| final Path path, final Iterable<? extends CharSequence> lines, |
| final Charset cs) throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(lines); |
| Objects.requireNonNull(cs); |
| |
| CharsetEncoder encoder = cs.newEncoder(); |
| try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build(); |
| BufferedWriter writer = |
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { |
| for (CharSequence line : lines) { |
| writer.append(line); |
| writer.newLine(); |
| } |
| } |
| return fileContext; |
| } |
| |
| /** |
| * Write a line of text to a file. Characters are encoded into bytes using the |
| * specified charset. This utility method opens the file for writing, creating |
| * the file if it does not exist, or overwrites an existing file. |
| * |
| * @param fs the file system with which to create the file |
| * @param path the path to the file |
| * @param charseq the char sequence to write to the file |
| * @param cs the charset to use for encoding |
| * |
| * @return the file system |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileSystem write(final FileSystem fs, final Path path, |
| final CharSequence charseq, final Charset cs) throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(charseq); |
| Objects.requireNonNull(cs); |
| |
| CharsetEncoder encoder = cs.newEncoder(); |
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build(); |
| BufferedWriter writer = |
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { |
| writer.append(charseq); |
| } |
| return fs; |
| } |
| |
| /** |
| * Write a line of text to a file. Characters are encoded into bytes using the |
| * specified charset. This utility method opens the file for writing, creating |
| * the file if it does not exist, or overwrites an existing file. |
| * |
| * @param FileContext the file context with which to create the file |
| * @param path the path to the file |
| * @param charseq the char sequence to write to the file |
| * @param cs the charset to use for encoding |
| * |
| * @return the file context |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileContext write(final FileContext fs, final Path path, |
| final CharSequence charseq, final Charset cs) throws IOException { |
| |
| Objects.requireNonNull(path); |
| Objects.requireNonNull(charseq); |
| Objects.requireNonNull(cs); |
| |
| CharsetEncoder encoder = cs.newEncoder(); |
| try (FSDataOutputStream out = fs.create(path).overwrite(true).build(); |
| BufferedWriter writer = |
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { |
| writer.append(charseq); |
| } |
| return fs; |
| } |
| |
| /** |
| * Write a line of text to a file. Characters are encoded into bytes using |
| * UTF-8. This utility method opens the file for writing, creating the file if |
| * it does not exist, or overwrites an existing file. |
| * |
| * @param fs the files system with which to create the file |
| * @param path the path to the file |
| * @param charseq the char sequence to write to the file |
| * |
| * @return the file system |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileSystem write(final FileSystem fs, final Path path, |
| final CharSequence charseq) throws IOException { |
| return write(fs, path, charseq, StandardCharsets.UTF_8); |
| } |
| |
| /** |
| * Write a line of text to a file. Characters are encoded into bytes using |
| * UTF-8. This utility method opens the file for writing, creating the file if |
| * it does not exist, or overwrites an existing file. |
| * |
| * @param fileContext the files system with which to create the file |
| * @param path the path to the file |
| * @param charseq the char sequence to write to the file |
| * |
| * @return the file context |
| * |
| * @throws NullPointerException if any of the arguments are {@code null} |
| * @throws IOException if an I/O error occurs creating or writing to the file |
| */ |
| public static FileContext write(final FileContext fileContext, |
| final Path path, final CharSequence charseq) throws IOException { |
| return write(fileContext, path, charseq, StandardCharsets.UTF_8); |
| } |
| } |
