| /** |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.ambari.fast_hdfs_resource; |
| |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.io.File; |
| import java.lang.System; |
| import java.util.ArrayList; |
| import java.util.HashSet; |
| |
| import org.apache.hadoop.fs.FileStatus; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.fs.permission.FsPermission; |
| import org.apache.hadoop.fs.FileSystem; |
| |
| /** |
| * Used to: |
| * 1) copy files/directories from localFS to hadoopFs |
| * 2) create empty |
| * 3) download files/directories from hadoopFs to localFS |
| * files/directories in hadoopFs |
| */ |
| public class Resource { |
| private String source; |
| private String target; |
| private String type; |
| private String action; |
| private String owner; |
| private String group; |
| private String mode; |
| private String nameservice; |
| private boolean recursiveChown; |
| private boolean recursiveChmod; |
| private boolean changePermissionforParents; |
| private boolean manageIfExists; |
| |
| public String getSource() { |
| return source; |
| } |
| |
| public void setSource(String source) { |
| this.source = source; |
| } |
| |
| public String getTarget() { |
| return target; |
| } |
| |
| public void setTarget(String target) { |
| this.target = target; |
| } |
| |
| public String getType() { |
| return type; |
| } |
| |
| public void setType(String type) { |
| this.type = type; |
| } |
| |
| public String getAction() { |
| return action; |
| } |
| |
| public void setAction(String action) { |
| this.action = action; |
| } |
| |
| public String getOwner() { |
| return owner; |
| } |
| |
| public void setOwner(String owner) { |
| this.owner = owner; |
| } |
| |
| public String getGroup() { |
| return group; |
| } |
| |
| public void setGroup(String group) { |
| this.group = group; |
| } |
| |
| public String getMode() { |
| return mode; |
| } |
| |
| public void setMode(String mode) { |
| this.mode = mode; |
| } |
| |
| public String getNameservice() { |
| return nameservice; |
| } |
| |
| public void setNameservice(String nameservice) { |
| this.nameservice = nameservice; |
| } |
| |
| public boolean isRecursiveChown() { |
| return recursiveChown; |
| } |
| |
| public void setRecursiveChown(boolean recursiveChown) { |
| this.recursiveChown = recursiveChown; |
| } |
| |
| public boolean isRecursiveChmod() { |
| return recursiveChmod; |
| } |
| |
| public void setRecursiveChmod(boolean recursiveChmod) { |
| this.recursiveChmod = recursiveChmod; |
| } |
| |
| public boolean isChangePermissionOnParents() { |
| return changePermissionforParents; |
| } |
| |
| public void setChangePermissionOnParents(boolean changePermissionforParents) { |
| this.changePermissionforParents = changePermissionforParents; |
| } |
| |
| public boolean isManageIfExists() { |
| return manageIfExists; |
| } |
| |
| public void setManageIfExists(boolean manageIfExists) { |
| this.manageIfExists = manageIfExists; |
| } |
| |
| @Override |
| public String toString() { |
| return "Resource [source=" + source + ", target=" + target + ", type=" |
| + type + ", action=" + action + ", owner=" + owner + ", group=" + group |
| + ", mode=" + mode + ", recursiveChown=" + recursiveChown |
| + ", recursiveChmod=" + recursiveChmod |
| + ", changePermissionforParents=" + changePermissionforParents |
| + ", manageIfExists=" + manageIfExists + "]"; |
| } |
| |
| /* |
| * Check if parameters are correctly set |
| */ |
| public static void checkResourceParameters(Resource resource, |
| FileSystem dfs) throws IllegalArgumentException, IOException { |
| |
| ArrayList<String> actionsAvailable = new ArrayList<String>(); |
| actionsAvailable.add("create"); |
| actionsAvailable.add("delete"); |
| actionsAvailable.add("download"); |
| ArrayList<String> typesAvailable = new ArrayList<String>(); |
| typesAvailable.add("file"); |
| typesAvailable.add("directory"); |
| |
| if (resource.getAction() == null || !actionsAvailable.contains(resource.getAction())) { |
| throw new IllegalArgumentException("Action is not supported."); |
| } |
| |
| String dfsPath = resource.getTarget(); |
| String localPath = resource.getSource(); |
| if (resource.getAction().equals("download")) { |
| dfsPath = resource.getSource(); |
| localPath = resource.getTarget(); |
| } |
| |
| if (dfsPath == null) { |
| throw new IllegalArgumentException("Path to resource in HadoopFs must be filled."); |
| } |
| |
| if (resource.getType() == null || !typesAvailable.contains(resource.getType())) { |
| throw new IllegalArgumentException("Type is not supported."); |
| } |
| |
| // Check consistency for ("type":"file" == file in hadoop) |
| if (dfs.isFile(new Path(dfsPath)) && !"file".equals(resource.getType())) { |
| throw new IllegalArgumentException( |
| "Cannot create a directory " + dfsPath + |
| " because file is present on the given path."); |
| } |
| // Check consistency for ("type":"directory" == directory in hadoop) |
| else if (dfs.isDirectory(new Path(dfsPath)) && !"directory".equals(resource.getType())) { |
| throw new IllegalArgumentException( |
| "Cannot create a file " + dfsPath + |
| " because directory is present on the given path."); |
| } |
| |
| if(localPath != null) { |
| File local = new File(localPath); |
| if(local.isFile() && !"file".equals(resource.getType())) { |
| throw new IllegalArgumentException( |
| "Cannot create a directory " + dfsPath + |
| " because source " + localPath + "is a file"); |
| } |
| else if(local.isDirectory() && !"directory".equals(resource.getType())) { |
| throw new IllegalArgumentException( |
| "Cannot create a file " + dfsPath + |
| " because source " + localPath + "is a directory"); |
| } |
| } |
| } |
| |
| /* |
| * Create/copy resource - {type} |
| */ |
| public static void createResource(Resource resource, |
| FileSystem dfs, Path pathHadoop) throws IOException { |
| |
| boolean isCreate = (resource.getSource() == null) ? true : false; |
| |
| if (isCreate && resource.getType().equals("directory")) { |
| dfs.mkdirs(pathHadoop); // empty dir(s) |
| } else if (isCreate && resource.getType().equals("file")) { |
| dfs.createNewFile(pathHadoop); // empty file |
| } else { |
| if(dfs.exists(pathHadoop) && dfs.getFileStatus(pathHadoop).isDir()) { |
| System.out.println("Skipping copy from local, as target " + pathHadoop + " is an existing directory."); // Copy from local to existing directory is not supported by dfs. |
| } else { |
| dfs.copyFromLocalFile(new Path(resource.getSource()), pathHadoop); |
| } |
| } |
| } |
| |
| /* |
| * Set permissions on resource - {mode} |
| */ |
| public static void setMode(Resource resource, |
| FileSystem dfs, Path pathHadoop) throws IOException { |
| |
| if (resource.getMode() != null) { |
| FsPermission permission = new FsPermission((short)Integer.parseInt(resource.getMode(), 8)); |
| dfs.setPermission(pathHadoop, permission); |
| |
| // Recursive |
| |
| // Get the list of sub-directories and files |
| HashSet<String> resultSet = new HashSet<String>(); |
| |
| if (resource.isRecursiveChmod()) |
| resource.fillDirectoryList(dfs, resource.getTarget(), resultSet); |
| |
| if(resource.isChangePermissionOnParents()) |
| resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet); |
| |
| for (String path : resultSet) { |
| dfs.setPermission(new Path(path), permission); |
| } |
| |
| } |
| } |
| |
| /* |
| * Set owner on resource - {owner} |
| */ |
| public static void setOwner(Resource resource, FileSystem dfs, |
| Path pathHadoop) throws IOException { |
| |
| if (!(resource.getOwner() == null && resource.getGroup() == null)) { |
| dfs.setOwner(pathHadoop, resource.getOwner(), resource.getGroup()); |
| |
| // Get the list of sub-directories and files |
| HashSet<String> resultSet = new HashSet<String>(); |
| if (resource.isRecursiveChown()) |
| resource.fillDirectoryList(dfs, resource.getTarget(), resultSet); |
| if(resource.isChangePermissionOnParents()) |
| resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet); |
| |
| for (String path : resultSet) { |
| dfs.setOwner(new Path(path), resource.getOwner(), resource.getGroup()); |
| } |
| } |
| } |
| |
| public void fillInParentDirectories(FileSystem dfs, String path, HashSet<String> resultSet) throws IOException { |
| Path filePath = new Path(path); |
| |
| while(true) { |
| filePath = filePath.getParent(); |
| |
| // if(filePath.isRoot()) { |
| if(filePath.getParent() == null) { |
| break; |
| } |
| resultSet.add(filePath.toString()); |
| } |
| } |
| |
| /* |
| * List all files and sub-directories recursively |
| */ |
| public void fillDirectoryList(FileSystem dfs, String path, |
| HashSet<String> resultSet) throws IOException { |
| |
| FileStatus[] fileStatus = dfs.listStatus(new Path(path)); |
| if (fileStatus != null) { |
| // Go through all resources in directory |
| for (FileStatus fs : fileStatus) { |
| String pathToResource = path + "/" + fs.getPath().getName(); |
| |
| resultSet.add(pathToResource); |
| |
| if (fs.isDir()) { |
| // recursive |
| fillDirectoryList(dfs, pathToResource, resultSet); |
| } |
| } |
| } |
| } |
| } |