contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ambari.view.utils.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.json.simple.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * Hdfs Business Delegate
 */
public class HdfsApi {
  private final static Logger LOG =
      LoggerFactory.getLogger(HdfsApi.class);

  private final Configuration conf;
  private final Map<String, String> authParams;

  private FileSystem fs;
  private UserGroupInformation ugi;

  /**
   * Constructor
   * @param configurationBuilder hdfs configuration builder
   * @throws IOException
   * @throws InterruptedException
   */
  public HdfsApi(ConfigurationBuilder configurationBuilder, String username) throws IOException,
      InterruptedException, HdfsApiException {
    this.authParams = configurationBuilder.buildAuthenticationConfig();
    conf = configurationBuilder.buildConfig();
    UserGroupInformation.setConfiguration(conf);
    ugi = UserGroupInformation.createProxyUser(username, getProxyUser());

    fs = execute(new PrivilegedExceptionAction<FileSystem>() {
      public FileSystem run() throws IOException {
        return FileSystem.get(conf);
      }
    });
  }

  private UserGroupInformation getProxyUser() throws IOException {
    UserGroupInformation proxyuser;
    if (authParams.containsKey("proxyuser")) {
      proxyuser = UserGroupInformation.createRemoteUser(authParams.get("proxyuser"));
    } else {
      proxyuser = UserGroupInformation.getCurrentUser();
    }

    proxyuser.setAuthenticationMethod(getAuthenticationMethod());
    return proxyuser;
  }

  private UserGroupInformation.AuthenticationMethod getAuthenticationMethod() {
    UserGroupInformation.AuthenticationMethod authMethod;
    if (authParams.containsKey("auth")) {
      String authName = authParams.get("auth");
      authMethod = UserGroupInformation.AuthenticationMethod.valueOf(authName.toUpperCase());
    } else {
      authMethod = UserGroupInformation.AuthenticationMethod.SIMPLE;
    }
    return authMethod;
  }

  /**
   * List dir operation
   * @param path path
   * @return array of FileStatus objects
   * @throws FileNotFoundException
   * @throws IOException
   * @throws InterruptedException
   */
  public FileStatus[] listdir(final String path) throws FileNotFoundException,
      IOException, InterruptedException {
    return execute(new PrivilegedExceptionAction<FileStatus[]>() {
      public FileStatus[] run() throws FileNotFoundException, Exception {
        return fs.listStatus(new Path(path));
      }
    });
  }

  /**
   * Get file status
   * @param path path
   * @return file status
   * @throws IOException
   * @throws FileNotFoundException
   * @throws InterruptedException
   */
  public FileStatus getFileStatus(final String path) throws IOException,
      FileNotFoundException, InterruptedException {
    return execute(new PrivilegedExceptionAction<FileStatus>() {
      public FileStatus run() throws FileNotFoundException, IOException {
        return fs.getFileStatus(new Path(path));
      }
    });
  }

  /**
   * Make directory
   * @param path path
   * @return success
   * @throws IOException
   * @throws InterruptedException
   */
  public boolean mkdir(final String path) throws IOException,
      InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return fs.mkdirs(new Path(path));
      }
    });
  }

  /**
   * Rename
   * @param src source path
   * @param dst destination path
   * @return success
   * @throws IOException
   * @throws InterruptedException
   */
  public boolean rename(final String src, final String dst) throws IOException,
      InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return fs.rename(new Path(src), new Path(dst));
      }
    });
  }

  /**
   * Check is trash enabled
   * @return true if trash is enabled
   * @throws Exception
   */
  public boolean trashEnabled() throws Exception {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws IOException {
        Trash tr = new Trash(fs, conf);
        return tr.isEnabled();
      }
    });
  }

  /**
   * Home directory
   * @return home directory
   * @throws Exception
   */
  public Path getHomeDir() throws Exception {
    return execute(new PrivilegedExceptionAction<Path>() {
      public Path run() throws IOException {
        return fs.getHomeDirectory();
      }
    });
  }

  /**
   * Hdfs Status
   * @return home directory
   * @throws Exception
   */
  public synchronized FsStatus getStatus() throws Exception {
    return execute(new PrivilegedExceptionAction<FsStatus>() {
      public FsStatus run() throws IOException {
        return fs.getStatus();
      }
    });
  }

  /**
   * Trash directory
   * @return trash directory
   * @throws Exception
   */
  public Path getTrashDir() throws Exception {
    return execute(new PrivilegedExceptionAction<Path>() {
      public Path run() throws IOException {
        TrashPolicy trashPolicy = TrashPolicy.getInstance(conf, fs,
            fs.getHomeDirectory());
        return trashPolicy.getCurrentTrashDir().getParent();
      }
    });
  }

  /**
   * Trash directory path.
   *
   * @return trash directory path
   * @throws Exception
   */
  public String getTrashDirPath() throws Exception {
    Path trashDir = getTrashDir();

    return trashDir.toUri().getRawPath();
  }

  /**
   * Trash directory path.
   *
   * @param    filePath        the path to the file
   * @return trash directory path for the file
   * @throws Exception
   */
  public String getTrashDirPath(String filePath) throws Exception {
    String trashDirPath = getTrashDirPath();

    Path path = new Path(filePath);
    trashDirPath = trashDirPath + "/" + path.getName();

    return trashDirPath;
  }

  /**
   * Empty trash
   * @return
   * @throws Exception
   */
  public Void emptyTrash() throws Exception {
    return execute(new PrivilegedExceptionAction<Void>() {
      public Void run() throws IOException {
        Trash tr = new Trash(fs, conf);
        tr.expunge();
        return null;
      }
    });
  }

  /**
   * Move to trash
   * @param path path
   * @return success
   * @throws IOException
   * @throws InterruptedException
   */
  public boolean moveToTrash(final String path) throws IOException,
      InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return Trash.moveToAppropriateTrash(fs, new Path(path), conf);
      }
    });
  }

  /**
   * Delete
   * @param path path
   * @param recursive delete recursive
   * @return success
   * @throws IOException
   * @throws InterruptedException
   */
  public boolean delete(final String path, final boolean recursive)
      throws IOException, InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return fs.delete(new Path(path), recursive);
      }
    });
  }

  /**
   * Create file
   * @param path path
   * @param overwrite overwrite existent file
   * @return output stream
   * @throws IOException
   * @throws InterruptedException
   */
  public FSDataOutputStream create(final String path, final boolean overwrite)
      throws IOException, InterruptedException {
    return execute(new PrivilegedExceptionAction<FSDataOutputStream>() {
      public FSDataOutputStream run() throws Exception {
        return fs.create(new Path(path), overwrite);
      }
    });
  }

  /**
   * Open file
   * @param path path
   * @return input stream
   * @throws IOException
   * @throws InterruptedException
   */
  public FSDataInputStream open(final String path) throws IOException,
      InterruptedException {
    return execute(new PrivilegedExceptionAction<FSDataInputStream>() {
      public FSDataInputStream run() throws Exception {
        return fs.open(new Path(path));
      }
    });
  }

  /**
   * Change permissions
   * @param path path
   * @param permissions permissions in format rwxrwxrwx
   * @throws IOException
   * @throws InterruptedException
   */
  public boolean chmod(final String path, final String permissions) throws IOException,
      InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        try {
          fs.setPermission(new Path(path), FsPermission.valueOf(permissions));
        } catch (Exception ex) {
          return false;
        }
        return true;
      }
    });
  }

  /**
   * Copy file
   * @param src source path
   * @param dest destination path
   * @throws java.io.IOException
   * @throws InterruptedException
   */
  public void copy(final String src, final String dest) throws IOException, InterruptedException, HdfsApiException {
    boolean result = execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return FileUtil.copy(fs, new Path(src), fs, new Path(dest), false, conf);
      }
    });

    if (!result) {
      throw new HdfsApiException("HDFS010 Can't copy source file from \" + src + \" to \" + dest");
    }
  }

  public boolean exists(final String newFilePath) throws IOException, InterruptedException {
    return execute(new PrivilegedExceptionAction<Boolean>() {
      public Boolean run() throws Exception {
        return fs.exists(new Path(newFilePath));
      }
    });
  }

  /**
   * Executes action on HDFS using doAs
   * @param action strategy object
   * @param <T> result type
   * @return result of operation
   * @throws IOException
   * @throws InterruptedException
   */
  public <T> T execute(PrivilegedExceptionAction<T> action)
      throws IOException, InterruptedException {
    T result = null;

    // Retry strategy applied here due to HDFS-1058. HDFS can throw random
    // IOException about retrieving block from DN if concurrent read/write
    // on specific file is performed (see details on HDFS-1058).
    int tryNumber = 0;
    boolean succeeded = false;
    do {
      tryNumber += 1;
      try {
        result = ugi.doAs(action);
        succeeded = true;
      } catch (IOException ex) {
        if (!ex.getMessage().contains("Cannot obtain block length for")) {
          throw ex;
        }
        if (tryNumber >= 3) {
          throw ex;
        }
        LOG.info("HDFS threw 'IOException: Cannot obtain block length' exception. " +
            "Retrying... Try #" + (tryNumber + 1));
        Thread.sleep(1000);  //retry after 1 second
      }
    } while (!succeeded);
    return result;
  }

  /**
   * Converts a Hadoop permission into a Unix permission symbolic representation
   * (i.e. -rwxr--r--) or default if the permission is NULL.
   *
   * @param p
   *          Hadoop permission.
   * @return the Unix permission symbolic representation or default if the
   *         permission is NULL.
   */
  private static String permissionToString(FsPermission p) {
    return (p == null) ? "default" : "-" + p.getUserAction().SYMBOL
        + p.getGroupAction().SYMBOL + p.getOtherAction().SYMBOL;
  }

  /**
   * Converts a Hadoop <code>FileStatus</code> object into a JSON array object.
   * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the
   * specified URL.
   * <p/>
   *
   * @param status
   *          Hadoop file status.
   * @return The JSON representation of the file status.
   */
  public Map<String, Object> fileStatusToJSON(FileStatus status) {
    Map<String, Object> json = new LinkedHashMap<String, Object>();
    json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath())
        .toString());
    json.put("replication", status.getReplication());
    json.put("isDirectory", status.isDirectory());
    json.put("len", status.getLen());
    json.put("owner", status.getOwner());
    json.put("group", status.getGroup());
    json.put("permission", permissionToString(status.getPermission()));
    json.put("accessTime", status.getAccessTime());
    json.put("modificationTime", status.getModificationTime());
    json.put("blockSize", status.getBlockSize());
    json.put("replication", status.getReplication());
    json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi));
    json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi));
    json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi));
    return json;
  }

  /**
   * Converts a Hadoop <code>FileStatus</code> array into a JSON array object.
   * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the
   * specified URL.
   * <p/>
   *
   * @param status
   *          Hadoop file status array.
   * @return The JSON representation of the file status array.
   */
  @SuppressWarnings("unchecked")
  public JSONArray fileStatusToJSON(FileStatus[] status) {
    JSONArray json = new JSONArray();
    if (status != null) {
      for (FileStatus s : status) {
        json.add(fileStatusToJSON(s));
      }
    }
    return json;
  }

  public static boolean checkAccessPermissions(FileStatus stat, FsAction mode, UserGroupInformation ugi) {
    FsPermission perm = stat.getPermission();
    String user = ugi.getShortUserName();
    List<String> groups = Arrays.asList(ugi.getGroupNames());
    if (user.equals(stat.getOwner())) {
      if (perm.getUserAction().implies(mode)) {
        return true;
      }
    } else if (groups.contains(stat.getGroup())) {
      if (perm.getGroupAction().implies(mode)) {
        return true;
      }
    } else {
      if (perm.getOtherAction().implies(mode)) {
        return true;
      }
    }
    return false;
  }
}