hadoop-mapreduce/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.raid;

import java.io.IOException;
import java.io.FileNotFoundException;
import java.util.Collection;
import java.util.Map;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.LinkedHashMap;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.net.InetSocketAddress;
import javax.security.auth.login.LoginException;

import org.apache.hadoop.ipc.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.HarFileSystem;

import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.DistributedRaidFileSystem;
import org.apache.hadoop.hdfs.RaidDFSUtil;

import org.apache.hadoop.raid.protocol.PolicyInfo;
import org.apache.hadoop.raid.protocol.PolicyList;
import org.apache.hadoop.raid.protocol.RaidProtocol;

/**
 * A {@link RaidShell} that allows browsing configured raid policies.
 */
public class RaidShell extends Configured implements Tool {
  static {
    Configuration.addDefaultResource("hdfs-default.xml");
    Configuration.addDefaultResource("hdfs-site.xml");
  }
  public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.RaidShell");
  public RaidProtocol raidnode;
  RaidProtocol rpcRaidnode;
  private UserGroupInformation ugi;
  volatile boolean clientRunning = true;
  private Configuration conf;

  /**
   * Start RaidShell.
   * <p>
   * The RaidShell connects to the specified RaidNode and performs basic
   * configuration options.
   * @throws IOException
   */
  public RaidShell(Configuration conf) throws IOException {
    super(conf);
    this.conf = conf;
  }

  void initializeRpc(Configuration conf, InetSocketAddress address) throws IOException {
    this.ugi = UserGroupInformation.getCurrentUser();
    this.rpcRaidnode = createRPCRaidnode(address, conf, ugi);
    this.raidnode = createRaidnode(rpcRaidnode);
  }

  void initializeLocal(Configuration conf) throws IOException {
    this.ugi = UserGroupInformation.getCurrentUser();
  }

  public static RaidProtocol createRaidnode(Configuration conf) throws IOException {
    return createRaidnode(RaidNode.getAddress(conf), conf);
  }

  public static RaidProtocol createRaidnode(InetSocketAddress raidNodeAddr,
      Configuration conf) throws IOException {
    return createRaidnode(createRPCRaidnode(raidNodeAddr, conf,
      UserGroupInformation.getCurrentUser()));
  }

  private static RaidProtocol createRPCRaidnode(InetSocketAddress raidNodeAddr,
      Configuration conf, UserGroupInformation ugi)
    throws IOException {
    LOG.debug("RaidShell connecting to " + raidNodeAddr);
    return (RaidProtocol)RPC.getProxy(RaidProtocol.class,
        RaidProtocol.versionID, raidNodeAddr, ugi, conf,
        NetUtils.getSocketFactory(conf, RaidProtocol.class));
  }

  private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode)
    throws IOException {
    RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
        5, 5000, TimeUnit.MILLISECONDS);

    Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(RemoteException.class,
        RetryPolicies.retryByRemoteException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

    methodNameToPolicyMap.put("create", methodPolicy);

    return (RaidProtocol) RetryProxy.create(RaidProtocol.class,
        rpcRaidnode, methodNameToPolicyMap);
  }

  private void checkOpen() throws IOException {
    if (!clientRunning) {
      IOException result = new IOException("RaidNode closed");
      throw result;
    }
  }

  /**
   * Close the connection to the raidNode.
   */
  public synchronized void close() throws IOException {
    if(clientRunning) {
      clientRunning = false;
      RPC.stopProxy(rpcRaidnode);
    }
  }

  /**
   * Displays format of commands.
   */
  private static void printUsage(String cmd) {
    String prefix = "Usage: java " + RaidShell.class.getSimpleName();
    if ("-showConfig".equals(cmd)) {
      System.err.println("Usage: java RaidShell" + 
                         " [-showConfig]"); 
    } else if ("-recover".equals(cmd)) {
      System.err.println("Usage: java RaidShell" +
                         " [-recover srcPath1 corruptOffset]");
    } else if ("-recoverBlocks".equals(cmd)) {
      System.err.println("Usage: java RaidShell" +
                         " [-recoverBlocks path1 path2...]");
    } else {
      System.err.println("Usage: java RaidShell");
      System.err.println("           [-showConfig ]");
      System.err.println("           [-help [cmd]]");
      System.err.println("           [-recover srcPath1 corruptOffset]");
      System.err.println("           [-recoverBlocks path1 path2...]");
      System.err.println("           [-fsck [path]]");
      System.err.println();
      ToolRunner.printGenericCommandUsage(System.err);
    }
  }

  /**
   * run
   */
  public int run(String argv[]) throws Exception {

    if (argv.length < 1) {
      printUsage("");
      return -1;
    }

    int exitCode = -1;
    int i = 0;
    String cmd = argv[i++];
    //
    // verify that we have enough command line parameters
    //
    if ("-showConfig".equals(cmd)) {
      if (argv.length < 1) {
        printUsage(cmd);
        return exitCode;
      }
    } else if ("-recover".equals(cmd)) {
      if (argv.length < 3) {
        printUsage(cmd);
        return exitCode;
      }
    } else if ("-fsck".equals(cmd)) {
      if ((argv.length < 1) || (argv.length > 2)) {
        printUsage(cmd);
        return exitCode;
      }
    }

    try {
      if ("-showConfig".equals(cmd)) {
        initializeRpc(conf, RaidNode.getAddress(conf));
        exitCode = showConfig(cmd, argv, i);
      } else if ("-recover".equals(cmd)) {
        initializeRpc(conf, RaidNode.getAddress(conf));
        exitCode = recoverAndPrint(cmd, argv, i);
      } else if ("-recoverBlocks".equals(cmd)) {
        initializeLocal(conf);
        recoverBlocks(argv, i);
        exitCode = 0;
      } else if ("-fsck".equals(cmd)) {
        if (argv.length == 1) {
          // if there are no args, check the whole file system
          exitCode = fsck("/");
        } else {
          // argv.length == 2
          // otherwise, check the path passed
          exitCode = fsck(argv[1]);
        }
      } else {
        exitCode = -1;
        System.err.println(cmd.substring(1) + ": Unknown command");
        printUsage("");
      }
    } catch (IllegalArgumentException arge) {
      exitCode = -1;
      System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
      printUsage(cmd);
    } catch (RemoteException e) {
      //
      // This is a error returned by raidnode server. Print
      // out the first line of the error mesage, ignore the stack trace.
      exitCode = -1;
      try {
        String[] content;
        content = e.getLocalizedMessage().split("\n");
        System.err.println(cmd.substring(1) + ": " +
                           content[0]);
      } catch (Exception ex) {
        System.err.println(cmd.substring(1) + ": " +
                           ex.getLocalizedMessage());
      }
    } catch (IOException e) {
      //
      // IO exception encountered locally.
      // 
      exitCode = -1;
      System.err.println(cmd.substring(1) + ": " +
                         e.getLocalizedMessage());
    } catch (Exception re) {
      exitCode = -1;
      System.err.println(cmd.substring(1) + ": " + re.getLocalizedMessage());
    } finally {
    }
    return exitCode;
  }

  /**
   * Apply operation specified by 'cmd' on all parameters
   * starting from argv[startindex].
   */
  private int showConfig(String cmd, String argv[], int startindex) throws IOException {
    int exitCode = 0;
    int i = startindex;
    PolicyList[] all = raidnode.getAllPolicies();
    for (PolicyList list: all) {
      for (PolicyInfo p : list.getAll()) {
        System.out.println(p);
      }
    }
    return exitCode;
  }

  /**
   * Recovers the specified path from the parity file
   */
  public Path[] recover(String cmd, String argv[], int startindex)
    throws IOException {
    Path[] paths = new Path[(argv.length - startindex) / 2];
    int j = 0;
    for (int i = startindex; i < argv.length; i = i + 2) {
      String path = argv[i];
      long corruptOffset = Long.parseLong(argv[i+1]);
      LOG.debug("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset);
      paths[j] = new Path(raidnode.recoverFile(path, corruptOffset));
      LOG.debug("Raidshell created recovery file " + paths[j]);
      j++;
    }
    return paths;
  }

  public int recoverAndPrint(String cmd, String argv[], int startindex)
    throws IOException {
    int exitCode = 0;
    for (Path p : recover(cmd,argv,startindex)) {
      System.out.println(p);
    }
    return exitCode;
  }

  public void recoverBlocks(String[] args, int startIndex)
    throws IOException {
    LOG.debug("Recovering blocks for " + (args.length - startIndex) + " files");
    BlockFixer.BlockFixerHelper fixer = new BlockFixer.BlockFixerHelper(conf);
    for (int i = startIndex; i < args.length; i++) {
      String path = args[i];
      fixer.fixFile(new Path(path));
    }
  }

  /**
   * checks whether a file has more than the allowable number of
   * corrupt blocks and must therefore be considered corrupt
   */
  private boolean isFileCorrupt(final DistributedFileSystem dfs,
                                final Path filePath)
    throws IOException {
    // corruptBlocksPerStripe:
    // map stripe # -> # of corrupt blocks in that stripe (data + parity)
    HashMap<Integer, Integer> corruptBlocksPerStripe =
      new LinkedHashMap<Integer, Integer>();

    // read conf
    final int stripeBlocks = RaidNode.getStripeLength(conf);

    // figure out which blocks are missing/corrupted
    final FileStatus fileStatus = dfs.getFileStatus(filePath);
    final long blockSize = fileStatus.getBlockSize();
    final long fileLength = fileStatus.getLen();
    final long fileLengthInBlocks = (fileLength / blockSize) +
      (((fileLength % blockSize) == 0) ? 0L : 1L);
    final long fileStripes = (fileLengthInBlocks / stripeBlocks) +
      (((fileLengthInBlocks % stripeBlocks) == 0) ? 0L : 1L);
    final BlockLocation[] fileBlocks =
      dfs.getFileBlockLocations(fileStatus, 0, fileLength);

    // figure out which stripes these corrupted blocks belong to
    for (BlockLocation fileBlock: fileBlocks) {
      int blockNo = (int) (fileBlock.getOffset() / blockSize);
      final int stripe = (int) (blockNo / stripeBlocks);
      if (fileBlock.isCorrupt() ||
          (fileBlock.getNames().length == 0 && fileBlock.getLength() > 0)) {
        if (corruptBlocksPerStripe.get(stripe) == null) {
          corruptBlocksPerStripe.put(stripe, 1);
        } else {
          corruptBlocksPerStripe.put(stripe, corruptBlocksPerStripe.
                                     get(stripe) + 1);
        }
        LOG.debug("file " + filePath.toString() + " corrupt in block " +
                 blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe +
                 "/" + fileStripes);
      } else {
        LOG.debug("file " + filePath.toString() + " OK in block " + blockNo +
                 "/" + fileLengthInBlocks + ", stripe " + stripe + "/" +
                 fileStripes);
      }
    }

    RaidInfo raidInfo = getFileRaidInfo(dfs, filePath);

    // now check parity blocks
    if (raidInfo.raidType != RaidType.NONE) {
      checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize,
                        fileStripes, raidInfo);
    }

    final int maxCorruptBlocksPerStripe = raidInfo.parityBlocksPerStripe;

    for (int corruptBlocksInStripe: corruptBlocksPerStripe.values()) {
      if (corruptBlocksInStripe > maxCorruptBlocksPerStripe) {
        return true;
      }
    }
    return false;
  }

  /**
   * holds the type of raid used for a particular file
   */
  private enum RaidType {
    XOR,
    RS,
    NONE
  }

  /**
   * holds raid type and parity file pair
   */
  private class RaidInfo {
    public RaidInfo(final RaidType raidType,
                    final RaidNode.ParityFilePair parityPair,
                    final int parityBlocksPerStripe) {
      this.raidType = raidType;
      this.parityPair = parityPair;
      this.parityBlocksPerStripe = parityBlocksPerStripe;
    }
    public final RaidType raidType;
    public final RaidNode.ParityFilePair parityPair;
    public final int parityBlocksPerStripe;
  }

  /**
   * returns the raid for a given file
   */
  private RaidInfo getFileRaidInfo(final DistributedFileSystem dfs,
                                   final Path filePath)
    throws IOException {
    // now look for the parity file
    Path destPath = null;
    RaidNode.ParityFilePair ppair = null;
    try {
      // look for xor parity file first
      destPath = RaidNode.xorDestinationPath(conf);
      ppair = RaidNode.getParityFile(destPath, filePath, conf);
    } catch (FileNotFoundException ignore) {
    }
    if (ppair != null) {
      return new RaidInfo(RaidType.XOR, ppair, 1);
    } else {
      // failing that, look for rs parity file
      try {
        destPath = RaidNode.rsDestinationPath(conf);
        ppair = RaidNode.getParityFile(destPath, filePath, conf);
      } catch (FileNotFoundException ignore) {
      }
      if (ppair != null) {
        return new RaidInfo(RaidType.RS, ppair, RaidNode.rsParityLength(conf));
      } else {
        return new RaidInfo(RaidType.NONE, null, 0);
      }
    }
  }

  /**
   * gets the parity blocks corresponding to file
   * returns the parity blocks in case of DFS
   * and the part blocks containing parity blocks
   * in case of HAR FS
   */
  private BlockLocation[] getParityBlocks(final Path filePath,
                                          final long blockSize,
                                          final long fileStripes,
                                          final RaidInfo raidInfo)
    throws IOException {


    final String parityPathStr = raidInfo.parityPair.getPath().toUri().
      getPath();
    FileSystem parityFS = raidInfo.parityPair.getFileSystem();

    // get parity file metadata
    FileStatus parityFileStatus = parityFS.
      getFileStatus(new Path(parityPathStr));
    long parityFileLength = parityFileStatus.getLen();

    if (parityFileLength != fileStripes * raidInfo.parityBlocksPerStripe *
        blockSize) {
      throw new IOException("expected parity file of length" +
                            (fileStripes * raidInfo.parityBlocksPerStripe *
                             blockSize) +
                            " but got parity file of length " +
                            parityFileLength);
    }

    BlockLocation[] parityBlocks =
      parityFS.getFileBlockLocations(parityFileStatus, 0L, parityFileLength);

    if (parityFS instanceof DistributedFileSystem ||
        parityFS instanceof DistributedRaidFileSystem) {
      long parityBlockSize = parityFileStatus.getBlockSize();
      if (parityBlockSize != blockSize) {
        throw new IOException("file block size is " + blockSize +
                              " but parity file block size is " +
                              parityBlockSize);
      }
    } else if (parityFS instanceof HarFileSystem) {
      LOG.debug("HAR FS found");
    } else {
      LOG.warn("parity file system is not of a supported type");
    }

    return parityBlocks;
  }

  /**
   * checks the parity blocks for a given file and modifies
   * corruptBlocksPerStripe accordingly
   */
  private void checkParityBlocks(final Path filePath,
                                 final HashMap<Integer, Integer>
                                 corruptBlocksPerStripe,
                                 final long blockSize,
                                 final long fileStripes,
                                 final RaidInfo raidInfo)
    throws IOException {

    // get the blocks of the parity file
    // because of har, multiple blocks may be returned as one container block
    BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize,
                                                      fileStripes, raidInfo);

    long parityStripeLength = blockSize *
      ((long) raidInfo.parityBlocksPerStripe);

    long parityFileLength = parityStripeLength * fileStripes;

    long parityBlocksFound = 0L;

    for (BlockLocation cb: containerBlocks) {
      if (cb.getLength() % blockSize != 0) {
        throw new IOException("container block size is not " +
                              "multiple of parity block size");
      }
      int blocksInContainer = (int) (cb.getLength() / blockSize);
      LOG.debug("found container with offset " + cb.getOffset() +
               ", length " + cb.getLength());

      for (long offset = cb.getOffset();
           offset < cb.getOffset() + cb.getLength();
           offset += blockSize) {
        long block = offset / blockSize;

        int stripe = (int) (offset / parityStripeLength);

        if (stripe < 0) {
          // before the beginning of the parity file
          continue;
        }
        if (stripe >= fileStripes) {
          // past the end of the parity file
          break;
        }

        parityBlocksFound++;

        if (cb.isCorrupt() ||
            (cb.getNames().length == 0 && cb.getLength() > 0)) {
          LOG.debug("parity file for " + filePath.toString() +
                   " corrupt in block " + block +
                   ", stripe " + stripe + "/" + fileStripes);

          if (corruptBlocksPerStripe.get(stripe) == null) {
            corruptBlocksPerStripe.put(stripe, 1);
          } else {
            corruptBlocksPerStripe.put(stripe,
                                       corruptBlocksPerStripe.get(stripe) +
                                       1);
          }
        } else {
          LOG.debug("parity file for " + filePath.toString() +
                   " OK in block " + block +
                   ", stripe " + stripe + "/" + fileStripes);
        }
      }
    }

    long parityBlocksExpected = raidInfo.parityBlocksPerStripe * fileStripes;
    if (parityBlocksFound != parityBlocksExpected ) {
      throw new IOException("expected " + parityBlocksExpected +
                            " parity blocks but got " + parityBlocksFound);
    }
  }


  /**
   * checks the raided file system, prints a list of corrupt files to
   * System.out and returns the number of corrupt files
   */
  public int fsck(final String path) throws IOException {

    FileSystem fs = (new Path(path)).getFileSystem(conf);

    // if we got a raid fs, get the underlying fs
    if (fs instanceof DistributedRaidFileSystem) {
      fs = ((DistributedRaidFileSystem) fs).getFileSystem();
    }

    // check that we have a distributed fs
    if (!(fs instanceof DistributedFileSystem)) {
      throw new IOException("expected DistributedFileSystem but got " +
                fs.getClass().getName());
    }
    final DistributedFileSystem dfs = (DistributedFileSystem) fs;

    // get conf settings
    String xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath();
    String rsPrefix = RaidNode.rsDestinationPath(conf).toUri().getPath();
    if (!xorPrefix.endsWith("/")) {
      xorPrefix = xorPrefix + "/";
    }
    if (!rsPrefix.endsWith("/")) {
      rsPrefix = rsPrefix + "/";
    }
    LOG.debug("prefixes: " + xorPrefix + ", " + rsPrefix);

    // get a list of corrupted files (not considering parity blocks just yet)
    // from the name node
    // these are the only files we need to consider:
    // if a file has no corrupted data blocks, it is OK even if some
    // of its parity blocks are corrupted, so no further checking is
    // necessary
    final String[] files = RaidDFSUtil.getCorruptFiles(dfs);
    final List<Path> corruptFileCandidates = new LinkedList<Path>();
    for (final String f: files) {
      final Path p = new Path(f);
      // if this file is a parity file
      // or if it does not start with the specified path,
      // ignore it
      if (!p.toString().startsWith(xorPrefix) &&
          !p.toString().startsWith(rsPrefix) &&
          p.toString().startsWith(path)) {
        corruptFileCandidates.add(p);
      }
    }
    // filter files marked for deletion
    RaidUtils.filterTrash(conf, corruptFileCandidates);

    int numberOfCorruptFiles = 0;

    for (final Path corruptFileCandidate: corruptFileCandidates) {
      if (isFileCorrupt(dfs, corruptFileCandidate)) {
        System.out.println(corruptFileCandidate.toString());
        numberOfCorruptFiles++;
      }
    }

    return numberOfCorruptFiles;
  }

  /**
   * main() has some simple utility methods
   */
  public static void main(String argv[]) throws Exception {
    RaidShell shell = null;
    try {
      shell = new RaidShell(new Configuration());
      int res = ToolRunner.run(shell, argv);
      System.exit(res);
    } catch (RPC.VersionMismatch v) {
      System.err.println("Version Mismatch between client and server" +
                         "... command aborted.");
      System.exit(-1);
    } catch (IOException e) {
      System.err.
        println("Bad connection to RaidNode or NameNode. command aborted.");
      System.err.println(e.getMessage());
      System.exit(-1);
    } finally {
      shell.close();
    }
  }
}