mapreduce/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.DataInput;
import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.hdfs.BlockMissingException;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.raid.Decoder;
import org.apache.hadoop.raid.RaidNode;
import org.apache.hadoop.raid.ReedSolomonDecoder;
import org.apache.hadoop.raid.XORDecoder;
import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;

/**
 * This is an implementation of the Hadoop  RAID Filesystem. This FileSystem 
 * wraps an instance of the DistributedFileSystem.
 * If a file is corrupted, this FileSystem uses the parity blocks to 
 * regenerate the bad block.
 */

public class DistributedRaidFileSystem extends FilterFileSystem {

  // these are alternate locations that can be used for read-only access
  DecodeInfo[] alternates;
  Configuration conf;
  int stripeLength;

  DistributedRaidFileSystem() throws IOException {
  }

  DistributedRaidFileSystem(FileSystem fs) throws IOException {
    super(fs);
    alternates = null;
    stripeLength = 0;
  }

  // Information required for decoding a source file
  static private class DecodeInfo {
    final Path destPath;
    final ErasureCodeType type;
    final Configuration conf;
    final int stripeLength;
    private DecodeInfo(Configuration conf, ErasureCodeType type, Path destPath) {
      this.conf = conf;
      this.type = type;
      this.destPath = destPath;
      this.stripeLength = RaidNode.getStripeLength(conf);
    }

    Decoder createDecoder() {
      if (this.type == ErasureCodeType.XOR) {
        return new XORDecoder(conf, stripeLength);
      } else if (this.type == ErasureCodeType.RS) {
        return new ReedSolomonDecoder(conf, stripeLength,
                              RaidNode.rsParityLength(conf));
      }
      return null;
    }
  }

  /* Initialize a Raid FileSystem
   */
  public void initialize(URI name, Configuration conf) throws IOException {
    this.conf = conf;

    Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl",
        DistributedFileSystem.class);
    if (clazz == null) {
      throw new IOException("No FileSystem for fs.raid.underlyingfs.impl.");
    }
    
    this.fs = (FileSystem)ReflectionUtils.newInstance(clazz, null); 
    super.initialize(name, conf);
    
    // find stripe length configured
    stripeLength = RaidNode.getStripeLength(conf);
    if (stripeLength == 0) {
      LOG.info("dfs.raid.stripeLength is incorrectly defined to be " + 
               stripeLength + " Ignoring...");
      return;
    }

    // Put XOR and RS in alternates
    alternates= new DecodeInfo[2];
    Path xorPath = RaidNode.xorDestinationPath(conf, fs);
    alternates[0] = new DecodeInfo(conf, ErasureCodeType.XOR, xorPath);
    Path rsPath = RaidNode.rsDestinationPath(conf, fs);
    alternates[1] = new DecodeInfo(conf, ErasureCodeType.RS, rsPath);
  }

  /*
   * Returns the underlying filesystem
   */
  public FileSystem getFileSystem() throws IOException {
    return fs;
  }

  @Override
  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
    ExtFSDataInputStream fd = new ExtFSDataInputStream(conf, this, alternates, f,
                                                       stripeLength, bufferSize);
    return fd;
  }

  public void close() throws IOException {
    if (fs != null) {
      try {
        fs.close();
      } catch(IOException ie) {
        //this might already be closed, ignore
      }
    }
    super.close();
  }

  /**
   * Layered filesystem input stream. This input stream tries reading
   * from alternate locations if it encoumters read errors in the primary location.
   */
  private static class ExtFSDataInputStream extends FSDataInputStream {
    /**
     * Create an input stream that wraps all the reads/positions/seeking.
     */
    private static class ExtFsInputStream extends FSInputStream {

      //The underlying data input stream that the
      // underlying filesystem will return.
      private FSDataInputStream underLyingStream;
      private byte[] oneBytebuff = new byte[1];
      private int nextLocation;
      private DistributedRaidFileSystem lfs;
      private Path path;
      private final DecodeInfo[] alternates;
      private final int buffersize;
      private final Configuration conf;
      private final int stripeLength;

      ExtFsInputStream(Configuration conf, DistributedRaidFileSystem lfs,
          DecodeInfo[] alternates, Path path, int stripeLength, int buffersize)
          throws IOException {
        this.underLyingStream = lfs.fs.open(path, buffersize);
        this.path = path;
        this.nextLocation = 0;
        this.alternates = alternates;
        this.buffersize = buffersize;
        this.conf = conf;
        this.lfs = lfs;
        this.stripeLength = stripeLength;
      }
      
      @Override
      public synchronized int available() throws IOException {
        int value = underLyingStream.available();
        nextLocation = 0;
        return value;
      }
      
      @Override
      public synchronized  void close() throws IOException {
        underLyingStream.close();
        super.close();
      }
      
      @Override
      public void mark(int readLimit) {
        underLyingStream.mark(readLimit);
        nextLocation = 0;
      }
      
      @Override
      public void reset() throws IOException {
        underLyingStream.reset();
        nextLocation = 0;
      }
      
      @Override
      public synchronized int read() throws IOException {
        long pos = underLyingStream.getPos();
        while (true) {
          try {
            int value = underLyingStream.read();
            nextLocation = 0;
            return value;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, pos);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }
      
      @Override
      public synchronized int read(byte[] b) throws IOException {
        long pos = underLyingStream.getPos();
        while (true) {
          try{
            int value = underLyingStream.read(b);
            nextLocation = 0;
            return value;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, pos);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }

      @Override
      public synchronized int read(byte[] b, int offset, int len) 
        throws IOException {
        long pos = underLyingStream.getPos();
        while (true) {
          try{
            int value = underLyingStream.read(b, offset, len);
            nextLocation = 0;
            return value;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, pos);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }
      
      @Override
      public synchronized int read(long position, byte[] b, int offset, int len) 
        throws IOException {
        long pos = underLyingStream.getPos();
        while (true) {
          try {
            int value = underLyingStream.read(position, b, offset, len);
            nextLocation = 0;
            return value;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, pos);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }
      
      @Override
      public synchronized long skip(long n) throws IOException {
        long value = underLyingStream.skip(n);
        nextLocation = 0;
        return value;
      }
      
      @Override
      public synchronized long getPos() throws IOException {
        long value = underLyingStream.getPos();
        nextLocation = 0;
        return value;
      }
      
      @Override
      public synchronized void seek(long pos) throws IOException {
        underLyingStream.seek(pos);
        nextLocation = 0;
      }

      @Override
      public boolean seekToNewSource(long targetPos) throws IOException {
        boolean value = underLyingStream.seekToNewSource(targetPos);
        nextLocation = 0;
        return value;
      }
      
      /**
       * position readable again.
       */
      @Override
      public void readFully(long pos, byte[] b, int offset, int length) 
        throws IOException {
        long post = underLyingStream.getPos();
        while (true) {
          try {
            underLyingStream.readFully(pos, b, offset, length);
            nextLocation = 0;
            return;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, post);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }
      
      @Override
      public void readFully(long pos, byte[] b) throws IOException {
        long post = underLyingStream.getPos();
        while (true) {
          try {
            underLyingStream.readFully(pos, b);
            nextLocation = 0;
            return;
          } catch (BlockMissingException e) {
            setAlternateLocations(e, post);
          } catch (ChecksumException e) {
            setAlternateLocations(e, pos);
          }
        }
      }

      /**
       * Extract good file from RAID
       * @param curpos curexp the current exception
       * @param curpos the position of the current operation to be retried
       * @throws IOException if all alternate locations are exhausted
       */
      private void setAlternateLocations(IOException curexp, long curpos) 
        throws IOException {
        while (alternates != null && nextLocation < alternates.length) {
          try {
            int idx = nextLocation++;
            long corruptOffset = underLyingStream.getPos();

            // Make sure we use DFS and not DistributedRaidFileSystem for unRaid.
            Configuration clientConf = new Configuration(conf);
            Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl",
                                                DistributedFileSystem.class);
            clientConf.set("fs.hdfs.impl", clazz.getName());
            // Disable caching so that a previously cached RaidDfs is not used.
            clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
            Path npath = RaidNode.unRaid(clientConf, path,
                         alternates[idx].destPath,
                         alternates[idx].createDecoder(),
                         stripeLength, corruptOffset);
            if (npath == null)
              continue;

            FileSystem fs1 = getUnderlyingFileSystem(conf);
            fs1.initialize(npath.toUri(), conf);
            LOG.info("Opening alternate path " + npath + " at offset " + curpos);
            FSDataInputStream fd = fs1.open(npath, buffersize);
            fd.seek(curpos);
            underLyingStream.close();
            underLyingStream = fd;
            lfs.fs = fs1;
            path = npath;
            return;
          } catch (Exception e) {
            LOG.info("Error in using alternate path " + path + ". " + e +
                     " Ignoring...");
          }
        }
        throw curexp;
      }

      /**
       * The name of the file system that is immediately below the
       * DistributedRaidFileSystem. This is specified by the
       * configuration parameter called fs.raid.underlyingfs.impl.
       * If this parameter is not specified in the configuration, then
       * the default class DistributedFileSystem is returned.
       * @param conf the configuration object
       * @return the filesystem object immediately below DistributedRaidFileSystem
       * @throws IOException if all alternate locations are exhausted
       */
      private FileSystem getUnderlyingFileSystem(Configuration conf) {
        Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl", DistributedFileSystem.class);
        FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
        return fs;
      }
    }
  
    /**
     * constructor for ext input stream.
     * @param fs the underlying filesystem
     * @param p the path in the underlying file system
     * @param buffersize the size of IO
     * @throws IOException
     */
    public ExtFSDataInputStream(Configuration conf, DistributedRaidFileSystem lfs,
      DecodeInfo[] alternates, Path  p, int stripeLength, int buffersize) throws IOException {
        super(new ExtFsInputStream(conf, lfs, alternates, p, stripeLength, buffersize));
    }
  }
}