branch-2.0.4-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hdfs;

import static org.junit.Assert.assertEquals;

import java.io.File;
import java.io.IOException;
import java.util.EnumMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.util.Holder;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;

/**
 * Unit test to make sure that Append properly logs the right
 * things to the edit log, such that files aren't lost or truncated
 * on restart.
 */
public class TestFileAppendRestart {
  private static final int BLOCK_SIZE = 4096;
  private static final String HADOOP_23_BROKEN_APPEND_TGZ =
      "image-with-buggy-append.tgz";
    
  private void writeAndAppend(FileSystem fs, Path p,
      int lengthForCreate, int lengthForAppend) throws IOException {
    // Creating a file with 4096 blockSize to write multiple blocks
    FSDataOutputStream stream = fs.create(
        p, true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
    try {
      AppendTestUtil.write(stream, 0, lengthForCreate);
      stream.close();
      
      stream = fs.append(p);
      AppendTestUtil.write(stream, lengthForCreate, lengthForAppend);
      stream.close();
    } finally {
      IOUtils.closeStream(stream);
    }
    
    int totalLength = lengthForCreate + lengthForAppend; 
    assertEquals(totalLength, fs.getFileStatus(p).getLen());
  }
    
  /**
   * Regression test for HDFS-2991. Creates and appends to files
   * where blocks start/end on block boundaries.
   */
  @Test
  public void testAppendRestart() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // Turn off persistent IPC, so that the DFSClient can survive NN restart
    conf.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
        0);
    MiniDFSCluster cluster = null;

    FSDataOutputStream stream = null;
    try {
      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
      FileSystem fs = cluster.getFileSystem();
      File editLog =
        new File(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(0),
            NNStorage.getInProgressEditsFileName(1));
      EnumMap<FSEditLogOpCodes, Holder<Integer>> counts;
      
      Path p1 = new Path("/block-boundaries");
      writeAndAppend(fs, p1, BLOCK_SIZE, BLOCK_SIZE);

      counts = FSImageTestUtil.countEditLogOpTypes(editLog);
      // OP_ADD to create file
      // OP_UPDATE_BLOCKS for first block
      // OP_CLOSE to close file
      // OP_ADD to reopen file
      // OP_UPDATE_BLOCKS for second block
      // OP_CLOSE to close file
      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);

      Path p2 = new Path("/not-block-boundaries");
      writeAndAppend(fs, p2, BLOCK_SIZE/2, BLOCK_SIZE);
      counts = FSImageTestUtil.countEditLogOpTypes(editLog);
      // OP_ADD to create file
      // OP_UPDATE_BLOCKS for first block
      // OP_CLOSE to close file
      // OP_ADD to re-establish the lease
      // OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block)
      // OP_UPDATE_BLOCKS at the start of the second block
      // OP_CLOSE to close file
      // Total: 2 OP_ADDs, 3 OP_UPDATE_BLOCKS, and 2 OP_CLOSEs in addition
      //        to the ones above
      assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
      assertEquals(2+3, (int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
      assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
      
      cluster.restartNameNode();
      
      AppendTestUtil.check(fs, p1, 2*BLOCK_SIZE);
      AppendTestUtil.check(fs, p2, 3*BLOCK_SIZE/2);
    } finally {
      IOUtils.closeStream(stream);
      if (cluster != null) { cluster.shutdown(); }
    }
  }

  /**
   * Earlier versions of HDFS had a bug (HDFS-2991) which caused
   * append(), when called exactly at a block boundary,
   * to not log an OP_ADD. This ensures that we can read from
   * such buggy versions correctly, by loading an image created
   * using a namesystem image created with 0.23.1-rc2 exhibiting
   * the issue.
   */
  @Test
  public void testLoadLogsFromBuggyEarlierVersions() throws IOException {
    final Configuration conf = new HdfsConfiguration();

    String tarFile = System.getProperty("test.cache.data", "build/test/cache")
      + "/" + HADOOP_23_BROKEN_APPEND_TGZ;
    String testDir = System.getProperty("test.build.data", "build/test/data");
    File dfsDir = new File(testDir, "image-with-buggy-append");
    if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
      throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
    }
    FileUtil.unTar(new File(tarFile), new File(testDir));

    File nameDir = new File(dfsDir, "name");
    GenericTestUtils.assertExists(nameDir);

    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath());

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
      .format(false)
      .manageDataDfsDirs(false)
      .manageNameDfsDirs(false)
      .numDataNodes(0)
      .waitSafeMode(false)
      .startupOption(StartupOption.UPGRADE)
      .build();
    try {
      FileSystem fs = cluster.getFileSystem();
      Path testPath = new Path("/tmp/io_data/test_io_0");
      assertEquals(2*1024*1024, fs.getFileStatus(testPath).getLen());
    } finally {
      cluster.shutdown();
    }
  }

  /**
   * Test to append to the file, when one of datanode in the existing pipeline is down.
   * @throws Exception
   */
  @Test
  public void testAppendWithPipelineRecovery() throws Exception {
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = null;
    try {
      cluster = new MiniDFSCluster.Builder(conf).manageDataDfsDirs(true)
          .manageNameDfsDirs(true).numDataNodes(4)
          .racks(new String[] { "/rack1", "/rack1", "/rack1", "/rack2" })
          .build();
      cluster.waitActive();

      DistributedFileSystem fs = cluster.getFileSystem();
      Path path = new Path("/test1");
      DFSTestUtil.createFile(fs, path, 1024, (short) 3, 1l);

      cluster.stopDataNode(3);
      DFSTestUtil.appendFile(fs, path, "hello");
    } finally {
      if (null != cluster) {
        cluster.shutdown();
      }
    }
  }
}