branch-2.0.4-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.web;

import java.net.InetSocketAddress;
import java.net.URI;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * Test WebHDFS with multiple NameNodes
 */
public class TestWebHdfsWithMultipleNameNodes {
  static final Log LOG = WebHdfsTestUtil.LOG;

  static private void setLogLevel() {
    ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL);
    ((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);

    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF);
    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF);
    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
  }

  private static final Configuration conf = new HdfsConfiguration();
  private static MiniDFSCluster cluster;
  private static WebHdfsFileSystem[] webhdfs;

  @BeforeClass
  public static void setupTest() {
    setLogLevel();
    try {
      setupCluster(4, 3);
    } catch(Exception e) {
      throw new RuntimeException(e);
    }
  }

  private static void setupCluster(final int nNameNodes, final int nDataNodes)
      throws Exception {
    LOG.info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes);

    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);

    cluster = new MiniDFSCluster.Builder(conf)
        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
        .numDataNodes(nDataNodes)
        .build();
    cluster.waitActive();
    
    webhdfs = new WebHdfsFileSystem[nNameNodes];
    for(int i = 0; i < webhdfs.length; i++) {
      final InetSocketAddress addr = cluster.getNameNode(i).getHttpAddress();
      final String uri = WebHdfsFileSystem.SCHEME  + "://"
          + addr.getHostName() + ":" + addr.getPort() + "/";
      webhdfs[i] = (WebHdfsFileSystem)FileSystem.get(new URI(uri), conf);
    }
  }

  @AfterClass
  public static void shutdownCluster() {
    if (cluster != null) {
      cluster.shutdown();
      cluster = null;
    }
  }

  private static String createString(String prefix, int i) {
    //The suffix is to make sure the strings have different lengths.
    final String suffix = "*********************".substring(0, i+1);
    return prefix + i + suffix + "\n";
  }

  private static String[] createStrings(String prefix, String name) {
    final String[] strings = new String[webhdfs.length]; 
    for(int i = 0; i < webhdfs.length; i++) {
      strings[i] = createString(prefix, i);
      LOG.info(name + "[" + i + "] = " + strings[i]);
    }
    return strings;
  }

  @Test
  public void testRedirect() throws Exception {
    final String dir = "/testRedirect/";
    final String filename = "file";
    final Path p = new Path(dir, filename);

    final String[] writeStrings = createStrings("write to webhdfs ", "write"); 
    final String[] appendStrings = createStrings("append to webhdfs ", "append"); 
    
    //test create: create a file for each namenode
    for(int i = 0; i < webhdfs.length; i++) {
      final FSDataOutputStream out = webhdfs[i].create(p);
      out.write(writeStrings[i].getBytes());
      out.close();
    }
    
    for(int i = 0; i < webhdfs.length; i++) {
      //check file length
      final long expected = writeStrings[i].length();
      Assert.assertEquals(expected, webhdfs[i].getFileStatus(p).getLen());
    }

    //test read: check file content for each namenode
    for(int i = 0; i < webhdfs.length; i++) {
      final FSDataInputStream in = webhdfs[i].open(p);
      for(int c, j = 0; (c = in.read()) != -1; j++) {
        Assert.assertEquals(writeStrings[i].charAt(j), c);
      }
      in.close();
    }

    //test append: append to the file for each namenode
    for(int i = 0; i < webhdfs.length; i++) {
      final FSDataOutputStream out = webhdfs[i].append(p);
      out.write(appendStrings[i].getBytes());
      out.close();
    }

    for(int i = 0; i < webhdfs.length; i++) {
      //check file length
      final long expected = writeStrings[i].length() + appendStrings[i].length();
      Assert.assertEquals(expected, webhdfs[i].getFileStatus(p).getLen());
    }

    //test read: check file content for each namenode
    for(int i = 0; i < webhdfs.length; i++) {
      final StringBuilder b = new StringBuilder(); 
      final FSDataInputStream in = webhdfs[i].open(p);
      for(int c; (c = in.read()) != -1; ) {
        b.append((char)c);
      }
      final int wlen = writeStrings[i].length();
      Assert.assertEquals(writeStrings[i], b.substring(0, wlen));
      Assert.assertEquals(appendStrings[i], b.substring(wlen));
      in.close();
    }
  }
}