hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package org.apache.hadoop.fs.swift;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
import org.apache.hadoop.fs.swift.util.DurationStats;
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;

import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;

/**
 * This is the base class for most of the Swift tests
 */
public class SwiftFileSystemBaseTest extends Assert implements
                                                    SwiftTestConstants {

  protected static final Log LOG =
          LogFactory.getLog(SwiftFileSystemBaseTest.class);
  protected SwiftNativeFileSystem fs;
  protected static SwiftNativeFileSystem lastFs;
  protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
  private Configuration conf;

  @Before
  public void setUp() throws Exception {
    noteAction("setup");
    final URI uri = getFilesystemURI();
    conf = createConfiguration();

    fs = createSwiftFS();
    try {
      fs.initialize(uri, conf);
    } catch (IOException e) {
      //FS init failed, set it to null so that teardown doesn't
      //attempt to use it
      fs = null;
      throw e;
    }
    //remember the last FS
    lastFs = fs;
    noteAction("setup complete");
  }

  /**
   * Configuration generator. May be overridden to inject
   * some custom options
   * @return a configuration with which to create FS instances
   */
  protected Configuration createConfiguration() {
    return new Configuration();
  }

  @After
  public void tearDown() throws Exception {
    cleanupInTeardown(fs, "/test");
  }

  @AfterClass
  public static void classTearDown() throws Exception {
    if (lastFs  != null) {
      List<DurationStats> statistics = lastFs.getOperationStatistics();
      for (DurationStats stat : statistics) {
        LOG.info(stat.toString());
      }
    }
  }

  /**
   * Get the configuration used to set up the FS
   * @return the configuration
   */
  public Configuration getConf() {
    return conf;
  }

  /**
   * Describe the test, combining some logging with details
   * for people reading the code
   *
   * @param description test description
   */
  protected void describe(String description) {
    noteAction(description);
  }

  protected URI getFilesystemURI() throws URISyntaxException, IOException {
    return getServiceURI(createConfiguration());
  }

  protected SwiftNativeFileSystem createSwiftFS() throws IOException {
    SwiftNativeFileSystem swiftNativeFileSystem =
      new SwiftNativeFileSystem();
    return swiftNativeFileSystem;
  }

  protected int getBlockSize() {
    return 1024;
  }

  /**
   * Is rename supported?
   * @return true
   */
  protected boolean renameSupported() {
    return true;
  }

  /**
   * assume in a test that rename is supported;
   * skip it if not
   */
  protected void assumeRenameSupported() {
    Assume.assumeTrue(renameSupported());
  }

  /**
   * Take an unqualified path, and qualify it w.r.t the
   * current filesystem
   * @param pathString source path
   * @return a qualified path instance
   */
  protected Path path(String pathString) {
    return new Path(pathString).makeQualified(fs);
  }

  /**
   * Get the filesystem
   * @return the current FS
   */
  public SwiftNativeFileSystem getFs() {
    return fs;
  }

  /**
   * Create a file using the standard {@link #data} bytes.
   *
   * @param path path to write
   * @throws IOException on any problem
   */
  protected void createFile(Path path) throws IOException {
    createFile(path, data);
  }

  /**
   * Create a file with the given data.
   *
   * @param path       path to write
   * @param sourceData source dataset
   * @throws IOException on any problem
   */
  protected void createFile(Path path, byte[] sourceData) throws IOException {
    FSDataOutputStream out = fs.create(path);
    out.write(sourceData, 0, sourceData.length);
    out.close();
  }

  /**
   * Create and then close a file
   * @param path path to create
   * @throws IOException on a failure
   */
  protected void createEmptyFile(Path path) throws IOException {
    FSDataOutputStream out = fs.create(path);
    out.close();
  }

  /**
   * Get the inner store -useful for lower level operations
   *
   * @return the store
   */
  protected SwiftNativeFileSystemStore getStore() {
    return fs.getStore();
  }

  /**
   * Rename a path
   * @param src source
   * @param dst dest
   * @param renameMustSucceed flag to say "this rename must exist"
   * @param srcExists add assert that the source exists afterwards
   * @param dstExists add assert the dest exists afterwards
   * @throws IOException IO trouble
   */
  protected void rename(Path src, Path dst, boolean renameMustSucceed,
                        boolean srcExists, boolean dstExists) throws IOException {
    if (renameMustSucceed) {
      renameToSuccess(src, dst, srcExists, dstExists);
    } else {
      renameToFailure(src, dst);
    }
  }

  /**
   * Get a string describing the outcome of a rename, by listing the dest
   * path and its parent along with some covering text
   * @param src source patj
   * @param dst dest path
   * @return a string for logs and exceptions
   * @throws IOException IO problems
   */
  private String getRenameOutcome(Path src, Path dst) throws IOException {
    String lsDst = ls(dst);
    Path parent = dst.getParent();
    String lsParent = parent != null ? ls(parent) : "";
    return "  result of " + src + " => " + dst
            + " - " + lsDst
            + " \n" + lsParent;
  }

  /**
   * Rename, expecting an exception to be thrown
   *
   * @param src source
   * @param dst dest
   * @throws IOException a failure other than an
   *                     expected SwiftRenameException or FileNotFoundException
   */
  protected void renameToFailure(Path src, Path dst) throws IOException {
    try {
      getStore().rename(src, dst);
      fail("Expected failure renaming " + src + " to " + dst
              + "- but got success");
    } catch (SwiftOperationFailedException e) {
      LOG.debug("Rename failed (expected):" + e);
    } catch (FileNotFoundException e) {
      LOG.debug("Rename failed (expected):" + e);
    }
  }

  /**
   * Rename to success
   *
   * @param src source
   * @param dst dest
   * @param srcExists add assert that the source exists afterwards
   * @param dstExists add assert the dest exists afterwards
   * @throws SwiftOperationFailedException operation failure
   * @throws IOException IO problems
   */
  protected void renameToSuccess(Path src, Path dst,
                                 boolean srcExists, boolean dstExists)
      throws SwiftOperationFailedException, IOException {
    getStore().rename(src, dst);
    String outcome = getRenameOutcome(src, dst);
    assertEquals("Source " + src + "exists: " + outcome,
            srcExists, fs.exists(src));
    assertEquals("Destination " + dstExists + " exists" + outcome,
            dstExists, fs.exists(dst));
  }

  /**
   * List a path in the test FS
   * @param path path to list
   * @return the contents of the path/dir
   * @throws IOException IO problems
   */
  protected String ls(Path path) throws IOException {
    return SwiftTestUtils.ls(fs, path);
  }

  /**
   * assert that a path exists
   * @param message message to use in an assertion
   * @param path path to probe
   * @throws IOException IO problems
   */
  public void assertExists(String message, Path path) throws IOException {
    assertPathExists(fs, message, path);
  }

  /**
   * assert that a path does not
   * @param message message to use in an assertion
   * @param path path to probe
   * @throws IOException IO problems
   */
  public void assertPathDoesNotExist(String message, Path path) throws
          IOException {
    SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
  }

  /**
   * Assert that a file exists and whose {@link FileStatus} entry
   * declares that this is a file and not a symlink or directory.
   *
   * @param filename name of the file
   * @throws IOException IO problems during file operations
   */
  protected void assertIsFile(Path filename) throws IOException {
    SwiftTestUtils.assertIsFile(fs, filename);
  }

  /**
   * Assert that a file exists and whose {@link FileStatus} entry
   * declares that this is a file and not a symlink or directory.
   *
   * @throws IOException IO problems during file operations
   */
  protected void mkdirs(Path path) throws IOException {
    assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
  }

  /**
   * Assert that a delete succeeded
   * @param path path to delete
   * @param recursive recursive flag
   * @throws IOException IO problems
   */
  protected void assertDeleted(Path path, boolean recursive) throws IOException {
    SwiftTestUtils.assertDeleted(fs, path, recursive);
  }

  /**
   * Assert that a value is not equal to the expected value
   * @param message message if the two values are equal
   * @param expected expected value
   * @param actual actual value
   */
  protected void assertNotEqual(String message, int expected, int actual) {
    assertTrue(message,
               actual != expected);
  }

  /**
   * Get the number of partitions written from the Swift Native FS APIs
   * @param out output stream
   * @return the number of partitioned files written by the stream
   */
  protected int getPartitionsWritten(FSDataOutputStream out) {
    return SwiftNativeFileSystem.getPartitionsWritten(out);
  }

  /**
   * Assert that the no. of partitions written matches expectations
   * @param action operation (for use in the assertions)
   * @param out output stream
   * @param expected expected no. of partitions
   */
  protected void assertPartitionsWritten(String action, FSDataOutputStream out,
                                         long expected) {
    OutputStream nativeStream = out.getWrappedStream();
    int written = getPartitionsWritten(out);
    if(written !=expected) {
    Assert.fail(action + ": " +
                TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT 
                + " + expected: " + expected + " actual: " + written
                + " -- " + nativeStream);
    }
  }

  /**
   * Assert that the result value == -1; which implies
   * that a read was successful
   * @param text text to include in a message (usually the operation)
   * @param result read result to validate
   */
  protected void assertMinusOne(String text, int result) {
    assertEquals(text + " wrong read result " + result, -1, result);
  }
}