| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.fs; |
| |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| |
| import junit.framework.TestCase; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.permission.FsPermission; |
| |
| /** |
| * <p> |
| * A collection of tests for the contract of the {@link FileSystem}. |
| * This test should be used for general-purpose implementations of |
| * {@link FileSystem}, that is, implementations that provide implementations |
| * of all of the functionality of {@link FileSystem}. |
| * </p> |
| * <p> |
| * To test a given {@link FileSystem} implementation create a subclass of this |
| * test and override {@link #setUp()} to initialize the <code>fs</code> |
| * {@link FileSystem} instance variable. |
| * </p> |
| */ |
| public abstract class FileSystemContractBaseTest extends TestCase { |
| private static final Log LOG = |
| LogFactory.getLog(FileSystemContractBaseTest.class); |
| |
| protected final static String TEST_UMASK = "062"; |
| protected FileSystem fs; |
| protected byte[] data = dataset(getBlockSize() * 2, 0, 255); |
| |
| @Override |
| protected void tearDown() throws Exception { |
| fs.delete(path("/test"), true); |
| } |
| |
| protected int getBlockSize() { |
| return 1024; |
| } |
| |
| protected String getDefaultWorkingDirectory() { |
| return "/user/" + System.getProperty("user.name"); |
| } |
| |
| protected boolean renameSupported() { |
| return true; |
| } |
| |
| public void testFsStatus() throws Exception { |
| FsStatus fsStatus = fs.getStatus(); |
| assertNotNull(fsStatus); |
| //used, free and capacity are non-negative longs |
| assertTrue(fsStatus.getUsed() >= 0); |
| assertTrue(fsStatus.getRemaining() >= 0); |
| assertTrue(fsStatus.getCapacity() >= 0); |
| } |
| |
| public void testWorkingDirectory() throws Exception { |
| |
| Path workDir = path(getDefaultWorkingDirectory()); |
| assertEquals(workDir, fs.getWorkingDirectory()); |
| |
| fs.setWorkingDirectory(path(".")); |
| assertEquals(workDir, fs.getWorkingDirectory()); |
| |
| fs.setWorkingDirectory(path("..")); |
| assertEquals(workDir.getParent(), fs.getWorkingDirectory()); |
| |
| Path relativeDir = path("hadoop"); |
| fs.setWorkingDirectory(relativeDir); |
| assertEquals(relativeDir, fs.getWorkingDirectory()); |
| |
| Path absoluteDir = path("/test/hadoop"); |
| fs.setWorkingDirectory(absoluteDir); |
| assertEquals(absoluteDir, fs.getWorkingDirectory()); |
| |
| } |
| |
| public void testMkdirs() throws Exception { |
| Path testDir = path("/test/hadoop"); |
| assertFalse(fs.exists(testDir)); |
| assertFalse(fs.isFile(testDir)); |
| |
| assertTrue(fs.mkdirs(testDir)); |
| |
| assertTrue(fs.exists(testDir)); |
| assertFalse(fs.isFile(testDir)); |
| |
| assertTrue(fs.mkdirs(testDir)); |
| |
| assertTrue(fs.exists(testDir)); |
| assertFalse(fs.isFile(testDir)); |
| |
| Path parentDir = testDir.getParent(); |
| assertTrue(fs.exists(parentDir)); |
| assertFalse(fs.isFile(parentDir)); |
| |
| Path grandparentDir = parentDir.getParent(); |
| assertTrue(fs.exists(grandparentDir)); |
| assertFalse(fs.isFile(grandparentDir)); |
| |
| } |
| |
| public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { |
| Path testDir = path("/test/hadoop"); |
| assertFalse(fs.exists(testDir)); |
| assertTrue(fs.mkdirs(testDir)); |
| assertTrue(fs.exists(testDir)); |
| |
| createFile(path("/test/hadoop/file")); |
| |
| Path testSubDir = path("/test/hadoop/file/subdir"); |
| try { |
| fs.mkdirs(testSubDir); |
| fail("Should throw IOException."); |
| } catch (IOException e) { |
| // expected |
| } |
| assertFalse(fs.exists(testSubDir)); |
| |
| Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir"); |
| try { |
| fs.mkdirs(testDeepSubDir); |
| fail("Should throw IOException."); |
| } catch (IOException e) { |
| // expected |
| } |
| assertFalse(fs.exists(testDeepSubDir)); |
| |
| } |
| |
| public void testMkdirsWithUmask() throws Exception { |
| if (fs.getScheme().equals("s3") || fs.getScheme().equals("s3n")) { |
| // skip permission tests for S3FileSystem until HDFS-1333 is fixed. |
| return; |
| } |
| Configuration conf = fs.getConf(); |
| String oldUmask = conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY); |
| try { |
| conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK); |
| final Path dir = new Path("/test/newDir"); |
| assertTrue(fs.mkdirs(dir, new FsPermission((short)0777))); |
| FileStatus status = fs.getFileStatus(dir); |
| assertTrue(status.isDirectory()); |
| assertEquals((short)0715, status.getPermission().toShort()); |
| } finally { |
| conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, oldUmask); |
| } |
| } |
| |
| public void testGetFileStatusThrowsExceptionForNonExistentFile() |
| throws Exception { |
| try { |
| fs.getFileStatus(path("/test/hadoop/file")); |
| fail("Should throw FileNotFoundException"); |
| } catch (FileNotFoundException e) { |
| // expected |
| } |
| } |
| |
| public void testListStatusThrowsExceptionForNonExistentFile() throws Exception { |
| try { |
| fs.listStatus(path("/test/hadoop/file")); |
| fail("Should throw FileNotFoundException"); |
| } catch (FileNotFoundException fnfe) { |
| // expected |
| } |
| } |
| |
| public void testListStatus() throws Exception { |
| Path[] testDirs = { path("/test/hadoop/a"), |
| path("/test/hadoop/b"), |
| path("/test/hadoop/c/1"), }; |
| assertFalse(fs.exists(testDirs[0])); |
| |
| for (Path path : testDirs) { |
| assertTrue(fs.mkdirs(path)); |
| } |
| |
| FileStatus[] paths = fs.listStatus(path("/test")); |
| assertEquals(1, paths.length); |
| assertEquals(path("/test/hadoop"), paths[0].getPath()); |
| |
| paths = fs.listStatus(path("/test/hadoop")); |
| assertEquals(3, paths.length); |
| assertEquals(path("/test/hadoop/a"), paths[0].getPath()); |
| assertEquals(path("/test/hadoop/b"), paths[1].getPath()); |
| assertEquals(path("/test/hadoop/c"), paths[2].getPath()); |
| |
| paths = fs.listStatus(path("/test/hadoop/a")); |
| assertEquals(0, paths.length); |
| } |
| |
| public void testWriteReadAndDeleteEmptyFile() throws Exception { |
| writeReadAndDelete(0); |
| } |
| |
| public void testWriteReadAndDeleteHalfABlock() throws Exception { |
| writeReadAndDelete(getBlockSize() / 2); |
| } |
| |
| public void testWriteReadAndDeleteOneBlock() throws Exception { |
| writeReadAndDelete(getBlockSize()); |
| } |
| |
| public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception { |
| writeReadAndDelete(getBlockSize() + (getBlockSize() / 2)); |
| } |
| |
| public void testWriteReadAndDeleteTwoBlocks() throws Exception { |
| writeReadAndDelete(getBlockSize() * 2); |
| } |
| |
| /** |
| * Write a dataset, read it back in and verify that they match. |
| * Afterwards, the file is deleted. |
| * @param len length of data |
| * @throws IOException on IO failures |
| */ |
| protected void writeReadAndDelete(int len) throws IOException { |
| Path path = path("/test/hadoop/file"); |
| writeAndRead(path, data, len, false, true); |
| } |
| |
| public void testOverwrite() throws IOException { |
| Path path = path("/test/hadoop/file"); |
| |
| fs.mkdirs(path.getParent()); |
| |
| createFile(path); |
| |
| assertTrue("Exists", fs.exists(path)); |
| assertEquals("Length", data.length, fs.getFileStatus(path).getLen()); |
| |
| try { |
| fs.create(path, false).close(); |
| fail("Should throw IOException."); |
| } catch (IOException e) { |
| // Expected |
| } |
| |
| FSDataOutputStream out = fs.create(path, true); |
| out.write(data, 0, data.length); |
| out.close(); |
| |
| assertTrue("Exists", fs.exists(path)); |
| assertEquals("Length", data.length, fs.getFileStatus(path).getLen()); |
| |
| } |
| |
| public void testWriteInNonExistentDirectory() throws IOException { |
| Path path = path("/test/hadoop/file"); |
| assertFalse("Parent doesn't exist", fs.exists(path.getParent())); |
| createFile(path); |
| |
| assertTrue("Exists", fs.exists(path)); |
| assertEquals("Length", data.length, fs.getFileStatus(path).getLen()); |
| assertTrue("Parent exists", fs.exists(path.getParent())); |
| } |
| |
| public void testDeleteNonExistentFile() throws IOException { |
| Path path = path("/test/hadoop/file"); |
| assertFalse("Doesn't exist", fs.exists(path)); |
| assertFalse("No deletion", fs.delete(path, true)); |
| } |
| |
| public void testDeleteRecursively() throws IOException { |
| Path dir = path("/test/hadoop"); |
| Path file = path("/test/hadoop/file"); |
| Path subdir = path("/test/hadoop/subdir"); |
| |
| createFile(file); |
| assertTrue("Created subdir", fs.mkdirs(subdir)); |
| |
| assertTrue("File exists", fs.exists(file)); |
| assertTrue("Dir exists", fs.exists(dir)); |
| assertTrue("Subdir exists", fs.exists(subdir)); |
| |
| try { |
| fs.delete(dir, false); |
| fail("Should throw IOException."); |
| } catch (IOException e) { |
| // expected |
| } |
| assertTrue("File still exists", fs.exists(file)); |
| assertTrue("Dir still exists", fs.exists(dir)); |
| assertTrue("Subdir still exists", fs.exists(subdir)); |
| |
| assertTrue("Deleted", fs.delete(dir, true)); |
| assertFalse("File doesn't exist", fs.exists(file)); |
| assertFalse("Dir doesn't exist", fs.exists(dir)); |
| assertFalse("Subdir doesn't exist", fs.exists(subdir)); |
| } |
| |
| public void testDeleteEmptyDirectory() throws IOException { |
| Path dir = path("/test/hadoop"); |
| assertTrue(fs.mkdirs(dir)); |
| assertTrue("Dir exists", fs.exists(dir)); |
| assertTrue("Deleted", fs.delete(dir, false)); |
| assertFalse("Dir doesn't exist", fs.exists(dir)); |
| } |
| |
| public void testRenameNonExistentPath() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/path"); |
| Path dst = path("/test/new/newpath"); |
| rename(src, dst, false, false, false); |
| } |
| |
| public void testRenameFileMoveToNonExistentDirectory() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/file"); |
| createFile(src); |
| Path dst = path("/test/new/newfile"); |
| rename(src, dst, false, true, false); |
| } |
| |
| public void testRenameFileMoveToExistingDirectory() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/file"); |
| createFile(src); |
| Path dst = path("/test/new/newfile"); |
| fs.mkdirs(dst.getParent()); |
| rename(src, dst, true, false, true); |
| } |
| |
| public void testRenameFileAsExistingFile() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/file"); |
| createFile(src); |
| Path dst = path("/test/new/newfile"); |
| createFile(dst); |
| rename(src, dst, false, true, true); |
| } |
| |
| public void testRenameFileAsExistingDirectory() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/file"); |
| createFile(src); |
| Path dst = path("/test/new/newdir"); |
| fs.mkdirs(dst); |
| rename(src, dst, true, false, true); |
| assertTrue("Destination changed", |
| fs.exists(path("/test/new/newdir/file"))); |
| } |
| |
| public void testRenameDirectoryMoveToNonExistentDirectory() |
| throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/dir"); |
| fs.mkdirs(src); |
| Path dst = path("/test/new/newdir"); |
| rename(src, dst, false, true, false); |
| } |
| |
| public void testRenameDirectoryMoveToExistingDirectory() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/dir"); |
| fs.mkdirs(src); |
| createFile(path("/test/hadoop/dir/file1")); |
| createFile(path("/test/hadoop/dir/subdir/file2")); |
| |
| Path dst = path("/test/new/newdir"); |
| fs.mkdirs(dst.getParent()); |
| rename(src, dst, true, false, true); |
| |
| assertFalse("Nested file1 exists", |
| fs.exists(path("/test/hadoop/dir/file1"))); |
| assertFalse("Nested file2 exists", |
| fs.exists(path("/test/hadoop/dir/subdir/file2"))); |
| assertTrue("Renamed nested file1 exists", |
| fs.exists(path("/test/new/newdir/file1"))); |
| assertTrue("Renamed nested exists", |
| fs.exists(path("/test/new/newdir/subdir/file2"))); |
| } |
| |
| public void testRenameDirectoryAsExistingFile() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/dir"); |
| fs.mkdirs(src); |
| Path dst = path("/test/new/newfile"); |
| createFile(dst); |
| rename(src, dst, false, true, true); |
| } |
| |
| public void testRenameDirectoryAsExistingDirectory() throws Exception { |
| if (!renameSupported()) return; |
| |
| Path src = path("/test/hadoop/dir"); |
| fs.mkdirs(src); |
| createFile(path("/test/hadoop/dir/file1")); |
| createFile(path("/test/hadoop/dir/subdir/file2")); |
| |
| Path dst = path("/test/new/newdir"); |
| fs.mkdirs(dst); |
| rename(src, dst, true, false, true); |
| assertTrue("Destination changed", |
| fs.exists(path("/test/new/newdir/dir"))); |
| assertFalse("Nested file1 exists", |
| fs.exists(path("/test/hadoop/dir/file1"))); |
| assertFalse("Nested file2 exists", |
| fs.exists(path("/test/hadoop/dir/subdir/file2"))); |
| assertTrue("Renamed nested file1 exists", |
| fs.exists(path("/test/new/newdir/dir/file1"))); |
| assertTrue("Renamed nested exists", |
| fs.exists(path("/test/new/newdir/dir/subdir/file2"))); |
| } |
| |
| public void testInputStreamClosedTwice() throws IOException { |
| //HADOOP-4760 according to Closeable#close() closing already-closed |
| //streams should have no effect. |
| Path src = path("/test/hadoop/file"); |
| createFile(src); |
| FSDataInputStream in = fs.open(src); |
| in.close(); |
| in.close(); |
| } |
| |
| public void testOutputStreamClosedTwice() throws IOException { |
| //HADOOP-4760 according to Closeable#close() closing already-closed |
| //streams should have no effect. |
| Path src = path("/test/hadoop/file"); |
| FSDataOutputStream out = fs.create(src); |
| out.writeChar('H'); //write some data |
| out.close(); |
| out.close(); |
| } |
| |
| protected Path path(String pathString) { |
| return new Path(pathString).makeQualified(fs); |
| } |
| |
| protected void createFile(Path path) throws IOException { |
| FSDataOutputStream out = fs.create(path); |
| out.write(data, 0, data.length); |
| out.close(); |
| } |
| |
| private void rename(Path src, Path dst, boolean renameSucceeded, |
| boolean srcExists, boolean dstExists) throws IOException { |
| assertEquals("Rename result", renameSucceeded, fs.rename(src, dst)); |
| assertEquals("Source exists", srcExists, fs.exists(src)); |
| assertEquals("Destination exists", dstExists, fs.exists(dst)); |
| } |
| |
| /** |
| * Verify that if you take an existing file and overwrite it, the new values |
| * get picked up. |
| * This is a test for the behavior of eventually consistent |
| * filesystems. |
| * |
| * @throws Exception on any failure |
| */ |
| |
| public void testOverWriteAndRead() throws Exception { |
| int blockSize = getBlockSize(); |
| |
| byte[] filedata1 = dataset(blockSize * 2, 'A', 26); |
| byte[] filedata2 = dataset(blockSize * 2, 'a', 26); |
| Path path = path("/test/hadoop/file-overwrite"); |
| writeAndRead(path, filedata1, blockSize, true, false); |
| writeAndRead(path, filedata2, blockSize, true, false); |
| writeAndRead(path, filedata1, blockSize * 2, true, false); |
| writeAndRead(path, filedata2, blockSize * 2, true, false); |
| writeAndRead(path, filedata1, blockSize, true, false); |
| writeAndRead(path, filedata2, blockSize * 2, true, false); |
| } |
| |
| /** |
| * |
| * Write a file and read it in, validating the result. Optional flags control |
| * whether file overwrite operations should be enabled, and whether the |
| * file should be deleted afterwards. |
| * |
| * If there is a mismatch between what was written and what was expected, |
| * a small range of bytes either side of the first error are logged to aid |
| * diagnosing what problem occurred -whether it was a previous file |
| * or a corrupting of the current file. This assumes that two |
| * sequential runs to the same path use datasets with different character |
| * moduli. |
| * |
| * @param path path to write to |
| * @param len length of data |
| * @param overwrite should the create option allow overwrites? |
| * @param delete should the file be deleted afterwards? -with a verification |
| * that it worked. Deletion is not attempted if an assertion has failed |
| * earlier -it is not in a <code>finally{}</code> block. |
| * @throws IOException IO problems |
| */ |
| protected void writeAndRead(Path path, byte[] src, int len, |
| boolean overwrite, |
| boolean delete) throws IOException { |
| assertTrue("Not enough data in source array to write " + len + " bytes", |
| src.length >= len); |
| fs.mkdirs(path.getParent()); |
| |
| FSDataOutputStream out = fs.create(path, overwrite, |
| fs.getConf().getInt("io.file.buffer.size", |
| 4096), |
| (short) 1, getBlockSize()); |
| out.write(src, 0, len); |
| out.close(); |
| |
| assertTrue("Exists", fs.exists(path)); |
| assertEquals("Length", len, fs.getFileStatus(path).getLen()); |
| |
| FSDataInputStream in = fs.open(path); |
| byte[] buf = new byte[len]; |
| in.readFully(0, buf); |
| in.close(); |
| |
| assertEquals(len, buf.length); |
| int errors = 0; |
| int first_error_byte = -1; |
| for (int i = 0; i < len; i++) { |
| if (src[i] != buf[i]) { |
| if (errors == 0) { |
| first_error_byte = i; |
| } |
| errors++; |
| } |
| } |
| |
| if (errors > 0) { |
| String message = String.format(" %d errors in file of length %d", |
| errors, len); |
| LOG.warn(message); |
| // the range either side of the first error to print |
| // this is a purely arbitrary number, to aid user debugging |
| final int overlap = 10; |
| for (int i = Math.max(0, first_error_byte - overlap); |
| i < Math.min(first_error_byte + overlap, len); |
| i++) { |
| byte actual = buf[i]; |
| byte expected = src[i]; |
| String letter = toChar(actual); |
| String line = String.format("[%04d] %2x %s\n", i, actual, letter); |
| if (expected != actual) { |
| line = String.format("[%04d] %2x %s -expected %2x %s\n", |
| i, |
| actual, |
| letter, |
| expected, |
| toChar(expected)); |
| } |
| LOG.warn(line); |
| } |
| fail(message); |
| } |
| |
| if (delete) { |
| boolean deleted = fs.delete(path, false); |
| assertTrue("Deleted", deleted); |
| assertFalse("No longer exists", fs.exists(path)); |
| } |
| } |
| |
| /** |
| * Convert a byte to a character for printing. If the |
| * byte value is < 32 -and hence unprintable- the byte is |
| * returned as a two digit hex value |
| * @param b byte |
| * @return the printable character string |
| */ |
| protected String toChar(byte b) { |
| if (b >= 0x20) { |
| return Character.toString((char) b); |
| } else { |
| return String.format("%02x", b); |
| } |
| } |
| |
| /** |
| * Create a dataset for use in the tests; all data is in the range |
| * base to (base+modulo-1) inclusive |
| * @param len length of data |
| * @param base base of the data |
| * @param modulo the modulo |
| * @return the newly generated dataset |
| */ |
| protected byte[] dataset(int len, int base, int modulo) { |
| byte[] dataset = new byte[len]; |
| for (int i = 0; i < len; i++) { |
| dataset[i] = (byte) (base + (i % modulo)); |
| } |
| return dataset; |
| } |
| } |