HADOOP-12304. Applications using FileContext fail with the default file system configured to be wasb/s3/etc. Contributed by Chris Nauroth.
(cherry picked from commit 3da0bedaec38714a62c3b7244e84bc4f981ddf45)
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 160a4f0..3cbc5d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -578,6 +578,9 @@
HDFS-8767. RawLocalFileSystem.listStatus() returns null for UNIX pipefile.
(kanaka kumar avvaru via wheat9)
+ HADOOP-12304. Applications using FileContext fail with the default file
+ system configured to be wasb/s3/etc. (cnauroth)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
index 6b7f387..388e64b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
@@ -46,12 +46,27 @@
Configuration conf, String supportedScheme, boolean authorityRequired)
throws IOException, URISyntaxException {
super(theUri, supportedScheme, authorityRequired,
- theFsImpl.getDefaultPort());
+ getDefaultPortIfDefined(theFsImpl));
fsImpl = theFsImpl;
fsImpl.initialize(theUri, conf);
fsImpl.statistics = getStatistics();
}
+ /**
+ * Returns the default port if the file system defines one.
+ * {@link FileSystem#getDefaultPort()} returns 0 to indicate the default port
+ * is undefined. However, the logic that consumes this value expects to
+ * receive -1 to indicate the port is undefined, which agrees with the
+ * contract of {@link URI#getPort()}.
+ *
+ * @param theFsImpl file system to check for default port
+ * @return default port, or -1 if default port is undefined
+ */
+ private static int getDefaultPortIfDefined(FileSystem theFsImpl) {
+ int defaultPort = theFsImpl.getDefaultPort();
+ return defaultPort != 0 ? defaultPort : -1;
+ }
+
@Override
public Path getInitialWorkingDirectory() {
return fsImpl.getInitialWorkingDirectory();
@@ -239,4 +254,4 @@
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return Arrays.asList(fsImpl.addDelegationTokens(renewer, null));
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
index a4ca6fd..25eee47 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.azure;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -35,6 +36,8 @@
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
@@ -362,7 +365,7 @@
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI(defaultScheme, authority, null, null, null);
- conf.set("fs.default.name", defaultUri.toString());
+ conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
// Add references to file system implementations for wasb and wasbs.
conf.addResource("azure-test.xml");
@@ -385,11 +388,34 @@
// authority for the Azure file system should throw.
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
- conf.set("fs.default.name", "file:///");
+ conf.set(FS_DEFAULT_NAME_KEY, "file:///");
try {
FileSystem.get(new URI("wasb:///random/path"), conf);
fail("Should've thrown.");
} catch (IllegalArgumentException e) {
}
}
+
+ @Test
+ public void testWasbAsDefaultFileSystemHasNoPort() throws Exception {
+ try {
+ testAccount = AzureBlobStorageTestAccount.createMock();
+ Configuration conf = testAccount.getFileSystem().getConf();
+ String authority = testAccount.getFileSystem().getUri().getAuthority();
+ URI defaultUri = new URI("wasb", authority, null, null, null);
+ conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
+ conf.addResource("azure-test.xml");
+
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs instanceof NativeAzureFileSystem);
+ assertEquals(-1, fs.getUri().getPort());
+
+ AbstractFileSystem afs = FileContext.getFileContext(conf)
+ .getDefaultFileSystem();
+ assertTrue(afs instanceof Wasb);
+ assertEquals(-1, afs.getUri().getPort());
+ } finally {
+ FileSystem.closeAll();
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
index 98e68c4..75b466d 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
@@ -35,4 +35,8 @@
<value>true</value>
</property>
-->
+ <property>
+ <name>fs.AbstractFileSystem.wasb.impl</name>
+ <value>org.apache.hadoop.fs.azure.Wasb</value>
+ </property>
</configuration>
