HADOOP-19146. S3A: noaa-cors-pds test bucket access with global endpoint fails (#6723)
HADOOP-19057 switched the hadoop-aws test bucket from landsat-pds to
noaa-cors-pds
This new bucket isn't accessible if the client configuration
sets an fs.s3a.endpoint/region value other than us-east-1.
Contributed by Viraj Jasani
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index 87a60c8..e6f258e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -44,6 +44,7 @@
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.*;
@@ -207,6 +208,11 @@ public void testBadCredentialsWithRemap() throws Exception {
@Test
public void testAnonymousProvider() throws Exception {
Configuration conf = createConf(AnonymousAWSCredentialsProvider.class);
+ if (isUsingDefaultExternalDataFile(conf)) {
+ removeBaseAndBucketOverrides(conf,
+ ENDPOINT);
+ conf.set(ENDPOINT, CENTRAL_ENDPOINT);
+ }
Path testFile = getExternalData(conf);
try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
Assertions.assertThat(fs)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
index 5f90115..b550fc5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -32,9 +32,6 @@
import org.apache.hadoop.fs.store.audit.AuditSpan;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.util.ArrayList;
@@ -45,7 +42,9 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData;
import static org.apache.hadoop.test.LambdaTestUtils.*;
import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
@@ -55,14 +54,15 @@
* ITest for failure handling, primarily multipart deletion.
*/
public class ITestS3AFailureHandling extends AbstractS3ATestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(ITestS3AFailureHandling.class);
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
S3ATestUtils.disableFilesystemCaching(conf);
conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true);
+ if (isUsingDefaultExternalDataFile(conf)) {
+ removeBaseAndBucketOverrides(conf, Constants.ENDPOINT);
+ }
return conf;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
index ce96248..5e6731e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
@@ -38,9 +38,11 @@
import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
@@ -84,7 +86,11 @@ public void setUp() throws Exception {
@Override
public Configuration createConfiguration() {
Configuration configuration = super.createConfiguration();
- S3ATestUtils.removeBaseAndBucketOverrides(configuration, PREFETCH_ENABLED_KEY);
+ if (isUsingDefaultExternalDataFile(configuration)) {
+ S3ATestUtils.removeBaseAndBucketOverrides(configuration,
+ PREFETCH_ENABLED_KEY,
+ ENDPOINT);
+ }
configuration.setBoolean(PREFETCH_ENABLED_KEY, true);
// use a small block size unless explicitly set in the test config.
configuration.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
index ba97463..4aaf35f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
@@ -53,12 +54,14 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyInt;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.terminateService;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.probeForAssumedRoleARN;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.assertSecurityEnabled;
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.closeUserFileSystems;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getOrcData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireAnonymousDataPath;
/**
@@ -251,6 +254,10 @@ public void testCommonCrawlLookup() throws Throwable {
public void testJobSubmissionCollectsTokens() throws Exception {
describe("Mock Job test");
JobConf conf = new JobConf(getConfiguration());
+ if (isUsingDefaultExternalDataFile(conf)) {
+ removeBaseAndBucketOverrides(conf,
+ Constants.ENDPOINT);
+ }
// the input here is the external file; which lets
// us differentiate source URI from dest URI
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index fd7a528..22a4727 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -118,12 +118,12 @@ protected int run(Object... args) throws Exception {
* Run a S3GuardTool command from a varags list, catch any raised
* ExitException and verify the status code matches that expected.
* @param status expected status code of the exception
+ * @param conf configuration object.
* @param args argument list
* @throws Exception any exception
*/
- protected void runToFailure(int status, Object... args)
+ protected void runToFailure(int status, Configuration conf, Object... args)
throws Exception {
- final Configuration conf = getConfiguration();
ExitUtil.ExitException ex =
intercept(ExitUtil.ExitException.class, () ->
runS3GuardCommand(conf, args));
@@ -247,7 +247,7 @@ public void testUnsupported() throws Throwable {
describe("Verify the unsupported tools are rejected");
for (String tool : UNSUPPORTED_COMMANDS) {
describe("Probing %s", tool);
- runToFailure(E_S3GUARD_UNSUPPORTED, tool);
+ runToFailure(E_S3GUARD_UNSUPPORTED, getConfiguration(), tool);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
index 18fdcca..5978761 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
@@ -48,6 +49,7 @@
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
/**
* Test S3Guard Tool CLI commands.
@@ -60,8 +62,13 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
@Test
public void testExternalBucketRequireUnencrypted() throws Throwable {
- removeBaseAndBucketOverrides(getConfiguration(), S3_ENCRYPTION_ALGORITHM);
- run(BucketInfo.NAME,
+ Configuration conf = getConfiguration();
+ if (isUsingDefaultExternalDataFile(conf)) {
+ removeBaseAndBucketOverrides(conf,
+ S3_ENCRYPTION_ALGORITHM,
+ ENDPOINT);
+ }
+ run(conf, BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG, "none",
externalBucket());
}
@@ -81,10 +88,17 @@ private String externalBucket() {
@Test
public void testExternalBucketRequireEncrypted() throws Throwable {
+ Configuration conf = getConfiguration();
+ if (isUsingDefaultExternalDataFile(conf)) {
+ removeBaseAndBucketOverrides(conf,
+ ENDPOINT);
+ }
runToFailure(E_BAD_STATE,
+ conf,
BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG,
- "AES256", externalBucket());
+ "AES256",
+ externalBucket());
}
@Test
@@ -212,9 +226,13 @@ public void testUploadListByAge() throws Throwable {
@Test
public void testUploadNegativeExpect() throws Throwable {
- runToFailure(E_BAD_STATE, Uploads.NAME, "-expect", "1",
- path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew")
- .toString());
+ Configuration conf = getConfiguration();
+ runToFailure(E_BAD_STATE,
+ conf,
+ Uploads.NAME,
+ "-expect",
+ "1",
+ path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew").toString());
}
private void assertNumUploads(Path path, int numUploads) throws Exception {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
index ae09452..a787f52 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -60,6 +60,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic;
@@ -99,7 +100,13 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
@Override
protected Configuration createScaleConfiguration() {
Configuration conf = super.createScaleConfiguration();
- S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+ S3ATestUtils.removeBaseAndBucketOverrides(conf,
+ PREFETCH_ENABLED_KEY);
+ if (isUsingDefaultExternalDataFile(conf)) {
+ S3ATestUtils.removeBaseAndBucketOverrides(
+ conf,
+ ENDPOINT);
+ }
conf.setBoolean(PREFETCH_ENABLED_KEY, false);
return conf;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
index 3835548..9400cfb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
@@ -134,6 +134,18 @@ public static String requireDefaultExternalDataFile(Configuration conf) {
}
/**
+ * To determine whether {@value S3ATestConstants#KEY_CSVTEST_FILE} is configured to be
+ * different from the default external file.
+ *
+ * @param conf Configuration object.
+ * @return True if the default external data file is being used.
+ */
+ public static boolean isUsingDefaultExternalDataFile(final Configuration conf) {
+ final String filename = getExternalData(conf).toUri().toString();
+ return DEFAULT_EXTERNAL_FILE.equals(filename);
+ }
+
+ /**
* Get the test external file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer
* read only).