HBASE-23099 - [HBOSS] Raise hadoop dependency version to 3.2.1 (#9)
HBASE-23099 - [HBOSS] Raise hadoop dependency version to 3.2.1
There's been some changes on S3Guard that comes with hadoop 3.2.1 and upcoming versions, such as HADOOP-16085, HADOOP-16233 and HADOOP-16380, among other recent commits. Some of these required additional properties or modified S3A expected behaviour for a few FS contract tests.
Signed-off-by: Josh Elser <elserj@apache.org>
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
index 1bb3710..21299db 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
@@ -52,6 +52,7 @@
@Before
public void setup() throws Exception {
Configuration conf = new Configuration();
+ conf.addResource("contract/s3a.xml");
hboss = TestUtils.getFileSystem(conf);
sync = hboss.getLockManager();
hboss.mkdirs(testPathRoot());
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
index 30f116f..9e4f5d9 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
@@ -102,7 +102,6 @@
return hboss;
} catch (Exception e) {
LOG.error(e.getMessage());
- e.printStackTrace();
throw e;
}
}
@@ -117,4 +116,17 @@
}
}
}
+
+ public static void runIfS3(boolean isS3, Configuration conf) {
+ try {
+ TestUtils.getFileSystem(conf);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ if(isS3) {
+ Assume.assumeTrue(TestUtils.fsIs(TestUtils.S3A, conf));
+ } else {
+ Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ }
+ }
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
index 6ad69ca..4ed4be0 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
@@ -52,6 +52,7 @@
public void setUp() throws Exception {
nameThread();
conf = new Configuration();
+ conf.addResource("contract/s3a.xml");
fs = TestUtils.getFileSystem(conf);
Assume.assumeNotNull(fs);
HBaseObjectStoreSemantics hboss = (HBaseObjectStoreSemantics)fs;
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
index 36adfcf..33d684d 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
@@ -21,7 +21,13 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hbase.oss.TestUtils;
+/**
+ * There is an S3A-specific extension of AbstractContractOpenTest
+ * that is similarly extended for HBOSS-on-S3A. This class remains to be run in
+ * the general case.
+ */
public class TestHBOSSContractOpen extends AbstractContractOpenTest {
@Override
@@ -31,6 +37,8 @@
@Override
protected AbstractFSContract createContract(Configuration conf) {
- return new HBOSSContract(conf);
+ AbstractFSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(false, conf);
+ return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java
new file mode 100644
index 0000000..cbf5b65
--- /dev/null
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hbase.oss.TestUtils;
+
+
+/**
+ * For S3A-specific extension of AbstractContractOpenTest, we needed to override
+ * areZeroByteFilesEncrypted() method to always return true.
+ * TestHBOSSContractOpen remains to be run in the general case.
+ */
+public class TestHBOSSContractOpenS3A extends AbstractContractOpenTest {
+
+ @Override
+ protected Configuration createConfiguration() {
+ return super.createConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ AbstractFSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(true, conf);
+ return contract;
+ }
+
+ /**
+ * S3A always declares zero byte files as encrypted.
+ * @return true, always.
+ */
+ protected boolean areZeroByteFilesEncrypted() {
+ return true;
+ }
+}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
index f1047c4..2ee68af 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
@@ -18,14 +18,10 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.hbase.oss.EmbeddedS3;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
-import org.junit.Test;
/**
* There is an S3A-specific extension of AbstractContractRenameTest
@@ -42,13 +38,7 @@
@Override
protected AbstractFSContract createContract(Configuration conf) {
HBOSSContract contract = new HBOSSContract(conf);
- try {
- TestUtils.getFileSystem(conf);
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception configuring FS: " + e);
- }
- Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ TestUtils.runIfS3(false, conf);
return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
index a441dbf..cb991fb 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
@@ -18,17 +18,13 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.hbase.oss.EmbeddedS3;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
-import org.junit.Test;
/**
* There is an S3A-specific extension of AbstractContractRenameTest, and this
@@ -45,13 +41,7 @@
@Override
protected AbstractFSContract createContract(Configuration conf) {
HBOSSContract contract = new HBOSSContract(conf);
- try {
- TestUtils.getFileSystem(conf);
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception configuring FS: " + e);
- }
- Assume.assumeTrue(TestUtils.fsIs(TestUtils.S3A, conf));
+ TestUtils.runIfS3(true, conf);
return contract;
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
index 189693a..f14036c 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
@@ -18,12 +18,10 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.oss.TestUtils;
public class TestHBOSSContractRootDirectory extends AbstractContractRootDirectoryTest {
@@ -34,6 +32,8 @@
@Override
protected AbstractFSContract createContract(Configuration conf) {
- return new HBOSSContract(conf);
+ HBOSSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(false, conf);
+ return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java
new file mode 100644
index 0000000..7fa151a
--- /dev/null
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hbase.oss.TestUtils;
+import org.junit.Ignore;
+
+/**
+ * For S3A-specific extension of AbstractContractRootDirectoryTest, this overrides
+ * testRmNonEmptyRootDirNonRecursive(), marking it as ignored, since
+ * HADOOP-16380 introduced changes on S3AFileSystem for delete not throw any exception.
+ * TestHBOSSContractRootDirectory remains to be run in the general case.
+ */
+public class TestHBOSSContractRootDirectoryS3A extends AbstractContractRootDirectoryTest {
+
+ @Override
+ protected Configuration createConfiguration() {
+ return super.createConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ HBOSSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(true, conf);
+ return contract;
+ }
+
+ //HADOOP-16380 introduced changes on S3AFileSystem for delete not throw any exception
+ @Override
+ @Ignore("S3 always return false when non-recursively remove root dir")
+ public void testRmNonEmptyRootDirNonRecursive() throws Throwable {
+ }
+}
diff --git a/hbase-oss/src/test/resources/contract/s3a.xml b/hbase-oss/src/test/resources/contract/s3a.xml
index 72aac03..98389a1 100644
--- a/hbase-oss/src/test/resources/contract/s3a.xml
+++ b/hbase-oss/src/test/resources/contract/s3a.xml
@@ -125,4 +125,8 @@
<value>false</value>
</property>
+ <property>
+ <name>fs.s3a.change.detection.version.required</name>
+ <value>false</value>
+ </property>
</configuration>
diff --git a/pom.xml b/pom.xml
index 1453038..1e822ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
<curator.version>4.2.0</curator.version>
<guava.version>23.0</guava.version>
<hadoop2.version>2.9.2</hadoop2.version>
- <hadoop3.version>3.2.0</hadoop3.version>
+ <hadoop3.version>3.2.1</hadoop3.version>
<hbase1.version>1.4.10</hbase1.version>
<hbase2.version>2.1.4</hbase2.version>
<hbase-thirdparty.version>2.2.1</hbase-thirdparty.version>
