Merge branch 'master' into HDDS-3698-nonrolling-upgrade
* master:
HDDS-3292. Support Hadoop 3.3 (#2356)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 772a53a..ca671b7 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -34,6 +34,11 @@
<artifactId>hdds-hadoop-dependency-client</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-hadoop-dependency-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
</dependency>
diff --git a/hadoop-hdds/hadoop-dependency-test/pom.xml b/hadoop-hdds/hadoop-dependency-test/pom.xml
index 34cf949..799fc54 100644
--- a/hadoop-hdds/hadoop-dependency-test/pom.xml
+++ b/hadoop-hdds/hadoop-dependency-test/pom.xml
@@ -54,5 +54,14 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 1f6f7e9..89edde4 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -118,8 +118,7 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index a72ab61..97e61ec 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -33,6 +33,7 @@
.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -648,7 +649,8 @@
new ContainerReportFromDatanode(datanodeOne, containerReport);
reportHandler.onMessage(containerReportFromDatanode, publisher);
- Mockito.verify(publisher, Mockito.times(1));
+ Mockito.verify(publisher, Mockito.times(1))
+ .fireEvent(Mockito.any(), Mockito.any(CommandForDatanode.class));
Assert.assertEquals(0, containerManager.getContainerReplicas(
containerOne.containerID()).size());
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index 7d3d457..3bdc6ab 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -47,7 +47,6 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import java.io.IOException;
@@ -61,7 +60,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -130,8 +128,8 @@
containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
Mockito.when(containerPlacementPolicy.chooseDatanodes(
- Mockito.anyListOf(DatanodeDetails.class),
- Mockito.anyListOf(DatanodeDetails.class),
+ Mockito.any(),
+ Mockito.any(),
Mockito.anyInt(), Mockito.anyLong()))
.thenAnswer(invocation -> {
int count = (int) invocation.getArguments()[2];
@@ -141,7 +139,7 @@
});
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.anyListOf(DatanodeDetails.class),
+ Mockito.any(),
Mockito.anyInt()
)).thenAnswer(invocation -> {
return new ContainerPlacementStatusDefault(2, 2, 3);
@@ -730,7 +728,7 @@
// test where there are 3 replicas. When there are 2 or 4 replicas
// the status returned will be healthy.
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(invocation -> {
return new ContainerPlacementStatusDefault(1, 2, 3);
@@ -799,7 +797,7 @@
// Ensure a mis-replicated status is returned for any containers in this
// test where there are exactly 3 replicas checked.
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
@@ -843,7 +841,7 @@
containerStateManager.updateContainerReplica(id, replicaFour);
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
@@ -883,8 +881,7 @@
containerStateManager.updateContainerReplica(id, replicaFive);
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new FunctionMatcher(list ->
- list != null && ((List) list).size() <= 4)),
+ Mockito.argThat(list -> list != null && list.size() <= 4),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
@@ -1188,31 +1185,4 @@
}
}
- static class ListOfNElements extends ArgumentMatcher<List> {
-
- private int expected;
-
- ListOfNElements(int expected) {
- this.expected = expected;
- }
-
- @Override
- public boolean matches(Object argument) {
- return ((List)argument).size() == expected;
- }
- }
-
- static class FunctionMatcher extends ArgumentMatcher<List> {
-
- private Function<Object, Boolean> function;
-
- FunctionMatcher(Function<Object, Boolean> function) {
- this.function = function;
- }
-
- @Override
- public boolean matches(Object argument) {
- return function.apply(argument);
- }
- }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index 1ce76e6..0799835 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -78,12 +78,10 @@
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
- <scope>test</scope>
- </dependency>
-
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
index cb7cf8a..70c74a9 100644
--- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
@@ -68,9 +68,7 @@
public void testDataNodeOperationalStateAndHealthIncludedInOutput()
throws Exception {
ScmClient scmClient = mock(ScmClient.class);
- Mockito.when(scmClient.queryNode(any(HddsProtos.NodeOperationalState.class),
- any(HddsProtos.NodeState.class), any(HddsProtos.QueryScope.class),
- Mockito.anyString()))
+ Mockito.when(scmClient.queryNode(any(), any(), any(), any()))
.thenAnswer(invocation -> getNodeDetails());
Mockito.when(scmClient.listPipelines())
.thenReturn(new ArrayList<>());
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt
index 11a3b74..8233966 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -57,6 +57,8 @@
share/ozone/lib/hadoop-common.jar
share/ozone/lib/hadoop-hdfs-client.jar
share/ozone/lib/hadoop-hdfs.jar
+share/ozone/lib/hadoop-shaded-guava.jar
+share/ozone/lib/hadoop-shaded-protobuf_3_7.jar
share/ozone/lib/hdds-client.jar
share/ozone/lib/hdds-common.jar
share/ozone/lib/hdds-config.jar
@@ -97,7 +99,6 @@
share/ozone/lib/jakarta.ws.rs-api.jar
share/ozone/lib/jakarta.xml.bind-api.jar
share/ozone/lib/javassist-GA.jar
-share/ozone/lib/javax.activation-api.jar
share/ozone/lib/javax.annotation-api.jar
share/ozone/lib/javax.el-api.jar
share/ozone/lib/javax.inject.jar
@@ -223,6 +224,7 @@
share/ozone/lib/slf4j-api.jar
share/ozone/lib/slf4j-log4j12.jar
share/ozone/lib/snakeyaml.jar
+share/ozone/lib/snappy-java.jar
share/ozone/lib/spring-beans.RELEASE.jar
share/ozone/lib/spring-core.RELEASE.jar
share/ozone/lib/spring-jcl.RELEASE.jar
diff --git a/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
index 4a92bd7..c500d57 100644
--- a/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
@@ -41,7 +41,7 @@
[arguments] ${url}
Run Keyword if '${SECURITY_ENABLED}' == 'true' Execute kdestroy
${result} = Execute curl --negotiate -u : -v -s -I ${url}
- Should contain ${result} 401 Unauthorized
+ Should contain ${result} 401 Authentication required
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab
${result} = Execute curl --negotiate -u : -v -s -I ${url}
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index 97e72ae..51328e4 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -135,8 +135,7 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 4ffa77b..108ccf8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -827,9 +827,8 @@
}
}
- @SuppressWarnings("visibilitymodifier")
- static class CloseContainerCommandMatcher
- extends ArgumentMatcher<CommandForDatanode> {
+ private static class CloseContainerCommandMatcher
+ implements ArgumentMatcher<CommandForDatanode> {
private final CommandForDatanode cmd;
private final UUID uuid;
@@ -840,8 +839,7 @@
}
@Override
- public boolean matches(Object argument) {
- CommandForDatanode cmdRight = (CommandForDatanode) argument;
+ public boolean matches(CommandForDatanode cmdRight) {
CloseContainerCommand left = (CloseContainerCommand) cmd.getCommand();
CloseContainerCommand right =
(CloseContainerCommand) cmdRight.getCommand();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 162ed8a..155738b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -139,7 +139,6 @@
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(numOfOMs)
.build();
- conf.setQuietMode(false);
cluster.waitForClusterToBeReady();
}
diff --git a/pom.xml b/pom.xml
index 2cdca42..0097604 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,7 @@
</organization>
<properties>
- <hadoop.version>3.2.2</hadoop.version>
+ <hadoop.version>3.3.1</hadoop.version>
<!-- version for hdds/ozone components -->
<hdds.version>${ozone.version}</hdds.version>
@@ -161,7 +161,6 @@
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
<okhttp.version>2.7.5</okhttp.version>
- <mockito1-hadoop.version>1.8.5</mockito1-hadoop.version>
<mockito1-powermock.version>1.10.19</mockito1-powermock.version>
<mockito2.version>2.28.2</mockito2.version>
<hamcrest.version>1.3</hamcrest.version>
@@ -1564,7 +1563,7 @@
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
- <version>3.8.0</version>
+ <version>3.12.2</version>
<scope>test</scope>
</dependency>
<dependency>
