HDDS-4985. [SCM HA Security] When Ratis enable, SCM secure cluster is not working. (#2052)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
index 6dc0f15..62951d5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
@@ -90,7 +90,8 @@
long startTime = Time.monotonicNowNanos();
Preconditions.checkNotNull(ratisHandler);
final SCMRatisResponse response = ratisHandler.submitRequest(
- SCMRatisRequest.of(requestType, method.getName(), args));
+ SCMRatisRequest.of(requestType, method.getName(),
+ method.getParameterTypes(), args));
LOG.info("Invoking method {} on target {}, cost {}us",
method, ratisHandler, (Time.monotonicNowNanos() - startTime) / 1000.0);
if (response.isSuccess()) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java
index 4277bb0..da8fadf 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
+import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.scm.ha.io.CodecFactory;
@@ -39,18 +40,22 @@
private final RequestType type;
private final String operation;
private final Object[] arguments;
+ private final Class<?>[] parameterTypes;
private SCMRatisRequest(final RequestType type, final String operation,
- final Object... arguments) {
+ final Class<?>[] parameterTypes, final Object... arguments) {
this.type = type;
this.operation = operation;
+ this.parameterTypes = parameterTypes;
this.arguments = arguments;
}
public static SCMRatisRequest of(final RequestType type,
- final String operation,
- final Object... arguments) {
- return new SCMRatisRequest(type, operation, arguments);
+ final String operation,
+ final Class<?>[] parameterTypes,
+ final Object... arguments) {
+ Preconditions.checkState(parameterTypes.length == arguments.length);
+ return new SCMRatisRequest(type, operation, parameterTypes, arguments);
}
/**
@@ -74,6 +79,9 @@
return arguments.clone();
}
+ public Class<?>[] getParameterTypes() {
+ return parameterTypes.clone();
+ }
/**
* Encodes the request into Ratis Message.
*/
@@ -86,9 +94,14 @@
methodBuilder.setName(operation);
final List<MethodArgument> args = new ArrayList<>();
+
+ int paramCounter = 0;
for (Object argument : arguments) {
final MethodArgument.Builder argBuilder = MethodArgument.newBuilder();
- argBuilder.setType(argument.getClass().getName());
+ // Set actual method parameter type, not actual argument type.
+ // This is done to avoid MethodNotFoundException in case if argument is
+ // subclass type, where as method is defined with super class type.
+ argBuilder.setType(parameterTypes[paramCounter++].getName());
argBuilder.setValue(CodecFactory.getCodec(argument.getClass())
.serialize(argument));
args.add(argBuilder.build());
@@ -109,9 +122,12 @@
SCMRatisRequestProto.parseFrom(message.getContent().toByteArray());
final Method method = requestProto.getMethod();
List<Object> args = new ArrayList<>();
+ Class<?>[] parameterTypes = new Class[method.getArgsCount()];
+ int paramCounter = 0;
for (MethodArgument argument : method.getArgsList()) {
try {
final Class<?> clazz = ReflectionUtil.getClass(argument.getType());
+ parameterTypes[paramCounter++] = clazz;
args.add(CodecFactory.getCodec(clazz)
.deserialize(clazz, argument.getValue()));
} catch (ClassNotFoundException ex) {
@@ -120,7 +136,7 @@
}
}
return new SCMRatisRequest(requestProto.getType(),
- method.getName(), args.toArray());
+ method.getName(), parameterTypes, args.toArray());
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index a1047b4..90ace48 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -19,9 +19,7 @@
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
import java.util.EnumMap;
-import java.util.List;
import java.util.Map;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
@@ -152,12 +150,8 @@
request.getType());
}
- final List<Class<?>> argumentTypes = new ArrayList<>();
- for(Object args : request.getArguments()) {
- argumentTypes.add(args.getClass());
- }
final Object result = handler.getClass().getMethod(
- request.getOperation(), argumentTypes.toArray(new Class<?>[0]))
+ request.getOperation(), request.getParameterTypes())
.invoke(handler, request.getArguments());
return SCMRatisResponse.encode(result);
} catch (NoSuchMethodException | SecurityException ex) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java
index 5295e0f..f5913aa 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java
@@ -40,7 +40,8 @@
PipelineID pipelineID = PipelineID.randomId();
Object[] args = new Object[] {pipelineID.getProtobuf()};
String operation = "test";
- SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation, args);
+ SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation,
+ new Class[]{pipelineID.getProtobuf().getClass()}, args);
Assert.assertEquals(operation,
SCMRatisRequest.decode(request.encode()).getOperation());
Assert.assertEquals(args[0],
@@ -52,7 +53,8 @@
PipelineID pipelineID = PipelineID.randomId();
// Non proto args
Object[] args = new Object[] {pipelineID};
- SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, "test", args);
+ SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, "test",
+ new Class[]{pipelineID.getClass()}, args);
// Should throw exception there.
request.encode();
}
@@ -73,7 +75,8 @@
pids.add(PipelineID.randomId().getProtobuf());
Object[] args = new Object[] {pids};
String operation = "test";
- SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation, args);
+ SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation,
+ new Class[]{pids.getClass()}, args);
Assert.assertEquals(operation,
SCMRatisRequest.decode(request.encode()).getOperation());
Assert.assertEquals(args[0],
@@ -84,7 +87,8 @@
public void testEncodeAndDecodeOfLong() throws Exception {
final Long value = 10L;
String operation = "test";
- SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation, value);
+ SCMRatisRequest request = SCMRatisRequest.of(PIPELINE, operation,
+ new Class[]{value.getClass()}, value);
Assert.assertEquals(operation,
SCMRatisRequest.decode(request.encode()).getOperation());
Assert.assertEquals(value,
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 31011fa..a4c8e80d 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -104,5 +104,6 @@
KERBEROS_KEYTABS: scm HTTP testuser testuser2
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}"
+ OZONE-SITE.XML_ozone.scm.ratis.enable: "${OZONE_SCM_RATIS_ENABLE:-false}"
OZONE_OPTS:
command: ["/opt/hadoop/bin/ozone","scm"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
index 9150b24..c482396 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
@@ -27,39 +27,42 @@
: ${OZONE_BUCKET_KEY_NAME:=key1}
-start_docker_env
+for enable in true false; do
-execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
+ start_docker_env 3 "${enable}"
-execute_robot_test scm kinit.robot
+ execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
-execute_robot_test scm basic
+ execute_robot_test scm kinit.robot
-execute_robot_test scm security
+ execute_robot_test scm basic
-for scheme in ofs o3fs; do
- for bucket in link bucket; do
- execute_robot_test scm -v SCHEME:${scheme} -v BUCKET_TYPE:${bucket} -N ozonefs-${scheme}-${bucket} ozonefs/ozonefs.robot
+ execute_robot_test scm security
+
+ for scheme in ofs o3fs; do
+ for bucket in link bucket; do
+ execute_robot_test scm -v SCHEME:${scheme} -v BUCKET_TYPE:${bucket} -N ozonefs-${scheme}-${bucket} ozonefs/ozonefs.robot
+ done
done
-done
-for bucket in link generated; do
- execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} s3
-done
+ for bucket in link generated; do
+ execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} s3
+ done
-#expects 4 pipelines, should be run before
-#admincli which creates STANDALONE pipeline
-execute_robot_test scm recon
+ #expects 4 pipelines, should be run before
+ #admincli which creates STANDALONE pipeline
+ execute_robot_test scm recon
-execute_robot_test scm admincli
-execute_robot_test scm spnego
+ execute_robot_test scm admincli
+ execute_robot_test scm spnego
-# test replication
-docker-compose up -d --scale datanode=2
-execute_robot_test scm -v container:1 -v count:2 replication/wait.robot
-docker-compose up -d --scale datanode=3
-execute_robot_test scm -v container:1 -v count:3 replication/wait.robot
+ # test replication
+ docker-compose up -d --scale datanode=2
+ execute_robot_test scm -v container:1 -v count:2 replication/wait.robot
+ docker-compose up -d --scale datanode=3
+ execute_robot_test scm -v container:1 -v count:3 replication/wait.robot
-stop_docker_env
+ stop_docker_env
-generate_report
+ generate_report
+done
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index acc9fb3..1a13219 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -138,6 +138,8 @@
create_results_dir
export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
+
+ export OZONE_SCM_RATIS_ENABLE=${2:-false}
docker-compose --no-ansi down
if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \
&& wait_for_safemode_exit \
