Prevent users from setting -Xmx parameter directly for spawned yarn processes
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
index 222ac5a..9789de5 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
@@ -5,7 +5,9 @@
import org.apache.s4.core.Main.InlineConfigParameterConverter;
+import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
public class CommonS4YarnArgs {
@@ -53,8 +55,18 @@
List<String> extraNamedParameters = new ArrayList<String>();
// TODO parse JVM parameters that include commas
- @Parameter(names = S4CLIYarnArgs.S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory is usually configured through "
- + S4_NODE_MEMORY, required = false)
+ @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory parameter -Xmx must be configured through "
+ + S4_NODE_MEMORY, required = false, validateWith = NodeJVMParametersValidator.class)
List<String> extraS4NodeJVMParams = new ArrayList<String>();
+ public static class NodeJVMParametersValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) throws ParameterException {
+ if (value.matches(".*-Xmx\\d+.*")) {
+ throw new ParameterException("-Xmx JVM parameter cannot be specified here. You must use the "
+ + S4_NODE_MEMORY + " parameter instead.");
+ }
+ }
+ }
}
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
index d7dc788..f1efb35 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
@@ -99,34 +99,34 @@
// For status update for clients - yet to be implemented
// Hostname of the container
- private String appMasterHostname = "";
+ private final String appMasterHostname = "";
// Port on which the app master listens for status update requests from clients
- private int appMasterRpcPort = 0;
+ private final int appMasterRpcPort = 0;
// Tracking url to which app master publishes info for clients to monitor
- private String appMasterTrackingUrl = "";
+ private final String appMasterTrackingUrl = "";
// Incremental counter for rpc calls to the RM
- private AtomicInteger rmRequestID = new AtomicInteger();
+ private final AtomicInteger rmRequestID = new AtomicInteger();
// Simple flag to denote whether all works is done
private boolean appDone = false;
// Counter for completed containers ( complete denotes successful or failed )
- private AtomicInteger numCompletedContainers = new AtomicInteger();
+ private final AtomicInteger numCompletedContainers = new AtomicInteger();
// Allocated container count so that we know how many containers has the RM
// allocated to us
- private AtomicInteger numAllocatedContainers = new AtomicInteger();
+ private final AtomicInteger numAllocatedContainers = new AtomicInteger();
// Count of failed containers
- private AtomicInteger numFailedContainers = new AtomicInteger();
+ private final AtomicInteger numFailedContainers = new AtomicInteger();
// Count of containers already requested from the RM
// Needed as once requested, we should not request for containers again and again.
// Only request for more if the original requirement changes.
- private AtomicInteger numRequestedContainers = new AtomicInteger();
+ private final AtomicInteger numRequestedContainers = new AtomicInteger();
// Containers to be released
- private CopyOnWriteArrayList<ContainerId> releasedContainers = new CopyOnWriteArrayList<ContainerId>();
+ private final CopyOnWriteArrayList<ContainerId> releasedContainers = new CopyOnWriteArrayList<ContainerId>();
// Launch threads
- private List<Thread> launchThreads = new ArrayList<Thread>();
+ private final List<Thread> launchThreads = new ArrayList<Thread>();
private int containerMemory;
@@ -144,8 +144,6 @@
Tools.parseArgs(yarnArgs, args);
- Thread.sleep(10000);
-
S4ApplicationMaster appMaster = new S4ApplicationMaster();
logger.info("Initializing ApplicationMaster with args " + Arrays.toString(args));
appMaster.init();
@@ -499,9 +497,6 @@
S4YarnClient.addListElementsToCommandLineBuffer(vargs, null, " ", yarnArgs.extraS4NodeJVMParams);
- if (!yarnArgs.extraS4NodeJVMParams.isEmpty()) {
- vargs.add(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS);
- }
// TODO add memory parameter
// vargs.add("-Xdebug");
// vargs.add("-Xrunjdwp:transport=dt_socket,address=8889,server=y");
diff --git a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
index a4b5a12..77c120c 100644
--- a/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
+++ b/subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
@@ -120,7 +120,7 @@
private static Logger logger = LoggerFactory.getLogger(S4YarnClient.class);
// Configuration
- private Configuration conf;
+ private final Configuration conf;
S4CLIYarnArgs yarnArgs;
@@ -325,8 +325,10 @@
vargs.add(CommonS4YarnArgs.NB_S4_NODES + " " + String.valueOf(yarnArgs.numContainers));
vargs.add(CommonS4YarnArgs.PRIORITY + " " + String.valueOf(yarnArgs.priority));
- addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, ",", yarnArgs.extraS4NodeJVMParams);
- addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.EXTRA_MODULES_CLASSES, ",", yarnArgs.extraModulesClasses);
+ addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, ",",
+ yarnArgs.extraS4NodeJVMParams);
+ addListElementsToCommandLineBuffer(vargs, CommonS4YarnArgs.EXTRA_MODULES_CLASSES, ",",
+ yarnArgs.extraModulesClasses);
addListElementsToCommandLineBuffer(vargs, "-namedStringParameters", ",", yarnArgs.extraNamedParameters);
vargs.add("-c " + String.valueOf(yarnArgs.cluster));
