Merge branch 'feature/SLIDER-565-test-JVM-env-passdown' into develop
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 706a34b..06efadf 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -22,7 +22,6 @@
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
@@ -162,6 +161,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@@ -2902,9 +2902,21 @@
}
println(builder.toString());
+ // Java properties
+ builder = new StringBuilder("JVM Properties\n");
+ Map<String, String> props =
+ SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
+ for (Entry<String, String> entry : props.entrySet()) {
+ builder.append(entry.getKey()).append("=")
+ .append(entry.getValue()).append("\n");
+ }
+
+ println(builder.toString());
+
// then the config
println("Slider client configuration:\n"
- + ConfigHelper.dumpConfigToString(config));
+ + ConfigHelper.dumpConfigToString(config));
+
}
SliderUtils.validateSliderClientEnvironment(log);
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 5f16e56..89cc263 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -190,4 +190,15 @@
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
+ /**
+ * Allowed port range
+ */
+ String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range";
+
+ /**
+ * env var for custom JVM options.
+ */
+ String SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS";
+
+ String SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 43dfbd0..0f622c9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -702,7 +702,29 @@
}
return map;
}
-
+
+ /**
+ * Take a map and produce a sorted equivalent
+ * @param source source map
+ * @return a map whose iterator returns the string-sorted ordering of entries
+ */
+ public static Map<String, String> sortedMap(Map<String, String> source) {
+ Map<String, String> out = new TreeMap<String, String>(source);
+ return out;
+ }
+
+ /**
+ * Convert a properties instance to a string map.
+ * @param properties source property object
+ * @return a string map
+ */
+ public static Map<String, String> toMap(Properties properties) {
+ Map<String, String> out = new HashMap<String, String>(properties.size());
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ out.put(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return out;
+ }
/**
* Merge in one map to another -all entries in the second map are
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 8af51b4..b44ae07 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -24,7 +24,6 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.common.tools.SliderUtils
import org.apache.tools.zip.ZipEntry
import org.apache.tools.zip.ZipOutputStream
import org.junit.Before
@@ -125,66 +124,15 @@
}
public static String findLineEntry(SliderShell shell, String[] locaters) {
- int index = 0;
- def output = shell.out
- output += shell.err
- for (String str in output) {
- if (str.contains("\"" + locaters[index] + "\"")) {
- if (locaters.size() == index + 1) {
- return str;
- } else {
- index++;
- }
- }
- }
-
- return null;
+ return shell.findLineEntry(locaters)
}
public static boolean containsString(SliderShell shell, String lookThisUp, int n = 1) {
- int count = 0
- def output = shell.out
- output += shell.err
- for (String str in output) {
- int subCount = countString(str, lookThisUp)
- count = count + subCount
- if (count == n) {
- return true;
- }
- }
-
- return false;
- }
-
- public static int countString(String str, String search) {
- int count = 0
- if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) {
- return count
- }
-
- int index = str.indexOf(search, 0)
- while (index > 0) {
- index = str.indexOf(search, index + 1)
- ++count
- }
- return count
+ return shell.outputContains(lookThisUp, n)
}
public static String findLineEntryValue(SliderShell shell, String[] locaters) {
- String line = findLineEntry(shell, locaters);
-
- if (line != null) {
- log.info("Parsing {} for value.", line)
- int dividerIndex = line.indexOf(":");
- if (dividerIndex > 0) {
- String value = line.substring(dividerIndex + 1).trim()
- if (value.endsWith(",")) {
- value = value.subSequence(0, value.length() - 1)
- }
- return value;
- }
- }
- return null;
+ return shell.findLineEntryValue(locaters)
}
public static void addDir(File dirObj, ZipOutputStream zipFile, String prefix) {
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 24bcc0e..21ec1f2 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -480,6 +480,26 @@
}
/**
+ * Assert that the stdout/stderr streams of the shell contain the string
+ * to look for.
+ * If the assertion does not hold, the output is logged before
+ * the assertion is thrown
+ * @param shell
+ * @param lookThisUp
+ * @param n number of times (default = 1)
+ */
+ public static void assertOutputContains(
+ SliderShell shell,
+ String lookThisUp,
+ int n = 1) {
+ if (!shell.outputContains(lookThisUp)) {
+ log.error("Missing $lookThisUp from:")
+ shell.dumpOutput()
+ assert shell.outputContains(lookThisUp)
+ }
+ }
+
+ /**
* Create a connection to the cluster by execing the status command
*
* @param clustername
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 86595dc..e278981 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -40,6 +40,8 @@
public static File scriptFile;
+ public File shellScript;
+
public static final List<String> slider_classpath_extra = []
/**
@@ -57,6 +59,7 @@
super(org.apache.hadoop.util.Shell.WINDOWS ? CMD : BASH)
assert confDir != null;
assert scriptFile != null;
+ shellScript = scriptFile;
command = scriptFile.absolutePath + " " + commands.join(" ")
}
@@ -280,4 +283,67 @@
return this
}
+
+ public String findLineEntry(String[] locaters) {
+ int index = 0;
+ def output = out +"\n"+ err
+ for (String str in output) {
+ if (str.contains("\"" + locaters[index] + "\"")) {
+ if (locaters.size() == index + 1) {
+ return str;
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public boolean outputContains(
+ String lookThisUp,
+ int n = 1) {
+ int count = 0
+ def output = out + "\n" + err
+ for (String str in output) {
+ int subCount = countString(str, lookThisUp)
+ count = count + subCount
+ if (count == n) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static int countString(String str, String search) {
+ int count = 0
+ if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) {
+ return count
+ }
+
+ int index = str.indexOf(search, 0)
+ while (index >= 0) {
+ ++count
+ index = str.indexOf(search, index + 1)
+ }
+ return count
+ }
+
+ public findLineEntryValue(String[] locaters) {
+ String line = findLineEntry(locaters);
+
+ if (line != null) {
+ log.info("Parsing {} for value.", line)
+ int dividerIndex = line.indexOf(":");
+ if (dividerIndex > 0) {
+ String value = line.substring(dividerIndex + 1).trim()
+ if (value.endsWith(",")) {
+ value = value.subSequence(0, value.length() - 1)
+ }
+ return value;
+ }
+ }
+ return null;
+ }
+
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
new file mode 100644
index 0000000..7680e49
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class CommandEnvironmentIT extends CommandTestBase {
+
+ File originalScript
+
+ @Before
+ public void cacheScript() {
+ originalScript = SliderShell.scriptFile
+ }
+
+ @After
+ public void restoreScript() {
+ SliderShell.scriptFile = originalScript
+ }
+
+ @Test
+ public void testJVMOptionPassdownBash() throws Throwable {
+ assume(!SliderShell.windows, "skip bash test on windows")
+ SliderShell.scriptFile = SLIDER_SCRIPT;
+ execScriptTest()
+ }
+
+ @Test
+ public void testJVMOptionPassdownPython() throws Throwable {
+ SliderShell.scriptFile = SLIDER_SCRIPT_PYTHON;
+ execScriptTest()
+ }
+
+ public void execScriptTest() {
+ SliderShell shell = new SliderShell([
+ SliderActions.ACTION_DIAGNOSTICS,
+ Arguments.ARG_CLIENT,
+ Arguments.ARG_VERBOSE
+ ])
+ def name = "testpropertySetInFuntest"
+ def val = "TestPropertyValue"
+ shell.setEnv(SliderKeys.SLIDER_JVM_OPTS, "-D" + name + "=" + val)
+ shell.execute(0)
+ assertOutputContains(shell, name, 2)
+ assertOutputContains(shell, val, 2)
+ assertOutputContains(shell, SliderKeys.PROPERTY_LIB_DIR)
+ assertOutputContains(shell, SliderKeys.PROPERTY_CONF_DIR)
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
index bc19367..da58dd2 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
@@ -41,7 +41,6 @@
println(shell.stdoutHistory)
println()
println(shell.stdErrHistory)
-
}
}
