SLIDER-875 address gour's comments and fix accumulo funtest failures
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 56014d7..81d290a 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -38,6 +38,8 @@
import org.junit.Before
import org.junit.Test
+import java.util.regex.Pattern
+
import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser
import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath
@@ -90,6 +92,7 @@
def keys = tree.credentials.get(jks)
assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " +
"in credentials"
+ jks = jks.replaceAll(Pattern.quote('${CLUSTER_NAME}'), clusterName)
Path jksPath = ProviderUtils.unnestUri(new URI(jks))
if (clusterFS.exists(jksPath)) {
clusterFS.delete(jksPath, false)
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 372ec48..78e9b89 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -721,7 +721,8 @@
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername, clusterDirectory);
try {
- checkForCredentials(getConfig(), instanceDefinition.getAppConf());
+ checkForCredentials(getConfig(), instanceDefinition.getAppConf(),
+ clustername);
} catch (IOException e) {
sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
throw e;
@@ -903,7 +904,7 @@
}
protected static void checkForCredentials(Configuration conf,
- ConfTree tree) throws IOException {
+ ConfTree tree, String clusterName) throws IOException {
if (tree.credentials == null || tree.credentials.isEmpty()) {
log.info("No credentials requested");
return;
@@ -912,7 +913,9 @@
BufferedReader br = null;
try {
for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
- String provider = cred.getKey();
+ String provider = cred.getKey()
+ .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+ .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
List<String> aliases = cred.getValue();
if (aliases == null || aliases.isEmpty()) {
continue;
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3921324..2c9be09 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -72,9 +72,15 @@
/**
* Key for component type. This MUST NOT be set in app_config/global {@value}
*/
- String COMPONENT_TYPE = "site.global.component_type";
- String EXTERNAL_COMPONENT = "external";
+ String COMPONENT_TYPE_KEY = "site.global.component_type";
+ /**
+ * A component type for an external app that has been predefined using the
+ * slider build command
+ */
+ String COMPONENT_TYPE_EXTERNAL_APP = "external_app";
String COMPONENT_SEPARATOR = "-";
+ String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX",
+ "site.fs.", "site.dfs."};
/**
* Key for application version. This must be set in app_config/global {@value}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b798d4d..aad8b45 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -492,11 +492,11 @@
FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
//list all paths in the src.
if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source dir not found " + srcFile);
+ throw new FileNotFoundException("Source file not found " + srcFile);
}
if (!srcFS.isFile(srcFile)) {
throw new FileNotFoundException(
- "Source dir not a file " + srcFile);
+ "Source file not a file " + srcFile);
}
FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
if (destFS.exists(destFile)) {
@@ -962,7 +962,7 @@
* @param prefixes prefixes to ignore
* @return 'first' merged with the second
*/
- public static Map<String, String> mergeMapsIgnorePrefixes(
+ public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes(
Map<String, String> first, Map<String, String> second,
String... prefixes) {
Preconditions.checkArgument(first != null, "Null 'first' value");
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 6a7975e..8155214 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -72,8 +72,8 @@
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
-import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE;
-import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT;
+import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_EXTERNAL_APP;
+import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_KEY;
import static org.apache.slider.common.tools.SliderUtils.isClusternameValid;
/**
@@ -260,10 +260,11 @@
}
- private void getExternalComponents(ConfTreeOperations ops,
- Set<String> externalComponents) throws BadConfigException {
- if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) {
- throw new BadConfigException(COMPONENT_TYPE + " must be " +
+ private Set<String> getExternalComponents(ConfTreeOperations ops)
+ throws BadConfigException {
+ Set<String> externalComponents = new HashSet<>();
+ if (ops.getGlobalOptions().containsKey(COMPONENT_TYPE_KEY)) {
+ throw new BadConfigException(COMPONENT_TYPE_KEY + " must be " +
"specified per-component, not in global");
}
@@ -273,15 +274,13 @@
continue;
}
Map<String, String> options = entry.getValue();
- if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
+ if (COMPONENT_TYPE_EXTERNAL_APP.equals(options.get(COMPONENT_TYPE_KEY))) {
externalComponents.add(entry.getKey());
}
}
+ return externalComponents;
}
- private static String[] PREFIXES_TO_SKIP = {"zookeeper.",
- "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."};
-
private void mergeExternalComponent(ConfTreeOperations ops,
ConfTreeOperations externalOps, String externalComponent,
Integer priority) throws BadConfigException {
@@ -297,11 +296,12 @@
if (priority == null) {
SliderUtils.mergeMaps(subComponentOps,
ops.getComponent(externalComponent).options);
- subComponentOps.remove(COMPONENT_TYPE);
+ subComponentOps.remove(COMPONENT_TYPE_KEY);
}
- SliderUtils.mergeMapsIgnorePrefixes(subComponentOps,
- externalOps.getComponent(subComponent), PREFIXES_TO_SKIP);
+ SliderUtils.mergeMapsIgnoreDuplicateKeysAndPrefixes(subComponentOps,
+ externalOps.getComponent(subComponent),
+ SliderKeys.COMPONENT_KEYS_TO_SKIP);
// add prefix to existing prefix
String existingPrefix = subComponentOps.get(ROLE_PREFIX);
@@ -348,8 +348,7 @@
}
}
- Set<String> externalComponents = new HashSet<>();
- getExternalComponents(appConf, externalComponents);
+ Set<String> externalComponents = getExternalComponents(appConf);
if (!externalComponents.isEmpty()) {
log.info("Found external components {}", externalComponents);
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8c0a2e4..1567fc9 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -74,8 +74,6 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
-import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName;
-
/** This class implements the client-side aspects of the agent deployer */
public class AgentClientProvider extends AbstractClientProvider
implements AgentKeys, SliderKeys {
@@ -184,7 +182,7 @@
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name,
+ AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
if (componentDef == null) {
throw new BadConfigException(
@@ -215,9 +213,8 @@
// fileSystem may be null for tests
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name,
+ AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
- // already checked it wasn't null
// ensure that intance count is 0 for client components
if ("CLIENT".equals(componentDef.getCategory())) {
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 565d73d..8341af4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -99,7 +99,7 @@
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
- String DEFAULT_METAINFO_MAP_KEY = "DEFAULT";
+ String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
index 853310d..b026b9d 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
@@ -6,7 +6,7 @@
},
"components": {
"test_sleep": {
- "site.global.component_type": "external"
+ "site.global.component_type": "external_app"
}
}
}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
index f67f83a..a9972cf 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
@@ -6,7 +6,7 @@
},
"components": {
"test-external-component": {
- "site.global.component_type": "external"
+ "site.global.component_type": "external_app"
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
index 8dd693e..13f5f3b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
@@ -67,7 +67,7 @@
SliderActions.ACTION_BUILD,
parent1,
[(echo): 1],
- [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"],
true,
false,
@@ -97,8 +97,8 @@
SliderActions.ACTION_BUILD,
parent2,
[(echo): 1],
- [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
- ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
+ ARG_COMP_OPT, parent1, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"],
true,
false,
