SLIDER-711 components/ and components/$component
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java
new file mode 100644
index 0000000..8eac3db
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+/**
+ * Serializable version of component data
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedComponentInformation {
+
+ public String name;
+ public int priority;
+ public int desired, actual, requested, releasing;
+ public int failed, started, startFailed, completed, totalRequested;
+ public String failureMessage;
+ public int placementPolicy;
+ public List<String> containers;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 7701cfe..4333a09 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -131,7 +131,7 @@
new RoleLaunchService.RoleLauncher(container,
role.getProviderRole(),
clusterSpec,
- clusterSpec.getResourceOperations() .getOrAddComponent(roleName),
+ clusterSpec.getResourceOperations().getOrAddComponent(roleName),
clusterSpec.getAppConfOperations().getOrAddComponent(roleName));
execute(launcher);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index e9d0371..9ae20a5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -839,8 +839,7 @@
}
return rs;
}
-
-
+
public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 3c860d6..74a1c9c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.SerializedComponentInformation;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
@@ -37,8 +38,7 @@
private final String name;
/**
- * Role key in the container details stored in the AM,
- * currently mapped to priority
+ * Role priority
*/
private final int key;
@@ -47,23 +47,6 @@
private int desired, actual, requested, releasing;
private int failed, started, startFailed, completed, totalRequested;
- /**
- * value to use when specifiying "no limit" for instances: {@value}
- */
- public static final int UNLIMITED_INSTANCES = 1;
-
- /**
- * minimum number of instances of a role permitted in a valid
- * configuration. Default: 0.
- */
- private int minimum = 0;
-
- /**
- * maximum number of instances of a role permitted in a valid
- * configuration. Default: unlimited.
- */
- private int maximum = UNLIMITED_INSTANCES;
-
private String failureMessage = "";
public RoleStatus(ProviderRole providerRole) {
@@ -260,8 +243,6 @@
return "RoleStatus{" +
"name='" + name + '\'' +
", key=" + key +
- ", minimum=" + minimum +
- ", maximum=" + maximum +
", desired=" + desired +
", actual=" + actual +
", requested=" + requested +
@@ -304,6 +285,22 @@
return stats;
}
+ public SerializedComponentInformation serialize() {
+ SerializedComponentInformation info = new SerializedComponentInformation();
+ info.name = name;
+ info.priority = getPriority();
+ info.desired = desired;
+ info.actual = actual;
+ info.requested = requested;
+ info.releasing = releasing;
+ info.failed = failed;
+ info.startFailed = startFailed;
+ info.requested = requested;
+ info.placementPolicy = getPlacementPolicy();
+ info.failureMessage = failureMessage;
+ return info;
+ }
+
/**
* Compare two role status entries by name
*/
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index ede46f0..dc07c10 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -30,6 +30,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
+import java.io.FileNotFoundException;
import java.net.URI;
import java.net.URL;
@@ -79,6 +80,8 @@
} catch (WebApplicationException e) {
// rethrow direct
throw e;
+ } catch (FileNotFoundException e) {
+ return new NotFoundException("Not found: " + path);
} catch (PathNotFoundException e) {
return new NotFoundException("Not found: " + path);
} catch (AuthenticationFailedException e) {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 28c0fab..011ec3a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -130,8 +130,10 @@
public static final String APPLICATION_WADL = "/application.wadl";
+ public static final String LIVE = "/live";
public static final String LIVE_RESOURCES = "/live/resources";
public static final String LIVE_CONTAINERS = "/live/containers";
+ public static final String LIVE_COMPONENTS = "/live/components";
public static final String MODEL_DESIRED = "/model/desired";
public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf";
public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources";
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index e4be96f..9419765 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -19,15 +19,23 @@
package org.apache.slider.server.appmaster.web.rest.application;
import com.google.common.collect.Lists;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.api.types.SerializedComponentInformation;
import org.apache.slider.api.types.SerializedContainerInformation;
import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent;
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,28 +43,43 @@
import javax.inject.Singleton;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Singleton
+@SuppressWarnings("unchecked")
public class ApplicationResource extends AbstractSliderResource {
private static final Logger log =
LoggerFactory.getLogger(ApplicationResource.class);
public static final int LIFESPAN = 1000;
+ public static final List<String> LIVE_ENTRIES = toJsonList("resources",
+ "containers",
+ "components",
+ "nodes",
+ "statistics",
+ "internal");
+ public static final List<String> ROOT_ENTRIES =
+ toJsonList("model", "live", "actions");
private final ContentCache cache = new ContentCache();
+ private final StateAccessForProviders state;
public ApplicationResource(WebAppApi slider) {
super(slider);
- StateAccessForProviders state = slider.getAppState();
+ state = slider.getAppState();
cache.put(RestPaths.LIVE_RESOURCES,
new CachedContent<ConfTree>(LIFESPAN,
new LiveResourcesRefresher(state)));
cache.put(RestPaths.LIVE_CONTAINERS,
new CachedContent<Map<String, SerializedContainerInformation>>(LIFESPAN,
- new ContainerListRefresher(state)));
+ new LiveContainersRefresher(state)));
+ cache.put(RestPaths.LIVE_COMPONENTS,
+ new CachedContent<Map<String, SerializedComponentInformation>> (LIFESPAN,
+ new LiveComponentsRefresher(state)));
}
/**
@@ -64,7 +87,7 @@
* @param elements elements
* @return something that can be returned
*/
- private List<String> toJsonList(String... elements) {
+ private static List<String> toJsonList(String... elements) {
return Lists.newArrayList(elements);
}
@@ -72,7 +95,7 @@
@Path("/")
@Produces({MediaType.APPLICATION_JSON})
public List<String> getRoot() {
- return toJsonList("model", "live", "actions");
+ return ROOT_ENTRIES;
}
@GET
@@ -86,12 +109,7 @@
@Path("/live")
@Produces({MediaType.APPLICATION_JSON})
public List<String> getLive() {
- return toJsonList("resources",
- "containers",
- "components",
- "nodes",
- "statistics",
- "internal");
+ return LIVE_ENTRIES;
}
@GET
@@ -104,6 +122,7 @@
throw buildException(RestPaths.LIVE_RESOURCES, e);
}
}
+
@GET
@Path(RestPaths.LIVE_CONTAINERS)
@Produces({MediaType.APPLICATION_JSON})
@@ -116,4 +135,65 @@
}
}
+ @GET
+ @Path(RestPaths.LIVE_CONTAINERS + "/{containerId}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public SerializedContainerInformation getLiveContainer(
+ @PathParam("containerId") String containerId) {
+ try {
+ RoleInstance id = state.getLiveInstanceByContainerID(containerId);
+ return id.serialize();
+ } catch (NoSuchNodeException e) {
+ throw new NotFoundException("Unknown container: " + containerId);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_CONTAINERS + "/"+ containerId, e);
+ }
+ }
+
+ @GET
+ @Path(RestPaths.LIVE_COMPONENTS)
+ @Produces({MediaType.APPLICATION_JSON})
+ public Map<String, SerializedComponentInformation> getLiveComponents() {
+ try {
+ return (Map<String, SerializedComponentInformation>) cache.lookup(
+ RestPaths.LIVE_COMPONENTS);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_COMPONENTS, e);
+ }
+ }
+
+ @GET
+ @Path(RestPaths.LIVE_COMPONENTS+"/{component}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public SerializedComponentInformation getLiveComponent(
+ @PathParam("component") String component) {
+ try {
+ RoleStatus roleStatus = state.lookupRoleStatus(component);
+ SerializedComponentInformation info = roleStatus.serialize();
+ List<RoleInstance> containers = lookupRoleContainers(component);
+ info.containers = new ArrayList<String>(containers.size());
+ for (RoleInstance container : containers) {
+ info.containers.add(container.id);
+ }
+ return info;
+ } catch (YarnRuntimeException e) {
+ throw new NotFoundException("Unknown component: " + component);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_CONTAINERS, e);
+ }
+ }
+
+ List<RoleInstance> lookupRoleContainers(String component) {
+ RoleStatus roleStatus = state.lookupRoleStatus(component);
+ List<RoleInstance> ownedContainerList = state.cloneOwnedContainerList();
+ List<RoleInstance> matching = new ArrayList<RoleInstance>(ownedContainerList.size());
+ int roleId = roleStatus.getPriority();
+ for (RoleInstance instance : ownedContainerList) {
+ if (instance.roleId == roleId) {
+ matching.add(instance);
+ }
+ }
+ return matching;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java
new file mode 100644
index 0000000..e543265
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.types.SerializedComponentInformation;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LiveComponentsRefresher
+ implements ResourceRefresher<Map<String, SerializedComponentInformation>> {
+
+ private final StateAccessForProviders state;
+
+ public LiveComponentsRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public Map<String, SerializedComponentInformation> refresh() {
+
+ Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap();
+ Map<String, SerializedComponentInformation> results =
+ new HashMap<String, SerializedComponentInformation>(
+ roleStatusMap.size());
+
+ for (RoleStatus status : roleStatusMap.values()) {
+ String name = status.getName();
+ SerializedComponentInformation info = status.serialize();
+ results.put(name, info);
+ }
+ return results;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
similarity index 90%
rename from slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
index 7e74062..39a543b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
@@ -29,11 +29,11 @@
/**
* Refresh the container list.
*/
-public class ContainerListRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
+public class LiveContainersRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
private final StateAccessForProviders state;
- public ContainerListRefresher(StateAccessForProviders state) {
+ public LiveContainersRefresher(StateAccessForProviders state) {
this.state = state;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 4312cf3..2ad7129 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -21,9 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.StateValues
+import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
@@ -118,21 +121,65 @@
describe "Application REST ${LIVE_CONTAINERS}"
- Map<String, SerializedContainerInformation> map =
+ Map<String, SerializedContainerInformation> containers =
fetchType(HashMap, appmaster, LIVE_CONTAINERS)
- assert map.size() == 1
- log.info "${map}"
- SerializedContainerInformation info = (SerializedContainerInformation) map.values()[0]
- assert info.containerId
- assert map[info.containerId]
+ assert containers.size() == 1
+ log.info "${containers}"
+ SerializedContainerInformation amContainerInfo = (SerializedContainerInformation) containers.values()[0]
+ assert amContainerInfo.containerId
- assert info.component == COMPONENT_AM
- assert info.createTime > 0
- assert info.exitCode == null
- assert info.output == null
- assert info.released == null
- assert info.state == StateValues.STATE_LIVE
+ def amContainerId = amContainerInfo.containerId
+ assert containers[amContainerId]
+
+ assert amContainerInfo.component == COMPONENT_AM
+ assert amContainerInfo.createTime > 0
+ assert amContainerInfo.exitCode == null
+ assert amContainerInfo.output == null
+ assert amContainerInfo.released == null
+ assert amContainerInfo.state == StateValues.STATE_LIVE
+
+ describe "base entry lists"
+ def list = fetchType(ArrayList, appmaster, LIVE)
+
+ def live_entries = ApplicationResource.LIVE_ENTRIES
+ assert list.size() == live_entries.size()
+ live_entries.containsAll(list)
+ describe "containers"
+
+ SerializedContainerInformation retrievedContainerInfo =
+ fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS +"/${amContainerId}")
+ assert retrievedContainerInfo.containerId == amContainerId
+
+ // fetch missing
+ try {
+ def result = fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS + "/unknown")
+ fail("expected an error, got $result")
+ } catch (NotFoundException e) {
+ // expected
+ }
+
+ describe "components"
+
+ Map<String, SerializedComponentInformation> components =
+ fetchType(HashMap, appmaster, LIVE_COMPONENTS)
+ // two components
+ assert components.size() == 1
+ log.info "${components}"
+
+ SerializedComponentInformation amComponentInfo =
+ (SerializedComponentInformation)components[COMPONENT_AM]
+
+ SerializedComponentInformation amFullInfo = fetchType(
+ SerializedComponentInformation,
+ appmaster,
+ LIVE_COMPONENTS +"/${COMPONENT_AM}")
+
+ assert amFullInfo.containers.size() == 1
+ assert amFullInfo.containers[0] == amContainerId
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index c188b9c..712445d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -19,10 +19,6 @@
package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
-import org.apache.hadoop.ipc.ProtocolSignature
-import org.apache.hadoop.yarn.exceptions.YarnException
-import org.apache.slider.api.SliderClusterProtocol
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
@@ -36,7 +32,7 @@
import org.apache.slider.server.appmaster.web.WebAppApiImpl
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher
import org.junit.Test
@@ -73,7 +69,7 @@
@Test
public void testContainerListRefresher() throws Throwable {
- ContainerListRefresher clr = new ContainerListRefresher(stateAccess)
+ LiveContainersRefresher clr = new LiveContainersRefresher(stateAccess)
def map = clr.refresh()
assert map.size() == 0
List<RoleInstance> instances = startNodes()
