Merge branch 'develop' into feature/SLIDER-82-pass-3.1
# Conflicts:
# slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
# slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
diff --git a/slider-assembly/src/conf/slider-client.xml b/slider-assembly/src/conf/slider-client.xml
index 3a42bee..9c2e76d 100644
--- a/slider-assembly/src/conf/slider-client.xml
+++ b/slider-assembly/src/conf/slider-client.xml
@@ -22,14 +22,6 @@
-->
<configuration>
- <!--
- The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
- Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
- for Slider client to work. The following list is not an exhaustive list but the minimal config
- needed to interact with a non-secure cluster.
- -->
-
- <!--
<property>
<name>slider.client.resource.origin</name>
<value>conf/slider-client.xml</value>
@@ -37,14 +29,41 @@
</property>
<property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
+ <name>slider.security.protocol.acl</name>
+ <value>*</value>
+ <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
</property>
<property>
- <name>slider.security.protocol.acl</name>
- <value>*</value>
- <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
+ <name>slider.yarn.queue</name>
+ <value/>
+ <description>the name of the YARN queue to use.</description>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue.priority</name>
+ <value>1</value>
+ <description>the priority of the application.</description>
+ </property>
+
+ <property>
+ <name>slider.am.login.keytab.required</name>
+ <value>false</value>
+ <description>Declare that a keytab must be provided.</description>
+ </property>
+
+ <!--
+ The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
+ Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
+ for Slider client to work. The following list is not an exhaustive list but the minimal config
+ needed to interact with a non-secure cluster.
+ -->
+
+ <!--
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
</property>
<property>
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index 025bd32..8358491 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -65,6 +65,12 @@
* As a wire format it is less efficient in both xfer and ser/deser than
* a binary format, but by having one unified format for wire and persistence,
* the code paths are simplified.
+ *
+ * This was the original single-file specification/model used in the Hoya
+ * precursor to Slider. Its now retained primarily as a way to publish
+ * the current state of the application, or at least a fraction thereof ...
+ * the larger set of information from the REST API is beyond the scope of
+ * this structure.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@@ -165,41 +171,35 @@
* cluster-specific options -to control both
* the Slider AM and the application that it deploys
*/
- public Map<String, String> options =
- new HashMap<String, String>();
+ public Map<String, String> options = new HashMap<>();
/**
* cluster information
* This is only valid when querying the cluster status.
*/
- public Map<String, String> info =
- new HashMap<String, String>();
+ public Map<String, String> info = new HashMap<>();
/**
* Statistics. This is only relevant when querying the cluster status
*/
- public Map<String, Map<String, Integer>> statistics =
- new HashMap<String, Map<String, Integer>>();
+ public Map<String, Map<String, Integer>> statistics = new HashMap<>();
/**
* Instances: role->count
*/
- public Map<String, List<String>> instances =
- new HashMap<String, List<String>>();
+ public Map<String, List<String>> instances = new HashMap<>();
/**
* Role options,
* role -> option -> value
*/
- public Map<String, Map<String, String>> roles =
- new HashMap<String, Map<String, String>>();
+ public Map<String, Map<String, String>> roles = new HashMap<>();
/**
* List of key-value pairs to add to a client config to set up the client
*/
- public Map<String, String> clientProperties =
- new HashMap<String, String>();
+ public Map<String, String> clientProperties = new HashMap<>();
/**
* Status information
@@ -218,7 +218,6 @@
public ClusterDescription() {
}
-
@Override
public String toString() {
try {
@@ -284,8 +283,8 @@
* @param dataOutputStream an outout stream that will always be closed
* @throws IOException any failure
*/
- private void writeJsonAsBytes(DataOutputStream dataOutputStream) throws
- IOException {
+ private void writeJsonAsBytes(DataOutputStream dataOutputStream)
+ throws IOException {
try {
String json = toJsonString();
byte[] b = json.getBytes(UTF_8);
@@ -303,7 +302,7 @@
* @throws IOException IO problems
*/
public static ClusterDescription load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
+ throws IOException, JsonParseException, JsonMappingException {
FileStatus status = fs.getFileStatus(path);
byte[] b = new byte[(int) status.getLen()];
FSDataInputStream dataInputStream = fs.open(path);
@@ -389,7 +388,7 @@
try {
return mapper.readValue(jsonFile, ClusterDescription.class);
} catch (IOException e) {
- log.error("Exception while parsing json file {}: {}" , jsonFile, e);
+ log.error("Exception while parsing json file {}" , jsonFile, e);
throw e;
}
}
@@ -528,20 +527,20 @@
}
String val = roleopts.get(option);
if (val == null) {
- throw new BadConfigException("Missing option '%s' in role %s ", option,
- role);
+ throw new BadConfigException("Missing option '%s' in role %s ", option, role);
}
return val;
}
- /**
+
+ /**
* Get a mandatory integer role option
* @param role role to get from
* @param option option name
* @return resolved value
* @throws BadConfigException if the option is not defined
*/
- public int getMandatoryRoleOptInt(String role, String option) throws
- BadConfigException {
+ public int getMandatoryRoleOptInt(String role, String option)
+ throws BadConfigException {
getMandatoryRoleOpt(role, option);
return getRoleOptInt(role, option, 0);
}
@@ -564,7 +563,7 @@
public Map<String, String> getOrAddRole(String role) {
Map<String, String> map = getRole(role);
if (map == null) {
- map = new HashMap<String, String>();
+ map = new HashMap<>();
}
roles.put(role, map);
return map;
@@ -575,7 +574,7 @@
*/
@JsonIgnore
public Set<String> getRoleNames() {
- return new HashSet<String>(roles.keySet());
+ return new HashSet<>(roles.keySet());
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
index 21ece2b..5b95414 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
@@ -80,8 +80,7 @@
}
private static void mergeInComponentMap(ClusterDescription cd,
- ConfTree confTree
- ) {
+ ConfTree confTree) {
Map<String, Map<String, String>> components = confTree.components;
for (Map.Entry<String, Map<String, String>> compEntry : components.entrySet()) {
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
index 2608cd7..d255db0 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
@@ -38,11 +38,11 @@
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
public final class ClusterNode implements Cloneable {
protected static final Logger
- LOG = LoggerFactory.getLogger(ClusterDescription.class);
+ LOG = LoggerFactory.getLogger(ClusterNode.class);
@JsonIgnore
public ContainerId containerId;
-
+
/**
* server name
*/
@@ -67,8 +67,7 @@
public boolean released;
public String host;
public String hostUrl;
-
-
+
/**
* state from {@link ClusterDescription}
*/
@@ -165,7 +164,7 @@
try {
return mapper.readValue(json, ClusterNode.class);
} catch (IOException e) {
- LOG.error("Exception while parsing json : " + e + "\n" + json, e);
+ LOG.error("Exception while parsing json : {}\n{}", e , json, e);
throw e;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 4512354..eda01ad 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -65,6 +65,11 @@
String ROLE_PREEMPTED_INSTANCES = "role.failed.preempted.instances";
/**
+ * Number of pending anti-affine instances: {@value}
+ */
+ String ROLE_PENDING_AA_INSTANCES = "role.pending.aa.instances";
+
+ /**
* Status report: number currently being released: {@value}
*/
String ROLE_FAILED_STARTING_INSTANCES = "role.failed.starting.instances";
@@ -98,19 +103,4 @@
*/
String ENV_PREFIX = "env.";
-
- /**
- * Default no. of cores in the AM {@value}
- */
- int DEFAULT_AM_V_CORES = 1;
-
- /**
- * The default memory of the AM: {@value}
- */
- int DEFAULT_AM_MEMORY = 1024;
-
- /**
- * The default heap of the AM: {@value}
- */
- String DEFAULT_AM_HEAP = "512M";
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
index 3668c66..d21785f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
+++ b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
@@ -22,6 +22,7 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
@@ -37,7 +38,7 @@
/**
* Get the aggregate desired model
* @return the aggregate configuration of what was asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getDesiredModel() throws IOException;
@@ -45,7 +46,7 @@
/**
* Get the desired application configuration
* @return the application configuration asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredAppconf() throws IOException;
@@ -53,7 +54,7 @@
/**
* Get the desired YARN resources
* @return the resources asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredResources() throws IOException;
@@ -69,7 +70,7 @@
/**
* Get the aggregate resolved model
* @return the aggregate configuration of what was asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getResolvedModel() throws IOException;
@@ -77,7 +78,7 @@
/**
* Get the resolved application configuration
* @return the application configuration asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedAppconf() throws IOException;
@@ -85,7 +86,7 @@
/**
* Get the resolved YARN resources
* @return the resources asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedResources() throws IOException;
@@ -126,13 +127,13 @@
* @throws IOException on any failure
*/
ComponentInformation getComponent(String componentName) throws IOException;
-
+
/**
* List all nodes into a map of [name:info]
- * @return a possibly empty map of nodes
+ * @return a possibly empty list of nodes
* @throws IOException on any failure
*/
- Map<String, NodeInformation> getLiveNodes() throws IOException;
+ NodeInformationList getLiveNodes() throws IOException;
/**
* Get information about a node
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
index 569660d..373d64d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -15110,6 +15110,16 @@
* <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
*/
int getPendingAntiAffineRequestCount();
+
+ // optional bool isAARequestOutstanding = 19;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean hasIsAARequestOutstanding();
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean getIsAARequestOutstanding();
}
/**
* Protobuf type {@code org.apache.slider.api.ComponentInformationProto}
@@ -15260,6 +15270,11 @@
pendingAntiAffineRequestCount_ = input.readInt32();
break;
}
+ case 152: {
+ bitField0_ |= 0x00020000;
+ isAARequestOutstanding_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -15659,6 +15674,22 @@
return pendingAntiAffineRequestCount_;
}
+ // optional bool isAARequestOutstanding = 19;
+ public static final int ISAAREQUESTOUTSTANDING_FIELD_NUMBER = 19;
+ private boolean isAARequestOutstanding_;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00020000) == 0x00020000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+
private void initFields() {
name_ = "";
priority_ = 0;
@@ -15678,6 +15709,7 @@
nodeFailed_ = 0;
preempted_ = 0;
pendingAntiAffineRequestCount_ = 0;
+ isAARequestOutstanding_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -15745,6 +15777,9 @@
if (((bitField0_ & 0x00010000) == 0x00010000)) {
output.writeInt32(18, pendingAntiAffineRequestCount_);
}
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ output.writeBool(19, isAARequestOutstanding_);
+ }
getUnknownFields().writeTo(output);
}
@@ -15831,6 +15866,10 @@
size += com.google.protobuf.CodedOutputStream
.computeInt32Size(18, pendingAntiAffineRequestCount_);
}
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(19, isAARequestOutstanding_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -15941,6 +15980,11 @@
result = result && (getPendingAntiAffineRequestCount()
== other.getPendingAntiAffineRequestCount());
}
+ result = result && (hasIsAARequestOutstanding() == other.hasIsAARequestOutstanding());
+ if (hasIsAARequestOutstanding()) {
+ result = result && (getIsAARequestOutstanding()
+ == other.getIsAARequestOutstanding());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -16026,6 +16070,10 @@
hash = (37 * hash) + PENDINGANTIAFFINEREQUESTCOUNT_FIELD_NUMBER;
hash = (53 * hash) + getPendingAntiAffineRequestCount();
}
+ if (hasIsAARequestOutstanding()) {
+ hash = (37 * hash) + ISAAREQUESTOUTSTANDING_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getIsAARequestOutstanding());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -16176,6 +16224,8 @@
bitField0_ = (bitField0_ & ~0x00010000);
pendingAntiAffineRequestCount_ = 0;
bitField0_ = (bitField0_ & ~0x00020000);
+ isAARequestOutstanding_ = false;
+ bitField0_ = (bitField0_ & ~0x00040000);
return this;
}
@@ -16278,6 +16328,10 @@
to_bitField0_ |= 0x00010000;
}
result.pendingAntiAffineRequestCount_ = pendingAntiAffineRequestCount_;
+ if (((from_bitField0_ & 0x00040000) == 0x00040000)) {
+ to_bitField0_ |= 0x00020000;
+ }
+ result.isAARequestOutstanding_ = isAARequestOutstanding_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -16359,6 +16413,9 @@
if (other.hasPendingAntiAffineRequestCount()) {
setPendingAntiAffineRequestCount(other.getPendingAntiAffineRequestCount());
}
+ if (other.hasIsAARequestOutstanding()) {
+ setIsAARequestOutstanding(other.getIsAARequestOutstanding());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -17122,6 +17179,39 @@
return this;
}
+ // optional bool isAARequestOutstanding = 19;
+ private boolean isAARequestOutstanding_ ;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00040000) == 0x00040000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder setIsAARequestOutstanding(boolean value) {
+ bitField0_ |= 0x00040000;
+ isAARequestOutstanding_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder clearIsAARequestOutstanding() {
+ bitField0_ = (bitField0_ & ~0x00040000);
+ isAARequestOutstanding_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:org.apache.slider.api.ComponentInformationProto)
}
@@ -20324,6 +20414,21 @@
* <code>required int64 lastUsed = 10;</code>
*/
long getLastUsed();
+
+ // required string name = 11;
+ /**
+ * <code>required string name = 11;</code>
+ */
+ boolean hasName();
+ /**
+ * <code>required string name = 11;</code>
+ */
+ java.lang.String getName();
+ /**
+ * <code>required string name = 11;</code>
+ */
+ com.google.protobuf.ByteString
+ getNameBytes();
}
/**
* Protobuf type {@code org.apache.slider.api.NodeEntryInformationProto}
@@ -20426,6 +20531,11 @@
lastUsed_ = input.readInt64();
break;
}
+ case 90: {
+ bitField0_ |= 0x00000400;
+ name_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -20626,6 +20736,49 @@
return lastUsed_;
}
+ // required string name = 11;
+ public static final int NAME_FIELD_NUMBER = 11;
+ private java.lang.Object name_;
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ name_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
private void initFields() {
priority_ = 0;
requested_ = 0;
@@ -20637,6 +20790,7 @@
live_ = 0;
releasing_ = 0;
lastUsed_ = 0L;
+ name_ = "";
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -20683,6 +20837,10 @@
memoizedIsInitialized = 0;
return false;
}
+ if (!hasName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
memoizedIsInitialized = 1;
return true;
}
@@ -20720,6 +20878,9 @@
if (((bitField0_ & 0x00000200) == 0x00000200)) {
output.writeInt64(10, lastUsed_);
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBytes(11, getNameBytes());
+ }
getUnknownFields().writeTo(output);
}
@@ -20769,6 +20930,10 @@
size += com.google.protobuf.CodedOutputStream
.computeInt64Size(10, lastUsed_);
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(11, getNameBytes());
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -20842,6 +21007,11 @@
result = result && (getLastUsed()
== other.getLastUsed());
}
+ result = result && (hasName() == other.hasName());
+ if (hasName()) {
+ result = result && getName()
+ .equals(other.getName());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -20895,6 +21065,10 @@
hash = (37 * hash) + LASTUSED_FIELD_NUMBER;
hash = (53 * hash) + hashLong(getLastUsed());
}
+ if (hasName()) {
+ hash = (37 * hash) + NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getName().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -21024,6 +21198,8 @@
bitField0_ = (bitField0_ & ~0x00000100);
lastUsed_ = 0L;
bitField0_ = (bitField0_ & ~0x00000200);
+ name_ = "";
+ bitField0_ = (bitField0_ & ~0x00000400);
return this;
}
@@ -21092,6 +21268,10 @@
to_bitField0_ |= 0x00000200;
}
result.lastUsed_ = lastUsed_;
+ if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.name_ = name_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -21138,6 +21318,11 @@
if (other.hasLastUsed()) {
setLastUsed(other.getLastUsed());
}
+ if (other.hasName()) {
+ bitField0_ |= 0x00000400;
+ name_ = other.name_;
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -21183,6 +21368,10 @@
return false;
}
+ if (!hasName()) {
+
+ return false;
+ }
return true;
}
@@ -21535,6 +21724,80 @@
return this;
}
+ // required string name = 11;
+ private java.lang.Object name_ = "";
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ name_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder setName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder clearName() {
+ bitField0_ = (bitField0_ & ~0x00000400);
+ name_ = getDefaultInstance().getName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder setNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:org.apache.slider.api.NodeEntryInformationProto)
}
@@ -32299,47 +32562,27 @@
public interface GetLiveNodesResponseProtoOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // repeated string names = 1;
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
/**
- * <code>repeated string names = 1;</code>
- */
- java.util.List<java.lang.String>
- getNamesList();
- /**
- * <code>repeated string names = 1;</code>
- */
- int getNamesCount();
- /**
- * <code>repeated string names = 1;</code>
- */
- java.lang.String getNames(int index);
- /**
- * <code>repeated string names = 1;</code>
- */
- com.google.protobuf.ByteString
- getNamesBytes(int index);
-
- // repeated .org.apache.slider.api.NodeInformationProto nodes = 2;
- /**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto>
getNodesList();
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index);
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
int getNodesCount();
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
getNodesOrBuilderList();
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
int index);
@@ -32397,16 +32640,8 @@
}
case 10: {
if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- names_ = new com.google.protobuf.LazyStringArrayList();
- mutable_bitField0_ |= 0x00000001;
- }
- names_.add(input.readBytes());
- break;
- }
- case 18: {
- if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
nodes_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeInformationProto>();
- mutable_bitField0_ |= 0x00000002;
+ mutable_bitField0_ |= 0x00000001;
}
nodes_.add(input.readMessage(org.apache.slider.api.proto.Messages.NodeInformationProto.PARSER, extensionRegistry));
break;
@@ -32420,9 +32655,6 @@
e.getMessage()).setUnfinishedMessage(this);
} finally {
if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- names_ = new com.google.protobuf.UnmodifiableLazyStringList(names_);
- }
- if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
nodes_ = java.util.Collections.unmodifiableList(nodes_);
}
this.unknownFields = unknownFields.build();
@@ -32456,66 +32688,36 @@
return PARSER;
}
- // repeated string names = 1;
- public static final int NAMES_FIELD_NUMBER = 1;
- private com.google.protobuf.LazyStringList names_;
- /**
- * <code>repeated string names = 1;</code>
- */
- public java.util.List<java.lang.String>
- getNamesList() {
- return names_;
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public int getNamesCount() {
- return names_.size();
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public java.lang.String getNames(int index) {
- return names_.get(index);
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public com.google.protobuf.ByteString
- getNamesBytes(int index) {
- return names_.getByteString(index);
- }
-
- // repeated .org.apache.slider.api.NodeInformationProto nodes = 2;
- public static final int NODES_FIELD_NUMBER = 2;
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
+ public static final int NODES_FIELD_NUMBER = 1;
private java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> nodes_;
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> getNodesList() {
return nodes_;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
getNodesOrBuilderList() {
return nodes_;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public int getNodesCount() {
return nodes_.size();
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index) {
return nodes_.get(index);
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
int index) {
@@ -32523,7 +32725,6 @@
}
private void initFields() {
- names_ = com.google.protobuf.LazyStringArrayList.EMPTY;
nodes_ = java.util.Collections.emptyList();
}
private byte memoizedIsInitialized = -1;
@@ -32544,11 +32745,8 @@
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
- for (int i = 0; i < names_.size(); i++) {
- output.writeBytes(1, names_.getByteString(i));
- }
for (int i = 0; i < nodes_.size(); i++) {
- output.writeMessage(2, nodes_.get(i));
+ output.writeMessage(1, nodes_.get(i));
}
getUnknownFields().writeTo(output);
}
@@ -32559,18 +32757,9 @@
if (size != -1) return size;
size = 0;
- {
- int dataSize = 0;
- for (int i = 0; i < names_.size(); i++) {
- dataSize += com.google.protobuf.CodedOutputStream
- .computeBytesSizeNoTag(names_.getByteString(i));
- }
- size += dataSize;
- size += 1 * getNamesList().size();
- }
for (int i = 0; i < nodes_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(2, nodes_.get(i));
+ .computeMessageSize(1, nodes_.get(i));
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
@@ -32595,8 +32784,6 @@
org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto other = (org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto) obj;
boolean result = true;
- result = result && getNamesList()
- .equals(other.getNamesList());
result = result && getNodesList()
.equals(other.getNodesList());
result = result &&
@@ -32612,10 +32799,6 @@
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
- if (getNamesCount() > 0) {
- hash = (37 * hash) + NAMES_FIELD_NUMBER;
- hash = (53 * hash) + getNamesList().hashCode();
- }
if (getNodesCount() > 0) {
hash = (37 * hash) + NODES_FIELD_NUMBER;
hash = (53 * hash) + getNodesList().hashCode();
@@ -32730,11 +32913,9 @@
public Builder clear() {
super.clear();
- names_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000001);
if (nodesBuilder_ == null) {
nodes_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
} else {
nodesBuilder_.clear();
}
@@ -32765,16 +32946,10 @@
public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto buildPartial() {
org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto result = new org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto(this);
int from_bitField0_ = bitField0_;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- names_ = new com.google.protobuf.UnmodifiableLazyStringList(
- names_);
- bitField0_ = (bitField0_ & ~0x00000001);
- }
- result.names_ = names_;
if (nodesBuilder_ == null) {
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
nodes_ = java.util.Collections.unmodifiableList(nodes_);
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
}
result.nodes_ = nodes_;
} else {
@@ -32795,21 +32970,11 @@
public Builder mergeFrom(org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto other) {
if (other == org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance()) return this;
- if (!other.names_.isEmpty()) {
- if (names_.isEmpty()) {
- names_ = other.names_;
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- ensureNamesIsMutable();
- names_.addAll(other.names_);
- }
- onChanged();
- }
if (nodesBuilder_ == null) {
if (!other.nodes_.isEmpty()) {
if (nodes_.isEmpty()) {
nodes_ = other.nodes_;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
} else {
ensureNodesIsMutable();
nodes_.addAll(other.nodes_);
@@ -32822,7 +32987,7 @@
nodesBuilder_.dispose();
nodesBuilder_ = null;
nodes_ = other.nodes_;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
nodesBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getNodesFieldBuilder() : null;
@@ -32864,106 +33029,13 @@
}
private int bitField0_;
- // repeated string names = 1;
- private com.google.protobuf.LazyStringList names_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- private void ensureNamesIsMutable() {
- if (!((bitField0_ & 0x00000001) == 0x00000001)) {
- names_ = new com.google.protobuf.LazyStringArrayList(names_);
- bitField0_ |= 0x00000001;
- }
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public java.util.List<java.lang.String>
- getNamesList() {
- return java.util.Collections.unmodifiableList(names_);
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public int getNamesCount() {
- return names_.size();
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public java.lang.String getNames(int index) {
- return names_.get(index);
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public com.google.protobuf.ByteString
- getNamesBytes(int index) {
- return names_.getByteString(index);
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public Builder setNames(
- int index, java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureNamesIsMutable();
- names_.set(index, value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public Builder addNames(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureNamesIsMutable();
- names_.add(value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public Builder addAllNames(
- java.lang.Iterable<java.lang.String> values) {
- ensureNamesIsMutable();
- super.addAll(values, names_);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public Builder clearNames() {
- names_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000001);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string names = 1;</code>
- */
- public Builder addNamesBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureNamesIsMutable();
- names_.add(value);
- onChanged();
- return this;
- }
-
- // repeated .org.apache.slider.api.NodeInformationProto nodes = 2;
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
private java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> nodes_ =
java.util.Collections.emptyList();
private void ensureNodesIsMutable() {
- if (!((bitField0_ & 0x00000002) == 0x00000002)) {
+ if (!((bitField0_ & 0x00000001) == 0x00000001)) {
nodes_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeInformationProto>(nodes_);
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000001;
}
}
@@ -32971,7 +33043,7 @@
org.apache.slider.api.proto.Messages.NodeInformationProto, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder> nodesBuilder_;
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> getNodesList() {
if (nodesBuilder_ == null) {
@@ -32981,7 +33053,7 @@
}
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public int getNodesCount() {
if (nodesBuilder_ == null) {
@@ -32991,7 +33063,7 @@
}
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index) {
if (nodesBuilder_ == null) {
@@ -33001,7 +33073,7 @@
}
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder setNodes(
int index, org.apache.slider.api.proto.Messages.NodeInformationProto value) {
@@ -33018,7 +33090,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder setNodes(
int index, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
@@ -33032,7 +33104,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder addNodes(org.apache.slider.api.proto.Messages.NodeInformationProto value) {
if (nodesBuilder_ == null) {
@@ -33048,7 +33120,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder addNodes(
int index, org.apache.slider.api.proto.Messages.NodeInformationProto value) {
@@ -33065,7 +33137,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder addNodes(
org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
@@ -33079,7 +33151,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder addNodes(
int index, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
@@ -33093,7 +33165,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder addAllNodes(
java.lang.Iterable<? extends org.apache.slider.api.proto.Messages.NodeInformationProto> values) {
@@ -33107,12 +33179,12 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder clearNodes() {
if (nodesBuilder_ == null) {
nodes_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
onChanged();
} else {
nodesBuilder_.clear();
@@ -33120,7 +33192,7 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public Builder removeNodes(int index) {
if (nodesBuilder_ == null) {
@@ -33133,14 +33205,14 @@
return this;
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder getNodesBuilder(
int index) {
return getNodesFieldBuilder().getBuilder(index);
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
int index) {
@@ -33150,7 +33222,7 @@
}
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
getNodesOrBuilderList() {
@@ -33161,14 +33233,14 @@
}
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder addNodesBuilder() {
return getNodesFieldBuilder().addBuilder(
org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance());
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder addNodesBuilder(
int index) {
@@ -33176,7 +33248,7 @@
index, org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance());
}
/**
- * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 2;</code>
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
*/
public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto.Builder>
getNodesBuilderList() {
@@ -33189,7 +33261,7 @@
nodesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
org.apache.slider.api.proto.Messages.NodeInformationProto, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>(
nodes_,
- ((bitField0_ & 0x00000002) == 0x00000002),
+ ((bitField0_ & 0x00000001) == 0x00000001),
getParentForChildren(),
isClean());
nodes_ = null;
@@ -34023,7 +34095,7 @@
" \002(\t\022\023\n\013application\030\003 \002(\t\"`\n#Application" +
"LivenessInformationProto\022\034\n\024allRequestsS" +
"atisfied\030\001 \001(\010\022\033\n\023requestsOutstanding\030\002 " +
- "\001(\005\"\216\003\n\031ComponentInformationProto\022\014\n\004nam",
+ "\001(\005\"\256\003\n\031ComponentInformationProto\022\014\n\004nam",
"e\030\001 \001(\t\022\020\n\010priority\030\002 \001(\005\022\017\n\007desired\030\003 \001" +
"(\005\022\016\n\006actual\030\004 \001(\005\022\021\n\treleasing\030\005 \001(\005\022\021\n" +
"\trequested\030\006 \001(\005\022\016\n\006failed\030\007 \001(\005\022\017\n\007star" +
@@ -34033,54 +34105,55 @@
"(\005\022\022\n\ncontainers\030\016 \003(\t\022\026\n\016failedRecently" +
"\030\017 \001(\005\022\022\n\nnodeFailed\030\020 \001(\005\022\021\n\tpreempted\030" +
"\021 \001(\005\022%\n\035pendingAntiAffineRequestCount\030\022" +
- " \001(\005\"\210\002\n\031ContainerInformationProto\022\023\n\013co",
- "ntainerId\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010re" +
- "leased\030\003 \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030" +
- "\005 \001(\005\022\023\n\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime" +
- "\030\007 \001(\003\022\021\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(" +
- "\t\022\014\n\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplac" +
- "ement\030\014 \001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024PingI" +
- "nformationProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 " +
- "\001(\t\022\014\n\004body\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\325\001\n\031Node" +
- "EntryInformationProto\022\020\n\010priority\030\001 \002(\005\022" +
- "\021\n\trequested\030\002 \002(\005\022\020\n\010starting\030\003 \002(\005\022\023\n\013",
- "startFailed\030\004 \002(\005\022\016\n\006failed\030\005 \002(\005\022\026\n\016fai" +
- "ledRecently\030\006 \002(\005\022\021\n\tpreempted\030\007 \002(\005\022\014\n\004" +
- "live\030\010 \002(\005\022\021\n\treleasing\030\t \002(\005\022\020\n\010lastUse" +
- "d\030\n \002(\003\"\334\001\n\024NodeInformationProto\022\020\n\010host" +
- "name\030\001 \002(\t\022\r\n\005state\030\002 \002(\t\022\023\n\013httpAddress" +
- "\030\003 \002(\t\022\020\n\010rackName\030\004 \002(\t\022\016\n\006labels\030\005 \002(\t" +
- "\022\024\n\014healthReport\030\006 \002(\t\022\023\n\013lastUpdated\030\007 " +
- "\002(\003\022A\n\007entries\030\010 \003(\01320.org.apache.slider" +
- ".api.NodeEntryInformationProto\"\026\n\024GetMod" +
- "elRequestProto\"\035\n\033GetModelDesiredRequest",
- "Proto\"$\n\"GetModelDesiredAppconfRequestPr" +
- "oto\"&\n$GetModelDesiredResourcesRequestPr" +
- "oto\"%\n#GetModelResolvedAppconfRequestPro" +
- "to\"\'\n%GetModelResolvedResourcesRequestPr" +
- "oto\"#\n!GetModelLiveResourcesRequestProto" +
- "\"\037\n\035GetLiveContainersRequestProto\"u\n\036Get" +
- "LiveContainersResponseProto\022\r\n\005names\030\001 \003" +
- "(\t\022D\n\ncontainers\030\002 \003(\01320.org.apache.slid" +
- "er.api.ContainerInformationProto\"3\n\034GetL" +
- "iveContainerRequestProto\022\023\n\013containerId\030",
- "\001 \002(\t\"\037\n\035GetLiveComponentsRequestProto\"u" +
- "\n\036GetLiveComponentsResponseProto\022\r\n\005name" +
- "s\030\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org.apache" +
- ".slider.api.ComponentInformationProto\",\n" +
- "\034GetLiveComponentRequestProto\022\014\n\004name\030\001 " +
- "\002(\t\"$\n\"GetApplicationLivenessRequestProt" +
- "o\"\023\n\021EmptyPayloadProto\" \n\020WrappedJsonPro" +
- "to\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificateStoreR" +
- "equestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013request" +
- "erId\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004 \002",
- "(\t\"1\n GetCertificateStoreResponseProto\022\r" +
- "\n\005store\030\001 \002(\014\"\032\n\030GetLiveNodesRequestProt" +
- "o\"f\n\031GetLiveNodesResponseProto\022\r\n\005names\030" +
- "\001 \003(\t\022:\n\005nodes\030\002 \003(\0132+.org.apache.slider" +
- ".api.NodeInformationProto\"\'\n\027GetLiveNode" +
- "RequestProto\022\014\n\004name\030\001 \002(\tB-\n\033org.apache" +
- ".slider.api.protoB\010Messages\210\001\001\240\001\001"
+ " \001(\005\022\036\n\026isAARequestOutstanding\030\023 \001(\010\"\210\002\n",
+ "\031ContainerInformationProto\022\023\n\013containerI" +
+ "d\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010released\030\003" +
+ " \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030\005 \001(\005\022\023\n" +
+ "\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime\030\007 \001(\003\022\021" +
+ "\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(\t\022\014\n\004hos" +
+ "t\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplacement\030\014 " +
+ "\001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024PingInformati" +
+ "onProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 \001(\t\022\014\n\004b" +
+ "ody\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\343\001\n\031NodeEntryInf" +
+ "ormationProto\022\020\n\010priority\030\001 \002(\005\022\021\n\treque",
+ "sted\030\002 \002(\005\022\020\n\010starting\030\003 \002(\005\022\023\n\013startFai" +
+ "led\030\004 \002(\005\022\016\n\006failed\030\005 \002(\005\022\026\n\016failedRecen" +
+ "tly\030\006 \002(\005\022\021\n\tpreempted\030\007 \002(\005\022\014\n\004live\030\010 \002" +
+ "(\005\022\021\n\treleasing\030\t \002(\005\022\020\n\010lastUsed\030\n \002(\003\022" +
+ "\014\n\004name\030\013 \002(\t\"\334\001\n\024NodeInformationProto\022\020" +
+ "\n\010hostname\030\001 \002(\t\022\r\n\005state\030\002 \002(\t\022\023\n\013httpA" +
+ "ddress\030\003 \002(\t\022\020\n\010rackName\030\004 \002(\t\022\016\n\006labels" +
+ "\030\005 \002(\t\022\024\n\014healthReport\030\006 \002(\t\022\023\n\013lastUpda" +
+ "ted\030\007 \002(\003\022A\n\007entries\030\010 \003(\01320.org.apache." +
+ "slider.api.NodeEntryInformationProto\"\026\n\024",
+ "GetModelRequestProto\"\035\n\033GetModelDesiredR" +
+ "equestProto\"$\n\"GetModelDesiredAppconfReq" +
+ "uestProto\"&\n$GetModelDesiredResourcesReq" +
+ "uestProto\"%\n#GetModelResolvedAppconfRequ" +
+ "estProto\"\'\n%GetModelResolvedResourcesReq" +
+ "uestProto\"#\n!GetModelLiveResourcesReques" +
+ "tProto\"\037\n\035GetLiveContainersRequestProto\"" +
+ "u\n\036GetLiveContainersResponseProto\022\r\n\005nam" +
+ "es\030\001 \003(\t\022D\n\ncontainers\030\002 \003(\01320.org.apach" +
+ "e.slider.api.ContainerInformationProto\"3",
+ "\n\034GetLiveContainerRequestProto\022\023\n\013contai" +
+ "nerId\030\001 \002(\t\"\037\n\035GetLiveComponentsRequestP" +
+ "roto\"u\n\036GetLiveComponentsResponseProto\022\r" +
+ "\n\005names\030\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org." +
+ "apache.slider.api.ComponentInformationPr" +
+ "oto\",\n\034GetLiveComponentRequestProto\022\014\n\004n" +
+ "ame\030\001 \002(\t\"$\n\"GetApplicationLivenessReque" +
+ "stProto\"\023\n\021EmptyPayloadProto\" \n\020WrappedJ" +
+ "sonProto\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificate" +
+ "StoreRequestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013r",
+ "equesterId\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004ty" +
+ "pe\030\004 \002(\t\"1\n GetCertificateStoreResponseP" +
+ "roto\022\r\n\005store\030\001 \002(\014\"\032\n\030GetLiveNodesReque" +
+ "stProto\"W\n\031GetLiveNodesResponseProto\022:\n\005" +
+ "nodes\030\001 \003(\0132+.org.apache.slider.api.Node" +
+ "InformationProto\"\'\n\027GetLiveNodeRequestPr" +
+ "oto\022\014\n\004name\030\001 \002(\tB-\n\033org.apache.slider.a" +
+ "pi.protoB\010Messages\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -34236,7 +34309,7 @@
internal_static_org_apache_slider_api_ComponentInformationProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_ComponentInformationProto_descriptor,
- new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", "PendingAntiAffineRequestCount", });
+ new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", "PendingAntiAffineRequestCount", "IsAARequestOutstanding", });
internal_static_org_apache_slider_api_ContainerInformationProto_descriptor =
getDescriptor().getMessageTypes().get(25);
internal_static_org_apache_slider_api_ContainerInformationProto_fieldAccessorTable = new
@@ -34254,7 +34327,7 @@
internal_static_org_apache_slider_api_NodeEntryInformationProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor,
- new java.lang.String[] { "Priority", "Requested", "Starting", "StartFailed", "Failed", "FailedRecently", "Preempted", "Live", "Releasing", "LastUsed", });
+ new java.lang.String[] { "Priority", "Requested", "Starting", "StartFailed", "Failed", "FailedRecently", "Preempted", "Live", "Releasing", "LastUsed", "Name", });
internal_static_org_apache_slider_api_NodeInformationProto_descriptor =
getDescriptor().getMessageTypes().get(28);
internal_static_org_apache_slider_api_NodeInformationProto_fieldAccessorTable = new
@@ -34380,7 +34453,7 @@
internal_static_org_apache_slider_api_GetLiveNodesResponseProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor,
- new java.lang.String[] { "Names", "Nodes", });
+ new java.lang.String[] { "Nodes", });
internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor =
getDescriptor().getMessageTypes().get(49);
internal_static_org_apache_slider_api_GetLiveNodeRequestProto_fieldAccessorTable = new
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 115405c..feebe1d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -38,7 +38,10 @@
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Class to handle marshalling of REST
@@ -84,8 +87,12 @@
if (wire.hasFailureMessage()) {
info.failureMessage = wire.getFailureMessage();
}
- info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
- info.pendingAntiAffineRequest = info.pendingAntiAffineRequestCount > 0;
+ if (wire.hasPendingAntiAffineRequestCount()) {
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ }
+ if (wire.hasIsAARequestOutstanding()) {
+ info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+ }
return info;
}
@@ -136,6 +143,7 @@
builder.addAllContainers(info.containers);
}
builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
+ builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
return builder.build();
}
@@ -145,33 +153,25 @@
Messages.NodeInformationProto.newBuilder();
builder.setHostname(info.hostname);
builder.setLastUpdated(info.lastUpdated);
- if (info.state != null) {
- builder.setState(info.state);
- }
- if (info.rackName != null) {
- builder.setRackName(info.rackName);
- }
- if (info.healthReport != null) {
- builder.setHealthReport(info.healthReport);
- }
- if (info.httpAddress != null) {
- builder.setHttpAddress(info.httpAddress);
- }
- if (info.labels != null) {
- builder.setLabels(info.labels);
- }
+ builder.setState(info.state != null? info.state : "unknown");
+ builder.setRackName(info.rackName != null ? info.rackName : "");
+ builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
+ builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
+ builder.setLabels(info.labels != null ? info.labels: "");
- List<NodeEntryInformation> entries = info.entries;
- if (entries != null) {
- for (NodeEntryInformation entry : entries) {
+
+ if (info.entries != null) {
+ for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
+ NodeEntryInformation entry = elt.getValue();
Messages.NodeEntryInformationProto.Builder node =
Messages.NodeEntryInformationProto.newBuilder();
+ node.setPriority(entry.priority);
+ node.setName(elt.getKey());
node.setFailed(entry.failed);
node.setFailedRecently(entry.failedRecently);
node.setLive(entry.live);
node.setLastUsed(entry.lastUsed);
node.setPreempted(entry.preempted);
- node.setPriority(entry.priority);
node.setRequested(entry.requested);
node.setReleasing(entry.releasing);
node.setStartFailed(entry.startFailed);
@@ -193,7 +193,7 @@
info.state = wire.getState();
List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
if (entriesList != null) {
- info.entries = new ArrayList<>(entriesList.size());
+ info.entries = new HashMap<>(entriesList.size());
for (Messages.NodeEntryInformationProto entry : entriesList) {
NodeEntryInformation nei = new NodeEntryInformation();
nei.failed = entry.getFailed();
@@ -206,7 +206,7 @@
nei.releasing = entry.getReleasing();
nei.startFailed = entry.getStartFailed();
nei.starting = entry.getStarting();
- info.entries.add(nei);
+ info.entries.put(entry.getName(), nei);
}
}
return info;
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
index 4dd2cb7..9879d05 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -30,9 +30,15 @@
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ApplicationLivenessInformation {
+ /** flag set if the cluster is at size */
public boolean allRequestsSatisfied;
+
+ /** number of outstanding requests: those needed to satisfy */
public int requestsOutstanding;
+ /** number of requests submitted to YARN */
+ public int activeRequests;
+
@Override
public String toString() {
final StringBuilder sb =
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 3b4b8bd..c46a59f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -52,8 +52,8 @@
public int requested;
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
- public boolean pendingAntiAffineRequest;
public int pendingAntiAffineRequestCount;
+ public boolean isAARequestOutstanding;
public String failureMessage;
public List<String> containers;
@@ -85,19 +85,20 @@
new StringBuilder("ComponentInformation{");
sb.append(", name='").append(name).append('\'');
sb.append(", actual=").append(actual);
- sb.append(", anti-affine pending").append(pendingAntiAffineRequestCount);
sb.append(", completed=").append(completed);
sb.append(", desired=").append(desired);
sb.append(", failed=").append(failed);
sb.append(", failureMessage='").append(failureMessage).append('\'');
sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
+ sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount);
sb.append(", priority=").append(priority);
sb.append(", releasing=").append(releasing);
sb.append(", requested=").append(requested);
sb.append(", started=").append(started);
sb.append(", startFailed=").append(startFailed);
sb.append(", totalRequested=").append(totalRequested);
- sb.append("container count='")
+ sb.append(", container count='")
.append(containers == null ? 0 : containers.size())
.append('\'');
sb.append('}');
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
index 482b0c7..8424be2 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
@@ -28,7 +28,6 @@
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class NodeEntryInformation {
-
/** incrementing counter of instances that failed */
public int failed;
@@ -58,4 +57,22 @@
/** number of starting instances */
public int starting;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "NodeEntryInformation{");
+ sb.append("priority=").append(priority);
+ sb.append(", live=").append(live);
+ sb.append(", requested=").append(requested);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", starting=").append(starting);
+ sb.append(", failed=").append(failed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", preempted=").append(preempted);
+ sb.append(", lastUsed=").append(lastUsed);
+ sb.append('}');
+ return sb.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
index 049ee52..4fe5b4c 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
@@ -22,7 +22,9 @@
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Serialized node information. Must be kept in sync with the protobuf equivalent.
@@ -31,12 +33,27 @@
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class NodeInformation {
- public String healthReport;
public String hostname;
- public String httpAddress;
- public String labels;
- public long lastUpdated;
- public String rackName;
public String state;
- public List<NodeEntryInformation> entries = new ArrayList<>();
+ public String labels;
+ public String rackName;
+ public String httpAddress;
+ public String healthReport;
+ public long lastUpdated;
+ public Map<String, NodeEntryInformation> entries = new HashMap<>();
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "NodeInformation{");
+ sb.append("hostname='").append(hostname).append('\'');
+ sb.append(", state='").append(state).append('\'');
+ sb.append(", labels='").append(labels).append('\'');
+ sb.append(", rackName='").append(rackName).append('\'');
+ sb.append(", httpAddress='").append(httpAddress).append('\'');
+ sb.append(", healthReport='").append(healthReport).append('\'');
+ sb.append(", lastUpdated=").append(lastUpdated);
+ sb.append('}');
+ return sb.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
new file mode 100644
index 0000000..741523e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.core.persist.JsonSerDeser;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+public class NodeInformationList extends ArrayList<NodeInformation> {
+ public NodeInformationList() {
+ }
+
+ public NodeInformationList(Collection<? extends NodeInformation> c) {
+ super(c);
+ }
+
+ public NodeInformationList(int initialCapacity) {
+ super(initialCapacity);
+ }
+
+ public static JsonSerDeser<NodeInformationList> createSerializer() {
+ return new JsonSerDeser<>(NodeInformationList.class);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
new file mode 100644
index 0000000..c926600
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+/**
+ * Simple role statistics for state views; can be generated by RoleStatus
+ * instances, and aggregated for summary information.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RoleStatistics {
+ public long activeAA = 0L;
+ public long actual = 0L;
+ public long completed = 0L;
+ public long desired = 0L;
+ public long failed = 0L;
+ public long failedRecently = 0L;
+ public long limitsExceeded = 0L;
+ public long nodeFailed = 0L;
+ public long preempted = 0L;
+ public long releasing = 0L;
+ public long requested = 0L;
+ public long started = 0L;
+ public long startFailed = 0L;
+ public long totalRequested = 0L;
+
+ /**
+ * Add another statistics instance
+ * @param that the other value
+ * @return this entry
+ */
+ public RoleStatistics add(final RoleStatistics that) {
+ activeAA += that.activeAA;
+ actual += that.actual;
+ completed += that.completed;
+ desired += that.desired;
+ failed += that.failed;
+ failedRecently += that.failedRecently;
+ limitsExceeded += that.limitsExceeded;
+ nodeFailed += that.nodeFailed;
+ preempted += that.preempted;
+ releasing += that.releasing;
+ requested += that.requested;
+ started += that.started;
+ startFailed += that.totalRequested;
+ totalRequested += that.totalRequested;
+ return this;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f44b739..da94dd4 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -64,11 +64,14 @@
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.client.ipc.SliderApplicationIpcClient;
import org.apache.slider.client.ipc.SliderClusterOperations;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
@@ -91,6 +94,7 @@
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionLookupArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
@@ -104,6 +108,7 @@
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
import org.apache.slider.core.build.InstanceBuilder;
import org.apache.slider.core.build.InstanceIO;
@@ -133,6 +138,7 @@
import org.apache.slider.core.persist.AppDefinitionPersister;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.SliderRegistryUtils;
import org.apache.slider.core.registry.YarnAppListClient;
@@ -166,7 +172,6 @@
import java.io.BufferedReader;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -333,7 +338,7 @@
if (isUnset(action)) {
throw new SliderException(EXIT_USAGE, serviceArgs.usage());
}
-
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -405,11 +410,15 @@
case ACTION_LIST:
exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
break;
-
+
case ACTION_LOOKUP:
exitCode = actionLookup(serviceArgs.getActionLookupArgs());
break;
+ case ACTION_NODES:
+ exitCode = actionNodes("", serviceArgs.getActionNodesArgs());
+ break;
+
case ACTION_PACKAGE:
exitCode = actionPackage(serviceArgs.getActionPackageArgs());
break;
@@ -2111,9 +2120,7 @@
commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
- /**
- * pass the registry binding
- */
+ // pass the registry binding
commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -2123,6 +2130,15 @@
// the relevant security settings go over
commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
+
+ // copy over any/all YARN RM client values, in case the server-side XML conf file
+ // has the 0.0.0.0 address
+ commandLine.addConfOptions(config,
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.RM_HOSTNAME,
+ YarnConfiguration.RM_PRINCIPAL);
+
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2137,12 +2153,8 @@
// Set the priority for the application master
-
- int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY,
- DEFAULT_YARN_QUEUE_PRIORITY);
-
-
- amLauncher.setPriority(amPriority);
+ amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY,
+ DEFAULT_YARN_QUEUE_PRIORITY));
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
@@ -2153,7 +2165,7 @@
log.info("Using queue {} for the application instance.", amQueue);
}
- if (amQueue != null) {
+ if (isSet(amQueue)) {
amLauncher.setQueue(amQueue);
}
@@ -4226,6 +4238,73 @@
throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs,
actionName));
}
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param args argument list
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ @Override
+ public NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
+ throws YarnException, IOException {
+ return yarnClient.listNodes(args.label, args.healthy);
+ }
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param args argument list
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public NodeInformationList listInstanceNodes(String instance, ActionNodesArgs args)
+ throws YarnException, IOException {
+ // TODO
+ log.info("listInstanceNodes {}", instance);
+ SliderClusterOperations clusterOps =
+ new SliderClusterOperations(bondToCluster(instance));
+ return clusterOps.getLiveNodes();
+ }
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ * Prints them to stdout unless the args names a file instead.
+ * @param args argument list
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public int actionNodes(String instance, ActionNodesArgs args) throws YarnException, IOException {
+
+ args.instance = instance;
+ NodeInformationList nodes;
+ if (SliderUtils.isUnset(instance)) {
+ nodes = listYarnClusterNodes(args);
+ } else {
+ nodes = listInstanceNodes(instance, args);
+ }
+ log.debug("Node listing for {} has {} nodes", args, nodes.size());
+ JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer();
+ if (args.outputFile != null) {
+ serDeser.save(nodes, args.outputFile);
+ } else {
+ println(serDeser.toJson(nodes));
+ }
+ return 0;
+ }
+
+ /**
+ * Create a new IPC client for talking to slider via what follows the REST API.
+ * Client must already be bonded to the cluster
+ * @return a new IPC client
+ */
+ public SliderApplicationApi createIpcClient()
+ throws IOException, YarnException {
+ return new SliderApplicationIpcClient(createClusterOperations());
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index d87b121..5c5d96b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
@@ -36,6 +37,7 @@
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
@@ -342,4 +344,14 @@
*/
int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException,
YarnException;
+
+ /**
+ * List the nodes
+ * @param args
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
+ throws YarnException, IOException;
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 3b7a65c..85a582b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -20,6 +20,7 @@
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -27,12 +28,18 @@
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -42,6 +49,8 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.BindException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -55,8 +64,7 @@
* from the slider entry point service
*/
public class SliderYarnClientImpl extends YarnClientImpl {
- protected static final Logger
- log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
+ protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
/**
* Keyword to use in the {@link #emergencyForceKill(String)}
@@ -65,6 +73,18 @@
*/
public static final String KILL_ALL = "all";
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // address isn't known; fail fast
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ + " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
+ }
+ super.serviceInit(conf);
+ }
+
/**
* Get the RM Client RPC interface
* @return an RPC interface valid after initialization and authentication
@@ -96,10 +116,10 @@
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
- Set<String> types = new HashSet<String>(1);
+ Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
- List<ApplicationReport> results = new ArrayList<ApplicationReport>();
+ List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
@@ -108,7 +128,6 @@
return results;
}
-
/**
* find all instances of a specific app -if there is more than one in the
* YARN cluster,
@@ -124,7 +143,7 @@
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getName().equals(appname)) {
results.add(report);
@@ -142,8 +161,7 @@
public boolean isApplicationLive(ApplicationReport app) {
Preconditions.checkArgument(app != null, "Null app report");
- return app.getYarnApplicationState().ordinal() <=
- YarnApplicationState.RUNNING.ordinal();
+ return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal();
}
@@ -330,8 +348,6 @@
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
Preconditions.checkArgument(desiredState != null, "Null desiredState");
- ApplicationReport found = null;
- ApplicationReport foundAndLive = null;
log.debug("Searching {} records for instance name {} in state '{}'",
instances.size(), appname, desiredState);
for (ApplicationReport app : instances) {
@@ -350,4 +366,45 @@
log.debug("No match");
return null;
}
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param label label to filter by -or "" for any
+ * @param live flag to request running nodes only
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public NodeInformationList listNodes(String label, boolean live)
+ throws IOException, YarnException {
+ Preconditions.checkArgument(label != null, "null label");
+ NodeState[] states;
+ if (live) {
+ states = new NodeState[1];
+ states[0] = NodeState.RUNNING;
+ } else {
+ states = new NodeState[0];
+ }
+ List<NodeReport> reports = getNodeReports(states);
+ NodeInformationList results = new NodeInformationList(reports.size());
+ for (NodeReport report : reports) {
+ if (live && report.getNodeState() != NodeState.RUNNING) {
+ continue;
+ }
+ if (!label.isEmpty() && !report.getNodeLabels().contains(label)) {
+ continue;
+ }
+ // build node info from report
+ NodeInformation info = new NodeInformation();
+ info.hostname = report.getNodeId().getHost();
+ info.healthReport = report.getHealthReport();
+ info.httpAddress = report.getHttpAddress();
+ info.labels = SliderUtils.extractNodeLabel(report);
+ info.rackName = report.getRackName();
+ info.state = report.getNodeState().toString();
+ results.add(info);
+ }
+ return results;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
index 291583d..a007326 100644
--- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
@@ -24,6 +24,7 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.core.conf.AggregateConf;
@@ -196,7 +197,7 @@
}
@Override
- public Map<String, NodeInformation> getLiveNodes() throws IOException {
+ public NodeInformationList getLiveNodes() throws IOException {
try {
return operations.getLiveNodes();
} catch (IOException e) {
diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index e1ec971..392f451 100644
--- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -19,6 +19,9 @@
package org.apache.slider.client.ipc;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -31,8 +34,10 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -237,7 +242,7 @@
/**
* Get the details on a list of uuids
- * @param uuids
+ * @param uuids instance IDs
* @return a possibly empty list of node details
* @throws IOException
* @throws YarnException
@@ -468,22 +473,16 @@
return unmarshall(proto);
}
- public Map<String, NodeInformation> getLiveNodes() throws IOException {
+ public NodeInformationList getLiveNodes() throws IOException {
Messages.GetLiveNodesResponseProto response =
appMaster.getLiveNodes(Messages.GetLiveNodesRequestProto.newBuilder().build());
- int namesCount = response.getNamesCount();
int records = response.getNodesCount();
- if (namesCount != records) {
- throw new IOException(
- "Number of names returned (" + namesCount + ")" +
- " does not match the number of records returned: " + records);
+ NodeInformationList nil = new NodeInformationList(records);
+ for (int i = 0; i < records; i++) {
+ nil.add(unmarshall(response.getNodes(i)));
}
- Map<String, NodeInformation> map = new HashMap<>(namesCount);
- for (int i = 0; i < namesCount; i++) {
- map.put(response.getNames(i), unmarshall(response.getNodes(i)));
- }
- return map;
+ return nil;
}
public NodeInformation getLiveNode(String hostname) throws IOException {
@@ -527,5 +526,4 @@
return unmarshall(response);
}
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
index ce2817a..4283ee8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
@@ -21,7 +21,6 @@
import com.google.common.base.Preconditions;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
@@ -32,11 +31,11 @@
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.core.restclient.HttpVerb;
import org.apache.slider.api.types.PingInformation;
import org.slf4j.Logger;
@@ -68,6 +67,19 @@
this.appResource = appResource;
}
+ /**
+ * Create an instance
+ * @param jerseyClient jersey client for operations
+ * @param appmaster URL of appmaster/proxy to AM
+ */
+ public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) {
+ super(jerseyClient);
+ WebResource amResource = jerseyClient.resource(appmaster);
+ amResource.type(MediaType.APPLICATION_JSON);
+ this.appResource = amResource.path(SLIDER_PATH_APPLICATION);
+ }
+
+
@Override
public String toString() {
final StringBuilder sb =
@@ -244,14 +256,13 @@
}
@Override
- public Map<String, NodeInformation> getLiveNodes() throws IOException {
- return getApplicationResource(LIVE_NODES,
- new GenericType<Map<String, NodeInformation>>() { });
+ public NodeInformationList getLiveNodes() throws IOException {
+ return getApplicationResource(LIVE_NODES, NodeInformationList.class);
}
@Override
public NodeInformation getLiveNode(String hostname) throws IOException {
- return getApplicationResource(LIVE_COMPONENTS + "/" + hostname,
+ return getApplicationResource(LIVE_NODES + "/" + hostname,
NodeInformation.class);
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 07214b2..26109a7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -125,6 +125,8 @@
"hadoop.http.filter.initializers";
String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ /** Declare that a keytab must be provided */
+ String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 17c235f..d0b1693 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -56,7 +56,7 @@
* This is the default parameter
*/
@Parameter
- public final List<String> parameters = new ArrayList<String>();
+ public final List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
@@ -77,7 +77,7 @@
*/
@Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<String>();
+ public final List<String> definitions = new ArrayList<>();
/**
* System properties
@@ -85,7 +85,7 @@
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
" These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<String>(0);
+ public final List<String> sysprops = new ArrayList<>(0);
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index f2e3c61..1c694bd 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -59,7 +59,7 @@
/**
* --image path
- the full path to a .tar or .tar.gz path containing an HBase image.
+ * the full path to a .tar or .tar.gz path containing an HBase image.
*/
@Parameter(names = ARG_IMAGE,
description = "The full path to a .tar or .tar.gz path containing the application",
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
index d6cabf5..5b4cfdc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
@@ -21,7 +21,6 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-
@Parameters(commandNames = {SliderActions.ACTION_AM_SUICIDE},
commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE)
public class ActionAMSuicideArgs extends AbstractActionArgs {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
index b0f53f8..d05f10b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
@@ -20,15 +20,10 @@
import com.beust.jcommander.Parameter;
-/*
-
-@Parameters(commandNames = {SliderActions.ACTION_KILL_CONTAINER},
- commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
-*/
public class ActionEchoArgs extends AbstractActionArgs {
@Override
public String getActionName() {
- return SliderActions.ACTION_DESTROY;
+ return SliderActions.ACTION_ECHO;
}
@Parameter(names = {ARG_MESSAGE},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 0bc5792..739b5fc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -57,7 +57,7 @@
@Parameter(names = {ARG_COMPONENTS}, variableArity = true,
description = "Filter containers by component names (used with " +
ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);;
+ public Set<String> components = new HashSet<>(0);
/**
* Get the min #of params expected
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
index 3b69e74..1b73522 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -43,7 +43,7 @@
}
@Parameter(names = {ARG_ID},
- description = "ID of the container")
+ description = "ID of the application")
public String id;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
new file mode 100644
index 0000000..5ddccf6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_NODES},
+ commandDescription = SliderActions.DESCRIBE_ACTION_NODES)
+public class ActionNodesArgs extends AbstractActionArgs {
+
+ /**
+ * Instance for API use; on CLI the name is derived from {@link #getClusterName()}.
+ */
+ public String instance;
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_NODES;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the information")
+ public File outputFile;
+
+ @Parameter(names = {ARG_LABEL})
+ public String label = "";
+
+ @Parameter(names = {ARG_HEALTHY} )
+ public boolean healthy;
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ActionNodesArgs{");
+ sb.append("instance='").append(instance).append('\'');
+ sb.append(", outputFile=").append(outputFile);
+ sb.append(", label='").append(label).append('\'');
+ sb.append(", healthy=").append(healthy);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
index a7edb65..00178df 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
@@ -25,13 +25,14 @@
commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
public class ActionStatusArgs extends AbstractActionArgs {
+
@Override
public String getActionName() {
return SliderActions.ACTION_STATUS;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the configuration data")
+ description = "Output file for the status information")
public String output;
public String getOutput() {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
index 832e1cc..6ef51b2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
@@ -61,11 +61,11 @@
@Parameter(names={ARG_CONTAINERS}, variableArity = true,
description = "stop specific containers")
- public List<String> containers = new ArrayList<String>(0);
+ public List<String> containers = new ArrayList<>(0);
@Parameter(names={ARG_COMPONENTS}, variableArity = true,
description = "stop all containers of specific components")
- public List<String> components = new ArrayList<String>(0);
+ public List<String> components = new ArrayList<>(0);
@Parameter(names = {ARG_FORCE},
description = "force spec upgrade operation")
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
index 65ebc4b..3ef8e19 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
@@ -34,7 +34,7 @@
arity = 2,
description = "--addon <name> <folder or package>",
splitter = DontSplitArguments.class)
- public List<String> addonTuples = new ArrayList<String>(0);
+ public List<String> addonTuples = new ArrayList<>(0);
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 1f07de3..248e4c2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -37,7 +37,7 @@
@Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
description = ARG_OPTION + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<String>(0);
+ public List<String> optionTuples = new ArrayList<>(0);
/**
@@ -47,7 +47,7 @@
description = "Component option " + ARG_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<String>(0);
+ public List<String> compOptTriples = new ArrayList<>(0);
/**
* Resource Options
@@ -55,7 +55,7 @@
@Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> resOptionTuples = new ArrayList<String>(0);
+ public List<String> resOptionTuples = new ArrayList<>(0);
/**
@@ -65,7 +65,7 @@
description = "Component resource option " + ARG_RES_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> resCompOptTriples = new ArrayList<String>(0);
+ public List<String> resCompOptTriples = new ArrayList<>(0);
public Map<String, String> getOptionsMap() throws
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index aeb2979..12a2032 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -44,20 +44,13 @@
* create a 3-tuple
*/
public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<Object>(3);
+ List<Object> l = new ArrayList<>(3);
l.add(msg);
l.add(min);
l.add(max);
return l;
}
- /**
- * Create a tuple
- */
- public static List<Object> tuple(String msg, int min) {
- return triple(msg, min, min);
- }
-
public static void applyFileSystemBinding(String filesystemBinding,
Configuration conf) {
if (filesystemBinding != null) {
@@ -100,7 +93,7 @@
public static Map<String, String> convertTupleListToMap(String description,
List<String> list) throws
BadCommandArgumentsException {
- Map<String, String> results = new HashMap<String, String>();
+ Map<String, String> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 2 != 0) {
@@ -131,10 +124,9 @@
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, Map<String, String>> results =
- new HashMap<String, Map<String, String>>();
+ List<String> list) throws BadCommandArgumentsException {
+
+ Map<String, Map<String, String>> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 3 != 0) {
@@ -149,7 +141,7 @@
Map<String, String> roleMap = results.get(role);
if (roleMap == null) {
//demand create new role map
- roleMap = new HashMap<String, String>();
+ roleMap = new HashMap<>();
results.put(role, roleMap);
}
if (roleMap.get(key) != null) {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index c2fa09c..ca22c24 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -60,6 +60,7 @@
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
+ String ARG_HEALTHY= "--healthy";
String ARG_HELP = "--help";
String ARG_HOSTNAME = "--hostname";
String ARG_ID = "--id";
@@ -70,6 +71,7 @@
String ARG_KEYTABINSTALL = "--install";
String ARG_KEYTABDELETE = "--delete";
String ARG_KEYTABLIST = "--list";
+ String ARG_LABEL = "--label";
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
@@ -125,10 +127,14 @@
/**
- * Deprecated
+ * Deprecated: use ARG_COMPONENT
*/
@Deprecated
String ARG_ROLE = "--role";
+
+ /**
+ * Deprecated: use ARG_COMP_OPT
+ */
@Deprecated
String ARG_ROLEOPT = "--roleopt";
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 3c430f3..ea1448b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -66,6 +66,7 @@
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
+ private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
@@ -86,31 +87,32 @@
protected void addActionArguments() {
addActions(
- actionPackageArgs,
- actionKeytabArgs,
+ actionAMSuicideArgs,
actionBuildArgs,
- actionDependencyArgs,
- actionCreateArgs,
- actionListArgs,
- actionStatusArgs,
- actionRegistryArgs,
actionClientArgs,
- actionFlexArgs,
+ actionCreateArgs,
+ actionDependencyArgs,
+ actionDestroyArgs,
actionDiagnosticArgs,
+ actionExistsArgs,
+ actionFlexArgs,
actionFreezeArgs,
+ actionHelpArgs,
+ actionInstallKeytabArgs,
+ actionInstallPackageArgs,
+ actionKeytabArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionLookupArgs,
+ actionNodesArgs,
+ actionPackageArgs,
+ actionRegistryArgs,
+ actionResolveArgs,
+ actionStatusArgs,
actionThawArgs,
actionUpdateArgs,
actionUpgradeArgs,
- actionDestroyArgs,
- actionExistsArgs,
- actionLookupArgs,
- actionResolveArgs,
- actionKillContainerArgs,
- actionAMSuicideArgs,
- actionInstallPackageArgs,
- actionInstallKeytabArgs,
- actionVersionArgs,
- actionHelpArgs
+ actionVersionArgs
);
}
@@ -196,6 +198,10 @@
return actionListArgs;
}
+ public ActionNodesArgs getActionNodesArgs() {
+ return actionNodesArgs;
+ }
+
public ActionLookupArgs getActionLookupArgs() {
return actionLookupArgs;
}
@@ -286,6 +292,9 @@
} else if (SliderActions.ACTION_LOOKUP.equals(action)) {
bindCoreAction(actionLookupArgs);
+ } else if (SliderActions.ACTION_NODES.equals(action)) {
+ bindCoreAction(actionNodesArgs);
+
} else if (SliderActions.ACTION_REGISTRY.equals(action)) {
bindCoreAction(actionRegistryArgs);
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 7e02eec..162a87d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -91,12 +91,12 @@
return coreAction.getClusterName();
}
- public CommonArgs(String[] args) {
+ protected CommonArgs(String[] args) {
this.args = args;
commander = new JCommander(this);
}
- public CommonArgs(Collection args) {
+ protected CommonArgs(Collection args) {
List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
commander = new JCommander(this);
@@ -123,7 +123,7 @@
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
- helperMessage.append("\nUsage: slider " + commandOfInterest);
+ helperMessage.append("\nUsage: slider ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
@@ -171,7 +171,7 @@
/**
* Add a command
* @param name action
- * @param arg
+ * @param arg value
*/
protected void addAction(String name, Object arg) {
commander.addCommand(name, arg);
@@ -249,7 +249,7 @@
throw new BadCommandArgumentsException(badArgMsgBuilder.toString());
} catch (UsageException e) {
StringBuilder badArgMsgBuilder = new StringBuilder();
- badArgMsgBuilder.append(e.toString() + "\n");
+ badArgMsgBuilder.append(e.toString()).append("\n");
badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
throw new UsageException(badArgMsgBuilder.toString());
}
@@ -289,12 +289,6 @@
return coreAction.manager;
}
-
-// public String getRmAddress() {
-// return rmAddress;
-// }
-
-
public String getAction() {
return commander.getParsedCommand();
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
index 494258e..abda53f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -34,7 +34,7 @@
arity = 2,
description = "--component <name> <count>",
splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<String>(0);
+ public List<String> componentTuples = new ArrayList<>(0);
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
index 3225133..0344305 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -27,7 +27,7 @@
@Override
public List<String> split(String value) {
- List<String> list = new ArrayList<String>(1);
+ List<String> list = new ArrayList<>(1);
list.add(value);
return list;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 54d2746..1d4e0ea 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -39,6 +39,7 @@
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
+ String ACTION_NODES = "nodes";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
@@ -80,6 +81,7 @@
"List running Slider applications";
String DESCRIBE_ACTION_LOOKUP =
"look up a YARN application";
+ String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
String DESCRIBE_ACTION_REGISTRY =
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
index 0ccca0f..6380d0c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -21,6 +21,9 @@
import java.io.Serializable;
import java.util.Comparator;
+/**
+ * Some general comparators
+ */
public class Comparators {
public static class LongComparator implements Comparator<Long>, Serializable {
@@ -30,12 +33,20 @@
// need to comparisons with a diff greater than integer size
if (result < 0 ) {
return -1;
- } else if (result >0) {
+ } else if (result > 0) {
return 1;
}
return 0;
}
}
+public static class InvertedLongComparator implements Comparator<Long>, Serializable {
+ private static final LongComparator inner = new LongComparator();
+ @Override
+ public int compare(Long o1, Long o2) {
+ return -inner.compare(o1, o2);
+ }
+ }
+
/**
* Little template class to reverse any comparitor
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 1f97982..712eb75 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -155,6 +156,7 @@
* name of docker program
*/
public static final String DOCKER = "docker";
+ public static final int NODE_LIST_LIMIT = 10;
private SliderUtils() {
}
@@ -746,10 +748,7 @@
* through
*/
public static boolean filter(String value, String filter) {
- if (StringUtils.isEmpty(filter) || filter.equals(value)) {
- return false;
- }
- return true;
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
}
/**
@@ -1776,6 +1775,19 @@
return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
}
+ /**
+ * Get a string node label value from a node report
+ * @param report node report
+ * @return a single trimmed label or ""
+ */
+ public static String extractNodeLabel(NodeReport report) {
+ Set<String> newlabels = report.getNodeLabels();
+ if (newlabels != null && !newlabels.isEmpty()) {
+ return newlabels.iterator().next().trim();
+ } else {
+ return "";
+ }
+ }
/**
* Callable for async/scheduled halt
@@ -2028,10 +2040,10 @@
errorText.append("No native IO library. ");
}
try {
- String path = Shell.getQualifiedBinPath("winutils.exe");
+ String path = Shell.getQualifiedBinPath(WINUTILS);
log.debug("winutils is at {}", path);
} catch (IOException e) {
- errorText.append("No WINUTILS.EXE. ");
+ errorText.append("No " + WINUTILS);
log.warn("No winutils: {}", e, e);
}
try {
@@ -2334,6 +2346,7 @@
public static String getClientConfigPath() {
URL path = ConfigHelper.class.getClassLoader().getResource(
SliderKeys.SLIDER_CLIENT_XML);
+ Preconditions.checkNotNull(path, "Failed to locate resource " + SliderKeys.SLIDER_CLIENT_XML);
return path.toString();
}
@@ -2470,7 +2483,7 @@
* @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
*/
public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? +1 : ((x == y) ? 0 : -1);
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
}
public static String getSystemEnv(String property) {
@@ -2482,7 +2495,8 @@
}
public static String requestToString(AMRMClient.ContainerRequest request) {
- StringBuffer buffer = new StringBuffer(request.toString());
+ Preconditions.checkArgument(request != null, "Null request");
+ StringBuilder buffer = new StringBuilder(request.toString());
buffer.append("; ");
buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; ");
String labels = request.getNodeLabelExpression();
@@ -2491,9 +2505,15 @@
}
List<String> nodes = request.getNodes();
if (nodes != null) {
- buffer.append("Nodes = [")
- .append(join(nodes, ", ", false))
- .append("]; ");
+ buffer.append("Nodes = [ ");
+ int size = nodes.size();
+ for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) {
+ buffer.append(nodes.get(i)).append(' ');
+ }
+ if (size > NODE_LIST_LIMIT) {
+ buffer.append(String.format("...(total %d entries)", size));
+ }
+ buffer.append("]; ");
}
List<String> racks = request.getRacks();
if (racks != null) {
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index bc116e7..58896ee 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -432,5 +432,14 @@
public void setComponentOpt(String role, String option, int val) {
setComponentOpt(role, option, Integer.toString(val));
}
+ /**
+ * Set a long role option, creating the role if necessary
+ * @param role role name
+ * @param option option name
+ * @param val long value
+ */
+ public void setComponentOpt(String role, String option, long val) {
+ setComponentOpt(role, option, Long.toString(val));
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index 57b8965..dbaa981 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -19,11 +19,8 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
import java.util.ArrayList;
import java.util.List;
@@ -104,56 +101,4 @@
return argumentList;
}
- public boolean addConfOption(Configuration conf, String key) {
- String val = conf.get(key);
- return defineIfSet(key, val);
- }
-
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI
- * @param key key
- * @param val value
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add(Arguments.ARG_DEFINE, key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index f8ea4ee..69b937d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -55,17 +55,13 @@
public UserGroupInformation setupUGI() {
UserGroupInformation user =
UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr =
- container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress =
- NetUtils.createSocketAddr(cmIpPortStr);
+ String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
- org.apache.hadoop.yarn.api.records.Token containerToken =
- container.getContainerToken();
+ org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
if (containerToken != null) {
Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken,
- cmAddress);
+ ConverterUtils.convertFromYarn(containerToken, cmAddress);
user.addToken(token);
}
return user;
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 0b3fa10..dcb322e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -20,11 +20,15 @@
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
/**
- * Command line builder purely for the Java CLI
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
*/
public class JavaCommandLineBuilder extends CommandLineBuilder {
@@ -80,4 +84,68 @@
sysprop("java.awt.headless", "true");
return this;
}
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String...keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
index 60717f6..9759205 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
@@ -101,7 +101,7 @@
* Make the persistent directory
* @throws IOException IO failure
*/
- private void mkPersistDir() throws IOException {
+ public void mkPersistDir() throws IOException {
coreFS.getFileSystem().mkdirs(persistDir);
}
@@ -165,13 +165,15 @@
* Acquire the writelock
* @throws IOException IO
* @throws LockAcquireFailedException
- * @throws FileNotFoundException if the target dir does not exist
+ * @throws FileNotFoundException if the target dir does not exist.
*/
@VisibleForTesting
boolean acquireReadLock() throws FileNotFoundException,
IOException,
LockAcquireFailedException {
if (!coreFS.getFileSystem().exists(persistDir)) {
+ // the dir is not there, so the data is not there, so there
+ // is nothing to read
throw new FileNotFoundException(persistDir.toString());
}
long now = System.currentTimeMillis();
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index c5908bb..4f60c06 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -91,10 +91,11 @@
*/
public T fromFile(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
+ File absoluteFile = jsonFile.getAbsoluteFile();
try {
- return mapper.readValue(jsonFile, classType);
+ return mapper.readValue(absoluteFile, classType);
} catch (IOException e) {
- log.error("Exception while parsing json file {}: {}", jsonFile, e);
+ log.error("Exception while parsing json file {}", absoluteFile, e);
throw e;
}
}
@@ -106,7 +107,6 @@
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
-/* JDK7
public T fromResource(String resource)
throws IOException, JsonParseException, JsonMappingException {
try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
@@ -115,34 +115,9 @@
}
return (T) (mapper.readValue(resStream, classType));
} catch (IOException e) {
- log.error("Exception while parsing json resource {}: {}", resource, e);
+ log.error("Exception while parsing json resource {}", resource, e);
throw e;
}
- }*/
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public synchronized T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- InputStream resStream = null;
- try {
- resStream = this.getClass().getResourceAsStream(resource);
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return (T) (mapper.readValue(resStream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json resource {}: {}", resource, e);
- throw e;
- } finally {
- IOUtils.closeStream(resStream);
- }
}
/**
@@ -155,7 +130,7 @@
try {
return (T) (mapper.readValue(stream, classType));
} catch (IOException e) {
- log.error("Exception while parsing json input stream: {}", e);
+ log.error("Exception while parsing json input stream", e);
throw e;
} finally {
IOUtils.closeStream(stream);
@@ -201,7 +176,7 @@
FSDataInputStream dataInputStream = fs.open(path);
int count = dataInputStream.read(b);
if (count != len) {
- throw new EOFException("Read finished prematurely");
+ throw new EOFException("Read of " + path +" finished prematurely");
}
return fromBytes(b);
}
@@ -230,9 +205,9 @@
*/
public void save(T instance, File file) throws
IOException {
- writeJsonAsBytes(instance, new FileOutputStream(file));
+ writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
}
-
+
/**
* Write the json as bytes -then close the file
* @param dataOutputStream an outout stream that will always be closed
@@ -251,7 +226,6 @@
}
}
-
/**
* Convert an object to a JSON string
* @param instance instance to convert
@@ -265,6 +239,5 @@
mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
return mapper.writeValueAsString(instance);
}
-
-
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7cba840..c701a55 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -81,7 +81,7 @@
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
- public AbstractProviderService(String name) {
+ protected AbstractProviderService(String name) {
super(name);
setStopIfNoChildServicesAtStartup(false);
}
@@ -343,12 +343,6 @@
return details;
}
-
- protected String getInfoAvoidingNull(ClusterDescription clusterDesc, String key) {
- String value = clusterDesc.getInfo(key);
-
- return null == value ? "N/A" : value;
- }
@Override
public void buildEndpointDetails(Map<String, String> details) {
@@ -363,10 +357,8 @@
if (!urls.isEmpty()) {
details.put(endpoint.api, urls.get(0).toString());
}
- } catch (InvalidRecordException ignored) {
+ } catch (InvalidRecordException | MalformedURLException ignored) {
// Ignored
- } catch (MalformedURLException ignored) {
- // ignored
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index 4e85a93..71d7566 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -27,7 +27,12 @@
/**
* Default value: history used, anti-affinity hinted at on rebuild/flex up
*/
- public static final int DEFAULT = 0;
+ public static final int NONE = 0;
+
+ /**
+ * Default value: history used, anti-affinity hinted at on rebuild/flex up
+ */
+ public static final int DEFAULT = NONE;
/**
* Strict placement: when asking for an instance for which there is
@@ -41,7 +46,7 @@
public static final int NO_DATA_LOCALITY = 2;
/**
- * Anti-affinity is mandatory. This is not supported in YARN
+ * Anti-affinity is mandatory.
*/
public static final int ANTI_AFFINITY_REQUIRED = 4;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 3009f50..1b95b42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -32,13 +32,15 @@
public int placementPolicy;
public int nodeFailureThreshold;
public final long placementTimeoutSeconds;
+ public final String labelExpression;
public ProviderRole(String name, int id) {
this(name,
id,
PlacementPolicy.DEFAULT,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+ ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
}
/**
@@ -49,18 +51,20 @@
* @param nodeFailureThreshold threshold for node failures (within a reset interval)
* after which a node failure is considered an app failure
* @param placementTimeoutSeconds for lax placement, timeout in seconds before
- * a relaxed placement request is generated.
+ * @param labelExpression label expression for requests; may be null
*/
public ProviderRole(String name,
int id,
int policy,
int nodeFailureThreshold,
- long placementTimeoutSeconds) {
+ long placementTimeoutSeconds,
+ String labelExpression) {
this.name = name;
this.id = id;
this.placementPolicy = policy;
this.nodeFailureThreshold = nodeFailureThreshold;
this.placementTimeoutSeconds = placementTimeoutSeconds;
+ this.labelExpression = labelExpression;
}
@Override
@@ -83,11 +87,14 @@
@Override
public String toString() {
- return "ProviderRole {" +
- "name='" + name + '\'' +
- ", id=" + id +
- ", policy=" + placementPolicy +
- ", nodeFailureThreshold=" + nodeFailureThreshold +
- '}';
+ final StringBuilder sb = new StringBuilder("ProviderRole{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", id=").append(id);
+ sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold);
+ sb.append(", placementTimeoutSeconds=").append(placementTimeoutSeconds);
+ sb.append(", labelExpression='").append(labelExpression).append('\'');
+ sb.append('}');
+ return sb.toString();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 9bd4dc9..e5430f2 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -90,7 +90,7 @@
new ProviderRole(COMPONENT_AM, KEY_AM,
PlacementPolicy.EXCLUDE_FROM_FLEXING,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- 0);
+ 0, "");
/**
* Initialize role list
@@ -218,11 +218,12 @@
* @param instanceDescription
* @param providerResources
* @throws IOException
+ * @throws BadConfigException if there's no keytab and it is explicitly required.
*/
protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem,
AggregateConf instanceDescription,
Map<String, LocalResource> providerResources)
- throws IOException {
+ throws IOException, BadConfigException {
if (UserGroupInformation.isSecurityEnabled()) {
String keytabPathOnHost = instanceDescription.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM).get(
@@ -243,10 +244,16 @@
providerResources.put(SliderKeys.KEYTAB_DIR + "/" +
amKeytabName, keytabRes);
} else {
- log.warn("No keytab file was found at {}. The AM will be "
- + "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
- + "operational beyond 24 hours.", keytabPath);
+ log.warn("No keytab file was found at {}.", keytabPath);
+ if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ throw new BadConfigException("No keytab file was found at %s.", keytabPath);
+
+ } else {
+ log.warn("The AM will be "
+ + "started without a kerberos authenticated identity. "
+ + "The application is therefore not guaranteed to remain "
+ + "operational beyond 24 hours.");
+ }
}
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index cee7a97..e382058 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -88,12 +88,11 @@
MapOperations resourceComponent,
MapOperations appComponent,
Path containerTmpDirPath) throws IOException, SliderException {
-
}
@Override
public List<ProviderRole> getRoles() {
- return new ArrayList<ProviderRole>(0);
+ return new ArrayList<>(0);
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
index 50b5dad..5d52441 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.slider.server.appmaster.state.AbstractClusterServices;
public class ProtobufClusterServices extends AbstractClusterServices {
@@ -27,4 +28,9 @@
public Resource newResource() {
return Records.newRecord(Resource.class);
}
+
+ @Override
+ public Resource newResource(int memory, int cores) {
+ return Resources.createResource(memory, cores);
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index fd9253e..18d5bfa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -57,6 +57,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -64,6 +65,7 @@
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -84,6 +86,7 @@
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.SliderClusterAPI;
+import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
@@ -143,6 +146,7 @@
import org.apache.slider.server.appmaster.rpc.SliderIPCService;
import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
@@ -168,6 +172,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
@@ -210,8 +215,7 @@
*/
protected static final Logger LOG_YARN = log;
- public static final String SERVICE_CLASSNAME_SHORT =
- "SliderAppMaster";
+ public static final String SERVICE_CLASSNAME_SHORT = "SliderAppMaster";
public static final String SERVICE_CLASSNAME =
"org.apache.slider.server.appmaster." + SERVICE_CLASSNAME_SHORT;
@@ -356,16 +360,6 @@
private RegistryOperations registryOperations;
/**
- * Record of the max no. of cores allowed in this cluster
- */
- private int containerMaxCores;
-
- /**
- * limit container memory
- */
- private int containerMaxMemory;
-
- /**
* The stop request received...the exit details are extracted
* from this
*/
@@ -415,6 +409,12 @@
*/
private boolean securityEnabled;
private ContentCache contentCache;
+ private SliderYarnClientImpl yarnClient;
+
+ /**
+ * resource limits
+ */
+ private Resource maximumResourceCapability;
/**
* Service Constructor
@@ -437,10 +437,8 @@
// Load in the server configuration - if it is actually on the Classpath
URL serverXmlUrl = ConfigHelper.getResourceUrl(SLIDER_SERVER_XML);
if (serverXmlUrl != null) {
-
log.info("Loading {} at {}", SLIDER_SERVER_XML, serverXmlUrl);
- Configuration serverConf =
- ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
+ Configuration serverConf = ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
ConfigHelper.mergeConfigurations(customConf, serverConf,
SLIDER_SERVER_XML, true);
}
@@ -496,8 +494,7 @@
metrics.registerAll(new GarbageCollectorMetricSet());
*/
- contentCache = ApplicationResouceContentCacheFactory.createContentCache(
- stateForProviders);
+ contentCache = ApplicationResouceContentCacheFactory.createContentCache(stateForProviders);
executorService = new WorkflowExecutorService<>("AmExecutor",
Executors.newFixedThreadPool(2,
@@ -505,7 +502,7 @@
addService(executorService);
addService(actionQueues);
-
+
//init all child services
super.serviceInit(conf);
}
@@ -536,8 +533,7 @@
* @param args argument list
*/
@Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
@@ -564,8 +560,7 @@
//dump the system properties if in debug mode
if (log.isDebugEnabled()) {
- log.debug("System properties:\n" +
- SliderUtils.propertiesToString(System.getProperties()));
+ log.debug("System properties:\n" + SliderUtils.propertiesToString(System.getProperties()));
}
//choose the action
@@ -630,13 +625,11 @@
Configuration serviceConf = getConfig();
- securityConfiguration = new SecurityConfiguration(
- serviceConf, instanceDefinition, clustername);
+ securityConfiguration = new SecurityConfiguration(serviceConf, instanceDefinition, clustername);
// obtain security state
securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
- instanceDefinition.getAppConfOperations().set(
- KEY_SECURITY_ENABLED, securityEnabled);
+ instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled);
// triggers resolution and snapshotting for agent
appState.setInitialInstanceDefinition(instanceDefinition);
@@ -654,44 +647,53 @@
InternalKeys.INTERNAL_PROVIDER_NAME);
log.info("Cluster provider type is {}", providerType);
SliderProviderFactory factory =
- SliderProviderFactory.createSliderProviderFactory(
- providerType);
+ SliderProviderFactory.createSliderProviderFactory(providerType);
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
- providerRMOperationHandler =
- new ProviderNotifyingOperationHandler(providerService);
+ providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService);
// create a slider AM provider
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ log.warn("Yarn RM address was unbound; attempting to fix up");
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container
*/
appMasterContainerID = ConverterUtils.toContainerId(
- SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name()));
+ SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
appAttemptID = appMasterContainerID.getApplicationAttemptId();
ApplicationId appid = appAttemptID.getApplicationId();
log.info("AM for ID {}", appid.getId());
- appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID,
- appMasterContainerID.toString());
- appInformation.put(StatusKeys.INFO_AM_APP_ID,
- appid.toString());
- appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
- appAttemptID.toString());
+ appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString());
+ appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString());
+ appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString());
Map<String, String> envVars;
List<Container> liveContainers;
+ List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
+ log.info("Yarn node report count: {}", nodeReports.size());
+
/*
* It is critical this section is synchronized, to stop async AM events
* arriving while registering a restarting AM.
@@ -715,7 +717,7 @@
if (securityEnabled) {
// fix up the ACLs if they are not set
- String acls = getConfig().get(KEY_PROTOCOL_ACL);
+ String acls = serviceConf.get(KEY_PROTOCOL_ACL);
if (acls == null) {
getConfig().set(KEY_PROTOCOL_ACL, "*");
}
@@ -731,8 +733,7 @@
appMasterHostname = rpcServiceAddress.getHostName();
appMasterRpcPort = rpcServiceAddress.getPort();
appMasterTrackingUrl = null;
- log.info("AM Server is listening at {}:{}", appMasterHostname,
- appMasterRpcPort);
+ log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort);
appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
@@ -776,24 +777,27 @@
// *****************************************************
log.info("Connecting to RM at {},address tracking URL={}",
appMasterRpcPort, appMasterTrackingUrl);
- amRegistrationData = asyncRMClient
- .registerApplicationMaster(appMasterHostname,
+ amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname,
appMasterRpcPort,
appMasterTrackingUrl);
- Resource maxResources =
- amRegistrationData.getMaximumResourceCapability();
- containerMaxMemory = maxResources.getMemory();
- containerMaxCores = maxResources.getVirtualCores();
- appState.setContainerLimits(maxResources.getMemory(),
- maxResources.getVirtualCores());
+ maximumResourceCapability = amRegistrationData.getMaximumResourceCapability();
+
+ int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ // validate scheduler vcores allocation setting
+ int minCores = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ int maxMemory = maximumResourceCapability.getMemory();
+ int maxCores = maximumResourceCapability.getVirtualCores();
+ appState.setContainerLimits(minMemory,maxMemory, minCores, maxCores );
// build the handler for RM request/release operations; this uses
// the max value as part of its lookup
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maxResources);
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability);
// set the RM-defined maximum cluster values
- appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
- appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
+ appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores));
+ appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory));
processAMCredentials(securityConfiguration);
@@ -837,15 +841,18 @@
Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);
//build the instance
- appState.buildInstance(instanceDefinition,
- serviceConf,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation,
- providerService.createContainerReleaseSelector());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = instanceDefinition;
+ binding.serviceConfig = serviceConf;
+ binding.publishedProviderConf = providerConf;
+ binding.roles = providerRoles;
+ binding.fs = fs.getFileSystem();
+ binding.historyPath = historyDir;
+ binding.liveContainers = liveContainers;
+ binding.applicationInfo = appInformation;
+ binding.releaseSelector = providerService.createContainerReleaseSelector();
+ binding.nodeReports = nodeReports;
+ appState.buildInstance(binding);
providerService.rebuildContainerDetails(liveContainers,
instanceDefinition.getName(), appState.getRolePriorityMap());
@@ -1877,7 +1884,15 @@
LOG_YARN.info("onNodesUpdated({})", updatedNodes.size());
log.info("Updated nodes {}", updatedNodes);
// Check if any nodes are lost or revived and update state accordingly
- appState.onNodesUpdated(updatedNodes);
+
+ AppState.NodeUpdatedOutcome outcome = appState.onNodesUpdated(updatedNodes);
+ if (!outcome.operations.isEmpty()) {
+ execute(outcome.operations);
+ }
+ // trigger a review if the cluster changed
+ if (outcome.clusterChanged) {
+ reviewRequestAndReleaseNodes("nodes updated");
+ }
}
/**
@@ -1937,15 +1952,6 @@
/* =================================================================== */
/**
- * Update the cluster description with anything interesting
- */
- public synchronized ClusterDescription updateClusterStatus() {
- Map<String, String> providerStatus = providerService.buildProviderStatus();
- assert providerStatus != null : "null provider status";
- return appState.refreshClusterStatus(providerStatus);
- }
-
- /**
* Launch the provider service
*
* @param instanceDefinition definition of the service
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 1aa9088..7446e82 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -77,7 +77,7 @@
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- List<AbstractRMOperation> opsList = new LinkedList<AbstractRMOperation>();
+ List<AbstractRMOperation> opsList = new LinkedList<>();
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 146dea4..34acade 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -58,20 +58,20 @@
* Immediate actions.
*/
public final BlockingDeque<AsyncAction> actionQueue =
- new LinkedBlockingDeque<AsyncAction>();
+ new LinkedBlockingDeque<>();
/**
* Actions to be scheduled in the future
*/
- public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<AsyncAction>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
/**
* Map of renewing actions by name ... this is to allow them to
* be cancelled by name
*/
private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
- = new ConcurrentHashMap<String, RenewingAction<? extends AsyncAction>>();
-
+ = new ConcurrentHashMap<>();
+
/**
* Create a queue instance with a single thread executor
*/
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
new file mode 100644
index 0000000..82bcd3a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+/**
+ * A metric which takes a predicate and returns 1 if the predicate evaluates
+ * to true. The predicate is evaluated whenever the metric is read.
+ */
+public class BoolMetricPredicate implements Metric, Gauge<Integer> {
+
+ private final Eval predicate;
+
+ public BoolMetricPredicate(Eval predicate) {
+ this.predicate = predicate;
+ }
+
+ @Override
+ public Integer getValue() {
+ return predicate.eval() ? 1: 0;
+ }
+
+ public interface Eval {
+ boolean eval();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
index 08f61ec..c93467b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
@@ -24,69 +24,75 @@
import java.util.concurrent.atomic.AtomicLong;
/**
- * This is a long which acts as a gauge
+ * This is a {@link AtomicLong} which acts as a metrics gauge: its state can be exposed as
+ * a metrics.
+ * It also exposes some of the same method names as the Codahale Counter class, so that
+ * it's easy to swap in.
+ *
*/
-public class LongGauge implements Metric, Gauge<Long> {
-
- private final AtomicLong value;
+public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
/**
* Instantiate
* @param val current value
*/
public LongGauge(long val) {
- this.value = new AtomicLong(val);
+ super(val);
}
+ /**
+ * Instantiate with value 0
+ */
public LongGauge() {
this(0);
}
/**
- * Set to a new value.
- * @param val value
+ * Get the value as a metric
+ * @return current value
*/
- public synchronized void set(long val) {
- value.set(val);
- }
-
- public void inc() {
- inc(1);
- }
-
- public void dec() {
- dec(1);
- }
-
- public synchronized void inc(int delta) {
- set(value.get() + delta);
- }
-
- public synchronized void dec(int delta) {
- set(value.get() - delta);
- }
-
- public long get() {
- return value.get();
- }
-
@Override
public Long getValue() {
return get();
}
- @Override
- public String toString() {
- return value.toString();
+ /**
+ * Method from {@Code counter}; used here for drop-in replacement
+ * without any recompile
+ * @return current value
+ */
+ public Long getCount() {
+ return get();
}
- @Override
- public int hashCode() {
- return value.hashCode();
+ /**
+ * {@code ++}
+ */
+ public void inc() {
+ incrementAndGet();
}
- @Override
- public boolean equals(Object obj) {
- return value.equals(obj);
+ /**
+ * {@code --}
+ */
+ public void dec() {
+ decrementAndGet();
+ }
+
+ /**
+ * Decrement to the floor of 0. Operations in parallel may cause confusion here,
+ * but it will still never go below zero
+ * @param delta delta
+ * @return the current value
+ */
+ public long decToFloor(long delta) {
+ long l = get();
+ long r = l - delta;
+ if (r < 0) {
+ r = 0;
+ }
+ // if this fails, the decrement has been lost
+ compareAndSet(l, r);
+ return get();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
new file mode 100644
index 0000000..1de7345
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+/**
+ * A metric which takes a function to generate a long value.
+ * The function is evaluated whenever the metric is read.
+ */
+public class LongMetricFunction implements Metric, Gauge<Long> {
+
+ private final Eval function;
+
+ public LongMetricFunction(Eval function) {
+ this.function = function;
+ }
+
+ @Override
+ public Long getValue() {
+ return function.eval();
+ }
+
+ public interface Eval {
+ long eval();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
index cced42a..37a8935 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -20,9 +20,12 @@
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.health.HealthCheckRegistry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -33,7 +36,8 @@
* Class for all metrics and monitoring
*/
public class MetricsAndMonitoring extends CompositeService {
-
+ protected static final Logger log =
+ LoggerFactory.getLogger(MetricsAndMonitoring.class);
public MetricsAndMonitoring(String name) {
super(name);
}
@@ -52,13 +56,15 @@
private final Map<String, MeterAndCounter> meterAndCounterMap
= new ConcurrentHashMap<>();
+ private final List<MetricSet> metricSets = new ArrayList<>();
+
/**
* List of recorded events
*/
private final List<RecordedEvent> eventHistory = new ArrayList<>(100);
public static final int EVENT_LIMIT = 1000;
-
+
public MetricRegistry getMetrics() {
return metrics;
}
@@ -74,6 +80,14 @@
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ for (MetricSet set : metricSets) {
+ unregister(set);
+ }
+ }
+
public MeterAndCounter getMeterAndCounter(String name) {
return meterAndCounterMap.get(name);
}
@@ -144,5 +158,38 @@
public synchronized List<RecordedEvent> cloneEventHistory() {
return new ArrayList<>(eventHistory);
}
+
+ /**
+ * Add a metric set for registering and deregistration on service stop
+ * @param metricSet metric set
+ */
+ public void addMetricSet(MetricSet metricSet) {
+ metricSets.add(metricSet);
+ metrics.registerAll(metricSet);
+ }
+
+ /**
+ * add a metric set, giving each entry a prefix
+ * @param prefix prefix (a trailing "." is automatically added)
+ * @param metricSet the metric set to register
+ */
+ public void addMetricSet(String prefix, MetricSet metricSet) {
+ addMetricSet(new PrefixedMetricsSet(prefix, metricSet));
+ }
+
+ /**
+ * Unregister a metric set; robust
+ * @param metricSet metric set to unregister
+ */
+ public void unregister(MetricSet metricSet) {
+ for (String s : metricSet.getMetrics().keySet()) {
+ try {
+ metrics.remove(s);
+ } catch (IllegalArgumentException e) {
+ // log but continue
+ log.info("Exception when trying to unregister {}", s, e);
+ }
+ }
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
index f8646bf..864a1cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
@@ -19,7 +19,9 @@
package org.apache.slider.server.appmaster.management;
import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.Slf4jReporter;
import com.google.common.base.Preconditions;
@@ -29,6 +31,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
@@ -79,7 +84,7 @@
JmxReporter jmxReporter;
jmxReporter = JmxReporter.forRegistry(metrics).build();
jmxReporter.start();
- addService(new ClosingService<JmxReporter>(jmxReporter));
+ addService(new ClosingService<>(jmxReporter));
// Ganglia
@@ -128,7 +133,7 @@
.convertDurationsTo(TimeUnit.MILLISECONDS)
.build();
reporter.start(interval, TimeUnit.MINUTES);
- addService(new ClosingService<ScheduledReporter>(reporter));
+ addService(new ClosingService<>(reporter));
summary.append(String.format(", SLF4J to log %s interval=%d",
logName, interval));
}
@@ -136,8 +141,11 @@
log.info(reportingDetails);
}
+
@Override
public String toString() {
return super.toString() + " " + reportingDetails;
}
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
index 31a82a3..fa6bfc0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
@@ -53,4 +53,6 @@
*/
public static final String CONTAINERS_START_FAILED = "containers.start-failed";
+ public static final String PREFIX_SLIDER_ROLES = "slider.roles.";
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
new file mode 100644
index 0000000..e9ad46a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * From an existing metrics set, generate a new metrics set with the
+ * prefix in front of every key.
+ *
+ * The prefix is added directly: if you want a '.' between prefix and metric
+ * keys, include it in the prefix.
+ */
+public class PrefixedMetricsSet implements MetricSet {
+
+ private final String prefix;
+ private final MetricSet source;
+
+ public PrefixedMetricsSet(String prefix, MetricSet source) {
+ this.prefix = prefix;
+ this.source = source;
+ }
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> sourceMetrics = source.getMetrics();
+ Map<String, Metric> metrics = new HashMap<>(sourceMetrics.size());
+ for (Map.Entry<String, Metric> entry : sourceMetrics.entrySet()) {
+ metrics.put(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ return metrics;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
index b5b27c5..ed3f197 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
@@ -26,5 +26,5 @@
* @param handler handler to perform the execution
*/
public abstract void execute(RMOperationHandlerActions handler);
-
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
index 08eb5bc..d7673d3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -18,7 +18,9 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
/**
* Cancel a container request
@@ -28,6 +30,7 @@
private final AMRMClient.ContainerRequest request;
public CancelSingleRequest(AMRMClient.ContainerRequest request) {
+ Preconditions.checkArgument(request != null, "Null container request");
this.request = request;
}
@@ -42,7 +45,9 @@
@Override
public String toString() {
- return "cancel single request for container at " + request.getPriority().toString();
+ return "Cancel container request"
+ + " for :" + ContainerPriority.toString(request.getPriority())
+ + " request " + request;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
index 46da536..4271d50 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
@@ -18,13 +18,16 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
public class ContainerReleaseOperation extends AbstractRMOperation {
private final ContainerId containerId;
public ContainerReleaseOperation(ContainerId containerId) {
+ Preconditions.checkArgument(containerId != null, "Null containerId");
this.containerId = containerId;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
index 6685b2a..e29ddd0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
@@ -54,7 +54,9 @@
@Override
public String toString() {
- return "request container for " + ContainerPriority.toString(getPriority())
+ return "request container for role "
+ + ContainerPriority.toString(getPriority())
+ + " request " + request
+ " relaxLocality=" + getRelaxLocality();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
index bb8f512..fda23aa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
@@ -31,6 +31,7 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.ServiceNotReadyException;
@@ -423,14 +424,12 @@
@Override
public Messages.GetLiveNodesResponseProto getLiveNodes(Messages.GetLiveNodesRequestProto request)
throws IOException {
- Map<String, NodeInformation> infoMap =
- (Map<String, NodeInformation>) cache.lookupWithIOE(LIVE_NODES);
+ NodeInformationList info = (NodeInformationList) cache.lookupWithIOE(LIVE_NODES);
Messages.GetLiveNodesResponseProto.Builder builder =
Messages.GetLiveNodesResponseProto.newBuilder();
- for (Map.Entry<String, NodeInformation> entry : infoMap.entrySet()) {
- builder.addNames(entry.getKey());
- builder.addNodes(marshall(entry.getValue()));
+ for (NodeInformation nodeInformation : info) {
+ builder.addNodes(marshall(nodeInformation));
}
return builder.build();
}
@@ -448,7 +447,6 @@
}
}
-
@Override
public Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException {
return lookupAggregateConf(MODEL_DESIRED);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
index 27e25f9..54f384b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
@@ -18,16 +18,44 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
/**
* Cluster services offered by the YARN infrastructure.
*/
public abstract class AbstractClusterServices {
+
+ private final DefaultResourceCalculator
+ defaultResourceCalculator = new DefaultResourceCalculator();
+
/**
* Create a resource for requests
* @return a resource which can be built up.
*/
public abstract Resource newResource();
+ public abstract Resource newResource(int memory, int cores);
+
+ /**
+ * Normalise memory, CPU and other resources according to the YARN AM-supplied
+ * values and the resource calculator in use (currently hard-coded to the
+ * {@link DefaultResourceCalculator}.
+ * Those resources which aren't normalized (currently: CPU) are left
+ * as is.
+ * @param resource resource requirements of a role
+ * @param minR minimum values of this queue
+ * @param maxR max values of this queue
+ * @return a normalized value.
+ */
+ public Resource normalize(Resource resource, Resource minR, Resource maxR) {
+ Preconditions.checkArgument(resource != null, "null resource");
+ Preconditions.checkArgument(minR != null, "null minR");
+ Preconditions.checkArgument(maxR != null, "null maxR");
+
+ Resource normalize = defaultResourceCalculator.normalize(resource, minR,
+ maxR, minR);
+ return newResource(normalize.getMemory(), resource.getVirtualCores());
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index eadb1dc..4b4d7a7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,12 +18,11 @@
package org.apache.slider.server.appmaster.state;
-import com.codahale.metrics.Counter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -36,6 +35,7 @@
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
@@ -46,6 +46,7 @@
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -64,6 +65,7 @@
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.LongGauge;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.MetricsConstants;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
@@ -85,9 +87,34 @@
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.slider.api.ResourceKeys.*;
-import static org.apache.slider.api.RoleKeys.*;
-import static org.apache.slider.api.StateValues.*;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_PRIORITY;
+import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_LABEL_EXPRESSION;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.PLACEMENT_ESCALATE_DELAY;
+import static org.apache.slider.api.ResourceKeys.YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.YARN_LABEL_EXPRESSION;
+import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.YARN_RESOURCE_MAX;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_RECENTLY_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_NODE_FAILED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_PENDING_AA_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_PREEMPTED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES;
+import static org.apache.slider.api.StateValues.STATE_CREATED;
+import static org.apache.slider.api.StateValues.STATE_DESTROYED;
+import static org.apache.slider.api.StateValues.STATE_LIVE;
+import static org.apache.slider.api.StateValues.STATE_SUBMITTED;
/**
@@ -142,7 +169,7 @@
private ConfTreeOperations resourcesSnapshot;
private ConfTreeOperations appConfSnapshot;
private ConfTreeOperations internalsSnapshot;
-
+
/**
* This is the status, the live model
*/
@@ -152,7 +179,7 @@
* Metadata provided by the AM for use in filling in status requests
*/
private Map<String, String> applicationInfo;
-
+
/**
* Client properties created via the provider -static for the life
* of the application
@@ -196,33 +223,33 @@
/**
* Counter for completed containers ( complete denotes successful or failed )
*/
- private final Counter completedContainerCount = new Counter();
+ private final LongGauge completedContainerCount = new LongGauge();
/**
* Count of failed containers
*/
- private final Counter failedContainerCount = new Counter();
+ private final LongGauge failedContainerCount = new LongGauge();
/**
* # of started containers
*/
- private final Counter startedContainers = new Counter();
+ private final LongGauge startedContainers = new LongGauge();
/**
* # of containers that failed to start
*/
- private final Counter startFailedContainerCount = new Counter();
+ private final LongGauge startFailedContainerCount = new LongGauge();
/**
* Track the number of surplus containers received and discarded
*/
- private final Counter surplusContainers = new Counter();
-
+ private final LongGauge surplusContainers = new LongGauge();
/**
- * Track the number of requested Containers
+ * Track the number of requested containers.
+ * Important: this does not include AA requests which are yet to be issued.
*/
- private final Counter outstandingContainerRequests = new Counter();
+ private final LongGauge outstandingContainerRequests = new LongGauge();
/**
* Map of requested nodes. This records the command used to start it,
@@ -265,28 +292,32 @@
/**
- * Record of the max no. of cores allowed in this cluster
+ * limits of container core numbers in this queue
*/
private int containerMaxCores;
+ private int containerMinCores;
/**
- * limit container memory
+ * limits of container memory in this queue
*/
private int containerMaxMemory;
-
+ private int containerMinMemory;
+
private RoleHistory roleHistory;
private Configuration publishedProviderConf;
private long startTimeThreshold;
-
+
private int failureThreshold = 10;
private int nodeFailureThreshold = 3;
-
+
private String logServerURL = "";
/**
* Selector of containers to release; application wide.
*/
private ContainerReleaseSelector containerReleaseSelector;
+ private Resource minResource;
+ private Resource maxResource;
/**
* Create an instance
@@ -295,6 +326,8 @@
*/
public AppState(AbstractClusterServices recordFactory,
MetricsAndMonitoring metricsAndMonitoring) {
+ Preconditions.checkArgument(recordFactory != null, "null recordFactory");
+ Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring");
this.recordFactory = recordFactory;
this.metricsAndMonitoring = metricsAndMonitoring;
@@ -307,7 +340,7 @@
register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount);
}
- private void register(String name, Counter counter) {
+ private void register(String name, Metric counter) {
this.metricsAndMonitoring.getMetrics().register(
MetricRegistry.name(AppState.class, name), counter);
}
@@ -374,7 +407,6 @@
return completedContainers;
}
-
public Map<ContainerId, RoleInstance> getFailedContainers() {
return failedContainers;
}
@@ -447,27 +479,31 @@
}
/**
- * Set the container limits -the max that can be asked for,
- * which are used when the "max" values are requested
+ * Set the container limits -the min and max values for
+ * resource requests. All requests must be multiples of the min
+ * values.
+ * @param minMemory min memory MB
* @param maxMemory maximum memory
+ * @param minCores min v core count
* @param maxCores maximum cores
*/
- public void setContainerLimits(int maxMemory, int maxCores) {
+ public void setContainerLimits(int minMemory,int maxMemory, int minCores, int maxCores) {
+ containerMinCores = minCores;
containerMaxCores = maxCores;
+ containerMinMemory = minMemory;
containerMaxMemory = maxMemory;
+ minResource = recordFactory.newResource(containerMinMemory, containerMinCores);
+ maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores);
}
-
public ConfTreeOperations getResourcesSnapshot() {
return resourcesSnapshot;
}
-
public ConfTreeOperations getAppConfSnapshot() {
return appConfSnapshot;
}
-
public ConfTreeOperations getInternalsSnapshot() {
return internalsSnapshot;
}
@@ -488,38 +524,17 @@
return unresolvedInstanceDefinition;
}
- /**
- * Build up the application state
- * @param instanceDefinition definition of the applicatin instance
- * @param appmasterConfig
- * @param publishedProviderConf any configuration info to be published by a provider
- * @param providerRoles roles offered by a provider
- * @param fs filesystem
- * @param historyDir directory containing history files
- * @param liveContainers list of live containers supplied on an AM restart
- * @param applicationInfo app info to retain for web views
- * @param releaseSelector selector of containers to release
- */
- public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration appmasterConfig,
- Configuration publishedProviderConf,
- List<ProviderRole> providerRoles,
- FileSystem fs,
- Path historyDir,
- List<Container> liveContainers,
- Map<String, String> applicationInfo,
- ContainerReleaseSelector releaseSelector)
- throws BadClusterStateException, BadConfigException, IOException {
- Preconditions.checkArgument(instanceDefinition != null);
- Preconditions.checkArgument(releaseSelector != null);
+ public synchronized void buildInstance(AppStateBindingInfo binding)
+ throws BadClusterStateException, BadConfigException, IOException {
+ binding.validate();
log.debug("Building application state");
- this.publishedProviderConf = publishedProviderConf;
- this.applicationInfo = applicationInfo != null ? applicationInfo
- : new HashMap<String, String>();
+ publishedProviderConf = binding.publishedProviderConf;
+ applicationInfo = binding.applicationInfo != null ? binding.applicationInfo
+ : new HashMap<String, String>();
clientProperties = new HashMap<>();
- containerReleaseSelector = releaseSelector;
+ containerReleaseSelector = binding.releaseSelector;
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
@@ -532,15 +547,15 @@
// set the cluster specification (once its dependency the client properties
// is out the way
- setInitialInstanceDefinition(instanceDefinition);
+ setInitialInstanceDefinition(binding.instanceDefinition);
//build the initial role list
- for (ProviderRole providerRole : providerRoles) {
+ List<ProviderRole> roleList = new ArrayList<>(binding.roles);
+ for (ProviderRole providerRole : roleList) {
buildRole(providerRole);
}
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
@@ -548,44 +563,43 @@
// this is a new value
log.info("Adding role {}", name);
MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole =
- createDynamicProviderRole(name, resComponent);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
- providerRoles.add(dynamicRole);
+ roleList.add(dynamicRole);
}
}
//then pick up the requirements
buildRoleRequirementsFromResources();
-
//set the livespan
- MapOperations globalResOpts =
- instanceDefinition.getResourceOperations().getGlobalOptions();
-
+ MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();
+
startTimeThreshold = globalResOpts.getOptionInt(
InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ CONTAINER_FAILURE_THRESHOLD,
+ DEFAULT_CONTAINER_FAILURE_THRESHOLD);
nodeFailureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD);
+ NODE_FAILURE_THRESHOLD,
+ DEFAULT_NODE_FAILURE_THRESHOLD);
initClusterStatus();
// set up the role history
- roleHistory = new RoleHistory(providerRoles);
+ roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
roleHistory.register(metricsAndMonitoring);
- roleHistory.onStart(fs, historyDir);
+ roleHistory.onStart(binding.fs, binding.historyPath);
+ // trigger first node update
+ roleHistory.onNodesUpdated(binding.nodeReports);
+
//rebuild any live containers
- rebuildModelFromRestart(liveContainers);
+ rebuildModelFromRestart(binding.liveContainers);
// any am config options to pick up
- logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
-
+ logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
//mark as live
applicationLive = true;
}
@@ -625,32 +639,31 @@
* @return a new provider role
* @throws BadConfigException bad configuration
*/
- public ProviderRole createDynamicProviderRole(String name,
- MapOperations component) throws
- BadConfigException {
- String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int priority = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PRIORITY,
- priOpt, 0, 1, -1);
- String placementOpt = component.getOption(
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ public ProviderRole createDynamicProviderRole(String name, MapOperations component)
+ throws BadConfigException {
+ String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY);
+ int priority = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1);
+
+ String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.DEFAULT));
- int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
- placementOpt, 0, 0, -1);
- int placementTimeout =
- component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
+ int placement = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1);
+
+ int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY,
+ DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
ProviderRole newRole = new ProviderRole(name,
priority,
placement,
getNodeFailureThresholdForRole(name),
- placementTimeout);
+ placementTimeout,
+ component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION));
log.info("New {} ", newRole);
return newRole;
}
-
/**
* Actions to perform when an instance definition is updated
* Currently:
@@ -665,56 +678,52 @@
*
* @throws BadConfigException
*/
- private synchronized void onInstanceDefinitionUpdated() throws
- BadConfigException,
- IOException {
+ private synchronized void onInstanceDefinitionUpdated()
+ throws BadConfigException, IOException {
+
log.debug("Instance definition updated");
//note the time
snapshotTime = now();
-
+
// resolve references if not already done
instanceDefinition.resolve();
// force in the AM desired state values
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
+
if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) {
resources.setComponentOpt(
- SliderKeys.COMPONENT_AM, ResourceKeys.COMPONENT_INSTANCES, "1");
+ SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1");
}
//snapshot all three sectons
- resourcesSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getResources());
- appConfSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
- internalsSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
+ resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources());
+ appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
+ internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
//build a new aggregate from the snapshots
instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree,
appConfSnapshot.confTree,
internalsSnapshot.confTree);
instanceDefinitionSnapshot.setName(instanceDefinition.getName());
- clusterStatusTemplate =
- ClusterDescriptionOperations.buildFromInstanceDefinition(
+ clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition(
instanceDefinition);
-
-// Add the -site configuration properties
+ // Add the -site configuration properties
for (Map.Entry<String, String> prop : clientProperties.entrySet()) {
clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue());
}
-
+
}
-
+
/**
* The resource configuration is updated -review and update state.
* @param resources updated resources specification
* @return a list of any dynamically added provider roles
* (purely for testing purposes)
*/
+ @VisibleForTesting
public synchronized List<ProviderRole> updateResourceDefinitions(ConfTree resources)
throws BadConfigException, IOException {
log.debug("Updating resources to {}", resources);
@@ -741,9 +750,9 @@
private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
List<ProviderRole> newRoles = new ArrayList<>(0);
-
- //now update every role's desired count.
- //if there are no instance values, that role count goes to zero
+
+ // now update every role's desired count.
+ // if there are no instance values, that role count goes to zero
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
@@ -754,10 +763,8 @@
// skip inflexible roles, e.g AM itself
continue;
}
- int currentDesired = roleStatus.getDesired();
+ long currentDesired = roleStatus.getDesired();
String role = roleStatus.getName();
- MapOperations comp =
- resources.getComponent(role);
int desiredInstanceCount = getDesiredInstanceCount(resources, role);
if (desiredInstanceCount == 0) {
log.info("Role {} has 0 instances specified", role);
@@ -769,23 +776,25 @@
}
}
- //now the dynamic ones. Iterate through the the cluster spec and
- //add any role status entries not in the role status
+ // now the dynamic ones. Iterate through the the cluster spec and
+ // add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
log.info("Adding new role {}", name);
MapOperations component = resources.getComponent(name);
- ProviderRole dynamicRole = createDynamicProviderRole(name,
- component);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, component);
RoleStatus roleStatus = buildRole(dynamicRole);
roleStatus.setDesired(getDesiredInstanceCount(resources, name));
log.info("New role {}", roleStatus);
- roleHistory.addNewProviderRole(dynamicRole);
+ roleHistory.addNewRole(roleStatus);
newRoles.add(dynamicRole);
}
}
+ // and fill in all those roles with their requirements
+ buildRoleResourceRequirements();
+
return newRoles;
}
@@ -799,7 +808,7 @@
private int getDesiredInstanceCount(ConfTreeOperations resources,
String role) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
log.error("Role {} has negative desired instances : {}", role,
@@ -821,7 +830,7 @@
* @throws BadConfigException if a role of that priority already exists
*/
public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException {
- //build role status map
+ // build role status map
int priority = providerRole.id;
if (roleStatusMap.containsKey(priority)) {
throw new BadConfigException("Duplicate Provider Key: %s and %s",
@@ -830,12 +839,26 @@
}
RoleStatus roleStatus = new RoleStatus(providerRole);
roleStatusMap.put(priority, roleStatus);
- roles.put(providerRole.name, providerRole);
+ String name = providerRole.name;
+ roles.put(name, providerRole);
rolePriorityMap.put(priority, providerRole);
+ // register its entries
+ metricsAndMonitoring.addMetricSet(MetricsConstants.PREFIX_SLIDER_ROLES + name, roleStatus);
return roleStatus;
}
/**
+ * Build up the requirements of every resource
+ */
+ private void buildRoleResourceRequirements() {
+ roleStatusMap.values();
+ for (RoleStatus role : roleStatusMap.values()) {
+ role.setResourceRequirements(
+ buildResourceRequirements(role, recordFactory.newResource()));
+ }
+ }
+
+ /**
* build up the special master node, which lives
* in the live node set but has a lifecycle bonded to the AM
* @param containerId the AM master
@@ -861,10 +884,9 @@
//it is also added to the set of live nodes
getLiveContainers().put(containerId, am);
putOwnedContainer(containerId, am);
-
+
// patch up the role status
- RoleStatus roleStatus = roleStatusMap.get(
- (SliderKeys.ROLE_AM_PRIORITY_INDEX));
+ RoleStatus roleStatus = roleStatusMap.get(SliderKeys.ROLE_AM_PRIORITY_INDEX);
roleStatus.setDesired(1);
roleStatus.incActual();
roleStatus.incStarted();
@@ -890,11 +912,12 @@
appMasterNode.state = STATE_LIVE;
}
- public RoleInstance getAppMasterNode() {
- return appMasterNode;
- }
-
-
+ /**
+ * Look up the status entry of a role or raise an exception
+ * @param key role ID
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
public RoleStatus lookupRoleStatus(int key) {
RoleStatus rs = getRoleStatusMap().get(key);
if (rs == null) {
@@ -902,8 +925,15 @@
}
return rs;
}
-
- public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
+
+ /**
+ * Look up the status entry of a container or raise an exception
+ *
+ * @param c container
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
+ public RoleStatus lookupRoleStatus(Container c) {
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
@@ -914,7 +944,7 @@
*/
public List<RoleStatus> cloneRoleStatusList() {
Collection<RoleStatus> statuses = roleStatusMap.values();
- List<RoleStatus> statusList = new ArrayList<RoleStatus>(statuses.size());
+ List<RoleStatus> statusList = new ArrayList<>(statuses.size());
try {
for (RoleStatus status : statuses) {
statusList.add((RoleStatus)(status.clone()));
@@ -926,6 +956,12 @@
}
+ /**
+ * Look up a role in the map
+ * @param name role name
+ * @return the instance
+ * @throws YarnRuntimeException if not found
+ */
public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
ProviderRole providerRole = roles.get(name);
if (providerRole == null) {
@@ -941,7 +977,7 @@
*/
public synchronized List<RoleInstance> cloneOwnedContainerList() {
Collection<RoleInstance> values = ownedContainers.values();
- return new ArrayList<RoleInstance>(values);
+ return new ArrayList<>(values);
}
/**
@@ -1040,7 +1076,6 @@
}
}
-
public synchronized List<RoleInstance> getLiveInstancesByContainerIDs(
Collection<String> containerIDs) {
//first, a hashmap of those containerIDs is built up
@@ -1094,7 +1129,6 @@
return nodes;
}
-
/**
* Build an instance map.
* @return the map of Role name to list of role instances
@@ -1111,8 +1145,7 @@
}
return map;
}
-
-
+
/**
* Build a map of role->nodename->node-info
*
@@ -1179,43 +1212,39 @@
roleHistory.onContainerReleaseSubmitted(container);
}
-
/**
- * Set up the resource requirements with all that this role needs,
- * then create the container request itself.
- * @param role role to ask an instance of
- * @param capability a resource to set up
- * @return the request for a new container
+ * Create a container request.
+ * Update internal state, such as the role request count.
+ * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
+ * This is where role history information will be used for placement decisions.
+ * @param role role
+ * @return the container request to submit or null if there is none
*/
- public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
- RoleStatus role,
- Resource capability) {
- buildResourceRequirements(role, capability);
- String labelExpression = getLabelExpression(role);
- //get the role history to select a suitable node, if available
- AMRMClient.ContainerRequest containerRequest =
- createContainerRequest(role, capability, labelExpression);
- return containerRequest;
+ private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
+ if (role.isAntiAffinePlacement()) {
+ return createAAContainerRequest(role);
+ } else {
+ incrementRequestCount(role);
+ return roleHistory.requestContainerForRole(role).getIssuedRequest();
+ }
}
/**
* Create a container request.
- * Update internal state, such as the role request count
- * This is where role history information will be used for placement decisions -
+ * Update internal state, such as the role request count.
+ * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
+ * This is where role history information will be used for placement decisions.
* @param role role
- * @param resource requirements
- * @param labelExpression label expression to satisfy
- * @return the container request to submit
+ * @return the container request to submit or null if there is none
*/
- private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
- Resource resource,
- String labelExpression) {
-
-
- AMRMClient.ContainerRequest request;
- request = roleHistory.requestNode(role, resource, labelExpression);
+ private AMRMClient.ContainerRequest createAAContainerRequest(RoleStatus role) {
+ OutstandingRequest request = roleHistory.requestContainerForAARole(role);
+ if (request == null) {
+ return null;
+ }
incrementRequestCount(role);
- return request;
+ role.setOutstandingAArequest(request);
+ return request.getIssuedRequest();
}
/**
@@ -1229,24 +1258,11 @@
incOutstandingContainerRequests();
}
-
- /**
- * dec requested count of a role
- * <p>
- * Also updates application state counters.
- * @param role role to decrement
- */
- protected synchronized void decrementRequestCount(RoleStatus role) {
- role.decRequested();
- }
-
/**
* Inc #of outstanding requests.
*/
private void incOutstandingContainerRequests() {
- synchronized (outstandingContainerRequests) {
- outstandingContainerRequests.inc();
- }
+ outstandingContainerRequests.inc();
}
/**
@@ -1279,11 +1295,11 @@
String option,
int defVal,
int maxVal) {
-
+
String val = resources.getComponentOpt(name, option,
Integer.toString(defVal));
Integer intVal;
- if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ if (YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
} else {
intVal = Integer.decode(val);
@@ -1291,15 +1307,15 @@
return intVal;
}
-
/**
* Build up the resource requirements for this role from the
* cluster specification, including substituing max allowed values
* if the specification asked for it.
* @param role role
- * @param capability capability to set up
+ * @param capability capability to set up. A new one may be created
+ * during normalization
*/
- public void buildResourceRequirements(RoleStatus role, Resource capability) {
+ public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
ConfTreeOperations resources = getResourcesSnapshot();
@@ -1314,17 +1330,16 @@
DEF_YARN_MEMORY,
containerMaxMemory);
capability.setMemory(ram);
- }
-
- /**
- * Extract the label expression for this role.
- * @param role role
- */
- public String getLabelExpression(RoleStatus role) {
- // Set up resource requirements from role values
- String name = role.getName();
- ConfTreeOperations resources = getResourcesSnapshot();
- return resources.getComponentOpt(name, YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION);
+ log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores);
+ Resource normalized = recordFactory.normalize(capability, minResource,
+ maxResource);
+ if (!Resources.equals(normalized, capability)) {
+ // resource requirements normalized to something other than asked for.
+ // LOG @ WARN so users can see why this is happening.
+ log.warn("Resource requirements of {} normalized" +
+ " from {} to {}", name, capability, normalized);
+ }
+ return normalized;
}
/**
@@ -1382,7 +1397,7 @@
RoleInstance starting = getStartingContainers().remove(containerId);
if (null == starting) {
throw new YarnRuntimeException(
- "Container "+ containerId +"%s is already started");
+ "Container "+ containerId +" is already started");
}
instance.state = STATE_LIVE;
RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
@@ -1427,11 +1442,31 @@
* Handle node update from the RM. This syncs up the node map with the RM's view
* @param updatedNodes updated nodes
*/
- public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
- roleHistory.onNodesUpdated(updatedNodes);
+ public synchronized NodeUpdatedOutcome onNodesUpdated(List<NodeReport> updatedNodes) {
+ boolean changed = roleHistory.onNodesUpdated(updatedNodes);
+ if (changed) {
+ log.info("YARN cluster changed —cancelling current AA requests");
+ List<AbstractRMOperation> operations = cancelOutstandingAARequests();
+ log.debug("Created {} cancel requests", operations.size());
+ return new NodeUpdatedOutcome(true, operations);
+ }
+ return new NodeUpdatedOutcome(false, new ArrayList<AbstractRMOperation>(0));
}
/**
+ * Return value of the {@link #onNodesUpdated(List)} call.
+ */
+ public static class NodeUpdatedOutcome {
+ public final boolean clusterChanged;
+ public final List<AbstractRMOperation> operations;
+
+ public NodeUpdatedOutcome(boolean clusterChanged,
+ List<AbstractRMOperation> operations) {
+ this.clusterChanged = clusterChanged;
+ this.operations = operations;
+ }
+ }
+ /**
* Is a role short lived by the threshold set for this application
* @param instance instance
* @return true if the instance is considered short lived
@@ -1475,7 +1510,6 @@
public int exitStatus = 0;
public boolean unknownNode = false;
-
public String toString() {
final StringBuilder sb =
new StringBuilder("NodeCompletionResult{");
@@ -1489,10 +1523,10 @@
return sb.toString();
}
}
-
+
/**
* handle completed node in the CD -move something from the live
- * server list to the completed server list
+ * server list to the completed server list.
* @param status the node that has just completed
* @return NodeCompletionResult
*/
@@ -1507,9 +1541,9 @@
log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- int releasing = roleStatus.decReleasing();
- int actual = roleStatus.decActual();
- int completedCount = roleStatus.incCompleted();
+ long releasing = roleStatus.decReleasing();
+ long actual = roleStatus.decActual();
+ long completedCount = roleStatus.incCompleted();
log.info("decrementing role count for role {} to {}; releasing={}, completed={}",
roleStatus.getName(),
actual,
@@ -1552,7 +1586,7 @@
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
message = String.format("Failure %s on host %s (%d): %s",
- roleInstance.getContainerId().toString(),
+ roleInstance.getContainerId(),
failedContainer.getNodeId().getHost(),
exitStatus,
completedLogsUrl);
@@ -1601,10 +1635,10 @@
log.warn("Received notification of completion of unknown node {}", id);
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
-
+
// and the active node list if present
removeOwnedContainer(containerId);
-
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
containerId) : "container still in release queue";
@@ -1639,9 +1673,6 @@
return completedLogsUrl;
}
-
-
-
/**
* Return the percentage done that Slider is to have YARN display in its
* Web UI
@@ -1649,7 +1680,7 @@
*/
public synchronized float getApplicationProgressPercentage() {
float percentage;
- int desired = 0;
+ long desired = 0;
float actual = 0;
for (RoleStatus role : getRoleStatusMap().values()) {
desired += role.getDesired();
@@ -1670,7 +1701,7 @@
public ClusterDescription refreshClusterStatus() {
return refreshClusterStatus(null);
}
-
+
/**
* Update the cluster description with the current application state
* @param providerStatus status from the provider for the cluster info section
@@ -1683,13 +1714,13 @@
now);
if (providerStatus != null) {
for (Map.Entry<String, String> entry : providerStatus.entrySet()) {
- cd.setInfo(entry.getKey(),entry.getValue());
+ cd.setInfo(entry.getKey(), entry.getValue());
}
}
MapOperations infoOps = new MapOperations("info", cd.info);
infoOps.mergeWithoutOverwrite(applicationInfo);
SliderUtils.addBuildInfo(infoOps, "status");
- cd.statistics = new HashMap<String, Map<String, Integer>>();
+ cd.statistics = new HashMap<>();
// build the map of node -> container IDs
Map<String, List<String>> instanceMap = createRoleToInstanceMap();
@@ -1698,7 +1729,7 @@
//build the map of node -> containers
Map<String, Map<String, ClusterNode>> clusterNodes =
createRoleToClusterNodeMap();
- cd.status = new HashMap<String, Object>();
+ cd.status = new HashMap<>();
cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
@@ -1706,7 +1737,7 @@
String rolename = role.getName();
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
- cd.setRoleOpt(rolename, ResourceKeys.COMPONENT_INSTANCES,
+ cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
@@ -1716,17 +1747,19 @@
cd.setRoleOpt(rolename, ROLE_FAILED_RECENTLY_INSTANCES, role.getFailedRecently());
cd.setRoleOpt(rolename, ROLE_NODE_FAILED_INSTANCES, role.getNodeFailed());
cd.setRoleOpt(rolename, ROLE_PREEMPTED_INSTANCES, role.getPreempted());
+ if (role.isAntiAffinePlacement()) {
+ cd.setRoleOpt(rolename, ROLE_PENDING_AA_INSTANCES, role.getPendingAntiAffineRequests());
+ }
Map<String, Integer> stats = role.buildStatistics();
cd.statistics.put(rolename, stats);
}
-
Map<String, Integer> sliderstats = getLiveStatistics();
cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats);
-
+
// liveness
cd.liveness = getApplicationLivenessInformation();
-
+
return cd;
}
@@ -1736,12 +1769,14 @@
*/
public ApplicationLivenessInformation getApplicationLivenessInformation() {
ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- int outstanding = (int) outstandingContainerRequests.getCount();
+ RoleStatistics stats = getRoleStatistics();
+ int outstanding = (int)(stats.desired - stats.actual);
li.requestsOutstanding = outstanding;
li.allRequestsSatisfied = outstanding <= 0;
+ li.activeRequests = (int)stats.requested;
return li;
}
-
+
/**
* Get the live statistics map
* @return a map of statistics values, defined in the {@link StatusKeys}
@@ -1752,21 +1787,33 @@
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE,
liveNodes.size());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
- (int)completedContainerCount.getCount());
+ completedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
- (int)failedContainerCount.getCount());
+ failedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED,
- (int)startedContainers.getCount());
+ startedContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED,
- (int) startFailedContainerCount.getCount());
+ startFailedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS,
- (int)surplusContainers.getCount());
+ surplusContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED,
completionOfUnknownContainerEvent.get());
return sliderstats;
}
/**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ public RoleStatistics getRoleStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ for (RoleStatus role : getRoleStatusMap().values()) {
+ stats.add(role.getStatistics());
+ }
+ return stats;
+ }
+
+ /**
* Get a snapshot of component information.
* <p>
* This does <i>not</i> include any container list, which
@@ -1776,8 +1823,7 @@
public Map<String, ComponentInformation> getComponentInfoSnapshot() {
Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
- Map<String, ComponentInformation> results =
- new HashMap<String, ComponentInformation>(
+ Map<String, ComponentInformation> results = new HashMap<>(
statusMap.size());
for (RoleStatus status : statusMap.values()) {
@@ -1794,7 +1840,7 @@
public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
throws SliderInternalStateException, TriggerClusterTeardownException {
log.debug("in reviewRequestAndReleaseNodes()");
- List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
+ List<AbstractRMOperation> allOperations = new ArrayList<>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
if (!roleStatus.isExcludeFromFlexing()) {
List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
@@ -1814,8 +1860,10 @@
throws TriggerClusterTeardownException {
long failures = role.getFailedRecently();
int threshold = getFailureThresholdForRole(role);
- log.debug("Failure count of component: {}: {}, threshold={}",
- role.getName(), failures, threshold);
+ if (log.isDebugEnabled() && failures > 0) {
+ log.debug("Failure count of component: {}: {}, threshold={}",
+ role.getName(), failures, threshold);
+ }
if (failures > threshold) {
throw new TriggerClusterTeardownException(
@@ -1841,7 +1889,7 @@
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
+ CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
@@ -1855,7 +1903,7 @@
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleName,
- ResourceKeys.NODE_FAILURE_THRESHOLD,
+ NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
@@ -1879,7 +1927,25 @@
public List<AbstractRMOperation> escalateOutstandingRequests() {
return roleHistory.escalateOutstandingRequests();
}
-
+
+ /**
+ * Cancel any outstanding AA Requests, building up the list of ops to
+ * cancel, removing them from RoleHistory structures and the RoleStatus
+ * entries.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
+ // get the list of cancel operations
+ List<AbstractRMOperation> operations = roleHistory.cancelOutstandingAARequests();
+ for (RoleStatus roleStatus : roleStatusMap.values()) {
+ if (roleStatus.isAARequestOutstanding()) {
+ log.info("Cancelling outstanding AA request for {}", roleStatus);
+ roleStatus.cancelOutstandingAARequest();
+ }
+ }
+ return operations;
+ }
+
/**
* Look at the allocation status of one role, and trigger add/release
* actions if the number of desired role instances doesn't equal
@@ -1896,15 +1962,16 @@
private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List<AbstractRMOperation> operations = new ArrayList<>();
- int delta;
- int expected;
+ long delta;
+ long expected;
String name = role.getName();
synchronized (role) {
delta = role.getDelta();
expected = role.getDesired();
}
- log.info("Reviewing {} : expected {}", role, expected);
+ log.info("Reviewing {} : ", role);
+ log.debug("Expected {}, Delta: {}", expected, delta);
checkFailureThreshold(role);
if (expected < 0 ) {
@@ -1917,38 +1984,53 @@
}
if (delta > 0) {
- log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
- delta, expected);
// more workers needed than we have -ask for more
- for (int i = 0; i < delta; i++) {
- Resource capability = recordFactory.newResource();
- AMRMClient.ContainerRequest containerAsk =
- buildContainerResourceAndRequest(role, capability);
- log.info("Container ask is {} and label = {}", containerAsk,
- containerAsk.getNodeLabelExpression());
- int askMemory = containerAsk.getCapability().getMemory();
- if (askMemory > this.containerMaxMemory) {
- log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, delta, expected);
+
+ if (role.isAntiAffinePlacement()) {
+ long pending = delta;
+ if (roleHistory.canPlaceAANodes()) {
+ // build one only if there is none outstanding, the role history knows
+ // enough about the cluster to ask, and there is somewhere to place
+ // the node
+ if (!role.isAARequestOutstanding()) {
+ // no outstanding AA; try to place things
+ AMRMClient.ContainerRequest request = createAAContainerRequest(role);
+ if (request != null) {
+ pending--;
+ log.info("Starting an anti-affine request sequence for {} nodes; pending={}",
+ delta, pending);
+ addContainerRequest(operations, request);
+ } else {
+ log.info("No location for anti-affine request");
+ }
+ }
+ } else {
+ log.warn("Awaiting node map before generating anti-affinity requests");
}
- operations.add(new ContainerRequestOperation(containerAsk));
+ log.info("Setting pending to {}", pending);
+ role.setPendingAntiAffineRequests(pending);
+ } else {
+
+ for (int i = 0; i < delta; i++) {
+ //get the role history to select a suitable node, if available
+ addContainerRequest(operations, createContainerRequest(role));
+ }
}
} else if (delta < 0) {
log.info("{}: Asking for {} fewer node(s) for a total of {}", name,
-delta,
expected);
// reduce the number expected (i.e. subtract the delta)
+ long excess = -delta;
- // then pick some containers to kill
- int excess = -delta;
-
- // how many requests are outstanding
- int outstandingRequests = role.getRequested();
+ // how many requests are outstanding? for AA roles, this includes pending
+ long outstandingRequests = role.getRequested() + role.getPendingAntiAffineRequests();
if (outstandingRequests > 0) {
// outstanding requests.
- int toCancel = Math.min(outstandingRequests, excess);
+ int toCancel = (int)Math.min(outstandingRequests, excess);
// Delegate to Role History
-
List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel);
log.info("Found {} outstanding requests to cancel", cancellations.size());
operations.addAll(cancellations);
@@ -1968,7 +2050,6 @@
}
}
-
// after the cancellation there may be no excess
if (excess > 0) {
@@ -1982,7 +2063,7 @@
log.info("No containers for component {}", roleId);
}
- // cut all release-in-progress nodes
+ // filter out all release-in-progress nodes
ListIterator<RoleInstance> li = containersToRelease.listIterator();
while (li.hasNext()) {
RoleInstance next = li.next();
@@ -2002,16 +2083,13 @@
// ask the release selector to sort the targets
containersToRelease = containerReleaseSelector.sortCandidates(
roleId,
- containersToRelease,
- excess);
+ containersToRelease);
- //crop to the excess
-
- List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
- ? containersToRelease.subList(0, excess)
+ // crop to the excess
+ List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
+ ? containersToRelease.subList(0, (int)excess)
: containersToRelease;
-
// then build up a release operation, logging each container as released
for (RoleInstance possible : finalCandidates) {
log.info("Targeting for release: {}", possible);
@@ -2020,13 +2098,43 @@
}
}
+ } else {
+ // actual + requested == desired
+ // there's a special case here: clear all pending AA requests
+ if (role.getPendingAntiAffineRequests() > 0) {
+ log.debug("Clearing outstanding pending AA requests");
+ role.setPendingAntiAffineRequests(0);
+ }
}
- // list of operations to execute
+ // there's now a list of operations to execute
+ log.debug("operations scheduled: {}; updated role: {}", operations.size(), role);
return operations;
}
/**
+ * Add a container request if the request is non-null
+ * @param operations operations to add the entry to
+ * @param containerAsk what to ask for
+ * @return true if a request was added
+ */
+ private boolean addContainerRequest(List<AbstractRMOperation> operations,
+ AMRMClient.ContainerRequest containerAsk) {
+ if (containerAsk != null) {
+ log.info("Container ask is {} and label = {}", containerAsk,
+ containerAsk.getNodeLabelExpression());
+ int askMemory = containerAsk.getCapability().getMemory();
+ if (askMemory > this.containerMaxMemory) {
+ log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ }
+ operations.add(new ContainerRequestOperation(containerAsk));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
* Releases a container based on container id
* @param containerId
* @return
@@ -2105,54 +2213,42 @@
* a list of operations to perform
* @param allocatedContainers the containers allocated
* @param assignments the assignments of roles to containers
- * @param releaseOperations any release operations
+ * @param operations any allocation or release operations
*/
public synchronized void onContainersAllocated(List<Container> allocatedContainers,
List<ContainerAssignment> assignments,
- List<AbstractRMOperation> releaseOperations) {
+ List<AbstractRMOperation> operations) {
assignments.clear();
- releaseOperations.clear();
+ operations.clear();
List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
log.debug("onContainersAllocated(): Total containers allocated = {}", ordered.size());
for (Container container : ordered) {
- String containerHostInfo = container.getNodeId().getHost()
- + ":" +
- container.getNodeId().getPort();
+ final NodeId nodeId = container.getNodeId();
+ String containerHostInfo = nodeId.getHost() + ":" + nodeId.getPort();
//get the role
final ContainerId cid = container.getId();
final RoleStatus role = lookupRoleStatus(container);
-
//dec requested count
- decrementRequestCount(role);
+ role.decRequested();
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
- final int allocated = role.incActual();
- final int desired = role.getDesired();
+ final long allocated = role.incActual();
+ final long desired = role.getDesired();
final String roleName = role.getName();
- final ContainerAllocation allocation =
+ final ContainerAllocationResults allocation =
roleHistory.onContainerAllocated(container, desired, allocated);
final ContainerAllocationOutcome outcome = allocation.outcome;
- // cancel an allocation request which granted this, so as to avoid repeated
- // requests
- if (allocation.origin != null && allocation.origin.getIssuedRequest() != null) {
- releaseOperations.add(allocation.origin.createCancelOperation());
- } else {
- // there's a request, but no idea what to cancel.
- // rather than try to recover from it inelegantly, (and cause more confusion),
- // log the event, but otherwise continue
- log.warn("Unexpected allocation of container "
- + SliderUtils.containerToString(container));
- }
+ // add all requests to the operations list
+ operations.addAll(allocation.operations);
//look for condition where we get more back than we asked
if (allocated > desired) {
- log.info("Discarding surplus {} container {} on {}", roleName, cid,
- containerHostInfo);
- releaseOperations.add(new ContainerReleaseOperation(cid));
+ log.info("Discarding surplus {} container {} on {}", roleName, cid, containerHostInfo);
+ operations.add(new ContainerReleaseOperation(cid));
//register as a surplus node
surplusNodes.add(cid);
surplusContainers.inc();
@@ -2168,13 +2264,34 @@
" on {}:{},",
roleName,
cid,
- container.getNodeId().getHost(),
- container.getNodeId().getPort()
- );
+ nodeId.getHost(),
+ nodeId.getPort());
assignments.add(new ContainerAssignment(container, role, outcome));
//add to the history
roleHistory.onContainerAssigned(container);
+ // now for AA requests, add some more
+ if (role.isAntiAffinePlacement()) {
+ role.completeOutstandingAARequest();
+ // check invariants. The new node must become unavailable.
+ NodeInstance node = roleHistory.getOrCreateNodeInstance(container);
+ if (node.canHost(role.getKey(), role.getLabelExpression())) {
+ log.error("Assigned node still declares as available {}", node.toFullString() );
+ }
+ if (role.getPendingAntiAffineRequests() > 0) {
+ // still an outstanding AA request: need to issue a new one.
+ log.info("Asking for next container for AA role {}", roleName);
+ if (!addContainerRequest(operations, createAAContainerRequest(role))) {
+ log.info("No capacity in cluster for new requests");
+ } else {
+ role.decPendingAntiAffineRequests();
+ }
+ log.debug("Current AA role status {}", role);
+ } else {
+ log.info("AA request sequence completed for role {}", role);
+ }
+ }
+
}
}
}
@@ -2237,7 +2354,7 @@
cid,
roleName,
containerHostInfo);
-
+
//update app state internal structures and maps
RoleInstance instance = new RoleInstance(container);
@@ -2257,4 +2374,32 @@
// now pretend it has just started
innerOnNodeManagerContainerStarted(cid);
}
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("AppState{");
+ sb.append("applicationLive=").append(applicationLive);
+ sb.append(", live nodes=").append(liveNodes.size());
+ sb.append(", startedContainers=").append(startedContainers);
+ sb.append(", startFailedContainerCount=").append(startFailedContainerCount);
+ sb.append(", surplusContainers=").append(surplusContainers);
+ sb.append(", failedContainerCount=").append(failedContainerCount);
+ sb.append(", outstanding non-AA Container Requests=")
+ .append(outstandingContainerRequests);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build map of role ID-> name
+ * @return
+ */
+ public Map<Integer, String> buildNamingMap() {
+ Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
+ Map<Integer, String> naming = new HashMap<>(statusMap.size());
+ for (Map.Entry<Integer, RoleStatus> entry : statusMap.entrySet()) {
+ naming.put(entry.getKey(), entry.getValue().getName());
+ }
+ return naming;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
new file mode 100644
index 0000000..a8aa1a2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Binding information for application states; designed to be extensible
+ * so that tests don't have to be massivley reworked when new arguments
+ * are added.
+ */
+public class AppStateBindingInfo {
+ public AggregateConf instanceDefinition;
+ public Configuration serviceConfig = new Configuration();
+ public Configuration publishedProviderConf = new Configuration(false);
+ public List<ProviderRole> roles = new ArrayList<>();
+ public FileSystem fs;
+ public Path historyPath;
+ public List<Container> liveContainers = new ArrayList<>(0);
+ public Map<String, String> applicationInfo = new HashMap<>();
+ public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
+ /** node reports off the RM. */
+ public List<NodeReport> nodeReports = new ArrayList<>(0);
+
+ public void validate() throws IllegalArgumentException {
+ Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition");
+ Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
+ Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf");
+ Preconditions.checkArgument(releaseSelector != null, "null releaseSelector");
+ Preconditions.checkArgument(roles != null, "null providerRoles");
+ Preconditions.checkArgument(fs != null, "null fs");
+ Preconditions.checkArgument(historyPath != null, "null historyDir");
+ Preconditions.checkArgument(nodeReports != null, "null nodeReports");
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
similarity index 77%
rename from slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
index 306ffb2..e80639e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
@@ -18,10 +18,15 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+
+import java.util.ArrayList;
+import java.util.List;
+
/**
* This is just a tuple of the outcome of a container allocation
*/
-public class ContainerAllocation {
+public class ContainerAllocationResults {
/**
* What was the outcome of this allocation: placed, escalated, ...
@@ -35,12 +40,11 @@
*/
public OutstandingRequest origin;
- public ContainerAllocation(ContainerAllocationOutcome outcome,
- OutstandingRequest origin) {
- this.outcome = outcome;
- this.origin = origin;
- }
+ /**
+ * A possibly empty list of requests to add to the follow-up actions
+ */
+ public List<AbstractRMOperation> operations = new ArrayList<>(0);
- public ContainerAllocation() {
+ public ContainerAllocationResults() {
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
index 3cc2106..0322f83 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -57,10 +57,9 @@
locationSpecified));
return pri;
}
-
-
+
public static int extractRole(int priority) {
- return priority >= NOLOCATION ? priority^NOLOCATION : priority;
+ return priority >= NOLOCATION ? priority ^ NOLOCATION : priority;
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
index 0cbc134..fafbada 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
@@ -30,9 +30,8 @@
* Given a list of candidate containers, return a sorted version of the priority
* in which they should be released.
* @param candidates candidate list ... everything considered suitable
- * @return
+ * @return the list of candidates
*/
List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect);
+ List<RoleInstance> candidates);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
index 9d936a1..38c5b8e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -32,8 +32,7 @@
@Override
public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect) {
+ List<RoleInstance> candidates) {
Collections.sort(candidates, new newerThan());
return candidates;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index 8ff0895..cf3881e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.slider.api.types.NodeEntryInformation;
/**
@@ -37,7 +38,7 @@
* <p>
*
*/
-public class NodeEntry {
+public class NodeEntry implements Cloneable {
public final int rolePriority;
@@ -89,7 +90,16 @@
* the number of instances > 1.
*/
public synchronized boolean isAvailable() {
- return getActive() == 0 && (requested == 0) && starting == 0;
+ return live + requested + starting - releasing <= 0;
+ }
+
+ /**
+ * Are the anti-affinity constraints held. That is, zero or one
+ * node running or starting
+ * @return true if the constraint holds.
+ */
+ public synchronized boolean isAntiAffinityConstraintHeld() {
+ return (live - releasing + starting) <= 1;
}
/**
@@ -256,10 +266,16 @@
return failedRecently;
}
+ @VisibleForTesting
+ public synchronized void setFailedRecently(int failedRecently) {
+ this.failedRecently = failedRecently;
+ }
+
public synchronized int getPreempted() {
return preempted;
}
+
/**
* Reset the failed recently count.
*/
@@ -301,4 +317,9 @@
info.lastUsed = lastUsed;
return info;
}
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 2afdc42..cc17cf0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -18,17 +18,20 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.slider.api.types.NodeEntryInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.common.tools.Comparators;
import org.apache.slider.common.tools.SliderUtils;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
+import java.util.Map;
/**
* A node instance -stores information about a node in the cluster.
@@ -45,6 +48,9 @@
*/
private NodeState nodeState = NodeState.RUNNING;
+ /**
+ * Last node report. If null: none
+ */
private NodeReport nodeReport = null;
/**
@@ -52,6 +58,18 @@
*/
private long nodeStateUpdateTime = 0;
+ /**
+ * Node labels.
+ *
+ * IMPORTANT: we assume that there is one label/node, which is the policy
+ * for Hadoop as of November 2015
+ */
+ private String nodeLabels = "";
+
+ /**
+ * An unordered list of node entries of specific roles. There's nothing
+ * indexed so as to support sparser datastructures.
+ */
private final List<NodeEntry> nodeEntries;
/**
@@ -63,18 +81,35 @@
nodeEntries = new ArrayList<>(roles);
}
-
/**
- * Update the node status
+ * Update the node status.
+ * The return code is true if the node state changed enough to
+ * trigger a re-evaluation of pending requests. That is, either a node
+ * became available when it was previously not, or the label changed
+ * on an available node.
+ *
+ * Transitions of a node from live to dead aren't treated as significant,
+ * nor label changes on a dead node.
+ *
* @param report latest node report
- * @return true if the node state changed
+ * @return true if the node state changed enough for a request evaluation.
*/
- public boolean updateNode(NodeReport report) {
+ public synchronized boolean updateNode(NodeReport report) {
+ nodeStateUpdateTime = report.getLastHealthReportTime();
nodeReport = report;
NodeState oldState = nodeState;
+ boolean oldStateUnusable = oldState.isUnusable();
nodeState = report.getNodeState();
- nodeStateUpdateTime = report.getLastHealthReportTime();
- return nodeState != oldState;
+ boolean newUsable = !nodeState.isUnusable();
+ boolean nodeNowAvailable = oldStateUnusable && newUsable;
+ String labels = this.nodeLabels;
+ nodeLabels = SliderUtils.extractNodeLabel(report);
+ return nodeNowAvailable
+ || newUsable && !this.nodeLabels.equals(labels);
+ }
+
+ public String getNodeLabels() {
+ return nodeLabels;
}
/**
@@ -108,6 +143,15 @@
}
/**
+ * Get the node entry matching a container on this node
+ * @param container container
+ * @return matching node instance for the role
+ */
+ public NodeEntry getOrCreate(Container container) {
+ return getOrCreate(ContainerPriority.extractRole(container));
+ }
+
+ /**
* Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
@@ -129,6 +173,14 @@
}
/**
+ * Is the node considered online
+ * @return the node
+ */
+ public boolean isOnline() {
+ return !nodeState.isUnusable();
+ }
+
+ /**
* Query for a node being considered unreliable
* @param role role key
* @param threshold threshold above which a node is considered unreliable
@@ -136,7 +188,6 @@
*/
public boolean isConsideredUnreliable(int role, int threshold) {
NodeEntry entry = get(role);
-
return entry != null && entry.getFailedRecently() > threshold;
}
@@ -158,7 +209,6 @@
nodeEntries.add(nodeEntry);
}
-
/**
* run through each entry; gc'ing & removing old ones that don't have
* a recent failure count (we care about those)
@@ -201,11 +251,12 @@
public String toFullString() {
final StringBuilder sb =
new StringBuilder(toString());
- int i = 0;
+ sb.append("{ ");
for (NodeEntry entry : nodeEntries) {
sb.append(String.format("\n [%02d] ", entry.rolePriority));
sb.append(entry.toString());
}
+ sb.append("} ");
return sb.toString();
}
@@ -249,39 +300,59 @@
/**
* Produced a serialized form which can be served up as JSON
+ * @param naming map of priority -> value for naming entries
* @return a summary of the current role status.
*/
- public synchronized NodeInformation serialize() {
+ public synchronized NodeInformation serialize(Map<Integer, String> naming) {
NodeInformation info = new NodeInformation();
info.hostname = hostname;
// null-handling state constructor
info.state = "" + nodeState;
info.lastUpdated = nodeStateUpdateTime;
+ info.labels = nodeLabels;
if (nodeReport != null) {
info.httpAddress = nodeReport.getHttpAddress();
info.rackName = nodeReport.getRackName();
- info.labels = SliderUtils.join(nodeReport.getNodeLabels(), ", ", false);
info.healthReport = nodeReport.getHealthReport();
}
- info.entries = new ArrayList<>(nodeEntries.size());
+ info.entries = new HashMap<>(nodeEntries.size());
for (NodeEntry nodeEntry : nodeEntries) {
- info.entries.add(nodeEntry.serialize());
+ String name = naming.get(nodeEntry.rolePriority);
+ if (name == null) {
+ name = Integer.toString(nodeEntry.rolePriority);
+ }
+ info.entries.put(name, nodeEntry.serialize());
}
return info;
}
/**
+ * Is this node instance a suitable candidate for the specific role?
+ * @param role role ID
+ * @param label label which must match, or "" for no label checks
+ * @return true if the node has space for this role, is running and the labels
+ * match.
+ */
+ public boolean canHost(int role, String label) {
+ return isOnline()
+ && (SliderUtils.isUnset(label) || label.equals(nodeLabels)) // label match
+ && getOrCreate(role).isAvailable(); // no live role
+ }
+
+ /**
* A comparator for sorting entries where the node is preferred over another.
- * <p>
- * The exact algorithm may change
- *
+ *
+ * The exact algorithm may change: current policy is "most recent first", so sorted
+ * on the lastUsed
+ *
* the comparision is a positive int if left is preferred to right;
* negative if right over left, 0 for equal
*/
- public static class Preferred implements Comparator<NodeInstance>,
- Serializable {
+ public static class Preferred implements Comparator<NodeInstance>, Serializable {
- final int role;
+ private static final Comparators.InvertedLongComparator comparator =
+ new Comparators.InvertedLongComparator();
+ private final int role;
public Preferred(int role) {
this.role = role;
@@ -291,16 +362,9 @@
public int compare(NodeInstance o1, NodeInstance o2) {
NodeEntry left = o1.get(role);
NodeEntry right = o2.get(role);
- long ageL = left != null ? left.getLastUsed() : 0;
- long ageR = right != null ? right.getLastUsed() : 0;
-
- if (ageL > ageR) {
- return -1;
- } else if (ageL < ageR) {
- return 1;
- }
- // equal
- return 0;
+ long ageL = left != null ? left.getLastUsed() : -1;
+ long ageR = right != null ? right.getLastUsed() : -1;
+ return comparator.compare(ageL, ageR);
}
}
@@ -313,7 +377,7 @@
public static class MoreActiveThan implements Comparator<NodeInstance>,
Serializable {
- final int role;
+ private final int role;
public MoreActiveThan(int role) {
this.role = role;
@@ -326,5 +390,20 @@
return activeRight - activeLeft;
}
}
+ /**
+ * A comparator for sorting entries alphabetically
+ */
+ public static class CompareNames implements Comparator<NodeInstance>,
+ Serializable {
+
+ public CompareNames() {
+ }
+
+ @Override
+ public int compare(NodeInstance left, NodeInstance right) {
+ return left.hostname.compareTo(right.hostname);
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index b631057..3858b68 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -92,20 +92,24 @@
}
}
-
/**
- * Update the node state
+ * Update the node state. Return true if the node state changed: either by
+ * being created, or by changing its internal state as defined
+ * by {@link NodeInstance#updateNode(NodeReport)}.
+ *
* @param hostname host name
* @param report latest node report
- * @return the updated node.
+ * @return true if the node state changed enough for a request evaluation.
*/
public boolean updateNode(String hostname, NodeReport report) {
- return getOrCreate(hostname).updateNode(report);
+ boolean nodeExisted = get(hostname) != null;
+ boolean updated = getOrCreate(hostname).updateNode(report);
+ return updated || !nodeExisted;
}
/**
* Clone point
- * @return
+ * @return a shallow clone
*/
@Override
public Object clone() {
@@ -123,4 +127,48 @@
put(node.hostname, node);
}
}
+
+ /**
+ * Test helper: build or update a cluster from a list of node reports
+ * @param reports the list of reports
+ * @return true if this has been considered to have changed the cluster
+ */
+ @VisibleForTesting
+ public boolean buildOrUpdate(List<NodeReport> reports) {
+ boolean updated = false;
+ for (NodeReport report : reports) {
+ updated |= getOrCreate(report.getNodeId().getHost()).updateNode(report);
+ }
+ return updated;
+ }
+
+ /**
+ * Scan the current node map for all nodes capable of hosting an instance
+ * @param role role ID
+ * @param label label which must match, or "" for no label checks
+ * @return a possibly empty list of node instances matching the criteria.
+ */
+ public List<NodeInstance> findAllNodesForRole(int role, String label) {
+ List<NodeInstance> nodes = new ArrayList<>(size());
+ for (NodeInstance instance : values()) {
+ if (instance.canHost(role, label)) {
+ nodes.add(instance);
+ }
+ }
+ Collections.sort(nodes, new NodeInstance.CompareNames());
+ return nodes;
+ }
+
+ @Override
+ public synchronized String toString() {
+ final StringBuilder sb = new StringBuilder("NodeMap{");
+ List<String> keys = new ArrayList<>(keySet());
+ Collections.sort(keys);
+ for (String key : keys) {
+ sb.append(key).append(": ");
+ sb.append(get(key).toFullString()).append("\n");
+ }
+ sb.append('}');
+ return sb.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index f5689dd..3a75f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -29,6 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -43,24 +44,21 @@
* instance constructed with (role, hostname) can be used to look up
* a complete request instance in the {@link OutstandingRequestTracker} map
*/
-public final class OutstandingRequest {
+public final class OutstandingRequest extends RoleHostnamePair {
protected static final Logger log =
LoggerFactory.getLogger(OutstandingRequest.class);
/**
- * requested role
- */
- public final int roleId;
-
- /**
* Node the request is for -may be null
*/
public final NodeInstance node;
-
+
/**
- * hostname -will be null if node==null
+ * A list of all possible nodes to list in an AA request. For a non-AA
+ * request where {@link #node} is set, element 0 of the list is the same
+ * value.
*/
- public final String hostname;
+ public final List<NodeInstance> nodes = new ArrayList<>(1);
/**
* Optional label. This is cached as the request option (explicit-location + label) is forbidden,
@@ -71,21 +69,21 @@
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private AMRMClient.ContainerRequest issuedRequest;
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long requestedTimeMillis;
/**
* Time in millis after which escalation should be triggered..
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long escalationTimeoutMillis;
@@ -105,15 +103,21 @@
private int priority = -1;
/**
+ * Is this an Anti-affine request which should be cancelled on
+ * a cluster resize?
+ */
+ private boolean antiAffine = false;
+
+ /**
* Create a request
* @param roleId role
* @param node node -can be null
*/
public OutstandingRequest(int roleId,
NodeInstance node) {
- this.roleId = roleId;
+ super(roleId, node != null ? node.hostname : null);
this.node = node;
- this.hostname = node != null ? node.hostname : null;
+ nodes.add(node);
}
/**
@@ -125,9 +129,21 @@
* @param hostname hostname
*/
public OutstandingRequest(int roleId, String hostname) {
+ super(roleId, hostname);
this.node = null;
- this.roleId = roleId;
- this.hostname = hostname;
+ }
+
+ /**
+ * Create an Anti-affine reques, including all listed nodes (there must be one)
+ * as targets.
+ * @param roleId role
+ * @param nodes list of nodes
+ */
+ public OutstandingRequest(int roleId, List<NodeInstance> nodes) {
+ super(roleId, nodes.get(0).hostname);
+ this.node = null;
+ this.antiAffine = true;
+ this.nodes.addAll(nodes);
}
/**
@@ -162,6 +178,14 @@
return priority;
}
+ public boolean isAntiAffine() {
+ return antiAffine;
+ }
+
+ public void setAntiAffine(boolean antiAffine) {
+ this.antiAffine = antiAffine;
+ }
+
/**
* Build a container request.
* <p>
@@ -178,16 +202,15 @@
* @param resource resource
* @param role role
* @param time time in millis to record as request time
- * @param labelExpression label to satisfy
* @return the request to raise
*/
public synchronized AMRMClient.ContainerRequest buildContainerRequest(
- Resource resource, RoleStatus role, long time, String labelExpression) {
+ Resource resource, RoleStatus role, long time) {
Preconditions.checkArgument(resource != null, "null `resource` arg");
Preconditions.checkArgument(role != null, "null `role` arg");
// cache label for escalation
- label = labelExpression;
+ label = role.getLabelExpression();
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
String[] hosts;
@@ -196,7 +219,23 @@
NodeInstance target = this.node;
String nodeLabels;
- if (target != null) {
+ if (isAntiAffine()) {
+ int size = nodes.size();
+ log.info("Creating anti-affine request across {} nodes; first node = {}",
+ size, hostname);
+ hosts = new String[size];
+ StringBuilder builder = new StringBuilder(size * 16);
+ int c = 0;
+ for (NodeInstance nodeInstance : nodes) {
+ hosts[c++] = nodeInstance.hostname;
+ builder.append(nodeInstance.hostname).append(" ");
+ }
+ log.debug("Full host list: [ {}]", builder);
+ escalated = false;
+ mayEscalate = false;
+ relaxLocality = false;
+ nodeLabels = null;
+ } else if (target != null) {
// placed request. Hostname is used in request
hosts = new String[1];
hosts[0] = target.hostname;
@@ -218,7 +257,7 @@
escalated = true;
// and forbid it happening
mayEscalate = false;
- nodeLabels = labelExpression;
+ nodeLabels = label;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
priority = pri.getPriority();
@@ -228,7 +267,6 @@
pri,
relaxLocality,
nodeLabels);
-
validate();
return issuedRequest;
}
@@ -254,7 +292,7 @@
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
- if (label == null && issuedRequestNodes != null) {
+ if (SliderUtils.isUnset(label) && issuedRequestNodes != null) {
nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
} else {
nodes = null;
@@ -302,58 +340,22 @@
return issuedRequest != null && issuedRequest.getCapability().equals(resource);
}
- /**
- * Equality is on hostname and role
- * @param o other
- * @return true on a match
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- OutstandingRequest request = (OutstandingRequest) o;
-
- if (roleId != request.roleId) {
- return false;
- }
- if (hostname != null
- ? !hostname.equals(request.hostname)
- : request.hostname != null) {
- return false;
- }
- return true;
- }
-
- /**
- * hash on hostname and role
- * @return hash code
- */
- @Override
- public int hashCode() {
- int result = roleId;
- result = 31 * result + (hostname != null ? hostname.hashCode() : 0);
- return result;
- }
-
@Override
public String toString() {
- int requestRoleId = ContainerPriority.extractRole(getPriority());
boolean requestHasLocation = ContainerPriority.hasLocation(getPriority());
final StringBuilder sb = new StringBuilder("OutstandingRequest{");
- sb.append("roleId=").append(this.roleId);
+ sb.append("roleId=").append(roleId);
+ if (hostname != null) {
+ sb.append(", hostname='").append(hostname).append('\'');
+ }
sb.append(", node=").append(node);
- sb.append(", hostname='").append(hostname).append('\'');
sb.append(", hasLocation=").append(requestHasLocation);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
- sb.append(", issuedRequest=").append(SliderUtils.requestToString(issuedRequest));
+ sb.append(", issuedRequest=").append(
+ issuedRequest != null ? SliderUtils.requestToString(issuedRequest) : "(null)");
sb.append('}');
return sb.toString();
}
@@ -367,14 +369,13 @@
return new CancelSingleRequest(issuedRequest);
}
-
/**
* Valid if a node label expression specified on container request is valid or
* not. Mimics the logic in AMRMClientImpl, so can be used for preflight checking
* and in mock tests
*
*/
- public void validate() throws InvalidContainerRequestException {
+ public void validate() throws InvalidContainerRequestException {
Preconditions.checkNotNull(issuedRequest, "request has not yet been built up");
AMRMClient.ContainerRequest containerRequest = issuedRequest;
String exp = containerRequest.getNodeLabelExpression();
@@ -390,7 +391,7 @@
+ " in a single node label expression: " + this);
}
- // Don't allow specify node label against ANY request
+ // Don't allow specify node label against ANY request listing hosts or racks
if ((containerRequest.getRacks() != null &&
(!containerRequest.getRacks().isEmpty()))
||
@@ -410,5 +411,12 @@
}
}
+ /**
+ * Create a new role/hostname pair for indexing.
+ * @return a new index.
+ */
+ public RoleHostnamePair getIndex() {
+ return new RoleHostnamePair(roleId, hostname);
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index a8787f0..64698f2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -35,10 +35,12 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
+import java.util.Set;
/**
* Tracks outstanding requests made with a specific placement option.
@@ -62,8 +64,7 @@
*/
private final List<AbstractRMOperation> NO_REQUESTS = new ArrayList<>(0);
- private Map<OutstandingRequest, OutstandingRequest> placedRequests =
- new HashMap<>();
+ private Map<RoleHostnamePair, OutstandingRequest> placedRequests = new HashMap<>();
/**
* List of open requests; no specific details on them.
@@ -82,10 +83,9 @@
* @return a new request
*/
public synchronized OutstandingRequest newRequest(NodeInstance instance, int role) {
- OutstandingRequest request =
- new OutstandingRequest(role, instance);
+ OutstandingRequest request = new OutstandingRequest(role, instance);
if (request.isLocated()) {
- placedRequests.put(request, request);
+ placedRequests.put(request.getIndex(), request);
} else {
openRequests.add(request);
}
@@ -93,6 +93,31 @@
}
/**
+ * Create a new Anti-affine request for the specific role
+ * <p>
+ * It is added to {@link #openRequests}
+ * <p>
+ * This does not update the node instance's role's request count
+ * @param role role index
+ * @param nodes list of suitable nodes
+ * @param label label to use
+ * @return a new request
+ */
+ public synchronized OutstandingRequest newAARequest(int role,
+ List<NodeInstance> nodes,
+ String label) {
+ Preconditions.checkArgument(!nodes.isEmpty());
+ // safety check to verify the allocation will hold
+ for (NodeInstance node : nodes) {
+ Preconditions.checkState(node.canHost(role, label),
+ "Cannot allocate role ID %d to node %s", role, node);
+ }
+ OutstandingRequest request = new OutstandingRequest(role, nodes);
+ openRequests.add(request);
+ return request;
+ }
+
+ /**
* Look up any oustanding request to a (role, hostname).
* @param role role index
* @param hostname hostname
@@ -101,7 +126,7 @@
@VisibleForTesting
public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) {
Preconditions.checkArgument(hostname != null, "null hostname");
- return placedRequests.get(new OutstandingRequest(role, hostname));
+ return placedRequests.get(new RoleHostnamePair(role, hostname));
}
/**
@@ -115,25 +140,30 @@
}
/**
- * Notification that a container has been allocated -drop it
- * from the {@link #placedRequests} structure.
+ * Notification that a container has been allocated
+ *
+ * <ol>
+ * <li>drop it from the {@link #placedRequests} structure.</li>
+ * <li>generate the cancellation request</li>
+ * <li>for AA placement, any actions needed</li>
+ * </ol>
+ *
* @param role role index
* @param hostname hostname
* @return the allocation outcome
*/
- public synchronized ContainerAllocation onContainerAllocated(int role,
+ public synchronized ContainerAllocationResults onContainerAllocated(int role,
String hostname,
Container container) {
final String containerDetails = SliderUtils.containerToString(container);
log.debug("Processing allocation for role {} on {}", role,
containerDetails);
- ContainerAllocation allocation = new ContainerAllocation();
+ ContainerAllocationResults allocation = new ContainerAllocationResults();
ContainerAllocationOutcome outcome;
- OutstandingRequest request =
- placedRequests.remove(new OutstandingRequest(role, hostname));
+ OutstandingRequest request = placedRequests.remove(new OutstandingRequest(role, hostname));
if (request != null) {
//satisfied request
- log.debug("Found placed request for container: {}", request);
+ log.debug("Found oustanding placed request for container: {}", request);
request.completed();
// derive outcome from status of tracked request
outcome = request.isEscalated()
@@ -144,16 +174,25 @@
// scan through all containers in the open request list
request = removeOpenRequest(container);
if (request != null) {
- log.debug("Found open request for container: {}", request);
+ log.debug("Found open outstanding request for container: {}", request);
request.completed();
outcome = ContainerAllocationOutcome.Open;
} else {
- log.warn("No open request found for container {}, outstanding queue has {} entries ",
+ log.warn("No oustanding request found for container {}, outstanding queue has {} entries ",
containerDetails,
openRequests.size());
outcome = ContainerAllocationOutcome.Unallocated;
}
}
+ if (request != null && request.getIssuedRequest() != null) {
+ allocation.operations.add(request.createCancelOperation());
+ } else {
+ // there's a request, but no idea what to cancel.
+ // rather than try to recover from it inelegantly, (and cause more confusion),
+ // log the event, but otherwise continue
+ log.warn("Unexpected allocation of container " + SliderUtils.containerToString(container));
+ }
+
allocation.origin = request;
allocation.outcome = outcome;
return allocation;
@@ -280,12 +319,12 @@
*/
public synchronized List<NodeInstance> resetOutstandingRequests(int role) {
List<NodeInstance> hosts = new ArrayList<>();
- Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
+ Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>> iterator =
placedRequests.entrySet().iterator();
while (iterator.hasNext()) {
- Map.Entry<OutstandingRequest, OutstandingRequest> next =
+ Map.Entry<RoleHostnamePair, OutstandingRequest> next =
iterator.next();
- OutstandingRequest request = next.getKey();
+ OutstandingRequest request = next.getValue();
if (request.roleId == role) {
iterator.remove();
request.completed();
@@ -350,6 +389,47 @@
}
/**
+ * Cancel all outstanding AA requests from the lists of requests.
+ *
+ * This does not remove them from the role status; they must be reset
+ * by the caller.
+ *
+ */
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
+
+ log.debug("Looking for AA request to cancel");
+ List<AbstractRMOperation> operations = new ArrayList<>();
+
+ // first, all placed requests
+ for (Map.Entry<RoleHostnamePair, OutstandingRequest> entry : placedRequests.entrySet()) {
+ OutstandingRequest outstandingRequest = entry.getValue();
+ synchronized (outstandingRequest) {
+ if (outstandingRequest.isAntiAffine()) {
+ // time to escalate
+ operations.add(outstandingRequest.createCancelOperation());
+ placedRequests.remove(entry.getKey());
+ }
+ }
+ }
+ // second, all open requests
+ ListIterator<OutstandingRequest> orit = openRequests.listIterator();
+ while (orit.hasNext()) {
+ OutstandingRequest outstandingRequest = orit.next();
+ synchronized (outstandingRequest) {
+ if (outstandingRequest.isAntiAffine()) {
+ // time to escalate
+ operations.add(outstandingRequest.createCancelOperation());
+ orit.remove();
+ }
+ }
+ }
+ log.info("Cancelling {} outstanding AA requests", operations.size());
+
+ return operations;
+ }
+
+ /**
* Extract a specific number of open requests for a role
* @param roleId role Id
* @param count count to extract
@@ -368,6 +448,7 @@
}
return results;
}
+
/**
* Extract a specific number of placed requests for a role
* @param roleId role Id
@@ -376,9 +457,10 @@
*/
public synchronized List<OutstandingRequest> extractPlacedRequestsForRole(int roleId, int count) {
List<OutstandingRequest> results = new ArrayList<>();
- Iterator<OutstandingRequest> iterator = placedRequests.keySet().iterator();
+ Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>>
+ iterator = placedRequests.entrySet().iterator();
while (iterator.hasNext() && count > 0) {
- OutstandingRequest request = iterator.next();
+ OutstandingRequest request = iterator.next().getValue();
if (request.roleId == roleId) {
results.add(request);
count--;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 82b2f2a..c409114 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -26,6 +26,7 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -290,12 +291,18 @@
@Override
public Map<String, NodeInformation> getNodeInformationSnapshot() {
- return appState.getRoleHistory().getNodeInformationSnapshot();
+ return appState.getRoleHistory()
+ .getNodeInformationSnapshot(appState.buildNamingMap());
}
@Override
public NodeInformation getNodeInformation(String hostname) {
- return appState.getRoleHistory().getNodeInformation(hostname);
+ return appState.getRoleHistory()
+ .getNodeInformation(hostname, appState.buildNamingMap());
}
+ @Override
+ public RoleStatistics getRoleStatistics() {
+ return appState.getRoleStatistics();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index df3983a..0584d30 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -19,19 +19,18 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.LongGauge;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.Timestamp;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
@@ -47,11 +46,10 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* The Role History.
@@ -71,12 +69,17 @@
protected static final Logger log =
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
+ /** the roles in here are shared with App State */
+ private final Map<Integer, RoleStatus> roleStatusMap = new HashMap<>();
+ private final AbstractClusterServices recordFactory;
+
private long startTime;
/** Time when saved */
private final Timestamp saveTime = new Timestamp(0);
- /** If the history was loaded, the time at which the history was saved */
+ /** If the history was loaded, the time at which the history was saved.
+ * That is: the time the data was valid */
private final Timestamp thawedDataTime = new Timestamp(0);
private NodeMap nodemap;
@@ -87,8 +90,8 @@
private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
/**
- * When were the nodes updated in a {@link #onNodesUpdated(List)} call.
- * If zero: never
+ * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
+ * If zero: never.
*/
private final Timestamp nodesUpdatedTime = new Timestamp(0);
private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
@@ -98,20 +101,24 @@
/**
* For each role, lists nodes that are available for data-local allocation,
- ordered by more recently released - To accelerate node selection
+ * ordered by more recently released - to accelerate node selection.
+ * That is, they are "recently used nodes"
*/
- private Map<Integer, LinkedList<NodeInstance>> availableNodes;
+ private Map<Integer, LinkedList<NodeInstance>> recentNodes;
/**
- * Track the failed nodes. Currently used to make wiser decision of container
- * ask with/without locality. Has other potential uses as well.
+ * Instantiate
+ * @param roles initial role list
+ * @param recordFactory yarn record factory
+ * @throws BadConfigException
*/
- private Set<String> failedNodes = new HashSet<>();
-
-
- public RoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
- this.providerRoles = providerRoles;
- roleSize = providerRoles.size();
+ public RoleHistory(Collection<RoleStatus> roles, AbstractClusterServices recordFactory) throws BadConfigException {
+ this.recordFactory = recordFactory;
+ roleSize = roles.size();
+ providerRoles = new ArrayList<>(roleSize);
+ for (RoleStatus role : roles) {
+ addNewRole(role);
+ }
reset();
}
@@ -122,15 +129,8 @@
protected synchronized void reset() throws BadConfigException {
nodemap = new NodeMap(roleSize);
- failedNodes = new HashSet<>();
resetAvailableNodeLists();
-
outstandingRequests = new OutstandingRequestTracker();
-
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
- for (ProviderRole providerRole : providerRoles) {
- checkProviderRole(roleStats, providerRole);
- }
}
/**
@@ -146,46 +146,33 @@
}
/**
- * safety check: make sure the provider role is unique amongst
+ * safety check: make sure the role is unique amongst
* the role stats...which is extended with the new role
- * @param roleStats role stats
- * @param providerRole role
+ * @param roleStatus role
* @throws ArrayIndexOutOfBoundsException
* @throws BadConfigException
*/
- protected void checkProviderRole(Map<Integer, RoleStatus> roleStats,
- ProviderRole providerRole)
- throws BadConfigException {
- int index = providerRole.id;
+ protected void putRole(RoleStatus roleStatus) throws BadConfigException {
+ int index = roleStatus.getKey();
if (index < 0) {
- throw new BadConfigException("Provider " + providerRole
- + " id is out of range");
+ throw new BadConfigException("Provider " + roleStatus + " id is out of range");
}
- if (roleStats.get(index) != null) {
+ if (roleStatusMap.get(index) != null) {
throw new BadConfigException(
- providerRole.toString() + " id duplicates that of " +
- roleStats.get(index));
+ roleStatus.toString() + " id duplicates that of " +
+ roleStatusMap.get(index));
}
- roleStats.put(index, new RoleStatus(providerRole));
+ roleStatusMap.put(index, roleStatus);
}
/**
- * Add a new provider role to the map
- * @param providerRole new provider role
+ * Add a new role
+ * @param roleStatus new role
*/
- public void addNewProviderRole(ProviderRole providerRole)
- throws BadConfigException {
- log.debug("Validating/adding new provider role to role history: {} ",
- providerRole);
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
-
- for (ProviderRole role : providerRoles) {
- roleStats.put(role.id, new RoleStatus(role));
- }
-
- checkProviderRole(roleStats, providerRole);
- log.debug("Check successful; adding role");
- this.providerRoles.add(providerRole);
+ public void addNewRole(RoleStatus roleStatus) throws BadConfigException {
+ log.debug("Validating/adding new role to role history: {} ", roleStatus);
+ putRole(roleStatus);
+ this.providerRoles.add(roleStatus.getProviderRole());
}
/**
@@ -206,7 +193,7 @@
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
- availableNodes = new HashMap<>(roleSize);
+ recentNodes = new ConcurrentHashMap<>(roleSize);
}
/**
@@ -322,12 +309,14 @@
/**
* Get snapshot of the node map
* @return a snapshot of the current node state
+ * @param naming naming map of priority to enty name; entries must be unique.
+ * It's OK to be incomplete, for those the list falls back to numbers.
*/
- public Map<String, NodeInformation> getNodeInformationSnapshot() {
- NodeMap map = cloneNodemap();
- Map<String, NodeInformation> result = new HashMap<>(map.size());
- for (Map.Entry<String, NodeInstance> entry : map.entrySet()) {
- result.put(entry.getKey(), entry.getValue().serialize());
+ public synchronized Map<String, NodeInformation> getNodeInformationSnapshot(
+ Map<Integer, String> naming) {
+ Map<String, NodeInformation> result = new HashMap<>(nodemap.size());
+ for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
+ result.put(entry.getKey(), entry.getValue().serialize(naming));
}
return result;
}
@@ -335,11 +324,14 @@
/**
* Get the information on a node
* @param hostname hostname
+ * @param naming naming map of priority to enty name; entries must be unique.
+ * It's OK to be incomplete, for those the list falls back to numbers.
* @return the information about that host, or null if there is none
*/
- public NodeInformation getNodeInformation(String hostname) {
+ public NodeInformation getNodeInformation(String hostname,
+ Map<Integer, String> naming) {
NodeInstance nodeInstance = nodemap.get(hostname);
- return nodeInstance != null ? nodeInstance.serialize() : null;
+ return nodeInstance != null ? nodeInstance.serialize(naming) : null;
}
/**
@@ -363,7 +355,7 @@
public synchronized void insert(Collection<NodeInstance> nodes) {
nodemap.insert(nodes);
}
-
+
/**
* Get current time. overrideable for test subclasses
* @return current time in millis
@@ -435,8 +427,7 @@
* @param historyDir path in FS for history
* @return true if the history was thawed
*/
- public boolean onStart(FileSystem fs, Path historyDir) throws
- BadConfigException {
+ public boolean onStart(FileSystem fs, Path historyDir) throws BadConfigException {
assert filesystem == null;
filesystem = fs;
historyPath = historyDir;
@@ -483,7 +474,7 @@
}
//start is then completed
- buildAvailableNodeLists();
+ buildRecentNodeLists();
} else {
//fallback to bootstrap procedure
onBootstrap();
@@ -496,7 +487,7 @@
* (After the start), rebuild the availability data structures
*/
@VisibleForTesting
- public synchronized void buildAvailableNodeLists() {
+ public synchronized void buildRecentNodeLists() {
resetAvailableNodeLists();
// build the list of available nodes
for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
@@ -505,13 +496,13 @@
NodeEntry nodeEntry = ni.get(i);
if (nodeEntry != null && nodeEntry.isAvailable()) {
log.debug("Adding {} for role {}", ni, i);
- getOrCreateNodesForRoleId(i).add(ni);
+ listRecentNodesForRoleId(i).add(ni);
}
}
}
// sort the resulting arrays
for (int i = 0; i < roleSize; i++) {
- sortAvailableNodeList(i);
+ sortRecentNodeList(i);
}
}
@@ -521,30 +512,35 @@
* @return potentially null list
*/
@VisibleForTesting
- public List<NodeInstance> getNodesForRoleId(int id) {
- return availableNodes.get(id);
+ public List<NodeInstance> getRecentNodesForRoleId(int id) {
+ return recentNodes.get(id);
}
-
+
/**
- * Get the nodes for an ID -may be null
+ * Get a possibly empty list of suggested nodes for a role.
* @param id role ID
* @return list
*/
- private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
- LinkedList<NodeInstance> instances = availableNodes.get(id);
+ private LinkedList<NodeInstance> listRecentNodesForRoleId(int id) {
+ LinkedList<NodeInstance> instances = recentNodes.get(id);
if (instances == null) {
- instances = new LinkedList<>();
- availableNodes.put(id, instances);
+ synchronized (this) {
+ // recheck in the synchronized block and recreate
+ if (recentNodes.get(id) == null) {
+ recentNodes.put(id, new LinkedList<NodeInstance>());
+ }
+ instances = recentNodes.get(id);
+ }
}
return instances;
}
-
+
/**
- * Sort an available node list
+ * Sort a the recent node list for a single role
* @param role role to sort
*/
- private void sortAvailableNodeList(int role) {
- List<NodeInstance> nodesForRoleId = getNodesForRoleId(role);
+ private void sortRecentNodeList(int role) {
+ List<NodeInstance> nodesForRoleId = getRecentNodesForRoleId(role);
if (nodesForRoleId != null) {
Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role));
}
@@ -556,7 +552,7 @@
* @return the instance, or null for none
*/
@VisibleForTesting
- public synchronized NodeInstance findNodeForNewInstance(RoleStatus role) {
+ public synchronized NodeInstance findRecentNodeForNewInstance(RoleStatus role) {
if (!role.isPlacementDesired()) {
// no data locality policy
return null;
@@ -566,7 +562,7 @@
NodeInstance nodeInstance = null;
// Get the list of possible targets.
// This is a live list: changes here are preserved
- List<NodeInstance> targets = getNodesForRoleId(roleId);
+ List<NodeInstance> targets = getRecentNodesForRoleId(roleId);
if (targets == null) {
// nothing to allocate on
return null;
@@ -578,7 +574,8 @@
NodeInstance candidate = targets.get(i);
if (candidate.getActiveRoleInstances(roleId) == 0) {
// no active instances: check failure statistics
- if (strictPlacement || !candidate.exceedsFailureThreshold(role)) {
+ if (strictPlacement
+ || (candidate.isOnline() && !candidate.exceedsFailureThreshold(role))) {
targets.remove(i);
// exit criteria for loop is now met
nodeInstance = candidate;
@@ -597,6 +594,18 @@
}
/**
+ * Find a node for use
+ * @param role role
+ * @return the instance, or null for none
+ */
+ @VisibleForTesting
+ public synchronized List<NodeInstance> findNodeForNewAAInstance(RoleStatus role) {
+ // all nodes that are live and can host the role; no attempt to exclude ones
+ // considered failing
+ return nodemap.findAllNodesForRole(role.getKey(), role.getLabelExpression());
+ }
+
+ /**
* Request an instance on a given node.
* An outstanding request is created & tracked, with the
* relevant node entry for that role updated.
@@ -606,47 +615,57 @@
* Returns the request that is now being tracked.
* If the node instance is not null, it's details about the role is incremented
*
- *
* @param node node to target or null for "any"
* @param role role to request
- * @param labelExpression label to satisfy
- * @return the container priority
+ * @return the request
*/
- public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
- NodeInstance node, RoleStatus role, Resource resource, String labelExpression) {
+ public synchronized OutstandingRequest requestInstanceOnNode(
+ NodeInstance node, RoleStatus role, Resource resource) {
OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
- return outstanding.buildContainerRequest(resource, role, now(), labelExpression);
- }
-
- /**
- * Find a node for a role and request an instance on that (or a location-less
- * instance) with a label expression
- * @param role role status
- * @param resource resource capabilities
- * @param labelExpression label to satisfy
- * @return a request ready to go
- */
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource,
- String labelExpression) {
- NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, labelExpression);
+ outstanding.buildContainerRequest(resource, role, now());
+ return outstanding;
}
/**
* Find a node for a role and request an instance on that (or a location-less
* instance)
* @param role role status
- * @param resource resource capabilities
- * @return a request ready to go
+ * @return a request ready to go, or null if this is an AA request and no
+ * location can be found.
*/
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource) {
- NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, null);
+ public synchronized OutstandingRequest requestContainerForRole(RoleStatus role) {
+
+ if (role.isAntiAffinePlacement()) {
+ return requestContainerForAARole(role);
+ } else {
+ Resource resource = recordFactory.newResource();
+ role.copyResourceRequirements(resource);
+ NodeInstance node = findRecentNodeForNewInstance(role);
+ return requestInstanceOnNode(node, role, resource);
+ }
}
/**
+ * Find a node for an AA role and request an instance on that (or a location-less
+ * instance)
+ * @param role role status
+ * @return a request ready to go, or null if no location can be found.
+ */
+ public synchronized OutstandingRequest requestContainerForAARole(RoleStatus role) {
+ List<NodeInstance> nodes = findNodeForNewAAInstance(role);
+ if (!nodes.isEmpty()) {
+ OutstandingRequest outstanding = outstandingRequests.newAARequest(
+ role.getKey(), nodes, role.getLabelExpression());
+ Resource resource = recordFactory.newResource();
+ role.copyResourceRequirements(resource);
+ outstanding.buildContainerRequest(resource, role, now());
+ return outstanding;
+ } else {
+ log.warn("No suitable location for {}", role.getName());
+ return null;
+ }
+ }
+ /**
* Get the list of active nodes ... walks the node map so
* is {@code O(nodes)}
* @param role role index
@@ -655,7 +674,7 @@
public synchronized List<NodeInstance> listActiveNodes(int role) {
return nodemap.listActiveNodes(role);
}
-
+
/**
* Get the node entry of a container
* @param container container to look up
@@ -663,8 +682,7 @@
* @throws RuntimeException if the container has no hostname
*/
public NodeEntry getOrCreateNodeEntry(Container container) {
- NodeInstance node = getOrCreateNodeInstance(container);
- return node.getOrCreate(ContainerPriority.extractRole(container));
+ return getOrCreateNodeInstance(container).getOrCreate(container);
}
/**
@@ -705,7 +723,7 @@
* @return list of containers potentially reordered
*/
public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) {
-
+
//partition into requested and unrequested
List<Container> requested =
new ArrayList<>(allocatedContainers.size());
@@ -717,7 +735,7 @@
requested.addAll(unrequested);
return requested;
}
-
+
/**
* A container has been allocated on a node -update the data structures
* @param container container
@@ -725,13 +743,14 @@
* @param actualCount current count of instances
* @return The allocation outcome
*/
- public synchronized ContainerAllocation onContainerAllocated(Container container,
- int desiredCount,
- int actualCount) {
+ public synchronized ContainerAllocationResults onContainerAllocated(Container container,
+ long desiredCount,
+ long actualCount) {
int role = ContainerPriority.extractRole(container);
+
String hostname = RoleHistoryUtils.hostnameOf(container);
- List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
- ContainerAllocation outcome =
+ List<NodeInstance> nodeInstances = listRecentNodesForRoleId(role);
+ ContainerAllocationResults outcome =
outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
@@ -741,7 +760,7 @@
//add the list
log.info("Adding {} hosts for role {}", hosts.size(), role);
nodeInstances.addAll(hosts);
- sortAvailableNodeList(role);
+ sortRecentNodeList(role);
}
}
return outcome;
@@ -752,8 +771,10 @@
* @param container container
*/
public void onContainerAssigned(Container container) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ NodeInstance node = getOrCreateNodeInstance(container);
+ NodeEntry nodeEntry = node.getOrCreate(container);
nodeEntry.onStarting();
+ log.debug("Node {} has updated NodeEntry {}", node, nodeEntry);
}
/**
@@ -763,9 +784,7 @@
*/
public void onContainerStartSubmitted(Container container,
RoleInstance instance) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
- int role = ContainerPriority.extractRole(container);
- // any actions we want here
+ // no actions here
}
/**
@@ -789,6 +808,16 @@
}
/**
+ * Does the RoleHistory have enough information about the YARN cluster
+ * to start placing AA requests? That is: has it the node map and
+ * any label information needed?
+ * @return true if the caller can start requesting AA nodes
+ */
+ public boolean canPlaceAANodes() {
+ return nodeUpdateReceived.get();
+ }
+
+ /**
* Get the last time the nodes were updated from YARN
* @return the update time or zero if never updated.
*/
@@ -798,33 +827,35 @@
/**
* Update failedNodes and nodemap based on the node state
- *
+ *
* @param updatedNodes list of updated nodes
+ * @return true if a review should be triggered.
*/
- public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
+ public synchronized boolean onNodesUpdated(List<NodeReport> updatedNodes) {
log.debug("Updating {} nodes", updatedNodes.size());
nodesUpdatedTime.set(now());
nodeUpdateReceived.set(true);
+ int printed = 0;
+ boolean triggerReview = false;
for (NodeReport updatedNode : updatedNodes) {
String hostname = updatedNode.getNodeId() == null
- ? null
- : updatedNode.getNodeId().getHost();
+ ? ""
+ : updatedNode.getNodeId().getHost();
NodeState nodeState = updatedNode.getNodeState();
- if (hostname == null || nodeState == null) {
+ if (hostname.isEmpty() || nodeState == null) {
+ log.warn("Ignoring incomplete update");
continue;
}
+ if (log.isDebugEnabled() && printed++ < 10) {
+ // log the first few, but avoid overloading the logs for a full cluster
+ // update
+ log.debug("Node \"{}\" is in state {}", hostname, nodeState);
+ }
// update the node; this also creates an instance if needed
boolean updated = nodemap.updateNode(hostname, updatedNode);
- if (updated) {
- log.debug("Updated host {} to state {}", hostname, nodeState);
- if (nodeState.isUnusable()) {
- log.info("Failed node {}", hostname);
- failedNodes.add(hostname);
- } else {
- failedNodes.remove(hostname);
- }
- }
+ triggerReview |= updated;
}
+ return triggerReview;
}
/**
@@ -862,7 +893,7 @@
/**
* Mark a container finished; if it was released then that is treated
- * differently. history is touch()ed
+ * differently. history is {@code touch()}-ed
*
*
* @param container completed container
@@ -892,7 +923,7 @@
/**
* If the node is marked as available; queue it for assignments.
- * Unsynced: expects caller to be in a sync block.
+ * Unsynced: requires caller to be in a sync block.
* @param container completed container
* @param nodeEntry node
* @param available available flag
@@ -907,7 +938,7 @@
NodeInstance ni = getOrCreateNodeInstance(container);
int roleId = ContainerPriority.extractRole(container);
log.debug("Node {} is now available for role id {}", ni, roleId);
- getOrCreateNodesForRoleId(roleId).addFirst(ni);
+ listRecentNodesForRoleId(roleId).addFirst(ni);
}
return available;
}
@@ -918,8 +949,7 @@
public synchronized void dump() {
for (ProviderRole role : providerRoles) {
log.info(role.toString());
- List<NodeInstance> instances =
- getOrCreateNodesForRoleId(role.id);
+ List<NodeInstance> instances = listRecentNodesForRoleId(role.id);
log.info(" available: " + instances.size()
+ " " + SliderUtils.joinWithInnerSeparator(" ", instances));
}
@@ -928,9 +958,6 @@
for (NodeInstance node : nodemap.values()) {
log.info(node.toFullString());
}
-
- log.info("Failed nodes: {}",
- SliderUtils.joinWithInnerSeparator(" ", failedNodes));
}
/**
@@ -951,8 +978,8 @@
* @return a clone of the list
*/
@VisibleForTesting
- public List<NodeInstance> cloneAvailableList(int role) {
- return new LinkedList<>(getOrCreateNodesForRoleId(role));
+ public List<NodeInstance> cloneRecentNodeList(int role) {
+ return new LinkedList<>(listRecentNodesForRoleId(role));
}
/**
@@ -974,31 +1001,42 @@
}
/**
- * Get a clone of the failedNodes
- *
- * @return the list
- */
- public List<String> cloneFailedNodes() {
- List<String> lst = new ArrayList<>();
- lst.addAll(failedNodes);
- return lst;
- }
-
- /**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
*/
public List<AbstractRMOperation> escalateOutstandingRequests() {
return outstandingRequests.escalateOutstandingRequests(now());
}
+ /**
+ * Escalate operation as triggered by external timer.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public List<AbstractRMOperation> cancelOutstandingAARequests() {
+ return outstandingRequests.cancelOutstandingAARequests();
+ }
+
+ /**
+ * Cancel a number of outstanding requests for a role -that is, not
+ * actual containers, just requests for new ones.
+ * @param role role
+ * @param toCancel number to cancel
+ * @return a list of cancellable operations.
+ */
+ public List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
+ return role.isAntiAffinePlacement() ?
+ cancelRequestsForAARole(role, toCancel)
+ : cancelRequestsForSimpleRole(role, toCancel);
+ }
/**
* Build the list of requests to cancel from the outstanding list.
- * @param role
- * @param toCancel
+ * @param role role
+ * @param toCancel number to cancel
* @return a list of cancellable operations.
*/
- public synchronized List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
+ private synchronized List<AbstractRMOperation> cancelRequestsForSimpleRole(RoleStatus role, int toCancel) {
+ Preconditions.checkArgument(toCancel > 0,
+ "trying to cancel invalid number of requests: " + toCancel);
List<AbstractRMOperation> results = new ArrayList<>(toCancel);
// first scan through the unplaced request list to find all of a role
int roleId = role.getKey();
@@ -1017,4 +1055,47 @@
return results;
}
+ /**
+ * Build the list of requests to cancel for an AA role. This reduces the number
+ * of outstanding pending requests first, then cancels any active request,
+ * before finally asking for any placed containers
+ * @param role role
+ * @param toCancel number to cancel
+ * @return a list of cancellable operations.
+ */
+ private synchronized List<AbstractRMOperation> cancelRequestsForAARole(RoleStatus role, int toCancel) {
+ List<AbstractRMOperation> results = new ArrayList<>(toCancel);
+ int roleId = role.getKey();
+ List<OutstandingRequest> requests = new ArrayList<>(toCancel);
+ // there may be pending requests which can be cancelled here
+ long pending = role.getPendingAntiAffineRequests();
+ if (pending > 0) {
+ // there are some pending ones which can be cancelled first
+ long pendingToCancel = Math.min(pending, toCancel);
+ log.info("Cancelling {} pending AA allocations, leaving {}", toCancel,
+ pendingToCancel);
+ role.setPendingAntiAffineRequests(pending - pendingToCancel);
+ toCancel -= pendingToCancel;
+ }
+ if (toCancel > 0 && role.isAARequestOutstanding()) {
+ // not enough
+ log.info("Cancelling current AA request");
+ // find the single entry which may be running
+ requests = outstandingRequests.extractOpenRequestsForRole(roleId, toCancel);
+ role.cancelOutstandingAARequest();
+ toCancel--;
+ }
+
+ // ask for some excess nodes
+ if (toCancel > 0) {
+ requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, toCancel));
+ }
+
+ // build cancellations
+ for (OutstandingRequest request : requests) {
+ results.add(request.createCancelOperation());
+ }
+ return results;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
new file mode 100644
index 0000000..920887a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.Objects;
+
+public class RoleHostnamePair {
+
+ /**
+ * requested role
+ */
+ public final int roleId;
+
+ /**
+ * hostname -will be null if node==null
+ */
+ public final String hostname;
+
+ public RoleHostnamePair(int roleId, String hostname) {
+ this.roleId = roleId;
+ this.hostname = hostname;
+ }
+
+ public int getRoleId() {
+ return roleId;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof RoleHostnamePair)) {
+ return false;
+ }
+ RoleHostnamePair that = (RoleHostnamePair) o;
+ return Objects.equals(roleId, that.roleId) &&
+ Objects.equals(hostname, that.hostname);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(roleId, hostname);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "RoleHostnamePair{");
+ sb.append("roleId=").append(roleId);
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 98a8311..656f96c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -18,22 +18,32 @@
package org.apache.slider.server.appmaster.state;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.BoolMetric;
+import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
+import org.apache.slider.server.appmaster.management.LongGauge;
import java.io.Serializable;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
- * Models the ongoing status of all nodes in
- * Nothing here is synchronized: grab the whole instance to update.
+ * Models the ongoing status of all nodes in an application.
+ *
+ * These structures are shared across the {@link AppState} and {@link RoleHistory} structures,
+ * and must be designed for synchronous access. Atomic counters are preferred to anything which
+ * requires synchronization. Where synchronized access is good is that it allows for
+ * the whole instance to be locked, for updating multiple entries.
*/
-public final class RoleStatus implements Cloneable {
+public final class RoleStatus implements Cloneable, MetricSet {
private final String name;
@@ -41,25 +51,31 @@
* Role priority
*/
private final int key;
-
private final ProviderRole providerRole;
- private int desired, actual, requested, releasing;
- private int failed, startFailed;
- private int started, completed, totalRequested;
- private final AtomicLong preempted = new AtomicLong(0);
- private final AtomicLong nodeFailed = new AtomicLong(0);
- private final AtomicLong failedRecently = new AtomicLong(0);
- private final AtomicLong limitsExceeded = new AtomicLong(0);
+ private final LongGauge actual = new LongGauge();
+ private final LongGauge completed = new LongGauge();
+ private final LongGauge desired = new LongGauge();
+ private final LongGauge failed = new LongGauge();
+ private final LongGauge failedRecently = new LongGauge(0);
+ private final LongGauge limitsExceeded = new LongGauge(0);
+ private final LongGauge nodeFailed = new LongGauge(0);
+ /** Number of AA requests queued. */
+ private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
+ private final LongGauge preempted = new LongGauge(0);
+ private final LongGauge releasing = new LongGauge();
+ private final LongGauge requested = new LongGauge();
+ private final LongGauge started = new LongGauge();
+ private final LongGauge startFailed = new LongGauge();
+ private final LongGauge totalRequested = new LongGauge();
- /** flag set to true if there is an outstanding anti-affine request */
- private final AtomicBoolean pendingAARequest = new AtomicBoolean(false);
+ /** resource requirements */
+ private Resource resourceRequirements;
- /**
- * Number of AA requests queued. These should be reduced first on a
- * flex down.
- */
- private int pendingAntiAffineRequestCount = 0;
+
+ /** any pending AA request */
+ private volatile OutstandingRequest outstandingAArequest = null;
+
private String failureMessage = "";
@@ -68,7 +84,37 @@
this.name = providerRole.name;
this.key = providerRole.id;
}
-
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> metrics = new HashMap<>(15);
+ metrics.put("actual", actual);
+ metrics.put("completed", completed );
+ metrics.put("desired", desired);
+ metrics.put("failed", failed);
+ metrics.put("limitsExceeded", limitsExceeded);
+ metrics.put("nodeFailed", nodeFailed);
+ metrics.put("preempted", preempted);
+ metrics.put("pendingAntiAffineRequests", pendingAntiAffineRequests);
+ metrics.put("releasing", releasing);
+ metrics.put("requested", requested);
+ metrics.put("preempted", preempted);
+ metrics.put("releasing", releasing );
+ metrics.put("requested", requested);
+ metrics.put("started", started);
+ metrics.put("startFailed", startFailed);
+ metrics.put("totalRequested", totalRequested);
+
+ metrics.put("outstandingAArequest",
+ new BoolMetricPredicate(new BoolMetricPredicate.Eval() {
+ @Override
+ public boolean eval() {
+ return isAARequestOutstanding();
+ }
+ }));
+ return metrics;
+ }
+
public String getName() {
return name;
}
@@ -123,63 +169,64 @@
return !hasPlacementPolicy(PlacementPolicy.NO_DATA_LOCALITY);
}
- public synchronized int getDesired() {
- return desired;
+ public long getDesired() {
+ return desired.get();
}
- public synchronized void setDesired(int desired) {
- this.desired = desired;
+ public void setDesired(long desired) {
+ this.desired.set(desired);
}
- public synchronized int getActual() {
- return actual;
+ public long getActual() {
+ return actual.get();
}
- public synchronized int incActual() {
- return ++actual;
+ public long incActual() {
+ return actual.incrementAndGet();
}
- public synchronized int decActual() {
- actual = Math.max(0, actual - 1);
- return actual;
+ public long decActual() {
+ return actual.decToFloor(1);
}
- public synchronized int getRequested() {
- return requested;
+ /**
+ * Get the request count.
+ * @return a count of requested containers
+ */
+ public long getRequested() {
+ return requested.get();
}
- public synchronized int incRequested() {
- totalRequested++;
- return ++requested;
+ public long incRequested() {
+ totalRequested.incrementAndGet();
+ return requested.incrementAndGet();
}
- public synchronized int cancel(int count) {
- requested = Math.max(0, requested - count);
- return requested;
- }
-
- public synchronized int decRequested() {
- return cancel(1);
+ public void cancel(long count) {
+ requested.decToFloor(count);
}
- public synchronized int getReleasing() {
- return releasing;
+ public void decRequested() {
+ cancel(1);
}
- public synchronized int incReleasing() {
- return ++releasing;
+ public long getReleasing() {
+ return releasing.get();
}
- public synchronized int decReleasing() {
- releasing = Math.max(0, releasing - 1);
- return releasing;
+ public long incReleasing() {
+ return releasing.incrementAndGet();
}
- public synchronized int getFailed() {
- return failed;
+ public long decReleasing() {
+ return releasing.decToFloor(1);
}
- public synchronized long getFailedRecently() {
+ public long getFailed() {
+ return failed.get();
+ }
+
+ public long getFailedRecently() {
return failedRecently.get();
}
@@ -195,6 +242,26 @@
return limitsExceeded.get();
}
+ public long incPendingAntiAffineRequests(long v) {
+ return pendingAntiAffineRequests.addAndGet(v);
+ }
+
+ /**
+ * Probe for an outstanding AA request being true
+ * @return true if there is an outstanding AA Request
+ */
+ public boolean isAARequestOutstanding() {
+ return outstandingAArequest != null;
+ }
+
+ /**
+ * expose the predicate {@link #isAARequestOutstanding()} as an integer,
+ * which is very convenient in tests
+ * @return 1 if there is an outstanding request; 0 if not
+ */
+ public int getOutstandingAARequestCount() {
+ return isAARequestOutstanding()? 1: 0;
+ }
/**
* Note that a role failed, text will
* be used in any diagnostics if an exception
@@ -215,7 +282,7 @@
case Node_failure:
nodeFailed.incrementAndGet();
- failed++;
+ failed.incrementAndGet();
break;
case Failed_limits_exceeded: // exceeded memory or CPU; app/configuration related
@@ -223,7 +290,7 @@
// fall through
case Failed: // application failure, possibly node related, possibly not
default: // anything else (future-proofing)
- failed++;
+ failed.incrementAndGet();
failedRecently.incrementAndGet();
//have a look to see if it short lived
if (startupFailure) {
@@ -233,39 +300,39 @@
}
}
- public synchronized int getStartFailed() {
- return startFailed;
+ public long getStartFailed() {
+ return startFailed.get();
}
public synchronized void incStartFailed() {
- startFailed++;
+ startFailed.getAndIncrement();
}
public synchronized String getFailureMessage() {
return failureMessage;
}
- public synchronized int getCompleted() {
- return completed;
+ public long getCompleted() {
+ return completed.get();
}
public synchronized void setCompleted(int completed) {
- this.completed = completed;
+ this.completed.set(completed);
}
- public synchronized int incCompleted() {
- return completed ++;
+ public long incCompleted() {
+ return completed.incrementAndGet();
}
- public synchronized int getStarted() {
- return started;
+ public long getStarted() {
+ return started.get();
}
public synchronized void incStarted() {
- started++;
+ started.incrementAndGet();
}
- public synchronized int getTotalRequested() {
- return totalRequested;
+ public long getTotalRequested() {
+ return totalRequested.get();
}
public long getPreempted() {
@@ -276,19 +343,58 @@
return nodeFailed.get();
}
+ public long getPendingAntiAffineRequests() {
+ return pendingAntiAffineRequests.get();
+ }
+
+ public void setPendingAntiAffineRequests(long pendingAntiAffineRequests) {
+ this.pendingAntiAffineRequests.set(pendingAntiAffineRequests);
+ }
+
+ public long decPendingAntiAffineRequests() {
+ return pendingAntiAffineRequests.decToFloor(1);
+ }
+
+ public OutstandingRequest getOutstandingAArequest() {
+ return outstandingAArequest;
+ }
+
+ public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) {
+ this.outstandingAArequest = outstandingAArequest;
+ }
+
+ /**
+ * Complete the outstanding AA request (there's no check for one in progress, caller
+ * expected to have done that).
+ */
+ public void completeOutstandingAARequest() {
+ setOutstandingAArequest(null);
+ }
+
+ /**
+ * Cancel any outstanding AA request. Harmless if the role is non-AA, or
+ * if there are no outstanding requests.
+ */
+ public void cancelOutstandingAARequest() {
+ if (outstandingAArequest != null) {
+ setOutstandingAArequest(null);
+ setPendingAntiAffineRequests(0);
+ decRequested();
+ }
+ }
+
/**
* Get the number of roles we are short of.
* nodes released are ignored.
* @return the positive or negative number of roles to add/release.
* 0 means "do nothing".
*/
- public synchronized int getDelta() {
- int inuse = getActualAndRequested();
- //don't know how to view these. Are they in-use or not?
- int delta = desired - inuse;
+ public long getDelta() {
+ long inuse = getActualAndRequested();
+ long delta = desired.get() - inuse;
if (delta < 0) {
//if we are releasing, remove the number that are already released.
- delta += releasing;
+ delta += releasing.get();
//but never switch to a positive
delta = Math.min(delta, 0);
}
@@ -296,32 +402,41 @@
}
/**
- * Get count of actual and requested containers
- * @return the size of the application when outstanding requests are included
+ * Get count of actual and requested containers. This includes pending ones
+ * @return the size of the application when outstanding requests are included.
*/
- public synchronized int getActualAndRequested() {
- return actual + requested;
+ public long getActualAndRequested() {
+ return actual.get() + requested.get();
}
@Override
- public synchronized String toString() {
- return "RoleStatus{" +
- "name='" + name + '\'' +
- ", key=" + key +
- ", desired=" + desired +
- ", actual=" + actual +
- ", requested=" + requested +
- ", releasing=" + releasing +
- ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount +
- ", failed=" + failed +
- ", failed recently=" + failedRecently.get() +
- ", node failed=" + nodeFailed.get() +
- ", pre-empted=" + preempted.get() +
- ", started=" + started +
- ", startFailed=" + startFailed +
- ", completed=" + completed +
- ", failureMessage='" + failureMessage + '\'' +
- '}';
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("RoleStatus{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", key=").append(key);
+ sb.append(", desired=").append(desired);
+ sb.append(", actual=").append(actual);
+ sb.append(", requested=").append(requested);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", failed=").append(failed);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", started=").append(started);
+ sb.append(", completed=").append(completed);
+ sb.append(", totalRequested=").append(totalRequested);
+ sb.append(", preempted=").append(preempted);
+ sb.append(", nodeFailed=").append(nodeFailed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", limitsExceeded=").append(limitsExceeded);
+ sb.append(", resourceRequirements=").append(resourceRequirements);
+ sb.append(", isAntiAffinePlacement=").append(isAntiAffinePlacement());
+ if (isAntiAffinePlacement()) {
+ sb.append(", pendingAntiAffineRequests=").append(pendingAntiAffineRequests);
+ sb.append(", outstandingAArequest=").append(outstandingAArequest);
+ }
+ sb.append(", failureMessage='").append(failureMessage).append('\'');
+ sb.append(", providerRole=").append(providerRole);
+ sb.append('}');
+ return sb.toString();
}
@Override
@@ -354,23 +469,39 @@
ComponentInformation info = new ComponentInformation();
info.name = name;
info.priority = getPriority();
- info.desired = desired;
- info.actual = actual;
- info.requested = requested;
- info.releasing = releasing;
- info.failed = failed;
- info.startFailed = startFailed;
+ info.desired = desired.intValue();
+ info.actual = actual.intValue();
+ info.requested = requested.intValue();
+ info.releasing = releasing.intValue();
+ info.failed = failed.intValue();
+ info.startFailed = startFailed.intValue();
info.placementPolicy = getPlacementPolicy();
info.failureMessage = failureMessage;
- info.totalRequested = totalRequested;
+ info.totalRequested = totalRequested.intValue();
info.failedRecently = failedRecently.intValue();
info.nodeFailed = nodeFailed.intValue();
info.preempted = preempted.intValue();
- info.pendingAntiAffineRequest = pendingAARequest.get();
- info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount;
+ info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue();
+ info.isAARequestOutstanding = isAARequestOutstanding();
return info;
}
-
+
+ /**
+ * Get the (possibly null) label expression for this role
+ * @return a string or null
+ */
+ public String getLabelExpression() {
+ return providerRole.labelExpression;
+ }
+
+ public Resource getResourceRequirements() {
+ return resourceRequirements;
+ }
+
+ public void setResourceRequirements(Resource resourceRequirements) {
+ this.resourceRequirements = resourceRequirements;
+ }
+
/**
* Compare two role status entries by name
*/
@@ -392,5 +523,35 @@
return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1));
}
}
-
+
+ /**
+ * Given a resource, set its requirements to those this role needs
+ * @param resource resource to configure
+ * @return the resource
+ */
+ public Resource copyResourceRequirements(Resource resource) {
+ Preconditions.checkNotNull(resourceRequirements,
+ "Role resource requirements have not been set");
+ resource.setMemory(resourceRequirements.getMemory());
+ resource.setVirtualCores(resourceRequirements.getVirtualCores());
+ return resource;
+ }
+
+ public synchronized RoleStatistics getStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ stats.activeAA = getOutstandingAARequestCount();
+ stats.actual = actual.get();
+ stats.desired = desired.get();
+ stats.failed = failed.get();
+ stats.limitsExceeded = limitsExceeded.get();
+ stats.nodeFailed = nodeFailed.get();
+ stats.preempted = preempted.get();
+ stats.releasing = releasing.get();
+ stats.requested = requested.get();
+ stats.started = started.get();
+ stats.startFailed = startFailed.get();
+ stats.totalRequested = totalRequested.get();
+ return stats;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
index b7f0e05..b848096 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
@@ -27,8 +27,7 @@
@Override
public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect) {
+ List<RoleInstance> candidates) {
return candidates;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 2fc00b2..ad91183 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -27,6 +27,7 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -303,4 +304,10 @@
* @return the information, or null if there is no information held.
*/
NodeInformation getNodeInformation(String hostname);
+
+ /**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ RoleStatistics getRoleStatistics();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 84f0eba..7ecc00c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -92,7 +92,7 @@
String regex = "(?!/ws)";
serveRegex(regex).with(SliderDefaultWrapperServlet.class);
- Map<String, String> params = new HashMap<String, String>();
+ Map<String, String> params = new HashMap<>();
params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index cfddb12..7ff83b6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -45,12 +45,11 @@
protected final WebAppApi slider;
protected final MetricsAndMonitoring metricsAndMonitoring;
- public AbstractSliderResource(WebAppApi slider) {
+ protected AbstractSliderResource(WebAppApi slider) {
this.slider = slider;
metricsAndMonitoring = slider.getMetricsAndMonitoring();
}
-
/**
* Generate a redirect to the WASL
* @param request to base the URL on
@@ -105,6 +104,7 @@
protected void mark(String verb, String path) {
metricsAndMonitoring.markMeterAndCounter(verb + "-" + path);
}
+
/**
* Mark an GET operation on a path
* @param verb HTTP Verb
@@ -129,6 +129,7 @@
protected void markGet(String path, String subpath) {
mark("GET", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -136,6 +137,7 @@
protected void markPost(String path, String subpath) {
mark("POST", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -143,6 +145,7 @@
protected void markPut(String path, String subpath) {
mark("PUT", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 424107c..b90eb62 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -148,4 +148,15 @@
public static final String ACTION = "/action";
public static final String ACTION_PING = ACTION + "/ping";
public static final String ACTION_STOP = ACTION + "/stop";
+
+ /**
+ * Path to a role
+ * @param name role name
+ * @return a path to it
+ */
+ public String pathToRole(String name) {
+
+ // ws/v1/slider/application/live/components/$name
+ return SLIDER_PATH_APPLICATION + LIVE_COMPONENTS + "/" + name;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index 1b54a31..52068d6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -26,6 +26,7 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -350,31 +351,31 @@
@GET
@Path(LIVE_NODES)
@Produces({APPLICATION_JSON})
- public Map<String, NodeInformation> getLiveNodes() {
+ public NodeInformationList getLiveNodes() {
markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS);
try {
- return (Map<String, NodeInformation>) cache.lookup(LIVE_NODES);
+ return (NodeInformationList) cache.lookup(LIVE_NODES);
} catch (Exception e) {
throw buildException(LIVE_COMPONENTS, e);
}
}
@GET
- @Path(LIVE_NODES + "/{node}")
+ @Path(LIVE_NODES + "/{hostname}")
@Produces({APPLICATION_JSON})
- public NodeInformation getLiveNode(@PathParam("node") String node) {
+ public NodeInformation getLiveNode(@PathParam("hostname") String hostname) {
markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS);
try {
- NodeInformation ni = state.getNodeInformation(node);
+ NodeInformation ni = state.getNodeInformation(hostname);
if (ni != null) {
return ni;
} else {
- throw new NotFoundException("Unknown node: " + node);
+ throw new NotFoundException("Unknown node: " + hostname);
}
} catch (NotFoundException e) {
throw e;
} catch (Exception e) {
- throw buildException(LIVE_CONTAINERS + "/" + node, e);
+ throw buildException(LIVE_COMPONENTS + "/" + hostname, e);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
index d4ab8fe..aeb7a11 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
@@ -18,16 +18,14 @@
package org.apache.slider.server.appmaster.web.rest.application.resources;
-import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import java.util.Map;
-
/**
* Update the live nodes map
*/
public class LiveNodesRefresher
- implements ResourceRefresher<Map<String, NodeInformation>> {
+ implements ResourceRefresher<NodeInformationList> {
private final StateAccessForProviders state;
@@ -36,7 +34,8 @@
}
@Override
- public Map<String, NodeInformation> refresh() {
- return state.getNodeInformationSnapshot();
+ public NodeInformationList refresh() {
+
+ return new NodeInformationList(state.getNodeInformationSnapshot().values());
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
index 75d417b..794daf9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -37,23 +37,13 @@
public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) {
if (uriBuilder != null) {
- this.href =
- uriBuilder.build().toASCIIString();
- resources =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "resources"));
- internal =
- ResourceFactory.createConfTreeResource(conf.getInternal(),
- uriBuilder.clone().path(
- "configurations").path(
- "internal"));
- appConf =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "appConf"));
+ this.href = uriBuilder.build().toASCIIString();
+ resources = ResourceFactory.createConfTreeResource(conf.getResources(),
+ uriBuilder.clone().path("configurations").path("resources"));
+ internal = ResourceFactory.createConfTreeResource(conf.getInternal(),
+ uriBuilder.clone().path("configurations").path("internal"));
+ appConf = ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path("configurations").path("appConf"));
initConfMap();
} else {
resources = null;
@@ -63,7 +53,7 @@
}
private void initConfMap() {
- confMap = new HashMap<String, ConfTreeResource>();
+ confMap = new HashMap<>();
confMap.put("internal", internal);
confMap.put("resources", resources);
confMap.put("appConf", appConf);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
index 137c476..2f02f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
@@ -18,23 +18,16 @@
import com.google.inject.Inject;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*
*/
-public class ClusterSpecificationBlock extends HtmlBlock {
- private static final Logger log = LoggerFactory.getLogger(ClusterSpecificationBlock.class);
-
- private StateAccessForProviders appState;
+public class ClusterSpecificationBlock extends SliderHamletBlock {
@Inject
public ClusterSpecificationBlock(WebAppApi slider) {
- this.appState = slider.getAppState();
+ super(slider);
}
@Override
@@ -50,7 +43,7 @@
pre().
_(getJson())._()._();
}
-
+
/**
* Get the JSON, catching any exceptions and returning error text instead
* @return
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 0896e2b..56285c2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -26,12 +26,10 @@
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import java.io.Serializable;
@@ -45,19 +43,18 @@
/**
*
*/
-public class ContainerStatsBlock extends HtmlBlock {
+public class ContainerStatsBlock extends SliderHamletBlock {
private static final String EVEN = "even", ODD = "odd", BOLD = "bold", SCHEME = "http://", PATH = "/node/container/";
// Some functions that help transform the data into an object we can use to abstract presentation specifics
protected static final Function<Entry<String,Integer>,Entry<TableContent,Integer>> stringIntPairFunc = toTableContentFunction();
+ protected static final Function<Entry<String,Long>,Entry<TableContent,Long>> stringLongPairFunc = toTableContentFunction();
protected static final Function<Entry<String,String>,Entry<TableContent,String>> stringStringPairFunc = toTableContentFunction();
- private WebAppApi slider;
-
@Inject
public ContainerStatsBlock(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
}
/**
@@ -92,7 +89,6 @@
@Override
protected void render(Block html) {
- StateAccessForProviders appState = slider.getAppState();
final Map<String,RoleInstance> containerInstances = getContainerInstances(
appState.cloneOwnedContainerList());
@@ -108,7 +104,7 @@
DIV<Hamlet> div = html.div("role-info ui-widget-content ui-corner-all");
List<ClusterNode> nodesInRole =
- new ArrayList<ClusterNode>(clusterNodesInRole.values());
+ new ArrayList<>(clusterNodesInRole.values());
div.h2(BOLD, StringUtils.capitalize(name));
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 5131b5e..b3be3bf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -21,50 +21,52 @@
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
+
/**
- *
+ * The main content on the Slider AM web page
*/
-public class IndexBlock extends HtmlBlock {
+public class IndexBlock extends SliderHamletBlock {
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
- private StateAccessForProviders appView;
- private ProviderService providerService;
+ /**
+ * Message printed when application is at full size.
+ *
+ * {@value}
+ */
+ public static final String ALL_CONTAINERS_ALLOCATED = "all containers allocated";
@Inject
public IndexBlock(WebAppApi slider) {
- this.appView = slider.getAppState();
- this.providerService = slider.getProviderService();
+ super(slider);
}
@Override
protected void render(Block html) {
- final String providerName = getProviderName();
-
- doIndex(html, providerName);
+ doIndex(html, getProviderName());
}
// An extra method to make testing easier since you can't make an instance of Block
@VisibleForTesting
protected void doIndex(Hamlet html, String providerName) {
- ClusterDescription clusterStatus = appView.getClusterStatus();
+ ClusterDescription clusterStatus = appState.getClusterStatus();
String name = clusterStatus.name;
if (name != null && (name.startsWith(" ") || name.endsWith(" "))) {
name = "'" + name + "'";
@@ -74,10 +76,9 @@
"Application: " + name);
ApplicationLivenessInformation liveness =
- appView.getApplicationLivenessInformation();
- String livestatus =
- liveness.allRequestsSatisfied
- ? "all containers allocated"
+ appState.getApplicationLivenessInformation();
+ String livestatus = liveness.allRequestsSatisfied
+ ? ALL_CONTAINERS_ALLOCATED
: String.format("Awaiting %d containers", liveness.requestsOutstanding);
Hamlet.TABLE<DIV<Hamlet>> table1 = div.table();
table1.tr()
@@ -86,7 +87,7 @@
._();
table1.tr()
.td("Total number of containers")
- .td(Integer.toString(appView.getNumOwnedContainers()))
+ .td(Integer.toString(appState.getNumOwnedContainers()))
._();
table1.tr()
.td("Create time: ")
@@ -108,44 +109,132 @@
.td("Application configuration path: ")
.td(clusterStatus.originConfigurationPath)
._();
- table1._()._();
+ table1._();
+ div._();
+ div = null;
+ DIV<Hamlet> containers = html.div("container_instances")
+ .h3("Component Instances");
- html.div("container_instances").h3("Component Instances");
+ int aaRoleWithNoSuitableLocations = 0;
+ int aaRoleWithOpenRequest = 0;
+ int roleWithOpenRequest = 0;
- Hamlet.TABLE<DIV<Hamlet>> table = div.table();
- table.tr()
- .td("Component")
- .td("Desired")
- .td("Actual")
- .td("Outstanding Requests")
- .td("Failed")
- .td("Failed to start")
- ._();
+ Hamlet.TABLE<DIV<Hamlet>> table = containers.table();
+ Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> header = table.thead().tr();
+ trb(header, "Component");
+ trb(header, "Desired");
+ trb(header, "Actual");
+ trb(header, "Outstanding Requests");
+ trb(header, "Failed");
+ trb(header, "Failed to start");
+ trb(header, "Placement");
+ header._()._(); // tr & thead
- List<RoleStatus> roleStatuses = appView.cloneRoleStatusList();
+ List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
Collections.sort(roleStatuses, new RoleStatus.CompareByName());
for (RoleStatus status : roleStatuses) {
+ String roleName = status.getName();
+ String nameUrl = apiPath(LIVE_COMPONENTS) + "/" + roleName;
+ String aatext;
+ if (status.isAntiAffinePlacement()) {
+ boolean aaRequestOutstanding = status.isAARequestOutstanding();
+ int pending = (int)status.getPendingAntiAffineRequests();
+ aatext = buildAADetails(aaRequestOutstanding, pending);
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext += " (label: " + status.getLabelExpression() + ")";
+ }
+ if (pending > 0 && !aaRequestOutstanding) {
+ aaRoleWithNoSuitableLocations ++;
+ } else if (aaRequestOutstanding) {
+ aaRoleWithOpenRequest++;
+ }
+ } else {
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext = "label: " + status.getLabelExpression();
+ } else {
+ aatext = "";
+ }
+ if (status.getRequested() > 0) {
+ roleWithOpenRequest ++;
+ }
+ }
table.tr()
- .td(status.getName())
- .td(String.format("%d", status.getDesired()))
- .td(String.format("%d", status.getActual()))
- .td(String.format("%d", status.getRequested()))
- .td(String.format("%d", status.getFailed()))
- .td(String.format("%d", status.getStartFailed()))
- ._();
+ .td().a(nameUrl, roleName)._()
+ .td(String.format("%d", status.getDesired()))
+ .td(String.format("%d", status.getActual()))
+ .td(String.format("%d", status.getRequested()))
+ .td(String.format("%d", status.getFailed()))
+ .td(String.format("%d", status.getStartFailed()))
+ .td(aatext)
+ ._();
}
- table._()._();
+ // empty row for some more spacing
+ table.tr()._();
+ // close table
+ table._();
+
+ containers._();
+ containers = null;
// some spacing
- html.p()._();
- html.p()._();
+ html.div()._();
+ html.div()._();
- html.div("provider_info").h3(providerName + " information");
- UL<DIV<Hamlet>> ul = div.ul();
+ DIV<Hamlet> diagnostics = html.div("diagnostics");
+
+ List<String> statusEntries = new ArrayList<>(0);
+ if (roleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d %s with requests unsatisfiable by cluster",
+ roleWithOpenRequest, plural(roleWithOpenRequest, "component")));
+ }
+ if (aaRoleWithNoSuitableLocations > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s no suitable nodes in the cluster",
+ aaRoleWithNoSuitableLocations,
+ plural(aaRoleWithNoSuitableLocations, "component has", "components have")));
+ }
+ if (aaRoleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s with requests unsatisfiable by cluster",
+ aaRoleWithOpenRequest,
+ plural(aaRoleWithOpenRequest, "component has", "components have")));
+
+ }
+ if (!statusEntries.isEmpty()) {
+ diagnostics.h3("Diagnostics");
+ Hamlet.TABLE<DIV<Hamlet>> diagnosticsTable = diagnostics.table();
+ for (String entry : statusEntries) {
+ diagnosticsTable.tr().td(entry)._();
+ }
+ diagnosticsTable._();
+ }
+ diagnostics._();
+
+ DIV<Hamlet> provider_info = html.div("provider_info");
+ provider_info.h3(providerName + " information");
+ UL<Hamlet> ul = html.ul();
addProviderServiceOptions(providerService, ul, clusterStatus);
- ul._()._();
+ ul._();
+ provider_info._();
+ }
+
+ @VisibleForTesting
+ String buildAADetails(boolean outstanding, int pending) {
+ return String.format("Anti-affinity:%s %d pending %s",
+ (outstanding ? " 1 active request and" : ""),
+ pending, plural(pending, "request"));
+ }
+
+ private String plural(int n, String singular) {
+ return plural(n, singular, singular + "s");
+ }
+ private String plural(int n, String singular, String plural) {
+ return n == 1 ? singular : plural;
+ }
+
+ private void trb(Hamlet.TR tr,
+ String text) {
+ tr.td().b(text)._();
}
private String getProviderName() {
@@ -153,14 +242,14 @@
}
private String getInfoAvoidingNulls(String key) {
- String createTime = appView.getClusterStatus().getInfo(key);
+ String createTime = appState.getClusterStatus().getInfo(key);
return null == createTime ? "N/A" : createTime;
}
- protected void addProviderServiceOptions(ProviderService providerService,
- UL<DIV<Hamlet>> ul, ClusterDescription clusterStatus) {
- Map<String, String> details = providerService.buildMonitorDetails(
+ protected void addProviderServiceOptions(ProviderService provider,
+ UL ul, ClusterDescription clusterStatus) {
+ Map<String, String> details = provider.buildMonitorDetails(
clusterStatus);
if (null == details) {
return;
@@ -176,4 +265,5 @@
}
}
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
index b2327ba..069d386 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
@@ -16,16 +16,21 @@
*/
package org.apache.slider.server.appmaster.web.view;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+
import static org.apache.slider.server.appmaster.web.SliderAMWebApp.*;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-
/**
*
*/
-public class NavBlock extends HtmlBlock {
+public class NavBlock extends SliderHamletBlock {
+
+ @Inject
+ public NavBlock(WebAppApi slider) {
+ super(slider);
+ }
@Override
protected void render(Block html) {
@@ -36,7 +41,6 @@
li().a(this.prefix(), "Overview")._().
li().a(relPath(CONTAINER_STATS), "Statistics")._().
li().a(relPath(CLUSTER_SPEC), "Specification")._().
- li().a(relPath(CLUSTER_SPEC), "Specification")._().
li().a(rootPath(SYSTEM_METRICS_JSON), "Metrics")._().
li().a(rootPath(SYSTEM_HEALTHCHECK), "Health")._().
li().a(rootPath(SYSTEM_THREADS), "Threads")._().
@@ -48,20 +52,11 @@
li().a(apiPath(LIVE_RESOURCES), "Resources")._().
li().a(apiPath(LIVE_COMPONENTS), "Components")._().
li().a(apiPath(LIVE_CONTAINERS), "Containers")._().
+ li().a(apiPath(LIVE_NODES), "Nodes")._().
+ li().a(apiPath(LIVE_STATISTICS), "Statistics")._().
li().a(apiPath(LIVE_LIVENESS), "Liveness")._()
._()
._();
}
- private String rootPath(String absolutePath) {
- return root_url(absolutePath);
- }
-
- private String relPath(String... args) {
- return ujoin(this.prefix(), args);
- }
- private String apiPath(String api) {
- return root_url(SLIDER_PATH_APPLICATION, api);
- }
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
new file mode 100644
index 0000000..82d7c8f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.view;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
+
+/**
+ * Anything we want to share across slider hamlet blocks
+ */
+public abstract class SliderHamletBlock extends HtmlBlock {
+
+ protected final StateAccessForProviders appState;
+ protected final ProviderService providerService;
+ protected final RestPaths restPaths = new RestPaths();
+
+ public SliderHamletBlock(WebAppApi slider) {
+ this.appState = slider.getAppState();
+ this.providerService = slider.getProviderService();
+ }
+
+ protected String rootPath(String absolutePath) {
+ return root_url(absolutePath);
+ }
+
+ protected String relPath(String... args) {
+ return ujoin(this.prefix(), args);
+ }
+
+ protected String apiPath(String api) {
+ return root_url(SLIDER_PATH_APPLICATION, api);
+ }
+
+}
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
index 50c10e4..b8bdc59 100644
--- a/slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -25,10 +25,7 @@
//import "Security.proto";
/*
- To debug compilation problems, bypass the maven build and invoke protoc
- from the command line
-
- protoc --java_out=target src/main/proto/SliderClusterMessages.proto
+ Look at SliderClusterProtocol.proto to see how to build this
*/
message RoleInstanceState {
@@ -258,6 +255,7 @@
optional int32 nodeFailed = 16;
optional int32 preempted = 17;
optional int32 pendingAntiAffineRequestCount = 18;
+ optional bool isAARequestOutstanding = 19;
}
/*
@@ -301,6 +299,7 @@
required int32 live = 8;
required int32 releasing = 9;
required int64 lastUsed = 10;
+ required string name = 11;
}
message NodeInformationProto {
@@ -389,8 +388,7 @@
}
message GetLiveNodesResponseProto {
- repeated string names = 1;
- repeated NodeInformationProto nodes = 2;
+ repeated NodeInformationProto nodes = 1;
}
message GetLiveNodeRequestProto {
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json
new file mode 100644
index 0000000..6e3ef14
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json
@@ -0,0 +1,7 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ }
+}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
index c8ada2c..073d1ff 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
@@ -1,16 +1,24 @@
{
- "schemaVersion": "2.1",
- "application": {
- "name": "SLEEPER",
- "components": [
- {
- "name": "SLEEP_100",
- "commands": [
- {
- "exec": "sleep 180"
- }
- ]
- }
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "SLEEPER",
+ "components": [
+ {
+ "name": "SLEEP_100",
+ "commands": [
+ {
+ "exec": "sleep 180"
+ }
]
- }
+ },
+ {
+ "name": "SLEEP_LONG",
+ "commands": [
+ {
+ "exec": "sleep 180000"
+ }
+ ]
+ }
+ ]
+ }
}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
index be7b962..e0ed16a 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
@@ -11,7 +11,13 @@
"SLEEP_100": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "128"
+ },
+ "SLEEP_LONG": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "0",
+ "yarn.memory": "128",
+ "yarn.component.placement.policy": "4"
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index a7bc0a3..b37ee3a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,15 +27,13 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.test.KeysForTests
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.AfterClass
import org.junit.BeforeClass
import org.junit.rules.TemporaryFolder
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
/**
* test base for agent clusters
@@ -43,8 +41,7 @@
@CompileStatic
@Slf4j
public abstract class AgentMiniClusterTestBase
-extends YarnZKMiniClusterTestBase {
- private static Logger LOG = LoggerFactory.getLogger(AgentMiniClusterTestBase)
+extends YarnZKMiniClusterTestBase implements KeysForTests {
protected static File agentConf
protected static File agentDef
protected static Map<String, String> agentDefOptions
@@ -92,7 +89,7 @@
@AfterClass
public static void cleanSubConfFiles() {
- def tempRoot
+ def tempRoot = ""
try {
tempRoot = tempFolder.root
if (tempRoot.exists()) {
@@ -101,7 +98,7 @@
} catch (IOException e) {
log.info("Failed to delete $tempRoot :$e", e)
} catch (IllegalStateException e) {
- log.warn("Temp folder deletion failed: $e")
+ log.warn("Temp folder deletion failed: $e", e)
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
index 6d1bcfc..6727a29 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
@@ -24,10 +24,8 @@
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.ComponentInformation
import org.apache.slider.api.types.ContainerInformation
-import org.apache.slider.api.types.NodeInformation
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.test.Outcome
-import org.junit.Test
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
@@ -210,10 +208,10 @@
describe "Node listing via $appAPI"
def liveNodes = appAPI.liveNodes
assert liveNodes.size() > 0
- def h = liveNodes.keySet()[0];
+ prettyPrintAsJson(liveNodes)
+ def h = liveNodes[0].hostname;
def localhost = appAPI.getLiveNode(h)
-
-
+ assert localhost.httpAddress == liveNodes[0].httpAddress
}
/**
@@ -239,7 +237,7 @@
testLiveContainers();
testRESTModel()
testAppLiveness()
-// testListNodes();
+ testListNodes();
}
public void testFlexOperation() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 97b3009..b0ae102 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -32,6 +32,7 @@
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.server.appmaster.rpc.RpcBinder
+import org.apache.slider.test.KeysForTests
import org.junit.Test
import static org.apache.slider.server.appmaster.management.MetricsKeys.*
@@ -39,8 +40,7 @@
@CompileStatic
@Slf4j
-class TestStandaloneREST extends AgentMiniClusterTestBase {
-
+class TestStandaloneREST extends AgentMiniClusterTestBase {
@Test
public void testStandaloneREST() throws Throwable {
@@ -65,15 +65,14 @@
ApplicationReport report = waitForClusterLive(client)
def proxyAM = report.trackingUrl
def directAM = report.originalTrackingUrl
-
-
+
// set up url config to match
initHttpTestSupport(launcher.configuration)
execOperation(WEB_STARTUP_TIME) {
GET(directAM)
}
-
+
execOperation(WEB_STARTUP_TIME) {
def metrics = GET(directAM, SYSTEM_METRICS_JSON)
log.info prettyPrintJson(metrics)
@@ -86,14 +85,12 @@
log.info GET(proxyAM, SYSTEM_HEALTHCHECK)
log.info GET(proxyAM, SYSTEM_METRICS_JSON)
- // using the metrics, await the first node status update
- /* SLIDER--82: disabled
- awaitGaugeValue(
- appendToURL(proxyAM, SYSTEM_METRICS_JSON),
- "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag",
+ // using the metrics, await the first node status update.
+ // this should be from AM launch itself
+ awaitGaugeValue(proxyAM,
+ NODES_UPDATED_FLAG_METRIC,
1,
WEB_STARTUP_TIME * 2, 500)
- */
// Is the back door required? If so, don't test complex verbs via the proxy
def proxyComplexVerbs = !SliderXmlConfKeys.X_DEV_INSECURE_REQUIRED
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 5495e62..6637c73 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -169,9 +169,5 @@
Map<String, String> masterRole = cd.getRole(master)
assert masterRole != null, "Role hbase-master must exist"
assert cd.roleNames.contains(master), "Role names must contain hbase-master"
-
- // and check liveness
- assert cd.liveness.allRequestsSatisfied
- assert 0 == cd.liveness.requestsOutstanding
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
index 1a90c88..17b176c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
@@ -21,14 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.exceptions.UsageException
-import org.apache.slider.core.exceptions.BadConfigException
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.junit.Test
@@ -38,13 +36,11 @@
@CompileStatic
@Slf4j
class TestClientBadArgs extends ServiceLauncherBaseTest {
-
- static String TEST_FILES = "./src/test/resources/org/apache/slider/providers/agent/tests/"
-
+
@Test
public void testNoAction() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: slider COMMAND",
[])
@@ -53,7 +49,7 @@
@Test
public void testUnknownAction() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"not-a-known-action",
["not-a-known-action"])
}
@@ -61,7 +57,7 @@
@Test
public void testActionWithoutOptions() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: slider build <application>",
[SliderActions.ACTION_BUILD])
}
@@ -69,7 +65,7 @@
@Test
public void testActionWithoutEnoughArgs() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
[SliderActions.ACTION_THAW])
}
@@ -77,7 +73,7 @@
@Test
public void testActionWithTooManyArgs() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
[SliderActions.ACTION_HELP,
"hello, world"])
@@ -86,7 +82,7 @@
@Test
public void testBadImageArg() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Unknown option: --image",
[SliderActions.ACTION_HELP,
Arguments.ARG_IMAGE])
@@ -95,7 +91,7 @@
@Test
public void testRegistryUsage() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"org.apache.slider.core.exceptions.UsageException: Argument --name missing",
[SliderActions.ACTION_REGISTRY])
assert exception instanceof UsageException
@@ -105,7 +101,7 @@
@Test
public void testRegistryExportBadUsage1() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected a value after parameter --getexp",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -118,7 +114,7 @@
@Test
public void testRegistryExportBadUsage2() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected a value after parameter --getexp",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -132,7 +128,7 @@
@Test
public void testRegistryExportBadUsage3() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: registry",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -147,7 +143,7 @@
@Test
public void testUpgradeUsage() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"org.apache.slider.core.exceptions.BadCommandArgumentsException: Not enough arguments for action: upgrade Expected minimum 1 but got 0",
[SliderActions.ACTION_UPGRADE])
assert exception instanceof BadCommandArgumentsException
@@ -158,7 +154,7 @@
public void testUpgradeWithTemplateOptionOnly() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --resources must be specified with option --template",
[SliderActions.ACTION_UPGRADE,
appName,
@@ -169,11 +165,17 @@
log.info(exception.toString())
}
+ public Configuration createTestConfig() {
+ def configuration = new Configuration()
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032")
+ return configuration
+ }
+
@Test
public void testUpgradeWithResourcesOptionOnly() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --template must be specified with option --resources",
[SliderActions.ACTION_UPGRADE,
appName,
@@ -188,7 +190,7 @@
public void testUpgradeWithTemplateResourcesAndContainersOption() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --containers cannot be "
+ "specified with --template or --resources",
[SliderActions.ACTION_UPGRADE,
@@ -208,7 +210,7 @@
public void testUpgradeWithTemplateResourcesAndComponentsOption() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --components cannot be "
+ "specified with --template or --resources",
[SliderActions.ACTION_UPGRADE,
@@ -228,7 +230,7 @@
public void testCreateAppWithAddonPkgBadArg1() throws Throwable {
//add on package without specifying add on package name
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected 2 values after --addon",
[SliderActions.ACTION_CREATE,
"cl1",
@@ -239,6 +241,15 @@
}
@Test
+ public void testNodesMissingFile() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ createTestConfig(),
+ "after parameter --out",
+ [SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT])
+ assert exception instanceof BadCommandArgumentsException
+ }
+
+ @Test
public void testFlexWithNoCompoents() throws Throwable {
def exception = launchExpectingException(SliderClient,
new Configuration(),
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
index 168415c..e244923 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
@@ -30,6 +30,7 @@
import org.apache.slider.core.launch.LaunchedApplication
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.apache.slider.core.persist.LockAcquireFailedException
+import org.apache.slider.server.appmaster.model.mock.MockApplicationId
import org.easymock.EasyMock
import org.junit.Assert
import org.junit.Test
@@ -137,7 +138,7 @@
AggregateConf instanceDefinition,
boolean debugAM)
throws YarnException, IOException {
- return new LaunchedApplication(clustername, new SliderYarnClientImpl());
+ return new LaunchedApplication(new MockApplicationId(1), new SliderYarnClientImpl());
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
index 8d5aef1..5295f9b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
@@ -82,10 +82,8 @@
new File(tmpDir, "resources.json").toURI()
])
fail("Upgrade command should have failed")
- } catch (SliderException e) {
- log.info(e.toString())
- assert e instanceof UnknownApplicationInstanceException
- assert e.getMessage().contains("Unknown application instance")
+ } catch (UnknownApplicationInstanceException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_UNKNOWN_INSTANCE, "Unknown application instance")
}
}
@@ -301,10 +299,6 @@
assert launcher.serviceExitCode == 0
}
- private File getTempLocation () {
- return new File(System.getProperty("user.dir") + "/target/_")
- }
-
static class TestSliderClient extends SliderClient {
public TestSliderClient() {
super()
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 5bf1c1f..e5951ac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -32,6 +32,7 @@
import org.junit.Before
import org.junit.Rule
import org.junit.rules.TemporaryFolder
+import org.slf4j.Logger
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@@ -53,7 +54,7 @@
*/
public static void assumeValidServerEnv() {
try {
- SliderUtils.validateSliderServerEnvironment(log, true)
+ SliderUtils.validateSliderServerEnvironment(log as Logger, true)
} catch (Exception e) {
skip(e.toString())
}
@@ -135,12 +136,8 @@
boolean create,
boolean blockUntilRunning) {
-
YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD,
@@ -166,11 +163,7 @@
List<String> extraArgs,
boolean deleteExistingData) {
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
SliderActions.ACTION_UPDATE,
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
new file mode 100644
index 0000000..855ed36
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import org.apache.slider.client.SliderClient
+
+/**
+ * Overridde the test actions with a sleep command, so that developers
+ * can see what the application is up to
+ */
+class DemoAgentAAEcho extends TestAgentAAEcho {
+
+ @Override
+ protected void postLaunchActions(
+ SliderClient sliderClient,
+ String clustername,
+ String rolename,
+ Map<String, Integer> roles,
+ String proxyAM) {
+
+ def url = proxyAM
+ // spin repeating the URl in the logs so YARN chatter doesn't lose it
+ describe("Web UI is at $url")
+
+ // run the superclass rest tests
+ // queryRestAPI(sliderClient, roles, proxyAM)
+
+ 5.times {
+ describe("Web UI is at $url")
+ sleep(60 *1000)
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
new file mode 100644
index 0000000..e193563
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.client.rest.SliderApplicationApiRestClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionNodesArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.server.appmaster.management.MetricsConstants
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_METRICS_JSON
+
+/**
+ * Tests an echo command
+ */
+@CompileStatic
+@Slf4j
+class TestAgentAAEcho extends TestAgentEcho {
+
+ @Test
+ public void testAgentEcho() throws Throwable {
+ assumeValidServerEnv()
+ def conf = configuration
+ conf.setBoolean(METRICS_LOGGING_ENABLED, true)
+ conf.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
+ String clustername = createMiniCluster("testaaecho",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ validatePaths()
+
+ def echo = "echo"
+ Map<String, Integer> roles = buildRoleMap(echo)
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ "" + PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ ARG_COMP_OPT, echo, SCRIPT_PATH, echo_py,
+ ARG_COMP_OPT, echo, SERVICE_NAME, "Agent",
+ ARG_DEFINE, SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false",
+ ARG_COMP_OPT, echo, TEST_RELAX_VERIFICATION, "true",
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ initHttpTestSupport(sliderClient.config)
+ def applicationReport = sliderClient.applicationReport
+ def proxyAM = applicationReport.trackingUrl
+ try {
+ postLaunchActions(sliderClient, clustername, echo, roles, proxyAM)
+ } catch (Exception ex) {
+ logMetricsQuietly(proxyAM)
+ throw ex;
+ }
+ }
+
+ /**
+ * retrieve cluster metrics and log quietly
+ * @param proxyAM
+ */
+ protected void logMetricsQuietly(String proxyAM) {
+ try {
+ log.error prettyPrintAsJson(GET(proxyAM, SYSTEM_METRICS_JSON));
+ } catch (Exception ex) {
+ log.warn("failed to get AM", ex)
+ }
+ }
+
+ /**
+ * Build the role map to use when creating teh cluster
+ * @param roleName the name used for the echo role
+ * @return the map
+ */
+ protected Map<String, Integer> buildRoleMap(String roleName) {
+ [
+ (roleName): 3,
+ ];
+ }
+
+ /**
+ * Any actions to perform after starting the agent cluster.
+ * HTTP client operations will have been set up already.
+ * @param sliderClient client for the cluster
+ * @param clustername cluster name
+ * @param rolename name of the echo role
+ * @param roles original set of roles
+ * @param proxyAM URl to proxy AM.
+ */
+ protected void postLaunchActions(
+ SliderClient sliderClient,
+ String clustername,
+ String rolename,
+ Map<String, Integer> roles,
+ String proxyAM) {
+ def onlyOneEcho = [(rolename): 1]
+ def requested = roles[rolename]
+
+ waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME)
+ //sleep a bit
+ sleep(5000)
+ //expect the role count to be the same
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+ def cd = sliderClient.getClusterDescription()
+ assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == requested - 1;
+ assert !cd.liveness.allRequestsSatisfied
+ assert cd.liveness.requestsOutstanding == requested - 1
+ def ipcClient = sliderClient.createIpcClient()
+
+ def echoInstances = sliderClient.listNodeUUIDsByRole(rolename)
+ queryRestAPI(sliderClient, roles, proxyAM)
+ // flex size
+ // while running, ask for many more, expect them to still be outstanding
+ sleep(5000)
+
+ requested = 50
+ def expectedPending = requested - 1
+
+ sliderClient.flex(clustername, [(rolename): requested]);
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+ sleep(4000)
+ def now = System.currentTimeMillis();
+ sleep(1000)
+
+ def componentInformation = ipcClient.getComponent(rolename)
+ assert !ipcClient.getComponent(rolename).isAARequestOutstanding
+
+ assert componentInformation.pendingAntiAffineRequestCount == expectedPending
+
+ cd = sliderClient.getClusterDescription()
+ assert !cd.liveness.allRequestsSatisfied
+ assert cd.liveness.requestsOutstanding == requested - 1
+ assert now <= Long.valueOf(cd.info.get(StatusKeys.INFO_STATUS_TIME_MILLIS))
+ assert expectedPending == cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1)
+
+ // while running, flex it to size = 1
+ sleep(1000)
+ sliderClient.flex(clustername, onlyOneEcho);
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+
+ def echoInstances2 = sliderClient.listNodeUUIDsByRole(rolename)
+ assertArrayEquals(echoInstances, echoInstances2)
+
+ assert !ipcClient.getComponent(rolename).isAARequestOutstanding
+ cd = sliderClient.getClusterDescription()
+ assert cd.liveness.allRequestsSatisfied
+
+
+ assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0;
+
+ def nodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs())
+ assert nodes.size() == 1
+ def activeNodes = sliderClient.listInstanceNodes(clustername, new ActionNodesArgs())
+ assert activeNodes[0].entries[rolename] && activeNodes[0].entries[rolename].live == 1
+ }
+
+ protected void queryRestAPI(SliderClient sliderClient, Map<String, Integer> roles, String proxyAM) {
+ GET(proxyAM)
+ describe "Proxy SliderRestClient Tests"
+ SliderApplicationApiRestClient restAPI =
+ new SliderApplicationApiRestClient(createUGIJerseyClient(), proxyAM)
+ awaitGaugeValue(proxyAM,
+ MetricsConstants.PREFIX_SLIDER_ROLES + "echo.pendingAntiAffineRequests",
+ 2,
+ WEB_STARTUP_TIME * 2, 500)
+
+ def echoInfo = restAPI.getComponent(ECHO)
+ assert echoInfo.pendingAntiAffineRequestCount == 2
+ // no active requests ... there's no capacity
+ assert !echoInfo.isAARequestOutstanding
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 973114d..23a7bbb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -22,11 +22,14 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionNodesArgs
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
import org.junit.Before
import org.junit.Test
@@ -40,6 +43,7 @@
@Slf4j
class TestAgentEcho extends AgentTestBase {
+ protected static final String ECHO = "echo"
File slider_core
String echo_py
File echo_py_path
@@ -59,7 +63,6 @@
agt_ver_path = new File(slider_core, agt_ver)
agt_conf = "agent.ini"
agt_conf_path = new File(slider_core, agt_conf)
-
}
@Override
@@ -68,7 +71,7 @@
}
@Test
- public void testEchoOperation() throws Throwable {
+ public void testAgentEcho() throws Throwable {
assumeValidServerEnv()
String clustername = createMiniCluster("",
@@ -79,12 +82,9 @@
true,
false)
- assert echo_py_path.exists()
- assert app_def_path.exists()
- assert agt_ver_path.exists()
- assert agt_conf_path.exists()
+ validatePaths()
- def role = "echo"
+ def role = ECHO
Map<String, Integer> roles = [
(role): 2,
];
@@ -127,5 +127,41 @@
assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative")
}
+
+ runNodemapTests(sliderClient)
+
+ }
+
+ /**
+ * do some nodemap checks, currently cluster-wide
+ * @param sliderClient
+ */
+ protected void runNodemapTests(SliderClient sliderClient) {
+ describe "slider nodes"
+ sliderClient.actionNodes("", new ActionNodesArgs())
+
+ def allNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs()).collect { it.httpAddress }
+ assert !allNodes.empty
+
+ // healthy only
+ def healthyNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(healthy: true))
+ assert healthyNodes.collect { it.httpAddress }.containsAll(allNodes)
+ // look for an unknown label and expect none
+ def gpuNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(label: "gpu"))
+ assert gpuNodes.empty
+ File t1 = createTempJsonFile()
+ sliderClient.actionNodes("", new ActionNodesArgs(outputFile: t1))
+ assert t1.exists()
+ JsonSerDeser<NodeInformationList> serDeser = new JsonSerDeser<>(NodeInformationList.class);
+ NodeInformationList loaded = serDeser.fromFile(t1)
+ assert allNodes.containsAll(loaded.collect { it.httpAddress })
+
+ }
+
+ protected void validatePaths() {
+ assert echo_py_path.exists()
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 264d260..60e9035 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -94,7 +94,9 @@
1,
true,
false)
- buildAgentCluster("test_build_basic_agent_node_only",
+
+ def cluster01 = clustername + "_01"
+ buildAgentCluster(cluster01,
[(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -110,7 +112,9 @@
def master = "hbase-master"
def rs = "hbase-rs"
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+
+ def cluster02 = clustername + "_02"
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(cluster02,
[
(ROLE_NODE): 1,
(master): 1,
@@ -149,10 +153,11 @@
def rscomponent = resource.getMandatoryComponent(rs)
assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
- // now create an instance with no role priority for the newnode role
+ describe "build a cluster with no role priority for the newnode role"
+
try {
- def name2 = clustername + "-2"
- buildAgentCluster(name2,
+ def cluster03 = clustername + "_03"
+ buildAgentCluster(cluster03,
[
(ROLE_NODE): 2,
"role3": 1,
@@ -166,14 +171,16 @@
],
true, false,
false)
- failWithBuildSucceeding(name2, "no priority for one role")
+ failWithBuildSucceeding(cluster03, "no priority for one role")
} catch (BadConfigException expected) {
}
+ describe "build a cluster with the number of agents out of range"
try {
- launcher = buildAgentCluster(clustername + "-10",
+ def cluster04 = clustername + "_04"
+ launcher = buildAgentCluster(cluster04,
[
- (ROLE_NODE): 4,
+ (ROLE_NODE): 2000,
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -185,15 +192,18 @@
],
true, false,
false)
- failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ failWithBuildSucceeding(cluster04, "too many instances")
} catch (BadConfigException expected) {
- assert expected.message.contains("Expected minimum is 1 and maximum is 2")
- assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Expected minimum is 1 and maximum is")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "out of range.")
}
- //duplicate priorities
+
+ describe "build a cluster with duplicate priorities for roles"
try {
- def name3 = clustername + "-3"
- buildAgentCluster(name3,
+ def cluster05 = clustername + "_05"
+ buildAgentCluster(cluster05,
[
(ROLE_NODE): 5,
(master): 1,
@@ -206,16 +216,16 @@
true, false,
false)
- failWithBuildSucceeding(name3, "duplicate priorities")
+ failWithBuildSucceeding(cluster05, "duplicate priorities")
} catch (BadConfigException expected) {
}
- def cluster4 = clustername + "-4"
+ def cluster06 = clustername + "_06"
def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
- buildAgentCluster(cluster4,
+ buildAgentCluster(cluster06,
[
(master): 1,
(rs): 5
@@ -235,7 +245,7 @@
false)
//now we want to look at the value
- AggregateConf instanceDefinition = loadInstanceDefinition(cluster4)
+ AggregateConf instanceDefinition = loadInstanceDefinition(cluster06)
def opt = instanceDefinition.getAppConfOperations().getComponentOpt(
SliderKeys.COMPONENT_AM,
RoleKeys.JVM_OPTS,
@@ -245,8 +255,8 @@
// now create an instance with no component options, hence no
// entry in the app config
- def name5 = clustername + "-5"
- buildAgentCluster(name5,
+ def name07 = clustername + "_07"
+ buildAgentCluster(name07,
[
"hbase-rs": 1,
],
@@ -403,9 +413,8 @@
def rscomponent2 = resource2.getMandatoryComponent(rs)
assert "6" == rscomponent2.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
}
-
+
public AggregateConf loadInstanceDefinition(String name) {
- def cluster4
def sliderFS = createSliderFileSystem()
def dirPath = sliderFS.buildClusterDirPath(name)
ConfPersister persister = new ConfPersister(sliderFS, dirPath)
@@ -697,10 +706,8 @@
}
public void failWithBuildSucceeding(String name, String reason) {
- def badArgs1
AggregateConf instanceDefinition = loadInstanceDefinition(name)
- log.error(
- "Build operation should have failed from $reason : \n$instanceDefinition")
+ log.error("Build operation should have failed from $reason : \n$instanceDefinition")
fail("Build operation should have failed from $reason")
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
new file mode 100644
index 0000000..7b3a65d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
+import org.apache.slider.server.appmaster.state.RoleStatus
+
+/**
+ * class for basis of Anti-affine placement tests; sets up role2
+ * for anti-affinity
+ */
+@CompileStatic
+@Slf4j
+class BaseMockAppStateAATest extends BaseMockAppStateTest
+ implements MockRoles {
+
+ /** Role status for the base AA role */
+ RoleStatus aaRole
+
+ /** Role status for the AA role requiring a node with the gpu label */
+ RoleStatus gpuRole
+
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [
+ MockFactory.PROVIDER_ROLE0,
+ MockFactory.AAROLE_1_GPU,
+ MockFactory.AAROLE_2,
+ ]
+ bindingInfo
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
new file mode 100644
index 0000000..a8c50d1
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement with a cluster of size 1
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateAAOvercapacity extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 1
+
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 1)
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(aaRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testOvercapacityRecovery() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ "verify the state. kill the allocated container and review")
+ //more than expected
+ long desired = 3
+ aaRole.desired = desired
+ assert appState.roleHistory.canPlaceAANodes()
+
+ //first request
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<AbstractRMOperation > operationsOut = []
+ // allocate and re-submit
+ def instances = submitOperations(operations, [], operationsOut)
+ assert 1 == instances.size()
+ assertAllContainersAA()
+
+ // expect an outstanding AA request to be unsatisfied
+ assert aaRole.actual < aaRole.desired
+ assert !aaRole.requested
+ assert !aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+
+ // now lets trigger a failure
+ def nodemap = cloneNodemap()
+ assert nodemap.size() == 1
+
+ def instance = instances[0]
+ def cid = instance.containerId
+
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid,
+ LauncherExitCodes.EXIT_TASK_LAUNCH_FAILURE))
+ assert result.containerFailed
+
+ assert aaRole.failed == 1
+ assert aaRole.actual == 0
+ def availablePlacements = appState.getRoleHistory().findNodeForNewAAInstance(aaRole)
+ assert availablePlacements.size() == 1
+ describe "expecting a successful review with available placements of $availablePlacements"
+ operations = appState.reviewRequestAndReleaseNodes()
+ assert operations.size() == 1
+ }
+
+ }
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
new file mode 100644
index 0000000..e43d894
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 3
+
+ /**
+ * The YARN engine has a cluster with very few nodes (3) and lots of containers, so
+ * if AA placement isn't working, there will be affine placements surfacing.
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 8)
+ }
+
+ /**
+ * This is the simplest AA allocation: no lables, so allocate anywhere
+ * @throws Throwable
+ */
+ @Test
+ public void testAllocateAANoLabel() throws Throwable {
+ assert cloneNodemap().size() > 0
+
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ AMRMClient.ContainerRequest request = getSingleRequest(ops)
+ assert !request.relaxLocality
+ assert request.nodes.size() == engine.cluster.clusterSize
+ assert request.racks == null
+ assert request.capability
+
+ Container allocated = engine.allocateContainer(request)
+
+ // notify the container ane expect
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated([allocated], assignments, operations)
+
+ def host = allocated.nodeId.host
+ def hostInstance = cloneNodemap().get(host)
+ assert hostInstance.get(aaRole.key).starting == 1
+ assert !hostInstance.canHost(aaRole.key, "")
+ assert !hostInstance.canHost(aaRole.key, null)
+
+ // assignment
+ assert assignments.size() == 1
+
+ // verify the release matches the allocation
+ assert operations.size() == 2
+ assert getCancel(operations, 0).capability.equals(allocated.resource)
+
+ // we also expect a new allocation request to have been issued
+
+ def req2 = getRequest(operations, 1)
+ assert req2.nodes.size() == engine.cluster.clusterSize - 1
+
+ assert !req2.nodes.contains(host)
+ assert !request.relaxLocality
+
+ // verify the pending couner is down
+ assert 0L == aaRole.pendingAntiAffineRequests
+ Container allocated2 = engine.allocateContainer(req2)
+
+ // placement must be on a different host
+ assert allocated2.nodeId != allocated.nodeId
+
+ ContainerAssignment assigned = assignments[0]
+ Container container = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ assert appState.onNodeManagerContainerStarted(container.id)
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+ assertAllContainersAA();
+
+ // identify those hosts with an aa role on
+ def naming = appState.buildNamingMap()
+ assert naming.size() == 3
+
+ def name = aaRole.name
+ assert name == naming[aaRole.key]
+ def info = appState.roleHistory.getNodeInformationSnapshot(naming);
+ assert info
+
+ def nodeInformation = info[host]
+ assert nodeInformation
+ assert nodeInformation.entries
+ assert nodeInformation.entries[name]
+ assert nodeInformation.entries[name].live
+ }
+
+ @Test
+ public void testAllocateFlexUp() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.requested == 1
+ assert aaRole.pendingAntiAffineRequests == 1
+ assert aaRole.actualAndRequested + aaRole.pendingAntiAffineRequests == aaRole.desired
+
+ // now trigger that flex up
+ aaRole.desired = 3
+
+ // expect: no new reqests, pending count ++
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ assert aaRole.actual + aaRole.pendingAntiAffineRequests + aaRole.outstandingAARequestCount ==
+ aaRole.desired
+
+ // 1 outstanding
+ assert aaRole.actual == 0
+ assert aaRole.AARequestOutstanding
+ // and one AA
+ assert aaRole.pendingAntiAffineRequests == 2
+ assertAllContainersAA()
+
+ // next iter
+ assert 1 == submitOperations(ops, [], ops2).size()
+ assert 2 == ops2.size()
+ assert aaRole.pendingAntiAffineRequests == 1
+ assertAllContainersAA()
+
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+ // now trigger the next execution cycle
+ List<AbstractRMOperation> ops3 = []
+ assert 1 == submitOperations(ops2, [], ops3).size()
+ assert 2 == ops3.size()
+ assert aaRole.pendingAntiAffineRequests == 0
+ assertAllContainersAA()
+
+ }
+
+ @Test
+ public void testAllocateFlexDownDecrementsPending() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.pendingAntiAffineRequests == 1
+ assert aaRole.AARequestOutstanding
+
+ // flex down so that the next request should be cancelled
+ aaRole.desired = 1
+
+ // expect: no new requests, pending count --
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ assert aaRole.AARequestOutstanding
+ assert aaRole.pendingAntiAffineRequests == 0
+ assertAllContainersAA()
+
+ // next iter
+ submitOperations(ops, [], ops2).size()
+ assert 1 == ops2.size()
+ assertAllContainersAA()
+ }
+
+ /**
+ * Here flex down while there is only one outstanding request.
+ * The outstanding flex should be cancelled
+ * @throws Throwable
+ */
+ @Test
+ public void testAllocateFlexDownForcesCancel() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 1
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.pendingAntiAffineRequests == 0
+ assert aaRole.AARequestOutstanding
+
+ // flex down so that the next request should be cancelled
+ aaRole.desired = 0
+ // expect: no new reqests, pending count --
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.pendingAntiAffineRequests == 0
+ assert !aaRole.AARequestOutstanding
+ assert ops2.size() == 1
+ getSingleCancel(ops2)
+
+ // next iter
+ submitOperations(ops, [], ops2).size()
+ assert 1 == ops2.size()
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(aaRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testAskForTooMany() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ " expect the final request to be unsatisfied until the cluster changes size")
+ //more than expected
+ aaRole.desired = NODES + 1
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert NODES == aaRole.pendingAntiAffineRequests
+ for (int i = 0; i < NODES; i++) {
+ def iter = "Iteration $i role = $aaRole"
+ log.info(iter)
+ List<AbstractRMOperation > operationsOut = []
+ assert 1 == submitOperations(operations, [], operationsOut).size(), iter
+ operations = operationsOut
+ if (i + 1 < NODES) {
+ assert operations.size() == 2
+ } else {
+ assert operations.size() == 1
+ }
+ assertAllContainersAA()
+ }
+ // expect an outstanding AA request to be unsatisfied
+ assert aaRole.actual < aaRole.desired
+ assert !aaRole.requested
+ assert !aaRole.AARequestOutstanding
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+ // in a review now, no more requests can be generated, as there is no space for AA placements,
+ // even though there is cluster capacity
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+
+ // now do a node update (this doesn't touch the YARN engine; the node isn't really there)
+ def outcome = addNewNode()
+ assert cloneNodemap().size() == NODES + 1
+ assert outcome.clusterChanged
+ // no active calls to empty
+ assert outcome.operations.empty
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ protected AppState.NodeUpdatedOutcome addNewNode() {
+ updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ }
+
+ @Test
+ public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
+ describe("Change the cluster size where the cluster size changes during a test sequence.")
+ aaRole.desired = NODES + 1
+ appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert NODES == aaRole.pendingAntiAffineRequests
+ def outcome = addNewNode()
+ assert outcome.clusterChanged
+ // one call to cancel
+ assert 1 == outcome.operations.size()
+ // and on a review, one more to rebuild
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ @Test
+ public void testBindingInfoMustHaveNodeMap() throws Throwable {
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.nodeReports = null;
+ try {
+ def state = new MockAppState(bindingInfo)
+ fail("Expected an exception, got $state")
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+
+ @Test
+ public void testAMRestart() throws Throwable {
+ def desiredAA = 3
+ aaRole.desired = desiredAA
+ List<RoleInstance> instances = createAndStartNodes()
+ List<Container> containers = instances.collect { it.container }
+
+ // now destroy the app state
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(0, 0, desiredAA)
+ ConfTreeOperations cto = new ConfTreeOperations(bindingInfo.instanceDefinition.resources)
+ cto.setComponentOpt(ROLE2,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED)
+ bindingInfo.liveContainers = containers
+ appState = new MockAppState(bindingInfo)
+
+ def aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ def gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ appState.reviewRequestAndReleaseNodes()
+ assert aaRole.antiAffinePlacement
+ assert aaRole.AARequestOutstanding
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index d36fdbc..6739623 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -41,11 +41,6 @@
@Slf4j
class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateAppRestIntegration"
- }
-
@Test
public void testCachedIntDocument() throws Throwable {
ContentCache cache = new ContentCache()
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 1b79115..3235827 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -58,7 +58,7 @@
*/
@Override
MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8000, 4)
+ return new MockYarnEngine(4, 8000)
}
@Override
@@ -95,7 +95,7 @@
//view the world
appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0)
assert queue.size() == 0
}
@@ -123,7 +123,7 @@
//view the world
appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0)
assert queue.size() == 1
}
@@ -148,7 +148,7 @@
RoleHistory history = appState.roleHistory
history.dump();
- List<NodeInstance> queue = history.cloneAvailableList(0)
+ List<NodeInstance> queue = history.cloneRecentNodeList(0)
assert queue.size() == 0
NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
index aa7bb11..e57f341 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -20,7 +20,6 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTreeOperations
@@ -28,7 +27,6 @@
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -36,7 +34,7 @@
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
@@ -47,11 +45,6 @@
class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateDynamicHistory"
- }
-
/**
* Small cluster with multiple containers per node,
* to guarantee many container allocations on each node
@@ -62,26 +55,6 @@
return new MockYarnEngine(8, 1)
}
- @Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0,0,0)
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
- }
-
-
@Test
public void testDynamicRoleHistory() throws Throwable {
@@ -199,7 +172,7 @@
assert !entry.live
- def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8)
+ def nodesForRoleId = roleHistory.getRecentNodesForRoleId(role_priority_8)
assert nodesForRoleId
// make sure new nodes will default to a different host in the engine
@@ -215,21 +188,16 @@
@Test(expected = BadConfigException.class)
public void testRoleHistoryRoleAdditions() throws Throwable {
MockRoleHistory roleHistory = new MockRoleHistory([])
- roleHistory.addNewProviderRole(new ProviderRole("one", 1))
- roleHistory.addNewProviderRole(new ProviderRole("two", 1))
+ roleHistory.addNewRole(new RoleStatus(new ProviderRole("one", 1)))
+ roleHistory.addNewRole(new RoleStatus(new ProviderRole("two", 1)))
roleHistory.dump()
- fail("should have raised an exception")
}
-
-
+
@Test(expected = BadConfigException.class)
public void testRoleHistoryRoleStartupConflict() throws Throwable {
MockRoleHistory roleHistory = new MockRoleHistory([
new ProviderRole("one", 1), new ProviderRole("two", 1)
])
roleHistory.dump()
- fail("should have raised an exception")
}
-
-
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index ee4abd6..d0163d2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -20,21 +20,17 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.RoleHistoryUtils
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
@@ -65,40 +61,27 @@
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- def instance = factory.newInstanceDefinition(0,0,0)
-
+ AggregateConf buildInstanceDefinition() {
+ def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
- (ResourceKeys.COMPONENT_PRIORITY): ROLE4,
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE4,
(ResourceKeys.COMPONENT_INSTANCES): "1",
]
- instance.resourceOperations.components[ROLE4]= opts
+ instance.resourceOperations.components[ROLE4] = opts
def opts5 = [
- (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
- (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
+ (ResourceKeys.COMPONENT_INSTANCES) : "1",
(ResourceKeys.COMPONENT_PLACEMENT_POLICY):
Integer.toString(PlacementPolicy.STRICT),
- (ResourceKeys.NODE_FAILURE_THRESHOLD):
+ (ResourceKeys.NODE_FAILURE_THRESHOLD) :
Integer.toString(2),
]
- instance.resourceOperations.components[ROLE5]= opts5
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
+ instance.resourceOperations.components[ROLE5] = opts5
+ instance
}
@Test
@@ -115,21 +98,16 @@
* @param actions source list
* @return found list
*/
- List<ContainerRequestOperation> findAllocationsForRole(int role,
+ Collection<ContainerRequestOperation> findAllocationsForRole(int role,
List<AbstractRMOperation> actions) {
- List <ContainerRequestOperation > results = []
- actions.each { AbstractRMOperation operation ->
- if (operation instanceof ContainerRequestOperation) {
- def req = (ContainerRequestOperation) operation;
- def reqrole = ContainerPriority.extractRole(req.request.priority)
- if (role == reqrole) {
- results << req
- }
- }
+ def requests = actions.findAll {
+ it instanceof ContainerRequestOperation}.collect {it as ContainerRequestOperation}
+
+ requests.findAll {
+ role == ContainerPriority.extractRole(it.request.priority)
}
- return results
- }
-
+ }
+
@Test
public void testStrictPlacementInitialRequest() throws Throwable {
log.info("Initial engine state = $engine")
@@ -141,7 +119,6 @@
assertRelaxLocalityFlag(ID5, null, true, actions)
}
-
@Test
public void testPolicyPropagation() throws Throwable {
assert !(appState.lookupRoleStatus(ROLE4).placementPolicy & PlacementPolicy.STRICT)
@@ -153,7 +130,6 @@
public void testNodeFailureThresholdPropagation() throws Throwable {
assert (appState.lookupRoleStatus(ROLE4).nodeFailureThreshold == 3)
assert (appState.lookupRoleStatus(ROLE5).nodeFailureThreshold == 2)
-
}
@Test
@@ -173,7 +149,6 @@
assert instanceA
def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
-
log.info("Allocated engine state = $engine")
assert engine.containerCount() == 1
@@ -183,8 +158,7 @@
role4.desired = 0
appState.lookupRoleStatus(ROLE4).desired = 0
def completionResults = []
- def containersToRelease = []
- instances = createStartAndStopNodes(completionResults)
+ createStartAndStopNodes(completionResults)
assert engine.containerCount() == 0
assert completionResults.size() == 1
@@ -215,19 +189,16 @@
}
assert instanceA
def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
-
-
log.info("Allocated engine state = $engine")
assert engine.containerCount() == 1
assert role5.actual == 1
- // shrinking cluster
+ // shrinking cluster
role5.desired = 0
def completionResults = []
- def containersToRelease = []
- instances = createStartAndStopNodes(completionResults)
+ createStartAndStopNodes(completionResults)
assert engine.containerCount() == 0
assert completionResults.size() == 1
assert role5.actual == 0
@@ -240,16 +211,15 @@
def nodes = cro.request.nodes
assert nodes.size() == 1
assert hostname == nodes[0]
-
}
public void assertRelaxLocalityFlag(
- int id,
+ int role,
String expectedHost,
boolean expectedRelaxFlag,
List<AbstractRMOperation> actions) {
def requests
- requests = findAllocationsForRole(id, actions)
+ requests = findAllocationsForRole(role, actions)
assert requests.size() == 1
def req = requests[0]
assert expectedRelaxFlag == req.request.relaxLocality
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 5d880b4..7bc6fe4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -20,21 +20,22 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that if you have more than one role, the right roles are chosen for release.
*/
@CompileStatic
@Slf4j
@@ -57,32 +58,25 @@
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector()
+ bindingInfo
+ }
+ @Override
+ AggregateConf buildInstanceDefinition() {
def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
]
instance.resourceOperations.components["dynamic-6"] = opts
-
-
- appState.buildInstance(instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new MostRecentContainerReleaseSelector())
+ instance
}
-
private ConfTreeOperations init() {
createAndStartNodes();
def resources = appState.instanceDefinition.resources;
@@ -181,16 +175,10 @@
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath2,
- null, null,
- new MostRecentContainerReleaseSelector())
+ def binding2 = buildBindingInfo()
+ binding2.instanceDefinition = factory.newInstanceDefinition(0, 0, 0)
+ binding2.historyPath = historyPath2
+ appState.buildInstance(binding2)
// on this read there won't be the right number of roles
def loadedRoleHistory = historyWriter.read(fs, history)
assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 257092a..548842c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,7 +20,6 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.slider.api.ClusterDescription
import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
@@ -76,7 +75,7 @@
ops = appState.reviewRequestAndReleaseNodes()
assert ops.empty
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ appState.innerOnNodeManagerContainerStarted(target.id)
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 9ac6fcf..ba7588a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -110,7 +110,6 @@
// four outstanding
assert role0.requested == 4
-
// flex cluster to 3
role0.desired = 3
ops = appState.reviewRequestAndReleaseNodes()
@@ -226,8 +225,7 @@
public void testFlexUpNoSpace() throws Throwable {
// engine only has two nodes, so > 2 will be outstanding
engine = new MockYarnEngine(1, 2)
- List<AbstractRMOperation> ops
- // role: desired = 2, requested = 1, actual=1
+ // role: desired = 2, requested = 1, actual=1
def role0 = role0Status
role0.desired = 4
createAndSubmitNodes()
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c310583..0e526b6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -20,14 +20,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.NodeMap
@@ -35,12 +32,11 @@
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that app state is rebuilt on a restart
*/
@CompileStatic
@Slf4j
-class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
+class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest implements MockRoles {
@Override
String getTestName() {
@@ -62,28 +58,15 @@
assert instances.size() == clusterSize
//clone the list
- List<RoleInstance> cloned = [];
- List<Container> containers = []
- instances.each { RoleInstance elt ->
- cloned.add(elt.clone() as RoleInstance)
- containers.add(elt.container)
- }
+ List<Container> containers = instances.collect { it.container }
NodeMap nodemap = appState.roleHistory.cloneNodemap()
- // now destroy the app state
- appState = new MockAppState()
-
//and rebuild
- appState.buildInstance(
- factory.newInstanceDefinition(r0, r1, r2),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- containers,
- null,
- new MostRecentContainerReleaseSelector())
+
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(r0, r1, r2)
+ bindingInfo.liveContainers = containers
+ appState = new MockAppState(bindingInfo)
assert appState.startedCountainerCount == clusterSize
@@ -107,22 +90,18 @@
assertNotNull("Null entry in original nodemap for " + hostname, orig)
for (int i = 0; i < ROLE_COUNT; i++) {
-
- assert (nodeInstance.getActiveRoleInstances(i) ==
- orig.getActiveRoleInstances(i))
+ assert (nodeInstance.getActiveRoleInstances(i) == orig.getActiveRoleInstances(i))
NodeEntry origRE = orig.getOrCreate(i)
NodeEntry newRE = nodeInstance.getOrCreate(i)
assert origRE.live == newRE.live
- assert newRE.starting == 0
+ assert 0 == newRE.starting
}
}
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
- def status = appState.getClusterStatus()
+ def status = appState.clusterStatus
// verify the AM restart container count was set
- String restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ String restarted = status.getInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null;
//and that the count == 1 master + the region servers
assert Integer.parseInt(restarted) == containers.size()
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 4ba0afd..93cce95 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
@@ -37,11 +38,6 @@
@Slf4j
class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
- @Override
- String getTestName() {
- "TestMockContainerResourceAllocations"
- }
-
@Test
public void testNormalAllocations() throws Throwable {
ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
@@ -71,7 +67,7 @@
assert ops.size() == 1
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
- assert requirements.memory == RM_MAX_RAM
+ assert requirements.memory == MockAppState.RM_MAX_RAM
assert requirements.virtualCores == 2
}
@@ -89,7 +85,7 @@
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
assert requirements.memory == 512
- assert requirements.virtualCores == RM_MAX_CORES
+ assert requirements.virtualCores == MockAppState.RM_MAX_CORES
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
new file mode 100644
index 0000000..f0fed95
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement
+ */
+@CompileStatic
+@Slf4j
+class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 3
+ private int GPU_NODES = 2
+ private String HOST0 = "00000000"
+ private String HOST1 = "00000001"
+
+
+ @Override
+ void setup() {
+ super.setup()
+ // node 1 is GPU
+
+ updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ }
+
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 8)
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(gpuRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testAskForTooMany() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ " expect the final request to be unsatisfied until the cluster changes size")
+ //more than expected
+ int size = GPU_NODES
+ gpuRole.desired = size + 1
+
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert gpuRole.AARequestOutstanding
+
+ assert gpuRole.pendingAntiAffineRequests == size
+ for (int i = 0; i < size; i++) {
+ def iter = "Iteration $i role = $aaRole"
+ describe iter
+ List<AbstractRMOperation > operationsOut = []
+
+ def roleInstances = submitOperations(operations, [], operationsOut)
+ // one instance per request
+ assert 1 == roleInstances.size()
+ appState.onNodeManagerContainerStarted(roleInstances[0].containerId)
+ assertAllContainersAA()
+ // there should be none left
+ log.debug(nodeInformationSnapshotAsString())
+ operations = operationsOut
+ if (i + 1 < size) {
+ assert operations.size() == 2
+ } else {
+ assert operations.size() == 1
+ }
+ }
+ // expect an outstanding AA request to be unsatisfied
+ assert gpuRole.actual < gpuRole.desired
+ assert !gpuRole.requested
+ assert !gpuRole.AARequestOutstanding
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+ // in a review now, no more requests can be generated, as there is no space for AA placements,
+ // even though there is cluster capacity
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+
+ // switch node 2 into being labelled
+ def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+
+ assert cloneNodemap().size() == NODES
+ assert outcome.clusterChanged
+ // no active calls to empty
+ assert outcome.operations.empty
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ protected AppState.NodeUpdatedOutcome addNewNode() {
+ updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ }
+
+ @Test
+ public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
+ describe("Change the cluster size where the cluster size changes during a test sequence.")
+ gpuRole.desired = GPU_NODES + 1
+ List<AbstractRMOperation> operations = appState.reviewRequestAndReleaseNodes()
+ assert gpuRole.AARequestOutstanding
+ assert GPU_NODES == gpuRole.pendingAntiAffineRequests
+ def outcome = addNewNode()
+ assert outcome.clusterChanged
+ // one call to cancel
+ assert 1 == outcome.operations.size()
+ // and on a review, one more to rebuild
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
new file mode 100644
index 0000000..db84b0b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.slider.api.proto.Messages
+import org.apache.slider.api.proto.RestTypeMarshalling
+import org.apache.slider.api.types.NodeInformation
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+/**
+ * Test anti-affine
+ */
+//@CompileStatic
+@Slf4j
+class TestRoleHistoryAA extends SliderTestBase {
+
+ List<String> hostnames = ["1", "2", "3"]
+ NodeMap nodeMap, gpuNodeMap
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
+
+
+ @Override
+ void setup() {
+ super.setup()
+ nodeMap = createNodeMap(hostnames, NodeState.RUNNING)
+ gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU")
+ }
+
+ @Test
+ public void testFindNodesInFullCluster() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, nodeMap.findAllNodesForRole(1, ""))
+ }
+
+ @Test
+ public void testFindNodesInUnhealthyCluster() throws Throwable {
+ // all three will surface at first
+ markNodeOneUnhealthy()
+ verifyResultSize(2, nodeMap.findAllNodesForRole(1, ""))
+ }
+
+ public boolean markNodeOneUnhealthy() {
+ return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY)
+ }
+
+ protected boolean setNodeState(NodeInstance node, NodeState state) {
+ node.updateNode(new MockNodeReport(node.hostname, state))
+ }
+
+ @Test
+ public void testFindNoNodesWrongLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
+ }
+
+ @Test
+ public void testFindSomeNodesSomeLabel() throws Throwable {
+ // all three will surface at first
+ update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
+ verifyResultSize(1, gpuNodes)
+ def instance = gpuNodes[0]
+ instance.getOrCreate(1).onStarting()
+ assert !instance.canHost(1, "GPU")
+ assert !instance.canHost(1, "")
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
+
+ }
+
+ @Test
+ public void testFindNoNodesRightLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"))
+ }
+
+ @Test
+ public void testFindNoNodesNoLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""))
+ }
+
+ @Test
+ public void testFindNoNodesClusterRequested() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.request()
+ }
+ assertNoAvailableNodes(1)
+ }
+
+ @Test
+ public void testFindNoNodesClusterBusy() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.request()
+ }
+ assertNoAvailableNodes(1)
+ }
+
+ /**
+ * Tag all nodes as starting, then walk one through a bit
+ * more of its lifecycle
+ */
+ @Test
+ public void testFindNoNodesLifecycle() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.onStarting()
+ }
+ assertNoAvailableNodes(1)
+
+ // walk one of the nodes through the lifecycle
+ def node1 = nodeMap.get("1")
+ assert !node1.canHost(1,"")
+ node1.get(1).onStartCompleted()
+ assert !node1.canHost(1,"")
+ assertNoAvailableNodes()
+ node1.get(1).release()
+ assert node1.canHost(1,"")
+ def list2 = verifyResultSize(1, nodeMap.findAllNodesForRole(1, ""))
+ assert list2[0].hostname == "1"
+
+ // now tag that node as unhealthy and expect it to go away
+ markNodeOneUnhealthy()
+ assertNoAvailableNodes()
+ }
+
+ @Test
+ public void testRolesIndependent() throws Throwable {
+ def node1 = nodeMap.get("1")
+ def role1 = node1.getOrCreate(1)
+ def role2 = node1.getOrCreate(2)
+ nodeMap.values().each {
+ it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ }
+ assertNoAvailableNodes(1)
+ assertNoAvailableNodes(2)
+ assert setNodeState(node1, NodeState.RUNNING)
+ // tag role 1 as busy
+ role1.onStarting()
+ assertNoAvailableNodes(1)
+
+ verifyResultSize(1, nodeMap.findAllNodesForRole(2, ""))
+ assert node1.canHost(2,"")
+ }
+
+ @Test
+ public void testNodeEntryAvailablity() throws Throwable {
+ def entry = new NodeEntry(1)
+ assert entry.available
+ entry.onStarting()
+ assert !entry.available
+ entry.onStartCompleted()
+ assert !entry.available
+ entry.release()
+ assert entry.available
+ entry.onStarting()
+ assert !entry.available
+ entry.onStartFailed()
+ assert entry.available
+ }
+
+ @Test
+ public void testNodeInstanceSerialization() throws Throwable {
+ def rh2 = new MockRoleHistory([])
+ rh2.getOrCreateNodeInstance("localhost")
+ def instance = rh2.getOrCreateNodeInstance("localhost")
+ instance.getOrCreate(1).onStartCompleted()
+ def Map<Integer, String> naming = [(1):"manager"]
+ def ni = instance.serialize(naming)
+ assert 1 == ni.entries["manager"].live
+ def ni2 = rh2.getNodeInformation("localhost", naming)
+ assert 1 == ni2.entries["manager"].live
+ def info = rh2.getNodeInformationSnapshot(naming)
+ assert 1 == info["localhost"].entries["manager"].live
+ def nil = new NodeInformationList(info.values());
+ assert 1 == nil[0].entries["manager"].live
+
+ def nodeInformationProto = RestTypeMarshalling.marshall(ni)
+ def entryProto = nodeInformationProto.getEntries(0)
+ assert entryProto && entryProto.getPriority() == 1
+ def unmarshalled = RestTypeMarshalling.unmarshall(nodeInformationProto)
+ assert unmarshalled.hostname == ni.hostname
+ assert unmarshalled.entries.keySet().containsAll(ni.entries.keySet())
+
+ }
+
+ @Test
+ public void testBuildRolenames() throws Throwable {
+
+ }
+ public List<NodeInstance> assertNoAvailableNodes(int role = 1, String label = "") {
+ return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label))
+ }
+
+ List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) {
+ if (list.size() != size) {
+ list.each { log.error(it.toFullString()) }
+ }
+ assert size == list.size()
+ list
+ }
+
+ def applyToNodeEntries(Collection<NodeInstance> list, Closure cl) {
+ list.each { it -> cl(it.getOrCreate(1)) }
+ }
+
+ def applyToNodeEntries(NodeMap nodeMap, Closure cl) {
+ applyToNodeEntries(nodeMap.values(), cl)
+ }
+
+ def NodeMap createNodeMap(List<NodeReport> nodeReports) {
+ NodeMap nodeMap = new NodeMap(1)
+ update(nodeMap, nodeReports)
+ nodeMap
+ }
+
+ protected boolean update(NodeMap nodeMap, List<NodeReport> nodeReports) {
+ nodeMap.buildOrUpdate(nodeReports)
+ }
+
+ def NodeMap createNodeMap(List<String> hosts, NodeState state,
+ String label = "") {
+ createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index fa10145..ca42546 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -28,13 +28,11 @@
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeId
import org.apache.slider.server.appmaster.state.*
-import org.junit.Before
import org.junit.Test
/**
@@ -60,15 +58,20 @@
String roleName = "test"
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory
Resource resource
- @Before
- public void setupRH() {
- roleHistory.onStart(fs, historyPath)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ roleHistory = appState.roleHistory
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
ResourceKeys.DEF_YARN_MEMORY);
}
@@ -76,11 +79,10 @@
@Test
public void testFindAndCreate() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes = request.getNodes()
assert nodes != null
@@ -113,14 +115,13 @@
@Test
public void testCreateAndRelease() throws Throwable {
int role = 1
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
//verify it is empty
assert roleHistory.listActiveNodes(role).empty
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
assert request.nodes == null
@@ -169,7 +170,7 @@
// ask for a container and expect to get the recently released one
AMRMClient.ContainerRequest request2 =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes2 = request2.nodes
assert nodes2 != null
@@ -205,11 +206,10 @@
@Test
public void testStartFailed() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -239,10 +239,10 @@
@Test
public void testStartFailedWithoutWarning() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -270,10 +270,10 @@
describe("fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -309,10 +309,10 @@
public void testContainerFailedWithoutWarning() throws Throwable {
describe( "fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -342,10 +342,10 @@
public void testAllocationListPrep() throws Throwable {
describe("test prepareAllocationList")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -374,9 +374,10 @@
describe("fail a node")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
- AMRMClient.ContainerRequest request = roleHistory.requestNode(roleStatus, resource);
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest;
String hostname = request.getNodes()[0]
assert age3Active0.hostname == hostname
@@ -399,7 +400,7 @@
int startSize = nodemap.size()
// now send a list of updated (failed) nodes event
- List<NodeReport> nodesUpdated = new ArrayList<NodeReport>();
+ List<NodeReport> nodesUpdated = new ArrayList<>();
NodeReport nodeReport = NodeReport.newInstance(
NodeId.newInstance(hostname, 0),
NodeState.LOST,
@@ -412,7 +413,7 @@
// as even unused nodes are added to the list, we expect the map size to be >1
assert startSize <= endSize
assert nodemap[hostname] != null
- assert roleHistory.cloneFailedNodes().contains(hostname)
+ assert !nodemap[hostname].online
// add a failure of a node we've never head of
def newhost = "newhost"
@@ -425,9 +426,8 @@
roleHistory.onNodesUpdated(nodesUpdated)
def nodemap2 = roleHistory.cloneNodemap()
- assert nodemap2.size() > endSize
- assert roleHistory.cloneFailedNodes().contains(newhost)
assert nodemap2[newhost]
+ assert !nodemap2[newhost].online
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
index 79d23e5..f36724e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.ContainerOutcome
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -32,10 +33,8 @@
/**
* Testing finding nodes for new instances.
- * These tests validate the (currently) suboptimal
- * behavior of not listing any known nodes when there
- * are none in the available list -even if there are nodes
- * known to be not running live instances in the cluster.
+ *
+ * This stresses the non-AA codepath
*/
@Slf4j
@CompileStatic
@@ -54,7 +53,7 @@
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
String roleName = "test"
RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
@@ -63,14 +62,14 @@
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
}
public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
List < NodeInstance > found = [];
for (int i = 0; i < count; i++) {
- NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus)
if (f) {
found << f
};
@@ -80,17 +79,17 @@
@Test
public void testFind1NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found")
assert [age3Active0].contains(found)
}
@Test
public void testFind2NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found")
assert [age2Active0, age3Active0].contains(found)
- NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found2 = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found2")
assert [age2Active0, age3Active0].contains(found2)
assert found != found2;
@@ -99,7 +98,7 @@
@Test
public void testFind3NodeR0ReturnsNull() throws Throwable {
assert 2== findNodes(2).size()
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
assert found == null;
}
@@ -123,7 +122,7 @@
assert age2Active0.getActiveRoleInstances(0) != 0
age3Active0.get(0).onStartCompleted()
assert age3Active0.getActiveRoleInstances(0) != 0
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info(found ?.toFullString())
assert found == null
}
@@ -147,7 +146,7 @@
assert age2Active0.exceedsFailureThreshold(roleStat)
// get the role & expect age3 to be picked up, even though it is older
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
assert age3Active0.is(found)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
index b26b2f0..bcd8f9f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -27,17 +28,20 @@
/**
* Unit test to verify the comparators sort as expected
*/
+@CompileStatic
class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
- NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
+ NodeInstance age1Active4 = nodeInstance(1001, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(1002, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(1003, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(1004, 1, 0, 0)
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
NodeInstance age6failing = nodeInstance(1006, 0, 0, 0)
- NodeInstance age1failing = nodeInstance(1000, 0, 0, 0)
+ NodeInstance age1failing = nodeInstance(1001, 0, 0, 0)
List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ List<NodeInstance> nodesPlusEmpty = [age2Active2, age4Active1, age1Active4, age3Active0, empty]
+ List<NodeInstance> allnodes = [age6failing, age2Active2, age4Active1, age1Active4, age3Active0, age1failing]
@Before
public void setup() {
@@ -51,43 +55,48 @@
}
@Test
- public void testNewerThan() throws Throwable {
-
+ public void testPreferred() throws Throwable {
Collections.sort(nodes, new NodeInstance.Preferred(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4])
+ assertListEquals(nodes, [age4Active1, age3Active0, age2Active2, age1Active4])
+ }
+
+ /**
+ * The preferred sort still includes failures; up to next phase in process
+ * to handle that
+ * @throws Throwable
+ */
+ @Test
+ public void testPreferredWithFailures() throws Throwable {
+ Collections.sort(allnodes, new NodeInstance.Preferred(0))
+ assert allnodes[0] == age6failing
+ assert allnodes[1] == age4Active1
}
@Test
- public void testFailureCountFirst() throws Throwable {
+ public void testPreferredComparatorDowngradesFailures() throws Throwable {
def preferred = new NodeInstance.Preferred(0)
assert preferred.compare(age6failing, age1failing) == -1
assert preferred.compare(age1failing, age6failing) == 1
}
-
+
@Test
public void testNewerThanNoRole() throws Throwable {
-
- nodes << empty
- Collections.sort(nodes, new NodeInstance.Preferred(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4, empty])
+ Collections.sort(nodesPlusEmpty, new NodeInstance.Preferred(0))
+ assertListEquals(nodesPlusEmpty, [age4Active1, age3Active0, age2Active2, age1Active4, empty])
}
@Test
public void testMoreActiveThan() throws Throwable {
Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0],)
+ assertListEquals(nodes, [age1Active4, age2Active2, age4Active1, age3Active0])
}
@Test
public void testMoreActiveThanEmpty() throws Throwable {
- nodes << empty
- Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0, empty])
+
+ Collections.sort(nodesPlusEmpty, new NodeInstance.MoreActiveThan(0))
+ assertListEquals(nodesPlusEmpty, [age1Active4, age2Active2, age4Active1, age3Active0, empty])
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 653af84..6969b38 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -35,16 +36,28 @@
import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
-class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+ public static final String WORKERS_LABEL = "workers"
NodeInstance host1 = new NodeInstance("host1", 3)
NodeInstance host2 = new NodeInstance("host2", 3)
+ def resource = factory.newResource(48, 1)
OutstandingRequestTracker tracker = new OutstandingRequestTracker()
-
+
+ public static final ProviderRole WORKER = new ProviderRole(
+ "worker",
+ 5,
+ PlacementPolicy.NONE,
+ 2,
+ 1,
+ WORKERS_LABEL)
+
@Override
- String getTestName() {
- return "TestOutstandingRequestTracker"
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [ WORKER ] + bindingInfo.roles
+ bindingInfo
}
@Test
@@ -57,10 +70,16 @@
@Test
public void testAddCompleteEntry() throws Throwable {
- tracker.newRequest(host1, 0)
- tracker.newRequest(host2, 0)
- tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1", null).outcome == ContainerAllocationOutcome.Placed
+ def req1 = tracker.newRequest(host1, 0)
+ req1.buildContainerRequest(resource, role0Status, 0)
+
+ tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0)
+ tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0)
+
+ def allocation = tracker.onContainerAllocated(1, "host1", null)
+ assert allocation.outcome == ContainerAllocationOutcome.Placed
+ assert allocation.operations[0] instanceof CancelSingleRequest
+
assert !tracker.lookupPlacedRequest(1, "host1")
assert tracker.lookupPlacedRequest(0, "host1")
}
@@ -80,46 +99,40 @@
@Test
public void testRemoveOpenRequestUnissued() throws Throwable {
def req1 = tracker.newRequest(null, 0)
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
- def c1 = factory.newContainer()
- c1.setPriority(new MockPriority(0))
+ def c1 = factory.newContainer(null, new MockPriority(0))
+ c1.resource = resource
- def resource = factory.newResource()
- resource.virtualCores=1
- resource.memory = 48;
- c1.setResource(resource)
- ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1).outcome
+ def allocation = tracker.onContainerAllocated(0, "host1", c1)
+ ContainerAllocationOutcome outcome = allocation.outcome
assert outcome == ContainerAllocationOutcome.Unallocated
+ assert allocation.operations.empty
assert tracker.listOpenRequests().size() == 1
}
@Test
public void testIssuedOpenRequest() throws Throwable {
def req1 = tracker.newRequest(null, 0)
- def resource = factory.newResource()
- resource.virtualCores = 1
- resource.memory = 48;
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, "")
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
- def c1 = factory.newContainer()
-
- def nodeId = factory.newNodeId()
- c1.nodeId = nodeId
- nodeId.host ="hostname-1"
def pri = ContainerPriority.buildPriority(0, false)
assert pri > 0
- c1.setPriority(new MockPriority(pri))
+ def nodeId = factory.newNodeId("hostname-1")
+ def c1 = factory.newContainer(nodeId, new MockPriority(pri))
- c1.setResource(resource)
+ c1.resource = resource
def issued = req1.issuedRequest
assert issued.capability == resource
- assert issued.priority.priority == c1.getPriority().getPriority()
+ assert issued.priority.priority == c1.priority.priority
assert req1.resourceRequirementsMatch(resource)
def allocation = tracker.onContainerAllocated(0, nodeId.host, c1)
assert tracker.listOpenRequests().size() == 0
+ assert allocation.operations[0] instanceof CancelSingleRequest
+
assert allocation.outcome == ContainerAllocationOutcome.Open
assert allocation.origin.is(req1)
}
@@ -146,7 +159,7 @@
// first request: default placement
assert role0Status.placementPolicy == PlacementPolicy.DEFAULT;
final def (res0, outstanding0) = newRequest(role0Status)
- final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0)
assert outstanding0.issuedRequest != null;
assert outstanding0.located
assert !outstanding0.escalated
@@ -157,7 +170,6 @@
// process skips entries which are in the list but have not been issued.
// ...which can be a race condition between request issuance & escalation.
// (not one observed outside test authoring, but retained for completeness)
- assert role2Status.placementPolicy == PlacementPolicy.ANTI_AFFINITY_REQUIRED
def (res2, outstanding2) = newRequest(role2Status)
// simulate some time escalation of role 1 MUST now be triggered
@@ -182,7 +194,7 @@
// build that second request from an anti-affine entry
// these get placed as well
now += interval
- final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now, null)
+ final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now)
// escalate a little bit more
final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now)
// and expect no new entries
@@ -223,18 +235,19 @@
resource.virtualCores = 1
resource.memory = 48;
- def label = "workers"
+ def workerRole = lookupRole(WORKER.name)
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
+ def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
+ // escalation
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == label)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
assert (yarnRequest2.relaxLocality)
}
/**
- * If the placement doesnt include a lablel, then the escalation request
+ * If the placement doesnt include a label, then the escalation request
* retains the node list, but sets relaxLocality==true
* @throws Throwable
*/
@@ -246,17 +259,44 @@
resource.virtualCores = 1
resource.memory = 48;
- def label = null
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
- assert (yarnRequest.nodes != null)
- assert (yarnRequest.nodeLabelExpression == null)
- assert (!yarnRequest.relaxLocality)
+ def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0)
+ assert yarnRequest.nodes != null
+ assert !yarnRequest.nodeLabelExpression
+ assert !yarnRequest.relaxLocality
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodes != null)
- assert (yarnRequest2.relaxLocality)
+ assert yarnRequest2.nodes != null
+ assert yarnRequest2.relaxLocality
}
+ @Test(expected = IllegalArgumentException)
+ public void testAARequestNoNodes() throws Throwable {
+ tracker.newAARequest(role0Status.key, [], "")
+ }
+
+ @Test
+ public void testAARequest() throws Throwable {
+ def role0 = role0Status.key
+ OutstandingRequest request = tracker.newAARequest(role0, [host1], "")
+ assert host1.hostname == request.hostname
+ assert !request.located
+ }
+
+ @Test
+ public void testAARequestPair() throws Throwable {
+ def role0 = role0Status.key
+ OutstandingRequest request = tracker.newAARequest(role0, [host1, host2], "")
+ assert host1.hostname == request.hostname
+ assert !request.located
+ def yarnRequest = request.buildContainerRequest(
+ role0Status.copyResourceRequirements(new MockResource(0, 0)),
+ role0Status,
+ 0)
+ assert !yarnRequest.relaxLocality
+ assert !request.mayEscalate()
+
+ assert yarnRequest.nodes.size() == 2
+ }
/**
* Create a new request (always against host1)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index c81c686..72e4240 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -22,15 +22,15 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -48,7 +48,8 @@
3,
PlacementPolicy.STRICT,
3,
- 3)
+ 3,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
@Override
String getTestName() {
@@ -57,7 +58,7 @@
@Test
public void testWriteReadEmpty() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
@@ -67,7 +68,7 @@
@Test
public void testWriteReadData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -77,7 +78,7 @@
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
def loadedRoleHistory = historyWriter.read(fs, history)
@@ -92,7 +93,7 @@
@Test
public void testWriteReadActiveData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
String addr = "localhost"
String addr2 = "rack1server5"
@@ -117,7 +118,7 @@
describe("Loaded")
log.info("testWriteReadActiveData in $history")
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
def loadedRoleHistory = historyWriter.read(fs, history)
assert 3 == loadedRoleHistory.size()
rh2.rebuild(loadedRoleHistory)
@@ -136,16 +137,16 @@
assert rh2.thawedDataTime == savetime
// now start it
- rh2.buildAvailableNodeLists();
+ rh2.buildRecentNodeLists();
describe("starting")
rh2.dump();
- List<NodeInstance> available0 = rh2.cloneAvailableList(0)
+ List<NodeInstance> available0 = rh2.cloneRecentNodeList(0)
assert available0.size() == 1
NodeInstance entry = available0.get(0)
assert entry.hostname == "localhost"
assert entry == localhost
- List<NodeInstance> available1 = rh2.cloneAvailableList(1)
+ List<NodeInstance> available1 = rh2.cloneRecentNodeList(1)
assert available1.size() == 2
//and verify that even if last used was set, the save time is picked up
assert entry.get(1).lastUsed == roleHistory.saveTime
@@ -154,7 +155,7 @@
@Test
public void testWriteThaw() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -164,7 +165,7 @@
Path history = roleHistory.saveHistory(time++)
long savetime =roleHistory.saveTime;
assert fs.isFile(history)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
assert rh2.onStart(fs, historyPath)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
@@ -211,7 +212,7 @@
@Test
public void testSkipEmptyFileOnRead() throws Throwable {
describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
time = 0
Path oldhistory = roleHistory.saveHistory(time++)
@@ -226,7 +227,7 @@
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
Path touched = touch(historyWriter, time++)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
assert rh2.onStart(fs, historyPath)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
@@ -240,7 +241,7 @@
@Test
public void testSkipBrokenFileOnRead() throws Throwable {
describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
time = 0
Path oldhistory = roleHistory.saveHistory(time++)
@@ -258,7 +259,7 @@
out.writeBytes("{broken:true}")
out.close()
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
describe("IGNORE STACK TRACE BELOW")
assert rh2.onStart(fs, historyPath)
@@ -285,7 +286,7 @@
def loadedRoleHistory = writer.read(source)
assert 4 == loadedRoleHistory.size()
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
@@ -300,7 +301,7 @@
def loadedRoleHistory = writer.read(source)
assert 6 == loadedRoleHistory.size()
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert 3 == roleHistory.rebuild(loadedRoleHistory)
}
@@ -318,7 +319,7 @@
assert 4 == loadedRoleHistory.size()
def expandedRoles = new ArrayList(MockFactory.ROLES)
expandedRoles << PROVIDER_ROLE3
- RoleHistory roleHistory = new RoleHistory(expandedRoles)
+ RoleHistory roleHistory = new MockRoleHistory(expandedRoles)
assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
index aef22fb..8a0c1ca 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
@@ -18,11 +18,13 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.common.SliderKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -34,14 +36,15 @@
import java.util.regex.Pattern
@Slf4j
+@CompileStatic
class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
- def paths = pathlist(
+ List<Path> paths = pathlist(
[
- "hdfs://localhost/history-0406c.json",
- "hdfs://localhost/history-5fffa.json",
- "hdfs://localhost/history-0001a.json",
- "hdfs://localhost/history-0001f.json",
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json",
]
)
Path h_0406c = paths[0]
@@ -50,9 +53,7 @@
List<Path> pathlist(List<String> pathnames) {
- def result = []
- pathnames.each { result << new Path(new URI(it as String)) }
- result
+ pathnames.collect{ new Path(new URI(it as String)) }
}
@Override
@@ -60,7 +61,6 @@
return "TestHistoryRWOrdering"
}
-
/**
* This tests regexp pattern matching. It uses the current time so isn't
* repeatable -but it does test a wider range of values in the process
@@ -79,13 +79,12 @@
}
}
-
@Test
public void testWriteSequenceReadData() throws Throwable {
describe "test that if multiple entries are written, the newest is picked up"
long time = System.currentTimeMillis();
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -94,7 +93,7 @@
Path history1 = roleHistory.saveHistory(time++)
Path history2 = roleHistory.saveHistory(time++)
- Path history3 = roleHistory.saveHistory(time++)
+ Path history3 = roleHistory.saveHistory(time)
//inject a later file with a different name
sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
@@ -116,7 +115,7 @@
public void testPathStructure() throws Throwable {
assert h_5fffa.getName() == "history-5fffa.json"
}
-
+
@Test
public void testPathnameComparator() throws Throwable {
@@ -137,10 +136,10 @@
RoleHistoryWriter.sortHistoryPaths(paths2)
assertListEquals(paths2,
[
- paths[1],
- paths[0],
- paths[3],
- paths[2]
+ paths[1],
+ paths[0],
+ paths[3],
+ paths[2]
])
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 9847992..2f160cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Resource
@@ -27,6 +28,7 @@
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -41,6 +43,7 @@
* get removed and added
*/
@Slf4j
+@CompileStatic
class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
String roleName = "test"
@@ -53,12 +56,17 @@
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
+ /** 1MB, 1 vcore*/
Resource resource = Resource.newInstance(1, 1)
ProviderRole provRole = new ProviderRole(roleName, 0)
RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
@Override
String getTestName() {
return "TestRoleHistoryAvailableList"
@@ -67,46 +75,44 @@
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
+ roleStatus.setResourceRequirements(Resource.newInstance(1, 1))
}
@Test
public void testAvailableListBuiltForRoles() throws Throwable {
- List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age3Active0, age2Active0], available0)
}
@Test
public void testRequestedNodeOffList() throws Throwable {
- List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
- NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert age3Active0 == ni
- AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
+ assertListEquals([age2Active0], roleHistory.cloneRecentNodeList(0))
+ roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
- assertListEquals([age2Active0], a2)
+ resource
+ )
}
@Test
public void testRequestedNodeOffListWithFailures() throws Throwable {
assert 0 == roleStatus.key
- assert !roleHistory.cloneAvailableList(0).isEmpty()
+ assert !roleHistory.cloneRecentNodeList(0).empty
NodeEntry age3role0 = recordAsFailed(age3Active0, 0, 4)
assert age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
recordAsFailed(age2Active0, 0, 4)
assert age2Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
// expect to get a null node back
- NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert !ni
// which is translated to a no-location request
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
+ resource).issuedRequest
assertNull(req.nodes)
@@ -115,57 +121,61 @@
roleHistory.dump()
assert 0 == age3role0.failedRecently
assert !age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
- assert !roleHistory.cloneAvailableList(0).isEmpty()
+ assert !roleHistory.cloneRecentNodeList(0).empty
// looking for a node should now find one
- ni = roleHistory.findNodeForNewInstance(roleStatus)
+ ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert ni == age3Active0
- req = roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource,
- "")
+ req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource).issuedRequest
assert 1 == req.nodes.size()
}
+ /**
+ * verify that strict placement policies generate requests for nodes irrespective
+ * of their failed status
+ * @throws Throwable
+ */
@Test
public void testStrictPlacementIgnoresFailures() throws Throwable {
def targetRole = role1Status
final ProviderRole providerRole1 = targetRole.providerRole
assert providerRole1.placementPolicy == PlacementPolicy.STRICT
- int key = targetRole.key
+ int key1 = targetRole.key
+ def key0 = role0Status.key
- recordAsFailed(age1Active4, key, 4)
- recordAsFailed(age2Active0, key, 4)
- recordAsFailed(age2Active2, key, 4)
- recordAsFailed(age3Active0, key, 4)
- recordAsFailed(age4Active1, key, 4)
+ def nodes = [age1Active4, age2Active0, age2Active2, age3Active0, age4Active1]
+ recordAllFailed(key0, 4, nodes)
+ recordAllFailed(key1, 4, nodes)
// trigger a list rebuild
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
+ def recentRole0 = roleHistory.cloneRecentNodeList(key0)
+ assert recentRole0.indexOf(age3Active0) < recentRole0.indexOf(age2Active0)
- assert !roleHistory.cloneAvailableList(key).isEmpty()
+ // the non-strict role has no suitable nodes
+ assert null == roleHistory.findRecentNodeForNewInstance(role0Status)
- NodeInstance ni = roleHistory.findNodeForNewInstance(targetRole)
- assert ni == age4Active1!= null
- // next lookup returns next node
- ni = roleHistory.findNodeForNewInstance(roleStatus)
- assert ni == age3Active0
+ def ni = roleHistory.findRecentNodeForNewInstance(targetRole)
+ assert ni
+
+ def ni2 = roleHistory.findRecentNodeForNewInstance(targetRole)
+ assert ni2
+ assert ni != ni2
}
-
@Test
public void testFindAndRequestNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
assert age3Active0.hostname == req.nodes[0]
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age2Active0], a2)
}
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
@@ -173,7 +183,7 @@
@Test
public void testCompletedRequestDropsNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
@@ -202,8 +212,8 @@
@Test
public void testTwoRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -216,9 +226,9 @@
@Test
public void testThreeRequestsOneUnsatisified() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -239,15 +249,15 @@
roleHistory.listOpenRequests().empty
// and the remainder goes onto the available list
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age2Active0], a2)
}
@Test
public void testThreeRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
assertOutstandingPlacedRequests(2)
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 29eefa5..69a98eb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -20,35 +20,38 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
import org.apache.slider.server.appmaster.state.ContainerOutcome
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.RoleStatus
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.appmaster.state.StateAccessForProviders
import org.apache.slider.test.SliderTestBase
-import org.junit.Before
@CompileStatic
@Slf4j
abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles {
- public static final int RM_MAX_RAM = 4096
- public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
MockAppState appState
MockYarnEngine engine
@@ -58,6 +61,16 @@
protected Path historyPath;
protected MockApplicationId applicationId;
protected MockApplicationAttemptId applicationAttemptId;
+ protected StateAccessForProviders stateAccess
+
+ /**
+ * Override point: called in setup() to create the YARN engine; can
+ * be changed for different sizes and options
+ * @return
+ */
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8, 8)
+ }
@Override
void setup() {
@@ -66,49 +79,46 @@
fs = HadoopFS.get(new URI("file:///"), conf)
sliderFileSystem = new SliderFileSystem(fs, conf)
engine = createYarnEngine()
+ initApp()
}
/**
- * Override point: called in setup() to create the YARN engine; can
- * be changed for different sizes and options
- * @return
+ * Initialize the application.
+ * This uses the binding information supplied by {@link #buildBindingInfo()}.
*/
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(64, 1)
- }
-
- @Before
void initApp(){
-
String historyDirName = testName;
-
-
- YarnConfiguration conf = SliderUtils.createConfiguration()
applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0)
applicationAttemptId = new MockApplicationAttemptId(
applicationId: applicationId,
attemptId: 1)
- fs = HadoopFS.get(new URI("file:///"), conf)
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- buildInstanceDefinition(),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new SimpleReleaseSelector())
+ appState = new MockAppState(buildBindingInfo())
+ stateAccess = new ProviderAppState(testName, appState)
+ }
+
+ /**
+ * Build the binding info from the default constructor values,
+ * the roles from {@link #factory}, and an instance definition
+ * from {@link #buildInstanceDefinition()}
+ * @return
+ */
+ AppStateBindingInfo buildBindingInfo() {
+ AppStateBindingInfo binding = new AppStateBindingInfo()
+ binding.instanceDefinition = buildInstanceDefinition();
+ binding.roles = factory.ROLES
+ binding.fs = fs
+ binding.historyPath = historyPath
+ binding.nodeReports = engine.nodeReports as List<NodeReport>
+ binding
}
/**
* Override point, define the instance definition
- * @return
+ * @return the instance definition
*/
public AggregateConf buildInstanceDefinition() {
factory.newInstanceDefinition(0, 0, 0)
@@ -116,22 +126,26 @@
/**
* Get the test name ... defaults to method name
- * @return
+ * @return the method name
*/
String getTestName() {
methodName.methodName;
}
public RoleStatus getRole0Status() {
- return appState.lookupRoleStatus(ROLE0)
+ lookupRole(ROLE0)
+ }
+
+ public RoleStatus lookupRole(String role) {
+ appState.lookupRoleStatus(role)
}
public RoleStatus getRole1Status() {
- return appState.lookupRoleStatus(ROLE1)
+ lookupRole(ROLE1)
}
public RoleStatus getRole2Status() {
- return appState.lookupRoleStatus(ROLE2)
+ lookupRole(ROLE2)
}
/**
@@ -147,7 +161,6 @@
return ri
}
-
public NodeInstance nodeInstance(long age, int live0, int live1=0, int live2=0) {
NodeInstance ni = new NodeInstance("age${age}-[${live0},${live1},$live2]",
MockFactory.ROLE_COUNT)
@@ -200,7 +213,6 @@
return createStartAndStopNodes([])
}
-
/**
* Create, Start and stop nodes
* @param completionResults List filled in with the status on all completed nodes
@@ -266,7 +278,7 @@
* @return a list of roles
*/
public List<RoleInstance> createAndSubmitNodes() {
- return createAndSubmitNodes([])
+ return createAndSubmitNodes([], [])
}
/**
@@ -275,35 +287,36 @@
* @return a list of roles allocated
*/
public List<RoleInstance> createAndSubmitNodes(
- List<ContainerId> released) {
+ List<ContainerId> containerIds,
+ List<AbstractRMOperation> operationsOut = []) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- return submitOperations(ops, released)
+ return submitOperations(ops, containerIds, operationsOut)
}
/**
* Process the RM operations and send <code>onContainersAllocated</code>
* events to the app state
- * @param ops
- * @param released
- * @return
+ * @param operationsIn list of incoming ops
+ * @param released released containers
+ * @return list of outbound operations
*/
public List<RoleInstance> submitOperations(
- List<AbstractRMOperation> ops,
- List<ContainerId> released) {
- List<Container> allocatedContainers = engine.execute(ops, released)
+ List<AbstractRMOperation> operationsIn,
+ List<ContainerId> released,
+ List<AbstractRMOperation> operationsOut = []) {
+ List<Container> allocatedContainers = engine.execute(operationsIn, released)
List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated(allocatedContainers, assignments, operations)
- List<RoleInstance> instances = []
- for (ContainerAssignment assigned : assignments) {
+ appState.onContainersAllocated(allocatedContainers, assignments, operationsOut)
+
+ assignments.collect {
+ ContainerAssignment assigned ->
Container container = assigned.container
RoleInstance ri = roleInstance(assigned)
- instances << ri
//tell the app it arrived
log.debug("Start submitted ${ri.role} on ${container.id} ")
appState.containerStartSubmitted(container, ri);
+ ri
}
- return instances
}
/**
@@ -326,13 +339,7 @@
List<ContainerId> extractContainerIds(
List<RoleInstance> instances,
int role) {
- List<ContainerId> cids = []
- instances.each { RoleInstance instance ->
- if (instance.roleId == role) {
- cids << instance.id
- }
- }
- return cids
+ instances.findAll { it.roleId == role }.collect { RoleInstance instance -> instance.id }
}
/**
@@ -351,4 +358,99 @@
}
entry
}
+
+ def recordAllFailed(int id, int count, List<NodeInstance> nodes) {
+ nodes.each { NodeInstance node -> recordAsFailed(node, id, count)}
+ }
+
+ /**
+ * Get the container request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof ContainerRequestOperation
+ ((ContainerRequestOperation) op).request
+ }
+
+ /**
+ * Get the cancel request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof CancelSingleRequest
+ ((CancelSingleRequest) op).request
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first ContainerRequestOperation
+ */
+ public AMRMClient.ContainerRequest getSingleRequest(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getRequest(ops, 0)
+ }
+
+ /**
+ * Get the node information as a large JSON String
+ * @return
+ */
+ String nodeInformationSnapshotAsString() {
+ prettyPrintAsJson(stateAccess.nodeInformationSnapshot)
+ }
+
+ /**
+ * Scan through all containers and assert that the assignment is AA
+ * @param index role index
+ */
+ void assertAllContainersAA(int index) {
+ cloneNodemap().each { name, info ->
+ def nodeEntry = info.get(index)
+ assert nodeEntry == null || nodeEntry.antiAffinityConstraintHeld
+ "too many instances on node $name"
+ }
+ }
+
+ List<NodeInstance> verifyNodeInstanceCount(int size, List<NodeInstance> list) {
+ if (list.size() != size) {
+ list.each { log.error(it.toFullString()) }
+ }
+ assert size == list.size()
+ list
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first operation
+ */
+ public AMRMClient.ContainerRequest getSingleCancel(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getCancel(ops, 0)
+ }
+
+ /**
+ * Get a snapshot of the nodemap of the application state
+ * @return a cloned nodemap
+ */
+ protected NodeMap cloneNodemap() {
+ appState.roleHistory.cloneNodemap()
+ }
+
+ /**
+ * Issue a nodes updated event
+ * @param report report to notify
+ * @return response of AM
+ */
+ protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) {
+ appState.onNodesUpdated([report])
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 6e21a38..c041ce5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -21,14 +21,18 @@
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
/**
* Extended app state that makes more things public
*/
class MockAppState extends AppState {
+ public static final int RM_MAX_RAM = 4096
+ public static final int RM_MAX_CORES = 64
public MockAppState(AbstractClusterServices recordFactory) {
super(recordFactory, new MetricsAndMonitoring());
+ setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES)
}
long time = 0;
@@ -37,7 +41,12 @@
* Instance with a mock record factory
*/
public MockAppState() {
- super(new MockClusterServices(), new MetricsAndMonitoring());
+ this(new MockClusterServices());
+ }
+
+ MockAppState(AppStateBindingInfo bindingInfo) {
+ this()
+ buildInstance(bindingInfo)
}
public Map<String, ProviderRole> getRoleMap() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
index d27a6bb..dfecc94 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
@@ -27,4 +27,9 @@
Resource newResource() {
return new MockResource()
}
+
+ @Override
+ Resource newResource(int memory, int cores) {
+ return new MockResource(memory, cores)
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 6071ef0..4bbfbd8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -44,26 +44,59 @@
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
+ /**
+ * basic role
+ */
public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
MockRoles.ROLE0,
0,
PlacementPolicy.DEFAULT,
2,
- 1)
- // role 1 is strict. timeout should be irrelevant; same as failures
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+ /**
+ * role 1 is strict. timeout should be irrelevant; same as failures
+ */
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
1,
PlacementPolicy.STRICT,
2,
- 1)
- // role 2: longer delay
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+
+ /**
+ * role 2: longer delay
+ */
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
+ PlacementPolicy.NO_DATA_LOCALITY,
+ 2,
+ 2,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+
+ /**
+ * Patch up a "role2" role to have anti-affinity set
+ */
+ public static final ProviderRole AAROLE_2 = new ProviderRole(
+ MockRoles.ROLE2,
+ 2,
PlacementPolicy.ANTI_AFFINITY_REQUIRED,
2,
- 2)
+ 2,
+ null)
+
+ /**
+ * Patch up a "role1" role to have anti-affinity set and GPI as the label
+ */
+ public static final ProviderRole AAROLE_1_GPU = new ProviderRole(
+ MockRoles.ROLE1,
+ 1,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 2,
+ 1,
+ MockRoles.LABEL_GPU)
int appIdCount;
int attemptIdCount;
@@ -80,7 +113,7 @@
PROVIDER_ROLE1,
PROVIDER_ROLE2,
]
-
+
public static final int ROLE_COUNT = ROLES.size();
MockContainerId newContainerId() {
@@ -107,8 +140,8 @@
return id;
}
- MockNodeId newNodeId() {
- new MockNodeId()
+ MockNodeId newNodeId(String host = null) {
+ new MockNodeId(host: host)
}
MockContainer newContainer(ContainerId cid) {
@@ -202,12 +235,11 @@
]
}
- MockResource newResource() {
- return new MockResource()
+ MockResource newResource(int memory = 0, int vcores = 0) {
+ return new MockResource(memory, vcores)
}
MockContainerStatus newContainerStatus() {
return new MockContainerStatus()
-
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
new file mode 100644
index 0000000..8c3b712
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.Resource
+
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockNodeReport extends NodeReport {
+ NodeId nodeId;
+ NodeState nodeState;
+ String httpAddress;
+ String rackName;
+ Resource used;
+ Resource capability;
+ int numContainers;
+ String healthReport;
+ long lastHealthReportTime;
+ Set<String> nodeLabels;
+
+ MockNodeReport() {
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ MockNodeReport(String hostname, NodeState nodeState, String label ="") {
+ nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ this.nodeState = nodeState
+ this.httpAddress = "http$hostname:80"
+ this.nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ static List<MockNodeReport> createInstances(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ new MockNodeReport(name, nodeState, label)}
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
index f4c54f3..91ab43b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
@@ -24,7 +24,7 @@
int memory
int virtualCores
- MockResource(int memory=0, int vcores=0) {
+ MockResource(int memory = 0, int vcores = 0) {
this.memory = memory
this.virtualCores = vcores
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
index c521697..4553e22 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
@@ -21,15 +21,22 @@
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
/**
* subclass to enable access to some of the protected methods
*/
class MockRoleHistory extends RoleHistory {
+ /**
+ * Take a list of provider roles and build the history from them, dynamically creating
+ * the role status entries on the way
+ * @param providerRoles provider role list
+ * @throws BadConfigException configuration problem with the role list
+ */
MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
- super(providerRoles)
+ super(providerRoles.collect { new RoleStatus(it) },
+ new MockClusterServices())
}
-
-
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
index b44482d..5ee8f24 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
@@ -24,4 +24,6 @@
String ROLE1 = "role1"
String ROLE2 = "role2"
int ROLE_COUNT = 3
+ String LABEL_GPU = "gpu"
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 6056e3a..265a796 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeState
/**
* Models the cluster itself: a set of mock cluster nodes.
@@ -143,21 +144,30 @@
}
return total;
}
-
+
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ nodes.collect { MockYarnClusterNode n -> n.nodeReport }
+ }
/**
* Model cluster nodes on the simpler "slot" model than the YARN-era
- * resource allocation model. Why? Makes it easier to implement.
+ * resource allocation model. Why? Easier to implement scheduling.
+ * Of course, if someone does want to implement the full process...
*
- * When a cluster is offline,
*/
public static class MockYarnClusterNode {
public final int nodeIndex
public final String hostname;
+ public List<String> labels = []
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
+ public MockNodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -170,6 +180,10 @@
MockContainerId mci = new MockContainerId(containerId: cid)
containers[i] = new MockYarnClusterContainer(mci)
}
+
+ nodeReport = new MockNodeReport()
+ nodeReport.nodeId = nodeId
+ nodeReport.nodeState = NodeState.RUNNING
}
/**
@@ -230,8 +244,6 @@
}
return result
}
-
-
/**
* Release a container
@@ -291,8 +303,8 @@
return (hostIndex << 8) | containerIndex & 0xff;
}
- public static final int extractHost(int cid) {
- return (cid >>> 8);
+ public static final int extractHost(long cid) {
+ return (cid >>> 8) & 0xffff;
}
public static final int extractContainer(int cid) {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 5860c6b..7ab97fa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.junit.Assert
@@ -113,7 +114,9 @@
ContainerId cid = cro.containerId
assert releaseContainer(cid);
released.add(cid)
- } else {
+ } else if (op instanceof CancelSingleRequest) {
+ // no-op
+ } else if (op instanceof ContainerRequestOperation) {
ContainerRequestOperation req = (ContainerRequestOperation) op
Container container = allocateContainer(req.request)
if (container != null) {
@@ -123,6 +126,8 @@
log.debug("Unsatisfied allocation $req")
pending.add(req)
}
+ } else {
+ log.warn("Unsupported operation $op")
}
}
return allocation
@@ -157,4 +162,11 @@
}
}
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ cluster.nodeReports
+ }
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index c2ea837..a818e53 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -24,8 +24,10 @@
import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest
import org.apache.slider.server.appmaster.model.mock.*
import org.apache.slider.server.appmaster.state.ContainerOutcome
+import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
@@ -34,7 +36,7 @@
@Slf4j
//@CompileStatic
-public class TestIndexBlock extends BaseMockAppStateTest {
+public class TestIndexBlock extends BaseMockAppStateAATest {
private IndexBlock indexBlock;
@@ -81,42 +83,80 @@
public void testIndex() {
def role0 = role0Status
def role1 = role1Status
- role0.desired = 8
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role1.incRequested()
- role1.incRequested()
- role1.incRequested()
+ def role2 = role2Status
+
+ def role0_desired = 8
+
+ role0.desired = role0_desired
+ int role0_actual = 5
+ int role0_requested = role0_desired - role0_actual
+ role0_actual.times {
+ role0.incActual()
+ }
+ assert role0.getActual() == role0_actual
+ role0_requested.times {
+ role0.incRequested()
+ }
+ assert role0.getRequested() == role0_requested
+
+ def role0_failures = 2
+
role0.noteFailed(false, "", ContainerOutcome.Failed)
role0.noteFailed(true, "", ContainerOutcome.Failed)
+ // all aa roles fields are in the
+ def aarole_desired = 200
+ aaRole.desired = aarole_desired
+ def aarole_actual = 90
+ def aarole_active = 1
+ def aarole_requested = aarole_desired - aarole_actual
+ def aarole_pending = aarole_requested - 1
+ def aarole_failures = 0
+ aarole_actual.times {
+ aaRole.incActual()
+ }
+ assert aaRole.actual == aarole_actual
+ aaRole.outstandingAArequest = new OutstandingRequest(2, "")
+ // add a requested
+ aaRole.incRequested()
+ aaRole.setPendingAntiAffineRequests(aarole_pending)
+ assert aaRole.pendingAntiAffineRequests == aarole_pending
+
+ assert aaRole.actualAndRequested == aarole_actual + 1
StringWriter sw = new StringWriter(64);
PrintWriter pw = new PrintWriter(sw);
Hamlet hamlet = new Hamlet(pw, 0, false);
-
- int level = hamlet.nestLevel();
+
indexBlock.doIndex(hamlet, "accumulo");
def body = sw.toString()
log.info(body)
- assertEquals(body, level, hamlet.nestLevel())
// verify role data came out
assert body.contains("role0")
- //
- assert body.contains("8")
- assert body.contains("5")
- assert body.contains("3")
- assert body.contains("2")
- assert body.contains("1")
-
+ assertContains(role0_desired, body)
+ assertContains(role0_actual, body)
+ assertContains(role0_requested, body)
+ assertContains(role0_failures, body)
+
assert body.contains("role1")
assert body.contains("role2")
+
+ assertContains(aarole_desired, body)
+ assertContains(aarole_actual, body)
+// assertContains(aarole_requested, body)
+ assertContains(aarole_failures, body)
+ assert body.contains(indexBlock.buildAADetails(true, aarole_pending))
+
// verify that the sorting took place
assert body.indexOf("role0") < body.indexOf("role1")
assert body.indexOf("role1") < body.indexOf("role2")
+
+ assert !body.contains(IndexBlock.ALL_CONTAINERS_ALLOCATED)
+ // role
+ }
+
+ def assertContains(int ex, String html) {
+ assertStringContains(Integer.toString(ex), html)
}
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
new file mode 100644
index 0000000..451bdae
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.management
+
+import org.apache.slider.server.appmaster.management.LongGauge
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+class TestGauges extends SliderTestBase {
+
+ @Test
+ public void testLongGaugeOperations() throws Throwable {
+ LongGauge gauge = new LongGauge();
+ assert gauge.get() == 0
+ gauge.inc()
+ assert gauge.get() == 1
+ gauge.inc()
+ assert gauge.get() == 2
+ gauge.inc()
+ assert gauge.get() == 3
+ assert gauge.getValue() == gauge.get()
+ assert gauge.count == gauge.get()
+
+ gauge.dec()
+ assert gauge.get() == 2
+ assert gauge.decToFloor(1) == 1
+ assert gauge.get() == 1
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(0) == 0
+
+ gauge.set(4)
+ assert gauge.decToFloor(8) == 0
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
index 4721552..b4a7ac9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -33,8 +33,9 @@
String USERNAME = "bigdataborat"
int WAIT_TIME = 120;
- String WAIT_TIME_ARG = WAIT_TIME.toString()
+ String WAIT_TIME_ARG = Integer.toString(WAIT_TIME)
String SLIDER_TEST_XML = "slider-test.xml"
+ String NODES_UPDATED_FLAG_METRIC = "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag"
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index ae07187..cb6ce0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -109,11 +109,29 @@
JsonOutput.prettyPrint(json)
}
+ /**
+ * Convert a JSON string to something readable
+ * @param json
+ * @return a string for printing
+ */
+ public static String prettyPrintAsJson(Object src) {
+ JsonOutput.prettyPrint(JsonOutput.toJson(src))
+ }
+
+ /**
+ * Skip the test with a message
+ * @param message message logged and thrown
+ */
public static void skip(String message) {
log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
+ /**
+ * Skip the test with a message if condition holds
+ * @param condition predicate
+ * @param message message logged and thrown
+ */
public static void assume(boolean condition, String message) {
if (!condition) {
skip(message)
@@ -123,18 +141,25 @@
/**
* Skip a test if not running on Windows
*/
- public assumeWindows() {
+ public static void assumeWindows() {
assume(Shell.WINDOWS, "not windows")
}
/**
* Skip a test if running on Windows
*/
- public assumeNotWindows() {
+ public static void assumeNotWindows() {
assume(!Shell.WINDOWS, "windows")
}
/**
+ * skip a test on windows
+ */
+ public static void skipOnWindows() {
+ assumeNotWindows();
+ }
+
+ /**
* Equality size for a list
* @param left
* @param right
@@ -234,15 +259,6 @@
}
/**
- * skip a test on windows
- */
- public static void skipOnWindows() {
- if (Shell.WINDOWS) {
- skip("Not supported on windows")
- }
- }
-
- /**
* Assert that any needed libraries being present. On Unix none are needed;
* on windows they must be present
*/
@@ -621,7 +637,6 @@
static UrlConnectionOperations connectionOperations
static UgiJerseyBinding jerseyBinding;
-
/**
* Static initializer of the connection operations
* @param conf config
@@ -869,6 +884,29 @@
log.info("$realkey = $val")
return val
}
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() {
+ return tmpFile(".json")
+ }
+
+ /**
+ * Create a temp file with the specific name. It's deleted after creation,
+ * to avoid "file exists exceptions"
+ * @param suffix suffix, e.g. ".txt"
+ * @return a path to a file which may be created
+ */
+ public static File tmpFile(String suffix) {
+ File reportFile = File.createTempFile(
+ "temp",
+ suffix,
+ new File("target"))
+ reportFile.delete()
+ return reportFile
+ }
/**
* Execute a closure, assert it fails with a given exit code and text
@@ -1442,15 +1480,16 @@
/**
* Await a specific gauge being of the desired value
- * @param target target URL
+ * @param am URL of appmaster
* @param gauge gauge name
* @param desiredValue desired value
* @param timeout timeout in millis
* @param sleepDur sleep in millis
*/
- public void awaitGaugeValue(String target, String gauge, int desiredValue,
+ public void awaitGaugeValue(String am, String gauge, int desiredValue,
int timeout,
int sleepDur) {
+ String target = appendToURL(am, SYSTEM_METRICS_JSON)
def text = "Probe $target for gauge $gauge == $desiredValue"
repeatUntilSuccess(text,
this.&probeMetricGaugeValue,
@@ -1465,6 +1504,13 @@
}
}
+ /**
+ * Probe for a metric gauge holding a value.
+ *
+ * Keys: "url:String", "gauge:String", "desiredValue:int"
+ * @param args argument map
+ * @return success on the desired value, retry if not; fail on IOE
+ */
Outcome probeMetricGaugeValue(Map args) {
String url = requiredMapValue(args, "url")
String gauge = requiredMapValue(args, "gauge")
@@ -1481,4 +1527,13 @@
}
}
+ public static void assertStringContains(String expected, String text) {
+ assertNotNull("null text", text)
+ if (!text.contains(expected)) {
+ def message = "id not find $expected in \"$text\""
+ log.error(message)
+ fail(message)
+ }
+
+ }
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 4c43168..7237ff4 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -24,10 +24,10 @@
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
@@ -35,10 +35,12 @@
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -71,26 +73,19 @@
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
- if (hostname.equals("localhost")) {
- return true;
- }
- return false;
+ return hostname.equals("localhost");
}
});
-
}
protected static final Logger log =
LoggerFactory.getLogger(TestAMAgentWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
public static final String AGENT_URL =
"https://localhost:${PORT}/ws/v1/slider/agents/";
static MockFactory factory = new MockFactory();
- private static Configuration conf = new Configuration();
private static WebAppApi slider;
private static FileSystem fs;
@@ -117,28 +112,16 @@
YarnConfiguration conf = SliderUtils.createConfiguration();
fs = FileSystem.get(new URI("file:///"), conf);
AppState appState = null;
- try {
- fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMAgentWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
- new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
- fs.delete(historyPath, true);
- appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
- } catch (Exception e) {
- log.error("Failed to set up app {}", e, e);
- }
+ File historyWorkDir = new File("target/history", "TestAMAgentWebServices");
+ Path historyPath = new Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
ProviderAppState providerAppState = new ProviderAppState("undefined",
appState);
@@ -173,7 +156,7 @@
WebResource webResource = client.resource(base_url + "test/register");
response = webResource.type(MediaType.APPLICATION_JSON)
.post(RegistrationResponse.class, createDummyJSONRegister());
- Assert.assertEquals(RegistrationStatus.OK, response.getResponseStatus());
+ assertEquals(RegistrationStatus.OK, response.getResponseStatus());
}
protected Client createTestClient() {
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index df7e002..49ad71a 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -31,7 +31,6 @@
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -39,13 +38,13 @@
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
@@ -67,15 +66,12 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
public class TestAMManagementWebServices extends JerseyTest {
protected static final Logger log =
LoggerFactory.getLogger(TestAMManagementWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
- public static final String EXAMPLES =
- "/org/apache/slider/core/conf/examples/";
+ public static final String EXAMPLES = "/org/apache/slider/core/conf/examples/";
static MockFactory factory = new MockFactory();
private static Configuration conf = new Configuration();
private static WebAppApi slider;
@@ -116,40 +112,27 @@
}
protected AggregateConf getAggregateConf() {
- JsonSerDeser<ConfTree> confTreeJsonSerDeser =
- new JsonSerDeser<ConfTree>(ConfTree.class);
- ConfTree internal = null;
- ConfTree app_conf = null;
- ConfTree resources = null;
try {
- internal =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES +"internal.json");
- app_conf =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "app_configuration.json");
- resources =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "resources.json");
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class);
+ AggregateConf aggregateConf = new AggregateConf(
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json")
+ );
+ aggregateConf.setName("test");
+ return aggregateConf;
} catch (IOException e) {
- fail(e.getMessage());
+ throw new AssertionError(e.getMessage(), e);
}
- AggregateConf aggregateConf = new AggregateConf(
- resources,
- app_conf,
- internal);
- aggregateConf.setName("test");
- return aggregateConf;
}
-
}
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
injector = createInjector();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- fs = FileSystem.get(new URI("file:///"), conf);
+ fs = FileSystem.get(new URI("file:///"), SliderUtils.createConfiguration());
}
private static Injector createInjector() {
@@ -160,23 +143,17 @@
AppState appState = null;
try {
fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMManagementWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
+ File historyWorkDir = new File("target/history", "TestAMManagementWebServices");
+ org.apache.hadoop.fs.Path historyPath =
new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
- appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
} catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
log.error("{}", e, e);
}
@@ -239,9 +216,19 @@
assertEquals("wrong href",
"http://localhost:9998/slideram/ws/v1/slider/mgmt/app/configurations/internal",
json.getHref());
- assertEquals("wrong description",
- "Internal configuration DO NOT EDIT",
- json.getMetadata().get("description"));
+
+ assertDescriptionContains("org/apache/slider/core/conf/examples/internal.json", json);
+ }
+
+ private void assertDescriptionContains(String expected, ConfTreeResource json) {
+
+ Map<String, Object> metadata = json.getMetadata();
+ assertNotNull("No metadata", metadata);
+ Object actual = metadata.get("description");
+ assertNotNull("No description", actual);
+
+ assertTrue(String.format("Did not find \"%s\" in \"%s\"", expected, actual),
+ actual.toString().contains(expected));
}
@Test
@@ -263,6 +250,7 @@
assertNotNull("no components", components);
assertEquals("incorrect number of components", 2, components.size());
assertNotNull("wrong component", components.get("worker"));
+ assertDescriptionContains("org/apache/slider/core/conf/examples/resources.json", json);
}
@Test
@@ -283,5 +271,7 @@
assertNotNull("no components", components);
assertEquals("incorrect number of components", 2, components.size());
assertNotNull("wrong component", components.get("worker"));
+ assertDescriptionContains("org/apache/slider/core/conf/examples/app_configuration.json", json);
+
}
}
diff --git a/slider-core/src/test/python/agent.py b/slider-core/src/test/python/agent.py
index 4177074..1c9ff5b 100644
--- a/slider-core/src/test/python/agent.py
+++ b/slider-core/src/test/python/agent.py
@@ -24,14 +24,21 @@
from optparse import OptionParser
import os
+
# A representative Agent code for the embedded agent
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/agent.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
+
(options, args) = parser.parse_args()
if options.log_folder:
@@ -43,9 +50,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 30
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
index 116d179..2eacc89 100755
--- a/slider-core/src/test/python/agent/main.py
+++ b/slider-core/src/test/python/agent/main.py
@@ -26,7 +26,7 @@
def main():
- print "Executing echo"
+ print "Executing src/test/python/agent/main.py"
try:
print 'Argument List: {0}'.format(str(sys.argv))
except AttributeError:
@@ -37,6 +37,8 @@
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
parser.add_option('--label', dest='label', help='label')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
parser.add_option('--zk-quorum', dest='host:2181', help='zookeeper quorum')
parser.add_option('--zk-reg-path', dest='/register/org-apache-slider/cl1', help='zookeeper registry path')
@@ -51,9 +53,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 300
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/echo.py b/slider-core/src/test/python/echo.py
index ea5e8ce..1dabf1c 100644
--- a/slider-core/src/test/python/echo.py
+++ b/slider-core/src/test/python/echo.py
@@ -26,13 +26,18 @@
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/echo.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
(options, args) = parser.parse_args()
if options.log_folder:
@@ -44,9 +49,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 300
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index cf4afe1..b7251d3 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -24,7 +24,7 @@
</comment>
<version>0.1</version>
<type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
+ <minHadoopVersion>2.6.0</minHadoopVersion>
<components>
<component>
<name>hbase-rs</name>
@@ -51,7 +51,7 @@
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
+ <maxInstanceCount>100</maxInstanceCount>
<commandScript>
<script>echo.py</script>
<scriptType>PYTHON</scriptType>
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
index abf42f9..ee9fc59 100644
--- a/slider-core/src/test/resources/example-slider-test.xml
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -33,12 +33,6 @@
</property>
<property>
- <name>slider.test.teardown.killall</name>
- <description>Kill all hbase/accumulo, etc processes on test teardown</description>
- <value>true</value>
- </property>
-
- <property>
<name>slider.test.thaw.wait.seconds</name>
<description>Time to wait for a start to work</description>
<value>60</value>
@@ -50,68 +44,4 @@
<value>60</value>
</property>
-
- <!-- Properties for the slider-hbase-provider only -not HBase-under-agent- -->
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.hbase.launch.wait.seconds</name>
- <description>Time to wait for the HBase application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <!-- Properties for the slider-accumulo-provider only -not HBase-under-agent- -->
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.launch.wait.seconds</name>
- <description>Time to wait for the accumulo application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>/home/slider/accumulo</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/Projects/accumulo/accumulo-1.6.0-bin.tar.gz</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>/home/slider/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>/home/slider/hadoop/</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
</configuration>
diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties
index ed19a5b..5858f55 100644
--- a/slider-core/src/test/resources/log4j.properties
+++ b/slider-core/src/test/resources/log4j.properties
@@ -38,6 +38,7 @@
#crank back on some noise
log4j.logger.org.apache.hadoop.util.Shell=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager=FATAL
log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
@@ -49,6 +50,7 @@
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl=ERROR
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
log4j.logger.org.apache.hadoop.metrics2=ERROR
log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
index 489acda..5690225 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
@@ -1,6 +1,8 @@
{
"schema": "http://example.org/specification/v2.0.0",
-
+ "metadata": {
+ "description": "org/apache/slider/core/conf/examples/app_configuration.json"
+ },
"global": {
"zookeeper.port": "2181",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
index 592b4dc..da53b94 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "Internal configuration DO NOT EDIT"
+ "description": "Internal resolved - org/apache/slider/core/conf/examples/internal-resolved.json"
},
"global": {
"application.name": "small_cluster",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
index 4c782fb..b628d10 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "Internal configuration DO NOT EDIT"
+ "description": "Internal unresolved - org/apache/slider/core/conf/examples/internal.json"
},
"global": {
"application.name": "small_cluster",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
index 9d1e916..206424d 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "example of a resources file"
+ "description": "example of a resources file: org/apache/slider/core/conf/examples/resources.json"
},
"global": {
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d3a6680..5fa4c2a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -29,15 +29,17 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
-import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.launch.SerializedApplicationReport
-import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.client.SliderClient
+import org.apache.slider.core.launch.SerializedApplicationReport
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.ApplicationReportSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.test.SliderTestUtils
import org.apache.slider.test.Outcome;
@@ -88,7 +90,6 @@
* Keytab for secure cluster
*/
public static final String TEST_AM_KEYTAB
- static File keytabFile
/**
* shell-escaped ~ symbol. On windows this does
@@ -207,7 +208,7 @@
return false;
}
}
-
+
/**
* Add a jar to the slider classpath by looking up a class and determining
* its containing JAR
@@ -326,7 +327,6 @@
])
}
-
static SliderShell freeze(
int exitCode,
String name,
@@ -360,9 +360,9 @@
static SliderShell killContainer(String name, String containerID) {
slider(0,
[
- ACTION_KILL_CONTAINER,
- name,
- containerID
+ ACTION_KILL_CONTAINER,
+ name,
+ containerID
])
}
@@ -441,9 +441,7 @@
}
static SliderShell registry(Collection<String> commands) {
- slider(0,
- [ACTION_REGISTRY] + commands
- )
+ slider(0, [ACTION_REGISTRY] + commands)
}
/**
@@ -573,7 +571,7 @@
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
- SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+ SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
if (blockUntilRunning) {
@@ -775,24 +773,6 @@
}
/**
- * Create a temp JSON file. After coming up with the name, the file
- * is deleted
- * @return the filename
- */
- public static File createTempJsonFile() {
- return tmpFile(".json")
- }
-
- public static File tmpFile(String suffix) {
- File reportFile = File.createTempFile(
- "launch",
- suffix,
- new File("target"))
- reportFile.delete()
- return reportFile
- }
-
- /**
* If the option is not null/empty, add the command and the option
* @param args arg list being built up
* @param command command to add option
@@ -813,20 +793,20 @@
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- }
+ }
return null;
- }
-
+ }
+
public static SerializedApplicationReport loadAppReport(File reportFile) {
- if (reportFile.exists() && reportFile.length()> 0) {
+ if (reportFile.exists() && reportFile.length() > 0) {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- } else {
+ }else {
throw new FileNotFoundException(reportFile.absolutePath)
- }
- }
-
+ }
+ }
+
public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
def report = maybeLoadAppReport(launchReport)
if (report) {
@@ -857,7 +837,50 @@
}
}
-
+ /**
+ * Lookup an application, return null if loading failed
+ * @param id application ID
+ * @return an application report or null
+ */
+ public static NodeInformationList listNodes(String name = "", boolean healthy = false, String label = "") {
+ File reportFile = createTempJsonFile();
+ try {
+ def shell = nodes(name, reportFile, healthy, label)
+ if (log.isDebugEnabled()) {
+ shell.dumpOutput()
+ }
+ JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer();
+ serDeser.fromFile(reportFile)
+ } finally {
+ reportFile.delete()
+ }
+ }
+
+ /**
+ * List cluster nodes
+ * @param name of cluster or null
+ * @param out output file (or null)
+ * @param healthy list healthy nodes only
+ * @param label label to filter on
+ * @return output
+ */
+ static SliderShell nodes(String name, File out = null, boolean healthy = false, String label = "") {
+ def commands = [ACTION_NODES]
+ if (label) {
+ commands += [ ARG_LABEL, label]
+ }
+ if (name) {
+ commands << name
+ }
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
+ if (healthy) {
+ commands << ARG_HEALTHY
+ }
+ slider(0, commands)
+ }
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
@@ -931,11 +954,7 @@
}
// trigger a failure on registry lookup
- SliderShell shell = registry(0, [
- ARG_NAME,
- application,
- ARG_LISTEXP
- ])
+ registry(0, [ARG_NAME, application, ARG_LISTEXP])
}
}
@@ -1206,7 +1225,6 @@
}
public ClusterDescription execStatus(String application) {
- ClusterDescription cd
File statusFile = File.createTempFile("status", ".json")
try {
slider(EXIT_SUCCESS,
@@ -1280,6 +1298,13 @@
}
}
+ /**
+ * Assert that exactly the number of containers are live
+ * @param clustername name of cluster
+ * @param component component to probe
+ * @param count count
+ * @return
+ */
public ClusterDescription assertContainersLive(String clustername,
String component,
int count) {
@@ -1426,8 +1451,8 @@
}
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for the output {@code command: List} containing {@code text}
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandOutputContains(Map args) {
@@ -1436,9 +1461,10 @@
SliderShell shell = slider(0, command)
return Outcome.fromBool(shell.outputContains(text))
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for a command {@code command: List} succeeeding
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandSucceeds(Map args) {
@@ -1446,9 +1472,11 @@
SliderShell shell = slider(command)
return Outcome.fromBool(shell.ret == 0)
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map
+ * probe for a command {@code command: List} generating a file 'filename'
+ * which must contain the text 'text'
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome generatedFileContains(Map args) {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
new file mode 100644
index 0000000..5de2b8e
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest
+
+/**
+ * The various resources used for test runs
+ */
+interface ResourcePaths {
+
+ String SLIDER_CORE_TEST_SRC= "../slider-core/src/test"
+ String SLIDER_CORE_APP_PACKAGES = "$SLIDER_CORE_TEST_SRC/app_packages"
+ String COMMAND_LOG_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources.json"
+ String COMMAND_LOG_RESOURCES_QUEUE_LABELS = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_queue_labels.json"
+ String COMMAND_LOG_RESOURCES_NO_ROLE = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_no_role.json"
+ String COMMAND_LOG_APPCONFIG_NO_HB = "$SLIDER_CORE_APP_PACKAGES/test_command_log/appConfig_no_hb.json"
+ String COMMAND_LOG_APPCONFIG_FAST_NO_REG = "$SLIDER_CORE_APP_PACKAGES/test_command_log/appConfig_fast_no_reg.json"
+
+ String PING_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/resources.json"
+ String PING_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/metainfo.json"
+ String PING_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/appConfig.json"
+
+ String SLEEP_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources.json"
+ String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json"
+ String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json"
+
+}
\ No newline at end of file
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
index 9826e97..5d069bc 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
@@ -37,7 +37,6 @@
*/
class ClusterConnectivityIT extends CommandTestBase {
-
public static final int CONNECT_TIMEOUT = 2000
@Test
@@ -60,7 +59,7 @@
tuples.each {
telnet(it.hostText, it.port)
}
-
+
}
@Test
@@ -73,7 +72,7 @@
telnet(rmAddr.hostName, rmAddr.port)
}
}
-
+
@Test
public void testRMBinding() throws Throwable {
SliderYarnClientImpl yarnClient = new SliderYarnClientImpl()
@@ -83,10 +82,10 @@
YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,5000)
SLIDER_CONFIG.setInt(
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,50)
-
+
yarnClient.init(SLIDER_CONFIG)
yarnClient.start();
- def instances = yarnClient.listInstances("")
+ def instances = yarnClient.listDeployedInstances("")
instances.each {it -> log.info("Instance $it.applicationId")}
} finally {
yarnClient.stop()
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
index cf74bbf..91797f9 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
@@ -23,9 +23,6 @@
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Before
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
index 728920e..e53c875 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -36,11 +36,10 @@
@Slf4j
class SyspropsIT extends SliderTestUtils{
-
@Test
public void testDumpSysprops() throws Throwable {
def sysprops = System.properties
- TreeSet<String> sorted = new TreeSet<String>();
+ TreeSet<String> sorted = new TreeSet<>();
sysprops.keys().each { String it -> sorted.add(it)}
sorted.each { String key ->
log.info("$key=\"${sysprops[key]}\"")
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
index 73912e6..666342e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
@@ -20,7 +20,6 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.Test
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index 916117c..b378110 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -22,7 +22,6 @@
import groovy.util.logging.Slf4j
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
-import org.junit.BeforeClass
import org.junit.Test
@CompileStatic
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
index 75d0634..e22c5f2 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
@@ -20,9 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.bigtop.itest.shell.Shell
import org.apache.slider.funtest.framework.CommandTestBase
-import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.common.params.SliderActions
import org.junit.Test
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
index 43275e6..b32275e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
@@ -18,28 +18,9 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.funtest.framework.FileUploader
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.After
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
new file mode 100644
index 0000000..c42edf8
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.types.NodeEntryInformation
+import org.apache.slider.api.types.NodeInformation
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.launch.SerializedApplicationReport
+import org.apache.slider.funtest.ResourcePaths
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AASleepIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ static String NAME = "test-aa-sleep"
+
+ static String TEST_RESOURCE = ResourcePaths.SLEEP_RESOURCES
+ static String TEST_METADATA = ResourcePaths.SLEEP_META
+ public static final String SLEEP_100 = "SLEEP_100"
+ public static final int SLEEP_LONG_PRIORITY = 3
+ public static final String SLEEP_LONG_PRIORITY_S = Integer.toString(SLEEP_LONG_PRIORITY)
+
+ public static final String SLEEP_LONG = "SLEEP_LONG"
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(NAME)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(NAME)
+ }
+
+ @Test
+ public void testAASleepIt() throws Throwable {
+ describe("Test Anti-Affinity Placement")
+
+ describe "diagnostics"
+
+ slider([ACTION_DIAGNOSTICS, ARG_VERBOSE, ARG_CLIENT, ARG_YARN, ARG_CREDENTIALS]).dumpOutput()
+
+ describe "list nodes"
+
+ def healthyNodes = listNodes("", true)
+
+ def healthyNodeCount = healthyNodes.size()
+ describe("Cluster nodes : ${healthyNodeCount}")
+ log.info(NodeInformationList.createSerializer().toJson(healthyNodes))
+
+ File launchReportFile = createTempJsonFile();
+
+ int desired = buildDesiredCount(healthyNodeCount)
+
+ SliderShell shell = createSliderApplicationMinPkg(NAME,
+ TEST_METADATA,
+ TEST_RESOURCE,
+ ResourcePaths.SLEEP_APPCONFIG,
+ [
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, Integer.toString( desired),
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_PRIORITY, SLEEP_LONG_PRIORITY_S
+ ],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ status(0, NAME)
+
+ def expected = buildExpectedCount(desired)
+ expectLiveContainerCountReached(NAME, SLEEP_100, expected,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ operations(NAME, loadAppReport(launchReportFile), desired, expected, healthyNodes)
+
+ //stop
+ freeze(0, NAME,
+ [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
+ destroy(0, NAME)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, NAME)
+ }
+
+ protected int buildExpectedCount(int desired) {
+ desired - 1
+ }
+
+ protected int buildDesiredCount(int clustersize) {
+ clustersize + 1
+ }
+
+ protected void operations(String name,
+ SerializedApplicationReport appReport,
+ int desired,
+ int expected,
+ NodeInformationList healthyNodes) {
+
+ // now here await for the cluster size to grow: if it does, there's a problem
+ // spin for a while and fail if the number ever goes above it.
+ ClusterDescription cd = null
+ 5.times {
+ cd = assertContainersLive(NAME, SLEEP_LONG, expected)
+ sleep(1000 * 10)
+ }
+
+ // here cluster is still 1 below expected
+ def role = cd.getRole(SLEEP_LONG)
+ assert "1" == role.get(RoleKeys.ROLE_PENDING_AA_INSTANCES)
+
+ // look through the nodes
+ def currentNodes = listNodes(name)
+ // assert that there is no entry of the sleep long priority on any node
+ currentNodes.each { NodeInformation it ->
+ def entry = it.entries[SLEEP_LONG]
+ assert entry == null || entry.live <= 1
+ }
+
+ // now reduce the cluster size and assert that the size stays the same
+
+
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
index 950dc68..05adebd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
@@ -33,7 +33,6 @@
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
-import org.junit.Assert
import org.junit.Test
import javax.net.ssl.TrustManager
@@ -47,7 +46,6 @@
import java.security.cert.CertificateException
import java.security.cert.X509Certificate
import com.google.common.io.Files
-import java.io.File
@CompileStatic
@Slf4j
@@ -107,7 +105,8 @@
String password = "welcome";
// ensure file doesn't exist
- new File(filename).delete();
+ def keystoreFile = new File(filename)
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -118,7 +117,7 @@
ARG_PASSWORD, password
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
KeyStore keystore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -126,7 +125,7 @@
filename = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -137,7 +136,7 @@
ARG_PASSWORD, password
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
KeyStore truststore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -164,7 +163,7 @@
log.info("Created credential provider $providerString for test")
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -176,7 +175,7 @@
ARG_PROVIDER, providerString
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
keystore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -184,7 +183,7 @@
filename = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -196,7 +195,7 @@
ARG_PROVIDER, providerString
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
truststore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -207,7 +206,7 @@
private static void validateKeystore(KeyStore keystore) {
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
+ assert certificate
String hostname = InetAddress.localHost.canonicalHostName;
@@ -217,9 +216,7 @@
// Get subject
Principal principal = x509cert.getSubjectDN();
String subjectDn = principal.getName();
- Assert.assertEquals("wrong DN",
- "CN=" + hostname + ", OU=" + APPLICATION_NAME + ", OU=client",
- subjectDn);
+ assert subjectDn == "CN=" + hostname + ", OU=" + APPLICATION_NAME + ", OU=client"
}
}
@@ -229,7 +226,7 @@
// obtain server cert
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
+ assert certificate
// validate keystore cert using trust store
TrustManagerFactory trustManagerFactory =
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
index deb2d97..6d1f759 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
@@ -21,19 +21,12 @@
import groovy.io.FileType
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.junit.After
-import org.junit.Before
import org.junit.Test
@CompileStatic
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index ca9f71b..faeb0a1 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -28,6 +28,7 @@
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -43,7 +44,7 @@
static String CLUSTER = "test-agent-cluster-lifecycle"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 39b5d6c..d73eb76 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -36,8 +37,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "two-container-fail-heartbeat"
- private static String APP_TEMPLATE3 =
- "../slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json"
+ private static String APP_TEMPLATE3 = ResourcePaths.COMMAND_LOG_APPCONFIG_NO_HB
@After
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 7d1be89..cfdff75 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -36,8 +37,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "one-container-fail-register"
- private static String APP_TEMPLATE2 =
- "../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
+ private static String APP_TEMPLATE2 = ResourcePaths.COMMAND_LOG_APPCONFIG_FAST_NO_REG
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
index 17eaf04..ee22360 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
@@ -21,6 +21,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.server.appmaster.SliderAppMaster
import static org.apache.slider.api.InternalKeys.*
@@ -41,7 +42,7 @@
static String CLUSTER = "test-agent-launchfail"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
index e7b0454..779316b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
@@ -20,18 +20,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.Outcome
-
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -47,9 +40,8 @@
static String CLUSTER = "test-agent-sleep-100"
- static String APP_RESOURCE11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json"
- static String APP_META11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json"
-
+ static String TEST_RESOURCE = ResourcePaths.SLEEP_RESOURCES
+ static String TEST_METADATA = ResourcePaths.SLEEP_META
@Before
public void prepareCluster() {
@@ -62,14 +54,14 @@
}
@Test
- public void testAgentRegistry() throws Throwable {
+ public void testAgentMinSleepIt() throws Throwable {
describe("Create a cluster using metainfo and resources only that executes sleep 100")
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
- APP_META11,
- APP_RESOURCE11,
+ TEST_METADATA,
+ TEST_RESOURCE,
null,
[],
launchReportFile)
@@ -91,7 +83,7 @@
CONTAINER_LAUNCH_TIMEOUT)
// sleep for some manual test
- describe("You may quickly perform manual tests against the application instance " + CLUSTER)
+ describe("You may quickly perform manual tests against the application instance $CLUSTER")
sleep(1000 * 30)
//stop
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
index ce2ed92..b10e792 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
@@ -22,18 +22,11 @@
import groovy.util.logging.Slf4j
import groovy.json.*
import org.apache.hadoop.net.NetUtils
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.Outcome
-
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -49,9 +42,9 @@
static String CLUSTER = "test-agent-ping-port"
- static String APP_RESOURCE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json"
- static String APP_META12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json"
- static String APP_TEMPLATE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json"
+ static String APP_RESOURCE12 = ResourcePaths.PING_RESOURCES
+ static String APP_META12 = ResourcePaths.PING_META
+ static String APP_TEMPLATE12 = ResourcePaths.PING_APPCONFIG
@Before
@@ -68,7 +61,7 @@
public void testAgentRegistry() throws Throwable {
describe("Create a cluster using metainfo, resources, and appConfig that calls nc to listen on a port")
assumeNotWindows()
- def clusterpath = buildClusterPath(CLUSTER)
+ buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index ff5e57e..7a03a05 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -28,6 +28,7 @@
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.test.Outcome
import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
@@ -46,8 +47,7 @@
static String CLUSTER = "test-agent-registry"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
-
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 62be615..d1f8f20 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -23,13 +23,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.api.RegistryOperations
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol
-import org.apache.hadoop.yarn.client.ClientRMProxy
-import org.apache.hadoop.yarn.webapp.ForbiddenException
import org.apache.slider.agent.rest.IpcApiClientTestDelegates
import org.apache.slider.agent.rest.JerseyTestDelegates
import org.apache.slider.agent.rest.AbstractRestTestDelegate
-import org.apache.slider.agent.rest.LowLevelRestTestDelegates
import org.apache.slider.agent.rest.RestAPIClientTestDelegates
import org.apache.slider.client.SliderClient
import org.apache.slider.client.ipc.SliderApplicationIpcClient
@@ -40,13 +36,13 @@
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.server.appmaster.rpc.RpcBinder
import org.junit.After
import org.junit.Before
-import org.junit.Ignore
import org.junit.Test
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_HEALTHCHECK
@@ -59,7 +55,7 @@
static String CLUSTER = "test-agent-web"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
@@ -72,12 +68,11 @@
}
@Test
- @Ignore("SLIDER-907")
public void testAgentWeb() throws Throwable {
describe("Web queries & REST operations against an AM")
// verify the ws/ path is open for all HTTP verbs
- def sliderConfiguration = ConfigHelper.loadSliderConfiguration();
+ ConfigHelper.loadSliderConfiguration();
/*
Is the back door required? If so, don't test complex verbs via the proxy
@@ -91,7 +86,7 @@
def directComplexVerbs = proxyComplexVerbs || SLIDER_CONFIG.getBoolean(
SliderXmlConfKeys.X_DEV_INSECURE_WS,
SliderXmlConfKeys.X_DEV_INSECURE_DEFAULT)
- def clusterpath = buildClusterPath(CLUSTER)
+ buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
@@ -110,77 +105,86 @@
def appId = ensureYarnApplicationIsUp(launchReportFile)
assert appId
awaitApplicationURLPublished(appId, instanceLaunchTime)
- File liveReportFile = createTempJsonFile();
- lookup(appId, liveReportFile)
- def report = loadAppReport(liveReportFile)
- assert report.url
+ def report = lookupApplication(appId)
+ assert report && report.url
def proxyAM = report.url
+ // decide whether or not to use https
+ def proxyTests = !proxyAM.toLowerCase(Locale.ENGLISH).startsWith("https:")
+
// here the URL has been published, but it may not be live yet, due to the time
// it takes the AM to build app state and boostrap the Web UI
def directAM = report.origTrackingUrl;
- describe "Proxy Jersey Tests"
-
Client jerseyClient = createUGIJerseyClient()
- JerseyTestDelegates proxyJerseyTests =
- new JerseyTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
+ if (proxyTests) {
+ describe "Proxy Jersey Tests"
- // wait it coming up
- awaitRestEndpointLive(proxyJerseyTests, instanceLaunchTime)
+ JerseyTestDelegates proxyJerseyTests =
+ new JerseyTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
- proxyJerseyTests.testSuiteGetOperations()
+ // wait it coming up
+ awaitRestEndpointLive(proxyJerseyTests, instanceLaunchTime)
+
+ proxyJerseyTests.testSuiteGetOperations()
+
+ describe "Proxy SliderRestClient Tests"
+ RestAPIClientTestDelegates proxySliderRestAPI =
+ new RestAPIClientTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
+ proxySliderRestAPI.testSuiteAll()
+
+
+ if (UserGroupInformation.securityEnabled) {
+ describe "Insecure Proxy Tests against a secure cluster"
+
+ // these tests use the Jersey client without the Hadoop-specific
+ // SPNEGO
+ JerseyTestDelegates basicJerseyClientTests =
+ new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
+ basicJerseyClientTests.testSuiteGetOperations()
+ }
+
+ // create the Rest client via the registry
+
+ //get a slider client against the cluster
+
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ RegistryOperations operations = sliderClient.registryOperations;
+ def restClientFactory = new RestClientFactory(
+ operations, jerseyClient,
+ "~", SliderKeys.APP_TYPE, CLUSTER)
+ def sliderApplicationApi = restClientFactory.createSliderAppApiClient();
+ sliderApplicationApi.desiredModel
+ sliderApplicationApi.resolvedModel
+ if (proxyComplexVerbs) {
+ sliderApplicationApi.ping("registry located")
+ }
+
+ } else {
+ describe "skipping tests against HTTPS RM proxy $proxyAM"
+ }
+
describe "Direct Jersey Tests"
JerseyTestDelegates directJerseyTests =
new JerseyTestDelegates(directAM, jerseyClient, directComplexVerbs)
directJerseyTests.testSuiteAll()
- describe "Proxy SliderRestClient Tests"
- RestAPIClientTestDelegates proxySliderRestAPI =
- new RestAPIClientTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
- proxySliderRestAPI.testSuiteAll()
-
describe "Direct SliderRestClient Tests"
RestAPIClientTestDelegates directSliderRestAPI =
new RestAPIClientTestDelegates(directAM, jerseyClient, directComplexVerbs)
directSliderRestAPI.testSuiteAll()
- if (UserGroupInformation.securityEnabled) {
- describe "Insecure Proxy Tests against a secure cluster"
-
- // these tests use the Jersey client without the Hadoop-specific
- // SPNEGO
- JerseyTestDelegates basicJerseyClientTests =
- new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
- basicJerseyClientTests.testSuiteGetOperations()
- }
-
- // create the Rest client via the registry
-
- //get a slider client against the cluster
-
- SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
- RegistryOperations operations = sliderClient.registryOperations;
- def restClientFactory = new RestClientFactory(
- operations, jerseyClient,
- "~", SliderKeys.APP_TYPE, CLUSTER)
- def sliderApplicationApi = restClientFactory.createSliderAppApiClient();
- sliderApplicationApi.desiredModel
- sliderApplicationApi.resolvedModel
- if (proxyComplexVerbs) {
- sliderApplicationApi.ping("registry located")
- }
// maybe execute IPC operations.
// these are skipped when security is enabled, until
// there's some code set up to do the tokens properly
- if (!UserGroupInformation.isSecurityEnabled()) {
+ if (!UserGroupInformation.securityEnabled) {
describe("IPC equivalent operations")
def sliderClusterProtocol =
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 0db775b..060af5d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -76,7 +76,8 @@
application,
ARG_COMPONENT,
COMMAND_LOGGER,
- "2"])
+ "2"
+ ])
// sleep till the new instance starts
sleep(1000 * 10)
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index 5758feb..666efff 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -67,8 +68,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "happy-path-with-queue-labels"
private static String TARGET_QUEUE = "labeled"
- private static String APP_RESOURCE4 =
- "../slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json"
+ private static String APP_RESOURCE4 = ResourcePaths.COMMAND_LOG_RESOURCES_QUEUE_LABELS
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
index 9fc8e6a..fa8b7a5 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -49,8 +50,7 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "app-upgrade-happy-path"
- private static String APP_RESOURCE =
- "../slider-core/src/test/app_packages/test_command_log/resources.json"
+ private static String APP_RESOURCE = ResourcePaths.COMMAND_LOG_RESOURCES
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index f03fb63..66bc10b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -37,13 +37,10 @@
public class ClusterBuildDestroyIT extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
-
static String CLUSTER = "test-cluster-build-destroy"
-
@BeforeClass
public static void prepareCluster() {
-
setupCluster(CLUSTER)
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
similarity index 60%
copy from slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
copy to slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
index 3dd7857..7d0ee46 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
@@ -20,26 +20,22 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.SliderShell
-import org.junit.Before
-import org.junit.Test
+import org.apache.slider.core.launch.SerializedApplicationReport
-/**
- * For a quick demo of a slider app; this starts the apps through agent test but
- * neglects to tear it down afterwards
- */
@CompileStatic
@Slf4j
-public class AppsThroughAgentDemo extends AppsThroughAgentIT {
+class DemoAASleep extends AASleepIT {
@Override
- void destroyCluster() {
-// super.destroyCluster()
+ protected void operations(
+ String name,
+ SerializedApplicationReport appReport,
+ int desired,
+ int expected) {
+ super.operations(name, appReport, desired, expected)
+
+ describe("cluster is live at ${appReport.url}")
+
+ sleep(10 * 60 * 1000)
}
-
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
similarity index 93%
rename from slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
rename to slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
index 3dd7857..ea10390 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
@@ -35,11 +35,11 @@
*/
@CompileStatic
@Slf4j
-public class AppsThroughAgentDemo extends AppsThroughAgentIT {
+public class DemoAppsThroughAgent extends AppsThroughAgentIT {
@Override
void destroyCluster() {
-// super.destroyCluster()
+
}
-
+
}
