Add base framework for api server. (#2094)
* Add base framework for api server.
* Fix build issues.
* Remove extra newlines.
* add config files as resources into api server jar
* package heron api server into tools
* Implement submit/kill/activate/deactivate for the rest api server.
* Remove excess logging and print statements from SubmitterMain.
* Add help flag and remove println and hardcoded directory.
* Add apiserver tests for the ConfigUtils.
* Add user and verion params for submit topology.
* Add release file flag for apiserver.
The api server will read the, build version, build user
and build tag from the release file and use those values
for submitting topologies.
* Set the submit_user when submitting a topology.
* adding URL based handling in heron submit
* Fix activate/deactivate/kill api paths in cli.
* support multiple deployment modes
* Return status 200 for a successful kill topology.
* Catch exceptions as opposed to runtime exceptions in topology resources.
* Update topology actions to check for status code 200.
kill/activate/deactivate endpoints will return a
http status code of 200 when successful.
* fix printing debug information
* Add script to start the api server.
* package heron api server
* get the actual binary since heron-apiserver is a symbolic link
* Add base-template flag and method to find where the jar is installed.
* Print error message if there is a parsing exception.
* Add error message when server fails to start.
* Update bind exception message.
* Add port flag for api server.
* added the service-url parameter
* - Remove HERON_VERSION from docker files
- Add error message reporting for service mode deployment
* remove debug print out
* Fix how yaml files are written to files.
* Pass extra parameters to update and restart extra parameters
Added exception handling for requests call
* Apply submit overrides and add version endpoint.
* Add restart rest endpoint.
* append a list element instead of extending it
* Add printing of args in debug mode
* Add rest update endpoint.
* Add verbose flag to apiserver.
* Allow update to accept overrides.
* Add more scheduler implementations.
* sort the version items and print it
* Add dryrun response for submit and update endpoints.
* fix python command line errors
* Fix api server activate handler.
* fixed submission to local cluster
* fix spelling mistake of function invocation
* Remove api depenedency from apiserver build file.
diff --git a/WORKSPACE b/WORKSPACE
index aa7d4c2..a5bf649 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -7,6 +7,12 @@
reef_version = "0.14.0"
slf4j_version = "1.7.7"
+# heron api server
+jetty_version = "9.4.6.v20170531"
+jersey_verion = "2.25.1"
+hk2_api = "2.5.0-b32"
+
+
maven_server(
name = "default",
url = "http://central.maven.org/maven2/",
@@ -471,3 +477,155 @@
artifact = "org.apache.pulsar:pulsar-client:jar:shaded:1.19.0-incubating"
)
# end Pulsar Client
+
+
+# heron api server
+# jetty
+maven_jar(
+ name = "org_eclipse_jetty_server",
+ artifact = "org.eclipse.jetty:jetty-server:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_http",
+ artifact = "org.eclipse.jetty:jetty-http:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_util",
+ artifact = "org.eclipse.jetty:jetty-util:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_io",
+ artifact = "org.eclipse.jetty:jetty-io:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_security",
+ artifact = "org.eclipse.jetty:jetty-security:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_servlet",
+ artifact = "org.eclipse.jetty:jetty-servlet:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_servlets",
+ artifact = "org.eclipse.jetty:jetty-servlets:" + jetty_version
+)
+
+maven_jar(
+ name = "org_eclipse_jetty_continuation",
+ artifact = "org.eclipse.jetty:jetty-continuation:" + jetty_version
+)
+
+maven_jar(
+ name = "javax_servlet_api",
+ artifact = "javax.servlet:javax.servlet-api:3.1.0"
+)
+# end jetty
+
+# jersey
+maven_jar(
+ name = "jersey_container_servlet_core",
+ artifact = "org.glassfish.jersey.containers:jersey-container-servlet-core:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_container_servlet",
+ artifact = "org.glassfish.jersey.containers:jersey-container-servlet:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_server",
+ artifact = "org.glassfish.jersey.core:jersey-server:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_client",
+ artifact = "org.glassfish.jersey.core:jersey-client:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_common",
+ artifact = "org.glassfish.jersey.core:jersey-common:jar:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_media_multipart",
+ artifact = "org.glassfish.jersey.media:jersey-media-multipart:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_media_jaxb",
+ artifact = "org.glassfish.jersey.media:jersey-media-jaxb:" + jersey_verion
+)
+
+maven_jar(
+ name = "jersey_guava",
+ artifact = "org.glassfish.jersey.bundles.repackaged:jersey-guava:" + jersey_verion
+)
+# end jersey
+
+maven_jar(
+ name = "javax_inject",
+ artifact = "org.glassfish.hk2.external:javax.inject:2.5.0-b32"
+)
+
+maven_jar(
+ name = "javax_annotation",
+ artifact = "javax.annotation:javax.annotation-api:1.2"
+)
+
+maven_jar(
+ name = "javax_validation",
+ artifact = "javax.validation:validation-api:1.1.0.Final"
+)
+
+maven_jar(
+ name = "javax_ws_rs_api",
+ artifact = "javax.ws.rs:javax.ws.rs-api:2.0.1"
+)
+
+maven_jar(
+ name = "hk2_api",
+ artifact = "org.glassfish.hk2:hk2-api:" + hk2_api
+)
+
+maven_jar(
+ name = "hk2_utils",
+ artifact = "org.glassfish.hk2:hk2-utils:" + hk2_api
+)
+
+maven_jar(
+ name = "hk2_aopalliance_repackaged",
+ artifact = "org.glassfish.hk2.external:aopalliance-repackaged:" + hk2_api
+)
+
+maven_jar(
+ name = "hk2_locator",
+ artifact = "org.glassfish.hk2:hk2-locator:" + hk2_api
+)
+
+maven_jar(
+ name = "hk2_osgi_resource_locator",
+ artifact = "org.glassfish.hk2:osgi-resource-locator:1.0.1"
+)
+
+maven_jar(
+ name = "org_javassit",
+ artifact = "org.javassist:javassist:3.20.0-GA"
+)
+
+maven_jar(
+ name = "mimepull",
+ artifact = "org.jvnet.mimepull:mimepull:1.9.7"
+)
+
+maven_jar(
+ name = "org_apache_commons_compress",
+ artifact = "org.apache.commons:commons-compress:1.14",
+)
+# end heron api server
diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7
index 19044d0..6a218c4 100644
--- a/docker/dist/Dockerfile.dist.centos7
+++ b/docker/dist/Dockerfile.dist.centos7
@@ -1,7 +1,5 @@
FROM centos:centos7
-ARG heronVersion
-
RUN yum -y upgrade
RUN yum -y install python; yum clean all
RUN yum -y install unzip; yum clean all
diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04
index c3f9073..0df770e 100644
--- a/docker/dist/Dockerfile.dist.ubuntu14.04
+++ b/docker/dist/Dockerfile.dist.ubuntu14.04
@@ -1,7 +1,5 @@
FROM ubuntu:14.04
-ARG heronVersion
-
RUN apt-get update
RUN apt-get -y install python ; apt-get clean all
RUN apt-get -y install unzip ; apt-get clean all
diff --git a/docker/dist/Dockerfile.dist.ubuntu15.10 b/docker/dist/Dockerfile.dist.ubuntu15.10
index 7375eae..5881348 100644
--- a/docker/dist/Dockerfile.dist.ubuntu15.10
+++ b/docker/dist/Dockerfile.dist.ubuntu15.10
@@ -1,7 +1,5 @@
FROM ubuntu:16.04
-ARG heronVersion
-
RUN apt-get update
RUN apt-get -y install python ; apt-get clean all
RUN apt-get -y install unzip ; apt-get clean all
diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04
index 7375eae..5881348 100644
--- a/docker/dist/Dockerfile.dist.ubuntu16.04
+++ b/docker/dist/Dockerfile.dist.ubuntu16.04
@@ -1,7 +1,5 @@
FROM ubuntu:16.04
-ARG heronVersion
-
RUN apt-get update
RUN apt-get -y install python ; apt-get clean all
RUN apt-get -y install unzip ; apt-get clean all
diff --git a/docker/scripts/build-docker.sh b/docker/scripts/build-docker.sh
index a0f8285..6e946e8 100755
--- a/docker/scripts/build-docker.sh
+++ b/docker/scripts/build-docker.sh
@@ -5,8 +5,7 @@
echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
}
-DOCKER_DIR=$(dirname $(realpath $0))
-PROJECT_DIR=$(dirname $DOCKER_DIR )
+DOCKER_DIR=$(dirname $(dirname $(realpath $0)))
SCRATCH_DIR="$HOME/.heron-docker"
cleanup() {
@@ -25,7 +24,7 @@
mkdir $1/artifacts
fi
- cp $DOCKER_DIR/* $1
+ cp $DOCKER_DIR/dist/* $1
}
run_build() {
@@ -37,13 +36,14 @@
setup_scratch_dir $SCRATCH_DIR
- echo "Building docker image with tag:$DOCKER_TAG"
#need to copy artifacts locally
+ echo "Building docker image with tag:$DOCKER_TAG"
cp -pr "$OUTPUT_DIRECTORY"/*$HERON_VERSION* "$SCRATCH_DIR/artifacts"
+ mv $SCRATCH_DIR/artifacts/heron-tools-install-$HERON_VERSION-$TARGET_PLATFORM.sh $SCRATCH_DIR/artifacts/heron-tools-install.sh
+ mv $SCRATCH_DIR/artifacts/heron-core-$HERON_VERSION-$TARGET_PLATFORM.tar.gz $SCRATCH_DIR/artifacts/heron-core.tar.gz
export HERON_VERSION
- docker build --build-arg heronVersion=$HERON_VERSION -t "$DOCKER_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
-
+ docker build -t "$DOCKER_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
}
case $# in
diff --git a/docker/scripts/ci-docker.sh b/docker/scripts/ci-docker.sh
index 71d89cb..79db0ac 100755
--- a/docker/scripts/ci-docker.sh
+++ b/docker/scripts/ci-docker.sh
@@ -65,9 +65,9 @@
# build the image
echo "Building docker image with tag:$DOCKER_TAG"
if [ "$HERON_VERSION" == "nightly" ]; then
- docker build --build-arg heronVersion=$HERON_VERSION -t "$DOCKER_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
+ docker build -t "$DOCKER_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
else
- docker build --build-arg heronVersion=$HERON_VERSION -t "$DOCKER_TAG" -t "$DOCKER_LATEST_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
+ docker build -t "$DOCKER_TAG" -t "$DOCKER_LATEST_TAG" -f "$DOCKER_FILE" "$SCRATCH_DIR"
fi
# save the image as a tar file
diff --git a/heron/config/src/yaml/BUILD b/heron/config/src/yaml/BUILD
index c918e66..606f74c 100644
--- a/heron/config/src/yaml/BUILD
+++ b/heron/config/src/yaml/BUILD
@@ -9,7 +9,7 @@
filegroup(
name = "conf-yaml",
- srcs = glob(["conf/*.yaml"]),
+ srcs = glob(["conf/**/*.yaml"]),
)
filegroup(
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerMain.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerMain.java
index 7fb40e6..3925fff 100644
--- a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerMain.java
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerMain.java
@@ -35,8 +35,7 @@
import com.twitter.heron.scheduler.client.ISchedulerClient;
import com.twitter.heron.scheduler.client.SchedulerClientFactory;
import com.twitter.heron.scheduler.dryrun.UpdateDryRunResponse;
-import com.twitter.heron.scheduler.dryrun.UpdateRawDryRunRenderer;
-import com.twitter.heron.scheduler.dryrun.UpdateTableDryRunRenderer;
+import com.twitter.heron.scheduler.utils.DryRunRenders;
import com.twitter.heron.spi.common.Config;
import com.twitter.heron.spi.common.ConfigLoader;
import com.twitter.heron.spi.common.Context;
@@ -321,7 +320,7 @@
LOG.log(Level.FINE, "Sending out dry-run response");
// Output may contain UTF-8 characters, so we should print using UTF-8 encoding
PrintStream out = new PrintStream(System.out, true, StandardCharsets.UTF_8.name());
- out.print(runtimeManagerMain.renderDryRunResponse(response));
+ out.print(DryRunRenders.render(response, Context.dryRunFormatType(config)));
// SUPPRESS CHECKSTYLE RegexpSinglelineJava
// Exit with status code 200 to indicate dry-run response is sent out
System.exit(200);
@@ -453,18 +452,4 @@
throws SchedulerException {
return new SchedulerClientFactory(config, runtime).getSchedulerClient();
}
-
- protected String renderDryRunResponse(UpdateDryRunResponse resp) {
- DryRunFormatType formatType = Context.dryRunFormatType(config);
- switch (formatType) {
- case RAW :
- return new UpdateRawDryRunRenderer(resp).render();
- case TABLE:
- return new UpdateTableDryRunRenderer(resp, false).render();
- case COLORED_TABLE:
- return new UpdateTableDryRunRenderer(resp, true).render();
- default: throw new IllegalArgumentException(
- String.format("Unexpected rendering format: %s", formatType));
- }
- }
}
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerRunner.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerRunner.java
index d090c0b..fe4f366 100644
--- a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerRunner.java
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/RuntimeManagerRunner.java
@@ -42,7 +42,7 @@
import com.twitter.heron.spi.utils.TMasterUtils;
public class RuntimeManagerRunner {
- static final String NEW_COMPONENT_PARALLELISM_KEY = "NEW_COMPONENT_PARALLELISM";
+ public static final String NEW_COMPONENT_PARALLELISM_KEY = "NEW_COMPONENT_PARALLELISM";
private static final Logger LOG = Logger.getLogger(RuntimeManagerRunner.class.getName());
private final Config config;
private final Config runtime;
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
index 6066d7a..f0ae589 100644
--- a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
@@ -32,13 +32,12 @@
import com.twitter.heron.api.generated.TopologyAPI;
import com.twitter.heron.common.basics.DryRunFormatType;
-import com.twitter.heron.common.basics.PackageType;
import com.twitter.heron.common.basics.SysUtils;
import com.twitter.heron.common.utils.logging.LoggingHelper;
import com.twitter.heron.scheduler.dryrun.SubmitDryRunResponse;
-import com.twitter.heron.scheduler.dryrun.SubmitRawDryRunRenderer;
-import com.twitter.heron.scheduler.dryrun.SubmitTableDryRunRenderer;
+import com.twitter.heron.scheduler.utils.DryRunRenders;
import com.twitter.heron.scheduler.utils.LauncherUtils;
+import com.twitter.heron.scheduler.utils.SubmitterUtils;
import com.twitter.heron.spi.common.Config;
import com.twitter.heron.spi.common.ConfigLoader;
import com.twitter.heron.spi.common.Context;
@@ -61,29 +60,6 @@
private static final Logger LOG = Logger.getLogger(SubmitterMain.class.getName());
/**
- * Load the topology config
- *
- * @param topologyPackage, tar ball containing user submitted jar/tar, defn and config
- * @param topologyBinaryFile, name of the user submitted topology jar/tar/pex file
- * @param topology, proto in memory version of topology definition
- * @return config, the topology config
- */
- protected static Config topologyConfigs(
- String topologyPackage, String topologyBinaryFile, String topologyDefnFile,
- TopologyAPI.Topology topology) {
- PackageType packageType = PackageType.getPackageType(topologyBinaryFile);
-
- return Config.newBuilder()
- .put(Key.TOPOLOGY_ID, topology.getId())
- .put(Key.TOPOLOGY_NAME, topology.getName())
- .put(Key.TOPOLOGY_DEFINITION_FILE, topologyDefnFile)
- .put(Key.TOPOLOGY_PACKAGE_FILE, topologyPackage)
- .put(Key.TOPOLOGY_BINARY_FILE, topologyBinaryFile)
- .put(Key.TOPOLOGY_PACKAGE_TYPE, packageType)
- .build();
- }
-
- /**
* Load the config parameters from the command line
*
* @param cluster, name of the cluster
@@ -258,6 +234,7 @@
return cmd.hasOption("v");
}
+ @SuppressWarnings("JavadocMethod")
@VisibleForTesting
public static Config loadConfig(CommandLine cmd, TopologyAPI.Topology topology) {
String cluster = cmd.getOptionValue("cluster");
@@ -293,8 +270,9 @@
return Config.toLocalMode(Config.newBuilder()
.putAll(ConfigLoader.loadConfig(heronHome, configPath, releaseFile, overrideConfigFile))
.putAll(commandLineConfigs(cluster, role, environ, submitUser, dryRun,
- dryRunFormat, isVerbose(cmd)))
- .putAll(topologyConfigs(topologyPackage, topologyBinaryFile, topologyDefnFile, topology))
+ dryRunFormat, isVerbose(cmd)))
+ .putAll(SubmitterUtils.topologyConfigs(topologyPackage, topologyBinaryFile,
+ topologyDefnFile, topology))
.build());
}
@@ -351,7 +329,7 @@
LOG.log(Level.FINE, "Sending out dry-run response");
// Output may contain UTF-8 characters, so we should print using UTF-8 encoding
PrintStream out = new PrintStream(System.out, true, StandardCharsets.UTF_8.name());
- out.print(submitterMain.renderDryRunResponse(response));
+ out.print(DryRunRenders.render(response, Context.dryRunFormatType(config)));
// Exit with status code 200 to indicate dry-run response is sent out
// SUPPRESS CHECKSTYLE RegexpSinglelineJava
System.exit(200);
@@ -570,15 +548,4 @@
LaunchRunner launchRunner = new LaunchRunner(config, runtime);
launchRunner.call();
}
-
- protected String renderDryRunResponse(SubmitDryRunResponse resp) {
- DryRunFormatType formatType = Context.dryRunFormatType(config);
- switch (formatType) {
- case RAW : return new SubmitRawDryRunRenderer(resp).render();
- case TABLE: return new SubmitTableDryRunRenderer(resp, false).render();
- case COLORED_TABLE: return new SubmitTableDryRunRenderer(resp, true).render();
- default: throw new IllegalArgumentException(
- String.format("Unexpected rendering format: %s", formatType));
- }
- }
}
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
new file mode 100644
index 0000000..d62fdfa
--- /dev/null
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
@@ -0,0 +1,54 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.utils;
+
+import com.twitter.heron.common.basics.DryRunFormatType;
+import com.twitter.heron.scheduler.dryrun.SubmitDryRunResponse;
+import com.twitter.heron.scheduler.dryrun.SubmitRawDryRunRenderer;
+import com.twitter.heron.scheduler.dryrun.SubmitTableDryRunRenderer;
+import com.twitter.heron.scheduler.dryrun.UpdateDryRunResponse;
+import com.twitter.heron.scheduler.dryrun.UpdateRawDryRunRenderer;
+import com.twitter.heron.scheduler.dryrun.UpdateTableDryRunRenderer;
+
+public final class DryRunRenders {
+
+ public static String render(SubmitDryRunResponse response, DryRunFormatType formatType) {
+ switch (formatType) {
+ case RAW :
+ return new SubmitRawDryRunRenderer(response).render();
+ case TABLE:
+ return new SubmitTableDryRunRenderer(response, false).render();
+ case COLORED_TABLE:
+ return new SubmitTableDryRunRenderer(response, true).render();
+ default: throw new IllegalArgumentException(
+ String.format("Unexpected rendering format: %s", formatType));
+ }
+ }
+
+ public static String render(UpdateDryRunResponse response, DryRunFormatType formatType) {
+ switch (formatType) {
+ case RAW :
+ return new UpdateRawDryRunRenderer(response).render();
+ case TABLE:
+ return new UpdateTableDryRunRenderer(response, false).render();
+ case COLORED_TABLE:
+ return new UpdateTableDryRunRenderer(response, true).render();
+ default: throw new IllegalArgumentException(
+ String.format("Unexpected rendering format: %s", formatType));
+ }
+ }
+
+ private DryRunRenders() {
+ }
+}
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SubmitterUtils.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SubmitterUtils.java
new file mode 100644
index 0000000..0e17f62
--- /dev/null
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SubmitterUtils.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.utils;
+
+import com.twitter.heron.api.generated.TopologyAPI;
+import com.twitter.heron.common.basics.PackageType;
+import com.twitter.heron.spi.common.Config;
+import com.twitter.heron.spi.common.Key;
+
+public final class SubmitterUtils {
+
+ /**
+ * Create the topology config
+ *
+ * @param topologyPackagePath path to the tar ball containing user submitted jar/tar, defn and config
+ * @param topologyBinaryFile name of the user submitted topology jar/tar/pex file
+ * @param topologyDefinitionPath path to the topology definition file
+ * @param topology proto in memory version of topology definition
+ * @return config the topology config
+ */
+ public static Config topologyConfigs(String topologyPackagePath, String topologyBinaryFile,
+ String topologyDefinitionPath, TopologyAPI.Topology topology) {
+ PackageType packageType = PackageType.getPackageType(topologyBinaryFile);
+
+ return Config.newBuilder()
+ .put(Key.TOPOLOGY_ID, topology.getId())
+ .put(Key.TOPOLOGY_NAME, topology.getName())
+ .put(Key.TOPOLOGY_DEFINITION_FILE, topologyDefinitionPath)
+ .put(Key.TOPOLOGY_PACKAGE_FILE, topologyPackagePath)
+ .put(Key.TOPOLOGY_BINARY_FILE, topologyBinaryFile)
+ .put(Key.TOPOLOGY_PACKAGE_TYPE, packageType)
+ .build();
+ }
+
+ private SubmitterUtils() {
+ }
+}
diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java
index 28f0d8d..5a82937 100644
--- a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java
+++ b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java
@@ -297,6 +297,10 @@
return cfgMap.keySet();
}
+ public Set<Map.Entry<String, Object>> getEntrySet() {
+ return cfgMap.entrySet();
+ }
+
@Override
public String toString() {
Map<String, Object> treeMap = new TreeMap<>(cfgMap);
diff --git a/heron/tools/apiserver/src/java/BUILD b/heron/tools/apiserver/src/java/BUILD
new file mode 100644
index 0000000..f193d31
--- /dev/null
+++ b/heron/tools/apiserver/src/java/BUILD
@@ -0,0 +1,63 @@
+package(default_visibility = ["//visibility:public"])
+
+load("/tools/rules/heron_deps", "heron_java_proto_files")
+
+api_deps_files = [
+ "//heron/spi/src/java:heron-spi",
+ "//heron/common/src/java:basics-java",
+]
+
+scheduler_deps_files = [
+ "//heron/scheduler-core/src/java:scheduler-java",
+ "//heron/schedulers/src/java:local-scheduler-java",
+ "//heron/schedulers/src/java:kubernetes-scheduler-java",
+ "//heron/schedulers/src/java:marathon-scheduler-java",
+ "//heron/schedulers/src/java:mesos-scheduler-java",
+ "//heron/schedulers/src/java:yarn-scheduler-java",
+ "//heron/schedulers/src/java:slurm-scheduler-java",
+]
+
+packing_deps_files = [
+ "//heron/packing/src/java:roundrobin-packing",
+ "//heron/packing/src/java:binpacking-packing",
+ "//heron/packing/src/java:builder"
+]
+
+uploader_deps_files = [
+ "//heron/uploaders/src/java:localfs-uploader-java",
+ "//heron/uploaders/src/java:gcs-uploader-java",
+ "//heron/uploaders/src/java:s3-uploader-java",
+]
+
+state_manager_deps_files = [
+ "//heron/statemgrs/src/java:statemgrs-java",
+]
+
+apiserver_deps_files = \
+ api_deps_files + \
+ heron_java_proto_files() + \
+ state_manager_deps_files + \
+ scheduler_deps_files + \
+ packing_deps_files + \
+ uploader_deps_files + [
+ "//third_party/java:cli",
+ "@org_yaml_snakeyaml//jar",
+ "//third_party/java:jetty-jersey-java",
+ "//third_party/java:commons-compress",
+ "//third_party/java:jackson",
+ "//third_party/java:logging",
+ ]
+
+java_binary(
+ name = 'heron-apiserver-unshaded',
+ srcs = glob(["**/apiserver/**/*.java"]),
+ main_class = "com.twitter.heron.apiserver.Runtime",
+ deps = apiserver_deps_files,
+)
+
+genrule(
+ name = "heron-apiserver",
+ srcs = [":heron-apiserver-unshaded_deploy.jar"],
+ outs = ["heron-apiserver.jar"],
+ cmd = "cp $< $@",
+)
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Constants.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Constants.java
new file mode 100644
index 0000000..1a09a9b
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Constants.java
@@ -0,0 +1,41 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver;
+
+import com.twitter.heron.spi.common.Key;
+
+public final class Constants {
+
+ static final int DEFAULT_PORT = 9000;
+
+ static final String DEFAULT_HERON_LOCAL = "~/.heron";
+
+ static final String DEFAULT_HERON_CLUSTER = "$HERON_HOME";
+
+ static final String DEFAULT_HERON_CONFIG_DIRECTORY = "conf";
+
+ static final String DEFAULT_HERON_RELEASE_FILE = "release.yaml";
+
+ public static final String DEFAULT_HERON_SANDBOX_CONFIG =
+ Key.HERON_CLUSTER_CONF.getDefaultString();
+
+ public static final String OVERRIDE_FILE = "override.yaml";
+
+ public static final String STATE_MANAGER_FILE = "statemgr.yaml";
+
+ public static final String DEFAULT_HERON_ENVIRONMENT = "default";
+
+ private Constants() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Resources.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Resources.java
new file mode 100644
index 0000000..6790006
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Resources.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+
+import com.twitter.heron.apiserver.resources.ConfigurationResource;
+import com.twitter.heron.apiserver.resources.NotFoundExceptionHandler;
+import com.twitter.heron.apiserver.resources.TopologyResource;
+
+public final class Resources {
+
+ private Resources() {
+ }
+
+ static Set<Class<?>> get() {
+ return new HashSet<>(getClasses());
+ }
+
+ private static List<Class<?>> getClasses() {
+ return Arrays.asList(
+ ConfigurationResource.class,
+ TopologyResource.class,
+ NotFoundExceptionHandler.class,
+ MultiPartFeature.class
+ );
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java
new file mode 100644
index 0000000..cb42006
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java
@@ -0,0 +1,289 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.twitter.heron.apiserver.resources.HeronResource;
+import com.twitter.heron.apiserver.utils.ConfigUtils;
+import com.twitter.heron.apiserver.utils.Logging;
+import com.twitter.heron.spi.common.Config;
+
+public final class Runtime {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Runtime.class);
+
+ private static final String API_BASE_PATH = "/api/v1/*";
+
+ private enum Flag {
+ Help("h"),
+ BaseTemplate("base-template"),
+ Cluster("cluster"),
+ ConfigPath("config-path"),
+ Port("port"),
+ Property("D"),
+ ReleaseFile("release-file"),
+ Verbose("verbose");
+
+ final String name;
+
+ Flag(String name) {
+ this.name = name;
+ }
+ }
+
+ private static Options createOptions() {
+ final Option cluster = Option.builder()
+ .desc("Cluster in which to deploy topologies")
+ .longOpt(Flag.Cluster.name)
+ .hasArg()
+ .argName(Flag.Cluster.name)
+ .required()
+ .build();
+
+ final Option baseTemplate = Option.builder()
+ .desc("Base configuration to use for deloying topologies")
+ .longOpt(Flag.BaseTemplate.name)
+ .hasArg()
+ .argName(Flag.BaseTemplate.name)
+ .required(false)
+ .build();
+
+ final Option config = Option.builder()
+ .desc("Path to the base configuration for deploying topologies")
+ .longOpt(Flag.ConfigPath.name)
+ .hasArg()
+ .argName(Flag.ConfigPath.name)
+ .required(false)
+ .build();
+
+ final Option port = Option.builder()
+ .desc("Port to bind to")
+ .longOpt(Flag.Port.name)
+ .hasArg()
+ .argName(Flag.Port.name)
+ .required(false)
+ .build();
+
+ final Option property = Option.builder(Flag.Property.name)
+ .argName("property=value")
+ .numberOfArgs(2)
+ .valueSeparator()
+ .desc("use value for given property")
+ .build();
+
+ final Option release = Option.builder()
+ .desc("Path to the release file")
+ .longOpt(Flag.ReleaseFile.name)
+ .hasArg()
+ .argName(Flag.ReleaseFile.name)
+ .required(false)
+ .build();
+
+ final Option verbose = Option.builder()
+ .desc("Verbose mode. Increases logging level to show debug messages.")
+ .longOpt(Flag.Verbose.name)
+ .hasArg(false)
+ .argName(Flag.Verbose.name)
+ .required(false)
+ .build();
+
+ return new Options()
+ .addOption(baseTemplate)
+ .addOption(cluster)
+ .addOption(config)
+ .addOption(port)
+ .addOption(release)
+ .addOption(property)
+ .addOption(verbose);
+ }
+
+ private static Options constructHelpOptions() {
+ Option help = Option.builder(Flag.Help.name)
+ .desc("List all options and their description")
+ .longOpt("help")
+ .hasArg(false)
+ .required(false)
+ .build();
+
+ return new Options()
+ .addOption(help);
+ }
+
+ // Print usage options
+ private static void usage(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("heron-apiserver", options);
+ }
+
+ private static String getConfigurationDirectory(String toolsHome, CommandLine cmd) {
+ if (cmd.hasOption(Flag.ConfigPath.name)) {
+ return cmd.getOptionValue(Flag.ConfigPath.name);
+ } else if (cmd.hasOption(Flag.BaseTemplate.name)) {
+ return Paths.get(toolsHome, Constants.DEFAULT_HERON_CONFIG_DIRECTORY,
+ cmd.getOptionValue(Flag.BaseTemplate.name)).toFile().getAbsolutePath();
+ }
+ return Paths.get(toolsHome, Constants.DEFAULT_HERON_CONFIG_DIRECTORY,
+ cmd.getOptionValue(Flag.Cluster.name)).toFile().getAbsolutePath();
+ }
+
+ private static String getHeronDirectory(CommandLine cmd) {
+ final String cluster = cmd.getOptionValue(Flag.Cluster.name);
+ return "local".equalsIgnoreCase(cluster)
+ ? Constants.DEFAULT_HERON_LOCAL : Constants.DEFAULT_HERON_CLUSTER;
+ }
+
+ private static String getReleaseFile(String toolsHome, CommandLine cmd) {
+ if (cmd.hasOption(Flag.ReleaseFile.name)) {
+ return cmd.getOptionValue(Flag.ReleaseFile.name);
+ }
+ return Paths.get(toolsHome, Constants.DEFAULT_HERON_RELEASE_FILE)
+ .toFile().getAbsolutePath();
+ }
+
+ private static int getPort(CommandLine cmd) {
+ if (cmd.hasOption(Flag.Port.name)) {
+ return Integer.valueOf(cmd.getOptionValue(Flag.Port.name));
+ }
+
+ return Constants.DEFAULT_PORT;
+ }
+
+ private static String loadOverrides(CommandLine cmd) throws IOException {
+ return ConfigUtils.createOverrideConfiguration(
+ cmd.getOptionProperties(Flag.Property.name));
+ }
+
+ private static String getToolsHome() throws URISyntaxException {
+ final String jarLocation =
+ Runtime.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
+ return Paths.get(jarLocation).getParent().getParent().toFile().getAbsolutePath();
+ }
+
+ private static Boolean isVerbose(CommandLine cmd) {
+ return cmd.hasOption(Flag.Verbose.name) ? Boolean.TRUE : Boolean.FALSE;
+ }
+
+ @SuppressWarnings({"IllegalCatch", "RegexpSinglelineJava"})
+ public static void main(String[] args) throws Exception {
+ final Options options = createOptions();
+ final Options helpOptions = constructHelpOptions();
+
+ CommandLineParser parser = new DefaultParser();
+
+ // parse the help options first.
+ CommandLine cmd = parser.parse(helpOptions, args, true);
+ if (cmd.hasOption(Flag.Help.name)) {
+ usage(options);
+ return;
+ }
+
+ try {
+ cmd = parser.parse(options, args);
+ } catch (ParseException pe) {
+ System.err.println(pe.getMessage());
+ usage(options);
+ return;
+ }
+
+ LOG.debug("apiserver overrides:\n {}", cmd.getOptionProperties(Flag.Property.name));
+
+ final String toolsHome = getToolsHome();
+
+ // read command line flags
+ final String heronConfigurationDirectory = getConfigurationDirectory(toolsHome, cmd);
+ final String heronDirectory = getHeronDirectory(cmd);
+ final String releaseFile = getReleaseFile(toolsHome, cmd);
+ final String configurationOverrides = loadOverrides(cmd);
+ final int port = getPort(cmd);
+
+
+ final Config baseConfiguration =
+ ConfigUtils.getBaseConfiguration(heronDirectory,
+ heronConfigurationDirectory,
+ releaseFile,
+ configurationOverrides);
+
+ final ResourceConfig config = new ResourceConfig(Resources.get());
+ final Server server = new Server(port);
+
+ final ServletContextHandler contextHandler =
+ new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+ contextHandler.setContextPath("/");
+
+ LOG.info("using configuration path: {}", heronConfigurationDirectory);
+
+ contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION, baseConfiguration);
+ contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_DIRECTORY,
+ heronConfigurationDirectory);
+ contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH,
+ configurationOverrides);
+
+ server.setHandler(contextHandler);
+
+ // set logging level
+ Logging.setVerbose(isVerbose(cmd));
+
+ final ServletHolder apiServlet =
+ new ServletHolder(new ServletContainer(config));
+
+ contextHandler.addServlet(apiServlet, API_BASE_PATH);
+
+ try {
+ server.start();
+
+ LOG.info("Heron apiserver started at {}", server.getURI());
+
+ server.join();
+ } catch (Exception ex) {
+ final String message = getErrorMessage(server, port, ex);
+ LOG.error(message);
+ System.err.println(message);
+ System.exit(1);
+ } finally {
+ server.destroy();
+ }
+ }
+
+ private static String getErrorMessage(Server server, int port, Exception ex) {
+ if (ex instanceof BindException) {
+ final URI uri = server.getURI();
+ return String.format("%s http://%s:%d", ex.getMessage(), uri.getHost(), port);
+ }
+
+ return ex.getMessage();
+ }
+
+ private Runtime() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Action.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Action.java
new file mode 100644
index 0000000..1ea520b
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Action.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+public interface Action {
+ void execute();
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactory.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactory.java
new file mode 100644
index 0000000..b795511
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactory.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.spi.common.Config;
+
+public interface ActionFactory {
+
+ Action createSubmitAction(Config config, String topologyPackagePath,
+ String topologyBinaryFileName, String topologyDefinitionPath);
+
+ Action createRuntimeAction(Config config, ActionType type);
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactoryImpl.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactoryImpl.java
new file mode 100644
index 0000000..67e807c
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionFactoryImpl.java
@@ -0,0 +1,43 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.api.generated.TopologyAPI;
+import com.twitter.heron.apiserver.utils.ConfigUtils;
+import com.twitter.heron.spi.common.Config;
+import com.twitter.heron.spi.utils.TopologyUtils;
+
+public class ActionFactoryImpl implements ActionFactory {
+
+ @Override
+ public Action createSubmitAction(Config config, String topologyPackagePath,
+ String topologyBinaryFileName, String topologyDefinitionPath) {
+ final TopologyAPI.Topology topology = TopologyUtils.getTopology(topologyDefinitionPath);
+ final Config topologyConfig =
+ ConfigUtils.getTopologyConfig(topologyPackagePath, topologyBinaryFileName,
+ topologyDefinitionPath, topology);
+ final Config configWithTopology =
+ Config.newBuilder()
+ .putAll(config)
+ .putAll(topologyConfig)
+ .build();
+
+ return new SubmitTopologyAction(configWithTopology, topology);
+ }
+
+ @Override
+ public Action createRuntimeAction(Config config, ActionType type) {
+ return new TopologyRuntimeAction(config, type.command);
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionType.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionType.java
new file mode 100644
index 0000000..e2db49b
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/ActionType.java
@@ -0,0 +1,30 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.scheduler.Command;
+
+public enum ActionType {
+ KILL(Command.KILL),
+ ACTIVATE(Command.ACTIVATE),
+ DEACTIVATE(Command.DEACTIVATE),
+ RESTART(Command.RESTART),
+ UPDATE(Command.UPDATE);
+
+ public final Command command;
+
+ ActionType(Command command) {
+ this.command = command;
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Keys.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Keys.java
new file mode 100644
index 0000000..f38fd34
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/Keys.java
@@ -0,0 +1,25 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.scheduler.RuntimeManagerRunner;
+
+public final class Keys {
+
+ public static final String NEW_COMPONENT_PARALLELISM_KEY =
+ RuntimeManagerRunner.NEW_COMPONENT_PARALLELISM_KEY;
+
+ private Keys() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/SubmitTopologyAction.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/SubmitTopologyAction.java
new file mode 100644
index 0000000..d7b78a5
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/SubmitTopologyAction.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.api.generated.TopologyAPI;
+import com.twitter.heron.scheduler.SubmitterMain;
+import com.twitter.heron.spi.common.Config;
+
+public class SubmitTopologyAction implements Action {
+
+ private final Config configuration;
+ private final TopologyAPI.Topology topology;
+
+ SubmitTopologyAction(Config configuration, TopologyAPI.Topology topology) {
+ this.configuration = configuration;
+ this.topology = topology;
+ }
+
+ @Override
+ public void execute() {
+ final SubmitterMain submitter = new SubmitterMain(configuration, topology);
+ submitter.submitTopology();
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/TopologyRuntimeAction.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/TopologyRuntimeAction.java
new file mode 100644
index 0000000..9c554a4
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/actions/TopologyRuntimeAction.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.actions;
+
+import com.twitter.heron.scheduler.Command;
+import com.twitter.heron.scheduler.RuntimeManagerMain;
+import com.twitter.heron.spi.common.Config;
+
+public class TopologyRuntimeAction implements Action {
+
+ private final Config config;
+ private final Command command;
+
+ TopologyRuntimeAction(Config config, Command command) {
+ this.config = config;
+ this.command = command;
+ }
+
+ @Override
+ public void execute() {
+ final RuntimeManagerMain runtimeManagerMain = new RuntimeManagerMain(config, command);
+ runtimeManagerMain.manageTopology();
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/ConfigurationResource.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/ConfigurationResource.java
new file mode 100644
index 0000000..701f3c2
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/ConfigurationResource.java
@@ -0,0 +1,57 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.resources;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import com.twitter.heron.spi.common.Config;
+
+@Path("/")
+public class ConfigurationResource extends HeronResource {
+
+ @Path("version")
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response release() throws JsonProcessingException {
+ final Config configuration = getBaseConfiguration();
+ final ObjectMapper mapper = new ObjectMapper();
+ final ObjectNode node = mapper.createObjectNode();
+ final Set<Map.Entry<String, Object>> sortedConfig = new TreeSet<>(Map.Entry.comparingByKey());
+ sortedConfig.addAll(configuration.getEntrySet());
+ for (Map.Entry<String, Object> entry : sortedConfig) {
+ if (entry.getKey().contains("heron.build")) {
+ node.put(entry.getKey(), entry.getValue().toString());
+ }
+ }
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(mapper
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(node))
+ .build();
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/Forms.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/Forms.java
new file mode 100644
index 0000000..ef57c71
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/Forms.java
@@ -0,0 +1,58 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.resources;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+
+import com.twitter.heron.apiserver.utils.FileHelper;
+
+final class Forms {
+
+ static String getString(FormDataMultiPart form, String key) {
+ return getValueAs(form, key, String.class);
+ }
+
+ static String getString(FormDataMultiPart form, String key, String defaultValue) {
+ return form.getFields().containsKey(key) ? getValueAs(form, key, String.class) : defaultValue;
+ }
+
+ static String getFirstOrDefault(MultivaluedMap<String, String> params, String key,
+ String defaultValue) {
+ return params.containsKey(key) ? params.getFirst(key) : defaultValue;
+ }
+
+ static File uploadFile(FormDataBodyPart part, String directory) throws IOException {
+ try (InputStream in = part.getValueAs(InputStream.class)) {
+ Path path = Paths.get(directory, part.getFormDataContentDisposition().getFileName());
+ FileHelper.copy(in, path);
+ return path.toFile();
+ }
+ }
+
+ private static <T> T getValueAs(FormDataMultiPart form, String key, Class<T> type) {
+ return form.getField(key).getValueAs(type);
+ }
+
+ private Forms() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java
new file mode 100644
index 0000000..313b120
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java
@@ -0,0 +1,57 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.resources;
+
+import javax.servlet.ServletContext;
+import javax.ws.rs.core.Context;
+
+import com.twitter.heron.spi.common.Config;
+
+public class HeronResource {
+
+ public static final String ATTRIBUTE_CONFIGURATION = "configuration";
+ public static final String ATTRIBUTE_CONFIGURATION_DIRECTORY = "configuration_directory";
+ public static final String ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH = "configuration_override";
+
+ @Context
+ protected ServletContext servletContext;
+
+ private Config baseConfiguration;
+ private String configurationDirectory;
+ private String configurationOverridePath;
+
+ Config getBaseConfiguration() {
+ if (baseConfiguration == null) {
+ baseConfiguration = (Config) servletContext.getAttribute(ATTRIBUTE_CONFIGURATION);
+ }
+ return baseConfiguration;
+ }
+
+ String getConfigurationDirectory() {
+ if (configurationDirectory == null) {
+ configurationDirectory =
+ (String) servletContext.getAttribute(ATTRIBUTE_CONFIGURATION_DIRECTORY);
+ }
+ return configurationDirectory;
+ }
+
+ String getConfigurationOverridePath() {
+ if (configurationOverridePath == null) {
+ configurationOverridePath =
+ (String) servletContext.getAttribute(ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH);
+ }
+
+ return configurationOverridePath;
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java
new file mode 100644
index 0000000..63eb4d8
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.resources;
+
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+@Provider
+public class NotFoundExceptionHandler implements ExceptionMapper<NotFoundException> {
+ @Override
+ public Response toResponse(NotFoundException exception) {
+ final ObjectMapper mapper = new ObjectMapper()
+ .enable(SerializationFeature.INDENT_OUTPUT)
+ .enable(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED);
+ final ObjectNode node = mapper.createObjectNode();
+ final ArrayNode arrayNode = node.putArray("paths");
+ arrayNode.add("/api/v1/topologies");
+ arrayNode.add("/api/v1/version");
+
+ final String response;
+ try {
+ response = mapper.writeValueAsString(node);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException("Error creating page not found response", e);
+ }
+
+ return Response.status(Response.Status.NOT_FOUND)
+ .entity(response)
+ .build();
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/TopologyResource.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/TopologyResource.java
new file mode 100644
index 0000000..41ee7cc
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/TopologyResource.java
@@ -0,0 +1,518 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.resources;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.twitter.heron.apiserver.Constants;
+import com.twitter.heron.apiserver.actions.ActionFactory;
+import com.twitter.heron.apiserver.actions.ActionFactoryImpl;
+import com.twitter.heron.apiserver.actions.ActionType;
+import com.twitter.heron.apiserver.actions.Keys;
+import com.twitter.heron.apiserver.utils.ConfigUtils;
+import com.twitter.heron.apiserver.utils.FileHelper;
+import com.twitter.heron.apiserver.utils.Logging;
+import com.twitter.heron.common.basics.DryRunFormatType;
+import com.twitter.heron.common.basics.FileUtils;
+import com.twitter.heron.common.basics.Pair;
+import com.twitter.heron.scheduler.dryrun.DryRunResponse;
+import com.twitter.heron.scheduler.dryrun.SubmitDryRunResponse;
+import com.twitter.heron.scheduler.dryrun.UpdateDryRunResponse;
+import com.twitter.heron.scheduler.utils.DryRunRenders;
+import com.twitter.heron.spi.common.Config;
+import com.twitter.heron.spi.common.Key;
+
+@Path("/topologies")
+public class TopologyResource extends HeronResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TopologyResource.class);
+
+ private static final String TOPOLOGY_TAR_GZ_FILENAME = "topology.tar.gz";
+
+ private static final int HTTP_UNPROCESSABLE_ENTITY_CODE = 422;
+
+ private static final String FORM_KEY_NAME = "name";
+ private static final String FORM_KEY_CLUSTER = "cluster";
+ private static final String FORM_KEY_ROLE = "role";
+ private static final String FORM_KEY_ENVIRONMENT = "environment";
+ private static final String FORM_KEY_DEFINITION = "definition";
+ private static final String FORM_KEY_TOPOLOGY = "topology";
+ private static final String FORM_KEY_USER = "user";
+
+ private static final Set<String> SUBMIT_TOPOLOGY_PARAMS = Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ FORM_KEY_NAME,
+ FORM_KEY_CLUSTER,
+ FORM_KEY_ROLE,
+ FORM_KEY_ENVIRONMENT,
+ FORM_KEY_DEFINITION,
+ FORM_KEY_TOPOLOGY,
+ FORM_KEY_USER
+ )
+ )
+ );
+
+ private static final String[] REQUIRED_SUBMIT_TOPOLOGY_PARAMS = {
+ FORM_KEY_NAME,
+ FORM_KEY_CLUSTER,
+ FORM_KEY_ROLE,
+ FORM_KEY_DEFINITION,
+ FORM_KEY_TOPOLOGY
+ };
+
+ private static final String PARAM_COMPONENT_PARALLELISM = "component_parallelism";
+ private static final String PARAM_DRY_RUN = "dry_run";
+ private static final String PARAM_DRY_RUN_FORMAT = "dry_run_format";
+ private static final String DEFAULT_DRY_RUN_FORMAT = DryRunFormatType.TABLE.toString();
+
+ // path format /topologies/{cluster}/{role}/{environment}/{name}
+ private static final String TOPOLOGY_PATH_FORMAT = "/topologies/%s/%s/%s/%s";
+
+ private final ActionFactory actionFactory = new ActionFactoryImpl();
+
+ @POST
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings({"IllegalCatch", "JavadocMethod"})
+ public Response submit(FormDataMultiPart form) throws IOException {
+ // verify that all we have all the required params
+ final List<String> missingDataKeys =
+ verifyKeys(form.getFields().keySet(), REQUIRED_SUBMIT_TOPOLOGY_PARAMS);
+ if (!missingDataKeys.isEmpty()) {
+ // return error since we are missing required parameters
+ final String message = String.format("Validation failed missing required params: %s",
+ missingDataKeys.toString());
+ return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE)
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createValidationError(message, missingDataKeys))
+ .build();
+ }
+
+ final String topologyName = Forms.getString(form, FORM_KEY_NAME);
+ final String cluster = Forms.getString(form, FORM_KEY_CLUSTER);
+ final String role = Forms.getString(form, FORM_KEY_ROLE);
+ final String environment =
+ Forms.getString(form, FORM_KEY_ENVIRONMENT, Constants.DEFAULT_HERON_ENVIRONMENT);
+ final String user = Forms.getString(form, FORM_KEY_USER, role);
+
+ // submit overrides are passed key=value
+ final Map<String, String> submitOverrides = getSubmitOverrides(form);
+
+
+ final String topologyDirectory =
+ Files.createTempDirectory(topologyName).toFile().getAbsolutePath();
+
+ try {
+ // upload the topology definition file to the topology directory
+ final FormDataBodyPart definitionFilePart = form.getField(FORM_KEY_DEFINITION);
+ final File topologyDefinitionFile = Forms.uploadFile(definitionFilePart, topologyDirectory);
+
+ // upload the topology binary file to the topology directory
+ final FormDataBodyPart topologyFilePart = form.getField(FORM_KEY_TOPOLOGY);
+ final File topologyBinaryFile = Forms.uploadFile(topologyFilePart, topologyDirectory);
+
+ final boolean isDryRun = form.getFields().containsKey(PARAM_DRY_RUN);
+ final Config config = configWithKeyValues(
+ Arrays.asList(
+ Pair.create(Key.CLUSTER.value(), cluster),
+ Pair.create(Key.TOPOLOGY_NAME.value(), topologyName),
+ Pair.create(Key.ROLE.value(), role),
+ Pair.create(Key.ENVIRON.value(), environment),
+ Pair.create(Key.SUBMIT_USER.value(), user),
+ Pair.create(Key.DRY_RUN.value(), isDryRun)
+ )
+ );
+
+ // copy configuration files to the sandbox config location
+ // topology-dir/<default-heron-sandbox-config>
+ FileHelper.copyDirectory(
+ Paths.get(getConfigurationDirectory()),
+ Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG));
+
+
+ final java.nio.file.Path overridesPath =
+ Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG,
+ Constants.OVERRIDE_FILE);
+ // copy override file into topology configuration directory
+ FileHelper.copy(Paths.get(getConfigurationOverridePath()), overridesPath);
+
+ // apply submit overrides
+ ConfigUtils.applyOverrides(overridesPath, submitOverrides);
+
+ // apply overrides to state manager config
+ ConfigUtils.applyOverridesToStateManagerConfig(overridesPath,
+ Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG,
+ Constants.STATE_MANAGER_FILE)
+ );
+
+ // create tar file from the contents of the topology directory
+ final File topologyPackageFile =
+ Paths.get(topologyDirectory, TOPOLOGY_TAR_GZ_FILENAME).toFile();
+ FileHelper.createTarGz(topologyPackageFile, FileHelper.getChildren(topologyDirectory));
+
+ // submit the topology
+ getActionFactory()
+ .createSubmitAction(config,
+ topologyPackageFile.getAbsolutePath(),
+ topologyBinaryFile.getName(),
+ topologyDefinitionFile.getAbsolutePath())
+ .execute();
+
+ return Response.created(
+ URI.create(String.format(TOPOLOGY_PATH_FORMAT,
+ cluster, role, environment, topologyName)))
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createdResponse(cluster, role, environment, topologyName)).build();
+ } catch (SubmitDryRunResponse response) {
+ return createDryRunResponse(response,
+ Forms.getString(form, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
+ } catch (Exception ex) {
+ return Response.serverError()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ } finally {
+ FileUtils.deleteDir(topologyDirectory);
+ }
+ }
+
+ @POST
+ @Path("/{cluster}/{role}/{environment}/{name}/activate")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("IllegalCatch")
+ public Response activate(
+ final @PathParam("cluster") String cluster,
+ final @PathParam("role") String role,
+ final @PathParam("environment") String environment,
+ final @PathParam("name") String name) {
+ try {
+ final Config config = getConfig(cluster, role, environment, name);
+ getActionFactory().createRuntimeAction(config, ActionType.ACTIVATE).execute();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(String.format("%s activated", name)))
+ .build();
+ } catch (Exception ex) {
+ return Response.serverError()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ }
+ }
+
+ @POST
+ @Path("/{cluster}/{role}/{environment}/{name}/deactivate")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("IllegalCatch")
+ public Response deactivate(
+ final @PathParam("cluster") String cluster,
+ final @PathParam("role") String role,
+ final @PathParam("environment") String environment,
+ final @PathParam("name") String name) {
+ try {
+ final Config config = getConfig(cluster, role, environment, name);
+ getActionFactory().createRuntimeAction(config, ActionType.DEACTIVATE).execute();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(String.format("%s deactivated", name)))
+ .build();
+ } catch (Exception ex) {
+ return Response.serverError()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ }
+ }
+
+ @POST
+ @Path("/{cluster}/{role}/{environment}/{name}/restart")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("IllegalCatch")
+ public Response restart(
+ final @PathParam("cluster") String cluster,
+ final @PathParam("role") String role,
+ final @PathParam("environment") String environment,
+ final @PathParam("name") String name,
+ final @DefaultValue("-1") @FormParam("container_id") int containerId) {
+ try {
+ final List<Pair<String, Object>> keyValues = new ArrayList<>(
+ Arrays.asList(
+ Pair.create(Key.CLUSTER.value(), cluster),
+ Pair.create(Key.ROLE.value(), role),
+ Pair.create(Key.ENVIRON.value(), environment),
+ Pair.create(Key.TOPOLOGY_NAME.value(), name),
+ Pair.create(Key.TOPOLOGY_CONTAINER_ID.value(), containerId)
+ )
+ );
+
+ final Config config = configWithKeyValues(keyValues);
+ getActionFactory().createRuntimeAction(config, ActionType.RESTART).execute();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(String.format("%s restarted", name)))
+ .build();
+ } catch (Exception ex) {
+ return Response.serverError()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ }
+ }
+
+ @POST
+ @Path("/{cluster}/{role}/{environment}/{name}/update")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings({"IllegalCatch", "JavadocMethod"})
+ public Response update(
+ final @PathParam("cluster") String cluster,
+ final @PathParam("role") String role,
+ final @PathParam("environment") String environment,
+ final @PathParam("name") String name,
+ MultivaluedMap<String, String> params) {
+ try {
+ if (params == null || !params.containsKey(PARAM_COMPONENT_PARALLELISM)) {
+ return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE)
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage("missing component_parallelism param"))
+ .build();
+ }
+
+ List<String> components = params.get(PARAM_COMPONENT_PARALLELISM);
+ final List<Pair<String, Object>> keyValues = new ArrayList<>(
+ Arrays.asList(
+ Pair.create(Key.CLUSTER.value(), cluster),
+ Pair.create(Key.ROLE.value(), role),
+ Pair.create(Key.ENVIRON.value(), environment),
+ Pair.create(Key.TOPOLOGY_NAME.value(), name),
+ Pair.create(Keys.NEW_COMPONENT_PARALLELISM_KEY,
+ String.join(",", components))
+ )
+ );
+
+ // has a dry run been requested?
+ if (params.containsKey(PARAM_DRY_RUN)) {
+ keyValues.add(Pair.create(Key.DRY_RUN.value(), Boolean.TRUE));
+ }
+
+ final Set<Pair<String, Object>> overrides = getUpdateOverrides(params);
+ // apply overrides if they exists
+ if (!overrides.isEmpty()) {
+ keyValues.addAll(overrides);
+ }
+
+ final Config config = configWithKeyValues(keyValues);
+ getActionFactory().createRuntimeAction(config, ActionType.UPDATE).execute();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(String.format("%s updated", name)))
+ .build();
+ } catch (UpdateDryRunResponse response) {
+ return createDryRunResponse(response,
+ Forms.getFirstOrDefault(params, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
+ } catch (Exception ex) {
+ LOG.error("error updating topology {}", name, ex);
+ return Response.serverError()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ }
+ }
+
+ @DELETE
+ @Path("/{cluster}/{role}/{environment}/{name}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("IllegalCatch")
+ public Response kill(
+ final @PathParam("cluster") String cluster,
+ final @PathParam("role") String role,
+ final @PathParam("environment") String environment,
+ final @PathParam("name") String name) {
+ try {
+ final Config config = getConfig(cluster, role, environment, name);
+ getActionFactory().createRuntimeAction(config, ActionType.KILL).execute();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(String.format("%s killed", name)))
+ .build();
+ } catch (Exception ex) {
+ final String message = ex.getMessage();
+ final Response.Status status = message.contains("does not exist")
+ ? Response.Status.NOT_FOUND : Response.Status.INTERNAL_SERVER_ERROR;
+ return Response.status(status)
+ .type(MediaType.APPLICATION_JSON)
+ .entity(createMessage(ex.getMessage()))
+ .build();
+ }
+ }
+
+ ActionFactory getActionFactory() {
+ return actionFactory;
+ }
+
+ private Config getConfig(String cluster, String role, String environment, String topologyName) {
+ return configWithKeyValues(
+ Arrays.asList(
+ Pair.create(Key.CLUSTER.value(), cluster),
+ Pair.create(Key.ROLE.value(), role),
+ Pair.create(Key.ENVIRON.value(), environment),
+ Pair.create(Key.TOPOLOGY_NAME.value(), topologyName)
+ ));
+ }
+
+ private Config configWithKeyValues(Collection<Pair<String, Object>> keyValues) {
+ final Config.Builder builder = Config.newBuilder().putAll(getBaseConfiguration());
+ for (Pair<String, Object> keyValue : keyValues) {
+ builder.put(keyValue.first, keyValue.second);
+ }
+ builder.put(Key.VERBOSE, Logging.isVerbose());
+ return Config.toLocalMode(builder.build());
+ }
+
+ private static List<String> verifyKeys(Set<String> keys, String... requiredKeys) {
+ final List<String> missingKeys = new ArrayList<>();
+ if (requiredKeys != null) {
+ for (String key : requiredKeys) {
+ if (!keys.contains(key)) {
+ missingKeys.add(key);
+ }
+ }
+ }
+ return missingKeys;
+ }
+
+ private static Map<String, String> getSubmitOverrides(FormDataMultiPart form) {
+ final Map<String, String> overrides = new HashMap<>();
+ for (String key : form.getFields().keySet()) {
+ if (!SUBMIT_TOPOLOGY_PARAMS.contains(key)) {
+ overrides.put(key, Forms.getString(form, key));
+ }
+ }
+ return overrides;
+ }
+
+ private static Set<Pair<String, Object>> getUpdateOverrides(
+ MultivaluedMap<String, String> params) {
+ final Set<Pair<String, Object>> overrides = new HashSet<>();
+ for (String key : params.keySet()) {
+ if (!PARAM_COMPONENT_PARALLELISM.equalsIgnoreCase(key)) {
+ overrides.add(Pair.create(key, params.getFirst(key)));
+ }
+ }
+ return overrides;
+ }
+
+ @SuppressWarnings("IllegalCatch")
+ private static DryRunFormatType getDryRunFormatType(String type) {
+ try {
+ if (type != null) {
+ return DryRunFormatType.valueOf(type);
+ }
+ } catch (Exception ex) {
+ LOG.warn("unknown dry format render type {} defaulting to table", type);
+ }
+ return DryRunFormatType.TABLE;
+ }
+
+ private static String getDryRunResponse(DryRunResponse response, String type) {
+ if (response instanceof SubmitDryRunResponse) {
+ return DryRunRenders.render((SubmitDryRunResponse) response,
+ getDryRunFormatType(type));
+ } else if (response instanceof UpdateDryRunResponse) {
+ return DryRunRenders.render((UpdateDryRunResponse) response,
+ getDryRunFormatType(type));
+ }
+ return "Unknown dry run response type " + response.getClass().getName();
+ }
+
+ private static Response createDryRunResponse(DryRunResponse response, String type) {
+ final String body = new ObjectMapper().createObjectNode()
+ .put("response", getDryRunResponse(response, type))
+ .toString();
+
+ return Response.ok()
+ .type(MediaType.APPLICATION_JSON)
+ .entity(body)
+ .build();
+ }
+
+ private static String createdResponse(String cluster, String role, String environment,
+ String topologyName) {
+ return new ObjectMapper().createObjectNode()
+ .put("name", topologyName)
+ .put("cluster", cluster)
+ .put("role", role)
+ .put("environment", environment)
+ .toString();
+ }
+
+ private static ObjectNode createBaseError(String message) {
+ final ObjectMapper mapper = new ObjectMapper();
+ return mapper.createObjectNode().put("message", message);
+ }
+
+ private static String createMessage(String message) {
+ return createBaseError(message).toString();
+ }
+
+ private static String createValidationError(String message, List<String> missing) {
+ ObjectNode node = createBaseError(message);
+ ObjectNode errors = node.putObject("errors");
+ ArrayNode missingParameters = errors.putArray("missing_parameters");
+ for (String param : missing) {
+ missingParameters.add(param);
+ }
+
+ return node.toString();
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java
new file mode 100644
index 0000000..caf3085
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java
@@ -0,0 +1,152 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.utils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import com.twitter.heron.api.generated.TopologyAPI;
+import com.twitter.heron.common.basics.SysUtils;
+import com.twitter.heron.scheduler.utils.SubmitterUtils;
+import com.twitter.heron.spi.common.Config;
+import com.twitter.heron.spi.common.ConfigLoader;
+
+public final class ConfigUtils {
+
+ private static final String CONFIG_SUFFIX = ".yaml";
+
+ public static Config.Builder builder(Config baseConfiguration) {
+ return Config.newBuilder().putAll(baseConfiguration);
+ }
+
+ public static String createOverrideConfiguration(Properties properties) throws IOException {
+ final Path overridesPath = Files.createTempFile("overrides-", CONFIG_SUFFIX);
+ try (Writer writer = Files.newBufferedWriter(overridesPath)) {
+ final Map<Object, Object> overrides = new HashMap<>();
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ overrides.put(entry.getKey(), entry.getValue());
+ }
+ final Yaml yaml = newYaml();
+ yaml.dump(overrides, writer);
+
+ return overridesPath.toFile().getAbsolutePath();
+ } finally {
+ overridesPath.toFile().deleteOnExit();
+ }
+ }
+
+ public static Config getBaseConfiguration(String heronDirectory,
+ String heronConfigDirectory,
+ String releaseFile,
+ String overrideConfigurationFile) {
+ // TODO add release file
+ return ConfigLoader.loadConfig(heronDirectory,
+ heronConfigDirectory,
+ releaseFile,
+ overrideConfigurationFile);
+ }
+
+ public static Config getTopologyConfig(String topologyPackage, String topologyBinaryFile,
+ String topologyDefinitionFile, TopologyAPI.Topology topology) {
+ return SubmitterUtils.topologyConfigs(
+ topologyPackage,
+ topologyBinaryFile,
+ topologyDefinitionFile,
+ topology);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void applyOverrides(Path overridesPath, Map<String, String> overrides)
+ throws IOException {
+ if (overrides.isEmpty()) {
+ return;
+ }
+ final Path tempOverridesPath = Files.createTempFile("overrides-", CONFIG_SUFFIX);
+
+ Reader overrideReader = null;
+ try (Writer writer = Files.newBufferedWriter(tempOverridesPath)) {
+ overrideReader = Files.newBufferedReader(overridesPath);
+ final Map<String, Object> currentOverrides =
+ (Map<String, Object>) new Yaml().load(overrideReader);
+ currentOverrides.putAll(overrides);
+
+ // write updated overrides
+ newYaml().dump(currentOverrides, writer);
+
+ // close override file so we can replace it with the updated overrides
+ overrideReader.close();
+
+ FileHelper.copy(tempOverridesPath, overridesPath);
+ } finally {
+ tempOverridesPath.toFile().delete();
+ SysUtils.closeIgnoringExceptions(overrideReader);
+ }
+ }
+
+ // this is needed because the heron executor ignores the override.yaml
+ @SuppressWarnings("unchecked")
+ public static void applyOverridesToStateManagerConfig(Path overridesPath,
+ Path stateManagerPath) throws IOException {
+ final Path tempStateManagerPath = Files.createTempFile("statemgr-", CONFIG_SUFFIX);
+ Reader stateManagerReader = null;
+ try (
+ Reader overrideReader = Files.newBufferedReader(overridesPath);
+ Writer writer = Files.newBufferedWriter(tempStateManagerPath);
+ ) {
+ stateManagerReader = Files.newBufferedReader(stateManagerPath);
+
+ final Map<String, Object> overrides = (Map<String, Object>) new Yaml().load(overrideReader);
+ final Map<String, Object> stateMangerConfig =
+ (Map<String, Object>) new Yaml().load(stateManagerReader);
+ // update the state manager config with the overrides
+ for (Map.Entry<String, Object> entry : overrides.entrySet()) {
+ // does this key have an override?
+ if (stateMangerConfig.containsKey(entry.getKey())) {
+ stateMangerConfig.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ // write new state manager config
+ newYaml().dump(stateMangerConfig, writer);
+
+ // close state manager file so we can replace it with the updated configuration
+ stateManagerReader.close();
+
+ FileHelper.copy(tempStateManagerPath, stateManagerPath);
+ } finally {
+ tempStateManagerPath.toFile().delete();
+ SysUtils.closeIgnoringExceptions(stateManagerReader);
+ }
+ }
+
+ private static Yaml newYaml() {
+ final DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+ options.setPrettyFlow(true);
+
+ return new Yaml(options);
+ }
+
+ private ConfigUtils() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/FileHelper.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/FileHelper.java
new file mode 100644
index 0000000..39a0294
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/FileHelper.java
@@ -0,0 +1,142 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.utils;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class FileHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FileHelper.class);
+
+ public static boolean copy(InputStream in, Path to) {
+ try {
+ Files.copy(in, to, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException ioe) {
+ LOG.error("Failed to copy file to {}", to, ioe);
+ return false;
+ }
+ return true;
+ }
+
+ public static boolean copy(Path from, Path to) {
+ try {
+ Files.copy(from, to, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException ioe) {
+ LOG.error("Failed to copy file from {} to {}", from, to, ioe);
+ return false;
+ }
+ return true;
+ }
+
+ public static File[] getChildren(String path) {
+ final File file = new File(path);
+ return file.isDirectory() ? file.listFiles() : new File[] {};
+ }
+
+ public static boolean copyDirectory(Path from, Path to) {
+ try {
+ Files.walkFileTree(from, new CopyDirectoryVisitor(from, to));
+ } catch (IOException ioe) {
+ LOG.error("Failed to copy directory from {} to {}", from, to, ioe);
+ return false;
+ }
+ return true;
+ }
+
+ public static boolean createTarGz(File archive, File... files) {
+ try (
+ FileOutputStream fileOutputStream = new FileOutputStream(archive);
+ BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
+ GzipCompressorOutputStream gzipOuputStream =
+ new GzipCompressorOutputStream(bufferedOutputStream);
+ TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(gzipOuputStream)
+ ) {
+ for (File file : files) {
+ addFileToArchive(archiveOutputStream, file, "");
+ }
+ archiveOutputStream.finish();
+ } catch (IOException ioe) {
+ LOG.error("Failed to create archive {} file.", archive, ioe);
+ return false;
+ }
+ return true;
+ }
+
+ private static void addFileToArchive(TarArchiveOutputStream archiveOutputStream, File file,
+ String base) throws IOException {
+ final File absoluteFile = file.getAbsoluteFile();
+ final String entryName = base + file.getName();
+ final TarArchiveEntry tarArchiveEntry = new TarArchiveEntry(file, entryName);
+ archiveOutputStream.putArchiveEntry(tarArchiveEntry);
+
+ if (absoluteFile.isFile()) {
+ Files.copy(file.toPath(), archiveOutputStream);
+ archiveOutputStream.closeArchiveEntry();
+ } else {
+ archiveOutputStream.closeArchiveEntry();
+ if (absoluteFile.listFiles() != null) {
+ for (File f : absoluteFile.listFiles()) {
+ addFileToArchive(archiveOutputStream, f, entryName + "/");
+ }
+ }
+ }
+ }
+
+ private static final class CopyDirectoryVisitor extends SimpleFileVisitor<Path> {
+ private final Path fromPath;
+ private final Path toPath;
+ private final StandardCopyOption copyOption = StandardCopyOption.REPLACE_EXISTING;
+
+ private CopyDirectoryVisitor(Path from, Path to) {
+ fromPath = from;
+ toPath = to;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
+ throws IOException {
+ Path targetPath = toPath.resolve(fromPath.relativize(dir));
+ if (!Files.exists(targetPath)) {
+ Files.createDirectory(targetPath);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ Files.copy(file, toPath.resolve(fromPath.relativize(file)), copyOption);
+ return FileVisitResult.CONTINUE;
+ }
+ }
+
+ private FileHelper() {
+ }
+}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/Logging.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/Logging.java
new file mode 100644
index 0000000..d71ac7d
--- /dev/null
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/Logging.java
@@ -0,0 +1,30 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.utils;
+
+public final class Logging {
+
+ private static volatile boolean isVerbose;
+
+ public static void setVerbose(boolean verbose) {
+ isVerbose = verbose;
+ }
+
+ public static boolean isVerbose() {
+ return isVerbose;
+ }
+
+ private Logging() {
+ }
+}
diff --git a/heron/tools/apiserver/src/shell/BUILD b/heron/tools/apiserver/src/shell/BUILD
new file mode 100644
index 0000000..d96eec8
--- /dev/null
+++ b/heron/tools/apiserver/src/shell/BUILD
@@ -0,0 +1,6 @@
+package(default_visibility = ["//visibility:public"])
+
+sh_binary(
+ name = "heron-apiserver",
+ srcs = [ "heron-apiserver.sh" ],
+)
diff --git a/heron/tools/apiserver/src/shell/heron-apiserver.sh b/heron/tools/apiserver/src/shell/heron-apiserver.sh
new file mode 100755
index 0000000..f217011
--- /dev/null
+++ b/heron/tools/apiserver/src/shell/heron-apiserver.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+BINDIR=$(dirname $(readlink $0))
+HERON_TOOLS_HOME=$(dirname ${BINDIR})
+HERON_APISERVER_JAR=${HERON_TOOLS_HOME}/lib/heron-apiserver.jar
+RELEASE_FILE=${HERON_TOOLS_HOME}/release.yaml
+
+# Check for the java to use
+if [[ -z $JAVA_HOME ]]; then
+ JAVA=$(which java)
+ if [ $? != 0 ]; then
+ echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2
+ exit 1
+ fi
+else
+ JAVA=${JAVA_HOME}/bin/java
+fi
+
+exec $JAVA -jar $HERON_APISERVER_JAR $@
diff --git a/heron/tools/apiserver/tests/java/BUILD b/heron/tools/apiserver/tests/java/BUILD
new file mode 100644
index 0000000..8017bbc
--- /dev/null
+++ b/heron/tools/apiserver/tests/java/BUILD
@@ -0,0 +1,29 @@
+load("/tools/rules/java_tests", "java_tests")
+
+load("/tools/rules/heron_deps", "heron_java_proto_files")
+
+common_deps_files = [
+ "//third_party/java:powermock",
+ "@commons_io_commons_io//jar",
+ "//third_party/java:mockito",
+ "//third_party/java:junit4",
+]
+
+apiserver_test_deps_files = \
+ common_deps_files + [
+ "//heron/tools/apiserver/src/java:heron-apiserver"
+ ]
+
+java_library(
+ name = "tests",
+ srcs = glob(["**/apiserver/**/*.java"]),
+ deps = apiserver_test_deps_files,
+)
+
+java_tests(
+ test_classes = [
+ "com.twitter.heron.apiserver.utils.ConfigUtilsTests",
+ ],
+ runtime_deps = [ ":tests" ],
+ size = "small",
+)
diff --git a/heron/tools/apiserver/tests/java/com/twitter/heron/apiserver/utils/ConfigUtilsTests.java b/heron/tools/apiserver/tests/java/com/twitter/heron/apiserver/utils/ConfigUtilsTests.java
new file mode 100644
index 0000000..d02f1c2
--- /dev/null
+++ b/heron/tools/apiserver/tests/java/com/twitter/heron/apiserver/utils/ConfigUtilsTests.java
@@ -0,0 +1,187 @@
+// Copyright 2017 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.apiserver.utils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.Test;
+
+import org.yaml.snakeyaml.Yaml;
+
+import com.twitter.heron.common.basics.Pair;
+
+import static org.junit.Assert.assertEquals;
+
+public class ConfigUtilsTests {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCreateOverrides() throws IOException {
+ final Properties overrideProperties = createOverrideProperties(
+ Pair.create("heron.statemgr.connection.string", "zookeeper:2181"),
+ Pair.create("heron.kubernetes.scheduler.uri", "http://localhost:8001")
+ );
+
+ final String overridesPath = ConfigUtils.createOverrideConfiguration(overrideProperties);
+ try (Reader reader = Files.newBufferedReader(Paths.get(overridesPath))) {
+ final Map<String, Object> overrides =
+ (Map<String, Object>) new Yaml().loadAs(reader, Map.class);
+ assertEquals(overrides.size(), overrideProperties.size());
+ for (String key : overrides.keySet()) {
+ assertEquals(overrides.get(key), overrideProperties.getProperty(key));
+ }
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStateManagerFileOverrides() throws IOException {
+ final Properties overrideProperties = createOverrideProperties(
+ Pair.create("heron.statemgr.connection.string", "zookeeper:2181"),
+ Pair.create("heron.kubernetes.scheduler.uri", "http://localhost:8001")
+ );
+
+ final String overridesPath = ConfigUtils.createOverrideConfiguration(overrideProperties);
+
+ // write default state manager config
+ final Path stateManagerPath = Files.createTempFile("statemgr-", ".yaml");
+ stateManagerPath.toFile().deleteOnExit();
+ try (Writer writer = Files.newBufferedWriter(stateManagerPath)) {
+ final Map<String, String> config = new HashMap<>();
+ config.put("heron.statemgr.connection.string", "<host>:<port>");
+ new Yaml().dump(config, writer);
+ }
+
+ // apply the overrides
+ ConfigUtils.applyOverridesToStateManagerConfig(Paths.get(overridesPath), stateManagerPath);
+
+ try (Reader reader = Files.newBufferedReader(stateManagerPath)) {
+ final Map<String, Object> stateManagerWithOverrides =
+ (Map<String, Object>) new Yaml().loadAs(reader, Map.class);
+ assertEquals(stateManagerWithOverrides.size(), 1);
+ assertEquals(stateManagerWithOverrides.get("heron.statemgr.connection.string"),
+ "zookeeper:2181");
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testNoOverridesAppliedToStateManager() throws IOException {
+ final Properties overrideProperties = createOverrideProperties(
+ Pair.create("heron.kubernetes.scheduler.uri", "http://localhost:8001")
+ );
+
+ final String overridesPath = ConfigUtils.createOverrideConfiguration(overrideProperties);
+
+ // write default state manager config
+ final Path stateManagerPath = Files.createTempFile("statemgr-", ".yaml");
+ stateManagerPath.toFile().deleteOnExit();
+ try (Writer writer = Files.newBufferedWriter(stateManagerPath)) {
+ final Map<String, String> config = new HashMap<>();
+ config.put("heron.statemgr.connection.string", "<host>:<port>");
+ new Yaml().dump(config, writer);
+ }
+
+ // apply the overrides
+ ConfigUtils.applyOverridesToStateManagerConfig(Paths.get(overridesPath), stateManagerPath);
+
+ try (Reader reader = Files.newBufferedReader(stateManagerPath)) {
+ final Map<String, Object> stateManagerWithOverrides =
+ (Map<String, Object>) new Yaml().loadAs(reader, Map.class);
+ assertEquals(stateManagerWithOverrides.size(), 1);
+ assertEquals(stateManagerWithOverrides.get("heron.statemgr.connection.string"),
+ "<host>:<port>");
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testApplyOverrides() throws IOException {
+ final Properties overrideProperties = createOverrideProperties(
+ Pair.create("heron.statemgr.connection.string", "zookeeper:2181"),
+ Pair.create("heron.kubernetes.scheduler.uri", "http://localhost:8001")
+ );
+
+ final String overridesPath = ConfigUtils.createOverrideConfiguration(overrideProperties);
+
+ final Map<String, String> overrides = createOverrides(
+ Pair.create("my.override.key", "my.override.value")
+ );
+
+ ConfigUtils.applyOverrides(Paths.get(overridesPath), overrides);
+
+ final Map<String, String> combinedOverrides = new HashMap<>();
+ combinedOverrides.putAll(overrides);
+ for (String key : overrideProperties.stringPropertyNames()) {
+ combinedOverrides.put(key, overrideProperties.getProperty(key));
+ }
+
+ try (Reader reader = Files.newBufferedReader(Paths.get(overridesPath))) {
+ final Map<String, Object> newOverrides =
+ (Map<String, Object>) new Yaml().loadAs(reader, Map.class);
+ assertEquals(newOverrides, combinedOverrides);
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testApplyEmptyOverrides() throws IOException {
+ final Properties overrideProperties = createOverrideProperties(
+ Pair.create("heron.statemgr.connection.string", "zookeeper:2181"),
+ Pair.create("heron.kubernetes.scheduler.uri", "http://localhost:8001")
+ );
+
+ final String overridesPath = ConfigUtils.createOverrideConfiguration(overrideProperties);
+
+ ConfigUtils.applyOverrides(Paths.get(overridesPath), new HashMap<>());
+
+ final Map<String, String> overrides = new HashMap<>();
+ for (String key : overrideProperties.stringPropertyNames()) {
+ overrides.put(key, overrideProperties.getProperty(key));
+ }
+
+ try (Reader reader = Files.newBufferedReader(Paths.get(overridesPath))) {
+ final Map<String, Object> newOverrides =
+ (Map<String, Object>) new Yaml().loadAs(reader, Map.class);
+ assertEquals(newOverrides, overrides);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> createOverrides(Pair<String, String>... keyValues) {
+ final Map<String, String> overrides = new HashMap<>();
+ for (Pair<String, String> kv : keyValues) {
+ overrides.put(kv.first, kv.second);
+ }
+ return overrides;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Properties createOverrideProperties(Pair<String, String>... props) {
+ final Properties properties = new Properties();
+ for (Pair<String, String> prop : props) {
+ properties.setProperty(prop.first, prop.second);
+ }
+
+ return properties;
+ }
+}
diff --git a/heron/tools/cli/src/python/BUILD b/heron/tools/cli/src/python/BUILD
index 718a922..23ff8bf 100644
--- a/heron/tools/cli/src/python/BUILD
+++ b/heron/tools/cli/src/python/BUILD
@@ -12,7 +12,11 @@
"//heron/tools/common/src/python:common-py",
"//heron/proto:proto-py",
],
- reqs = ["PyYAML==3.10", "enum34==1.1.6"],
+ reqs = [
+ "PyYAML==3.10",
+ "enum34==1.1.6",
+ "requests==2.18.1",
+ ],
)
pex_binary(
diff --git a/heron/tools/cli/src/python/activate.py b/heron/tools/cli/src/python/activate.py
index 306509e..365d6c9 100644
--- a/heron/tools/cli/src/python/activate.py
+++ b/heron/tools/cli/src/python/activate.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' activate '''
+from heron.common.src.python.utils.log import Log
import heron.tools.cli.src.python.cli_helper as cli_helper
def create_parser(subparsers):
@@ -31,4 +32,5 @@
:param unknown_args:
:return:
'''
+ Log.debug("Activate Args: %s", cl_args)
return cli_helper.run(command, cl_args, "activate topology")
diff --git a/heron/tools/cli/src/python/args.py b/heron/tools/cli/src/python/args.py
index 5f7feb6..ab33fde 100644
--- a/heron/tools/cli/src/python/args.py
+++ b/heron/tools/cli/src/python/args.py
@@ -85,6 +85,16 @@
)
return parser
+def add_service_url(parser):
+ '''
+ :param parser:
+ :return:
+ '''
+ parser.add_argument(
+ '--service-url',
+ default="",
+ help='API service end point')
+ return parser
def add_config(parser):
'''
diff --git a/heron/tools/cli/src/python/cdefs.py b/heron/tools/cli/src/python/cdefs.py
new file mode 100644
index 0000000..8b52304
--- /dev/null
+++ b/heron/tools/cli/src/python/cdefs.py
@@ -0,0 +1,63 @@
+# Copyright 2016 Twitter. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+''' cdefs.py '''
+import os
+import yaml
+
+import heron.common.src.python.utils.log as Log
+import heron.tools.common.src.python.utils.config as config
+
+################################################################################
+def read_server_mode_cluster_definition(cluster, cl_args, config_file):
+ '''
+ Read the cluster definition for server mode
+ :param cluster:
+ :param cl_args:
+ :param config_file:
+ :return:
+ '''
+
+ client_confs = dict()
+
+ # check if the config file exists, if it does, read it
+ if os.path.isfile(config_file):
+ with open(config_file, 'r') as conf_file:
+ client_confs = yaml.load(conf_file)
+
+ if not client_confs:
+ client_confs = dict()
+ client_confs[cluster] = dict()
+
+ # now check if the service-url from command line is set, if so override it
+ if cl_args['service_url']:
+ client_confs[cluster]['service_url'] = cl_args['service_url']
+
+ # the return value of yaml.load can be None if conf_file is an empty file
+ # or there is no service-url in command line, if needed.
+
+ return client_confs
+
+################################################################################
+def check_direct_mode_cluster_definition(cluster, config_path):
+ '''
+ Check the cluster definition for direct mode
+ :param cluster:
+ :param config_path:
+ :return:
+ '''
+ config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
+ if not os.path.isdir(config_path):
+ Log.error("Cluster config directory \'%s\' does not exist", config_path)
+ return False
+ return True
diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py
index 7385f9f..c33d93fd 100644
--- a/heron/tools/cli/src/python/cli_helper.py
+++ b/heron/tools/cli/src/python/cli_helper.py
@@ -13,10 +13,13 @@
# limitations under the License.
''' cli_helper.py '''
import logging
+import requests
import heron.tools.common.src.python.utils.config as config
+from heron.tools.cli.src.python.result import SimpleResult, Status
import heron.tools.cli.src.python.args as args
import heron.tools.cli.src.python.execute as execute
import heron.tools.cli.src.python.jars as jars
+import heron.tools.cli.src.python.rest as rest
from heron.common.src.python.utils.log import Log
@@ -39,15 +42,64 @@
args.add_topology(parser)
args.add_config(parser)
+ args.add_service_url(parser)
args.add_verbose(parser)
parser.set_defaults(subcommand=action)
return parser
+################################################################################
+def flatten_args(fargs):
+ temp_args = []
+ for k, v in fargs.iteritems():
+ if isinstance(v, list):
+ temp_args.extend([(k, value) for value in v])
+ else:
+ temp_args.append((k, v))
+ return temp_args
################################################################################
# pylint: disable=dangerous-default-value
-def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
+def run_server(command, cl_args, action, extra_args=dict()):
+ '''
+ helper function to take action on topologies using REST API
+ :param command:
+ :param cl_args:
+ :param action: description of action taken
+ :return:
+ '''
+ topology_name = cl_args['topology-name']
+
+ service_endpoint = cl_args['service_url']
+ apiroute = rest.ROUTE_SIGNATURES[command][1] % (
+ cl_args['cluster'],
+ cl_args['role'],
+ cl_args['environ'],
+ topology_name
+ )
+ service_apiurl = service_endpoint + apiroute
+ service_method = rest.ROUTE_SIGNATURES[command][0]
+
+ # convert the dictionary to a list of tuples
+ data = flatten_args(extra_args)
+
+ err_msg = "Failed to %s: %s" % (action, topology_name)
+ succ_msg = "Successfully %s: %s" % (action, topology_name)
+
+ try:
+ r = service_method(service_apiurl, data=data)
+ s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError
+ if r.status_code != requests.codes.ok:
+ Log.error(r.json().get('message', "Unknown error from api server %d" % r.status_code))
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
+ Log.error(err)
+ return SimpleResult(Status.HeronError, err_msg, succ_msg)
+
+ return SimpleResult(s, err_msg, succ_msg)
+
+################################################################################
+# pylint: disable=dangerous-default-value
+def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
'''
helper function to take action on topologies
:param command:
@@ -89,3 +141,10 @@
succ_msg = "Successfully %s: %s" % (action, topology_name)
result.add_context(err_msg, succ_msg)
return result
+
+################################################################################
+def run(command, cl_args, action, extra_lib_jars=[]):
+ if cl_args['deploy_mode'] == config.SERVER_MODE:
+ return run_server(command, cl_args, action, extra_args=dict())
+ else:
+ return run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=extra_lib_jars)
diff --git a/heron/tools/cli/src/python/deactivate.py b/heron/tools/cli/src/python/deactivate.py
index 08dc04a..be23bee 100644
--- a/heron/tools/cli/src/python/deactivate.py
+++ b/heron/tools/cli/src/python/deactivate.py
@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' deactivate.py '''
+from heron.common.src.python.utils.log import Log
import heron.tools.cli.src.python.cli_helper as cli_helper
-
def create_parser(subparsers):
'''
:param subparsers:
@@ -32,4 +32,5 @@
:param unknown_args:
:return:
'''
+ Log.debug("Deactivate Args: %s", cl_args)
return cli_helper.run(command, cl_args, "deactivate topology")
diff --git a/heron/tools/cli/src/python/kill.py b/heron/tools/cli/src/python/kill.py
index 928c4d5..ac91070 100644
--- a/heron/tools/cli/src/python/kill.py
+++ b/heron/tools/cli/src/python/kill.py
@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' kill.py '''
+from heron.common.src.python.utils.log import Log
import heron.tools.cli.src.python.cli_helper as cli_helper
-
def create_parser(subparsers):
'''
:param subparsers:
@@ -32,4 +32,5 @@
:param unknown_args:
:return:
'''
+ Log.debug("Kill Args: %s", cl_args)
return cli_helper.run(command, cl_args, "kill topology")
diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py
index f77a879..e7f0a21 100644
--- a/heron/tools/cli/src/python/main.py
+++ b/heron/tools/cli/src/python/main.py
@@ -25,6 +25,7 @@
import heron.common.src.python.utils.log as log
import heron.tools.common.src.python.utils.config as config
+import heron.tools.cli.src.python.cdefs as cdefs
import heron.tools.cli.src.python.help as cli_help
import heron.tools.cli.src.python.activate as activate
import heron.tools.cli.src.python.deactivate as deactivate
@@ -144,6 +145,118 @@
if not config.check_release_file_exists():
sys.exit(1)
+################################################################################
+# pylint: disable=unused-argument
+def server_deployment_mode(command, parser, cluster, cl_args):
+ '''
+ check the server deployment mode for the given cluster
+ if it is valid return the valid set of args
+ :param cluster:
+ :param cl_args:
+ :return:
+ '''
+ config_file = config.heron_rc_file()
+
+ # Read the cluster definition, if not found
+ client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args, config_file)
+ if not client_confs[cluster]:
+ return dict()
+
+ # tell the user which definition that we are using
+ if not cl_args['service_url']:
+ Log.info("Using cluster definition from file %s" % config_file)
+ else:
+ Log.info("Using cluster service url %s" % cl_args['service_url'])
+
+ # if cluster definition exists, but service_url is not set, it is an error
+ if not 'service_url' in client_confs[cluster]:
+ Log.error('No service url for %s cluster in %s', cluster, config_file)
+ sys.exit(1)
+
+ try:
+ cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])
+ config.server_mode_cluster_role_env(cluster_role_env, client_confs, config_file)
+ cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
+ except Exception as ex:
+ Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
+ sys.exit(1)
+
+ new_cl_args = dict()
+ new_cl_args['cluster'] = cluster_tuple[0]
+ new_cl_args['role'] = cluster_tuple[1]
+ new_cl_args['environ'] = cluster_tuple[2]
+ new_cl_args['service_url'] = client_confs[cluster]['service_url'].rstrip('/')
+ new_cl_args['deploy_mode'] = config.SERVER_MODE
+
+ cl_args.update(new_cl_args)
+ return cl_args
+
+################################################################################
+def direct_deployment_mode(command, parser, cluster, cl_args):
+ '''
+ check the direct deployment mode for the given cluster
+ if it is valid return the valid set of args
+ :param command:
+ :param parser:
+ :param cluster:
+ :param cl_args:
+ :return:
+ '''
+
+ cluster = cl_args['cluster']
+ try:
+ config_path = cl_args['config_path']
+ override_config_file = config.parse_override_config(cl_args['config_property'])
+ except KeyError:
+ # if some of the arguments are not found, print error and exit
+ subparser = config.get_subparser(parser, command)
+ print subparser.format_help()
+ return dict()
+
+ # check if the cluster config directory exists
+ if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):
+ return dict()
+
+ config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
+ if not os.path.isdir(config_path):
+ Log.error("Cluster config directory \'%s\' does not exist", config_path)
+ return dict()
+
+ Log.info("Using cluster definition in %s" % config_path)
+
+ try:
+ cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])
+ config.direct_mode_cluster_role_env(cluster_role_env, config_path)
+ cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
+ except Exception as ex:
+ Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
+ return dict()
+
+ new_cl_args = dict()
+ new_cl_args['cluster'] = cluster_tuple[0]
+ new_cl_args['role'] = cluster_tuple[1]
+ new_cl_args['environ'] = cluster_tuple[2]
+ new_cl_args['config_path'] = config_path
+ new_cl_args['override_config_file'] = override_config_file
+ new_cl_args['deploy_mode'] = config.DIRECT_MODE
+
+ cl_args.update(new_cl_args)
+ return cl_args
+
+################################################################################
+def deployment_mode(command, parser, cl_args):
+ # first check if it is server mode
+ new_cl_args = server_deployment_mode(command, parser, cl_args['cluster'], cl_args)
+ if len(new_cl_args) > 0:
+ return new_cl_args
+
+ # now check if it is direct mode
+ new_cl_args = direct_deployment_mode(command, parser, cl_args['cluster'], cl_args)
+ if len(new_cl_args) > 0:
+ return new_cl_args
+
+ return dict()
+
################################################################################
def extract_common_args(command, parser, cl_args):
@@ -156,37 +269,25 @@
'''
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
- config_path = cl_args['config_path']
- override_config_file = config.parse_override_config(cl_args['config_property'])
except KeyError:
- # if some of the arguments are not found, print error and exit
- subparser = config.get_subparser(parser, command)
- print subparser.format_help()
- return dict()
-
- cluster = config.get_heron_cluster(cluster_role_env)
- config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
- if not os.path.isdir(config_path):
- Log.error("Config path cluster directory does not exist: %s", config_path)
- return dict()
+ try:
+ cluster_role_env = cl_args.pop('cluster') # for version command
+ except KeyError:
+ # if some of the arguments are not found, print error and exit
+ subparser = config.get_subparser(parser, command)
+ print subparser.format_help()
+ return dict()
new_cl_args = dict()
- try:
- cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
- new_cl_args['cluster'] = cluster_tuple[0]
- new_cl_args['role'] = cluster_tuple[1]
- new_cl_args['environ'] = cluster_tuple[2]
- new_cl_args['submit_user'] = getpass.getuser()
- new_cl_args['config_path'] = config_path
- new_cl_args['override_config_file'] = override_config_file
- except Exception as ex:
- Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
- return dict()
+ cluster_tuple = config.get_cluster_role_env(cluster_role_env)
+ new_cl_args['cluster'] = cluster_tuple[0]
+ new_cl_args['role'] = cluster_tuple[1]
+ new_cl_args['environ'] = cluster_tuple[2]
+ new_cl_args['submit_user'] = getpass.getuser()
cl_args.update(new_cl_args)
return cl_args
-
################################################################################
def main():
'''
@@ -220,19 +321,29 @@
# command to be execute
command = command_line_args['subcommand']
+ if command == 'version':
+ results = run(command, parser, command_line_args, unknown_args)
+ return 0 if result.is_successful(results) else 1
+
if command not in ('help', 'version'):
log.set_logging_level(command_line_args)
+ Log.debug("Input Command Line Args: %s", command_line_args)
+
+ # determine the mode of deployment
command_line_args = extract_common_args(command, parser, command_line_args)
+ command_line_args = deployment_mode(command, parser, command_line_args)
+
# bail out if args are empty
if not command_line_args:
return 1
- # register dirs cleanup function during exit
- cleaned_up_files.append(command_line_args['override_config_file'])
- atexit.register(cleanup, cleaned_up_files)
+ # register dirs cleanup function during exit
+ if command_line_args['deploy_mode'] == config.DIRECT_MODE and command != "version":
+ cleaned_up_files.append(command_line_args['override_config_file'])
+ atexit.register(cleanup, cleaned_up_files)
# print the input parameters, if verbose is enabled
- Log.debug(command_line_args)
+ Log.debug("Processed Command Line Args: %s", command_line_args)
start = time.time()
results = run(command, parser, command_line_args, unknown_args)
@@ -242,7 +353,7 @@
if command not in ('help', 'version'):
sys.stdout.flush()
- Log.debug('Elapsed time: %.3fs.', (end - start))
+ Log.info('Elapsed time: %.3fs.', (end - start))
return 0 if result.is_successful(results) else 1
diff --git a/heron/tools/cli/src/python/rest.py b/heron/tools/cli/src/python/rest.py
new file mode 100644
index 0000000..6b878b1
--- /dev/null
+++ b/heron/tools/cli/src/python/rest.py
@@ -0,0 +1,25 @@
+# Copyright 2016 Twitter. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+''' rest.py '''
+import requests
+
+ROUTE_SIGNATURES = {
+ 'activate': (requests.post, '/api/v1/topologies/%s/%s/%s/%s/activate', []),
+ 'deactivate': (requests.post, '/api/v1/topologies/%s/%s/%s/%s/deactivate', []),
+ 'kill': (requests.delete, '/api/v1/topologies/%s/%s/%s/%s', []),
+ 'restart': (requests.post, '/api/v1/topologies/%s/%s/%s/%s/restart', []),
+ 'submit': (requests.post, '/api/v1/topologies', ['name', 'cluster', 'role', 'env', 'user']),
+ 'update': (requests.post, '/api/v1/topologies/%s/%s/%s/%s/update', []),
+ 'version': (requests.get, '/api/v1/version', []),
+}
diff --git a/heron/tools/cli/src/python/restart.py b/heron/tools/cli/src/python/restart.py
index bc77ec2..ec2575a 100644
--- a/heron/tools/cli/src/python/restart.py
+++ b/heron/tools/cli/src/python/restart.py
@@ -12,9 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' restart.py '''
+from heron.common.src.python.utils.log import Log
import heron.tools.cli.src.python.args as args
import heron.tools.cli.src.python.cli_helper as cli_helper
-
+import heron.tools.common.src.python.utils.config as config
def create_parser(subparsers):
'''
@@ -39,6 +40,7 @@
help='Identifier of the container to be restarted')
args.add_config(parser)
+ args.add_service_url(parser)
args.add_verbose(parser)
parser.set_defaults(subcommand='restart')
@@ -54,7 +56,12 @@
:param unknown_args:
:return:
'''
+ Log.debug("Restart Args: %s", cl_args)
container_id = cl_args['container-id']
- extra_args = ["--container_id", str(container_id)]
- return cli_helper.run(command, cl_args, "restart topology", extra_args=extra_args)
+ if cl_args['deploy_mode'] == config.SERVER_MODE:
+ dict_extra_args = {"container_id": str(container_id)}
+ return cli_helper.run_server(command, cl_args, "restart topology", extra_args=dict_extra_args)
+ else:
+ list_extra_args = ["--container_id", str(container_id)]
+ return cli_helper.run_direct(command, cl_args, "restart topology", extra_args=list_extra_args)
diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py
index 834b201..b0ca038 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -16,6 +16,7 @@
import logging
import os
import tempfile
+import requests
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
@@ -25,12 +26,24 @@
import heron.tools.cli.src.python.jars as jars
import heron.tools.cli.src.python.opts as opts
import heron.tools.cli.src.python.result as result
+import heron.tools.cli.src.python.rest as rest
import heron.tools.common.src.python.utils.config as config
import heron.tools.common.src.python.utils.classpath as classpath
# pylint: disable=too-many-return-statements
################################################################################
+def launch_mode_msg(cl_args):
+ '''
+ Depending on the mode of launching a topology provide a message
+ :param cl_args:
+ :return:
+ '''
+ if cl_args['dry_run']:
+ return "in dry-run mode"
+ return ""
+
+################################################################################
def create_parser(subparsers):
'''
Create a subparser for the submit command
@@ -51,9 +64,10 @@
cli_args.add_topology_class(parser)
cli_args.add_config(parser)
cli_args.add_deactive_deploy(parser)
- cli_args.add_extra_launch_classpath(parser)
- cli_args.add_system_property(parser)
cli_args.add_dry_run(parser)
+ cli_args.add_extra_launch_classpath(parser)
+ cli_args.add_service_url(parser)
+ cli_args.add_system_property(parser)
cli_args.add_verbose(parser)
parser.set_defaults(subcommand='submit')
@@ -68,6 +82,7 @@
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
+ :param topology_name:
:return:
'''
# get the normalized path for topology.tar.gz
@@ -119,16 +134,61 @@
extra_jars=extra_jars,
args=args,
java_defines=[])
- err_context = "Failed to launch topology '%s'" % topology_name
- if cl_args["dry_run"]:
- err_context += " in dry-run mode"
- succ_context = "Successfully launched topology '%s'" % topology_name
- if cl_args["dry_run"]:
- succ_context += " in dry-run mode"
- res.add_context(err_context, succ_context)
+
+ err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
+ succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
+
+ res.add_context(err_ctxt, succ_ctxt)
return res
################################################################################
+def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):
+ '''
+ Launch a topology given topology jar, its definition file and configurations
+ :param cl_args:
+ :param topology_file:
+ :param topology_defn_file:
+ :param topology_name:
+ :return:
+ '''
+ service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]
+ service_method = rest.ROUTE_SIGNATURES['submit'][0]
+ data = dict(
+ name=topology_name,
+ cluster=cl_args['cluster'],
+ role=cl_args['role'],
+ environment=cl_args['environ'],
+ user=cl_args['submit_user'],
+ )
+
+ if cl_args['dry_run']:
+ data["dry_run"] = True
+
+ files = dict(
+ definition=open(topology_defn_file, 'rb'),
+ topology=open(topology_file, 'rb'),
+ )
+
+ err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
+ succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
+
+ try:
+ r = service_method(service_apiurl, data=data, files=files)
+ ok = r.status_code is requests.codes.ok
+ created = r.status_code is requests.codes.created
+ s = Status.Ok if created or ok else Status.HeronError
+ if s is Status.HeronError:
+ Log.error(r.json().get('message', "Unknown error from api server %d" % r.status_code))
+ elif ok:
+ # this case happens when we request a dry_run
+ print r.json().get("response")
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
+ Log.error(err)
+ return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
+ return SimpleResult(s, err_ctxt, succ_ctxt)
+
+
+################################################################################
def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
@@ -154,12 +214,19 @@
except Exception as e:
err_context = "Cannot load topology definition '%s': %s" % (defn_file, e)
return SimpleResult(Status.HeronError, err_context)
+
# launch the topology
- mode = " in dry-run mode" if cl_args['dry_run'] else ''
- Log.info("Launching topology: \'%s\'%s", topology_defn.name, mode)
- res = launch_a_topology(
- cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)
+ Log.info("Launching topology: \'%s\'%s", topology_defn.name, launch_mode_msg(cl_args))
+
+ # check if we have to do server or direct based deployment
+ if cl_args['deploy_mode'] == config.SERVER_MODE:
+ res = launch_topology_server(
+ cl_args, topology_file, defn_file, topology_defn.name)
+ else:
+ res = launch_a_topology(
+ cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)
results.append(res)
+
return results
@@ -283,6 +350,8 @@
:param unknown_args:
:return:
'''
+ Log.debug("Submit Args %s", cl_args)
+
# get the topology file name
topology_file = cl_args['topology-file-name']
diff --git a/heron/tools/cli/src/python/update.py b/heron/tools/cli/src/python/update.py
index 4a914e4..202f13a 100644
--- a/heron/tools/cli/src/python/update.py
+++ b/heron/tools/cli/src/python/update.py
@@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' restart.py '''
+from heron.common.src.python.utils.log import Log
import heron.tools.cli.src.python.args as args
import heron.tools.cli.src.python.cli_helper as cli_helper
import heron.tools.cli.src.python.jars as jars
+import heron.tools.common.src.python.utils.config as config
import argparse
import re
@@ -49,6 +51,7 @@
args.add_config(parser)
args.add_dry_run(parser)
+ args.add_service_url(parser)
args.add_verbose(parser)
parser.set_defaults(subcommand='update')
@@ -58,12 +61,24 @@
# pylint: disable=unused-argument
def run(command, parser, cl_args, unknown_args):
""" run the update command """
- extra_args = ["--component_parallelism", ','.join(cl_args['component_parallelism'])]
+
+ Log.debug("Update Args: %s", cl_args)
extra_lib_jars = jars.packing_jars()
action = "update topology%s" % (' in dry-run mode' if cl_args["dry_run"] else '')
- if cl_args["dry_run"]:
- extra_args.append('--dry_run')
- if "dry_run_format" in cl_args:
- extra_args += ["--dry_run_format", cl_args["dry_run_format"]]
- return cli_helper.run(command, cl_args, action, extra_args, extra_lib_jars)
+ if cl_args['deploy_mode'] == config.SERVER_MODE:
+ dict_extra_args = {"component_parallelism": cl_args['component_parallelism']}
+ if cl_args["dry_run"]:
+ dict_extra_args.update({'dry_run': True})
+ if "dry_run_format" in cl_args:
+ dict_extra_args.update({"dry_run_format", cl_args["dry_run_format"]})
+
+ return cli_helper.run_server(command, cl_args, action, dict_extra_args)
+ else:
+ list_extra_args = ["--component_parallelism", ','.join(cl_args['component_parallelism'])]
+ if cl_args["dry_run"]:
+ list_extra_args.append('--dry_run')
+ if "dry_run_format" in cl_args:
+ list_extra_args += ["--dry_run_format", cl_args["dry_run_format"]]
+
+ return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)
diff --git a/heron/tools/cli/src/python/version.py b/heron/tools/cli/src/python/version.py
index 8cf487d..d9b6607 100644
--- a/heron/tools/cli/src/python/version.py
+++ b/heron/tools/cli/src/python/version.py
@@ -12,10 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' version.py '''
+from heron.common.src.python.utils.log import Log
from heron.tools.cli.src.python.result import SimpleResult, Status
import heron.tools.cli.src.python.args as cli_args
import heron.tools.common.src.python.utils.config as config
+import heron.tools.cli.src.python.cdefs as cdefs
+import heron.tools.cli.src.python.rest as rest
+import sys
+import requests
+
+def add_version_titles(parser):
+ '''
+ :param parser:
+ :return:
+ '''
+ # pylint: disable=protected-access
+ parser._positionals.title = "Optional positional arguments"
+ parser._optionals.title = "Optional arguments"
+ return parser
def create_parser(subparsers):
'''
@@ -25,17 +40,25 @@
parser = subparsers.add_parser(
'version',
help='Print version of heron-cli',
- usage="%(prog)s",
+ usage="%(prog)s [options] [cluster]",
add_help=True)
- cli_args.add_titles(parser)
+ add_version_titles(parser)
+
+ parser.add_argument(
+ 'cluster',
+ nargs='?',
+ type=str,
+ default="",
+ help='Name of the cluster')
+
+ cli_args.add_service_url(parser)
parser.set_defaults(subcommand='version')
return parser
-
# pylint: disable=unused-argument
-def run(command, parser, args, unknown_args):
+def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
@@ -43,5 +66,40 @@
:param unknown_args:
:return:
'''
- config.print_build_info()
+ cluster = cl_args['cluster']
+
+ # server mode
+ if cluster:
+ config_file = config.heron_rc_file()
+ client_confs = dict()
+
+ # Read the cluster definition, if not found
+ client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args, config_file)
+
+ if not client_confs[cluster]:
+ Log.error('Neither service url nor %s cluster definition in %s file', cluster, config_file)
+ return SimpleResult(Status.HeronError)
+
+ # if cluster definition exists, but service_url is not set, it is an error
+ if not 'service_url' in client_confs[cluster]:
+ Log.error('No service url for %s cluster in %s', cluster, config_file)
+ sys.exit(1)
+
+ service_endpoint = cl_args['service_url']
+ service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][1]
+ service_method = rest.ROUTE_SIGNATURES[command][0]
+
+ try:
+ r = service_method(service_apiurl)
+ if r.status_code != requests.codes.ok:
+ Log.error(r.json().get('message', "Unknown error from api server %d" % r.status_code))
+ sorted_items = sorted(r.json().items(), key=lambda tup: tup[0])
+ for key, value in sorted_items:
+ print "%s : %s" % (key, value)
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
+ Log.error(err)
+ return SimpleResult(Status.HeronError)
+ else:
+ config.print_build_info()
+
return SimpleResult(Status.Ok)
diff --git a/heron/tools/cli/tests/python/client_command_unittest.py b/heron/tools/cli/tests/python/client_command_unittest.py
index 7859970..c747492 100644
--- a/heron/tools/cli/tests/python/client_command_unittest.py
+++ b/heron/tools/cli/tests/python/client_command_unittest.py
@@ -21,6 +21,7 @@
import sys
import tempfile
import heron.tools.cli.src.python.main as main
+import heron.tools.cli.src.python.cdefs as cdefs
import heron.tools.cli.src.python.submit as submit
import heron.tools.cli.src.python.result as result
import heron.tools.common.src.python.utils.config as config
@@ -51,6 +52,8 @@
os.path.isdir = MagicMock(return_value=True)
os.path.isfile = MagicMock(return_value=True)
os.environ.copy = MagicMock(return_value={})
+ main.server_deployment_mode = MagicMock(return_value=dict())
+ cdefs.check_direct_mode_cluster_definition = MagicMock(return_value=True)
def run_test(self, command, issued_commands, environ):
calls = []
diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py
index 9f827de..12e0de7 100644
--- a/heron/tools/common/src/python/utils/config.py
+++ b/heron/tools/common/src/python/utils/config.py
@@ -40,16 +40,23 @@
ZIPPED_RELEASE_YAML = "scripts/packages/release.yaml"
OVERRIDE_YAML = "override.yaml"
+# mode of deployment
+DIRECT_MODE = 'direct'
+SERVER_MODE = 'server'
+
# directories for heron sandbox
SANDBOX_CONF_DIR = "./heron-conf"
# config file for heron cli
CLIENT_YAML = "client.yaml"
-# cli configs for role and env
-IS_ROLE_REQUIRED = "heron.config.is.role.required"
-IS_ENV_REQUIRED = "heron.config.is.env.required"
+# client configs for role and env for direct deployment
+ROLE_REQUIRED = "heron.config.is.role.required"
+ENV_REQUIRED = "heron.config.is.env.required"
+# client config for role and env for server deployment
+ROLE_KEY = "role.required"
+ENVIRON_KEY = "env.required"
def create_tar(tar_filename, files, config_dir, config_files):
'''
@@ -240,12 +247,15 @@
"""Get the cluster to which topology is submitted"""
return cluster_role_env.split('/')[0]
+def heron_rc_file():
+ """Get the full path name of the .heronrc file"""
+ return os.path.join(os.path.expanduser('~'), '.heronrc')
+################################################################################
# pylint: disable=too-many-branches
def parse_cluster_role_env(cluster_role_env, config_path):
"""Parse cluster/[role]/[environ], supply default, if not provided, not required"""
parts = cluster_role_env.split('/')[:3]
- Log.debug("Using config file under %s" % config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
raise Exception("Invalid config path")
@@ -273,17 +283,17 @@
# if role is required but not provided, raise exception
if len(parts) == 1:
- if (IS_ROLE_REQUIRED in cli_confs) and (cli_confs[IS_ROLE_REQUIRED] is True):
+ if (ROLE_REQUIRED in cli_confs) and (cli_confs[ROLE_REQUIRED] is True):
raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s"
- % (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
+ % (cluster_role_env, ROLE_REQUIRED, cli_conf_file))
else:
parts.append(getpass.getuser())
# if environ is required but not provided, raise exception
if len(parts) == 2:
- if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
+ if (ENV_REQUIRED in cli_confs) and (cli_confs[ENV_REQUIRED] is True):
raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s"
- % (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
+ % (cluster_role_env, ENV_REQUIRED, cli_conf_file))
else:
parts.append(ENVIRON)
@@ -294,6 +304,84 @@
return (parts[0], parts[1], parts[2])
+################################################################################
+def get_cluster_role_env(cluster_role_env):
+ """Parse cluster/[role]/[environ], supply empty string, if not provided"""
+ parts = cluster_role_env.split('/')[:3]
+ if len(parts) == 3:
+ return (parts[0], parts[1], parts[2])
+
+ if len(parts) == 2:
+ return (parts[0], parts[1], "")
+
+ if len(parts) == 1:
+ return (parts[0], "", "")
+
+ return ("", "", "")
+
+################################################################################
+def direct_mode_cluster_role_env(cluster_role_env, config_path):
+ """Check cluster/[role]/[environ], if they are required"""
+
+ # otherwise, get the client.yaml file
+ cli_conf_file = os.path.join(config_path, CLIENT_YAML)
+
+ # if client conf doesn't exist, use default value
+ if not os.path.isfile(cli_conf_file):
+ return True
+
+ client_confs = {}
+ with open(cli_conf_file, 'r') as conf_file:
+ client_confs = yaml.load(conf_file)
+
+ # the return value of yaml.load can be None if conf_file is an empty file
+ if not client_confs:
+ return True
+
+ # if role is required but not provided, raise exception
+ role_present = True if len(cluster_role_env[1]) > 0 else False
+ if ROLE_REQUIRED in client_confs and client_confs[ROLE_REQUIRED] and not role_present:
+ raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s"
+ % (cluster_role_env, ROLE_REQUIRED, cli_conf_file))
+
+ # if environ is required but not provided, raise exception
+ environ_present = True if len(cluster_role_env[2]) > 0 else False
+ if ENV_REQUIRED in client_confs and client_confs[ENV_REQUIRED] and not environ_present:
+ raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s"
+ % (cluster_role_env, ENV_REQUIRED, cli_conf_file))
+
+ return True
+
+################################################################################
+def server_mode_cluster_role_env(cluster_role_env, config_map, config_file):
+ """Check cluster/[role]/[environ], if they are required"""
+
+ cmap = config_map[cluster_role_env[0]]
+
+ # if role is required but not provided, raise exception
+ role_present = True if len(cluster_role_env[1]) > 0 else False
+ if ROLE_KEY in cmap and cmap[ROLE_KEY] and not role_present:
+ raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s"
+ % (cluster_role_env, ROLE_KEY, config_file))
+
+ # if environ is required but not provided, raise exception
+ environ_present = True if len(cluster_role_env[2]) > 0 else False
+ if ENVIRON_KEY in cmap and cmap[ENVIRON_KEY] and not environ_present:
+ raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s"
+ % (cluster_role_env, ENVIRON_KEY, config_file))
+
+ return True
+
+################################################################################
+def defaults_cluster_role_env(cluster_role_env):
+ """
+ if role is not provided, supply userid
+ if environ is not provided, supply 'default'
+ """
+ if len(cluster_role_env[1]) == 0 and len(cluster_role_env[2]) == 0:
+ return (cluster_role_env[0], getpass.getuser(), ENVIRON)
+
+ return (cluster_role_env[0], cluster_role_env[1], ENVIRON)
################################################################################
# Parse the command line for overriding the defaults
@@ -354,9 +442,12 @@
release_file = get_zipped_heron_release_file()
else:
release_file = get_heron_release_file()
+
with open(release_file) as release_info:
- for line in release_info:
- print line,
+ release_map = yaml.load(release_info)
+ release_items = sorted(release_map.items(), key=lambda tup: tup[0])
+ for key, value in release_items:
+ print "%s : %s" % (key, value)
def get_version_number(zipped_pex=False):
"""Print version from release.yaml
diff --git a/scripts/get_all_heron_paths.sh b/scripts/get_all_heron_paths.sh
index 2e7f689..20679e9 100755
--- a/scripts/get_all_heron_paths.sh
+++ b/scripts/get_all_heron_paths.sh
@@ -67,7 +67,7 @@
}
function get_heron_java_paths() {
- local java_paths=$(find {heron,tools,integration_test,contrib} -name "*.java" | sed "s|/src/java/.*$|/src/java|"| sed "s|/java/src/.*$|/java/src|" | sed "s|/tests/java/.*$|/tests/java|" | sort -u | fgrep -v "heron/scheduler/" | fgrep -v "heron/scheduler/" )
+ local java_paths=$(find {heron,heron/tools,tools,integration_test,contrib} -name "*.java" | sed "s|/src/java/.*$|/src/java|"| sed "s|/java/src/.*$|/java/src|" | sed "s|/tests/java/.*$|/tests/java|" | sort -u | fgrep -v "heron/scheduler/" | fgrep -v "heron/scheduler/" )
if [ "$(uname -s | tr 'A-Z' 'a-z')" != "darwin" ]; then
java_paths=$(echo "${java_paths}" | fgrep -v "/objc_tools/")
fi
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index 2af67f9..5a7e76f 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -161,9 +161,24 @@
)
pkg_tar(
+ name = "heron-tools-lib",
+ package_dir = "lib",
+ files = heron_tools_lib_files(),
+)
+
+pkg_tar(
name = "heron-tools-conf",
package_dir = "conf",
- files = heron_tools_conf_files(),
+ files = heron_tools_conf_files()
+)
+
+pkg_tar(
+ name = "heron-tools-cluster-conf",
+ strip_prefix = "/heron/config/src/yaml/conf",
+ package_dir = "conf",
+ files = [
+ "//heron/config/src/yaml:conf-yaml"
+ ],
)
pkg_tar(
@@ -172,7 +187,9 @@
files = generated_release_files,
deps = [
":heron-tools-bin",
+ ":heron-tools-lib",
":heron-tools-conf",
+ ":heron-tools-cluster-conf",
],
)
@@ -228,65 +245,12 @@
)
pkg_tar(
- name = "heron-client-conf-local",
- package_dir = "conf/local",
- files = [
- "//heron/config/src/yaml:conf-local-yaml",
- ],
-)
-
-pkg_tar(
- name = "heron-client-conf-slurm",
- package_dir = "conf/slurm",
- files = [
- "//heron/config/src/yaml:conf-slurm-yaml",
- ],
-)
-
-pkg_tar(
- name = "heron-client-conf-aurora",
- package_dir = "conf/aurora",
- files = [
- "//heron/config/src/yaml:conf-aurora-yaml",
- ],
-)
-
-pkg_tar(
- name = "heron-client-conf-yarn",
- package_dir = "conf/yarn",
- files = [
- "//heron/config/src/yaml:conf-yarn-yaml",
- ],
-)
-
-pkg_tar(
- name = "heron-client-conf-mesos",
- package_dir = "conf/mesos",
- files = [
- "//heron/config/src/yaml:conf-mesos-yaml",
- ],
-)
-
-pkg_tar(
- name = "heron-client-conf-marathon",
- package_dir = "conf/marathon",
- files = [
- "//heron/config/src/yaml:conf-marathon-yaml",
- ]
-)
-
-pkg_tar(
- name = "heron-client-conf-kubernetes",
- package_dir = "conf/kubernetes",
- files = [
- "//heron/config/src/yaml:conf-kubernetes-yaml",
- ]
-)
-
-pkg_tar(
name = "heron-client-conf",
+ strip_prefix = "/heron/config/src/yaml/conf",
package_dir = "conf",
- files = heron_client_conf_files(),
+ files = [
+ "//heron/config/src/yaml:conf-yaml",
+ ]
)
pkg_tar(
@@ -302,13 +266,6 @@
deps = [
":heron-client-bin",
":heron-client-conf",
- ":heron-client-conf-local",
- ":heron-client-conf-aurora",
- ":heron-client-conf-slurm",
- ":heron-client-conf-yarn",
- ":heron-client-conf-mesos",
- ":heron-client-conf-marathon",
- ":heron-client-conf-kubernetes",
":heron-client-dist",
":heron-client-examples",
":heron-client-lib-third_party",
diff --git a/scripts/packages/client_template_bin.sh b/scripts/packages/client_template_bin.sh
index 0235092..12c2764 100755
--- a/scripts/packages/client_template_bin.sh
+++ b/scripts/packages/client_template_bin.sh
@@ -19,7 +19,6 @@
# Installation and etc prefix can be overriden from command line
install_prefix=${1:-"/usr/local/heron"}
-heronrc=${2:-"/usr/local/heron/etc/heron.heronrc"}
progname="$0"
@@ -31,9 +30,8 @@
echo "Usage: $progname [options]" >&2
echo "Options are:" >&2
echo " --prefix=/some/path set the prefix path (default=/usr/local)." >&2
- echo " --heronrc= set the heronrc path (default=/usr/local/heron/etc/heron.heronrc)." >&2
echo " --user configure for user install, expands to" >&2
- echo ' `--prefix=$HOME/.heron --heronrc=$HOME/.heronrc`.' >&2
+ echo ' `--prefix=$HOME/.heron`.' >&2
exit 1
}
@@ -41,20 +39,15 @@
bin="%prefix%/bin"
base="%prefix%/heron"
conf="%prefix%/heron/conf"
-heronrc="%prefix%/heron/etc/heron.heronrc"
for opt in "${@}"; do
case $opt in
--prefix=*)
prefix="$(echo "$opt" | cut -d '=' -f 2-)"
;;
- --heronrc=*)
- heronrc="$(echo "$opt" | cut -d '=' -f 2-)"
- ;;
--user)
bin="$HOME/bin"
base="$HOME/.heron"
- heronrc="$HOME/.heronrc"
;;
*)
usage
@@ -64,14 +57,12 @@
bin="${bin//%prefix%/${prefix}}"
base="${base//%prefix%/${prefix}}"
-heronrc="${heronrc//%prefix%/${prefix}}"
check_unzip; check_tar; check_java
# Test for write access
test_write "${bin}"
test_write "${base}"
-test_write "${heronrc}"
# Do the actual installation
echo -n "Uncompressing."
@@ -111,16 +102,6 @@
ln -s "${base}/bin/heron-explorer" "${bin}/heron-explorer"
echo -n .
-if [ -f "${heronrc}" ]; then
- echo
- echo "${heronrc} already exists, not modifying it"
-else
- touch "${heronrc}"
- if [ "${UID}" -eq 0 ]; then
- chmod 0644 "${heronrc}"
- fi
-fi
-
rm "${base}/heron-client.tar.gz"
cat <<EOF
diff --git a/scripts/packages/tools_template_bin.sh b/scripts/packages/tools_template_bin.sh
index 4b050bf..4b48efc 100755
--- a/scripts/packages/tools_template_bin.sh
+++ b/scripts/packages/tools_template_bin.sh
@@ -73,6 +73,9 @@
if [ -L "${bin}/heron-ui" ]; then
rm -f "${bin}/heron-ui"
fi
+if [ -L "${bin}/heron-apiserver" ]; then
+ rm -f "${bin}/heron-apiserver"
+fi
if [ -d "${base}" -a -x "${base}/bin/heron-tracker" ]; then
rm -fr "${base}"
fi
@@ -84,14 +87,16 @@
untar ${base}/heron-tools.tar.gz ${base}
echo -n .
chmod 0755 ${base}/bin/heron-tracker ${base}/bin/heron-ui
+chmod 0755 ${base}/bin/heron-apiserver
echo -n .
chmod -R og-w "${base}"
chmod -R og+rX "${base}"
chmod -R u+rwX "${base}"
echo -n .
-ln -s "${base}/bin/heron-tracker" "${bin}/heron-tracker"
-ln -s "${base}/bin/heron-ui" "${bin}/heron-ui"
+ln -s "${base}/bin/heron-tracker" "${bin}/heron-tracker"
+ln -s "${base}/bin/heron-ui" "${bin}/heron-ui"
+ln -s "${base}/bin/heron-apiserver" "${bin}/heron-apiserver"
echo -n .
rm "${base}/heron-tools.tar.gz"
diff --git a/third_party/java/BUILD b/third_party/java/BUILD
index 7efd80e..752ee35 100644
--- a/third_party/java/BUILD
+++ b/third_party/java/BUILD
@@ -230,3 +230,79 @@
"@apache_pulsar_client//jar",
],
)
+
+java_library(
+ name = "commons-compress",
+ srcs = [ "Empty.java" ],
+ exports = [
+ "@org_apache_commons_compress//jar",
+ ],
+ deps = [
+ "@org_apache_commons_compress//jar",
+ ]
+)
+
+java_library(
+ name = "jetty-jersey-java",
+ srcs = [ "Empty.java" ],
+ exports = [
+ "@org_eclipse_jetty_server//jar",
+ "@org_eclipse_jetty_http//jar",
+ "@org_eclipse_jetty_util//jar",
+ "@org_eclipse_jetty_io//jar",
+ "@org_eclipse_jetty_security//jar",
+ "@org_eclipse_jetty_continuation//jar",
+ "@org_eclipse_jetty_servlet//jar",
+ "@org_eclipse_jetty_servlets//jar",
+ "@javax_servlet_api//jar",
+ "@jersey_container_servlet_core//jar",
+ "@jersey_container_servlet//jar",
+ "@jersey_server//jar",
+ "@jersey_client//jar",
+ "@jersey_common//jar",
+ "@jersey_guava//jar",
+ "@jersey_media_multipart//jar",
+ "@jersey_media_jaxb//jar",
+ "@javax_inject//jar",
+ "@javax_annotation//jar",
+ "@javax_validation//jar",
+ "@javax_ws_rs_api//jar",
+ "@hk2_api//jar",
+ "@hk2_utils//jar",
+ "@hk2_locator//jar",
+ "@hk2_aopalliance_repackaged//jar",
+ "@hk2_osgi_resource_locator//jar",
+ "@org_javassit//jar",
+ "@mimepull//jar",
+ ],
+ deps = [
+ "@org_eclipse_jetty_server//jar",
+ "@org_eclipse_jetty_http//jar",
+ "@org_eclipse_jetty_util//jar",
+ "@org_eclipse_jetty_io//jar",
+ "@org_eclipse_jetty_security//jar",
+ "@org_eclipse_jetty_continuation//jar",
+ "@org_eclipse_jetty_servlet//jar",
+ "@org_eclipse_jetty_servlets//jar",
+ "@javax_servlet_api//jar",
+ "@jersey_container_servlet_core//jar",
+ "@jersey_container_servlet//jar",
+ "@jersey_server//jar",
+ "@jersey_client//jar",
+ "@jersey_common//jar",
+ "@jersey_guava//jar",
+ "@jersey_media_multipart//jar",
+ "@jersey_media_jaxb//jar",
+ "@javax_inject//jar",
+ "@javax_annotation//jar",
+ "@javax_validation//jar",
+ "@javax_ws_rs_api//jar",
+ "@hk2_api//jar",
+ "@hk2_utils//jar",
+ "@hk2_locator//jar",
+ "@hk2_aopalliance_repackaged//jar",
+ "@hk2_osgi_resource_locator//jar",
+ "@org_javassit//jar",
+ "@mimepull//jar",
+ ],
+)
diff --git a/tools/rules/heron_tools.bzl b/tools/rules/heron_tools.bzl
index dfb79aa..dbeae82 100644
--- a/tools/rules/heron_tools.bzl
+++ b/tools/rules/heron_tools.bzl
@@ -8,12 +8,15 @@
return [
"//heron/tools/tracker/src/python:heron-tracker",
"//heron/tools/ui/src/python:heron-ui",
+ "//heron/tools/apiserver/src/shell:heron-apiserver"
+ ]
+
+def heron_tools_lib_files():
+ return [
+ "//heron/tools/apiserver/src/java:heron-apiserver",
]
def heron_tools_conf_files():
return [
"//heron/tools/config/src/yaml:tracker-yaml",
]
-
-def heron_tools_lib_files():
- return []
