Allowing API Server to distribute heron core package (#2724)
* Allowing API Server to distribute heron core package
* modifying endpoint
diff --git a/heron/config/src/yaml/conf/nomad/client.yaml b/heron/config/src/yaml/conf/nomad/client.yaml
index 6a50658..e51afc4 100644
--- a/heron/config/src/yaml/conf/nomad/client.yaml
+++ b/heron/config/src/yaml/conf/nomad/client.yaml
@@ -2,3 +2,7 @@
heron.package.core.uri: file://${HERON_DIST}/heron-core.tar.gz
heron.package.use_core_uri: true
+
+# Can use API Server to distribute core
+#heron.package.core.uri: http://localhost:9000/api/v1/file/download/core
+
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java
index 14de304..84a39a1 100644
--- a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/Runtime.java
@@ -55,7 +55,8 @@
Property("D"),
ReleaseFile("release-file"),
Verbose("verbose"),
- DownloadHostName("download-hostname");
+ DownloadHostName("download-hostname"),
+ HeronCorePackagePath("heron-core-package-path");
final String name;
@@ -128,6 +129,14 @@
.required(false)
.build();
+ final Option downloadHeronCoreName = Option.builder()
+ .desc("Path to Heron Core Package. API Server can serve Heron Core Package")
+ .longOpt(Flag.HeronCorePackagePath.name)
+ .hasArg()
+ .argName(Flag.HeronCorePackagePath.name)
+ .required(false)
+ .build();
+
return new Options()
.addOption(baseTemplate)
.addOption(cluster)
@@ -136,7 +145,8 @@
.addOption(release)
.addOption(property)
.addOption(verbose)
- .addOption(downloadHostName);
+ .addOption(downloadHostName)
+ .addOption(downloadHeronCoreName);
}
private static Options constructHelpOptions() {
@@ -204,6 +214,13 @@
return null;
}
+ private static String getHeronCorePackagePath(CommandLine cmd) {
+ if (cmd.hasOption(Flag.HeronCorePackagePath.name)) {
+ return String.valueOf(cmd.getOptionValue(Flag.HeronCorePackagePath.name));
+ }
+ return null;
+ }
+
private static String loadOverrides(CommandLine cmd) throws IOException {
return ConfigUtils.createOverrideConfiguration(
cmd.getOptionProperties(Flag.Property.name));
@@ -258,6 +275,7 @@
final String configurationOverrides = loadOverrides(cmd);
final int port = getPort(cmd);
final String downloadHostName = getDownloadHostName(cmd);
+ final String heronCorePackagePath = getHeronCorePackagePath(cmd);
final Config baseConfiguration =
ConfigUtils.getBaseConfiguration(heronDirectory,
@@ -265,6 +283,8 @@
releaseFile,
configurationOverrides);
+ LOG.info("heronCorePackagePath: " + heronCorePackagePath);
+
final ResourceConfig config = new ResourceConfig(Resources.get());
final Server server = new Server(port);
@@ -284,6 +304,8 @@
String.valueOf(port));
contextHandler.setAttribute(HeronResource.ATTRIBUTE_DOWNLOAD_HOSTNAME,
String.valueOf(downloadHostName));
+ contextHandler.setAttribute(HeronResource.ATTRIBUTE_HERON_CORE_PACKAGE_PATH,
+ String.valueOf(heronCorePackagePath));
server.setHandler(contextHandler);
@@ -292,8 +314,6 @@
contextHandler.addServlet(apiServlet, API_BASE_PATH);
-
-
try {
server.start();
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/FileResource.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/FileResource.java
index f2dce0c..79e32fd 100644
--- a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/FileResource.java
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/FileResource.java
@@ -18,10 +18,9 @@
import java.io.InputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.UUID;
+import javax.activation.MimetypesFileTypeMap;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@@ -29,7 +28,6 @@
import javax.ws.rs.PathParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import org.eclipse.jetty.util.StringUtil;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -124,7 +122,7 @@
*/
@GET
@Path("/download/{file}")
- public Response downloadPdfFile(final @PathParam("file") String file) {
+ public Response downloadFile(final @PathParam("file") String file) {
Config config = createConfig();
String uploadDir = config.getStringValue(FILE_SYSTEM_DIRECTORY);
String filePath = uploadDir + "/" + file;
@@ -132,19 +130,30 @@
LOG.debug("Download request file " + file + " doesn't exist at " + uploadDir);
return Response.status(Response.Status.NOT_FOUND).build();
}
- StreamingOutput fileStream = new StreamingOutput() {
- @Override
- public void write(java.io.OutputStream output) throws IOException {
- java.nio.file.Path path = Paths.get(filePath);
- byte[] data = Files.readAllBytes(path);
- output.write(data);
- output.flush();
- }
- };
- return Response
- .ok(fileStream, MediaType.APPLICATION_OCTET_STREAM)
- .header("content-disposition", "attachment; filename = " + file)
- .build();
+
+ String mimeType = new MimetypesFileTypeMap().getContentType(file);
+ Response.ResponseBuilder rb = Response.ok(file, mimeType);
+ rb.header("content-disposition", "attachment; filename = "
+ + file);
+ return rb.build();
+ }
+
+ /**
+ * Endpoint for downloading Heron Core
+ */
+ @GET
+ @Path("/download/core")
+ public Response downloadHeronCore() {
+ String corePath = getHeronCorePackagePath();
+ File file = new File(corePath);
+ if (!file.exists()) {
+ return Response.status(Response.Status.NOT_FOUND).build();
+ }
+ String mimeType = new MimetypesFileTypeMap().getContentType(file);
+ Response.ResponseBuilder rb = Response.ok(file, mimeType);
+ rb.header("content-disposition", "attachment; filename = "
+ + file.getName());
+ return rb.build();
}
private Config createConfig() {
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java
index 6ed1e66..7620b81 100644
--- a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/HeronResource.java
@@ -26,6 +26,7 @@
public static final String ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH = "configuration_override";
public static final String ATTRIBUTE_PORT = "port";
public static final String ATTRIBUTE_DOWNLOAD_HOSTNAME = "download_hostname";
+ public static final String ATTRIBUTE_HERON_CORE_PACKAGE_PATH = "heron_core_package_path";
@Context
protected ServletContext servletContext;
@@ -36,6 +37,7 @@
private String cluster;
private String port;
private String downloadHostname;
+ private String heronCorePackagePath;
Config getBaseConfiguration() {
if (baseConfiguration == null) {
@@ -85,4 +87,13 @@
return downloadHostname;
}
+ String getHeronCorePackagePath() {
+ if (heronCorePackagePath == null) {
+ heronCorePackagePath
+ = (String) servletContext.getAttribute(ATTRIBUTE_HERON_CORE_PACKAGE_PATH);
+ }
+
+ return heronCorePackagePath;
+ }
+
}
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java
index 63eb4d8..956b639 100644
--- a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/resources/NotFoundExceptionHandler.java
@@ -35,6 +35,7 @@
final ArrayNode arrayNode = node.putArray("paths");
arrayNode.add("/api/v1/topologies");
arrayNode.add("/api/v1/version");
+ arrayNode.add("/api/v1/file");
final String response;
try {
