Merge branch 'master' into 0.14.6
diff --git a/heron/common/src/java/com/twitter/heron/common/utils/logging/LoggingHelper.java b/heron/common/src/java/com/twitter/heron/common/utils/logging/LoggingHelper.java
index d4f76aa..b245eb0 100644
--- a/heron/common/src/java/com/twitter/heron/common/utils/logging/LoggingHelper.java
+++ b/heron/common/src/java/com/twitter/heron/common/utils/logging/LoggingHelper.java
@@ -34,7 +34,7 @@
*/
public final class LoggingHelper {
public static final String FORMAT_PROP_KEY = "java.util.logging.SimpleFormatter.format";
- public static final String DEFAULT_FORMAT = "[%1$tF %1$tT %1$tz] %3$s %4$s: %5$s %6$s %n";
+ public static final String DEFAULT_FORMAT = "[%1$tF %1$tT %1$tz] [%4$s] %3$s: %5$s %6$s %n";
private LoggingHelper() {
}
diff --git a/heron/common/src/python/utils/log.py b/heron/common/src/python/utils/log.py
index 01c5a96..533c698 100644
--- a/heron/common/src/python/utils/log.py
+++ b/heron/common/src/python/utils/log.py
@@ -24,9 +24,9 @@
# time formatter - date - time - UTC offset
# e.g. "08/16/1988 21:30:00 +1030"
# see time formatter documentation for more
-date_format = "%m/%d/%Y %H:%M:%S %z"
+date_format = "%Y-%m-%d %H:%M:%S %z"
-def configure(level=logging.INFO, logfile=None, with_time=False):
+def configure(level=logging.INFO, logfile=None):
""" Configure logger which dumps log on terminal
:param level: logging level: info, warning, verbose...
@@ -46,20 +46,14 @@
# if logfile is specified, FileHandler is used
if logfile is not None:
- if with_time:
- log_format = "%(asctime)s:%(levelname)s: %(message)s"
- else:
- log_format = "%(levelname)s: %(message)s"
+ log_format = "[%(asctime)s] [%(levelname)s]: %(message)s"
formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
file_handler = logging.FileHandler(logfile)
file_handler.setFormatter(formatter)
Log.addHandler(file_handler)
# otherwise, use StreamHandler to output to stream (stdout, stderr...)
else:
- if with_time:
- log_format = "%(log_color)s%(levelname)s:%(reset)s %(asctime)s %(message)s"
- else:
- log_format = "%(log_color)s%(levelname)s:%(reset)s %(message)s"
+ log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s"
# pylint: disable=redefined-variable-type
formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)
stream_handler = logging.StreamHandler()
@@ -76,7 +70,7 @@
logging.basicConfig()
root_logger = logging.getLogger()
- log_format = "%(asctime)s:%(levelname)s:%(filename)s: %(message)s"
+ log_format = "[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s"
root_logger.setLevel(level)
handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)
@@ -89,7 +83,7 @@
root_logger.debug("Removing StreamHandler: " + str(handler))
root_logger.handlers.remove(handler)
-def set_logging_level(cl_args, with_time=False):
+def set_logging_level(cl_args):
"""simply set verbose level based on command-line args
:param cl_args: CLI arguments
@@ -97,7 +91,7 @@
:return: None
:rtype: None
"""
- if cl_args['verbose']:
- configure(logging.DEBUG, with_time=with_time)
+ if 'verbose' in cl_args and cl_args['verbose']:
+ configure(logging.DEBUG)
else:
- configure(logging.INFO, with_time=with_time)
+ configure(logging.INFO)
diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py
index e70d388..4e58588 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -735,7 +735,7 @@
def setup(shardid):
# Redirect stdout and stderr to files in append mode
# The filename format is heron-executor-<container_id>.stdxxx
- log.configure(logfile='heron-executor-%s.stdout' % shardid, with_time=True)
+ log.configure(logfile='heron-executor-%s.stdout' % shardid)
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
index f95f656..5b68273 100644
--- a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/SubmitterMain.java
@@ -375,6 +375,14 @@
*
*/
public void submitTopology() throws TopologySubmissionException {
+ // build primary runtime config first
+ Config primaryRuntime = Config.newBuilder()
+ .putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
+ // call launcher directly here if in dry-run mode
+ if (Context.dryRun(config)) {
+ callLauncherRunner(primaryRuntime);
+ return;
+ }
// 1. Do prepare work
// create an instance of state manager
String statemgrClass = Context.stateManagerClass(config);
@@ -414,16 +422,6 @@
// Put it in a try block so that we can always clean resources
try {
- // Build the basic runtime config
- Config primaryRuntime = Config.newBuilder()
- .putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
-
- // Bypass validation and upload if in dry-run mode
- if (Context.dryRun(config)) {
- callLauncherRunner(primaryRuntime);
- return;
- }
-
// initialize the state manager
statemgr.initialize(config);
@@ -435,7 +433,6 @@
LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
- // First, create the basic runtime config to generate the packing plan
Config runtimeWithoutPackageURI = Config.newBuilder()
.putAll(primaryRuntime)
.putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor))
diff --git a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SchedulerUtils.java b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SchedulerUtils.java
index 3aab25b..78c0591 100644
--- a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SchedulerUtils.java
+++ b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/SchedulerUtils.java
@@ -281,7 +281,7 @@
Scheduler.SchedulerLocation location = builder.build();
- LOG.log(Level.INFO, "Setting SchedulerLocation: {0}", location);
+ LOG.log(Level.INFO, "Setting Scheduler locations: {0}", location);
SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
Boolean result =
statemgr.setSchedulerLocation(location, Runtime.topologyName(runtime));
diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/aurora/AuroraCLIController.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/aurora/AuroraCLIController.java
index c559704..6840720 100644
--- a/heron/schedulers/src/java/com/twitter/heron/scheduler/aurora/AuroraCLIController.java
+++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/aurora/AuroraCLIController.java
@@ -126,7 +126,7 @@
StringBuilder stdout = new StringBuilder();
StringBuilder stderr = new StringBuilder();
int status =
- ShellUtils.runProcess(auroraCmd.toArray(new String[auroraCmd.size()]), stdout, stderr);
+ ShellUtils.runProcess(auroraCmd.toArray(new String[auroraCmd.size()]), stderr);
if (status != 0) {
LOG.severe(String.format(
diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/slurm/SlurmController.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/slurm/SlurmController.java
index 3ee754d..acce7dc 100644
--- a/heron/schedulers/src/java/com/twitter/heron/scheduler/slurm/SlurmController.java
+++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/slurm/SlurmController.java
@@ -71,11 +71,8 @@
String[] slurmCmdArray = slurmCmd.toArray(new String[0]);
LOG.log(Level.INFO, "Executing job [" + topologyWorkingDirectory + "]:",
Arrays.toString(slurmCmdArray));
- StringBuilder stdout = new StringBuilder();
StringBuilder stderr = new StringBuilder();
- boolean ret = runProcess(topologyWorkingDirectory, slurmCmdArray, stdout, stderr);
- LOG.log(Level.FINE, "Stdout for Slurm script: ", stdout);
- LOG.log(Level.FINE, "Stderror for Slurm script: ", stderr);
+ boolean ret = runProcess(topologyWorkingDirectory, slurmCmdArray, stderr);
return ret;
}
@@ -124,9 +121,9 @@
* This is for unit testing
*/
protected boolean runProcess(String topologyWorkingDirectory, String[] slurmCmd,
- StringBuilder stdout, StringBuilder stderr) {
+ StringBuilder stderr) {
File file = topologyWorkingDirectory == null ? null : new File(topologyWorkingDirectory);
- return 0 == ShellUtils.runSyncProcess(isVerbose, false, slurmCmd, stdout, stderr, file);
+ return 0 == ShellUtils.runSyncProcess(false, slurmCmd, stderr, file);
}
/**
@@ -140,7 +137,7 @@
List<String> jobIdFileContent = readFromFile(jobIdFile);
if (jobIdFileContent.size() > 0) {
String[] slurmCmd = new String[]{"scancel", jobIdFileContent.get(0)};
- return runProcess(null, slurmCmd, new StringBuilder(), new StringBuilder());
+ return runProcess(null, slurmCmd, new StringBuilder());
} else {
LOG.log(Level.SEVERE, "Failed to read the Slurm Job id from file: {0}", jobIdFile);
return false;
diff --git a/heron/schedulers/tests/java/com/twitter/heron/scheduler/slurm/SlurmControllerTest.java b/heron/schedulers/tests/java/com/twitter/heron/scheduler/slurm/SlurmControllerTest.java
index a5ba594..0770456 100644
--- a/heron/schedulers/tests/java/com/twitter/heron/scheduler/slurm/SlurmControllerTest.java
+++ b/heron/schedulers/tests/java/com/twitter/heron/scheduler/slurm/SlurmControllerTest.java
@@ -65,15 +65,13 @@
// Failed
Mockito.doReturn(false).when(controller).runProcess(Matchers.anyString(),
- Matchers.any(String[].class), Matchers.any(StringBuilder.class),
- Matchers.any(StringBuilder.class));
+ Matchers.any(String[].class), Matchers.any(StringBuilder.class));
Assert.assertFalse(controller.createJob(slurmFileName, "slurm",
expectedCommand, WORKING_DIRECTORY, 1));
// Happy path
Mockito.doReturn(true).when(controller).runProcess(Matchers.anyString(),
- Matchers.any(String[].class), Matchers.any(StringBuilder.class),
- Matchers.any(StringBuilder.class));
+ Matchers.any(String[].class), Matchers.any(StringBuilder.class));
Assert.assertTrue(controller.createJob(slurmFileName, "slurm",
expectedCommand, WORKING_DIRECTORY, 1));
}
@@ -87,14 +85,12 @@
Assert.assertFalse(controller.killJob(jobIdFile));
// fail if process creation fails
Mockito.doReturn(false).when(controller).runProcess(Matchers.anyString(),
- Matchers.any(String[].class), Matchers.any(StringBuilder.class),
- Matchers.any(StringBuilder.class));
+ Matchers.any(String[].class), Matchers.any(StringBuilder.class));
Mockito.doReturn(jobIds).when(controller).readFromFile(jobIdFile);
Assert.assertFalse(controller.killJob(jobIdFile));
// happy path
Mockito.doReturn(true).when(controller).runProcess(Matchers.anyString(),
- Matchers.any(String[].class), Matchers.any(StringBuilder.class),
- Matchers.any(StringBuilder.class));
+ Matchers.any(String[].class), Matchers.any(StringBuilder.class));
Mockito.doReturn(jobIds).when(controller).readFromFile(jobIdFile);
Assert.assertTrue(controller.killJob(jobIdFile));
}
diff --git a/heron/spi/src/java/com/twitter/heron/spi/utils/ShellUtils.java b/heron/spi/src/java/com/twitter/heron/spi/utils/ShellUtils.java
index 94aa6d7..160e990 100644
--- a/heron/spi/src/java/com/twitter/heron/spi/utils/ShellUtils.java
+++ b/heron/spi/src/java/com/twitter/heron/spi/utils/ShellUtils.java
@@ -14,6 +14,7 @@
package com.twitter.heron.spi.utils;
+import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -21,7 +22,6 @@
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;
@@ -60,42 +60,64 @@
return builder.toString();
}
- public static int runProcess(String[] cmdline, StringBuilder stdout, StringBuilder stderr) {
- return runSyncProcess(false, false, cmdline, stdout, stderr, null);
+ public static int runProcess(String[] cmdline, StringBuilder outputBuilder) {
+ return runSyncProcess(false, cmdline, outputBuilder, null);
}
public static int runProcess(
- boolean verbose, String cmdline, StringBuilder stdout, StringBuilder stderr) {
- return runSyncProcess(verbose, false, splitTokens(cmdline), stdout, stderr, null);
+ String cmdline, StringBuilder outputBuilder) {
+ return runSyncProcess(false, splitTokens(cmdline), outputBuilder, null);
}
public static int runSyncProcess(
- boolean verbose, boolean isInheritIO, String[] cmdline, StringBuilder stdout,
- StringBuilder stderr, File workingDirectory) {
- return runSyncProcess(isInheritIO, cmdline, stdout, stderr, workingDirectory,
+ boolean isInheritIO, String[] cmdline, StringBuilder outputBuilder, File workingDirectory) {
+ return runSyncProcess(isInheritIO, cmdline, outputBuilder, workingDirectory,
new HashMap<String, String>());
}
/**
- * Start a daemon thread to read data from "input" to "out".
+ * Read and print line from input, and save each line in a string builder
+ * line read from input is printed to stderr directly instead of using LOG (which will
+ * add redundant timestamp and class name info).
+ *
+ * This method does not start the thread. Instead, caller should start this thread.
+ *
+ * @param input input stream
+ * @param processOutputStringBuilder string builder used to save input lines
+ * @return thread
*/
- private static Thread asyncProcessStream(final InputStream input, final StringBuilder out) {
+ private static Thread createAsyncStreamThread(final InputStream input,
+ final StringBuilder processOutputStringBuilder) {
Thread thread = new Thread() {
@Override
public void run() {
- try {
- out.append(inputstreamToString(input));
- } finally {
+ // do not buffer
+ LOG.log(Level.INFO, "Process output (stdout+stderr):");
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1);
+ while (true) {
+ String line = null;
try {
- input.close();
+ line = reader.readLine();
} catch (IOException e) {
- LOG.log(Level.WARNING, "Failed to close the input stream", e);
+ LOG.log(Level.SEVERE, "Error when reading line from subprocess", e);
}
+ if (line == null) {
+ break;
+ } else {
+ System.err.println(line);
+ if (processOutputStringBuilder != null) {
+ processOutputStringBuilder.append(line);
+ }
+ }
+ }
+ try {
+ input.close();
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Failed to close the input stream", e);
}
}
};
thread.setDaemon(true);
- thread.start();
return thread;
}
@@ -103,61 +125,51 @@
* run sync process
*/
private static int runSyncProcess(
- boolean isInheritIO, String[] cmdline, StringBuilder stdout,
- StringBuilder stderr, File workingDirectory, Map<String, String> envs) {
- final StringBuilder pStdOut = stdout == null ? new StringBuilder() : stdout;
- final StringBuilder pStdErr = stderr == null ? new StringBuilder() : stderr;
+ boolean isInheritIO, String[] cmdline,
+ StringBuilder outputBuilder, File workingDirectory, Map<String, String> envs) {
+ final StringBuilder builder = outputBuilder == null ? new StringBuilder() : outputBuilder;
// Log the command for debugging
- LOG.log(Level.FINE, "Process command: `$ {0}`", Arrays.toString(cmdline));
+ LOG.log(Level.INFO, "Running synced process: ``{0}''''", String.join(" ", cmdline));
ProcessBuilder pb = getProcessBuilder(isInheritIO, cmdline, workingDirectory, envs);
+ /* combine input stream and error stream into stderr because
+ 1. this preserves order of process's stdout/stderr message
+ 2. there is no need to distinguish stderr from stdout
+ 3. follow one basic pattern of the design of Python<~>Java I/O redirection:
+ stdout contains useful messages Java program needs to propagate back, stderr
+ contains all other information */
+ pb.redirectErrorStream(true);
Process process;
try {
process = pb.start();
} catch (IOException e) {
- LOG.log(Level.SEVERE, "Failed to run Sync Process ", e);
+ LOG.log(Level.SEVERE, "Failed to run synced process", e);
return -1;
}
- // Launching threads to consume stdout and stderr before "waitFor". Otherwise, output from the
- // "process" can exhaust the available buffer for the output or error stream because neither
- // stream is read while waiting for the process to complete. If either buffer becomes full, it
- // can block the "process" as well, preventing all progress for both the "process" and the
- // current thread.
- Thread stdoutThread = asyncProcessStream(process.getInputStream(), pStdOut);
- Thread stderrThread = asyncProcessStream(process.getErrorStream(), pStdErr);
-
- int exitValue;
+ // Launching threads to consume combined stdout and stderr before "waitFor".
+ // Otherwise, output from the "process" can exhaust the available buffer for the combined stream
+ // because stream is not read while waiting for the process to complete.
+ // If buffer becomes full, it can block the "process" as well,
+ // preventing all progress for both the "process" and the current thread.
+ Thread outputsThread = createAsyncStreamThread(process.getInputStream(), builder);
try {
- exitValue = process.waitFor();
- // Make sure `pStdOut` and `pStdErr` get the buffered data
- stdoutThread.join();
- stderrThread.join();
+ outputsThread.start();
+ int exitValue = process.waitFor();
+ outputsThread.join();
+ return exitValue;
} catch (InterruptedException e) {
// The current thread is interrupted, so try to interrupt reading threads and kill
// the process to return quickly.
- stdoutThread.interrupt();
- stderrThread.interrupt();
+ outputsThread.interrupt();
process.destroy();
- LOG.log(Level.SEVERE, "Running Sync Process was interrupted", e);
+ LOG.log(Level.SEVERE, "Running synced process was interrupted", e);
// Reset the interrupt status to allow other codes noticing it.
Thread.currentThread().interrupt();
return -1;
}
-
- String stdoutString = pStdOut.toString();
- String stderrString = pStdErr.toString();
- if (!stdoutString.isEmpty()) {
- LOG.log(Level.FINE, "\nSTDOUT:\n {0}", stdoutString);
- }
-
- if (!stderrString.isEmpty()) {
- LOG.log(Level.FINE, "\nSTDERR:\n {0}", stderrString);
- }
-
- return exitValue;
}
public static Process runASyncProcess(
@@ -183,7 +195,7 @@
private static Process runASyncProcess(String[] command, File workingDirectory,
Map<String, String> envs, String logFileUuid, boolean logStderr) {
- LOG.log(Level.FINE, "$> {0}", Arrays.toString(command));
+ LOG.log(Level.INFO, "Running async process: ``{0}''''", String.join(" ", command));
// the log file can help people to find out what happened between pb.start()
// and the async process started
@@ -209,7 +221,7 @@
try {
process = pb.start();
} catch (IOException e) {
- LOG.log(Level.SEVERE, "Failed to run Async Process ", e);
+ LOG.log(Level.SEVERE, "Failed to run async process", e);
}
return process;
@@ -259,6 +271,7 @@
}
static Process establishSocksProxyProcess(String proxyHost, int proxyPort) {
+ LOG.info(String.format("Establishing SOCKS proxy to: %s:%d", proxyHost, proxyPort));
return ShellUtils.runASyncProcess(String.format("ssh -ND %d %s", proxyPort, proxyHost));
}
@@ -278,8 +291,8 @@
// using curl copy the url to the target file
String cmd = String.format("curl %s -o %s", uri, destination);
- int ret = runSyncProcess(isVerbose, isInheritIO,
- splitTokens(cmd), new StringBuilder(), new StringBuilder(), parentDirectory);
+ int ret = runSyncProcess(isInheritIO,
+ splitTokens(cmd), new StringBuilder(), parentDirectory);
return ret == 0;
}
@@ -296,8 +309,8 @@
String packageName, String targetFolder, boolean isVerbose, boolean isInheritIO) {
String cmd = String.format("tar -xvf %s", packageName);
- int ret = runSyncProcess(isVerbose, isInheritIO,
- splitTokens(cmd), new StringBuilder(), new StringBuilder(), new File(targetFolder));
+ int ret = runSyncProcess(isInheritIO,
+ splitTokens(cmd), new StringBuilder(), new File(targetFolder));
return ret == 0;
}
diff --git a/heron/spi/src/java/com/twitter/heron/spi/utils/TopologyUtils.java b/heron/spi/src/java/com/twitter/heron/spi/utils/TopologyUtils.java
index 96596ee..c37bb3c 100644
--- a/heron/spi/src/java/com/twitter/heron/spi/utils/TopologyUtils.java
+++ b/heron/spi/src/java/com/twitter/heron/spi/utils/TopologyUtils.java
@@ -50,7 +50,7 @@
return topology;
} catch (IOException e) {
- throw new RuntimeException("Failed to read/parse content of " + topologyDefnFile);
+ throw new RuntimeException("Failed to read/parse content of " + topologyDefnFile, e);
}
}
diff --git a/heron/spi/tests/java/com/twitter/heron/spi/utils/ShellUtilsTest.java b/heron/spi/tests/java/com/twitter/heron/spi/utils/ShellUtilsTest.java
index f727392..474fe50 100644
--- a/heron/spi/tests/java/com/twitter/heron/spi/utils/ShellUtilsTest.java
+++ b/heron/spi/tests/java/com/twitter/heron/spi/utils/ShellUtilsTest.java
@@ -53,11 +53,10 @@
@Test
public void testRunProcess() {
String testString = "testString";
- StringBuilder stdout = new StringBuilder();
StringBuilder stderr = new StringBuilder();
- Assert.assertEquals(0, ShellUtils.runProcess(true, "echo " + testString, stdout, stderr));
- Assert.assertEquals(testString, stdout.toString().trim());
- Assert.assertTrue(stderr.toString().trim().isEmpty());
+ Assert.assertEquals(0, ShellUtils.runProcess("echo " + testString, stderr));
+ Assert.assertEquals(testString, stderr.toString().trim());
+ Assert.assertTrue(!stderr.toString().trim().isEmpty());
}
@Test(timeout = 60000)
@@ -73,13 +72,11 @@
input.close();
}
Assert.assertTrue("Cannot make the test script executable", testScript.setExecutable(true));
- StringBuilder stdout = new StringBuilder();
StringBuilder stderr = new StringBuilder();
Assert.assertEquals(0,
- ShellUtils.runProcess(true,
- "/bin/bash -c " + testScript.getAbsolutePath(), stdout, stderr));
+ ShellUtils.runProcess(
+ "/bin/bash -c " + testScript.getAbsolutePath(), stderr));
// Only checks stdout and stderr are not empty. Correctness is checked in "testRunProcess".
- Assert.assertTrue(!stdout.toString().trim().isEmpty());
Assert.assertTrue(!stderr.toString().trim().isEmpty());
} finally {
testScript.delete();
diff --git a/heron/statemgrs/src/java/com/twitter/heron/statemgr/zookeeper/curator/CuratorStateManager.java b/heron/statemgrs/src/java/com/twitter/heron/statemgr/zookeeper/curator/CuratorStateManager.java
index eb774b7..e699fcb 100644
--- a/heron/statemgrs/src/java/com/twitter/heron/statemgr/zookeeper/curator/CuratorStateManager.java
+++ b/heron/statemgrs/src/java/com/twitter/heron/statemgr/zookeeper/curator/CuratorStateManager.java
@@ -79,8 +79,8 @@
String newConnectionString = tunneledResults.first;
if (newConnectionString.isEmpty()) {
- throw new IllegalArgumentException("Cannot connect to tunnel host: "
- + tunnelConfig.getTunnelHost());
+ throw new IllegalArgumentException("Failed to connect to tunnel host '"
+ + tunnelConfig.getTunnelHost() + "'");
}
// Use the new connection string
@@ -90,7 +90,7 @@
// Start it
client = getCuratorClient();
- LOG.info("Starting client to: " + connectionString);
+ LOG.info("Starting Curator client connecting to: " + connectionString);
client.start();
try {
@@ -170,7 +170,7 @@
protected void initTree() {
// Make necessary directories
for (StateLocation location : StateLocation.values()) {
- LOG.info(String.format("%s directory: %s", location.getName(), getStateDirectory(location)));
+ LOG.fine(String.format("%s directory: %s", location.getName(), getStateDirectory(location)));
}
try {
@@ -196,8 +196,10 @@
// Close the tunneling
LOG.info("Closing the tunnel processes");
- for (Process process : tunnelProcesses) {
- process.destroy();
+ if (tunnelProcesses != null) {
+ for (Process process : tunnelProcesses) {
+ process.destroy();
+ }
}
}
diff --git a/heron/tools/cli/src/python/BUILD b/heron/tools/cli/src/python/BUILD
index 846b05a..558e206 100644
--- a/heron/tools/cli/src/python/BUILD
+++ b/heron/tools/cli/src/python/BUILD
@@ -13,7 +13,10 @@
"//heron/tools/common/src/python:common-py",
"//heron/proto:proto-py",
],
- reqs = ["pyyaml==3.10"],
+ reqs = [
+ "pyyaml==3.10",
+ "enum34==1.1.6"
+ ],
)
pex_binary(
diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py
index c3eff55..c47cb60 100644
--- a/heron/tools/cli/src/python/cli_helper.py
+++ b/heron/tools/cli/src/python/cli_helper.py
@@ -77,7 +77,7 @@
new_args.append("--verbose")
# invoke the runtime manager to kill the topology
- resp = execute.heron_class(
+ result = execute.heron_class(
'com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
@@ -86,5 +86,5 @@
err_msg = "Failed to %s %s" % (action, topology_name)
succ_msg = "Successfully %s %s" % (action, topology_name)
- resp.add_context(err_msg, succ_msg)
- return resp
+ result.add_context(err_msg, succ_msg)
+ return result
diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py
index 9343642..6136393 100644
--- a/heron/tools/cli/src/python/execute.py
+++ b/heron/tools/cli/src/python/execute.py
@@ -20,7 +20,7 @@
import traceback
from heron.common.src.python.utils.log import Log
-from heron.tools.cli.src.python.response import Response, Status
+from heron.tools.cli.src.python.result import SimpleResult, ProcessResult, Status
import heron.common.src.python.pex_loader as pex_loader
import heron.tools.cli.src.python.opts as opts
import heron.tools.cli.src.python.jars as jars
@@ -67,17 +67,11 @@
Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
- proc = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ proc = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, bufsize=1)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
- msg, detailed_msg = proc.communicate()
- # remove trailing newlines
- if msg and msg[-1] == '\n':
- msg = msg[:-1]
- if detailed_msg and detailed_msg[-1] == '\n':
- detailed_msg = detailed_msg[:-1]
- Log.debug("shelled-out program's return code: %d", proc.returncode)
- return Response(proc.returncode, msg, detailed_msg)
+ return ProcessResult(proc)
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
'''
@@ -118,23 +112,16 @@
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
-
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
-
# invoke the command with subprocess and print error message, if any
- proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
- proc.communicate()
- retcode = proc.returncode
- if retcode != 0:
- err_str = "Topology class %s failed to be loaded from the given pex" % topology_class_name
- return Response(retcode, err_str, None)
- else:
- return Response(retcode)
+ return ProcessResult(proc)
else:
try:
# loading topology from Topology's subclass (no main method)
@@ -147,8 +134,9 @@
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
- return Response(Status.Ok)
+ return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
- err_str = "Topology %s failed to be loaded from the given pex" % topology_class_name
- return Response(Status.HeronError, err_str, str(ex))
+ err_context = "Topology %s failed to be loaded from the given pex: %s" %\
+ (topology_class_name, ex)
+ return SimpleResult(Status.HeronError, err_context)
diff --git a/heron/tools/cli/src/python/help.py b/heron/tools/cli/src/python/help.py
index f923273..b9b04d3 100644
--- a/heron/tools/cli/src/python/help.py
+++ b/heron/tools/cli/src/python/help.py
@@ -13,7 +13,7 @@
# limitations under the License.
''' help.py '''
from heron.common.src.python.utils.log import Log
-from heron.tools.cli.src.python.response import Response, Status
+from heron.tools.cli.src.python.result import SimpleResult, Status
import heron.tools.common.src.python.utils.config as config
def create_parser(subparsers):
@@ -55,13 +55,13 @@
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
- return Response(Status.Ok)
+ return SimpleResult(Status.Ok)
# get the subparser for the specific command
subparser = config.get_subparser(parser, command_help)
if subparser:
print subparser.format_help()
- return Response(Status.Ok)
+ return SimpleResult(Status.Ok)
else:
Log.error("Unknown subcommand \'%s\'", command_help)
- return Response(Status.InvocationError)
+ return SimpleResult(Status.InvocationError)
diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py
index 7160df9..b665620 100644
--- a/heron/tools/cli/src/python/main.py
+++ b/heron/tools/cli/src/python/main.py
@@ -29,12 +29,14 @@
import heron.tools.cli.src.python.activate as activate
import heron.tools.cli.src.python.deactivate as deactivate
import heron.tools.cli.src.python.kill as kill
-import heron.tools.cli.src.python.response as response
+import heron.tools.cli.src.python.result as result
import heron.tools.cli.src.python.restart as restart
import heron.tools.cli.src.python.submit as submit
import heron.tools.cli.src.python.update as update
import heron.tools.cli.src.python.version as version
+from heron.tools.cli.src.python.opts import cleaned_up_files
+
Log = log.Log
HELP_EPILOG = '''Getting more help:
@@ -116,8 +118,8 @@
if command in runners:
return runners[command].run(command, parser, command_args, unknown_args)
else:
- detailed_msg = 'Unknown subcommand: %s' % command
- return response.Response(response.Status.InvocationError, detailed_msg=detailed_msg)
+ err_context = 'Unknown subcommand: %s' % command
+ return result.SimpleResult(result.Status.InvocationError, err_context)
def cleanup(files):
'''
@@ -125,7 +127,10 @@
:return:
'''
for cur_file in files:
- shutil.rmtree(os.path.dirname(cur_file))
+ if os.path.isdir(cur_file):
+ shutil.rmtree(cur_file)
+ else:
+ shutil.rmtree(os.path.dirname(cur_file))
################################################################################
@@ -215,9 +220,6 @@
# command to be execute
command = command_line_args['subcommand']
- # file resources to be cleaned when exit
- files = []
-
if command not in ('help', 'version'):
log.set_logging_level(command_line_args)
command_line_args = extract_common_args(command, parser, command_line_args)
@@ -225,26 +227,24 @@
if not command_line_args:
return 1
# register dirs cleanup function during exit
- files.append(command_line_args['override_config_file'])
+ cleaned_up_files.append(command_line_args['override_config_file'])
- atexit.register(cleanup, files)
+ atexit.register(cleanup, cleaned_up_files)
# print the input parameters, if verbose is enabled
Log.debug(command_line_args)
start = time.time()
- resp = run(command, parser, command_line_args, unknown_args)
+ results = run(command, parser, command_line_args, unknown_args)
if command not in ('help', 'version'):
- response.render(resp)
+ result.render(results)
end = time.time()
if command not in ('help', 'version'):
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.', (end - start))
- return 0 if response.isAllSuccessful(resp) else 1
- else:
- return 0 if resp else 1
+ return 0 if result.isAllSuccessful(results) else 1
if __name__ == "__main__":
sys.exit(main())
diff --git a/heron/tools/cli/src/python/opts.py b/heron/tools/cli/src/python/opts.py
index 309ed55..edb2c30 100644
--- a/heron/tools/cli/src/python/opts.py
+++ b/heron/tools/cli/src/python/opts.py
@@ -21,6 +21,8 @@
verbose_flag = False
trace_execution_flag = False
+cleaned_up_files = []
+
################################################################################
def get_heron_config():
diff --git a/heron/tools/cli/src/python/response.py b/heron/tools/cli/src/python/response.py
deleted file mode 100644
index 1196094..0000000
--- a/heron/tools/cli/src/python/response.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2016 Twitter. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-'''response.py'''
-from heron.common.src.python.utils.log import Log
-
-# Meaning of exit status code:
-# - status code = 0:
-# program exits without error
-# - 0 < status code < 100:
-# program fails to execute before program execution. For example,
-# JVM cannot find or load main class
-# - 100 <= status code < 200:
-# program fails to launch after program execution. For example,
-# topology definition file fails to be loaded
-# - status code == 200:
-# program sends out dry-run response
-
-# Definition corresponds to definition in com.twitter.heron.scheduler.AbstractMain
-
-# pylint: disable=no-init
-class Status(object):
- """Status code enum"""
- Ok = 0
- InvocationError = 1
- HeronError = 100
- DryRun = 200
-
-def status_type(status_code):
- if status_code == 0:
- return Status.Ok
- elif status_code < 100:
- return Status.InvocationError
- elif status_code == 200:
- return Status.DryRun
- else:
- return Status.HeronError
-
-class Response(object):
- """Response class that captures result of executing an action
-
- If the response object is generated by a statement that
- shells out program, `msg` is stdout, `detailed_msg` is stderr
- """
- def __init__(self, status_code, msg=None, detailed_msg=None):
- self.status = status_type(status_code)
- self.msg = msg
- self.detailed_msg = detailed_msg
- self.err_context = None
- self.succ_context = None
-
- def add_context(self, err_context, succ_context=None):
- """ Prepend msg to add some context information
-
- :param pmsg: context info
- :return: None
- """
- self.err_context = err_context
- self.succ_context = succ_context
-
-def render(resp):
- def do_log(log, msg):
- if msg:
- log(msg)
- if isinstance(resp, list):
- for r in resp:
- render(r)
- elif isinstance(resp, Response):
- if resp.status == Status.Ok:
- do_log(Log.info, resp.succ_context)
- do_log(Log.info, resp.msg)
- do_log(Log.debug, resp.detailed_msg)
- elif resp.status == Status.HeronError:
- do_log(Log.error, resp.err_context)
- do_log(Log.error, resp.msg)
- do_log(Log.debug, resp.detailed_msg)
- # If status code is InvocationError, invocation of shelled-out program fails. The error
- # message will be in stderr, so we log.error detailed message(stderr) only
- elif resp.status == Status.InvocationError:
- do_log(Log.error, resp.detailed_msg)
- elif resp.status == Status.DryRun:
- do_log(Log.info, resp.succ_context)
- # No need to prefix [INFO] here. We want to display dry-run response in a clean way
- print resp.msg
- do_log(Log.debug, resp.detailed_msg)
- else:
- raise RuntimeError("Unknown status type of value %d", resp.status)
- else:
- raise RuntimeError("Unknown response instance: %s", str(resp.__class__))
-
-# check if all responses are successful
-def isAllSuccessful(resps):
- if isinstance(resps, list):
- return all([resp.status == Status.Ok for resp in resps])
- elif isinstance(resps, Response):
- return resps.status == Status.Ok
- else:
- raise RuntimeError("Unknown response instance: %s", str(resps.__class__))
diff --git a/heron/tools/cli/src/python/result.py b/heron/tools/cli/src/python/result.py
new file mode 100644
index 0000000..e1643dc
--- /dev/null
+++ b/heron/tools/cli/src/python/result.py
@@ -0,0 +1,188 @@
+# Copyright 2016 Twitter. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''result.py'''
+import abc
+import sys
+from enum import Enum
+
+from heron.common.src.python.utils.log import Log
+
+# Meaning of exit status code:
+# - status code = 0:
+# program exits without error
+# - 0 < status code < 100:
+# program fails to execute before program execution. For example,
+# JVM cannot find or load main class
+# - 100 <= status code < 200:
+# program fails to launch after program execution. For example,
+# topology definition file fails to be loaded
+# - status code == 200:
+# program sends out dry-run response
+
+# Definition corresponds to definition in com.twitter.heron.scheduler.AbstractMain
+
+# pylint: disable=no-init
+class Status(Enum):
+ """Status code enum"""
+ Ok = 0
+ InvocationError = 1
+ HeronError = 100
+ DryRun = 200
+
+def status_type(status_code):
+ if status_code == 0:
+ return Status.Ok
+ elif status_code < 100:
+ return Status.InvocationError
+ elif status_code == 200:
+ return Status.DryRun
+ else:
+ return Status.HeronError
+
+class Result(object):
+ """Result class"""
+ def __init__(self, status=None, err_context=None, succ_context=None):
+ self.status = status
+ self.err_context = err_context
+ self.succ_context = succ_context
+
+ @staticmethod
+ def _do_log(log_f, msg):
+ if msg:
+ if msg[-1] == '\n':
+ msg = msg[:-1]
+ log_f(msg)
+
+ def _log_context(self):
+ # render context only after process exits
+ assert self.status is not None
+ if self.status == Status.Ok or self.status == Status.DryRun:
+ self._do_log(Log.info, self.succ_context)
+ elif self.status == Status.HeronError:
+ self._do_log(Log.error, self.err_context)
+ elif self.status == Status.InvocationError:
+ # invocation error has no context
+ pass
+ else:
+ raise RuntimeError(
+ "Unknown status type of value %d. Expected value: %s", self.status.value, list(Status))
+
+ def add_context(self, err_context, succ_context=None):
+ """ Prepend msg to add some context information
+
+ :param pmsg: context info
+ :return: None
+ """
+ self.err_context = err_context
+ self.succ_context = succ_context
+
+ def is_successful(self):
+ return self.status == Status.Ok
+
+ @abc.abstractmethod
+ def render(self):
+ pass
+
+
+class SimpleResult(Result):
+ """Simple result: result that already and only
+ contains status of the result"""
+ def __init__(self, status):
+ super(SimpleResult, self).__init__(status)
+
+ def render(self):
+ self._log_context()
+
+
+class ProcessResult(Result):
+ """Process result: a wrapper of result class"""
+ def __init__(self, proc):
+ super(ProcessResult, self).__init__()
+ self.proc = proc
+
+ def renderProcessStdErr(self, stderr_line):
+ """ render stderr of shelled-out process
+ stderr could be error message of failure of invoking process or
+ normal stderr output from successfully shelled-out process.
+ In the first case, ``Popen'' should fail fast and we should be able to
+ get return code immediately. We then render the failure message.
+ In the second case, we simply print stderr line in stderr.
+ The way to handle the first case is shaky but should be the best we can
+ do since we have conflicts of design goals here.
+ :param stderr_line: one line from shelled-out process
+ :return:
+ """
+ retcode = self.proc.poll()
+ if retcode is not None and status_type(retcode) == Status.InvocationError:
+ self._do_log(Log.error, stderr_line)
+ else:
+ print >> sys.stderr, stderr_line,
+
+ def renderProcessStdOut(self, stdout):
+ """ render stdout of shelled-out process
+ stdout always contains information Java process wants to
+ propagate back to cli, so we do special rendering here
+ :param stdout: all lines from shelled-out process
+ :return:
+ """
+ # since we render stdout line based on Java process return code,
+ # ``status'' has to be already set
+ assert self.status is not None
+ # remove pending newline
+ if self.status == Status.Ok:
+ self._do_log(Log.info, stdout)
+ elif self.status == Status.HeronError:
+ # remove last newline since logging will append newline
+ self._do_log(Log.error, stdout)
+ # No need to prefix [INFO] here. We want to display dry-run response in a clean way
+ elif self.status == Status.DryRun:
+ print >> sys.stdout, stdout,
+ else:
+ raise RuntimeError(
+ "Unknown status type of value %d. Expected value: %s", self.status.value, list(Status))
+
+ def render(self):
+ while True:
+ stderr_line = self.proc.stderr.readline()
+ if not stderr_line:
+ if self.proc.poll() is None:
+ continue
+ else:
+ break
+ else:
+ self.renderProcessStdErr(stderr_line)
+ self.proc.wait()
+ self.status = status_type(self.proc.returncode)
+ stdout = "".join(self.proc.stdout.readlines())
+ self.renderProcessStdOut(stdout)
+ self._log_context()
+
+
+def render(results):
+ if isinstance(results, Result):
+ results.render()
+ elif isinstance(results, list):
+ for r in results:
+ r.render()
+ else:
+ raise RuntimeError("Unknown result instance: %s", str(results.__class__))
+
+# check if all results are successful
+def isAllSuccessful(results):
+ if isinstance(results, list):
+ return all([result.is_successful() for result in results])
+ elif isinstance(results, Result):
+ return results.is_successful()
+ else:
+ raise RuntimeError("Unknown result instance: %s", str(results.__class__))
diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py
index da2bb37..9d0ba76 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -15,16 +15,16 @@
import glob
import logging
import os
-import shutil
import tempfile
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
-from heron.tools.cli.src.python.response import Response, Status
+from heron.tools.cli.src.python.result import SimpleResult, Status
import heron.tools.cli.src.python.args as cli_args
import heron.tools.cli.src.python.execute as execute
import heron.tools.cli.src.python.jars as jars
import heron.tools.cli.src.python.opts as opts
+import heron.tools.cli.src.python.result as result
import heron.tools.common.src.python.utils.config as config
import heron.tools.common.src.python.utils.classpath as classpath
@@ -112,7 +112,7 @@
# invoke the submitter to submit and launch the topology
main_class = 'com.twitter.heron.scheduler.SubmitterMain'
- resp = execute.heron_class(
+ res = execute.heron_class(
class_name=main_class,
lib_jars=lib_jars,
extra_jars=extra_jars,
@@ -124,8 +124,8 @@
succ_context = "Successfully launched topology '%s'" % topology_name
if cl_args["dry_run"]:
succ_context += " in dry-run mode"
- resp.add_context(err_context, succ_context)
- return resp
+ res.add_context(err_context, succ_context)
+ return res
################################################################################
def launch_topologies(cl_args, topology_file, tmp_dir):
@@ -140,9 +140,9 @@
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
- return Response(Status.HeronError, "No topologies found under %s" % tmp_dir)
+ return SimpleResult(Status.HeronError, "No topologies found under %s" % tmp_dir)
- responses = []
+ results = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
@@ -151,15 +151,14 @@
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
- msg = "Cannot load topology definition '%s'" % defn_file
- return Response(Status.HeronError, msg, str(e))
-
+ err_context = "Cannot load topology definition '%s': %s" % (defn_file, e)
+ return SimpleResult(Status.HeronError, err_context)
# launch the topology
Log.info("Launching topology: \'%s\'", topology_defn.name)
- resp = launch_a_topology(
+ res = launch_a_topology(
cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)
- responses.append(resp)
- return responses
+ results.append(res)
+ return results
################################################################################
@@ -183,23 +182,25 @@
topology_file = cl_args['topology-file-name']
main_class = cl_args['topology-class-name']
- resp = execute.heron_class(
+
+ res = execute.heron_class(
class_name=main_class,
lib_jars=config.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
- if resp.status != Status.Ok:
+ result.render(res)
+
+ if not res.is_successful():
err_context = "Failed to create topology definition \
file when executing class '%s' of file '%s'" % (main_class, topology_file)
- resp.add_context(err_context)
- return resp
+ res.add_context(err_context)
+ return res
- responses = launch_topologies(cl_args, topology_file, tmp_dir)
- shutil.rmtree(tmp_dir)
+ results = launch_topologies(cl_args, topology_file, tmp_dir)
- return responses
+ return results
################################################################################
@@ -226,23 +227,22 @@
topology_file = cl_args['topology-file-name']
java_defines = cl_args['topology_main_jvm_property']
main_class = cl_args['topology-class-name']
- resp = execute.heron_tar(
+ res = execute.heron_tar(
main_class,
topology_file,
tuple(unknown_args),
tmp_dir,
java_defines)
- if resp.status != Status.Ok:
+ res.render()
+
+ if not res.is_successful():
err_context = "Failed to create topology definition \
file when executing class '%s' of file '%s'" % (main_class, topology_file)
- resp.add_context(err_context)
- return resp
+ res.add_context(err_context)
+ return res
- responses = launch_topologies(cl_args, topology_file, tmp_dir)
- shutil.rmtree(tmp_dir)
-
- return responses
+ return launch_topologies(cl_args, topology_file, tmp_dir)
################################################################################
# Execute the pex file to create topology definition file by running
@@ -253,18 +253,18 @@
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
topology_class_name = cl_args['topology-class-name']
- resp = execute.heron_pex(
+ res = execute.heron_pex(
topology_file, topology_class_name, tuple(unknown_args))
- if resp.status != Status.Ok:
+
+ res.render()
+
+ if not res.is_successful():
err_context = "Failed to create topology definition \
file when executing class '%s' of file '%s'" % (topology_class_name, topology_file)
- resp.add_context(err_context)
- return resp
+ res.add_context(err_context)
+ return res
- responses = launch_topologies(cl_args, topology_file, tmp_dir)
- shutil.rmtree(tmp_dir)
-
- return responses
+ return launch_topologies(cl_args, topology_file, tmp_dir)
################################################################################
# pylint: disable=unused-argument
@@ -287,8 +287,8 @@
# check to see if the topology file exists
if not os.path.isfile(topology_file):
- err_msg = "Topology jar|tar|pex file '%s' does not exist" % topology_file
- return Response(Status.InvocationError, detailed_msg=err_msg)
+ err_context = "Topology jar|tar|pex file '%s' does not exist" % topology_file
+ return SimpleResult(Status.InvocationError, err_context)
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
@@ -296,19 +296,21 @@
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
ext_name = os.path.splitext(topology_file)
- err_msg = "Unknown file type '%s'. Please use .tar or .tar.gz or .jar or .pex file" % ext_name
- return Response(Status.InvocationError, detailed_msg=err_msg)
+ err_context = "Unknown file type '%s'. Please use .tar or .tar.gz or .jar or .pex file"\
+ % ext_name
+ return SimpleResult(Status.InvocationError, err_context)
# check if extra launch classpath is provided and if it is validate
if cl_args['extra_launch_classpath']:
valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath'])
if not valid_classpath:
- err_msg = "One of jar or directory in extra launch classpath does not exist: %s" % \
+ err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \
cl_args['extra_launch_classpath']
- return Response(Status.InvocationError, detailed_msg=err_msg)
+ return SimpleResult(Status.InvocationError, err_context)
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
+ opts.cleaned_up_files.append(tmp_dir)
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
@@ -323,10 +325,7 @@
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
-
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
-
else:
return submit_pex(cl_args, unknown_args, tmp_dir)
-
diff --git a/heron/tools/cli/src/python/version.py b/heron/tools/cli/src/python/version.py
index 01030ae..8cf487d 100644
--- a/heron/tools/cli/src/python/version.py
+++ b/heron/tools/cli/src/python/version.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
''' version.py '''
-from heron.tools.cli.src.python.response import Response, Status
+from heron.tools.cli.src.python.result import SimpleResult, Status
import heron.tools.cli.src.python.args as cli_args
import heron.tools.common.src.python.utils.config as config
@@ -44,4 +44,4 @@
:return:
'''
config.print_build_info()
- return Response(Status.Ok)
+ return SimpleResult(Status.Ok)
diff --git a/heron/tools/tracker/src/python/main.py b/heron/tools/tracker/src/python/main.py
index 363fbf2..fb9a93d 100644
--- a/heron/tools/tracker/src/python/main.py
+++ b/heron/tools/tracker/src/python/main.py
@@ -197,7 +197,7 @@
namespace = vars(args)
- log.set_logging_level(namespace, with_time=True)
+ log.set_logging_level(namespace)
# set Tornado global option
define_options(namespace['port'], namespace['config_file'])
diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py
index 703a475..02857a4 100644
--- a/heron/tools/ui/src/python/main.py
+++ b/heron/tools/ui/src/python/main.py
@@ -125,7 +125,7 @@
:param argv:
:return:
'''
- log.configure(logging.DEBUG, with_time=True)
+ log.configure(logging.DEBUG)
tornado.log.enable_pretty_logging()
# create the parser and parse the arguments
diff --git a/heron/uploaders/src/java/com/twitter/heron/uploader/hdfs/HdfsController.java b/heron/uploaders/src/java/com/twitter/heron/uploader/hdfs/HdfsController.java
index 3058b72..2b5bd8a 100644
--- a/heron/uploaders/src/java/com/twitter/heron/uploader/hdfs/HdfsController.java
+++ b/heron/uploaders/src/java/com/twitter/heron/uploader/hdfs/HdfsController.java
@@ -27,22 +27,22 @@
public boolean exists(String filePath) {
String command = String.format("hadoop --config %s fs -test -e %s", configDir, filePath);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
public boolean mkdirs(String dir) {
String command = String.format("hadoop --config %s fs -mkdir -p %s", configDir, dir);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
public boolean copyFromLocalFile(String source, String destination) {
String command = String.format("hadoop --config %s fs -copyFromLocal -f %s %s",
configDir, source, destination);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
public boolean delete(String filePath) {
String command = String.format("hadoop --config %s fs -rm %s", configDir, filePath);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
}
diff --git a/heron/uploaders/src/java/com/twitter/heron/uploader/scp/ScpController.java b/heron/uploaders/src/java/com/twitter/heron/uploader/scp/ScpController.java
index 989bf48..16a22ec 100644
--- a/heron/uploaders/src/java/com/twitter/heron/uploader/scp/ScpController.java
+++ b/heron/uploaders/src/java/com/twitter/heron/uploader/scp/ScpController.java
@@ -41,7 +41,7 @@
// an example ssh command created by the format looks like this:
// ssh -i ~/.ssh/id_rsa -p 23 user@example.com mkdir -p /heron/repository/...
String command = String.format("ssh %s %s mkdir -p %s", sshOptions, sshConnection, dir);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
public boolean copyFromLocalFile(String source, String destination) {
@@ -49,11 +49,11 @@
// scp -i ~/.ssh/id_rsa -p 23 ./foo.tar.gz user@example.com:/heron/foo.tar.gz
String command =
String.format("scp %s %s %s:%s", scpOptions, source, scpConnection, destination);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
public boolean delete(String filePath) {
String command = String.format("ssh %s %s rm -rf %s", sshOptions, sshConnection, filePath);
- return 0 == ShellUtils.runProcess(isVerbose, command, null, null);
+ return 0 == ShellUtils.runProcess(command, null);
}
}
diff --git a/scripts/centos/BUILD b/scripts/centos/BUILD
deleted file mode 100644
index b3229e7..0000000
--- a/scripts/centos/BUILD
+++ /dev/null
@@ -1,491 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-load("/tools/rules/pex_rules", "pex_binary")
-
-release_version = "unversioned"
-
-filegroup(
- name = "tarpkgs",
- srcs = [
- ":heron-api",
- ":heron-core",
- ":heron-client",
- ":heron-tools",
- ],
-)
-
-filegroup(
- name = "binpkgs",
- srcs = [
- ":heron-api-install.sh",
- ":heron-client-install.sh",
- ":heron-tools-install.sh",
- ],
-)
-
-################################################################################
-# Heron core packaging
-################################################################################
-genrule(
- name = "heron-core",
- srcs = [
- ":release.yaml",
- ":hexecutor",
- ":hshell",
- ":hstmgr",
- ":htmaster",
- ":hpyinstance",
- ":hscheduler",
- ":hscheduler-aurora",
- ":hscheduler-local",
- ":hbppacking",
- ":hrrpacking",
- ":hmetricsmgr",
- ":hlfsstatemgr",
- ":hzkstatemgr",
- ":hinstance",
- ],
- outs = [
- "heron-core.tar.gz",
- ],
- cmd = " ".join([
- "export GENDIR=$(GENDIR);",
- "export BINDIR=$(BINDIR);",
- "$(location package_release.sh) $(location heron-core.tar.gz)",
- "--cp $(location release.yaml) heron-core/release.yaml",
- "--cp $(location hexecutor) heron-core/bin/heron-executor",
- "--cp $(location hshell) heron-core/bin/heron-shell",
- "--cp $(location hstmgr) heron-core/bin/heron-stmgr",
- "--cp $(location htmaster) heron-core/bin/heron-tmaster",
- "--cp $(location hpyinstance) heron-core/bin/heron-python-instance",
- "--cp $(location hscheduler) heron-core/lib/scheduler/heron-scheduler.jar",
- "--cp $(location hscheduler-aurora) heron-core/lib/scheduler/heron-aurora-scheduler.jar",
- "--cp $(location hscheduler-local) heron-core/lib/scheduler/heron-local-scheduler.jar",
- "--cp $(location hbppacking) heron-core/lib/packing/heron-binpacking-packing.jar",
- "--cp $(location hrrpacking) heron-core/lib/packing/heron-roundrobin-packing.jar",
- "--cp $(location hmetricsmgr) heron-core/lib/metricsmgr/heron-metricsmgr.jar",
- "--cp $(location hlfsstatemgr) heron-core/lib/statemgr/heron-localfs-statemgr.jar",
- "--cp $(location hzkstatemgr) heron-core/lib/statemgr/heron-zookeeper-statemgr.jar",
- "--cp $(location hinstance) heron-core/lib/instance/heron-instance.jar",
- ]),
- heuristic_label_expansion = False,
- tags = ["manual"],
- tools = ["package_release.sh"],
-)
-
-################################################################################
-# Heron client packaging
-################################################################################
-genrule(
- name = "heron-client",
- srcs = [
- ":release.yaml",
- ":heron-core",
- ":conf-local-heron-internals",
- ":conf-local-metrics-sinks",
- ":conf-local-client",
- ":conf-local-packing",
- ":conf-local-scheduler",
- ":conf-local-statemgr",
- ":conf-local-uploader",
- ":hcli",
- ":hexamples",
- ":hscheduler",
- ":hscheduler-aurora",
- ":hscheduler-local",
- ":hbppacking",
- ":hrrpacking",
- ":hlfsstatemgr",
- ":hzkstatemgr",
- ":huploader-localfs",
- ":huploader-null",
- ":protobuf-java",
- ":slf4j-api-java",
- ":slf4j-jdk-java",
- ],
- outs = [
- "heron-client.tar.gz",
- ],
- cmd = " ".join([
- "export GENDIR=$(GENDIR);",
- "export BINDIR=$(BINDIR);",
- "$(location package_release.sh) $(location heron-client.tar.gz)",
- "--cp $(location release.yaml) release.yaml",
- "--cp $(location hcli) bin/heron",
- "--cp $(location conf-local-heron-internals) conf/local/heron_internals.yaml",
- "--cp $(location conf-local-metrics-sinks) conf/local/metrics_sinks.yaml",
- "--cp $(location conf-local-client) conf/local/client.yaml",
- "--cp $(location conf-local-packing) conf/local/packing.yaml",
- "--cp $(location conf-local-scheduler) conf/local/scheduler.yaml",
- "--cp $(location conf-local-statemgr) conf/local/statemgr.yaml",
- "--cp $(location conf-local-uploader) conf/local/uploader.yaml",
- "--cp $(location heron-core) dist/heron-core.tar.gz",
- "--cp $(location hexamples) examples/heron-examples.jar",
- "--cp $(location hscheduler) lib/scheduler/heron-scheduler.jar",
- "--cp $(location hscheduler-aurora) lib/scheduler/heron-aurora-scheduler.jar",
- "--cp $(location hscheduler-local) lib/scheduler/heron-local-scheduler.jar",
- "--cp $(location hbppacking) lib/packing/heron-binpacking-packing.jar",
- "--cp $(location hrrpacking) lib/packing/heron-roundrobin-packing.jar",
- "--cp $(location hlfsstatemgr) lib/statemgr/heron-localfs-statemgr.jar",
- "--cp $(location hzkstatemgr) lib/statemgr/heron-zookeeper-statemgr.jar",
- "--cp $(location huploader-localfs) lib/uploader/heron-localfs-uploader.jar",
- "--cp $(location huploader-null) lib/uploader/heron-null-uploader.jar",
- "--cp $(location protobuf-java) lib/third_party/$$(basename $(location protobuf-java))",
- "--cp $(location slf4j-api-java) lib/third_party/$$(basename $(location slf4j-api-java))",
- "--cp $(location slf4j-jdk-java) lib/third_party/$$(basename $(location slf4j-jdk-java))",
- ]),
- heuristic_label_expansion = False,
- tags = ["manual"],
- tools = ["package_release.sh"],
-)
-
-################################################################################
-# Heron tools packaging
-################################################################################
-genrule(
- name = "heron-tools",
- srcs = [
- ":release.yaml",
- ":htracker",
- ":hui",
- ],
- outs = [
- "heron-tools.tar.gz",
- ],
- cmd = " ".join([
- "export GENDIR=$(GENDIR);",
- "export BINDIR=$(BINDIR);",
- "$(location package_release.sh) $(location heron-tools.tar.gz)",
- "--cp $(location release.yaml) release.yaml",
- "--cp $(location htracker) bin/heron-tracker",
- "--cp $(location hui) bin/heron-ui",
- ]),
- heuristic_label_expansion = False,
- tags = ["manual"],
- tools = ["package_release.sh"],
-)
-
-################################################################################
-# Heron api packaging
-################################################################################
-genrule(
- name = "heron-api",
- srcs = [
- ":release.yaml",
- ":hapi",
- ":hspi",
- ":hstorm",
- ],
- outs = [
- "heron-api.tar.gz",
- ],
- cmd = " ".join([
- "export GENDIR=$(GENDIR);",
- "export BINDIR=$(BINDIR);",
- "$(location package_release.sh) $(location heron-api.tar.gz)",
- "--cp $(location release.yaml) release.yaml",
- "--cp $(location hapi) heron-api.jar",
- "--cp $(location hspi) heron-spi.jar",
- "--cp $(location hstorm) heron-storm.jar",
- ]),
- heuristic_label_expansion = False,
- tags = ["manual"],
- tools = ["package_release.sh"],
-)
-
-filegroup(
- name = "conf-local-heron-internals",
- srcs = ["//heron/tools/config/src/yaml:conf-local-heron-internals"],
-)
-
-filegroup(
- name = "conf-local-metrics-sinks",
- srcs = ["//heron/tools/config/src/yaml:conf-local-metrics-sinks"],
-)
-
-filegroup(
- name = "conf-local-client",
- srcs = ["//heron/tools/config/src/yaml:conf-local-client"],
-)
-
-filegroup(
- name = "conf-local-packing",
- srcs = ["//heron/tools/config/src/yaml:conf-local-packing"],
-)
-
-filegroup(
- name = "conf-local-scheduler",
- srcs = ["//heron/tools/config/src/yaml:conf-local-scheduler"],
-)
-
-filegroup(
- name = "conf-local-statemgr",
- srcs = ["//heron/tools/config/src/yaml:conf-local-statemgr"],
-)
-
-filegroup(
- name = "conf-local-uploader",
- srcs = ["//heron/tools/config/src/yaml:conf-local-uploader"],
-)
-
-filegroup(
- name = "hexamples",
- srcs = ["//heron/examples/src/java:heron-examples"],
-)
-
-filegroup(
- name = "htmaster",
- srcs = ["//heron/tmaster/src/cpp:heron-tmaster"],
-)
-
-filegroup(
- name = "hpyinstance",
- srcs = ["//heron/instance/src/python/instance:heron-python-instance"],
-)
-
-filegroup(
- name = "hstmgr",
- srcs = ["//heron/stmgr/src/cpp:heron-stmgr"],
-)
-
-filegroup(
- name = "hinstance",
- srcs = ["//heron/instance/src/java:heron-instance"],
-)
-
-filegroup(
- name = "hlogging",
- srcs = ["//heron/instance/src/java:aurora-logging-properties"],
-)
-
-filegroup(
- name = "hscheduler",
- srcs = ["//heron/scheduler-core/src/java:heron-scheduler"],
-)
-
-filegroup(
- name = "hscheduler-aurora",
- srcs = ["//heron/schedulers/src/java:heron-aurora-scheduler"],
-)
-
-filegroup(
- name = "hscheduler-local",
- srcs = ["//heron/schedulers/src/java:heron-local-scheduler"],
-)
-
-filegroup(
- name = "hbppacking",
- srcs = ["//heron/packing/src/java:heron-binpacking-packing"],
-)
-
-filegroup(
- name = "hrrpacking",
- srcs = ["//heron/packing/src/java:heron-roundrobin-packing"],
-)
-
-filegroup(
- name = "hmetricsmgr",
- srcs = ["//heron/metricsmgr/src/java:heron-metricsmgr"],
-)
-
-filegroup(
- name = "hexecutor",
- srcs = ["//heron/executor/src/python:heron-executor"],
-)
-
-filegroup(
- name = "hshell",
- srcs = ["//heron/shell/src/python:heron-shell"],
-)
-
-filegroup(
- name = "hcli",
- srcs = ["//heron/tools/cli/src/python:heron"],
-)
-
-filegroup(
- name = "haurora-job",
- srcs = ["//heron/tools/cli/src/python:heron-aurora"],
-)
-
-filegroup(
- name = "hinternals-config",
- srcs = ["//heron/config:config-internals-yaml"],
-)
-
-filegroup(
- name = "hcli2",
- srcs = ["//heron/cli2/src/python:heron-cli2"],
-)
-
-filegroup(
- name = "hscheduler-config",
- srcs = ["//heron/cli2/src/python:scheduler-config"],
-)
-
-filegroup(
- name = "haurora-scheduler-config",
- srcs = ["//heron/cli2/src/python:aurora-scheduler-config"],
-)
-
-filegroup(
- name = "hlocal-scheduler-config",
- srcs = ["//heron/cli2/src/python:local-scheduler-config"],
-)
-
-filegroup(
- name = "hmesos-scheduler-config",
- srcs = ["//heron/cli2/src/python:mesos-scheduler-config"],
-)
-
-filegroup(
- name = "hlfsstatemgr",
- srcs = ["//heron/statemgrs/src/java:heron-localfs-statemgr"],
-)
-
-filegroup(
- name = "hzkstatemgr",
- srcs = ["//heron/statemgrs/src/java:heron-zookeeper-statemgr"],
-)
-
-filegroup(
- name = "protobuf-java",
- srcs = ["@com_google_protobuf_protobuf_java//jar"],
-)
-
-filegroup(
- name = "slf4j-api-java",
- srcs = ["@org_slf4j_slf4j_api//jar"],
-)
-
-filegroup(
- name = "slf4j-jdk-java",
- srcs = ["@org_slf4j_slf4j_jdk14//jar"],
-)
-
-filegroup(
- name = "hapi",
- srcs = ["//heron/api/src/java:heron-api"],
-)
-
-filegroup(
- name = "hspi",
- srcs = ["//heron/spi/src/java:heron-spi"],
-)
-
-filegroup(
- name = "hmetrics-api",
- srcs = ["//heron/metricsmgr-api/src/java:metricsmgr-api-java"],
-)
-
-filegroup(
- name = "hstorm",
- srcs = ["//heron/storm/src/java:heron-storm"],
-)
-
-filegroup(
- name = "hviz",
- srcs = ["//heron/viz/src/python:heron-viz"],
-)
-
-filegroup(
- name = "htracker",
- srcs = ["//heron/tools/tracker/src/python:heron-tracker"],
-)
-
-filegroup(
- name = "hui",
- srcs = ["//heron/tools/ui/src/python:heron-ui"],
-)
-
-filegroup(
- name = "huploader-localfs",
- srcs = ["//heron/uploaders/src/java:heron-localfs-uploader"],
-)
-
-filegroup(
- name = "huploader-null",
- srcs = ["//heron/uploaders/src/java:heron-null-uploader"],
-)
-
-genrule(
- name = "generate-package-info",
- outs = ["release.yaml"],
- cmd = "$(location //scripts/packages:package-info-generator) $$(find . -name '*status*.txt') >$@",
- stamp = 1,
- tools = ["//scripts/packages:package-info-generator"],
-)
-
-genrule(
- name = "generate-api-launcher",
- srcs = [
- ":release.yaml",
- "//scripts/packages:api-template-bin.sh",
- "//scripts/packages:bin-common.sh"
- ],
- outs = ["api_launcher_bin.sh"],
- cmd = """
- release_info="$$(cat $(location :release.yaml))"
- bin_common="$$(cat $(location //scripts/packages:bin-common.sh))"
- template="$$(cat $(location //scripts/packages:api-template-bin.sh))"
- echo "$${bin_common}\n\n$${template//%release_info%/$${release_info}}" >$@
- """,
-)
-
-genrule(
- name = "generate-client-launcher",
- srcs = [
- ":release.yaml",
- "//scripts/packages:client-template-bin.sh",
- "//scripts/packages:bin-common.sh"
- ],
- outs = ["client_launcher_bin.sh"],
- cmd = """
- release_info="$$(cat $(location :release.yaml))"
- bin_common="$$(cat $(location //scripts/packages:bin-common.sh))"
- template="$$(cat $(location //scripts/packages:client-template-bin.sh))"
- echo "$${bin_common}\n\n$${template//%release_info%/$${release_info}}" >$@
- """,
-)
-
-genrule(
- name = "generate-tools-launcher",
- srcs = [
- ":release.yaml",
- "//scripts/packages:tools-template-bin.sh",
- "//scripts/packages:bin-common.sh"
- ],
- outs = ["tools_launcher_bin.sh"],
- cmd = """
- release_info="$$(cat $(location :release.yaml))"
- bin_common="$$(cat $(location //scripts/packages:bin-common.sh))"
- template="$$(cat $(location //scripts/packages:tools-template-bin.sh))"
- echo "$${bin_common}\n\n$${template//%release_info%/$${release_info}}" >$@
- """,
-)
-
-load("/scripts/packages/self_extract_binary", "self_extract_binary")
-
-self_extract_binary(
- name = "heron-api-install.sh",
- flatten_resources = [
- ":heron-api",
- ],
- launcher = ":api_launcher_bin.sh",
-)
-
-self_extract_binary(
- name = "heron-client-install.sh",
- flatten_resources = [
- ":heron-client",
- ],
- launcher = ":client_launcher_bin.sh",
-)
-
-self_extract_binary(
- name = "heron-tools-install.sh",
- flatten_resources = [
- ":heron-tools",
- ],
- launcher = ":tools_launcher_bin.sh",
-)
diff --git a/scripts/centos/package_release.sh b/scripts/centos/package_release.sh
deleted file mode 100755
index 71ee2d5..0000000
--- a/scripts/centos/package_release.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/bash -e
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script to package a release tar and create its associated .md5 checksum.
-#
-# Usage: package_release.sh <path-to-output-tar.gz> [package contents]
-#
-# In the simplest case, each file given will be placed in the root of the
-# resulting archive. The --relpath, --path, and --cp flags change this behavior
-# so that file paths can be structured.
-#
-# --path <path>: Each file is copied to ARCHIVE_ROOT/<path>/$(basename file).
-# --relpaths <prefix>: Strip $GENBIR, $BINDIR and then <prefix> from each
-# file's path. The resulting path is used for the file
-# inside of the archive. This combines with --path to
-# change the root of the resulting file path.
-# --cp <path> <path>: Copy the first file to the archive using exactly the
-# second path.
-#
-# Example:
-# BINDIR=bazel-bin/ \
-# package_release.sh /tmp/b.tar README.adoc LICENSE \
-# --path some/path/for/docs kythe/docs/kythe-{overview,storage}.txt \
-# --relpaths kythe/docs bazel-bin/kythe/docs/schema/schema.html \
-# --cp CONTRIBUTING.md kythe/docs/how-to-contribute.md
-#
-# Resulting tree in /tmp/b.tar:
-# README.adoc
-# LICENSE
-# kythe/docs/
-# kythe-overview.txt
-# kythe-storage.txt
-# schema.html
-# how-to-contribute.md
-
-OUT="$1"
-shift
-
-PBASE="$OUT.dir"
-P=$PBASE
-
-mkdir -p "$PBASE"
-trap "rm -rf '$PWD/$OUT.dir'" EXIT ERR INT
-
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --relpaths)
- RELPATHS=$2
- shift
- ;;
- --path)
- P="$PBASE/$2"
- mkdir -p "$P"
- shift
- ;;
- --cp)
- mkdir -p "$PBASE/$(dirname "$3")"
- cp "$2" "$PBASE/$3"
- shift 2
- ;;
- *)
- if [[ -z "$RELPATHS" ]]; then
- cp "$1" "$P"/
- else
- rp="${1#$GENDIR/}"
- rp="${rp#$BINDIR/}"
- rp="$(dirname "${rp#$RELPATHS/}")"
- mkdir -p "$P/$rp"
- cp "$1" "$P/$rp"
- fi
- ;;
- esac
- shift
-done
-
-tar czf "$OUT" -C "$OUT.dir" .
-
-cd "$(dirname "$OUT")"
-# md5sum "$(basename "$OUT")".gz > "$(basename "$OUT").gz.md5"
