Merge branch 'S4-130' into dev
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 7aafe90..fb8f6e4 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -19,16 +19,20 @@
import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.s4.core.util.ArchiveFetchException;
import org.apache.s4.core.util.ParametersInjectionModule;
+import org.apache.s4.core.util.ParsingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import com.google.common.io.Resources;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -68,10 +72,15 @@
}
});
+ // inject parameter from the command line, including zk string
+ Map<String, String> inlineParameters = Maps.newHashMap(ParsingUtils
+ .convertListArgsToMap(nodeArgs.extraNamedParameters));
+ inlineParameters.put("s4.cluster.zk_address", nodeArgs.zkConnectionString);
+
Injector injector = Guice.createInjector(Modules.override(
new BaseModule(Resources.getResource("default.s4.base.properties").openStream(), nodeArgs.clusterName))
- .with(new ParametersInjectionModule(ImmutableMap.of("s4.cluster.zk_address",
- nodeArgs.zkConnectionString))));
+ .with(new ParametersInjectionModule(inlineParameters)));
+
S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
try {
bootstrap.start(injector);
@@ -96,5 +105,11 @@
@Parameter(names = "-zk", description = "Zookeeper connection string", required = false)
String zkConnectionString = "localhost:2181";
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of "
+ + "inline configuration parameters, taking precedence over homonymous configuration parameters from "
+ + "configuration files. Syntax: '-p=name1=value1,name2=value2 '. "
+ + "NOTE: application parameters should be injected in the application configuration/deployment step."
+ + "Only parameters relevant to the node should be injected here, e.g. metrics logging configuration", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
+ List<String> extraNamedParameters = new ArrayList<String>();
}
}
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
new file mode 100644
index 0000000..37e18de
--- /dev/null
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s4.core.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * Utilities for parsing command line parameters
+ */
+public class ParsingUtils {
+
+ public static Map<String, String> convertListArgsToMap(List<String> args) {
+ Map<String, String> result = Maps.newHashMap();
+ for (String arg : args) {
+ String[] split = arg.split("[=]");
+ if (!(split.length == 2)) {
+ throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
+ }
+ result.put(split[0], split[1]);
+ }
+ return result;
+ }
+
+ public static class InlineConfigParameterConverter implements IStringConverter<String> {
+
+ private static Logger logger = LoggerFactory.getLogger(InlineConfigParameterConverter.class);
+
+ @Override
+ public String convert(String arg) {
+ Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
+ logger.info("processing inline configuration parameter {}", arg);
+ Matcher parameterMatcher = parameterPattern.matcher(arg);
+ if (!parameterMatcher.find()) {
+ throw new IllegalArgumentException("Cannot understand parameter " + arg);
+ }
+ return parameterMatcher.group(1);
+ }
+ }
+
+}
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/S4Metrics.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/S4Metrics.java
index 6544c36..eb637a2 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/S4Metrics.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/S4Metrics.java
@@ -106,10 +106,10 @@
} else {
String group1 = matcher.group(1);
- if ("csv".equals(group1)) {
- String outputDir = matcher.group(2);
- long period = Long.valueOf(matcher.group(3));
- TimeUnit timeUnit = TimeUnit.valueOf(matcher.group(4));
+ if (group1.startsWith("csv")) {
+ String outputDir = group1.substring("csv:".length());
+ long period = Long.valueOf(matcher.group(2));
+ TimeUnit timeUnit = TimeUnit.valueOf(matcher.group(3));
logger.info("Reporting metrics through csv files in directory [{}] with frequency of [{}] [{}]",
new String[] { outputDir, String.valueOf(period), timeUnit.name() });
CsvReporter.enable(new File(outputDir), period, timeUnit);
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
index 796a781..e562800 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.-->
<diezelImplementation xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>Builder</name>
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
index 90764b2..c019141 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. -->
<diezel xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>S4DSL</name>
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index f78b866..695faf3 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -23,13 +23,11 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.I0Itec.zkclient.ZkClient;
import org.apache.s4.comm.topology.ZNRecordSerializer;
import org.apache.s4.core.util.AppConfig;
+import org.apache.s4.core.util.ParsingUtils;
import org.apache.s4.deploy.DeploymentUtils;
import org.gradle.tooling.BuildLauncher;
import org.gradle.tooling.GradleConnector;
@@ -37,10 +35,8 @@
import org.gradle.tooling.ProjectConnection;
import org.slf4j.LoggerFactory;
-import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Maps;
import com.google.common.base.Strings;
/**
@@ -86,8 +82,8 @@
.appURI(s4rURI == null ? null : s4rURI.toString())
.customModulesNames(deployArgs.modulesClassesNames)
.customModulesURIs(deployArgs.modulesURIs).appClassName(deployArgs.appClass)
- .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
- deployArgs.clusterName, false, deployArgs.zkConnectionString);
+ .namedParameters(ParsingUtils.convertListArgsToMap(deployArgs.extraNamedParameters))
+ .build(), deployArgs.clusterName, false, deployArgs.zkConnectionString);
// Explicitly shutdown the JVM since Gradle leaves non-daemon threads running that delay the termination
if (!deployArgs.testMode) {
System.exit(0);
@@ -98,18 +94,6 @@
}
- private static Map<String, String> convertListArgsToMap(List<String> args) {
- Map<String, String> result = Maps.newHashMap();
- for (String arg : args) {
- String[] split = arg.split("[=]");
- if (!(split.length == 2)) {
- throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
- }
- result.put(split[0], split[1]);
- }
- return result;
- }
-
@Parameters(commandNames = "s4 deploy", commandDescription = "Package and deploy application to S4 cluster", separators = "=")
static class DeployAppArgs extends S4ArgsBase {
@@ -137,7 +121,7 @@
@Parameter(names = { "-modulesClasses", "-emc", "-mc" }, description = "Fully qualified class names of custom modules")
List<String> modulesClassesNames = new ArrayList<String>();
- @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = InlineConfigParameterConverter.class)
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
List<String> extraNamedParameters = new ArrayList<String>();
@Parameter(names = "-testMode", description = "Special mode for regression testing", hidden = true)
@@ -147,24 +131,6 @@
boolean debug = false;
}
- /**
- * Parameters parsing utility.
- *
- */
- public static class InlineConfigParameterConverter implements IStringConverter<String> {
-
- @Override
- public String convert(String arg) {
- Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
- logger.info("processing inline configuration parameter {}", arg);
- Matcher parameterMatcher = parameterPattern.matcher(arg);
- if (!parameterMatcher.find()) {
- throw new IllegalArgumentException("Cannot understand parameter " + arg);
- }
- return parameterMatcher.group(1);
- }
- }
-
static class ExecGradle {
public static void exec(File buildFile, String taskName, String[] params, boolean debug) throws Exception {
diff --git a/test-apps/twitter-adapter/build.gradle b/test-apps/twitter-adapter/build.gradle
index 0d93fd0..e654957 100644
--- a/test-apps/twitter-adapter/build.gradle
+++ b/test-apps/twitter-adapter/build.gradle
@@ -46,8 +46,6 @@
project.ext["distRootFolder"] = "$archivesBaseName-${-> version}"
-// Append the suffix 'SNAPSHOT' when the build is not for release.
-version = new Version(major: 0, minor: 0, bugfix: 0, isRelease: false)
group = 'org.apache.s4'
apply plugin: 'java'
diff --git a/website/content/doc/0.6.0/metrics.md b/website/content/doc/0.6.0/metrics.md
index 552df5d..cdc2469 100644
--- a/website/content/doc/0.6.0/metrics.md
+++ b/website/content/doc/0.6.0/metrics.md
@@ -24,18 +24,22 @@
By default, metrics are exposed by each node through JMX.
-The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as an application parameter, and must match the following regular expression:
+The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as a node or application parameter [^1], and must match the following regular expression:
(csv:.+|console):(\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)
Examples:
# dump metrics to csv files to /path/to/directory every 10 seconds
- csv:file://path/to/directory:10:SECONDS
+ # (recommendation: use a different and clean directory for each of the nodes)
+ csv:/path/to/directory:10:SECONDS
# dump metrics to the console every minute
console:1:MINUTES
-Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
\ No newline at end of file
+Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
+
+
+[^1]: csv-based metrics logging are usually specified as a node configuration, and must point to a clean directory, different for each node. Otherwise the metrics logging system will try to create multiple new files with the same name, and depending on your setup, that may not be possible.
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/twitter_trending_example.md b/website/content/doc/0.6.0/twitter_trending_example.md
index 0787d89..f7fabf4 100644
--- a/website/content/doc/0.6.0/twitter_trending_example.md
+++ b/website/content/doc/0.6.0/twitter_trending_example.md
@@ -47,12 +47,14 @@
* Build and deploy twitter-counter app
./s4 s4r -b=`pwd`/test-apps/twitter-counter/build.gradle -appClass=org.apache.s4.example.twitter.TwitterCounterApp twitter-counter
+
./s4 deploy -appName=twitter-counter -c=cluster1 -s4r=`pwd`/test-apps/twitter-counter/build/libs/twitter-counter.s4r
-* Build and deploy twitter-adapter app. In this example, we don't directly specify the app class of the adapter, we use the deployment approach for apps (remember, the adapter is also an app). Notice that the twitter-adapter package also has a different naming scheme. [^1]
+* Build and deploy twitter-adapter app. In this example, we don't directly specify the app class of the adapter, we use the deployment approach for apps (remember, the adapter is also an app).
./s4 s4r -b=`pwd`/test-apps/twitter-adapter/build.gradle -appClass=org.apache.s4.example.twitter.TwitterInputAdapter twitter-adapter
- ./s4 deploy -appName=twitter-adapter -c=cluster2 -s4r=`pwd`/test-apps/twitter-adapter/build/libs/twitter-adapter-0.0.0-SNAPSHOT.s4r -p=s4.adapter.output.stream=RawStatus
+
+ ./s4 deploy -appName=twitter-adapter -c=cluster2 -s4r=`pwd`/test-apps/twitter-adapter/build/libs/twitter-adapter.s4r -p=s4.adapter.output.stream=RawStatus
* Observe the current 10 most popular topics in file TopNTopics.txt. The file gets updated at regular intervals, and only outputs topics with a minimum of 10 occurrences, so you may have to wait a little before the file is updated :
@@ -79,7 +81,3 @@
We hope this will help you start rapidly, and remember: we're happy to help!
----
-
-###Footnotes
-
-[^1]: Modifying the `build.gradle` script you can change several aspects of the build process. By default the name of the `s4r` package is the application name provided in the packaging step, but you can attach the version automatically as in this example.
\ No newline at end of file
