S4-130 Update and document parameters initialization from nodes
- applicable to csv metrics logging configuration
- also: license update for some xml files
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 7aafe90..fb8f6e4 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -19,16 +19,20 @@
import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.s4.core.util.ArchiveFetchException;
import org.apache.s4.core.util.ParametersInjectionModule;
+import org.apache.s4.core.util.ParsingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import com.google.common.io.Resources;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -68,10 +72,15 @@
}
});
+ // inject parameter from the command line, including zk string
+ Map<String, String> inlineParameters = Maps.newHashMap(ParsingUtils
+ .convertListArgsToMap(nodeArgs.extraNamedParameters));
+ inlineParameters.put("s4.cluster.zk_address", nodeArgs.zkConnectionString);
+
Injector injector = Guice.createInjector(Modules.override(
new BaseModule(Resources.getResource("default.s4.base.properties").openStream(), nodeArgs.clusterName))
- .with(new ParametersInjectionModule(ImmutableMap.of("s4.cluster.zk_address",
- nodeArgs.zkConnectionString))));
+ .with(new ParametersInjectionModule(inlineParameters)));
+
S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
try {
bootstrap.start(injector);
@@ -96,5 +105,11 @@
@Parameter(names = "-zk", description = "Zookeeper connection string", required = false)
String zkConnectionString = "localhost:2181";
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of "
+ + "inline configuration parameters, taking precedence over homonymous configuration parameters from "
+ + "configuration files. Syntax: '-p=name1=value1,name2=value2 '. "
+ + "NOTE: application parameters should be injected in the application configuration/deployment step."
+ + "Only parameters relevant to the node should be injected here, e.g. metrics logging configuration", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
+ List<String> extraNamedParameters = new ArrayList<String>();
}
}
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
new file mode 100644
index 0000000..37e18de
--- /dev/null
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s4.core.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * Utilities for parsing command line parameters
+ */
+public class ParsingUtils {
+
+ public static Map<String, String> convertListArgsToMap(List<String> args) {
+ Map<String, String> result = Maps.newHashMap();
+ for (String arg : args) {
+ String[] split = arg.split("[=]");
+ if (!(split.length == 2)) {
+ throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
+ }
+ result.put(split[0], split[1]);
+ }
+ return result;
+ }
+
+ public static class InlineConfigParameterConverter implements IStringConverter<String> {
+
+ private static Logger logger = LoggerFactory.getLogger(InlineConfigParameterConverter.class);
+
+ @Override
+ public String convert(String arg) {
+ Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
+ logger.info("processing inline configuration parameter {}", arg);
+ Matcher parameterMatcher = parameterPattern.matcher(arg);
+ if (!parameterMatcher.find()) {
+ throw new IllegalArgumentException("Cannot understand parameter " + arg);
+ }
+ return parameterMatcher.group(1);
+ }
+ }
+
+}
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
index 796a781..e562800 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.-->
<diezelImplementation xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>Builder</name>
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
index 90764b2..c019141 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. -->
<diezel xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>S4DSL</name>
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index f78b866..695faf3 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -23,13 +23,11 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.I0Itec.zkclient.ZkClient;
import org.apache.s4.comm.topology.ZNRecordSerializer;
import org.apache.s4.core.util.AppConfig;
+import org.apache.s4.core.util.ParsingUtils;
import org.apache.s4.deploy.DeploymentUtils;
import org.gradle.tooling.BuildLauncher;
import org.gradle.tooling.GradleConnector;
@@ -37,10 +35,8 @@
import org.gradle.tooling.ProjectConnection;
import org.slf4j.LoggerFactory;
-import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Maps;
import com.google.common.base.Strings;
/**
@@ -86,8 +82,8 @@
.appURI(s4rURI == null ? null : s4rURI.toString())
.customModulesNames(deployArgs.modulesClassesNames)
.customModulesURIs(deployArgs.modulesURIs).appClassName(deployArgs.appClass)
- .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
- deployArgs.clusterName, false, deployArgs.zkConnectionString);
+ .namedParameters(ParsingUtils.convertListArgsToMap(deployArgs.extraNamedParameters))
+ .build(), deployArgs.clusterName, false, deployArgs.zkConnectionString);
// Explicitly shutdown the JVM since Gradle leaves non-daemon threads running that delay the termination
if (!deployArgs.testMode) {
System.exit(0);
@@ -98,18 +94,6 @@
}
- private static Map<String, String> convertListArgsToMap(List<String> args) {
- Map<String, String> result = Maps.newHashMap();
- for (String arg : args) {
- String[] split = arg.split("[=]");
- if (!(split.length == 2)) {
- throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
- }
- result.put(split[0], split[1]);
- }
- return result;
- }
-
@Parameters(commandNames = "s4 deploy", commandDescription = "Package and deploy application to S4 cluster", separators = "=")
static class DeployAppArgs extends S4ArgsBase {
@@ -137,7 +121,7 @@
@Parameter(names = { "-modulesClasses", "-emc", "-mc" }, description = "Fully qualified class names of custom modules")
List<String> modulesClassesNames = new ArrayList<String>();
- @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = InlineConfigParameterConverter.class)
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
List<String> extraNamedParameters = new ArrayList<String>();
@Parameter(names = "-testMode", description = "Special mode for regression testing", hidden = true)
@@ -147,24 +131,6 @@
boolean debug = false;
}
- /**
- * Parameters parsing utility.
- *
- */
- public static class InlineConfigParameterConverter implements IStringConverter<String> {
-
- @Override
- public String convert(String arg) {
- Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
- logger.info("processing inline configuration parameter {}", arg);
- Matcher parameterMatcher = parameterPattern.matcher(arg);
- if (!parameterMatcher.find()) {
- throw new IllegalArgumentException("Cannot understand parameter " + arg);
- }
- return parameterMatcher.group(1);
- }
- }
-
static class ExecGradle {
public static void exec(File buildFile, String taskName, String[] params, boolean debug) throws Exception {
diff --git a/website/content/doc/0.6.0/metrics.md b/website/content/doc/0.6.0/metrics.md
index 70d08d5..cdc2469 100644
--- a/website/content/doc/0.6.0/metrics.md
+++ b/website/content/doc/0.6.0/metrics.md
@@ -24,14 +24,14 @@
By default, metrics are exposed by each node through JMX.
-The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as an application parameter, and must match the following regular expression:
+The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as a node or application parameter [^1], and must match the following regular expression:
(csv:.+|console):(\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)
Examples:
# dump metrics to csv files to /path/to/directory every 10 seconds
- # (recommendation: use a clean directory)
+ # (recommendation: use a different and clean directory for each of the nodes)
csv:/path/to/directory:10:SECONDS
# dump metrics to the console every minute
@@ -39,4 +39,7 @@
-Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
\ No newline at end of file
+Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
+
+
+[^1]: csv-based metrics logging are usually specified as a node configuration, and must point to a clean directory, different for each node. Otherwise the metrics logging system will try to create multiple new files with the same name, and depending on your setup, that may not be possible.
\ No newline at end of file
