Merge branch 'master' into blur-0.2.4-parcel
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index d8cf3e2..46a649b 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -343,7 +343,7 @@
public static synchronized Configuration getSystemConfiguration() {
if (_systemConfiguration == null) {
- _systemConfiguration = BlurUtil.newHadoopConfiguration();
+ _systemConfiguration = BlurUtil.newHadoopConfiguration(null);
}
return new Configuration(_systemConfiguration);
}
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index e46a791..a1c0f88 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -46,9 +46,6 @@
import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_DEFAULT_MAX_FRAME_SIZE;
import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
import static org.apache.blur.utils.BlurConstants.BLUR_TMP_PATH;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
import static org.apache.blur.utils.BlurUtil.quietClose;
import java.io.File;
@@ -81,7 +78,6 @@
import org.apache.blur.utils.BlurUtil;
import org.apache.blur.utils.GCWatcher;
import org.apache.blur.utils.MemoryReporter;
-import org.apache.blur.zookeeper.ZkUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.zookeeper.ZooKeeper;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -111,7 +107,7 @@
}
public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration) throws Exception {
- Configuration config = BlurUtil.newHadoopConfiguration();
+ Configuration config = BlurUtil.newHadoopConfiguration(configuration);
TableContext.setSystemBlurConfiguration(configuration);
TableContext.setSystemConfiguration(config);
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 7809ffa..15d861c 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -53,9 +53,6 @@
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS;
import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_DEFAULT_MAX_FRAME_SIZE;
import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
import static org.apache.blur.utils.BlurUtil.quietClose;
import java.io.Closeable;
@@ -112,7 +109,6 @@
import org.apache.blur.utils.BlurUtil;
import org.apache.blur.utils.GCWatcher;
import org.apache.blur.utils.MemoryReporter;
-import org.apache.blur.zookeeper.ZkUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.search.BooleanQuery;
import org.apache.zookeeper.ZooKeeper;
@@ -148,7 +144,7 @@
}
public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration) throws Exception {
- Configuration config = BlurUtil.newHadoopConfiguration();
+ Configuration config = BlurUtil.newHadoopConfiguration(configuration);
TableContext.setSystemBlurConfiguration(configuration);
TableContext.setSystemConfiguration(config);
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index e4db03b..9406e66 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -21,6 +21,7 @@
import static org.apache.blur.metrics.MetricsConstants.THRIFT_CALLS;
import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_FILTERED_SERVER_CLASS;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTERED_SERVER_CLASS;
+import static org.apache.blur.utils.BlurConstants.HADOOP_CONF;
import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX;
import java.io.ByteArrayInputStream;
@@ -48,6 +49,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
@@ -703,7 +705,7 @@
FileSystem fileSystem = tablePath.getFileSystem(configuration);
if (createPath(fileSystem, tablePath)) {
LOG.info("Table uri existed.");
-// validateShardCount(shardCount, fileSystem, tablePath);
+ // validateShardCount(shardCount, fileSystem, tablePath);
}
ThreadWatcher.resetStatus();
}
@@ -1286,14 +1288,26 @@
throw new RuntimeException(e);
}
}
-
- public static Configuration newHadoopConfiguration() {
- return addHdfsConfig(new Configuration());
+
+ public static Configuration newHadoopConfiguration(BlurConfiguration blurConfiguration) {
+ return addHdfsConfig(new Configuration(), blurConfiguration);
}
- public static Configuration addHdfsConfig(Configuration configuration) {
+ public static Configuration addHdfsConfig(Configuration configuration, BlurConfiguration blurConfiguration) {
configuration.addResource("hdfs-default.xml");
configuration.addResource("hdfs-site.xml");
+ if (blurConfiguration != null) {
+ Map<String, String> properties = blurConfiguration.getProperties();
+ for (Entry<String, String> e : properties.entrySet()) {
+ String key = e.getKey();
+ if (key.startsWith(HADOOP_CONF)) {
+ String hadoopKey = key.substring(HADOOP_CONF.length());
+ String hadoopValue = e.getValue();
+ LOG.info("Adding hadoop configuration item [{0}] [{1}]", hadoopKey, hadoopValue);
+ configuration.set(hadoopKey, hadoopValue);
+ }
+ }
+ }
return configuration;
}
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index f4e7074..d197b90 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -320,7 +320,7 @@
for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
Path path = new Path(output, ShardUtil.getShardName(i));
Collection<Path> commitedTasks = getCommitedTasks(path);
- assertTrue(multiple >= commitedTasks.size());
+ assertTrue(commitedTasks.size() >= multiple);
for (Path p : commitedTasks) {
DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, p));
total += reader.numDocs();
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java
index 72db77b..5e244fd 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java
@@ -18,7 +18,6 @@
package org.apache.blur.shell;
-import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml
index 748f6c0..49e7137 100644
--- a/blur-thrift/pom.xml
+++ b/blur-thrift/pom.xml
@@ -51,7 +51,7 @@
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>${servlet-api.version}</version>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
</dependencies>
@@ -149,14 +149,6 @@
<name>hadoop2-mr1</name>
</property>
</activation>
- <dependencies>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- <version>${servlet-api.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
</profile>
<profile>
<id>hadoop2</id>
@@ -165,14 +157,6 @@
<name>hadoop2</name>
</property>
</activation>
- <dependencies>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- <version>${servlet-api.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
</profile>
</profiles>
</project>
diff --git a/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java
new file mode 100644
index 0000000..752f266
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+
+public class BlurPropertyParser {
+
+ public Map<String, List<BlurProp>> parse() throws IOException {
+ InputStream inputStream = BlurPropertyParser.class.getResourceAsStream("/blur-default.properties");
+ BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+ String line;
+ String prevLine = null;
+
+ String key = "|||General-Server-Properties|||";
+
+ Set<String> requiredProperties = getRequiredProperties();
+
+ Map<String, List<BlurProp>> map = new HashMap<String, List<BlurProp>>();
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.equals("### Shard Server Configuration")) {
+ key = "|||Shard-Server-Properties|||";
+ } else if (line.equals("### Controller Server Configuration")) {
+ key = "|||Controller-Server-Properties|||";
+ }
+ if (!line.startsWith("#") && !line.isEmpty()) {
+ String desc = getDesc(prevLine);
+ String name = getName(line);
+ String value = getValue(line);
+ String type = getType(value);
+ List<BlurProp> props = map.get(key);
+ if (props == null) {
+ props = Lists.newArrayList();
+ map.put(key, props);
+ }
+ BlurProp p = new BlurProp();
+ p.setName(name);
+ p.setDefaultVal(value);
+ p.setDescription(desc);
+ p.setType(type); // infer type...
+ p.setRequired(requiredProperties.contains(name));
+ props.add(p);
+ }
+ prevLine = line;
+ }
+ return map;
+ }
+
+ private Set<String> getRequiredProperties() throws IOException {
+ InputStream inputStream = getClass().getResourceAsStream("/blur-site.properties");
+ Properties properties = new Properties();
+ properties.load(inputStream);
+ inputStream.close();
+ Set<String> result = new HashSet<String>();
+ for (Object o : properties.keySet()) {
+ result.add(o.toString());
+ }
+ return result;
+ }
+
+ String getType(String value) {
+ if (value == null || value.isEmpty()) {
+ return "string";
+ }
+
+ if (isNumeric(value)) {
+ return "long";
+ }
+
+ if ("true".equals(value) || "false".equals(value)) {
+ return "boolean";
+ }
+
+ if (isDouble(value)) {
+ return "double";
+ }
+ return "string";
+ }
+
+ private boolean isDouble(String value) {
+ try {
+ Double.parseDouble(value);
+ return true;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ private boolean isNumeric(String value) {
+ byte[] chars = value.getBytes();
+
+ int start = 0;
+ if (value.charAt(0) == '-') {
+ start = 1;
+ }
+
+ for (int i = start; i < chars.length; i++) {
+ if (!Character.isDigit(chars[i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String getValue(String line) {
+ int index = line.indexOf('=');
+ if (index < 0) {
+ throw new RuntimeException();
+ }
+ return line.substring(index + 1);
+ }
+
+ private static String getName(String line) {
+ int index = line.indexOf('=');
+ if (index < 0) {
+ throw new RuntimeException();
+ }
+ return line.substring(0, index);
+ }
+
+ private static String getDesc(String prevLine) {
+ return prevLine.substring(1).trim();
+ }
+
+ public static class BlurProp {
+ private String name;
+ private String description;
+ private String defaultVal;
+ private String type;
+ private boolean required;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getDefaultVal() {
+ return defaultVal;
+ }
+
+ public void setDefaultVal(String defaultVal) {
+ this.defaultVal = defaultVal;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public boolean isRequired() {
+ return required;
+ }
+
+ public void setRequired(boolean required) {
+ this.required = required;
+ }
+
+ }
+
+}
diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java
index d867f31..7b860f4 100644
--- a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java
+++ b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java
@@ -1,5 +1,3 @@
-package org.apache.blur.doc;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,6 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.blur.doc;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
index bcba22b..fb81a00 100644
--- a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
+++ b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
@@ -24,45 +24,36 @@
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import org.apache.blur.doc.BlurPropertyParser.BlurProp;
+
public class CreateBlurServerSetupHtmlPage {
public static void main(String[] args) throws IOException {
- InputStream inputStream = CreateBlurServerSetupHtmlPage.class.getResourceAsStream("/blur-default.properties");
- BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
- String line;
- String prevLine = null;
-
- String key = "|||General-Server-Properties|||";
+ BlurPropertyParser parser = new BlurPropertyParser();
+ Map<String, List<BlurProp>> props = parser.parse();
Map<String, StringBuffer> map = new HashMap<String, StringBuffer>();
- while ((line = reader.readLine()) != null) {
- line = line.trim();
- if (line.equals("### Shard Server Configuration")) {
- key = "|||Shard-Server-Properties|||";
- } else if (line.equals("### Controller Server Configuration")) {
- key = "|||Controller-Server-Properties|||";
+
+ for (Map.Entry<String, List<BlurProp>> prop : props.entrySet()) {
+
+ StringBuffer buffer = map.get(prop.getKey());
+ if (buffer == null) {
+ buffer = new StringBuffer();
+ map.put(prop.getKey(), buffer);
}
- if (!line.startsWith("#") && !line.isEmpty()) {
- System.out.println(prevLine);
- System.out.println(line);
- String desc = getDesc(prevLine);
- String name = getName(line);
- String value = getValue(line);
- StringBuffer buffer = map.get(key);
- if (buffer == null) {
- buffer = new StringBuffer();
- map.put(key, buffer);
+ for (BlurProp p : prop.getValue()) {
+
+ buffer.append("<tr><td>").append(p.getName());
+ if (!p.getDefaultVal().trim().isEmpty()) {
+ buffer.append(" (").append(p.getDefaultVal()).append(")");
}
- buffer.append("<tr><td>").append(name);
- if (!value.trim().isEmpty()) {
- buffer.append(" (").append(value).append(")");
- }
- buffer.append("</td><td>").append(desc).append("</td></tr>");
+ buffer.append("</td><td>").append(p.getDescription()).append("</td></tr>");
}
- prevLine = line;
+
}
- reader.close();
+
String source = args[0];
String dest = args[1];
replaceValuesInFile(source, dest, map);
@@ -91,24 +82,4 @@
}
- private static String getValue(String line) {
- int index = line.indexOf('=');
- if (index < 0) {
- throw new RuntimeException();
- }
- return line.substring(index + 1);
- }
-
- private static String getName(String line) {
- int index = line.indexOf('=');
- if (index < 0) {
- throw new RuntimeException();
- }
- return line.substring(0, index);
- }
-
- private static String getDesc(String prevLine) {
- return prevLine.substring(1).trim();
- }
-
}
diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java b/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java
new file mode 100644
index 0000000..83d31d0
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.blur.doc.BlurPropertyParser.BlurProp;
+
+public class CreateCSDDescriptor {
+
+ private static final String BLUR_VERSION = "|||BLUR-VERSION|||";
+
+ public static void main(String[] args) throws IOException {
+ BlurPropertyParser parser = new BlurPropertyParser();
+ Map<String, List<BlurProp>> props = parser.parse();
+ Map<String, StringBuffer> map = new HashMap<String, StringBuffer>();
+
+ JsonPropertyFormatter formatter = new JsonPropertyFormatter();
+
+ for (Map.Entry<String, List<BlurProp>> prop : props.entrySet()) {
+
+ StringBuffer buffer = map.get(prop.getKey());
+ if (buffer == null) {
+ buffer = new StringBuffer();
+ map.put(prop.getKey(), buffer);
+ }
+ boolean first = true;
+
+ for (BlurProp p : prop.getValue()) {
+ if (!first) {
+ buffer.append(formatter.separator());
+ }
+ buffer.append(formatter.format(p));
+ first = false;
+ }
+ }
+
+ String source = args[0];
+ String dest = args[1];
+ String blurVersion = args[2];
+
+ replaceValuesInFile(source, dest, map, formatVersion(blurVersion));
+ }
+
+ public static String formatVersion(String blurVersion) {
+ return blurVersion.replace("-", ".");
+ }
+
+ private static void replaceValuesInFile(String s, String o, Map<String, StringBuffer> replacements, String blurVersion)
+ throws IOException {
+
+ File source = new File(s);
+ File output = new File(o);
+ System.out.println("Source[" + source.getAbsolutePath() + "]");
+ System.out.println("Output[" + output.getAbsolutePath() + "]");
+ PrintWriter writer = new PrintWriter(output);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(source)));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ StringBuffer newData = replacements.get(line.trim());
+ if (line.contains(BLUR_VERSION)) {
+ writer.println(line.replace(BLUR_VERSION, blurVersion));
+ } else if (newData != null) {
+ writer.println(newData.toString());
+ } else {
+ writer.println(line);
+ }
+ }
+ writer.close();
+ reader.close();
+ }
+
+}
diff --git a/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java b/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java
new file mode 100644
index 0000000..b0cee3c
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import org.apache.blur.doc.BlurPropertyParser.BlurProp;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.google.common.base.Splitter;
+
+public class JsonPropertyFormatter {
+
+ private static final String BLUR = "blur";
+ private static final String DEFAULT = "default";
+ private static final String TYPE = "type";
+ private static final String CONFIGURABLE_IN_WIZARD = "configurableInWizard";
+ private static final String REQUIRED = "required";
+ private static final String CONFIG_NAME = "configName";
+ private static final String DESCRIPTION = "description";
+ private static final String LABEL = "label";
+ private static final String NAME = "name";
+
+ public String separator() {
+ return ",";
+ }
+
+ public String format(BlurProp prop) {
+ JSONObject jsonObject = new JSONObject();
+ try {
+ jsonObject.put(NAME, prop.getName().replace(".", "_"));
+ jsonObject.put(LABEL, pretty(prop.getName()));
+ jsonObject.put(DESCRIPTION, prop.getDescription());
+ jsonObject.put(CONFIG_NAME, prop.getName());
+ jsonObject.put(REQUIRED, prop.isRequired());
+ jsonObject.put(CONFIGURABLE_IN_WIZARD, prop.isRequired());
+ jsonObject.put(TYPE, prop.getType());
+ jsonObject.put(DEFAULT, prop.getDefaultVal());
+ return jsonObject.toString(1);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String pretty(String s) {
+ Splitter splitter = Splitter.on('.');
+ StringBuilder builder = new StringBuilder();
+ for (String split : splitter.split(s)) {
+ if (builder.length() == 0 && split.equals(BLUR)) {
+ // skip
+ } else {
+ if (builder.length() != 0) {
+ builder.append(' ');
+ }
+ builder.append(split.substring(0, 1).toUpperCase()).append(split.substring(1));
+ }
+ }
+ return builder.toString();
+ }
+}
diff --git a/blur-util/src/main/java/org/apache/blur/doc/ParcelJsonTemplate.java b/blur-util/src/main/java/org/apache/blur/doc/ParcelJsonTemplate.java
new file mode 100644
index 0000000..f165963
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/doc/ParcelJsonTemplate.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+
+public class ParcelJsonTemplate {
+
+ private static final String BLUR_VERSION = "|||BLUR-VERSION|||";
+
+ public static void main(String[] args) throws IOException {
+
+ String source = args[0];
+ String dest = args[1];
+ String blurVersion = args[2];
+
+ replaceValuesInFile(source, dest, blurVersion);
+ }
+
+ private static void replaceValuesInFile(String s, String o, String blurVersion) throws IOException {
+ File source = new File(s);
+ File output = new File(o);
+ System.out.println("Source[" + source.getAbsolutePath() + "]");
+ System.out.println("Output[" + output.getAbsolutePath() + "]");
+ PrintWriter writer = new PrintWriter(output);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(source)));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (line.contains(BLUR_VERSION)) {
+ writer.println(line.replace(BLUR_VERSION, blurVersion));
+ } else {
+ writer.println(line);
+ }
+ }
+ writer.close();
+ reader.close();
+ }
+
+}
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 955d6cf..08fb745 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -197,6 +197,8 @@
public static final String SHARED_MERGE_SCHEDULER_PREFIX = "shared-merge-scheduler";
public static final String BLUR_FILTER_ALIAS = "blur.filter.alias.";
+
+ public static final String HADOOP_CONF = "hadoop_conf.";
static {
try {
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 999389b..209ffdf 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -76,13 +76,13 @@
# The command lib path where the controller and shard server processes will poll for new commands to enable.
blur.tmp.path=
-### Shard Server Configuration
# The map reduce working path for map reduce incremental updates.
blur.bulk.update.working.path=
# The desired permission on the blur.bulk.update.working.path and sub directories. If blank the default permissions will be applied.
blur.bulk.update.working.path.permission=
+### Shard Server Configuration
# The hostname for the shard, if blank the hostname is automatically detected
blur.shard.hostname=
diff --git a/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java b/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java
new file mode 100644
index 0000000..e8ecc89
--- /dev/null
+++ b/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class BlurPropertyParserTest {
+ private BlurPropertyParser parser;
+
+ @Before
+ public void setUp() throws Exception {
+ parser = new BlurPropertyParser();
+ }
+
+ @Test
+ public void testDefaultValNull() {
+ assertEquals("string", parser.getType(""));
+ assertEquals("string", parser.getType(null));
+ }
+
+ @Test
+ public void testDefaultValInt() {
+ assertEquals("long", parser.getType(Integer.toString(Integer.MIN_VALUE)));
+ assertEquals("long", parser.getType(Integer.toString(Integer.MAX_VALUE)));
+ }
+
+ @Test
+ public void testDefaultValBoolean() {
+ assertEquals("boolean", parser.getType("true"));
+ assertEquals("boolean", parser.getType("false"));
+ }
+
+ @Test
+ public void testDefaultValDouble() {
+ assertEquals("double", parser.getType("0.75"));
+ assertEquals("double", parser.getType("-0.75"));
+ }
+
+}
diff --git a/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java b/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java
new file mode 100644
index 0000000..9015cbb
--- /dev/null
+++ b/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Properties;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class CreateBlurServerSetupHtmlPageTest {
+ private File source;
+ private File dest = new File("target/gen.prop.doc.html");
+
+ @Before
+ public void before() {
+ source = new File(CreateBlurServerSetupHtmlPageTest.class.getResource("/prop.doc.base.html").getFile());
+ }
+
+ // Not a great test but makes sure i get it mostly put back together.
+ @Test
+ public void testGeneration() throws IOException {
+ CreateBlurServerSetupHtmlPage.main(new String[] { source.getAbsolutePath(), dest.getAbsolutePath() });
+
+ Properties defaultProperties = new Properties();
+ defaultProperties.load(CreateBlurServerSetupHtmlPageTest.class.getResourceAsStream("/blur-default.properties"));
+ int numProps = defaultProperties.size();
+ int numDocumentedProps = getDocumentedProps();
+
+ assertEquals(numProps, numDocumentedProps);
+ }
+
+ private int getDocumentedProps() throws IOException {
+ int count = 0;
+ String docs = readGeneratedDocs();
+ String[] props = docs.split("<tr>");
+
+ // System.out.println("PROPERTIES:");
+
+ for (String p : props) {
+ if (!p.isEmpty()) {
+ count++;
+ // System.out.println("PROP: " + p);
+ }
+ }
+
+ return count;
+ }
+
+ private String readGeneratedDocs() throws IOException {
+ StringBuffer buffer = new StringBuffer();
+ FileInputStream fis = new FileInputStream(dest);
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new InputStreamReader(fis));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ }
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ return buffer.toString();
+ }
+
+}
diff --git a/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java b/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java
new file mode 100644
index 0000000..cc9217e
--- /dev/null
+++ b/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Properties;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class CreateCSDDescriptorTest {
+ private File source;
+ private File dest = new File("target/service.sdl");
+
+ @Before
+ public void before() {
+ source = new File(CreateCSDDescriptorTest.class.getResource("/service.sdl.template").getFile());
+ }
+
+ // Not a great test but makes sure i get it mostly put back together.
+ @Test
+ public void testGeneration() throws IOException {
+ Properties defaultProperties = new Properties();
+ defaultProperties.load(CreateCSDDescriptorTest.class.getResourceAsStream("/blur-default.properties"));
+ int numProps = defaultProperties.size();
+ int numDocumentedProps = getDocumentedProps();
+
+ assertEquals(numProps, numDocumentedProps);
+ }
+
+ private int getDocumentedProps() throws IOException {
+ int count = 0;
+ String docs = readGeneratedDocs();
+ String[] props = docs.split("configurableInWizard");
+
+ // System.out.println("PROPERTIES:");
+
+ for (String p : props) {
+ if (!p.isEmpty()) {
+ count++;
+ // System.out.println("PROP: " + p);
+ }
+ }
+
+ return count;
+ }
+
+ private String readGeneratedDocs() throws IOException {
+ StringBuffer buffer = new StringBuffer();
+ FileInputStream fis = new FileInputStream(dest);
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new InputStreamReader(fis));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ }
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ return buffer.toString();
+ }
+
+}
diff --git a/blur-util/src/test/resources/prop.doc.base.html b/blur-util/src/test/resources/prop.doc.base.html
new file mode 100644
index 0000000..2e85566
--- /dev/null
+++ b/blur-util/src/test/resources/prop.doc.base.html
@@ -0,0 +1,5 @@
+|||General-Server-Properties|||
+
+|||Shard-Server-Properties|||
+
+|||Controller-Server-Properties|||
\ No newline at end of file
diff --git a/blur-util/src/test/resources/service.sdl.template b/blur-util/src/test/resources/service.sdl.template
new file mode 100644
index 0000000..3f9f6dc
--- /dev/null
+++ b/blur-util/src/test/resources/service.sdl.template
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+{
+ "name" : "BLUR",
+ "label" : "Blur",
+ "description": "Apache Blur Search Service",
+ "version" : "0.2.4",
+ "runAs" : {
+ "user" : "blur",
+ "group" : "blur"
+ },
+ "rolesWithExternalLinks":["BLUR_CONTROLLER","BLUR_SHARD"],
+ "hdfsDirs" : [
+ {
+ "name" : "CreateBlurTablesDir",
+ "label": "Create Blur Tables Directory",
+ "description" : "Creates the Blur Tables Directory",
+ "directoryDescription": "Location of Blur tables.",
+ "path":"/user/${user}/${blur_cluster_name}/tables",
+ "permissions":"0755"
+ },
+ {
+ "name" : "CreateBlurCommandsDir",
+ "label": "Create Blur Commands Directory",
+ "description" : "Creates the Blur Commands Directory",
+ "directoryDescription": "Path that Blur will look for installed commands.",
+ "path":"/user/${user}/${blur_cluster_name}/commands",
+ "permissions":"0755"
+ }
+ ],
+ "serviceInit": {
+ "preStartSteps": [
+ {"commandName":"CreateBlurTablesDir"},
+ {"commandName":"CreateBlurCommandsDir"}
+ ]
+ },
+
+ "parameters": [
+
+ {
+ "name":"blur_cluster_name",
+ "label":"Blur Cluster Name",
+ "description": "Name of Blur Cluster",
+ "configName":"blur.cluster.name",
+ "required":"true",
+ "type":"string",
+ "default":"default",
+ "configurableInWizard":true
+ },
+ {
+ "name":"blur_zookeeper_root",
+ "label":"Zookeeper Root",
+ "description": "Advanced: Root ZK connection to alternate path.",
+ "configName":"blur.zookeeper.root",
+ "required":"false",
+ "type":"string",
+ "default":"",
+ "configurableInWizard":true
+ },
+
+|||General-Server-Properties|||
+ ],
+
+ "roles" : [
+ {
+ "name" : "BLUR_CONTROLLER",
+ "label" : "Blur Controller Server",
+ "pluralLabel" : "Blur Controller Servers",
+ "startRunner" : {
+ "program" : "scripts/control.sh",
+ "args" : [ "start-controller" ],
+ "environmentVariables" : {
+ "BLUR_ZK_ROOT": "${blur_zookeeper_root}",
+ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
+ "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
+ "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
+ "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}"
+ }
+ },
+ "externalLink" : {
+ "name":"blur_controller_ui",
+ "label":"Blur Controller Status UI",
+ "url":"http://${host}:${blur_gui_controller_port}"
+ },
+ "logging": {
+ "dir":"/var/log/blur",
+ "filename": "blur--controller-server-${host}-0_main.log",
+ "configName" :"blur.log.dir",
+ "isModifiable":true,
+ "loggingType":"log4j"
+ },
+ "configWriter" : {
+ "generators" : [
+ {
+ "filename" : "blur-site.properties",
+ "configFormat" : "properties",
+ "excludedParams": ["blur_controller_jvm_options"]
+ }
+ ]
+ },
+ "parameters": [
+ {
+ "name":"blur_controller_jvm_options",
+ "label":"JVM Options for Controllers",
+ "description": "JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here.",
+ "configName":"blur.controller.jvm.options",
+ "required":"false",
+ "type":"string",
+ "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true",
+ "configurableInWizard":true
+ },
+|||Controller-Server-Properties|||
+ ]
+ },
+ {
+ "name" : "BLUR_SHARD",
+ "label" : "Blur Shard Server",
+ "pluralLabel" : "Blur Shard Servers",
+ "startRunner" : {
+ "program" : "scripts/control.sh",
+ "args" : [ "start-shard" ],
+ "environmentVariables" : {
+ "BLUR_ZK_ROOT": "${blur_zookeeper_root}",
+ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
+ "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
+ "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
+ "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}"
+ }
+ },
+ "externalLink" : {
+ "name":"blur_shard_ui",
+ "label":"Blur Shard Status UI",
+ "url":"http://${host}:${blur_gui_shard_port}"
+ },
+ "logging": {
+ "dir":"/var/log/blur",
+ "filename": "blur--shard-server-${host}-0_main.log",
+ "configName" :"blur.log.dir",
+ "isModifiable":true,
+ "loggingType":"log4j"
+ },
+ "configWriter" : {
+ "generators" : [
+ {
+ "filename" : "blur-site.properties",
+ "configFormat" : "properties",
+ "excludedParams": ["blur_shard_jvm_options"]
+ }
+ ]
+ },
+ "parameters": [
+ {
+ "name":"blur_shard_jvm_options",
+ "label":"JVM OPTIONS for Shards",
+ "description": "JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here.",
+ "configName":"blur.shard.jvm.options",
+ "required":"false",
+ "type":"string",
+ "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m",
+ "configurableInWizard":true
+ },
+|||Shard-Server-Properties|||
+ ]
+ }
+ ],
+ "parcel" : {
+ "requiredTags": ["blurcontroller"],
+ "optionalTags": ["blurcontroller", "blurshard", "blurconsole"]
+ },
+ "serviceDependencies" : [
+ {"name":"HDFS", "required":"true"},
+ {"name":"ZOOKEEPER", "required":"true"},
+ {"name":"YARN", "required":"true"}
+ ]
+
+
+}
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 97707df..19be49e 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -60,6 +60,43 @@
<build>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <id>create-service-descriptor</id>
+ <phase>package</phase>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ <configuration>
+ <mainClass>org.apache.blur.doc.CreateCSDDescriptor</mainClass>
+ <arguments>
+ <argument>${project.build.scriptSourceDirectory}/../../assemble/cdh/csd/descriptor/service.sdl.template</argument>
+ <argument>${project.build.directory}/service.sdl</argument>
+ <argument>${project.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>create-parcel-json</id>
+ <phase>package</phase>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ <configuration>
+ <mainClass>org.apache.blur.doc.ParcelJsonTemplate</mainClass>
+ <arguments>
+ <argument>${project.build.scriptSourceDirectory}/../../assemble/cdh/parcel/meta/parcel.json.template</argument>
+ <argument>${project.build.directory}/parcel.json</argument>
+ <argument>${project.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
@@ -87,6 +124,35 @@
</descriptors>
</configuration>
</execution>
+ <execution>
+ <id>distro-assembly-parcel</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>${parcel.assembly.file}</descriptor>
+ </descriptors>
+ <finalName>blur-${project.version}.parcel</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <ignoreDirFormatExtensions>false</ignoreDirFormatExtensions>
+ </configuration>
+ </execution>
+ <execution>
+ <id>distro-assembly-csd</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>${csd.assembly.file}</descriptor>
+ </descriptors>
+ <finalName>BLUR-${project.version}</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ </execution>
</executions>
</plugin>
</plugins>
@@ -124,6 +190,9 @@
</activation>
<properties>
<bin.assembly.file>src/assemble/bin-hadoop2.xml</bin.assembly.file>
+
+ <csd.assembly.file>src/assemble/csd-hadoop2.xml</csd.assembly.file>
+ <parcel.assembly.file>src/assemble/parcel-hadoop2.xml</parcel.assembly.file>
</properties>
</profile>
</profiles>
diff --git a/distribution/run_parcel_server.sh b/distribution/run_parcel_server.sh
new file mode 100755
index 0000000..1cec125
--- /dev/null
+++ b/distribution/run_parcel_server.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT_DIR=`dirname "$0"`
+PROJECT_DIR=`cd "$PROJECT_DIR"; pwd`
+
+BLUR_VERSION=`mvn help:evaluate -Dexpression=project.version -Dhadoop2 | grep -Ev '(^\[|Download\w+:)'`
+echo "BLUR_VERSION=${BLUR_VERSION}"
+TARGET="${PROJECT_DIR}/target"
+
+LAST_UPDATED_SEC=`date +%s`
+LAST_UPDATED="${LAST_UPDATED_SEC}0000"
+
+PARCEL="${TARGET}/blur-${BLUR_VERSION}.parcel.tar.gz"
+PARCEL_SHA="${PARCEL}.sha"
+BLUR_CSD="${TARGET}/BLUR-${BLUR_VERSION}.jar"
+
+HTTP_DIR="${TARGET}/http"
+MANIFEST="${HTTP_DIR}/manifest.json"
+
+rm -r $HTTP_DIR
+mkdir $HTTP_DIR
+
+sha1sum $PARCEL | awk '{print $1}' > $PARCEL_SHA
+HASH=`cat $PARCEL_SHA`
+echo "{\"lastUpdated\":${LAST_UPDATED},\"parcels\": [" > $MANIFEST
+for DISTRO in el5 el6 sles11 lucid precise trusty squeeze wheezy
+do
+ if [ $DISTRO != "el5" ] ; then
+ echo "," >> $MANIFEST
+ fi
+ DISTRO_PARCEL="blur-${BLUR_VERSION}-${DISTRO}.parcel"
+ DISTRO_PARCEL_SHA="blur-${BLUR_VERSION}-${DISTRO}.parcel.sha"
+ ln $PARCEL "${HTTP_DIR}/${DISTRO_PARCEL}"
+ ln $PARCEL_SHA "${HTTP_DIR}/${DISTRO_PARCEL_SHA}"
+ echo "{\"parcelName\":\"${DISTRO_PARCEL}\",\"components\": [{\"name\" : \"blur\",\"version\" : \"${BLUR_VERSION}\",\"pkg_version\": \"${BLUR_VERSION}\"}],\"hash\":\"${HASH}\"}" >> $MANIFEST
+done
+echo "]}" >> $MANIFEST
+CSD_BLUR_VERSION=`echo ${BLUR_VERSION} | tr - .`
+ln $BLUR_CSD "${HTTP_DIR}/BLUR-${CSD_BLUR_VERSION}.jar"
+cd ${HTTP_DIR}
+python -m SimpleHTTPServer
+
+
diff --git a/distribution/src/assemble/cdh/README.txt b/distribution/src/assemble/cdh/README.txt
new file mode 100644
index 0000000..588cbfa
--- /dev/null
+++ b/distribution/src/assemble/cdh/README.txt
@@ -0,0 +1,37 @@
+== Overview ==
+This provides some info on testing the CSD/Parcel packaging for Apache Blur.
+
+== CSD ==
+ sudo cp distribution/target/BLUR-0.2.4.jar /opt/cloudera/csd/BLUR-0.2.4.jar
+ sudo service cloudera-scm-server restart
+
+== Parcel ==
+Note that the '.tar.gz' extension is ripped off in the copy
+sudo cp distribution/target/blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.tar.gz /opt/cloudera/parcel-repo/blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel
+
+Then, in /opt/cloudera/parcel-repo:
+sha1sum blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel | awk '{print $1}' > blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.sha && cat blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.sha
+
+... which will write the sha file and print it to the screen. You can then copy that into a manifest in the
+ same directory [/opt/cloudera/parcel-repo/manifest.json] and substitute it in the hash value, which will look
+ like this:
+
+
+{
+ "lastUpdated":14286139390000,
+ "parcels": [
+ {
+ "parcelName":"blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel",
+ "components": [
+ {
+ "name" : "blur",
+ "version" : "0.2.4",
+ "pkg_version": "0.2.4"
+ }
+ ],
+ "hash":"632bd8d320f27ba68b9595f39ca2d99a203dd43c"
+ }
+ ]
+}
+
+Now you should be able to click the "Check for new parcels" in CM and begin activating it.
\ No newline at end of file
diff --git a/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template
new file mode 100644
index 0000000..af96568
--- /dev/null
+++ b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template
@@ -0,0 +1,170 @@
+{
+ "name" : "BLUR",
+ "label" : "Blur",
+ "description": "Apache Blur Search Service",
+ "version" : "|||BLUR-VERSION|||",
+ "runAs" : {
+ "user" : "blur",
+ "group" : "blur"
+ },
+ "rolesWithExternalLinks":["BLUR_CONTROLLER","BLUR_SHARD"],
+ "hdfsDirs" : [
+ {
+ "name" : "CreateBlurTablesDir",
+ "label": "Create Blur Tables Directory",
+ "description" : "Creates the Blur Tables Directory",
+ "directoryDescription": "Location of Blur tables.",
+ "path":"/user/${user}/${blur_cluster_name}/tables",
+ "permissions":"0755"
+ },
+ {
+ "name" : "CreateBlurCommandsDir",
+ "label": "Create Blur Commands Directory",
+ "description" : "Creates the Blur Commands Directory",
+ "directoryDescription": "Path that Blur will look for installed commands.",
+ "path":"/user/${user}/${blur_cluster_name}/commands",
+ "permissions":"0755"
+ }
+ ],
+ "serviceInit": {
+ "preStartSteps": [
+ {"commandName":"CreateBlurTablesDir"},
+ {"commandName":"CreateBlurCommandsDir"}
+ ]
+ },
+
+ "parameters": [
+
+ {
+ "name":"blur_cluster_name",
+ "label":"Blur Cluster Name",
+ "description": "Name of Blur Cluster",
+ "configName":"blur.cluster.name",
+ "required":"true",
+ "type":"string",
+ "default":"default",
+ "configurableInWizard":true
+ },
+ {
+ "name":"blur_zookeeper_root",
+ "label":"Zookeeper Root",
+ "description": "Advanced: Root ZK connection to alternate path.",
+ "configName":"blur.zookeeper.root",
+ "required":"false",
+ "type":"string",
+ "default":"",
+ "configurableInWizard":true
+ },
+
+|||General-Server-Properties|||
+ ],
+
+ "roles" : [
+ {
+ "name" : "BLUR_CONTROLLER",
+ "label" : "Blur Controller Server",
+ "pluralLabel" : "Blur Controller Servers",
+ "startRunner" : {
+ "program" : "scripts/control.sh",
+ "args" : [ "start-controller" ],
+ "environmentVariables" : {
+ "BLUR_ZK_ROOT": "${blur_zookeeper_root}",
+ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
+ "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
+ "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
+ "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}"
+ }
+ },
+ "externalLink" : {
+ "name":"blur_controller_ui",
+ "label":"Blur Controller Status UI",
+ "url":"http://${host}:${blur_gui_controller_port}"
+ },
+ "logging": {
+ "dir":"/var/log/blur",
+ "filename": "blur-controller-server-${host}.log",
+ "loggingType":"log4j"
+ },
+ "configWriter" : {
+ "generators" : [
+ {
+ "filename" : "blur-site.properties",
+ "configFormat" : "properties",
+ "excludedParams": ["blur_controller_jvm_options"]
+ }
+ ]
+ },
+ "parameters": [
+ {
+ "name":"blur_controller_jvm_options",
+ "label":"JVM Options for Controllers",
+ "description": "JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here.",
+ "configName":"blur.controller.jvm.options",
+ "required":"false",
+ "type":"string",
+ "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true",
+ "configurableInWizard":true
+ },
+|||Controller-Server-Properties|||
+ ]
+ },
+ {
+ "name" : "BLUR_SHARD",
+ "label" : "Blur Shard Server",
+ "pluralLabel" : "Blur Shard Servers",
+ "startRunner" : {
+ "program" : "scripts/control.sh",
+ "args" : [ "start-shard" ],
+ "environmentVariables" : {
+ "BLUR_ZK_ROOT": "${blur_zookeeper_root}",
+ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
+ "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
+ "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
+ "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}"
+ }
+ },
+ "additionalExternalLinks" : [{
+ "name":"blur_shard_ui",
+ "label":"Blur Shard Status UI",
+ "url":"http://${host}:${blur_gui_shard_port}"
+ }],
+ "logging": {
+ "dir":"/var/log/blur",
+ "filename": "blur-shard-server-${host}.log",
+ "loggingType":"log4j"
+ },
+ "configWriter" : {
+ "generators" : [
+ {
+ "filename" : "blur-site.properties",
+ "configFormat" : "properties",
+ "excludedParams": ["blur_shard_jvm_options"]
+ }
+ ]
+ },
+ "parameters": [
+ {
+ "name":"blur_shard_jvm_options",
+ "label":"JVM OPTIONS for Shards",
+ "description": "JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here.",
+ "configName":"blur.shard.jvm.options",
+ "required":"false",
+ "type":"string",
+ "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m",
+ "configurableInWizard":true
+ },
+|||Shard-Server-Properties|||
+ ]
+ }
+ ],
+ "parcel" : {
+ "requiredTags": ["blurcontroller"],
+ "optionalTags": ["blurcontroller", "blurshard", "blurconsole"]
+ },
+ "serviceDependencies" : [
+ {"name":"HDFS", "required":"true"},
+ {"name":"ZOOKEEPER", "required":"true"}
+ ]
+
+
+}
diff --git a/distribution/src/assemble/cdh/csd/scripts/control.sh b/distribution/src/assemble/cdh/csd/scripts/control.sh
new file mode 100644
index 0000000..b5c18e6
--- /dev/null
+++ b/distribution/src/assemble/cdh/csd/scripts/control.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+: <<DOCUMENTATION
+ --------------------------------------------------------------------------
+| control.sh
+ --------------------------------------------------------------------------
+Script used by CM to control services of Blur. This should mostly be
+bridge
+ --------------------------------------------------------------------------
+DOCUMENTATION
+
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+this="$bin/$script"
+
+set -e
+set -u
+set -o pipefail
+
+IFS=$'\n\t'
+
+scratch=$(mktemp -d -t scratch.tmp.XXXXXXXXXX)
+
+function finish {
+ rm -rf "${scratch}"
+ # Your cleanup code here
+}
+trap finish EXIT
+
+#__args
+declare -A OPT_ARGS
+usage() { echo "Usage: $0 <action>" 1>&2; exit 1; }
+
+#__functions
+
+write_to_blur_site_props() {
+ prop="$1=$2"
+ echo -n "Adding property[$prop] to blur-site.properties..."
+ #For now, rely on last one in
+ echo $prop >> $BLUR_SITE
+ echo " done."
+}
+
+setup_environment() {
+ blur_tmp_path="${CONF_DIR}/tmp"
+ blur_conf_dir="${CONF_DIR}"
+
+ mkdir -p $blur_tmp_path
+ mkdir -p $blur_conf_dir
+
+ #Wacky, yo!
+ chown blur:blur $blur_tmp_path
+
+ BLUR_SITE=${blur_conf_dir}/blur-site.properties
+
+# cp ${BLUR_HOME}/conf/log* ${blur_conf_dir}/
+
+ if [ -z "$BLUR_ZK_ROOT" ]; then
+ echo "Rooting zookeeper at [${BLUR_ZK_ROOT}]"
+ fi
+
+ write_to_blur_site_props blur.zookeeper.connection "$ZK_QUORUM/$BLUR_ZK_ROOT"
+
+ DFS_PATH=$(hdfs getconf -confKey fs.defaultFS)
+
+ #TODO: Should allow BLUR_DEFAULT_TABLE_PATH to be absolute to another cluster too.
+ write_to_blur_site_props "blur.cluster.${BLUR_CLUSTER_NAME}.table.uri" "$DFS_PATH/$BLUR_DEFAULT_TABLE_PATH"
+ write_to_blur_site_props blur.command.lib.path "$DFS_PATH/$BLUR_COMMAND_PATH"
+
+ export BLUR_CONF_DIR=$blur_conf_dir
+ export BLUR_LOGS="/var/log/blur"
+ export HADOOP_CLASSPATH=$(hadoop classpath)
+}
+
+start_controller() {
+ setup_environment
+ exec $BLUR_HOME/bin/start-supervised-controller-server.sh
+}
+
+start_shard() {
+ setup_environment
+ exec $BLUR_HOME/bin/start-supervised-shard-server.sh
+}
+
+#__main
+action="$1"
+
+if [ "${action}" == "" ] ;then
+ usage
+fi
+
+echo "Executing [$action] with BLUR_HOME [$BLUR_HOME]"
+
+export BLUR_LOGS=${CONF_DIR}/logs
+
+case ${action} in
+ (start-controller)
+ start_controller
+ ;;
+ (start-shard)
+ start_shard
+ ;;
+ (*)
+ echo "Unknown command[${action}]"
+ ;;
+esac
+
+
+
+
+
+
diff --git a/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh b/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh
new file mode 100644
index 0000000..ff839e7
--- /dev/null
+++ b/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HADOOP_HOME=$CDH_HADOOP_HOME
+export BLUR_HOME=$PARCELS_ROOT/$PARCEL_DIRNAME
+
+source $COMMON_SCRIPT
+set_hadoop_classpath
+HCP=$(hadoop classpath)
+export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HCP
+echo "***** PREFIX:"
+env
+echo "**** DONE."
+
diff --git a/distribution/src/assemble/cdh/parcel/meta/parcel.json.template b/distribution/src/assemble/cdh/parcel/meta/parcel.json.template
new file mode 100644
index 0000000..67ae8f2
--- /dev/null
+++ b/distribution/src/assemble/cdh/parcel/meta/parcel.json.template
@@ -0,0 +1,42 @@
+{
+ "schema_version": 1,
+ "name": "blur",
+ "version" : "|||BLUR-VERSION|||",
+ "setActiveSymlink": true,
+
+ "depends": "",
+ "replaces":"blur",
+ "conflicts":"",
+
+ "provides": [
+ "blurcontroller",
+ "blurshard",
+ "blurconsole"
+ ],
+
+ "scripts": {
+ "defines":"blur_parcel_env.sh"
+ },
+
+ "components": [
+ {
+ "name" : "blur",
+ "version" : "|||BLUR-VERSION|||",
+ "pkg_version": "|||BLUR-VERSION|||"
+ }
+ ],
+
+ "packages" : [],
+
+ "users": {
+ "blur": {
+ "longname" : "Blur",
+ "home" : "/var/lib/blur",
+ "shell" : "/bin/bash",
+ "extra_groups": []
+
+ }
+ },
+ "groups": []
+
+}
diff --git a/distribution/src/assemble/csd-hadoop2.xml b/distribution/src/assemble/csd-hadoop2.xml
new file mode 100644
index 0000000..13e638e
--- /dev/null
+++ b/distribution/src/assemble/csd-hadoop2.xml
@@ -0,0 +1,55 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>csd</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.scriptSourceDirectory}/../../assemble/cdh/csd/scripts</directory>
+ <outputDirectory>scripts</outputDirectory>
+ <excludes>
+ <exclude>**/.empty</exclude>
+ </excludes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>${project.build.directory}/service.sdl</source>
+ <outputDirectory>descriptor</outputDirectory>
+ <destName>service.sdl</destName>
+ </file>
+ <file>
+ <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/NOTICE-bin.txt</source>
+ <outputDirectory></outputDirectory>
+ <destName>NOTICE</destName>
+ </file>
+ <file>
+ <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/LICENSE-bin.txt</source>
+ <outputDirectory></outputDirectory>
+ <destName>LICENSE</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/distribution/src/assemble/parcel-hadoop2.xml b/distribution/src/assemble/parcel-hadoop2.xml
new file mode 100644
index 0000000..50a290b
--- /dev/null
+++ b/distribution/src/assemble/parcel-hadoop2.xml
@@ -0,0 +1,103 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>parcel</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>blur-${project.version}/lib</outputDirectory>
+ <unpack>false</unpack>
+ <includes>
+ <include>org.apache.blur:*</include>
+
+ <include>org.json:json</include>
+ <include>com.yammer.metrics:*</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.httpcomponents:*</include>
+ <include>org.apache.lucene:*</include>
+ <include>com.spatial4j:spatial4j</include>
+ <include>commons-cli:commons-cli</include>
+ <include>org.eclipse.jetty:*</include>
+ <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include>
+ <include>jline:jline</include>
+ <include>com.fasterxml.jackson.core:*</include>
+ <include>javax.servlet:*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.scriptSourceDirectory}/conf</directory>
+ <outputDirectory>blur-${project.version}/conf</outputDirectory>
+ <excludes>
+ <exclude>**/.empty</exclude>
+ <exclude>**/log4j*.xml</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.scriptSourceDirectory}/bin</directory>
+ <outputDirectory>blur-${project.version}/bin</outputDirectory>
+ <excludes>
+ <exclude>**/.empty</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.scriptSourceDirectory}/../../assemble/cdh/parcel/meta</directory>
+ <outputDirectory>blur-${project.version}/meta</outputDirectory>
+ <excludes>
+ <exclude>**/.empty</exclude>
+ <exclude>**/parcel.json.template</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.scriptSourceDirectory}/../../../../</directory>
+ <outputDirectory>blur-${project.version}/</outputDirectory>
+ <includes>
+ <include>DISCLAIMER</include>
+ <include>README</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>${project.build.directory}/parcel.json</source>
+ <outputDirectory>blur-${project.version}/meta</outputDirectory>
+ <destName>parcel.json</destName>
+ </file>
+ <file>
+ <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/NOTICE-bin.txt</source>
+ <outputDirectory>blur-${project.version}</outputDirectory>
+ <destName>NOTICE</destName>
+ </file>
+ <file>
+ <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/LICENSE-bin.txt</source>
+ <outputDirectory>blur-${project.version}</outputDirectory>
+ <destName>LICENSE</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/distribution/src/main/scripts/bin/blur-supervised-config.sh b/distribution/src/main/scripts/bin/blur-supervised-config.sh
new file mode 100755
index 0000000..f6e594b
--- /dev/null
+++ b/distribution/src/main/scripts/bin/blur-supervised-config.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+export BLUR_HOME="$bin"/..
+export BLUR_HOME_CONF=$BLUR_HOME/conf
+
+#. $BLUR_HOME/conf/blur-env.sh
+if [ -z "$JAVA_HOME" ]; then
+ if which java >/dev/null 2>&1 ; then
+ export JAVA_HOME=`java -cp $bin/../lib/blur-util-*.jar org.apache.blur.FindJavaHome`
+ fi
+fi
+if [ -z "$JAVA_HOME" ]; then
+ cat 1>&2 <<EOF
++======================================================================+
+| Error: JAVA_HOME is not set and Java could not be found |
++----------------------------------------------------------------------+
+| Please download the latest Sun JDK from the Sun Java web site |
+| > http://java.sun.com/javase/downloads/ < |
+| |
+| Hadoop and Blur requires Java 1.6 or later. |
+| NOTE: This script will find Sun Java whether you install using the |
+| binary or the RPM based installer. |
++======================================================================+
+EOF
+ exit 1
+fi
+
+export JAVA=$JAVA_HOME/bin/java
+
+if [ -z "$BLUR_CONF_DIR" ]; then
+ BLUR_CONF_DIR=$BLUR_HOME/conf
+fi
+
+if [[ -z "$BLUR_CLASSPATH" ]]; then
+ BLUR_CLASSPATH=$BLUR_CONF_DIR
+else
+ BLUR_CLASSPATH=$BLUR_CLASSPATH:$BLUR_CONF_DIR
+fi
+
+for f in $BLUR_HOME/lib/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+done
+
+for f in $BLUR_HOME/lib/*.war; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+done
+
+BLUR_CORE_FILE=`ls -d1 $BLUR_HOME/lib/blur-core-*.jar | head -1`
+
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$HADOOP_HOME/etc/hadoop
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$HADOOP_CLASSPATH
+
+ for f in $HADOOP_HOME/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/lib/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/share/hadoop/yarn/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/share/hadoop/yarn/lib/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/share/hadoop/common/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/share/hadoop/common/lib/*.jar; do
+ BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f;
+ done
+
+
+
+export BLUR_CLASSPATH
+
+echo "BLUR_CLASSPATH: [$BLUR_CLASSPATH]"
+
+HOSTNAME=`hostname`
diff --git a/distribution/src/main/scripts/bin/start-blocking-controller-server.sh b/distribution/src/main/scripts/bin/start-blocking-controller-server.sh
new file mode 100755
index 0000000..f3df832
--- /dev/null
+++ b/distribution/src/main/scripts/bin/start-blocking-controller-server.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/blur-config.sh
+
+INSTANCE=0
+while [ $INSTANCE -lt $BLUR_NUMBER_OF_CONTROLLER_SERVER_INSTANCES_PER_MACHINE ]; do
+ PID_FILE=$BLUR_HOME/pids/controller-$INSTANCE.pid
+
+ if [ -f $PID_FILE ]; then
+ if kill -0 `cat $PID_FILE` > /dev/null 2>&1; then
+ echo Controller server already running as process `cat $PID_FILE`. Stop it first.
+ let INSTANCE=INSTANCE+1
+ continue
+ fi
+ fi
+
+ PROC_NAME=controller-server-$HOSTNAME-$INSTANCE
+ "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-controller-$INSTANCE $BLUR_CONTROLLER_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$USER-$PROC_NAME -cp $BLUR_CLASSPATH org.apache.blur.thrift.ThriftBlurControllerServer -s $INSTANCE > "$BLUR_LOGS/blur-$USER-$PROC_NAME.out" 2>&1 < /dev/null
+ echo $! > $PID_FILE
+ echo Controller [$INSTANCE] starting as process `cat $PID_FILE`.
+
+ let INSTANCE=INSTANCE+1
+done
\ No newline at end of file
diff --git a/distribution/src/main/scripts/bin/start-supervised-controller-server.sh b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh
new file mode 100755
index 0000000..7afac4c
--- /dev/null
+++ b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+echo "Sourcing configs.."
+. $BLUR_HOME/bin/blur-supervised-config.sh
+
+PROC_NAME=controller-server-$HOSTNAME-0
+echo "Launching controller [$PROC_NAME] now..."
+echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}"
+exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Dblur-controller -Dlog4j.configuration=log4j.properties $BLUR_CONTROLLER_JVM_OPTIONS -Djava.library.path=$JAVA_LIBRARY_PATH -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurControllerServer -s 0
diff --git a/distribution/src/main/scripts/bin/start-supervised-shard-server.sh b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh
new file mode 100755
index 0000000..88e5352
--- /dev/null
+++ b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+echo "Sourcing configs.."
+. $BLUR_HOME/bin/blur-supervised-config.sh
+
+PROC_NAME=shard-server-$HOSTNAME-0
+echo "Launching shard [$PROC_NAME] now..."
+echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}"
+exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Dblur-shard -Dlog4j.configuration=log4j.properties $BLUR_SHARD_JVM_OPTIONS -Djava.library.path=$JAVA_LIBRARY_PATH -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurShardServer -s 0
diff --git a/pom.xml b/pom.xml
index 23e2479..54cf474 100644
--- a/pom.xml
+++ b/pom.xml
@@ -382,6 +382,10 @@
<!-- Used to create empty directories in git -->
<exclude>**/.empty</exclude>
+
+ <exclude>**/test/**/prop.doc.base.html</exclude>
+ <exclude>**/parcel.json.template</exclude>
+ <exclude>**/service.sdl.template</exclude>
</excludes>
</configuration>
<executions>
