WHIRR-738. Create a service for Druid.
Contributed by Russell Jurney
diff --git a/CHANGES.txt b/CHANGES.txt
index a52fd3d..204af4c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@
NEW FEATURES
+ WHIRR-738. Create a service for Druid (Russell Jurney via abayer)
+
IMPROVEMENTS
WHIRR-726. Allow specifying a Kerberos user other than
diff --git a/cli/pom.xml b/cli/pom.xml
index 4accf70..14e29a5 100644
--- a/cli/pom.xml
+++ b/cli/pom.xml
@@ -121,6 +121,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-druid</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
diff --git a/pom.xml b/pom.xml
index af35d96..1f3c234 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,6 +58,7 @@
<module>services/pig</module>
<module>services/solr</module>
<module>services/kerberos</module>
+ <module>services/druid</module>
</modules>
diff --git a/recipes/druid.properties b/recipes/druid.properties
new file mode 100644
index 0000000..f105abd
--- /dev/null
+++ b/recipes/druid.properties
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Deploy a Druid Cluster
+#
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=druid
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 zookeeper+druid-mysql+druid-master+druid-broker+druid-compute+druid-realtime
+# whirr.instance-templates=3 zookeeper,1 druid-mysql,2 druid-realtime,2 druid-broker,2 druid-master,5 druid-compute
+
+# Which version of druid to load
+whirr.druid.version=0.5.54
+
+# S3 bucket to store segments in
+whirr.druid.pusher.s3.bucket=dummy_s3_bucket
+
+# The realtime.spec file to use to configure a realtime node
+# whirr.druid.realtime.spec.path=/path/to/druid/examples/config/realtime/realtime.spec
diff --git a/services/druid/pom.xml b/services/druid/pom.xml
new file mode 100644
index 0000000..b72ee04
--- /dev/null
+++ b/services/druid/pom.xml
@@ -0,0 +1,120 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-druid</artifactId>
+ <packaging>bundle</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Apache Whirr Druid</name>
+ <properties>
+ <osgi.import>
+ !org.apache.whirr.service.druid*,
+ org.apache.commons.configuration*;version="[1.6,2)",
+ *
+ </osgi.import>
+ <osgi.export>
+ org.apache.whirr.service.druid*;version="${project.version}"
+ </osgi.export>
+ <osgi.bundle.activator>org.apache.whirr.service.druid.osgi.Activator</osgi.bundle.activator>
+ </properties>
+ <repositories>
+ <repository>
+ <id>pub-libs</id>
+ <name>pub-libs-local</name>
+ <url>https://metamx.artifactoryonline.com/metamx/pub-libs-releases-local</url>
+ </repository>
+ </repositories>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-zookeeper</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidBrokerClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidBrokerClusterActionHandler.java
new file mode 100644
index 0000000..56528e0
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidBrokerClusterActionHandler.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+/**
+ * Broker subclass of DruidClusterActionHandler.
+ */
+public class DruidBrokerClusterActionHandler extends DruidClusterActionHandler {
+
+ public static final String ROLE = "druid-broker";
+ public static final Integer PORT = 8080;
+
+ @Override
+ public final String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ public final Integer getPort() {
+ return PORT;
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidCluster.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidCluster.java
new file mode 100644
index 0000000..2cea0c1
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidCluster.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.druid;
+
+import org.apache.whirr.Cluster;
+import org.apache.whirr.RolePredicates;
+
+import java.io.IOException;
+
+/**
+ * DruidCluster - master class of the Druid cluster, with static methods.
+ */
+public class DruidCluster {
+ /**
+ * Returns the public address of the MySQL node.
+ * @param cluster the Cluster object
+ * @return mysql ip
+ * @throws IOException
+ */
+ public static String getMySQLPublicAddress(Cluster cluster)
+ throws IOException {
+ return cluster.getInstanceMatching(
+ RolePredicates.role(DruidMySQLClusterActionHandler.ROLE))
+ .getPrivateIp();
+ }
+
+ /**
+ * Converts a version to a url.
+ * @param version - the druid release
+ * @return version url
+ */
+ public static String getDownloadUrl(String version) {
+ return "http://static.druid.io/artifacts/releases/druid-services-"
+ + version + "-bin.tar.gz";
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidClusterActionHandler.java
new file mode 100644
index 0000000..cdf365f
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidClusterActionHandler.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.Map;
+
+import org.apache.whirr.Cluster;
+import org.apache.whirr.service.ClusterActionEvent;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+import org.apache.whirr.service.FirewallManager;
+import org.apache.whirr.service.zookeeper.ZooKeeperCluster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.whirr.RolePredicates.role;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+public abstract class DruidClusterActionHandler
+ extends ClusterActionHandlerSupport {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DruidClusterActionHandler.class);
+
+ public abstract String getRole();
+ public abstract Integer getPort();
+
+ private String readFile( String file ) throws IOException {
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ String line = null;
+ StringBuilder stringBuilder = new StringBuilder();
+ String ls = System.getProperty("line.separator");
+
+ while((line = reader.readLine()) != null) {
+ stringBuilder.append(line);
+ stringBuilder.append(ls);
+ }
+
+ return stringBuilder.toString();
+ }
+
+ // Always over-ridden in subclass
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event)
+ throws IOException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Cluster cluster = event.getCluster();
+ Configuration conf = getConfiguration(clusterSpec);
+
+ LOG.info("Role: [" + getRole() + "] Port: [" + getPort() + "]");
+
+ // Open a port for the service
+ event.getFirewallManager().addRule(
+ FirewallManager.Rule.create().destination(role(getRole())).port(getPort())
+ );
+
+ handleFirewallRules(event);
+
+ // Zookeeper quorum
+ String quorum = ZooKeeperCluster.getHosts(cluster, true);
+ LOG.info("ZookeeperCluster.getHosts(cluster): " + quorum);
+
+ // Get MySQL Server address
+ String mysqlAddress = DruidCluster.getMySQLPublicAddress(cluster);
+ LOG.info("DruidCluster.getMySQLPublicAddress(cluster).getHostAddress(): " + mysqlAddress);
+
+ // Get Blobstore and bucket
+ Map<String, String> env = System.getenv();
+ String identity = clusterSpec.getBlobStoreIdentity();
+ String credential = clusterSpec.getBlobStoreCredential();
+ String s3Bucket = conf.getString("whirr.druid.pusher.s3.bucket");
+ LOG.info("whirr.druid.pusher.s3.bucket: " + s3Bucket);
+
+ addStatement(event, call("retry_helpers"));
+ addStatement(event, call("configure_hostnames"));
+ addStatement(event, call("configure_druid",
+ getRole(),
+ quorum,
+ getPort().toString(),
+ mysqlAddress,
+ identity,
+ credential,
+ s3Bucket
+ ));
+
+ // Configure the realtime spec for realtime nodes
+ if(getRole().equals("druid-realtime")) {
+ String specPath = (String)conf.getProperty("whirr.druid.realtime.spec.path");
+ LOG.info("whirr.druid.realtime.spec.path" + specPath);
+
+ if(specPath == null || specPath.equals("")) {
+ // Default to the included realtime.spec
+ specPath = DruidClusterActionHandler.class.getResource("/" + "realtime.spec").getPath();
+ // prepareRemoteFileUrl(event, specPath);
+ }
+
+ // Quorum is a variable in the realtime.spec
+ String realtimeSpec = "'" + readFile(specPath) + "'";
+ addStatement(event, call("configure_realtime", quorum, realtimeSpec));
+ }
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+
+ addStatement(event, call("install_openjdk"));
+ addStatement(event, call("retry_helpers"));
+ addStatement(event, call("configure_hostnames"));
+
+ //addStatement(event, call(getInstallFunction(conf, "java", "install_oracle_jdk7")));
+
+ // Get the right version of druid and map this to the tarUrl
+ String druidVersion = (String)conf.getProperty("whirr.druid.version");
+ LOG.info("whirr.druid.version: " + druidVersion);
+ String tarUrl = DruidCluster.getDownloadUrl(druidVersion);
+ LOG.info("whirr tarUrl: " + tarUrl);
+
+ addStatement(event, call("install_druid", tarUrl));
+ }
+
+ protected synchronized Configuration getConfiguration(ClusterSpec clusterSpec) throws IOException {
+ return getConfiguration(clusterSpec, "whirr-druid-default.properties");
+ }
+
+ protected String getConfigureFunction(Configuration config) {
+ return "configure_druid";
+ }
+
+ @Override
+ protected void beforeStart(ClusterActionEvent event) throws IOException {
+ Configuration config = getConfiguration(event.getClusterSpec());
+ String configureFunction = getConfigureFunction(config);
+
+ if (configureFunction.equals("configure_druid")) {
+ addStatement(event, call(getStartFunction(config), getRole()));
+ } else {
+ }
+ }
+
+ @Override
+ protected void beforeStop(ClusterActionEvent event) throws IOException {
+ addStatement(event, call("stop_druid"));
+ }
+
+ @Override
+ protected void beforeCleanup(ClusterActionEvent event) throws IOException {
+ addStatement(event, call("cleanup_druid"));
+ }
+
+ protected String getStartFunction(Configuration config) {
+ return getStartFunction(config, getRole(), "start_druid");
+ }
+
+
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidComputeClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidComputeClusterActionHandler.java
new file mode 100644
index 0000000..145d5fa
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidComputeClusterActionHandler.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+public class DruidComputeClusterActionHandler extends DruidClusterActionHandler {
+ public static final String ROLE = "druid-compute";
+ public static final Integer PORT = 8083;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ public Integer getPort() {
+ return PORT;
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMasterClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMasterClusterActionHandler.java
new file mode 100644
index 0000000..9c308de
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMasterClusterActionHandler.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+public class DruidMasterClusterActionHandler extends DruidClusterActionHandler {
+ public static final String ROLE = "druid-master";
+ public static final Integer PORT = 8081;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ public Integer getPort() {
+ return PORT;
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMySQLClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMySQLClusterActionHandler.java
new file mode 100644
index 0000000..9d91ee4
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidMySQLClusterActionHandler.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.ClusterActionEvent;
+
+import java.io.IOException;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+public class DruidMySQLClusterActionHandler extends DruidClusterActionHandler {
+ public static final String ROLE = "druid-mysql";
+ public static final Integer PORT = 3306;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ public Integer getPort() {
+ return PORT;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+
+ addStatement(event, call("configure_hostnames"));
+ addStatement(event, call("install_mysql")); // installed/run via apt-get
+ }
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException {
+ // No-op
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/DruidRealtimeClusterActionHandler.java b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidRealtimeClusterActionHandler.java
new file mode 100644
index 0000000..4fb7ebd
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/DruidRealtimeClusterActionHandler.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid;
+
+public class DruidRealtimeClusterActionHandler extends DruidClusterActionHandler {
+ public static final String ROLE = "druid-realtime";
+ public static final Integer PORT = 8082;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ public Integer getPort() {
+ return PORT;
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/Activator.java b/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/Activator.java
new file mode 100644
index 0000000..76b4801
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/Activator.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid.osgi;
+
+import org.apache.whirr.service.ClusterActionHandler;
+import org.apache.whirr.service.druid.DruidBrokerClusterActionHandler;
+import org.apache.whirr.service.druid.DruidComputeClusterActionHandler;
+import org.apache.whirr.service.druid.DruidMasterClusterActionHandler;
+import org.apache.whirr.service.druid.DruidMySQLClusterActionHandler;
+import org.apache.whirr.service.druid.DruidRealtimeClusterActionHandler;
+import org.jclouds.scriptbuilder.functionloader.osgi.BundleFunctionLoader;
+import org.osgi.framework.BundleActivator;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.ServiceRegistration;
+
+import java.util.Properties;
+
+public class Activator implements BundleActivator {
+
+ private BundleFunctionLoader functionLoader;
+
+ private final ClusterActionHandler druidBrokerClusterActionHandler = new
+ DruidBrokerClusterActionHandler();
+ private ServiceRegistration druidBrokerRegistration;
+
+ private final ClusterActionHandler druidMasterClusterActionHandler = new
+ DruidMasterClusterActionHandler();
+ private ServiceRegistration druidMasterRegistration;
+
+ private final ClusterActionHandler druidComputeClusterActionHandler = new
+ DruidComputeClusterActionHandler();
+ private ServiceRegistration druidComputeRegistration;
+
+ private final ClusterActionHandler druidRealtimeClusterActionHandler = new
+ DruidRealtimeClusterActionHandler();
+ private ServiceRegistration druidRealtimeRegistration;
+
+ private final ClusterActionHandler druidMySQLClusterActionHandler = new
+ DruidMySQLClusterActionHandler();
+ private ServiceRegistration druidMySQLRegistration;
+
+ /**
+ * Called when this bundle is started so the Framework can perform the
+ * bundle-specific activities necessary to start this bundle. This method
+ * can be used to register services or to allocate any resources that this
+ * bundle needs.
+ * <p/>
+ * <p/>
+ * This method must complete and return to its caller in a timely manner.
+ *
+ * @param context The execution context of the bundle being started.
+ * @throws Exception If this method throws an exception, this
+ * bundle is marked as stopped and the Framework will remove this
+ * bundle's listeners, unregister all services registered by this
+ * bundle, and release all services used by this bundle.
+ */
+ @Override
+ public void start(BundleContext context) throws Exception {
+ //Initialize OSGi based FunctionLoader
+ functionLoader = new BundleFunctionLoader(context);
+ functionLoader.start();
+
+ Properties brokerProps = new Properties();
+ brokerProps.put("name", "druid-broker");
+ druidBrokerRegistration = context.registerService(
+ ClusterActionHandler.class.getName(),
+ druidBrokerClusterActionHandler,
+ brokerProps
+ );
+
+ Properties masterProps = new Properties();
+ masterProps.put("name", "druid-master");
+ druidMasterRegistration = context.registerService(
+ ClusterActionHandler.class.getName(),
+ druidMasterClusterActionHandler, masterProps
+ );
+
+ Properties computeProps = new Properties();
+ computeProps.put("name", "druid-compute");
+ druidComputeRegistration = context.registerService(
+ ClusterActionHandler.class.getName(),
+ druidComputeClusterActionHandler,
+ computeProps
+ );
+
+ Properties realtimeProps = new Properties();
+ realtimeProps.put("name", "druid-realtime");
+ druidRealtimeRegistration = context.registerService(
+ ClusterActionHandler.class.getName(),
+ druidRealtimeClusterActionHandler,
+ realtimeProps
+ );
+
+ Properties mysqlProps = new Properties();
+ realtimeProps.put("name", "druid-mysql");
+ druidRealtimeRegistration = context.registerService(
+ ClusterActionHandler.class.getName(),
+ druidRealtimeClusterActionHandler,
+ mysqlProps
+ );
+
+ }
+
+ /**
+ * Called when this bundle is stopped so the Framework can perform the
+ * bundle-specific activities necessary to stop the bundle. In general, this
+ * method should undo the work that the <code>BundleActivator.start</code>
+ * method started. There should be no active threads that were started by
+ * this bundle when this bundle returns. A stopped bundle must not call any
+ * Framework objects.
+ * <p/>
+ * <p/>
+ * This method must complete and return to its caller in a timely manner.
+ *
+ * @param context The execution context of the bundle being stopped.
+ * @throws Exception If this method throws an exception, the
+ * bundle is still marked as stopped, and the Framework will remove
+ * the bundle's listeners, unregister all services registered by the
+ * bundle, and release all services used by the bundle.
+ */
+ @Override
+ public void stop(BundleContext context) throws Exception {
+ if (druidBrokerRegistration != null) {
+ druidBrokerRegistration.unregister();
+ }
+ if (druidMasterRegistration != null) {
+ druidMasterRegistration.unregister();
+ }
+ if (druidComputeRegistration != null) {
+ druidComputeRegistration.unregister();
+ }
+ if (druidRealtimeRegistration != null) {
+ druidRealtimeRegistration.unregister();
+ }
+ if (druidMySQLRegistration != null) {
+ druidMySQLRegistration.unregister();
+ }
+ if (functionLoader != null) {
+ functionLoader.stop();
+ }
+ }
+}
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/package-info.java b/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/package-info.java
new file mode 100644
index 0000000..cc2723f
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/osgi/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * org.apache.whirr.service.druid.osgi.
+ */
+package org.apache.whirr.service.druid.osgi;
diff --git a/services/druid/src/main/java/org/apache/whirr/service/druid/package-info.java b/services/druid/src/main/java/org/apache/whirr/service/druid/package-info.java
new file mode 100644
index 0000000..138a8c5
--- /dev/null
+++ b/services/druid/src/main/java/org/apache/whirr/service/druid/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Package for the Apache Whirr service for Druid
+ */
+package org.apache.whirr.service.druid;
diff --git a/services/druid/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler b/services/druid/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
new file mode 100644
index 0000000..31417eb
--- /dev/null
+++ b/services/druid/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.druid.DruidBrokerClusterActionHandler
+org.apache.whirr.service.druid.DruidMasterClusterActionHandler
+org.apache.whirr.service.druid.DruidComputeClusterActionHandler
+org.apache.whirr.service.druid.DruidRealtimeClusterActionHandler
+org.apache.whirr.service.druid.DruidMySQLClusterActionHandler
diff --git a/services/druid/src/main/resources/functions/configure_druid.sh b/services/druid/src/main/resources/functions/configure_druid.sh
new file mode 100644
index 0000000..c19daca
--- /dev/null
+++ b/services/druid/src/main/resources/functions/configure_druid.sh
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function configure_druid() {
+
+ ROLE=$1
+ ZOOKEEPER_QUORUM=$2
+ PORT=$3
+ MYSQL_HOSTNAME=$4
+ IDENTITY=$5
+ CREDENTIAL=$6
+ S3_BUCKET=$7
+ HOSTNAME=$PRIVATE_IP
+ ROLE_NAME=${ROLE/druid-//}
+
+ echo "ROLE: $ROLE, ZOOKEEPER_QUORUM: $ZOOKEEPER_QUORUM, PORT: $PORT, MYSQL_HOSTNAME=$MYSQL_HOSTNAME, HOSTNAME=$HOSTNAME, ROLE_NAME=$ROLE_NAME"
+
+ # Configure runtime.properties with Zookeeper address
+ cat > /usr/local/druid-services-0.5.7/config/$ROLE_NAME/runtime.properties <<EOF
+
+# Druid base config
+com.metamx.emitter.logging=true
+
+druid.processing.formatString=processing_%s
+druid.processing.numThreads=1
+druid.processing.buffer.sizeBytes=10000000
+
+#emitting, opaque marker
+druid.service=example
+
+druid.request.logging.dir=/tmp/example/log
+druid.realtime.specFile=realtime.spec
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# below are dummy values when operating a realtime only node
+com.metamx.aws.accessKey=$IDENTITY
+com.metamx.aws.secretKey=$CREDENTIAL
+druid.pusher.s3.bucket=${S3_BUCKET}
+
+druid.client.http.connections=30
+druid.zk.service.host=$ZOOKEEPER_QUORUM
+druid.server.maxSize=300000000000
+druid.zk.paths.base=/druid
+druid.database.segmentTable=prod_segments
+druid.database.user=druid
+druid.database.password=diurd
+druid.database.connectURI=jdbc:mysql://$MYSQL_HOSTNAME:3306/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+druid.database.ruleTable=rules
+druid.database.configTable=config
+
+# Path on local FS for storage of segments; dir will be created if needed
+druid.paths.indexCache=/tmp/druid/indexCache
+# Path on local FS for storage of segment metadata; dir will be created if needed
+druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache
+druid.pusher.local.storageDirectory=/tmp/druid/localStorage
+druid.pusher.local=true
+
+druid.host=$HOSTNAME:$PORT
+druid.port=$PORT
+EOF
+
+}
\ No newline at end of file
diff --git a/services/druid/src/main/resources/functions/configure_realtime.sh b/services/druid/src/main/resources/functions/configure_realtime.sh
new file mode 100644
index 0000000..9b01881
--- /dev/null
+++ b/services/druid/src/main/resources/functions/configure_realtime.sh
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function configure_realtime() {
+
+ ZOOKEEPER_QUORUM=$1
+ REALTIME_SPEC=$2
+
+ echo "PWD: `pwd`"
+
+ cat > /usr/local/druid-services-0.5.7/config/realtime/realtime.spec << EOF
+${REALTIME_SPEC}
+EOF
+}
\ No newline at end of file
diff --git a/services/druid/src/main/resources/functions/install_druid.sh b/services/druid/src/main/resources/functions/install_druid.sh
new file mode 100644
index 0000000..1ce4136
--- /dev/null
+++ b/services/druid/src/main/resources/functions/install_druid.sh
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function install_tarball_no_md5_no_overwrite() {
+ cd /usr/local
+ if [[ "$1" != "" ]]; then
+ # Download a .tar.gz file and extract to target dir
+
+ local tar_url=$1
+ local tar_file=`basename $tar_url`
+
+ local target=/usr/local/
+ mkdir -p $target
+
+ local curl="curl -L --silent --show-error --fail --connect-timeout 10 --max-time 600 --retry 5"
+ # any download should take less than 10 minutes
+
+ for retry_count in `seq 1 3`;
+ do
+ $curl -O $tar_url || true
+
+ if [ ! $retry_count -eq "3" ]; then
+ sleep 10
+ fi
+ done
+
+ if [ ! -e $tar_file ]; then
+ echo "Failed to download $tar_file. Aborting."
+ exit 1
+ fi
+
+ # Using -k so we don't overwrite any other roles' druid configs
+ tar kxzf $tar_file -C $target
+ rm -f $tar_file
+ fi
+}
+
+function install_druid() {
+ install_tarball_no_md5_no_overwrite http://static.druid.io/artifacts/releases/druid-services-0.5.7-bin.tar.gz
+}
\ No newline at end of file
diff --git a/services/druid/src/main/resources/functions/install_mysql.sh b/services/druid/src/main/resources/functions/install_mysql.sh
new file mode 100644
index 0000000..1d1821a
--- /dev/null
+++ b/services/druid/src/main/resources/functions/install_mysql.sh
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Installing mysql also starts/activates it, and then we create the Druid-specific tables below.
+function install_mysql() {
+ # Install MySQL
+ export DEBIAN_FRONTEND=noninteractive
+ sudo debconf-set-selections <<< 'mysql-server-5.1 mysql-server/root_password password diurd'
+ sudo debconf-set-selections <<< 'mysql-server-5.1 mysql-server/root_password_again password diurd'
+ sudo apt-get -q -y -V --force-yes --reinstall install mysql-server-5.1
+ sudo apt-get -q -y -V --force-yes --reinstall install mysql-client-5.1
+
+ # Remove binding to localhost so we can accept external connections
+ sudo sed -i "s/bind-address/# bind-address/" /etc/mysql/my.cnf
+
+ # Restart mysql
+ sudo restart mysql
+
+ # Setup druid tables
+ mysql -u root -pdiurd -e "CREATE USER 'druid'@'%' IDENTIFIED BY 'diurd'"; 2>&1 > /dev/null
+ mysql -u root -pdiurd -e "GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd'; CREATE database druid;" 2>&1 > /dev/null
+ mysql -u root -pdiurd -e "GRANT ALL ON druid.* TO 'druid'@'%' IDENTIFIED BY 'diurd'"; 2>&1 > /dev/null
+ mysql -u root -pdiurd -e "FLUSH PRIVILEGES;"
+}
\ No newline at end of file
diff --git a/services/druid/src/main/resources/functions/start_druid.sh b/services/druid/src/main/resources/functions/start_druid.sh
new file mode 100644
index 0000000..9c89aea
--- /dev/null
+++ b/services/druid/src/main/resources/functions/start_druid.sh
@@ -0,0 +1,53 @@
+function start_druid() {
+
+ ROLE=$1
+ echo "Inside start_druid(), ROLE=$ROLE"
+
+ # Make logs directory
+ echo "Creating log directory..."
+ if [ ! -d "/usr/local/druid-services-0.5.7/logs" ]; then
+ mkdir /usr/local/druid-services-0.5.7/logs
+ fi
+
+ # Start the appropriate role
+ echo "Executing druid $ROLE..."
+ case $ROLE in
+ druid)
+ # Run the realtime node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=/usr/local/druid-services-0.5.7/config/realtime/realtime.spec -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/realtime com.metamx.druid.realtime.RealtimeMain 2>&1 > /usr/local/druid-services-0.5.7/logs/realtime.log &
+
+ # And a master node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/master com.metamx.druid.http.MasterMain 2>&1 > /usr/local/druid-services-0.5.7/logs/master.log &
+
+ # And a compute node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/compute com.metamx.druid.http.ComputeMain 2>&1 > /usr/local/druid-services-0.5.7/logs/compute.log &
+
+ # And a broker node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/broker com.metamx.druid.http.BrokerMain 2>&1 > /usr/local/druid-services-0.5.7/logs/broker.log &
+
+ ;;
+ druid-broker)
+ # Run the broker node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/broker com.metamx.druid.http.BrokerMain 2>&1 > /usr/local/druid-services-0.5.7/logs/broker.log &
+ ;;
+ druid-master)
+ # Run the master node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/master com.metamx.druid.http.MasterMain 2>&1 > /usr/local/druid-services-0.5.7/logs/master.log &
+ ;;
+ druid-compute)
+ # Run the compute node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/compute com.metamx.druid.http.ComputeMain 2>&1 > /usr/local/druid-services-0.5.7/logs/compute.log &
+ ;;
+ druid-realtime)
+ # Run the realtime node
+ nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=/usr/local/druid-services-0.5.7/config/realtime/realtime.spec -classpath /usr/local/druid-services-0.5.7/lib/druid-services-0.5.7-selfcontained.jar:/usr/local/druid-services-0.5.7/config/realtime com.metamx.druid.realtime.RealtimeMain 2>&1 > /usr/local/druid-services-0.5.7/logs/realtime.log &
+ ;;
+ druid-mysql)
+ # Noop - MySQL runs automatically after apt-get installed
+ ;;
+ *)
+ echo $"Usage: $0 {druid|druid-broker|druid-master|druid-compute|druid-realtime|druid-mysql}"
+ exit 1
+ esac
+
+}
\ No newline at end of file
diff --git a/services/druid/src/main/resources/functions/stop_druid.sh b/services/druid/src/main/resources/functions/stop_druid.sh
new file mode 100644
index 0000000..512c8cf
--- /dev/null
+++ b/services/druid/src/main/resources/functions/stop_druid.sh
@@ -0,0 +1,8 @@
+function stop_druid() {
+
+ ps -eaf | grep RealtimeMain | grep -v grep | awk '{print $2}' | xargs kill
+ ps -eaf | grep MasterMain | grep -v grep | awk '{print $2}' | xargs kill
+ ps -eaf | grep ComputeMain | grep -v grep | awk '{print $2}' | xargs kill
+ ps -eaf | grep BrokerMain | grep -v grep | awk '{print $2}' | xargs kill
+
+}
diff --git a/services/druid/src/main/resources/realtime.spec b/services/druid/src/main/resources/realtime.spec
new file mode 100644
index 0000000..0309b3b
--- /dev/null
+++ b/services/druid/src/main/resources/realtime.spec
@@ -0,0 +1,29 @@
+[{
+ "schema" : { "dataSource":"druidtest",
+ "aggregators":[ {"type":"count", "name":"impressions"},
+ {"type":"doubleSum","name":"wp","fieldName":"wp"}],
+ "indexGranularity":"minute",
+ "shardSpec" : { "type": "none" } },
+ "config" : { "maxRowsInMemory" : 500000,
+ "intermediatePersistPeriod" : "PT10m" },
+ "firehose" : { "type" : "kafka-0.7.2",
+ "consumerProps" : { "zk.connect" : "localhost:2181",
+ "zk.connectiontimeout.ms" : "15000",
+ "zk.sessiontimeout.ms" : "15000",
+ "zk.synctime.ms" : "5000",
+ "groupid" : "topic-pixel-local",
+ "fetch.size" : "1048586",
+ "autooffset.reset" : "largest",
+ "autocommit.enable" : "false" },
+ "feed" : "druidtest",
+ "parser" : { "timestampSpec" : { "column" : "utcdt", "format" : "iso" },
+ "data" : { "format" : "json" },
+ "dimensionExclusions" : ["wp"] } },
+ "plumber" : { "type" : "realtime",
+ "windowPeriod" : "PT10m",
+ "segmentGranularity":"hour",
+ "basePersistDirectory" : "/tmp/realtime/basePersist",
+ "rejectionPolicy": {"type": "messageTime"} }
+
+}]
+
diff --git a/services/druid/src/main/resources/whirr-druid-default.properties b/services/druid/src/main/resources/whirr-druid-default.properties
new file mode 100644
index 0000000..f0621a5
--- /dev/null
+++ b/services/druid/src/main/resources/whirr-druid-default.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.druid.version=0.5.7
+whirr.com.metamx.emitter.logging=true
+whirr.druid.pusher.s3.bucket=dummy_s3_bucket
diff --git a/services/druid/src/test/java/org/apache/whirr/service/druid/integration/DruidServiceTest.java b/services/druid/src/test/java/org/apache/whirr/service/druid/integration/DruidServiceTest.java
new file mode 100644
index 0000000..1d16308
--- /dev/null
+++ b/services/druid/src/test/java/org/apache/whirr/service/druid/integration/DruidServiceTest.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.druid.integration;
+
+import com.google.common.collect.Iterables;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.TestConstants;
+import org.apache.whirr.service.druid.DruidRealtimeClusterActionHandler;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.URL;
+import java.net.URLConnection;
+
+import static org.apache.whirr.RolePredicates.role;
+
+public class DruidServiceTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DruidRealtimeClusterActionHandler.class);
+
+ private ClusterSpec clusterSpec;
+ private ClusterController controller;
+ private Cluster cluster;
+
+ @Before
+ public void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ config.addConfiguration(new PropertiesConfiguration("whirr-druid-test.properties"));
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+ //controller = new ClusterControllerFactory().create(clusterSpec.getServiceName());
+ cluster = controller.launchCluster(clusterSpec);
+ }
+
+ @Test(timeout = TestConstants.ITEST_TIMEOUT)
+ public void testSegmentMetadata() {
+ try {
+ String response = getQueryInfo();
+ return;
+ }
+ catch(Exception e)
+ {
+ LOG.debug("Caught exception contacting cluster");
+ }
+ }
+
+ private String getQueryInfo() throws Exception {
+ for(int i=0; i<20; i++) {
+ try {
+ Cluster.Instance instance = Iterables.get(
+ cluster.getInstancesMatching(role(DruidRealtimeClusterActionHandler.ROLE)), 0);
+ String address = instance.getPublicAddress().getHostAddress();
+ String port = DruidRealtimeClusterActionHandler.PORT.toString();
+
+ URL url = new URL(String.format("http://%s:%s/druid/v2", address, port));
+ String query = "{\n" +
+ " \"queryType\":\"segmentMetadata\",\n" +
+ " \"dataSource\":\"sample_datasource\",\n" +
+ " \"intervals\":[\"2013-01-01/2014-01-01\"],\n" +
+ "}";
+ URLConnection conn = url.openConnection();
+ conn.setDoOutput(true);
+
+ OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream());
+
+ writer.write(query);
+ writer.flush();
+
+ String line;
+ BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+ StringBuilder builder = new StringBuilder();
+
+ while ((line = reader.readLine()) != null) {
+ builder.append(line);
+ }
+ writer.close();
+ reader.close();
+
+ return builder.toString();
+
+ } catch(IOException e) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e1) {}
+ }
+ }
+ throw new Exception("Unable to get cluster metadata info.");
+ }
+
+ @After
+ public void tearDown() throws IOException, InterruptedException {
+ controller.destroyCluster(clusterSpec);
+ }
+}
diff --git a/services/druid/src/test/resources/whirr-druid-test.properties b/services/druid/src/test/resources/whirr-druid-test.properties
new file mode 100644
index 0000000..a326afc
--- /dev/null
+++ b/services/druid/src/test/resources/whirr-druid-test.properties
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Deploy a Druid Cluster
+#
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=druid
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 zookeeper+druid-realtime+druid-mysql
+
+# Credentials
+whirr.provider=ec2
+
+# Which version of druid to load
+whirr.druid.version=0.5.54
+
+# S3 bucket to store segments in
+whirr.druid.pusher.s3.bucket=dummy_s3_bucket
\ No newline at end of file
