HBASE-21002 make an assembly for hbase-connectors
diff --git a/bin/hbase-connectors b/bin/hbase-connectors
index cb31ec5..50d1b4f 100755
--- a/bin/hbase-connectors
+++ b/bin/hbase-connectors
@@ -107,17 +107,13 @@
-#if [[ -n "$HBASE_CONNECTOR_HEAPSIZE" ]]; then
-# JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_CONNECTOR_HEAPSIZE)"
-#fi
-#
-#if [[ -n "$HBASE_CONNECTOR_OFFHEAPSIZE" ]]; then
-# JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)"
-#fi
+if [[ -n "$HBASE_CONNECTOR_HEAPSIZE" ]]; then
+ JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_CONNECTOR_HEAPSIZE)"
+fi
-
-
-
+if [[ -n "$HBASE_CONNECTOR_OFFHEAPSIZE" ]]; then
+ JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)"
+fi
# so that filenames w/ spaces are handled correctly in loops below
ORIG_IFS=$IFS
@@ -126,11 +122,15 @@
# CLASSPATH initially contains $HBASE_CONNECTOR_CONF_DIR
PASS_CLASSPATH="${HBASE_CONNECTOR_CONF_DIR}"
-#CLASSPATH=${PASS_CLASSPATH}:$JAVA_HOME/lib/tools.jar
-
+CLASSPATH=${PASS_CLASSPATH}:$JAVA_HOME/lib/tools.jar
HBASE_IN_PATH=$(which hbase 2>/dev/null)
+if [ "$HBASE_IN_PATH" = "" ]; then
+ echo "hbase command must be in the path.. aborting"
+ exit 1
+fi
+
# default log directory & file
if [ "$HBASE_CONNECTOR_LOG_DIR" = "" ]; then
HBASE_CONNECTOR_LOG_DIR="$HBASE_CONNECTOR_HOME/logs"
@@ -258,17 +258,12 @@
HBASE_CONNECTOR_OPTS="$HBASE_CONNECTOR_OPTS -Dhbase.connector.id.str=$HBASE_CONNECTOR_IDENT_STRING"
HBASE_CONNECTOR_OPTS="$HBASE_CONNECTOR_OPTS -Dhbase.connector.root.logger=${HBASE_CONNECTOR_ROOT_LOGGER:-INFO,console}"
-
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
HBASE_CONNECTOR_OPTS="$HBASE_CONNECTOR_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH"
fi
-#HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
-
-
-
-
+HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
# by now if we're running a command it means we need logging
for f in ${HBASE_CONNECTOR_HOME}/lib/client-facing-thirdparty/slf4j-*.jar; do
@@ -278,13 +273,10 @@
fi
done
-
-
CLASSPATH=$PASS_CLASSPATH:`$HBASE_IN_PATH classpath`
export CLASSPATH
-
if [ "${HBASE_CONNECTOR_NOEXEC}" != "" ]; then
"$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_CONNECTOR_OPTS $CLASS "$@"
else
diff --git a/hbase-connectors-assembly/src/main/assembly/connector-components.xml b/hbase-connectors-assembly/src/main/assembly/connector-components.xml
index 84e99be..a645f20 100755
--- a/hbase-connectors-assembly/src/main/assembly/connector-components.xml
+++ b/hbase-connectors-assembly/src/main/assembly/connector-components.xml
@@ -36,7 +36,8 @@
<outputDirectory>bin</outputDirectory>
<includes>
<include>hbase-connectors</include>
- <include>hbase-connectors-config.sh</include>
+ <include>hbase-connectors-config.sh</include>
+ <include>hbase-connectors-daemon.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
diff --git a/kafka/README b/kafka/README
deleted file mode 100755
index 7685095..0000000
--- a/kafka/README
+++ /dev/null
@@ -1,126 +0,0 @@
-Hbase Kafka Proxy
-
-Welcome to the hbase kafka proxy. The purpose of this proxy is to act as a 'fake peer'. It
-receives replication events from it's peer and applies a set of rules (stored in
-kafka-route-rules.xml) to determine if the event should be forwared to a topic. If the
-mutation matches one of the rules, the mutation is converted to an avro object and the
-item is placed into the topic.
-
-The service sets up a bare bones region server, so it will use the values in hbase-site.xml. If
-you wish to override those values, pass them as -Dkey=value.
-
-To Use:
-
-1. Make sure the hbase command is in your path. The proxy uses the 'hbase classpath' command to
-find the hbase libraries.
-
-2. Create any topics in your kafka broker that you wish to use.
-
-3. set up kafka-route-rules.xml. This file controls how the mutations are routed. There are
-two kinds of rules: route and drop. drop: any mutation that matches this rule will be dropped.
-route: any mutation that matches this rule will be routed to the configured topic.
-
-Each rule has the following parameters:
-- table
-- columnFamily
-- qualifier
-
-The qualifier parameter can contain simple wildcard expressions (start and end only).
-
-Examples
-
-<rules>
- <rule action="route" table="default:mytable" topic="foo" />
-</rules>
-
-
-This causes all mutations done to default:mytable to be routed to kafka topic 'foo'
-
-
-<rules>
- <rule action="route" table="default:mytable" columnFamily="mycf" qualifier="myqualifier"
- topic="mykafkatopic"/>
-</rules>
-
-This will cause all mutations from default:mytable columnFamily mycf and qualifier myqualifier
-to be routed to mykafkatopic.
-
-
-<rules>
- <rule action="drop" table="default:mytable" columnFamily="mycf" qualifier="secret*"/>
- <rule action="route" table="default:mytable" columnFamily="mycf" topic="mykafkatopic"/>
-</rules>
-
-This combination will route all mutations from default:mytable columnFamily mycf to mykafkatopic
-unless they start with 'secret'. Items matching that rule will be dropped. The way the rule is
-written, all other mutations for column family mycf will be routed to the 'mykafka' topic.
-
-4. Service arguments
-
---kafkabrokers (or -b) <kafka brokers (comma delmited)>
---routerulesfile (or -r) <file with rules to route to kafka (defaults to kafka-route-rules.xml)>
---kafkaproperties (or -f) <Path to properties file that has the kafka connection properties>
---peername (or -p) name of hbase peer to use (defaults to hbasekafka)
---znode (or -z) root znode (defaults to /kafkaproxy)
---enablepeer (or -e) enable peer on startup (defaults to false)]
---auto (or -a) auto create peer
-
-
-5. start the service.
- - make sure the hbase command is in your path
- - ny default, the service looks for route-rules.xml in the conf directory, you can specify a
- differeent file or location with the -r argument
-
-bin/hbase-connectors-daemon.sh start kafkaproxy -a -e -p wootman -b localhost:9092 -r ~/kafka-route-rules.xml
-
-this:
-- starts the kafka proxy
-- passes the -a. The proxy will create the replication peer specified by -p if it does not exist
- (not required, but it savecs some busy work).
-- enables the peer (-e) the proxy will enable the peer when the service starts (not required, you can
- manually enable the peer in the hbase shell)
-
-
-Notes:
-1. The proxy will connect to the zookeeper in hbase-site.xml by default. You can override this by
- passing -Dhbase.zookeeper.quorum
-
- bin/hbase-connectors-daemon.sh start kafkaproxy -Dhbase.zookeeper.quorum=localhost:1234 ..... other args ....
-
-2. route rules only support unicode characters.
-3. I do not have access to a secured hadoop clsuter to test this on.
-
-Message format
-
-Messages are in avro format, this is the schema:
-
-{"namespace": "org.apache.hadoop.hbase.kafka",
- "type": "record",
- "name": "HBaseKafkaEvent",
- "fields": [
- {"name": "key", "type": "bytes"},
- {"name": "timestamp", "type": "long" },
- {"name": "delete", "type": "boolean" },
- {"name": "value", "type": "bytes"},
- {"name": "qualifier", "type": "bytes"},
- {"name": "family", "type": "bytes"},
- {"name": "table", "type": "bytes"}
- ]
-}
-
-Any language that supports Avro should be able to consume the messages off the topic.
-
-
-Testing Utility
-
-A utility is included to test the routing rules.
-
-bin/hbase-connectors-daemon.sh start kafkaproxytest -k <kafka.broker> -t <topic to listen to>
-
-the messages will be dumped in string format under logs/
-
-TODO:
-1. Some properties passed into the region server are hard-coded.
-2. The avro objects should be generic
-3. Allow rules to be refreshed without a restart
-4. Get this tested on a secure (TLS & Kerberos) enabled cluster.
\ No newline at end of file
diff --git a/kafka/hbase-kafka-proxy/pom.xml b/kafka/hbase-kafka-proxy/pom.xml
index 6ebace5..8259bc9 100755
--- a/kafka/hbase-kafka-proxy/pom.xml
+++ b/kafka/hbase-kafka-proxy/pom.xml
@@ -92,25 +92,17 @@
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-annotations</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
- <groupId>org.apache.yetus</groupId>
- <artifactId>audience-annotations</artifactId>
- <version>${audience-annotations.version}</version>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/kafka/hbase-kafka-proxy/src/main/java/org/apache/hadoop/hbase/kafka/KafkaProxy.java b/kafka/hbase-kafka-proxy/src/main/java/org/apache/hadoop/hbase/kafka/KafkaProxy.java
index fda5cf2..4637bd0 100755
--- a/kafka/hbase-kafka-proxy/src/main/java/org/apache/hadoop/hbase/kafka/KafkaProxy.java
+++ b/kafka/hbase-kafka-proxy/src/main/java/org/apache/hadoop/hbase/kafka/KafkaProxy.java
@@ -44,7 +44,6 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hbase.thirdparty.org.apache.commons.cli.OptionGroup;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
diff --git a/pom.xml b/pom.xml
index da176d5..69838c0 100755
--- a/pom.xml
+++ b/pom.xml
@@ -133,16 +133,6 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <version>${hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-annotations</artifactId>
- <version>${hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
<version>${hbase.version}</version>
<type>test-jar</type>
@@ -155,36 +145,25 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <artifactId>hbase-zookeeper</artifactId>
- <groupId>org.apache.hbase</groupId>
- <version>${hbase.version}</version>
- </dependency>
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
+ <scope>provided</scope>
</dependency>
+
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
+ <groupId>org.apache.hbase.connectors</groupId>
+ <artifactId>hbase-kafka-proxy</artifactId>
+ <version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.hbase.connectors</groupId>
- <artifactId>hbase-kafka-proxy</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase.connectors</groupId>
- <artifactId>hbase-kafka-model</artifactId>
- <version>${project.version}</version>
- </dependency>
-
-
+ <dependency>
+ <groupId>org.apache.hbase.connectors</groupId>
+ <artifactId>hbase-kafka-model</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>
