CRUNCH-664 Fixes HBase configuration properties being overwritten
Signed-off-by: Josh Wills <jwills@apache.org>
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
index f287d5e..f4f134d 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
@@ -100,7 +100,7 @@
@Override
public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
final Configuration conf = job.getConfiguration();
- HBaseConfiguration.addHbaseResources(conf);
+ HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
conf.setStrings("io.serializations", conf.get("io.serializations"),
MutationSerialization.class.getName());
Class<?> typeClass = ptype.getTypeClass(); // Either Put or Delete
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
index 41d56ff..8593a76 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
@@ -56,7 +56,7 @@
@Override
public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
Configuration conf = job.getConfiguration();
- HBaseConfiguration.addHbaseResources(conf);
+ HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
conf.setStrings("io.serializations", conf.get("io.serializations"),
KeyValueSerialization.class.getName());
super.configureForMapReduce(job, ptype, outputPath, name);
diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java
new file mode 100644
index 0000000..8faa27d
--- /dev/null
+++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class HBaseTargetTest {
+
+ @Test
+ public void testConfigureForMapReduce() throws IOException {
+ Job job = Job.getInstance();
+ // Add the test config file. We can't just call job.getConfiguration().set() because
+ // setting a configuration with .set will always to precedence.
+ job.getConfiguration().addResource("test-hbase-conf.xml");
+
+ HBaseTarget target = new HBaseTarget("testTable");
+ target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name");
+
+ assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period"));
+ }
+}
diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java
new file mode 100644
index 0000000..fbe1a72
--- /dev/null
+++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class HFileTargetTest {
+
+ @Test
+ public void testConfigureForMapReduce() throws IOException {
+ Job job = Job.getInstance();
+ // Add the test config file. We can't just call job.getConfiguration().set() because
+ // setting a configuration with .set will always to precedence.
+ job.getConfiguration().addResource("test-hbase-conf.xml");
+
+ HFileTarget target = new HFileTarget("/");
+ target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name");
+
+ assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period"));
+ }
+}
diff --git a/crunch-hbase/src/test/resources/test-hbase-conf.xml b/crunch-hbase/src/test/resources/test-hbase-conf.xml
new file mode 100644
index 0000000..a6e6dc0
--- /dev/null
+++ b/crunch-hbase/src/test/resources/test-hbase-conf.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>hbase.client.scanner.timeout.period</name>
+ <value>12345</value>
+ </property>
+</configuration>
