HCATALOG-522 Make HCatHadoopShims.commitJob() generic
git-svn-id: https://svn.apache.org/repos/asf/incubator/hcatalog/trunk@1393629 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 923aeec..4c76264 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -119,6 +119,8 @@
OPTIMIZATIONS
BUG FIXES
+ HCAT-522 Make HCatHadoopShims.commitJob() generic (traviscrawford)
+
HCAT-451 Partitions are created even when Jobs are aborted (avandana)
HCAT-513 Data Store onto HCatalog table fails for dynamic partitioning as the temporary directory gets deleted by the completed map tasks (amalakar via toffer)
diff --git a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
index 848b2fb..54c174b 100644
--- a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
+++ b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
@@ -156,7 +156,7 @@
@Override
public void storeSchema(ResourceSchema schema, String arg1, Job job) throws IOException {
- HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), schema, arg1, job);
+ HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), job);
}
@Override
diff --git a/ivy.xml b/ivy.xml
index 3ddab03..c19972d 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -36,7 +36,7 @@
<dependencies>
<!-- hadoop20 dependencies -->
- <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="default->*"/>
+ <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="test->*"/>
<dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="default->*"/>
<dependency org="javax.management.j2ee" name="management-api" rev="${javax-mgmt.version}"/>
<dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}"/>
diff --git a/server-extensions/pom.xml b/server-extensions/pom.xml
index ee974ad..3f6ee20 100644
--- a/server-extensions/pom.xml
+++ b/server-extensions/pom.xml
@@ -79,5 +79,11 @@
<version>${hadoop20.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>${pig.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/shims/ivy.xml b/shims/ivy.xml
index 773d094..29eb330 100644
--- a/shims/ivy.xml
+++ b/shims/ivy.xml
@@ -11,8 +11,6 @@
<conf name="hadoop23" visibility="private"/>
</configurations>
<dependencies>
- <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="hadoop20->*;hadoop23->*"/>
-
<!-- Hadoop 0.20 shim dependencies. Used for building 0.20 shims. -->
<dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="hadoop20->*">
<exclude org="commons-daemon" module="commons-daemon"/>
diff --git a/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java b/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
index 5deea06..d10a581 100644
--- a/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
+++ b/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
@@ -33,7 +33,6 @@
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
public class HCatHadoopShims20S implements HCatHadoopShims {
@Override
@@ -90,8 +89,7 @@
}
@Override
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException {
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException {
if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
try {
//In local mode, mapreduce will not call OutputCommitter.cleanupJob.
diff --git a/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java b/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
index 80d2042..dfefd40 100644
--- a/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
+++ b/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
@@ -32,7 +32,6 @@
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.net.NetUtils;
@@ -88,8 +87,7 @@
}
@Override
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException {
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException {
// Do nothing as this was fixed by MAPREDUCE-1447.
}
diff --git a/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java b/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
index ee693b4..b5da1c5 100644
--- a/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
+++ b/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
@@ -30,7 +30,6 @@
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
/**
* Shim layer to abstract differences between Hadoop 0.20 and 0.23
@@ -81,8 +80,7 @@
public org.apache.hadoop.mapred.JobContext createJobContext(JobConf conf, JobID jobId, Progressable progressable);
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException;
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException;
public void abortJob(OutputFormat outputFormat, Job job) throws IOException;
