PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark mode after PIG-5157 (nkollar via szita)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1802675 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 3540884..2f262b1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -38,6 +38,8 @@
BUG FIXES
+PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark mode after PIG-5157 (nkollar via szita)
+
PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java b/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
index 03dea96..7bf059b 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
@@ -175,7 +175,6 @@
SparkPigStats sparkStats = (SparkPigStats) pigContext
.getExecutionEngine().instantiatePigStats();
sparkStats.initialize(pigContext, sparkplan, jobConf);
- UDFContext.getUDFContext().addJobConf(jobConf);
PigStats.start(sparkStats);
startSparkIfNeeded(jobConf, pigContext);
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java b/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
index b81bb1a..cff68ea 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
@@ -60,7 +60,12 @@
public static SparkShims getInstance() {
if (sparkShims == null) {
- String sparkVersion = UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+ String sparkVersion;
+ if (UDFContext.getUDFContext().isFrontend()) {
+ sparkVersion = SparkContext.getOrCreate().version();
+ } else {
+ sparkVersion = UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+ }
LOG.info("Initializing SparkShims for Spark version: " + sparkVersion);
String sparkMajorVersion = getSparkMajorVersion(sparkVersion);
try {
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java b/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
index 64f2fc2..75a630c 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
@@ -232,8 +232,8 @@
}
return result;
- } catch (Exception e) {
- log.warn(e);
+ } catch (ExecException e) {
+ log.error(e);
return null;
}
}