PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1801860 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 2348898..13fc9e9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -36,6 +36,8 @@
BUG FIXES
+PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
+
PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)
PIG-5263: Using wildcard doesn't work with OrcStorage (satishsaley via rohini)
diff --git a/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java b/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
index de34e0d..733ea70 100644
--- a/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
+++ b/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
@@ -116,10 +116,6 @@
return false;
loFilter = (LOFilter)succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter)!=null)
- return false;
-
// we have to check more only if LoadFunc implements LoadMetada
loadFunc = loLoad.getLoadFunc();
if(!( loadFunc instanceof LoadMetadata ) ) {
diff --git a/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java b/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
index d964b3d..ba7c741 100644
--- a/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
+++ b/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
@@ -107,10 +107,6 @@
return false;
loFilter = (LOFilter) succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter) != null)
- return false;
-
// we have to check more only if LoadFunc implements LoadPredicatePushdown
loadFunc = loLoad.getLoadFunc();
if (!(loadFunc instanceof LoadPredicatePushdown)) {
diff --git a/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java b/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
index 2dcfe05..5e06e4d 100644
--- a/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
+++ b/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
@@ -61,10 +61,12 @@
import org.apache.pig.newplan.logical.expression.NotExpression;
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
import org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
@@ -467,10 +469,14 @@
Operator op = newLogicalPlan.getSinks().get(0);
LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
- String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
- .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
- Assert.assertEquals("checking trimmed filter expression:",
- filterExpr, actual);
+ if (unsupportedExpr) {
+ String actual = getTestExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0));
+ Assert.assertEquals("checking trimmed filter expression:", filterExpr, actual);
+ } else {
+ String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
+ .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
+ Assert.assertEquals("checking trimmed filter expression:", filterExpr, actual);
+ }
} else {
Iterator<Operator> it = newLogicalPlan.getOperators();
while( it.hasNext() ) {
@@ -707,6 +713,14 @@
"(not (browser#'type' is null))", true);
}
+ @Test
+ public void testScalarExpressions() throws Exception {
+ String q = "z = load '1line' as (z1:int); " +
+ query3 + "b = filter a by srcid != 10 and srcid == z.z1;" +
+ "store b into 'out';";
+ testFull(q, "(srcid != 10)", "(srcid == z.z1)", true);
+ }
+
//// helper methods ///////
private PartitionFilterExtractor test(String query, List<String> partitionCols,
String expPartFilterString, String expFilterString)
@@ -917,6 +931,10 @@
} else if (op instanceof NotExpression) {
String expr = getTestExpression(((NotExpression) op).getExpression());
return braketize("not " + expr);
+ } else if (op instanceof ScalarExpression) {
+ ScalarExpression scalar = (ScalarExpression) op;
+ return ((LogicalRelationalOperator)scalar.getImplicitReferencedOperator()).getAlias() +
+ "." + scalar.getFieldSchema().alias;
} else {
throw new FrontendException("Unsupported conversion of LogicalExpression to Expression: " + op.getName());
}
