PIG-5078: Script fails with error - POStoreTez only accepts MROutput (rohini)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1779222 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 2231723..7f7f771 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -181,6 +181,8 @@
BUG FIXES
+PIG-5078: Script fails with error - POStoreTez only accepts MROutput (rohini)
+
PIG-5088: HashValuePartitioner has skew when there is only map fields (rohini)
PIG-5043: Slowstart not applied in Tez with PARALLEL clause (rohini)
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
index c73ce42..6e88528 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
@@ -415,7 +415,11 @@
tezOp.getVertexGroupInfo().setVertexGroup(vertexGroup);
POStore store = tezOp.getVertexGroupInfo().getStore();
if (store != null) {
- vertexGroup.addDataSink(store.getOperatorKey().toString(),
+ String outputKey = store.getOperatorKey().toString();
+ if (store instanceof POStoreTez) {
+ outputKey = ((POStoreTez) store).getOutputKey();
+ }
+ vertexGroup.addDataSink(outputKey,
DataSinkDescriptor.create(tezOp.getVertexGroupInfo().getStoreOutputDescriptor(),
OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), dag.getCredentials()));
}
diff --git a/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java b/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
index 7e444d0..f68aeb8 100644
--- a/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
+++ b/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
@@ -259,9 +259,24 @@
for (TezOperator succ : successors) {
if (succ.isVertexGroup() && unionStoreOutputs.get(i).getSFile().equals(succ.getVertexGroupInfo().getSFile())) {
existingVertexGroup = succ;
+ break;
}
}
}
+ if (existingVertexGroup == null) {
+ // In the case of union + split + union + store, the different stores in the Split
+ // will be writing to same location after second union operator is optimized.
+ // So while optimizing the first union, we should just make it write to one vertex group
+ for (int j = 0; j < i; j++) {
+ if (unionStoreOutputs.get(i).getSFile().equals(storeVertexGroupOps[j].getVertexGroupInfo().getSFile())) {
+ storeVertexGroupOps[i] = storeVertexGroupOps[j];
+ break;
+ }
+ }
+ if (storeVertexGroupOps[i] != null) {
+ continue;
+ }
+ }
if (existingVertexGroup != null) {
storeVertexGroupOps[i] = existingVertexGroup;
existingVertexGroup.getVertexGroupMembers().remove(unionOp.getOperatorKey());
@@ -292,6 +307,15 @@
TezOperator[] outputVertexGroupOps = new TezOperator[unionOutputKeys.size()];
String[] newOutputKeys = new String[unionOutputKeys.size()];
for (int i=0; i < outputVertexGroupOps.length; i++) {
+ for (int j = 0; j < i; j++) {
+ if (unionOutputKeys.get(i).equals(unionOutputKeys.get(j))) {
+ outputVertexGroupOps[i] = outputVertexGroupOps[j];
+ break;
+ }
+ }
+ if (outputVertexGroupOps[i] != null) {
+ continue;
+ }
outputVertexGroupOps[i] = new TezOperator(OperatorKey.genOpKey(scope));
outputVertexGroupOps[i].setVertexGroupInfo(new VertexGroupInfo());
outputVertexGroupOps[i].getVertexGroupInfo().setOutput(unionOutputKeys.get(i));
@@ -537,15 +561,24 @@
// Connect predecessor to the storeVertexGroups
int i = 0;
for (TezOperator storeVertexGroup : storeVertexGroupOps) {
+ // Skip connecting if they are already connected. Can happen in case of
+ // union + split + union + store. Because of the split all the stores
+ // will be writing to same location
+ List<OperatorKey> inputs = storeVertexGroup.getVertexGroupInfo().getInputs();
+ if (inputs == null || !inputs.contains(pred.getOperatorKey())) {
+ tezPlan.connect(pred, storeVertexGroup);
+ }
storeVertexGroup.getVertexGroupInfo().addInput(pred.getOperatorKey());
pred.addVertexGroupStore(clonedUnionStoreOutputs.get(i++).getOperatorKey(),
storeVertexGroup.getOperatorKey());
- tezPlan.connect(pred, storeVertexGroup);
}
for (TezOperator outputVertexGroup : outputVertexGroupOps) {
+ List<OperatorKey> inputs = outputVertexGroup.getVertexGroupInfo().getInputs();
+ if (inputs == null || !inputs.contains(pred.getOperatorKey())) {
+ tezPlan.connect(pred, outputVertexGroup);
+ }
outputVertexGroup.getVertexGroupInfo().addInput(pred.getOperatorKey());
- tezPlan.connect(pred, outputVertexGroup);
}
copyOperatorProperties(pred, unionOp);
diff --git a/test/e2e/pig/tests/multiquery.conf b/test/e2e/pig/tests/multiquery.conf
index 0ac20ef..4bef81a 100644
--- a/test/e2e/pig/tests/multiquery.conf
+++ b/test/e2e/pig/tests/multiquery.conf
@@ -741,6 +741,38 @@
d = join b by name, c1 by name using 'replicated';
e = join d by b::name, c2 by name using 'replicated';
store e into ':OUTPATH:';\,
+ },
+ {
+ # Multiple Union + Multiple Split + Single store
+ 'num' => 13,
+ 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:int, gpa);
+b = load ':INPATH:/singlefile/studenttab10k' as (name, age:int, gpa);
+u1 = union onschema a, b;
+SPLIT u1 INTO r IF age < 30, s OTHERWISE;
+c = load ':INPATH:/singlefile/voternulltab10k' as (votername, voterage, registration, contributions);
+d = JOIN r BY name LEFT, c BY votername;
+u2 = UNION ONSCHEMA d, s;
+e = FILTER u2 BY name == 'nick miller';
+f = FILTER u2 BY age > 70 ;
+u3 = UNION ONSCHEMA e, f;
+store u3 into ':OUTPATH:';\,
+ },
+ {
+ # PIG-5082. Similar to MultiQuery_Union_13 but for non-store vertex group
+ 'num' => 14,
+ 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:int, gpa);
+b = load ':INPATH:/singlefile/studenttab10k' as (name, age:int, gpa);
+u1 = union onschema a, b;
+SPLIT u1 INTO r IF age < 30, s OTHERWISE;
+c = load ':INPATH:/singlefile/voternulltab10k' as (votername, voterage, registration, contributions);
+d = JOIN r BY name LEFT, c BY votername;
+u2 = UNION ONSCHEMA d, s;
+e = FILTER u2 BY name == 'nick miller';
+f = FILTER u2 BY age > 70 ;
+u3 = UNION ONSCHEMA e, f;
+SPLIT u3 INTO t if age > 75, u OTHERWISE;
+v = JOIN t BY name LEFT, c BY votername;
+store v into ':OUTPATH:';\,
}
] # end of tests
},
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19-OPTOFF.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19-OPTOFF.gld
new file mode 100644
index 0000000..72b501e
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19-OPTOFF.gld
@@ -0,0 +1,107 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-38 -> Tez vertex scope-40,
+Tez vertex scope-39 -> Tez vertex scope-40,
+Tez vertex scope-40 -> Tez vertex scope-47,Tez vertex scope-49,Tez vertex scope-55,Tez vertex scope-57,
+Tez vertex scope-47 -> Tez vertex scope-49,
+Tez vertex scope-49
+Tez vertex scope-55 -> Tez vertex scope-57,
+Tez vertex scope-57
+
+Tez vertex scope-38
+# Plan on vertex
+POValueOutputTez - scope-42 -> [scope-40]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-39
+# Plan on vertex
+POValueOutputTez - scope-43 -> [scope-40]
+|
+|---c: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][1] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][0] - scope-12
+ |
+ |---b: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-40
+# Plan on vertex
+1-2: Split - scope-61
+| |
+| POValueOutputTez - scope-51 -> [scope-49]
+| |
+| |---d: Filter[bag] - scope-18
+| | |
+| | Less Than or Equal[boolean] - scope-21
+| | |
+| | |---Project[int][0] - scope-19
+| | |
+| | |---Constant(5) - scope-20
+| |
+| POValueOutputTez - scope-59 -> [scope-57]
+| |
+| |---f: Filter[bag] - scope-28
+| | |
+| | Greater Than[boolean] - scope-31
+| | |
+| | |---Project[int][0] - scope-29
+| | |
+| | |---Constant(10) - scope-30
+| |
+| POValueOutputTez - scope-44 -> [scope-47, scope-55]
+|
+|---POShuffledValueInputTez - scope-41 <- [scope-38, scope-39]
+Tez vertex scope-47
+# Plan on vertex
+POValueOutputTez - scope-52 -> [scope-49]
+|
+|---e: Filter[bag] - scope-22
+ | |
+ | Less Than or Equal[boolean] - scope-25
+ | |
+ | |---Project[int][0] - scope-23
+ | |
+ | |---Constant(10) - scope-24
+ |
+ |---POValueInputTez - scope-48 <- scope-40
+Tez vertex scope-49
+# Plan on vertex
+h: Store(file:///tmp/pigoutput/1:org.apache.pig.builtin.PigStorage) - scope-27
+|
+|---POShuffledValueInputTez - scope-50 <- [scope-40, scope-47]
+Tez vertex scope-55
+# Plan on vertex
+POValueOutputTez - scope-60 -> [scope-57]
+|
+|---g: Filter[bag] - scope-32
+ | |
+ | Equal To[boolean] - scope-35
+ | |
+ | |---Project[chararray][1] - scope-33
+ | |
+ | |---Constant(6) - scope-34
+ |
+ |---POValueInputTez - scope-56 <- scope-40
+Tez vertex scope-57
+# Plan on vertex
+i: Store(file:///tmp/pigoutput/2:org.apache.pig.builtin.PigStorage) - scope-37
+|
+|---POShuffledValueInputTez - scope-58 <- [scope-40, scope-55]
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19.gld
new file mode 100644
index 0000000..7cf8914
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19.gld
@@ -0,0 +1,125 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-38 -> Tez vertex group scope-66,Tez vertex group scope-67,
+Tez vertex scope-39 -> Tez vertex group scope-66,Tez vertex group scope-67,
+Tez vertex group scope-66
+Tez vertex group scope-67
+
+Tez vertex scope-38
+# Plan on vertex
+1-2: Split - scope-68
+| |
+| h: Store(file:///tmp/pigoutput/1:org.apache.pig.builtin.PigStorage) - scope-73 -> scope-27
+| |
+| |---d: Filter[bag] - scope-69
+| | |
+| | Less Than or Equal[boolean] - scope-72
+| | |
+| | |---Project[int][0] - scope-70
+| | |
+| | |---Constant(5) - scope-71
+| |
+| h: Store(file:///tmp/pigoutput/1:org.apache.pig.builtin.PigStorage) - scope-78 -> scope-27
+| |
+| |---e: Filter[bag] - scope-74
+| | |
+| | Less Than or Equal[boolean] - scope-77
+| | |
+| | |---Project[int][0] - scope-75
+| | |
+| | |---Constant(10) - scope-76
+| |
+| i: Store(file:///tmp/pigoutput/2:org.apache.pig.builtin.PigStorage) - scope-83 -> scope-37
+| |
+| |---f: Filter[bag] - scope-79
+| | |
+| | Greater Than[boolean] - scope-82
+| | |
+| | |---Project[int][0] - scope-80
+| | |
+| | |---Constant(10) - scope-81
+| |
+| i: Store(file:///tmp/pigoutput/2:org.apache.pig.builtin.PigStorage) - scope-88 -> scope-37
+| |
+| |---g: Filter[bag] - scope-84
+| | |
+| | Equal To[boolean] - scope-87
+| | |
+| | |---Project[chararray][1] - scope-85
+| | |
+| | |---Constant(6) - scope-86
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-39
+# Plan on vertex
+1-2: Split - scope-89
+| |
+| h: Store(file:///tmp/pigoutput/1:org.apache.pig.builtin.PigStorage) - scope-94 -> scope-27
+| |
+| |---d: Filter[bag] - scope-90
+| | |
+| | Less Than or Equal[boolean] - scope-93
+| | |
+| | |---Project[int][0] - scope-91
+| | |
+| | |---Constant(5) - scope-92
+| |
+| h: Store(file:///tmp/pigoutput/1:org.apache.pig.builtin.PigStorage) - scope-99 -> scope-27
+| |
+| |---e: Filter[bag] - scope-95
+| | |
+| | Less Than or Equal[boolean] - scope-98
+| | |
+| | |---Project[int][0] - scope-96
+| | |
+| | |---Constant(10) - scope-97
+| |
+| i: Store(file:///tmp/pigoutput/2:org.apache.pig.builtin.PigStorage) - scope-104 -> scope-37
+| |
+| |---f: Filter[bag] - scope-100
+| | |
+| | Greater Than[boolean] - scope-103
+| | |
+| | |---Project[int][0] - scope-101
+| | |
+| | |---Constant(10) - scope-102
+| |
+| i: Store(file:///tmp/pigoutput/2:org.apache.pig.builtin.PigStorage) - scope-109 -> scope-37
+| |
+| |---g: Filter[bag] - scope-105
+| | |
+| | Equal To[boolean] - scope-108
+| | |
+| | |---Project[chararray][1] - scope-106
+| | |
+| | |---Constant(6) - scope-107
+|
+|---c: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][1] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][0] - scope-12
+ |
+ |---b: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex group scope-66 <- [scope-38, scope-38, scope-39, scope-39] -> null
+# No plan on vertex group
+Tez vertex group scope-67 <- [scope-38, scope-38, scope-39, scope-39] -> null
+# No plan on vertex group
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20-OPTOFF.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20-OPTOFF.gld
new file mode 100644
index 0000000..3509a04
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20-OPTOFF.gld
@@ -0,0 +1,145 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-77 -> Tez vertex scope-78,
+Tez vertex scope-68 -> Tez vertex scope-70,
+Tez vertex scope-69 -> Tez vertex scope-70,
+Tez vertex scope-70 -> Tez vertex scope-78,Tez vertex scope-81,
+Tez vertex scope-78 -> Tez vertex scope-81,
+Tez vertex scope-81 -> Tez vertex scope-88,Tez vertex scope-90,
+Tez vertex scope-88 -> Tez vertex scope-90,
+Tez vertex scope-90
+
+Tez vertex scope-77
+# Plan on vertex
+d: Local Rearrange[tuple]{chararray}(false) - scope-29 -> scope-78
+| |
+| Project[chararray][0] - scope-30
+|
+|---c: New For Each(false)[bag] - scope-22
+ | |
+ | Cast[chararray] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ |
+ |---c: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-18
+Tez vertex scope-68
+# Plan on vertex
+POValueOutputTez - scope-72 -> [scope-70]
+|
+|---a: New For Each(false)[bag] - scope-6
+ | |
+ | Cast[chararray] - scope-4
+ | |
+ | |---Project[bytearray][0] - scope-3
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-2
+Tez vertex scope-69
+# Plan on vertex
+POValueOutputTez - scope-73 -> [scope-70]
+|
+|---b: New For Each(false)[bag] - scope-11
+ | |
+ | Cast[chararray] - scope-9
+ | |
+ | |---Project[bytearray][0] - scope-8
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-7
+Tez vertex scope-70
+# Plan on vertex
+1-3: Split - scope-95
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-27 -> scope-78
+| | |
+| | Project[chararray][0] - scope-28
+| |
+| |---r: Filter[bag] - scope-14
+| | |
+| | Not Equal To[boolean] - scope-17
+| | |
+| | |---Project[chararray][0] - scope-15
+| | |
+| | |---Constant() - scope-16
+| |
+| POValueOutputTez - scope-84 -> [scope-81]
+| |
+| |---u2: New For Each(false,false)[bag] - scope-51
+| | |
+| | Project[chararray][0] - scope-48
+| | |
+| | Constant(DummyVal) - scope-50
+| |
+| |---s: Filter[bag] - scope-43
+| | |
+| | Not[boolean] - scope-47
+| | |
+| | |---Not Equal To[boolean] - scope-46
+| | |
+| | |---Project[chararray][0] - scope-44
+| | |
+| | |---Constant() - scope-45
+|
+|---POShuffledValueInputTez - scope-71 <- [scope-68, scope-69]
+Tez vertex scope-78
+# Plan on vertex
+POValueOutputTez - scope-83 -> [scope-81]
+|
+|---u2: New For Each(false,false)[bag] - scope-42
+ | |
+ | Project[chararray][0] - scope-38
+ | |
+ | Project[chararray][1] - scope-40
+ |
+ |---d: New For Each(true,true)[tuple] - scope-37
+ | |
+ | Project[bag][1] - scope-31
+ | |
+ | POBinCond[bag] - scope-36
+ | |
+ | |---Project[bag][2] - scope-32
+ | |
+ | |---POUserFunc(org.apache.pig.builtin.IsEmpty)[boolean] - scope-34
+ | | |
+ | | |---Project[bag][2] - scope-33
+ | |
+ | |---Constant({()}) - scope-35
+ |
+ |---d: Package(Packager)[tuple]{chararray} - scope-26
+Tez vertex scope-81
+# Plan on vertex
+u2: Split - scope-94
+| |
+| POValueOutputTez - scope-92 -> [scope-90]
+| |
+| |---e: Filter[bag] - scope-56
+| | |
+| | Equal To[boolean] - scope-59
+| | |
+| | |---Project[chararray][0] - scope-57
+| | |
+| | |---Constant() - scope-58
+| |
+| POValueOutputTez - scope-85 -> [scope-88]
+|
+|---POShuffledValueInputTez - scope-82 <- [scope-70, scope-78]
+Tez vertex scope-88
+# Plan on vertex
+POValueOutputTez - scope-93 -> [scope-90]
+|
+|---f: Filter[bag] - scope-62
+ | |
+ | Equal To[boolean] - scope-65
+ | |
+ | |---Project[chararray][0] - scope-63
+ | |
+ | |---Constant(m) - scope-64
+ |
+ |---POValueInputTez - scope-89 <- scope-81
+Tez vertex scope-90
+# Plan on vertex
+u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-67
+|
+|---POShuffledValueInputTez - scope-91 <- [scope-81, scope-88]
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20.gld
new file mode 100644
index 0000000..ffc1bf0
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20.gld
@@ -0,0 +1,197 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-1
+#--------------------------------------------------
+Tez vertex scope-139 -> Tez vertex scope-140,
+Tez vertex scope-130 -> Tez vertex group scope-160,Tez vertex group scope-183,
+Tez vertex scope-131 -> Tez vertex group scope-160,Tez vertex group scope-183,
+Tez vertex group scope-183 -> Tez vertex scope-140,
+Tez vertex scope-140 -> Tez vertex group scope-160,
+Tez vertex group scope-160
+
+Tez vertex scope-139
+# Plan on vertex
+d: Local Rearrange[tuple]{chararray}(false) - scope-91 -> scope-140
+| |
+| Project[chararray][0] - scope-92
+|
+|---c: New For Each(false)[bag] - scope-84
+ | |
+ | Cast[chararray] - scope-82
+ | |
+ | |---Project[bytearray][0] - scope-81
+ |
+ |---c: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-80
+Tez vertex scope-130
+# Plan on vertex
+1-10: Split - scope-184
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-189 -> scope-140
+| | |
+| | Project[chararray][0] - scope-190
+| |
+| |---r: Filter[bag] - scope-185
+| | |
+| | Not Equal To[boolean] - scope-188
+| | |
+| | |---Project[chararray][0] - scope-186
+| | |
+| | |---Constant() - scope-187
+| |
+| u2: Split - scope-199
+| | |
+| | u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-204 -> scope-129
+| | |
+| | |---e: Filter[bag] - scope-200
+| | | |
+| | | Equal To[boolean] - scope-203
+| | | |
+| | | |---Project[chararray][0] - scope-201
+| | | |
+| | | |---Constant() - scope-202
+| | |
+| | u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-129
+| | |
+| | |---f: Filter[bag] - scope-205
+| | | |
+| | | Equal To[boolean] - scope-208
+| | | |
+| | | |---Project[chararray][0] - scope-206
+| | | |
+| | | |---Constant(m) - scope-207
+| |
+| |---u2: New For Each(false,false)[bag] - scope-198
+| | |
+| | Project[chararray][0] - scope-196
+| | |
+| | Constant(DummyVal) - scope-197
+| |
+| |---s: Filter[bag] - scope-191
+| | |
+| | Not[boolean] - scope-195
+| | |
+| | |---Not Equal To[boolean] - scope-194
+| | |
+| | |---Project[chararray][0] - scope-192
+| | |
+| | |---Constant() - scope-193
+|
+|---a: New For Each(false)[bag] - scope-68
+ | |
+ | Cast[chararray] - scope-66
+ | |
+ | |---Project[bytearray][0] - scope-65
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-64
+Tez vertex scope-131
+# Plan on vertex
+1-10: Split - scope-210
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-215 -> scope-140
+| | |
+| | Project[chararray][0] - scope-216
+| |
+| |---r: Filter[bag] - scope-211
+| | |
+| | Not Equal To[boolean] - scope-214
+| | |
+| | |---Project[chararray][0] - scope-212
+| | |
+| | |---Constant() - scope-213
+| |
+| u2: Split - scope-225
+| | |
+| | u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-230 -> scope-129
+| | |
+| | |---e: Filter[bag] - scope-226
+| | | |
+| | | Equal To[boolean] - scope-229
+| | | |
+| | | |---Project[chararray][0] - scope-227
+| | | |
+| | | |---Constant() - scope-228
+| | |
+| | u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-235 -> scope-129
+| | |
+| | |---f: Filter[bag] - scope-231
+| | | |
+| | | Equal To[boolean] - scope-234
+| | | |
+| | | |---Project[chararray][0] - scope-232
+| | | |
+| | | |---Constant(m) - scope-233
+| |
+| |---u2: New For Each(false,false)[bag] - scope-224
+| | |
+| | Project[chararray][0] - scope-222
+| | |
+| | Constant(DummyVal) - scope-223
+| |
+| |---s: Filter[bag] - scope-217
+| | |
+| | Not[boolean] - scope-221
+| | |
+| | |---Not Equal To[boolean] - scope-220
+| | |
+| | |---Project[chararray][0] - scope-218
+| | |
+| | |---Constant() - scope-219
+|
+|---b: New For Each(false)[bag] - scope-73
+ | |
+ | Cast[chararray] - scope-71
+ | |
+ | |---Project[bytearray][0] - scope-70
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-69
+Tez vertex group scope-183 <- [scope-130, scope-131] -> scope-140
+# No plan on vertex group
+Tez vertex scope-140
+# Plan on vertex
+u2: Split - scope-161
+| |
+| u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-166 -> scope-129
+| |
+| |---e: Filter[bag] - scope-162
+| | |
+| | Equal To[boolean] - scope-165
+| | |
+| | |---Project[chararray][0] - scope-163
+| | |
+| | |---Constant() - scope-164
+| |
+| u3: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-171 -> scope-129
+| |
+| |---f: Filter[bag] - scope-167
+| | |
+| | Equal To[boolean] - scope-170
+| | |
+| | |---Project[chararray][0] - scope-168
+| | |
+| | |---Constant(m) - scope-169
+|
+|---u2: New For Each(false,false)[bag] - scope-104
+ | |
+ | Project[chararray][0] - scope-100
+ | |
+ | Project[chararray][1] - scope-102
+ |
+ |---d: New For Each(true,true)[tuple] - scope-99
+ | |
+ | Project[bag][1] - scope-93
+ | |
+ | POBinCond[bag] - scope-98
+ | |
+ | |---Project[bag][2] - scope-94
+ | |
+ | |---POUserFunc(org.apache.pig.builtin.IsEmpty)[boolean] - scope-96
+ | | |
+ | | |---Project[bag][2] - scope-95
+ | |
+ | |---Constant({()}) - scope-97
+ |
+ |---d: Package(Packager)[tuple]{chararray} - scope-88
+Tez vertex group scope-160 <- [scope-140, scope-140, scope-130, scope-130, scope-131, scope-131] -> null
+# No plan on vertex group
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21-OPTOFF.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21-OPTOFF.gld
new file mode 100644
index 0000000..428ce4b
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21-OPTOFF.gld
@@ -0,0 +1,163 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-83 -> Tez vertex scope-84,
+Tez vertex scope-74 -> Tez vertex scope-76,
+Tez vertex scope-75 -> Tez vertex scope-76,
+Tez vertex scope-76 -> Tez vertex scope-84,Tez vertex scope-87,
+Tez vertex scope-84 -> Tez vertex scope-87,
+Tez vertex scope-87 -> Tez vertex scope-94,Tez vertex scope-96,
+Tez vertex scope-94 -> Tez vertex scope-96,
+Tez vertex scope-96 -> Tez vertex scope-104,
+Tez vertex scope-104
+
+Tez vertex scope-83
+# Plan on vertex
+d: Local Rearrange[tuple]{chararray}(false) - scope-29 -> scope-84
+| |
+| Project[chararray][0] - scope-30
+|
+|---c: New For Each(false)[bag] - scope-22
+ | |
+ | Cast[chararray] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ |
+ |---c: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-18
+Tez vertex scope-74
+# Plan on vertex
+POValueOutputTez - scope-78 -> [scope-76]
+|
+|---a: New For Each(false)[bag] - scope-6
+ | |
+ | Cast[chararray] - scope-4
+ | |
+ | |---Project[bytearray][0] - scope-3
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-2
+Tez vertex scope-75
+# Plan on vertex
+POValueOutputTez - scope-79 -> [scope-76]
+|
+|---b: New For Each(false)[bag] - scope-11
+ | |
+ | Cast[chararray] - scope-9
+ | |
+ | |---Project[bytearray][0] - scope-8
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-7
+Tez vertex scope-76
+# Plan on vertex
+1-3: Split - scope-108
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-27 -> scope-84
+| | |
+| | Project[chararray][0] - scope-28
+| |
+| |---r: Filter[bag] - scope-14
+| | |
+| | Not Equal To[boolean] - scope-17
+| | |
+| | |---Project[chararray][0] - scope-15
+| | |
+| | |---Constant() - scope-16
+| |
+| POValueOutputTez - scope-90 -> [scope-87]
+| |
+| |---u2: New For Each(false,false)[bag] - scope-51
+| | |
+| | Project[chararray][0] - scope-48
+| | |
+| | Constant(DummyVal) - scope-50
+| |
+| |---s: Filter[bag] - scope-43
+| | |
+| | Not[boolean] - scope-47
+| | |
+| | |---Not Equal To[boolean] - scope-46
+| | |
+| | |---Project[chararray][0] - scope-44
+| | |
+| | |---Constant() - scope-45
+|
+|---POShuffledValueInputTez - scope-77 <- [scope-74, scope-75]
+Tez vertex scope-84
+# Plan on vertex
+POValueOutputTez - scope-89 -> [scope-87]
+|
+|---u2: New For Each(false,false)[bag] - scope-42
+ | |
+ | Project[chararray][0] - scope-38
+ | |
+ | Project[chararray][1] - scope-40
+ |
+ |---d: New For Each(true,true)[tuple] - scope-37
+ | |
+ | Project[bag][1] - scope-31
+ | |
+ | POBinCond[bag] - scope-36
+ | |
+ | |---Project[bag][2] - scope-32
+ | |
+ | |---POUserFunc(org.apache.pig.builtin.IsEmpty)[boolean] - scope-34
+ | | |
+ | | |---Project[bag][2] - scope-33
+ | |
+ | |---Constant({()}) - scope-35
+ |
+ |---d: Package(Packager)[tuple]{chararray} - scope-26
+Tez vertex scope-87
+# Plan on vertex
+u2: Split - scope-107
+| |
+| POValueOutputTez - scope-98 -> [scope-96]
+| |
+| |---e: Filter[bag] - scope-56
+| | |
+| | Equal To[boolean] - scope-59
+| | |
+| | |---Project[chararray][0] - scope-57
+| | |
+| | |---Constant() - scope-58
+| |
+| POValueOutputTez - scope-91 -> [scope-94]
+|
+|---POShuffledValueInputTez - scope-88 <- [scope-76, scope-84]
+Tez vertex scope-94
+# Plan on vertex
+POValueOutputTez - scope-99 -> [scope-96]
+|
+|---f: Filter[bag] - scope-62
+ | |
+ | Equal To[boolean] - scope-65
+ | |
+ | |---Project[chararray][0] - scope-63
+ | |
+ | |---Constant(m) - scope-64
+ |
+ |---POValueInputTez - scope-95 <- scope-87
+Tez vertex scope-96
+# Plan on vertex
+POValueOutputTez - scope-103 -> [scope-104]
+|
+|---v: Limit - scope-72
+ |
+ |---t: Filter[bag] - scope-68
+ | |
+ | Not Equal To[boolean] - scope-71
+ | |
+ | |---Project[chararray][0] - scope-69
+ | |
+ | |---Constant() - scope-70
+ |
+ |---POShuffledValueInputTez - scope-97 <- [scope-87, scope-94]
+Tez vertex scope-104
+# Plan on vertex
+v: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-73
+|
+|---v: Limit - scope-106
+ |
+ |---POValueInputTez - scope-105 <- scope-96
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21.gld
new file mode 100644
index 0000000..b4645b6
--- /dev/null
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21.gld
@@ -0,0 +1,265 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-83 -> Tez vertex scope-84,
+Tez vertex scope-74 -> Tez vertex group scope-121,Tez vertex group scope-164,
+Tez vertex scope-75 -> Tez vertex group scope-121,Tez vertex group scope-164,
+Tez vertex group scope-164 -> Tez vertex scope-84,
+Tez vertex scope-84 -> Tez vertex group scope-121,
+Tez vertex group scope-121 -> Tez vertex scope-104,
+Tez vertex scope-104
+
+Tez vertex scope-83
+# Plan on vertex
+d: Local Rearrange[tuple]{chararray}(false) - scope-29 -> scope-84
+| |
+| Project[chararray][0] - scope-30
+|
+|---c: New For Each(false)[bag] - scope-22
+ | |
+ | Cast[chararray] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ |
+ |---c: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-18
+Tez vertex scope-74
+# Plan on vertex
+1-3: Split - scope-166
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-171 -> scope-84
+| | |
+| | Project[chararray][0] - scope-172
+| |
+| |---r: Filter[bag] - scope-167
+| | |
+| | Not Equal To[boolean] - scope-170
+| | |
+| | |---Project[chararray][0] - scope-168
+| | |
+| | |---Constant() - scope-169
+| |
+| u2: Split - scope-181
+| | |
+| | POValueOutputTez - scope-191 -> [scope-104]
+| | |
+| | |---v: Limit - scope-190
+| | |
+| | |---t: Filter[bag] - scope-186
+| | | |
+| | | Not Equal To[boolean] - scope-189
+| | | |
+| | | |---Project[chararray][0] - scope-187
+| | | |
+| | | |---Constant() - scope-188
+| | |
+| | |---e: Filter[bag] - scope-182
+| | | |
+| | | Equal To[boolean] - scope-185
+| | | |
+| | | |---Project[chararray][0] - scope-183
+| | | |
+| | | |---Constant() - scope-184
+| | |
+| | POValueOutputTez - scope-201 -> [scope-104]
+| | |
+| | |---v: Limit - scope-200
+| | |
+| | |---t: Filter[bag] - scope-196
+| | | |
+| | | Not Equal To[boolean] - scope-199
+| | | |
+| | | |---Project[chararray][0] - scope-197
+| | | |
+| | | |---Constant() - scope-198
+| | |
+| | |---f: Filter[bag] - scope-192
+| | | |
+| | | Equal To[boolean] - scope-195
+| | | |
+| | | |---Project[chararray][0] - scope-193
+| | | |
+| | | |---Constant(m) - scope-194
+| |
+| |---u2: New For Each(false,false)[bag] - scope-180
+| | |
+| | Project[chararray][0] - scope-178
+| | |
+| | Constant(DummyVal) - scope-179
+| |
+| |---s: Filter[bag] - scope-173
+| | |
+| | Not[boolean] - scope-177
+| | |
+| | |---Not Equal To[boolean] - scope-176
+| | |
+| | |---Project[chararray][0] - scope-174
+| | |
+| | |---Constant() - scope-175
+|
+|---a: New For Each(false)[bag] - scope-6
+ | |
+ | Cast[chararray] - scope-4
+ | |
+ | |---Project[bytearray][0] - scope-3
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-2
+Tez vertex scope-75
+# Plan on vertex
+1-3: Split - scope-202
+| |
+| d: Local Rearrange[tuple]{chararray}(false) - scope-207 -> scope-84
+| | |
+| | Project[chararray][0] - scope-208
+| |
+| |---r: Filter[bag] - scope-203
+| | |
+| | Not Equal To[boolean] - scope-206
+| | |
+| | |---Project[chararray][0] - scope-204
+| | |
+| | |---Constant() - scope-205
+| |
+| u2: Split - scope-217
+| | |
+| | POValueOutputTez - scope-227 -> [scope-104]
+| | |
+| | |---v: Limit - scope-226
+| | |
+| | |---t: Filter[bag] - scope-222
+| | | |
+| | | Not Equal To[boolean] - scope-225
+| | | |
+| | | |---Project[chararray][0] - scope-223
+| | | |
+| | | |---Constant() - scope-224
+| | |
+| | |---e: Filter[bag] - scope-218
+| | | |
+| | | Equal To[boolean] - scope-221
+| | | |
+| | | |---Project[chararray][0] - scope-219
+| | | |
+| | | |---Constant() - scope-220
+| | |
+| | POValueOutputTez - scope-237 -> [scope-104]
+| | |
+| | |---v: Limit - scope-236
+| | |
+| | |---t: Filter[bag] - scope-232
+| | | |
+| | | Not Equal To[boolean] - scope-235
+| | | |
+| | | |---Project[chararray][0] - scope-233
+| | | |
+| | | |---Constant() - scope-234
+| | |
+| | |---f: Filter[bag] - scope-228
+| | | |
+| | | Equal To[boolean] - scope-231
+| | | |
+| | | |---Project[chararray][0] - scope-229
+| | | |
+| | | |---Constant(m) - scope-230
+| |
+| |---u2: New For Each(false,false)[bag] - scope-216
+| | |
+| | Project[chararray][0] - scope-214
+| | |
+| | Constant(DummyVal) - scope-215
+| |
+| |---s: Filter[bag] - scope-209
+| | |
+| | Not[boolean] - scope-213
+| | |
+| | |---Not Equal To[boolean] - scope-212
+| | |
+| | |---Project[chararray][0] - scope-210
+| | |
+| | |---Constant() - scope-211
+|
+|---b: New For Each(false)[bag] - scope-11
+ | |
+ | Cast[chararray] - scope-9
+ | |
+ | |---Project[bytearray][0] - scope-8
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-7
+Tez vertex group scope-164 <- [scope-74, scope-75] -> scope-84
+# No plan on vertex group
+Tez vertex scope-84
+# Plan on vertex
+u2: Split - scope-122
+| |
+| POValueOutputTez - scope-132 -> [scope-104]
+| |
+| |---v: Limit - scope-131
+| |
+| |---t: Filter[bag] - scope-127
+| | |
+| | Not Equal To[boolean] - scope-130
+| | |
+| | |---Project[chararray][0] - scope-128
+| | |
+| | |---Constant() - scope-129
+| |
+| |---e: Filter[bag] - scope-123
+| | |
+| | Equal To[boolean] - scope-126
+| | |
+| | |---Project[chararray][0] - scope-124
+| | |
+| | |---Constant() - scope-125
+| |
+| POValueOutputTez - scope-142 -> [scope-104]
+| |
+| |---v: Limit - scope-141
+| |
+| |---t: Filter[bag] - scope-137
+| | |
+| | Not Equal To[boolean] - scope-140
+| | |
+| | |---Project[chararray][0] - scope-138
+| | |
+| | |---Constant() - scope-139
+| |
+| |---f: Filter[bag] - scope-133
+| | |
+| | Equal To[boolean] - scope-136
+| | |
+| | |---Project[chararray][0] - scope-134
+| | |
+| | |---Constant(m) - scope-135
+|
+|---u2: New For Each(false,false)[bag] - scope-42
+ | |
+ | Project[chararray][0] - scope-38
+ | |
+ | Project[chararray][1] - scope-40
+ |
+ |---d: New For Each(true,true)[tuple] - scope-37
+ | |
+ | Project[bag][1] - scope-31
+ | |
+ | POBinCond[bag] - scope-36
+ | |
+ | |---Project[bag][2] - scope-32
+ | |
+ | |---POUserFunc(org.apache.pig.builtin.IsEmpty)[boolean] - scope-34
+ | | |
+ | | |---Project[bag][2] - scope-33
+ | |
+ | |---Constant({()}) - scope-35
+ |
+ |---d: Package(Packager)[tuple]{chararray} - scope-26
+Tez vertex group scope-121 <- [scope-84, scope-84, scope-76, scope-74, scope-75] -> scope-104
+# No plan on vertex group
+Tez vertex scope-104
+# Plan on vertex
+v: Store(file:///tmp/pigoutput:org.apache.pig.builtin.PigStorage) - scope-73
+|
+|---v: Limit - scope-106
+ |
+ |---POValueInputTez - scope-105 <- scope-121
diff --git a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
index ced251d..d34c18f 100644
--- a/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
+++ b/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
@@ -4,7 +4,7 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-70,Tez vertex group scope-71,Tez vertex group scope-71,
+Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-71,
Tez vertex scope-56 -> Tez vertex group scope-70,Tez vertex group scope-71,
Tez vertex group scope-70
Tez vertex group scope-71
diff --git a/test/org/apache/pig/tez/TestTezCompiler.java b/test/org/apache/pig/tez/TestTezCompiler.java
index cefd1fb..d01a4c2 100644
--- a/test/org/apache/pig/tez/TestTezCompiler.java
+++ b/test/org/apache/pig/tez/TestTezCompiler.java
@@ -1063,6 +1063,71 @@
}
@Test
+ public void testUnionSplitUnionStore() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input1' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "split c into d if x <= 5, e if x <= 10, f if x >10, g if y == '6';" +
+ "h = union onschema d, e;" +
+ "i = union onschema f, g;" +
+ "store h into 'file:///tmp/pigoutput/1';" +
+ "store i into 'file:///tmp/pigoutput/2';";
+
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-19-OPTOFF.gld");
+
+ // With a join in between
+ query =
+ "a = load 'file:///tmp/input' as (x:chararray);" +
+ "b = load 'file:///tmp/input' as (x:chararray);" +
+ "c = load 'file:///tmp/input' as (y:chararray);" +
+ "u1 = union onschema a, b;" +
+ "SPLIT u1 INTO r IF x != '', s OTHERWISE;" +
+ "d = JOIN r BY x LEFT, c BY y;" +
+ "u2 = UNION ONSCHEMA d, s;" +
+ "e = FILTER u2 BY x == '';" +
+ "f = FILTER u2 BY x == 'm';" +
+ "u3 = UNION ONSCHEMA e, f;" +
+ "store u3 into 'file:///tmp/pigoutput';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-20-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionSplitUnionLimitStore() throws Exception {
+ // Similar to previous testcase but a LIMIT at the end to test a non-store vertex group
+ String query =
+ "a = load 'file:///tmp/input' as (x:chararray);" +
+ "b = load 'file:///tmp/input' as (x:chararray);" +
+ "c = load 'file:///tmp/input' as (y:chararray);" +
+ "u1 = union onschema a, b;" +
+ "SPLIT u1 INTO r IF x != '', s OTHERWISE;" +
+ "d = JOIN r BY x LEFT, c BY y;" +
+ "u2 = UNION ONSCHEMA d, s;" +
+ "e = FILTER u2 BY x == '';" +
+ "f = FILTER u2 BY x == 'm';" +
+ "u3 = UNION ONSCHEMA e, f;" +
+ "SPLIT u3 INTO t if x != '', u OTHERWISE;" +
+ "v = LIMIT t 10;" +
+ "store v into 'file:///tmp/pigoutput';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-21-OPTOFF.gld");
+ }
+
+ @Test
public void testRank() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +
