PIG-4880: Overlapping of parameter substitution names inside&outside a macro fails with NPE (knoguchi)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1743527 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 16a5301..cfc219c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -119,6 +119,8 @@
BUG FIXES
+PIG-4880: Overlapping of parameter substitution names inside&outside a macro fails with NPE (knoguchi)
+
PIG-4881: TestBuiltin.testUniqueID failing on hadoop-1.x (knoguchi)
PIG-4888: Line number off when reporting syntax error inside a macro (knoguchi)
diff --git a/src/org/apache/pig/parser/PigMacro.java b/src/org/apache/pig/parser/PigMacro.java
index dc277ab..1f84af4 100644
--- a/src/org/apache/pig/parser/PigMacro.java
+++ b/src/org/apache/pig/parser/PigMacro.java
@@ -168,14 +168,9 @@
Map<String, String> paramVal = pc.getParamVal();
for (Map.Entry<String, String> e : pigContext.getParamVal().entrySet()) {
- if (paramVal.containsKey(e.getKey())) {
- throw new ParserException(
- "Macro contains argument or return value " + e.getKey() + " which conflicts " +
- "with a Pig parameter of the same name."
- );
- } else {
- paramVal.put(e.getKey(), e.getValue());
- }
+ // overwrite=false since macro parameters should have precedence
+ // over commandline parameters (if keys overlap)
+ pc.processOrdLine(e.getKey(), e.getValue(), false);
}
ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(pc);
diff --git a/test/org/apache/pig/test/TestMacroExpansion.java b/test/org/apache/pig/test/TestMacroExpansion.java
index 99c405a..0328b07 100644
--- a/test/org/apache/pig/test/TestMacroExpansion.java
+++ b/test/org/apache/pig/test/TestMacroExpansion.java
@@ -2280,6 +2280,135 @@
verify(script, expected);
}
+ // When declare-in-macro, macro param and command-line param contain the
+ // same name, last declare wins
+ @Test
+ public void testParamOverLap1() throws Exception {
+ String macro =
+ "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
+ " %declare number 333;\n" +
+ " $G = GROUP $REL by $key parallel $number;\n" +
+ "};";
+ createFile("my_macro.pig", macro);
+
+ String script =
+ "%declare number 111;\n" +
+ "IMPORT 'my_macro.pig';\n" +
+ "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
+ "result = mygroupby(data, i, 222);\n" +
+ "STORE result INTO 'test.out' USING PigStorage();";
+
+ String expected =
+ "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
+ "result = GROUP data by (i) parallel 333;\n" +
+ "STORE result INTO 'test.out' USING PigStorage();\n";
+
+ verify(script, expected);
+ }
+
+ // When default-in-macro, macro param and command-line param contain the
+ // same name, then default should be ignored and macro param to be taken
+ @Test
+ public void testParamOverLap2() throws Exception {
+ String macro =
+ "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
+ " %default number 333;\n" +
+ " $G = GROUP $REL by $key parallel $number;\n" +
+ "};";
+ createFile("my_macro.pig", macro);
+
+ String script =
+ "%declare number 111;\n" +
+ "IMPORT 'my_macro.pig';\n" +
+ "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
+ "result = mygroupby(data, i, 222);\n" +
+ "STORE result INTO 'test.out' USING PigStorage();";
+
+ String expected =
+ "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
+ "result = GROUP data by (i) parallel 222;\n" +
+ "STORE result INTO 'test.out' USING PigStorage();\n";
+
+ verify(script, expected);
+ }
+
+ // Overlapping of macro param and command-line param used to be disallowed.
+ // Now, simply taking the macro param when this happens
+ @Test
+ public void testParamOverLap3() throws Exception {
+ String macro =
+ "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
+ " $G = GROUP $REL by $key parallel $number;\n" +
+ "};";
+ createFile("my_macro.pig", macro);
+
+ String script =
+ "%default number 111;\n" +
+ "IMPORT 'my_macro.pig';\n" +
+ "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
+ "result = mygroupby(data, i, 222);\n" +
+ "STORE result INTO 'test.out' USING PigStorage();";
+
+ String expected =
+ "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
+ "result = GROUP data by (i) parallel 222;\n" +
+ "STORE result INTO 'test.out' USING PigStorage();\n";
+
+ verify(script, expected);
+ }
+
+ // Testing inline declare and commandline param overlap.
+ // testParamOverLap1 should cover this case as well but creating a specific
+ // case since this pair used to fail with NPE
+ @Test
+ public void testParamOverLap4() throws Exception {
+ String macro =
+ "DEFINE mygroupby(REL, key) RETURNS G {\n" +
+ " %declare number 333;\n" +
+ " $G = GROUP $REL by $key parallel $number;\n" +
+ "};";
+ createFile("my_macro.pig", macro);
+
+ String script =
+ "%default number 111;\n" +
+ "IMPORT 'my_macro.pig';\n" +
+ "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
+ "result = mygroupby(data, i);\n" +
+ "STORE result INTO 'test.out' USING PigStorage();";
+
+ String expected =
+ "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
+ "result = GROUP data by (i) parallel 333;\n" +
+ "STORE result INTO 'test.out' USING PigStorage();\n";
+
+ verify(script, expected);
+ }
+
+ // default-in-macro should yield to command-line param
+ @Test
+ public void testParamOverLap5() throws Exception {
+ String macro =
+ "DEFINE mygroupby(REL, key) RETURNS G {\n" +
+ " %default number 333;\n" +
+ " $G = GROUP $REL by $key parallel $number;\n" +
+ "};";
+ createFile("my_macro.pig", macro);
+
+ String script =
+ "%declare number 111;\n" +
+ "IMPORT 'my_macro.pig';\n" +
+ "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
+ "result = mygroupby(data, i);\n" +
+ "STORE result INTO 'test.out' USING PigStorage();";
+
+ String expected =
+ "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
+ "result = GROUP data by (i) parallel 111;\n" +
+ "STORE result INTO 'test.out' USING PigStorage();\n";
+
+ verify(script, expected);
+ }
+
//-------------------------------------------------------------------------
private void testMacro(String content) throws Exception {
