PIG-3772: Syntax error when casting an inner schema of a bag and line break involved (ssvinarchukhorton via knoguchi)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1743524 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index f4225ba..cfa4493 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -119,6 +119,8 @@
BUG FIXES
+PIG-3772: Syntax error when casting an inner schema of a bag and line break involved (ssvinarchukhorton via knoguchi)
+
PIG-4892: removing /tmp/output before UT (daijy)
PIG-4882: Remove hardcoded groovy.grape.report.downloads=true from DownloadResolver (erwaman via daijy)
diff --git a/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj b/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
index 85537e9..af882ea 100644
--- a/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
+++ b/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
@@ -267,7 +267,7 @@
<"'"> {prevState = PIG_START;} : IN_STRING
| <"`"> {prevState = PIG_START;} : IN_COMMAND
| <(" " | "\t")+["A","a"]["S","s"](" " | "\t")+ > {prevState = PIG_START;} : SCHEMA_DEFINITION
-| <(" " | "\t")+["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"](" " | "\t")+ > {prevState = PIG_START;} : GENERATE
+| <(" " | "\t")+["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"](" " | "\t" | "\r" | "\n")+ > {prevState = PIG_START;} : GENERATE
| <"{"> {pigBlockLevel = 1;} : IN_BLOCK
| <"}"> {if (true) throw new TokenMgrError("Unmatched '}'", TokenMgrError.LEXICAL_ERROR);}
| <";"> : PIG_END
@@ -395,7 +395,7 @@
{
<"\""> {prevState = IN_BLOCK;} : IN_DOUBLE_QUOTED_STRING
| <(" " | "\t")+["A","a"]["S","s"](" " | "\t")+ > {prevState = IN_BLOCK;} : SCHEMA_DEFINITION
-| <(" " | "\t")+["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"](" " | "\t")+> {prevState = IN_BLOCK;} : GENERATE
+| <(" " | "\t")+["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"](" " | "\t" | "\r" | "\n")+> {prevState = IN_BLOCK;} : GENERATE
| <"{"> {pigBlockLevel++;}
| <"}"(";")?> {pigBlockLevel--; if (pigBlockLevel == 0) SwitchTo(PIG_END);}
| <"'"> {prevState = IN_BLOCK;} : IN_STRING
diff --git a/test/org/apache/pig/TestMain.java b/test/org/apache/pig/TestMain.java
index e1c8111..daae526 100644
--- a/test/org/apache/pig/TestMain.java
+++ b/test/org/apache/pig/TestMain.java
@@ -24,8 +24,10 @@
import static org.junit.Assert.fail;
import java.io.BufferedWriter;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
+import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
@@ -35,6 +37,7 @@
import org.apache.pig.parser.ParserException;
import org.apache.pig.parser.SourceLocation;
import org.apache.pig.test.TestPigRunner.TestNotificationListener;
+import org.apache.pig.test.Util;
import org.apache.pig.tools.parameters.ParameterSubstitutionException;
import org.apache.pig.tools.pigstats.PigStats;
import org.junit.Test;
@@ -152,6 +155,35 @@
}
+ @Test
+ public void testParseInputScript() throws Exception {
+ File input = Util.createInputFile("tmp", "",
+ new String[]{"{(1,1.0)}\ttestinputstring1",
+ "{(2,2.0)}\ttestinputstring1",
+ "{(3,3.0)}\ttestinputstring1",
+ "{(4,4.0)}\ttestinputstring1"}
+ );
+ File out = new File(System.getProperty("java.io.tmpdir")+"/testParseInputScriptOut");
+ File scriptFile = Util.createInputFile("pigScript", "",
+ new String[]{"A = load '"+input.getAbsolutePath()+"' as (a:{(x:chararray, y:float)}, b:chararray);",
+ "B = foreach A generate\n" +
+ " b,\n" +
+ " (bag{tuple(long)}) a.x as ax:{(x:long)};",
+ "store B into '"+out.getAbsolutePath()+"';"}
+ );
+
+ Main.run(new String[]{"-x", "local", scriptFile.getAbsolutePath()}, null);
+ BufferedReader file = new BufferedReader(new FileReader(new File(out.getAbsolutePath()+"/part-m-00000")));
+ String line;
+ int count = 0;
+ while(( line = file.readLine()) != null) {
+ count++;
+ }
+ assertEquals(4,count);
+ Util.deleteDirectory(new File(out.getAbsolutePath()));
+ assertTrue(!new File(out.getAbsolutePath()).exists());
+ }
+
public static class TestNotificationListener2 extends TestNotificationListener {
protected boolean hadArgs = false;
public TestNotificationListener2() {}
