/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.pig.parser;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;

import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigServer;
import org.apache.pig.test.Util;
import org.apache.pig.tools.pigstats.PigStats;
import org.junit.Assert;
import org.junit.Test;

public class TestQueryParser {

    @Test
    public void test() throws IOException, RecognitionException  {
        CharStream input = new QueryParserFileStream( "test/org/apache/pig/parser/TestParser.pig" );
        QueryLexer lexer = new QueryLexer(input);
        CommonTokenStream tokens = new  CommonTokenStream(lexer);

        QueryParser parser = new QueryParser(tokens);
        QueryParser.query_return result = parser.query();

        Tree ast = (Tree)result.getTree();

        System.out.println( ast.toStringTree() );
        TreePrinter.printTree( (CommonTree)ast, 0 );
        Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
        Assert.assertEquals( 0, parser.getNumberOfSyntaxErrors() );
    }

    @Test
    // After PIG-438, realias statement is valid
    public void testNegative1() throws IOException, RecognitionException {
        shouldPass("A = load 'x'; B=A;");
    }
    
    @Test(expected=RecognitionException.class)
    public void testNegative2() throws IOException, RecognitionException {
        shouldFail("A = load 'x'; B=(A);");
    }

    @Test(expected=RecognitionException.class)
    public void testNegative3() throws IOException, RecognitionException {
        shouldFail("A = load 'x';B = (A) as (a:int, b:long);");
    }

    @Test(expected=RecognitionException.class)
    public void testNegative4() throws IOException, RecognitionException {
        shouldFail("A = load 'x'; B = ( filter A by $0 == 0 ) as (a:bytearray, b:long);");
    }
    
    @Test(expected=RecognitionException.class)
    public void testNegative5() throws IOException, RecognitionException {
        shouldFail("A = load 'x'; D = group A by $0:long;");
    }
    
    @Test(expected=RecognitionException.class)
    public void testNegative6() throws IOException, RecognitionException {
        shouldFail("A = load '/Users/gates/test/data/studenttab10'; B = foreach A generate $0, 3.0e10.1;");
    }
    
    @Test // test error message with file name
    public void testNagative7() throws IOException {
        File f1 = new File("myscript.pig");
        f1.deleteOnExit();
        
        FileWriter fw1 = new FileWriter(f1);
        fw1.append("A = loadd '1.txt';");
        fw1.close();
        
        String[] args = { "-x", "local", "-c", "myscript.pig" };
        PigStats stats = PigRunner.run(args, null);
       
        Assert.assertFalse(stats.isSuccessful());
        
        String expected = "<file myscript.pig, line 1, column 0>";
        String msg = stats.getErrorMessage();
        
        Assert.assertFalse(msg == null);
        Assert.assertTrue(msg.startsWith(expected));
    }
    
    // See PIG-2238
    @Test
    public void testDependentNullAlias() throws IOException, RecognitionException {
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        try {
            pigServer.registerQuery( "F = limit F 20;store F into 'out';" );
        } catch(Exception ex) {
            Assert.assertTrue(ex.getMessage().contains("Unrecognized alias F"));
            return;
        }
        Assert.fail();
    }
    
    @Test
    public void test2() throws IOException, RecognitionException {
        shouldPass("A = load '/Users/gates/test/data/studenttab10'; B = foreach A generate ( $0 == 0 ? 1 : 0 );");
    }

    @Test
    public void test3() throws IOException, RecognitionException {
        String query = "a = load '1.txt' as (a0);" +
                       "b = foreach a generate flatten( (bag{tuple(map[])})a0 ) as b0:map[];" +
                       "c = foreach b generate (long)b0#'key1';";
        shouldPass( query );
    }

    @Test
    public void test4() throws IOException, RecognitionException {
        String query = "a = load '1.txt'  as (name, age, gpa); b = group a by name;" +
                       "c = foreach b generate group, COUNT(a.age);" +
                       "store c into 'y';";
        shouldPass( query );
    }
    
    @Test
    public void test5() throws IOException, RecognitionException {
        String query = "a = load 'x' as (name, age, gpa);" +
            "b = foreach a generate name, age +  2L, 3.125F, 3.4e2;" +
            " store b into 'y'; ";
        shouldPass( query );
    }
    
    @Test
    public void test6() throws IOException, RecognitionException {
        String query = "a = load '/user/pig/tests/data/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);" +
                       "b = foreach a generate (int)((int)gpa/((int)gpa - 1)) as norm_gpa:int;" + 
                       "c = foreach b generate (norm_gpa is not null? norm_gpa: 0);" +
                       "store c into '/user/pig/out/jianyong.1297229709/Types_37.out';";
        shouldPass( query );
    }
    
    @Test
    public void test7() throws IOException, RecognitionException {
        String query = "a = load '/user/pig/tests/data/singlefile/studenttab10k';" +
                       "b = group a by $0;" +
                       "c = foreach b {c1 = order $1 by * using org.apache.pig.test.udf.orderby.OrdDesc; generate flatten(c1); };" +
                       "store c into '/user/pig/out/jianyong.1297305352/Order_15.out';";
        shouldPass( query );
    }
    
    @Test
    public void test8() throws IOException, RecognitionException {
        String query = "a = load '/user/pig/tests/data/singlefile/studenttab10k';" +
                       "b = group a by $0;" +
                       "c = foreach b {c1 = order $1 by $1; generate flatten(c1), MAX($1.$1); };" +
                       "store c into '/user/pig/out/jianyong.1297305352/Order_17.out';";
        shouldPass( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative1() throws IOException, RecognitionException {
	// cube keyword used as alias
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "cube = cube x by (a, b, c);";
    	shouldFail( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative2() throws IOException, RecognitionException {
	// syntax error - brackets missing
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "y = cube x by a, b, c;";
    	shouldFail( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative3() throws IOException, RecognitionException {
	// syntax error - BY missing
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "y = cube x (a, b, c);";
    	shouldFail( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative4() throws IOException, RecognitionException {
	// syntax error - UDF at the end 
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "y = cube x by (a, b, c), UDF(c);";
    	shouldFail( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative5() throws IOException, RecognitionException {
	// syntax error - specifying just dimensions 
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "y = cube x by (a, b, c), CUBE(c);";
    	shouldFail( query );
    }
    
    @Test(expected=RecognitionException.class)
    public void testCubeNegative6() throws IOException, RecognitionException {
	// syntax error - dangling dimension 
    	String query = "x = load 'cubedata' as (a, b, c, d); " +
    				   "y = cube x by CUBE(a, b, c), y, ROLLUP(c);";
    	shouldFail( query );
    }
    
    @Test
    public void testCubePositive1() throws IOException, RecognitionException {
	// syntactically correct
    	String query = "x = load 'cubedata' as (a, b, c, d);" + 
    				   "y = cube x by cube(a, b, c);" +
    				   "z = foreach y generate flatten(group) as (a, b, c), COUNT(x) as count;" +
    				   "store z into 'cube_output';";
    	shouldPass( query );
    }
    
    @Test
    public void testCubePositive2() throws IOException, RecognitionException {
	// all columns using *
    	String query = "x = load 'cubedata' as (a, b, c, d);" + 
    				   "y = cube x by rollup(*), rollup($2..$3);" +
    				   "z = foreach y generate flatten(group) as (a, b, c, d), COUNT(x) as count;" +
    				   "store z into 'cube_output';";
    	shouldPass( query );
    }
    
    
    @Test
    public void testCubePositive3() throws IOException, RecognitionException {
	// range projection
    	String query = "x = load 'cubedata' as (a, b, c, d);" + 
    				   "y = cube x by cube($0, $1);" +
    				   "z = foreach y generate flatten(group) as (a, b), COUNT(x) as count;" +
    				   "store z into 'cube_output';";
    	shouldPass( query );
    }
    
    @Test
    public void test9() throws IOException, RecognitionException {
        String query = "a = load 'x' as (u,v);" +
                       "b = load 'y' as (u,w);" +
                       "c = join a by u, b by u;" +
                       "d = foreach c generate a::u, b::u, w;";
        shouldPass( query );
    }

    @Test
    public void test10() throws IOException, RecognitionException {
        String query = "a = load 'x' as (name, age, gpa);" +
            "b = FOREACH C GENERATE group, flatten( ( 1 == 2 ? 2 : 3 ) );" +
            " store b into 'y'; ";
        shouldPass( query );
    }

    // 'repl' and such, shouldn't be treated as a constant. So, the following should pass.
    @Test
    public void test11() throws IOException, RecognitionException {
        String query = "a = load 'repl' as (name, age, gpa);" +
            "b = FOREACH C GENERATE group, flatten( ( 1 == 2 ? 2 : 3 ) );" +
            " store b into 'skewed'; ";
        shouldPass( query );
    }

    @Test
    public void testBagType() throws IOException, RecognitionException {
        String query = "a = load '1.txt' as ( u : bag{}, v : bag{tuple(x, y)} );" +
            "b = load '2.x' as ( t : {}, u : {(r,s)}, v : bag{ T : tuple( x, y ) }, w : bag{(z1, z2)} );" +
            "c = load '3.x' as p : int;";
        int errorCount = parse( query );
        Assert.assertTrue( errorCount == 0 );
    }

    @Test
    public void testFlatten() throws IOException, RecognitionException {
        String query = "a = load '1.txt' as ( u, v, w : int );" +
            "b = foreach a generate * as ( x, y, z ), flatten( u ) as ( r, s ), flatten( v ) as d, w + 5 as e:int;";
        int errorCount = parse( query );
        Assert.assertTrue( errorCount == 0 );
    }
    
    @Test //PIG-2083
    public void testNullInBinCondNoSpace() throws IOException{
        String query = "a = load '1.txt' as (a0, a1);" +
        "b = foreach a generate (a0==0?null:2);"; //no space around the null keyword, so the lexer doesn't emit a NULL token
        PigServer pig = new PigServer(ExecType.LOCAL);
        Util.registerMultiLineQuery(pig, query);
        pig.explain("b", System.out);
    }

    @Test
    public void testAST() throws IOException, RecognitionException  {
        CharStream input = new QueryParserFileStream( "test/org/apache/pig/parser/TestAST.pig" );
        QueryLexer lexer = new QueryLexer(input);
        CommonTokenStream tokens = new  CommonTokenStream(lexer);

        QueryParser parser = new QueryParser(tokens);
        QueryParser.query_return result = parser.query();

        Tree ast = (Tree)result.getTree();

        System.out.println( ast.toStringTree() );
        TreePrinter.printTree( (CommonTree)ast, 0 );
        Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
        Assert.assertEquals( 0, parser.getNumberOfSyntaxErrors() );
   
        Assert.assertEquals( "QUERY", ast.getText() );
        Assert.assertEquals( 5, ast.getChildCount() );
        
        for( int i = 0; i < ast.getChildCount(); i++ ) {
            Tree c = ast.getChild( i );
            Assert.assertEquals( "STATEMENT", c.getText() );
        }
        
        Tree stmt = ast.getChild( 0 );
        Assert.assertEquals( "A", stmt.getChild( 0 ).getText() ); // alias
        Assert.assertTrue( "LOAD".equalsIgnoreCase( stmt.getChild( 1 ).getText() ) );
        
        stmt = ast.getChild( 1 );
        Assert.assertEquals( "B", stmt.getChild( 0 ).getText() ); // alias
        Assert.assertTrue( "FOREACH".equalsIgnoreCase( stmt.getChild( 1 ).getText() ) );
        
        stmt = ast.getChild( 2 );
        Assert.assertEquals( "C", stmt.getChild( 0 ).getText() ); // alias
        Assert.assertTrue( "FILTER".equalsIgnoreCase( stmt.getChild( 1 ).getText() ) );

        stmt = ast.getChild( 3 );
        Assert.assertEquals( "D", stmt.getChild( 0 ).getText() ); // alias
        Assert.assertTrue( "LIMIT".equalsIgnoreCase( stmt.getChild( 1 ).getText() ) );

        stmt = ast.getChild( 4 );
        Assert.assertTrue( "STORE".equalsIgnoreCase( stmt.getChild( 0 ).getText() ) );
    }

    @Test
    public void testMultilineFunctionArguments() throws RecognitionException, IOException {
        final String pre = "STORE data INTO 'testOut' \n" +
                           "USING PigStorage (\n";

        String lotsOfNewLines = "'{\"debug\": 5,\n" +
                                "  \"data\": \"/user/lguo/testOut/ComponentActTracking4/part-m-00000.avro\",\n" +
                                "  \"field0\": \"int\",\n" +
                                "  \"field1\": \"def:browser_id\",\n" +
                                "  \"field3\": \"def:act_content\" }\n '\n";

        String [] queries = { lotsOfNewLines,
                            "'notsplitatall'",
                            "'see you\nnext line'",
                            "'surrounded \n by spaces'",
                            "'\nleading newline'",
                            "'trailing newline\n'",
                            "'\n'",
                            "'repeated\n\n\n\n\n\n\n\n\nnewlines'",
                            "'also\ris\rsupported\r'"};

        final String post = ");";

        for(String q : queries) {
            shouldPass(pre + q + post);
        }
    }

    private void shouldPass(String query) throws RecognitionException, IOException {
        System.out.println("Testing: " + query);
        Assert.assertEquals(query + " should have passed", 0, parse(query));
    }

    private void shouldFail(String query) throws RecognitionException, IOException {
        System.out.println("Testing: " + query);
        parse( query );
    }
    
    private int parse(String query) throws IOException, RecognitionException  {
        CharStream input = new QueryParserStringStream( query, null );
        QueryLexer lexer = new QueryLexer(input);
        CommonTokenStream tokens = new  CommonTokenStream(lexer);

        QueryParser parser = QueryParserUtils.createParser(tokens);
        QueryParser.query_return result = parser.query();

        Tree ast = (Tree)result.getTree();

        System.out.println( ast.toStringTree() );
        TreePrinter.printTree((CommonTree) ast, 0);
        Assert.assertEquals(0, lexer.getNumberOfSyntaxErrors());
        return parser.getNumberOfSyntaxErrors();
    }

    //PIG-2267
    @Test
    public void testThatColNameIsGeneratedProperly() throws IOException {
        String query = "a = load '1.txt' as (int,(long,[]),{([])});"
                     + "b = foreach a generate val_0, tuple_0, bag_0;"
                     + "c = foreach b generate val_0, flatten(tuple_0), flatten(bag_0);"
                     + "d = foreach c generate val_0, tuple_0::val_0, bag_0::map_0;";
        PigServer pig = new PigServer(ExecType.LOCAL);
        Util.registerMultiLineQuery(pig, query);
    }


    //RANK
    @Test
    public void testRankPositive1() throws IOException, RecognitionException {
        shouldPass("B = rank A;");
    }

    @Test
    public void testRankPositive2() throws IOException, RecognitionException {
        shouldPass("B = rank A by x;");
    }

    @Test
    public void testRankPositive3() throws IOException, RecognitionException {
        shouldPass("B = rank A by x DESC;");
    }

    @Test
    public void testRankPositive4() throws IOException, RecognitionException {
        shouldPass("B = rank A by x, y ASC, w DESC, z ASC;");
    }

    @Test
    public void testRankPositive5() throws IOException, RecognitionException {
        String query = "A = load 'data' as (x:int, y:chararray, z:int, rz:chararray);";
        query += "B = rank A by x..z;";
        shouldPass(query);
    }

    @Test
    public void testRankPositive6() throws IOException, RecognitionException {
        String query = "A = load 'data' as (x:int, y:chararray, z:int, rz:chararray);";
        query += "B = rank A by *;";
        shouldPass(query);
    }

    @Test
    public void testRankPositive7() throws IOException, RecognitionException {
        String query = "A = load 'data' as (x:int, y:chararray, z:int, rz:chararray);";
        query += "B = rank A by x DESC DENSE;";
        shouldPass(query);
    }

    @Test
    public void testRankPositive8() throws IOException, RecognitionException {
        String query = "A = load 'data' as (x:int, y:chararray, z:int, rz:chararray);";
        query += "B = rank A by x DESC,y ASC DENSE;";
        shouldPass(query);
    }

    @Test
    public void testRankPositive9() throws IOException, RecognitionException {
        String query = "A = load 'data' as (x:int, y:chararray, z:int, rz:chararray);";
        query += "B = rank A by * DENSE;";
        shouldPass(query);
    }
    
    @Test // PIG-2769
    public void testSlowQuery() throws Exception {
        String query = "A = load 'A.txt' using PigStorage() AS (m: int);";
        query += "B = FOREACH A { days_str = (chararray) (m == 1 ? 31: (m == 2 ? 28: (m == 3 ? 31: (m == 4 ? 30: (m == 5 ? 31: (m == 6 ? 30: (m == 7 ? 31: (m == 8 ? 31: (m == 9 ? 30: (m == 10 ? 31: (m == 11 ? 30:31))))))))))); GENERATE days_str as days_str; }";
        query += "store B into 'B';";
        shouldPass(query);
    }
    
    @Test
    public void testFunction1() throws Exception {
        shouldPass("B = foreach A generate org.apache.pig.builtin.CONCAT(b, c);");
    }
    
    @Test
    public void testFunction2() throws Exception {
        shouldPass("B = foreach A generate org.apache.pig.builtin.CONCAT(b, c, 'x', 'y');");
    }
    
    @Test
    public void testFunction3() throws Exception {
        shouldPass("B = foreach A generate flatten(myudfs.Swap(name, age)), gpa;");
    }
    
    @Test
    public void testFilter1() throws Exception {
        shouldPass("E = FILTER D BY not IsEmpty(B);");
    }
    
    @Test
    public void testFilter2() throws Exception {
        shouldPass("C = filter B by 2 > 1;");
    }
    
    @Test
    public void testFilter3() throws Exception {
        shouldPass("C = filter B by a is null;");
    }
    
    @Test
    public void testFilter4() throws Exception {
        shouldPass("C = filter B by a is not null;");
    }
    
    @Test
    public void testGroup1() throws Exception {
        shouldPass("B = group A by ( a, $2 );");
    }
    
    @Test
    public void testCast1() throws Exception {
        shouldPass("B = FOREACH A GENERATE (int)$0 + 1;");
    }
    
    @Test
    public void testCast2() throws Exception {
        shouldPass("B = FOREACH A GENERATE (tuple(int,int,float))fld;");
    }
    
    @Test
    public void testCast3() throws Exception {
        shouldPass("B = FOREACH A GENERATE (bag{tuple(long)})fld; ");
    }
    
    @Test
    public void testCast4() throws Exception {
        shouldPass("B = FOREACH A GENERATE (map[])fld;");
    }
    
    @Test
    public void testCast5() throws Exception {
        shouldPass("E = foreach D generate userid, clicks/(double)C.total, cnt;");
    }
    
    @Test
    public void testCast6() throws Exception {
        shouldPass("X = FOREACH B GENERATE group, (chararray)COUNT(A) AS total;");
    }
    
    @Test
    public void testCast7() throws Exception {
        shouldPass("B = FOREACH A GENERATE a + (int)null;");
    }
    
    @Test
    public void testCast8() throws Exception {
        shouldPass("a = load '1.txt' as(map[int]); --Map value is int");
    }
    
    @Test
    public void testCast9() throws Exception {
        shouldPass("b = foreach a generate (map[(i:int)])a0; -- Map value is tuple");
    }
    
    @Test
    public void testCast10() throws Exception {
        shouldPass("b = stream a through `cat` as (m:map[{(i:int,j:chararray)}]); -- Map value is bag");
    }
    
    @Test
    public void testNull1() throws Exception {
        shouldPass("B = FOREACH A GENERATE a, null;");
    }
    
    @Test
    public void testNull2() throws Exception {
        shouldPass("D = FOREACH C GENERATE FLATTEN((IsEmpty(A) ? null : A)), FLATTEN((IsEmpty(B) ? null : B));");
    }
    
    @Test
    public void testNull3() throws Exception {
        shouldPass("B = FOREACH A GENERATE a + null;");
    }
    
    @Test
    public void testStar1() throws Exception {
        shouldPass("B = FOREACH A GENERATE *, MyUDF(name, age);");
    }
    
    @Test
    public void testStar2() throws Exception {
        shouldPass("C = FOREACH A GENERATE name, age, MyUDF(*);");
    }
    
    @Test
    public void testProjectRange1() throws Exception {
        shouldPass("F = foreach IN generate (int)col0, col1 .. col3; ");
    }
    
    @Test
    public void testProjectRange2() throws Exception {
        shouldPass("SORT = order IN by col2 .. col3, col0, col4 ..; ");
    }

    @Test
    public void testProjectRange3() throws Exception {
        shouldPass("J = join IN1 by $0 .. $3, IN2 by $0 .. $3; ");
    }
    
    @Test
    public void testProjectRange4() throws Exception {
        shouldPass("g = group l1 by b .. c; ");
    }
    
    @Test
    public void testProjection1() throws Exception {
        shouldPass("b = foreach a generate flatten(group), SUM($1.$2);");
    }
    
    @Test
    public void testProjection2() throws Exception {
        shouldPass("a = group (load '/var/folders/bs/cy3sndf95ng5ljgy5nxs1j080000gn/T/test6322762304144938425txt') by ($0,$1);");
    }
    
    @Test
    public void testPartition() throws Exception {
        shouldPass("B = group A by $0 PARTITION BY org.apache.pig.test.utils.SimpleCustomPartitioner parallel 2;");
    }
    
    @Test
    public void testBoolean1() throws Exception {
        shouldPass("C = foreach B generate group,  " +
                "((org.apache.pig.test.utils.AccumulatorBagCount(A)>1 and " +
                "org.apache.pig.test.utils.AccumulatorBagCount(A)<3)?0:1);");
    }
    
    @Test
    public void testBoolean2() throws Exception {
        shouldPass("C = foreach B generate group,  " +
                "((org.apache.pig.test.utils.AccumulatorBagCount(A)>3 or " +
                "org.apache.pig.test.utils.AccumulatorBagCount(A)<2)?0:1);");
    }
    
    @Test
    public void testSplit1() throws Exception {
        shouldPass("split a into b if id > 3, c if id < 3, d otherwise;");
    }
    
    @Test
    public void testSplit2() throws Exception {
        shouldPass("SPLIT logs INTO logins IF command == 'login', all_quits IF command == 'quit';");
    }

    @Test
    public void testBigDecimalParsing() throws Exception {
        shouldPass("B = FILTER A BY $1 < 1234567890.123456789BD;");
    }

    @Test
    public void testBigIntegerParsing() throws Exception {
        shouldPass("B = FILTER A BY $1 < 1234567890123456789BI;");
    }
}