horaedb-sql-javacc/src/main/javacc/CeresDBSqlParser.jj

/**
 * Defines a loose parser, refer to ClickHouse(https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj).
 * It cannot be used to validate SQL on behalf of server, but only for the following purposes:
 * 1) recognize type of each statement(DDL/DML/DCL/TCL, query or mutation etc.)
 * 2) extract table and parameters from a statement
 */
options {
    // DEBUG_LOOKAHEAD = true;
    // DEBUG_PARSER = true;
    // DEBUG_TOKEN_MANAGER = true;

    ERROR_REPORTING = false;
    UNICODE_INPUT = true;
    COMMON_TOKEN_ACTION = true;
}

PARSER_BEGIN(CeresDBSqlParser)

package org.apache.horaedb.sql;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.StringReader;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class CeresDBSqlParser {
    private static final boolean DEBUG = false;

    private static final Logger log = LoggerFactory.getLogger(CeresDBSqlParser.class);

    private final List<CeresDBSqlStatement> statements = new ArrayList<>();

    private ParseHandler handler;

    private boolean tokenIn(int tokenIndex, int... tokens) {
        boolean matched = false;

        int t = getToken(tokenIndex).kind;
        if (tokens != null) {
            for (int i : tokens) {
                if (t == i) {
                    matched = true;
                    break;
                }
            }
        }

        return matched;
    }

    // FIXME ugly workaround but performs better than adding another lexical state for <BETWEEN>...<AND>
    private boolean noAndWithinBetween() {
        return !(getToken(1).kind == AND && token_source.parentToken == BETWEEN);
    }

    public static CeresDBSqlStatement[] parse(String sql) {
        return parse(sql, null);
    }

    public static CeresDBSqlStatement[] parse(String sql, ParseHandler handler) {
        CeresDBSqlStatement[] stmts = new CeresDBSqlStatement[] { new CeresDBSqlStatement(sql, StatementType.UNKNOWN) };

        if (sql == null || sql.isEmpty()) {
            return stmts;
        }

        CeresDBSqlParser p = new CeresDBSqlParser(sql, handler);
        try {
            stmts = p.sql();
        } catch (Exception e) {
            if (DEBUG) {
                throw new IllegalArgumentException(e);
            } else {
                log.warn("{}. If you believe the SQL is valid, please feel free to open an issue on Github with this warning and the following SQL attached.\n{}", e.getMessage(), sql);
            }
        }

        return stmts;
    }

    public CeresDBSqlParser(String sql, ParseHandler handler) {
        this(new StringReader(sql));

        this.handler = handler;
    }

    public void addStatement() {
        if (token_source.isValid()) {
            CeresDBSqlStatement sqlStmt = token_source.build(handler);
            // FIXME remove the restriction once we can handle insertion with format well
            if (statements.size() == 0 || sqlStmt.isRecognized()) {
                statements.add(sqlStmt);
            }
        } else {
            token_source.reset();
        }
    }
}

PARSER_END(CeresDBSqlParser)

TOKEN_MGR_DECLS: {
    // whitespaces and comments are invalid
    private int validTokens = 0;
    // see http://www.engr.mun.ca/~theo/JavaCC-FAQ/javacc-faq-moz.htm#tth_sEc3.17
    private int commentNestingDepth = 0;

    final java.util.Deque<Integer> stack = new java.util.LinkedList<>();
    int parentToken = -1;

    final StringBuilder builder = new StringBuilder();

    StatementType stmtType = StatementType.UNKNOWN;
    String database = null;

    final List<String> tables = new ArrayList<>();
    final List<Integer> parameters = new ArrayList<>();
    final Map<String, Integer> positions = new HashMap<>();

    public void CommonTokenAction(Token t) {
        if (t.kind != CeresDBSqlParserConstants.SEMICOLON) {
            builder.append(t.image);

            if (t.kind != CeresDBSqlParserConstants.EOF) {
                validTokens++;
            }
        }
    }

    void enterToken(int tokenKind) {
        if (tokenKind < 0) {
            return;
        }

        stack.push(parentToken = tokenKind);
    }

    void leaveToken(int tokenKind) {
        if (parentToken == tokenKind) {
            stack.pop();
        }

        parentToken = stack.isEmpty() ? -1 : stack.getLast();
    }

    void processMacro(String name, List<String> params, ParseHandler handler) {
        StringBuilder m = new StringBuilder();
        m.append('#').append(name);

        int startPos = builder.lastIndexOf(m.toString());
        int endPos = params.size() > 0 ? builder.indexOf(")", startPos) + 1 : startPos + m.length();

        builder.delete(startPos, endPos);
        if (handler != null) {
            String replacement = handler.handleMacro(name, params);
            if (replacement != null && !replacement.isEmpty()) {
                builder.insert(startPos, replacement);
            }
        }
    }

    void processParameter(String str, ParseHandler handler) {
        int pos = builder.lastIndexOf(str);
        parameters.add(pos);

        if (handler != null) {
            String replacement = handler.handleParameter(database, tables, parameters.size());
            if (replacement != null && !replacement.isEmpty()) {
                builder.deleteCharAt(pos);
                builder.insert(pos, replacement);
            }
        }
    }

    void append(StringBuilder str) {
        builder.append(str.toString());
    }

    void reset() {
        stack.clear();
        parentToken = -1;

        builder.setLength(validTokens = 0);

        stmtType = StatementType.UNKNOWN;
        database = null;
        tables.clear();
        parameters.clear();
        positions.clear();
    }

    CeresDBSqlStatement build(ParseHandler handler) {
        String sqlStmt = builder.toString();
        CeresDBSqlStatement s = null;
        if (handler != null) {
            s = handler.handleStatement(
                sqlStmt, stmtType, database, tables, parameters, positions);
        }

        if (s == null) {
            s = new CeresDBSqlStatement(
                sqlStmt, stmtType, database, tables, parameters, positions);
        }
        
        // reset variables
        reset();

        return s;
    }

    boolean isValid() {
        return validTokens > 0;
    }

    void setPosition(String keyword) {
        if (keyword == null || keyword.isEmpty()) {
            return;
        }

        this.positions.put(keyword, builder.lastIndexOf(keyword));
    }
}

SKIP: {
    <WHITESPACE: " " | "\t" | "\n" | "\r" | "\u000b" | "\u000c" | "\u0085" | "\u00a0" | "\u180e"
        | "\u2000" | "\u200a" | "\u200b" | "\u200c" | "\u200d" 
        | "\u2028" | "\u2029" | "\u202f" | "\u205f" | "\u2060" | "\u3000" | "\ufeff">
    { append(image); }
    | <SINGLE_LINE_COMMENT: ("--" | "//") (~["\r", "\n"])*> { append(image); }
    | "/*" { commentNestingDepth = 1; append(image); }: MULTI_LINE_COMMENT
}

<MULTI_LINE_COMMENT> SKIP: {
    "/*" { commentNestingDepth += 1; append(image); }
    | "*/" { SwitchTo(--commentNestingDepth == 0 ? DEFAULT : MULTI_LINE_COMMENT); append(image); }
    | < ~[] > { append(image); }
}

// top-level statements
CeresDBSqlStatement[] sql(): {} {
    stmts()
    { addStatement(); }
    (
        (LOOKAHEAD(2) <SEMICOLON>)+
        (stmts())?
        { addStatement(); }
    )*
    <EOF>
    { return statements.toArray(new CeresDBSqlStatement[statements.size()]); }
}

void stmts(): { Token t; } {
    LOOKAHEAD(2) stmt()
    | LOOKAHEAD(2) anyExprList() // in case there's anything new
}

void stmt(): {} {
    alterStmt()      { if (token_source.stmtType == StatementType.UNKNOWN) token_source.stmtType = StatementType.ALTER; }
    | createStmt()   { token_source.stmtType = StatementType.CREATE;   }
    | deleteStmt()   { token_source.stmtType = StatementType.DELETE;   }
    | describeStmt() { token_source.stmtType = StatementType.DESCRIBE; }
    | dropStmt()     { token_source.stmtType = StatementType.DROP;     }
    | insertStmt()   { token_source.stmtType = StatementType.INSERT;   }
    | selectStmt()   { token_source.stmtType = StatementType.SELECT;   }
    | showStmt()     { token_source.stmtType = StatementType.SHOW;     }
    | existsStmt()   { token_source.stmtType = StatementType.EXISTS;   }
}

void alterStmt(): {} {
    <ALTER> <TABLE> tableIdentifier(true) anyExprList()
}

void createStmt(): { Token t; } {
    <CREATE> (
        LOOKAHEAD(2) <TABLE>
        (
            LOOKAHEAD(2)
            <IF> <NOT> t = <EXISTS> { token_source.setPosition(t.image); }
        )?
    )?
    tableIdentifier(true)
    anyExprList() // not interested
}

void deleteStmt(): {} {
    <DELETE> <FROM> tableIdentifier(true) (<WHERE> anyExprList())?
}

void describeStmt(): {} {
    (<DESCRIBE> | <DESC>)
    (LOOKAHEAD({ getToken(1).kind == TABLE }) <TABLE>)? tableIdentifier(true) (anyExprList())?
}

void dropStmt(): { Token t; } {
    <DROP> (
        LOOKAHEAD(2) <TABLE>
        (
            LOOKAHEAD(2)
            <IF> t = <EXISTS> { token_source.setPosition(t.image); }
        )?
    )?
    tableIdentifier(true) (anyExprList())?
}

void insertStmt(): {} {
    <INSERT> <INTO>
    (
        LOOKAHEAD({ getToken(1).kind == FUNCTION }) <FUNCTION> functionExpr() 
        | (LOOKAHEAD(2) <TABLE>)? tableIdentifier(true)
    )
    (LOOKAHEAD(2) <LPAREN> columnExprList() <RPAREN>)?
    dataClause()
}

void dataClause(): { Token t; } {
    try {
        anyIdentifier() (LOOKAHEAD(2) anyExprList())?
        | LOOKAHEAD(2) t = <VALUES> { token_source.setPosition(t.image); }
            <LPAREN> columnExprList() <RPAREN>
            (
                LOOKAHEAD(2)
                (<COMMA>)?
                <LPAREN> columnExprList() <RPAREN>
            )*
        | anyExprList() // not interested
    } catch (ParseException e) {
        Token nextToken;
        do {
            nextToken = getNextToken();
        } while(nextToken.kind != SEMICOLON && nextToken.kind != EOF);
    }
}

void selectStmt(): {} {
    (withClause())?
    <SELECT> (LOOKAHEAD(2) <DISTINCT>)? (LOOKAHEAD(2) topClause())? columnExprList()
    (LOOKAHEAD(2) <FROM> tableClause(true) (<COMMA> tableClause(true))*)?
    (LOOKAHEAD(2) <INNER> <JOIN> tableClause(true) (<COMMA> tableClause(true))*)?
    (LOOKAHEAD(2) <LEFT> <JOIN> tableClause(true) (<COMMA> tableClause(true))*)?
    (LOOKAHEAD(2) <RIGHT> <JOIN> tableClause(true) (<COMMA> tableClause(true))*)?
    (LOOKAHEAD(2) <JOIN> tableClause(true) (<COMMA> tableClause(true))*)?
    (LOOKAHEAD(2) anyExprList())? // everything else
}

void withClause(): {} {
    <WITH> { token_source.enterToken(WITH); } withExpr() (<COMMA> withExpr())*
    { token_source.leaveToken(WITH); }
}

void topClause(): {} { <TOP> <DECIMAL_LITERAL> (LOOKAHEAD(2) <WITH> <TIES>)? }

void tableClause(boolean record): { Token t; } {
    (LOOKAHEAD(2) tableIdentifier(record) | (<LPAREN> anyExprList() <RPAREN>))
    (LOOKAHEAD(2) t = aliasExpr() { if (record && t != null) token_source.tables.add(SqlUtils.unescape(t.image)); } )?
    (LOOKAHEAD(2) <FINAL>)?
    (LOOKAHEAD(2) <SAMPLE> numberLiteral() (LOOKAHEAD(2) <SLASH> numberLiteral())?
        (LOOKAHEAD(2) <OFFSET> numberLiteral() (LOOKAHEAD(2) <SLASH> numberLiteral())?)?
    )?
}

void showStmt(): {} {
    <SHOW>
    (<CREATE> <TABLE>)? tableIdentifier(true)
    (LOOKAHEAD(2) anyExprList())?
}

void existsStmt(): {} {
    <EXISTS>
    (<TABLE>)? tableIdentifier(true)
    (LOOKAHEAD(2) anyExprList())?
}

// columns
void columnExprList(): {} {
    columnsExpr() (<COMMA> columnsExpr())*
}

void withExpr(): {} {
    nestedExpr()
    (
        (
            LOOKAHEAD({ getToken(1).kind == FLOATING_LITERAL })
            <FLOATING_LITERAL> | <DOT> <DECIMAL_LITERAL>
        )+
        | (LOOKAHEAD(2) <LBRACKET> anyExprList() <RBRACKET>)+
        | LOOKAHEAD(2) <IS> (<NOT>)? <NULL>
        | LOOKAHEAD(2) (<NOT>)? betweenExpr()
        | LOOKAHEAD(2) (<NOT>)? (<ILIKE> | <LIKE>) nestedExpr()
        | LOOKAHEAD(2, { noAndWithinBetween() }) (LOOKAHEAD(2, { noAndWithinBetween() }) calcExpr())+
        | LOOKAHEAD(2) (<NOT>)? <IN> nestedExpr()
        | LOOKAHEAD(2) <QUERY> nestedExpr() <COLON> nestedExpr()
        | LOOKAHEAD(2) <AS> columnExpr()
    )?
}

void columnsExpr(): {} {
    LOOKAHEAD(allColumnsExpr()) allColumnsExpr()
    (
        LOOKAHEAD(2) (<APPLY> | <EXCEPT> | <REPLACE>) <LPAREN> anyExprList() <RPAREN>
    )*
    | nestedExpr()
    (
        (
            LOOKAHEAD({ getToken(1).kind == FLOATING_LITERAL })
            <FLOATING_LITERAL> | <DOT> <DECIMAL_LITERAL>
        )+
        | (LOOKAHEAD(2) <LBRACKET> anyExprList() <RBRACKET>)+
        | LOOKAHEAD(2) <IS> (<NOT>)? <NULL>
        | LOOKAHEAD(2) (<NOT>)? betweenExpr()
        | LOOKAHEAD(2) (<NOT>)? (<ILIKE> | <LIKE>) nestedExpr()
        | LOOKAHEAD(2, { noAndWithinBetween() }) (LOOKAHEAD(2, { noAndWithinBetween() }) calcExpr())+
        | LOOKAHEAD(2) (<NOT>)? <IN> nestedExpr()
        | LOOKAHEAD(2) <QUERY> nestedExpr() <COLON> nestedExpr()
        | LOOKAHEAD(2) aliasExpr()
    )?
}

void allColumnsExpr(): {} {
    <ASTERISK> | anyIdentifier() <DOT> (LOOKAHEAD(2) anyIdentifier() <DOT>)? <ASTERISK>
}

void nestedExpr(): {} {
    LOOKAHEAD(2) (<DASH> | <NOT>) nestedExpr()
    | LOOKAHEAD(2) <CASE> (LOOKAHEAD({ getToken(1).kind != WHEN }) nestedExpr())? 
        (<WHEN> nestedExpr() <THEN> nestedExpr())+ (<ELSE> nestedExpr())? <END>
    | LOOKAHEAD(2) <INTERVAL> (LOOKAHEAD(2) <STRING_LITERAL> | nestedExpr() interval())
    | columnExpr()
    (
        (
            <FLOATING_LITERAL> | <DOT> <DECIMAL_LITERAL>
        )+
        | (LOOKAHEAD(2) <LBRACKET> anyExprList() <RBRACKET>)+
        | LOOKAHEAD(2) <IS> (<NOT>)? <NULL>
        | LOOKAHEAD(2) (<NOT>)? betweenExpr()
        | LOOKAHEAD(2) (<NOT>)? (<ILIKE> | <LIKE>) nestedExpr()
        | LOOKAHEAD(2, { noAndWithinBetween() }) (LOOKAHEAD(2, { noAndWithinBetween() }) calcExpr())+
        | LOOKAHEAD(2) (<NOT>)? <IN> nestedExpr()
        | LOOKAHEAD(2) <QUERY> nestedExpr() <COLON> nestedExpr()
    )?
}

void calcExpr(): {} {
    (<AND> | <OR> | operator()) nestedExpr()
}

void betweenExpr(): {} {
    <BETWEEN> { token_source.enterToken(BETWEEN); }
    nestedExpr() 
    <AND>  { token_source.leaveToken(BETWEEN); }
    nestedExpr()
}

void functionExpr(): {} {
    anyIdentifier() <LPAREN> (anyExprList())? <RPAREN>
    (LOOKAHEAD(2) <LPAREN> (anyExprList())? <RPAREN>)?
}

void columnExpr(): { Token t; } {
    t = <QUERY> { token_source.processParameter(t.image, handler); }
    | <LBRACE> (LOOKAHEAD(2) anyExprList())? <RBRACE>
    | <LBRACKET> (LOOKAHEAD(2) anyExprList())? <RBRACKET>
    | <LPAREN> anyExprList() <RPAREN>
    | (LOOKAHEAD(2) macro())+
    | LOOKAHEAD(2, { !(tokenIn(1, INF, NAN, NULL) && tokenIn(2, DOT)) }) literal()
    | LOOKAHEAD(2, { getToken(2).kind == LPAREN }) functionExpr()
    | anyIdentifier() (LOOKAHEAD(2) <DOT> anyIdentifier())*
}

void withTotalPart(): { Token t; } {
    <WITH> (LOOKAHEAD(2) t = <TOTALS> { token_source.setPosition(t.image); })?
}

// expressions
void anyExprList(): {} {
    anyExpr() (LOOKAHEAD(2) <COMMA> | anyExpr())*
}

void anyExpr(): {} {
    anyNestedExpr() (
        LOOKAHEAD(2)
        (
            // TODO needs to extract parameters(and exclude ternary operator) here
            <AT> | <AND> | <OR> | <COLON> | <QUERY> | operator()
        )? anyNestedExpr()
    )*
}

void anyNestedExpr(): {} {
    LOOKAHEAD(2) withTotalPart()
    | (LOOKAHEAD(2) <DASH>)? anyColumnExpr() (
        LOOKAHEAD({ getToken(1).kind == FLOATING_LITERAL })
        <FLOATING_LITERAL> | <DOT> <DECIMAL_LITERAL>
    )*
}

void anyColumnExpr(): { Token t; } {
    // <ASTERISK>
    t = <QUERY> { token_source.processParameter(t.image, handler); }
    | <LBRACE> (LOOKAHEAD(2) anyExprList())? <RBRACE>
    | <LBRACKET> (LOOKAHEAD(2) anyExprList())? <RBRACKET>
    | <LPAREN> (LOOKAHEAD(2) anyExprList())? <RPAREN>
    | (LOOKAHEAD(2) macro())+
    | LOOKAHEAD(2, { !(tokenIn(1, INF, NAN, NULL) && tokenIn(2, DOT)) }) literal()
    // | (LOOKAHEAD(2, { !(tokenIn(1, INF, NAN, NULL)) }) <FLOATING_LITERAL> | literal())
    | nestedIdentifier()
}

Token aliasExpr(): { Token t = null; } {
    (
        LOOKAHEAD(2) <AS> t = anyIdentifier()
        | t = identifier()
    )
    { return t; }
}

void nestedIdentifier(): {} {
    (<ASTERISK> | anyIdentifier()) (LOOKAHEAD(2) <DOT> (<ASTERISK> | anyIdentifier()))*
}

void tableIdentifier(boolean record): { Token t; } {
    (
        (LOOKAHEAD(2) databaseIdentifier(record) <DOT>)? t = anyIdentifier()
        (LOOKAHEAD(2) <LPAREN> anyExprList() <RPAREN>)?
    )
    {
        if (record && t != null) {
            token_source.tables.add(SqlUtils.unescape(t.image));
        }
    }
}

void databaseIdentifier(boolean record): { Token t; } {
    t = anyIdentifier() { if (record) token_source.database = SqlUtils.unescape(t.image); }
}

void settingExprList(): {} {
    settingExpr() (<COMMA> settingExpr())*
}

void settingExpr(): {} {
    identifier() <EQ_SINGLE> literal()
}

// basics
Token anyIdentifier(): { Token t; } {
    (
        t = <BACK_QUOTED_NAME>
        | t = <DOUBLE_QUOTED_NAME>
        | t = variable()
        | t = <IDENTIFIER>
        | t = anyKeyword()
    )
    { return t; }
}

Token identifier(): { Token t; } {
    (
        t = <BACK_QUOTED_NAME>
        | t = <DOUBLE_QUOTED_NAME>
        | t = variable()
        | t = <IDENTIFIER>
        | t = keyword()
    )
    { return t; }
}

void interval(): {} {
    <SECOND> | <MINUTE> | <HOUR> | <DAY> | <WEEK> | <MONTH> | <QUARTER> | <YEAR>
}

Token literal(): { Token t; } {
    (
        t = dateLiteral()
        | t = numberLiteral()
        | t = <STRING_LITERAL>
        | t = <NULL>
    )
    { return t; }
}

Token dateLiteral(): { Token t; String prefix; } {
    (t = <DATE> | t = <TIMESTAMP>) { prefix = t.image; }
    t = <STRING_LITERAL>
    { return Token.newToken(0, prefix + " " + t.image); }
}

Token numberLiteral(): { Token t = null; StringBuilder sb = new StringBuilder(); } {
    (
        (t = <PLUS> | t = <DASH>)? { if (t != null) sb.append(t.image); }
        (
            LOOKAHEAD(2)
            t = <DECIMAL_LITERAL> | t = <HEXADECIMAL_LITERAL> | t = <FLOATING_LITERAL> | t = <INF> | t = <NAN>
        ) { sb.append(t.image); }
    )
    { return Token.newToken(0, sb.toString()); }
}

void operator(): {} {
    (<ARROW> | <ASTERISK> | <CONCAT> | <DASH> | <EQ_DOUBLE> | <EQ_SINGLE>
    | <GE> | <GT> | <LE> | <LT> | <NOT_EQ> | <PERCENT> | <PLUS> | <SLASH>)
}

void macro(): { 
    Token t; 
    String name; 
    List<String> params = new ArrayList<>();
} {
    (
        <HASH>
        (t = anyKeyword() | t = <IDENTIFIER>) { name = t.image; }
        (
            LOOKAHEAD(2) <LPAREN>
            t = <STRING_LITERAL> { params.add(SqlUtils.unescape(t.image)); }
            (<COMMA> t = <STRING_LITERAL> { params.add(SqlUtils.unescape(t.image)); })*
            <RPAREN>
        )?
    )
    { token_source.processMacro(name, params, handler); }
}

Token variable(): { Token t; } {
    (<AT> <AT> (t = anyKeyword() | t = <IDENTIFIER>))
    {
        return Token.newToken(0, "@@" + t.image);
    }
}

Token anyKeyword(): { Token t; } {
    (
        // leading keywords(except with)
        t = <ALTER> | t = <ATTACH> | t = <CHECK> | t = <CREATE> | t = <DELETE> | t = <DESC> | t = <DESCRIBE>
        | t = <DETACH> | t = <DROP> | t = <EXISTS> | t = <EXPLAIN> | t = <GRANT> | t = <INSERT> | t = <KILL>
        | t = <OPTIMIZE> | t = <RENAME> | t = <REVOKE> | t = <SELECT> | t = <SET> | t = <SHOW> | t = <SYSTEM>
        | t = <TRUNCATE> | t = <UPDATE> | t = <USE> | t = <WATCH>
        // others
        | t = <ALL> | t = <AND> | t = <APPLY> | t = <ARRAY> | t = <AS> | t = <ASOF> | t = <BETWEEN> | t = <CASE> 
        | t = <DATE> | t = <DATABASE> | t = <DATABASES> | t = <DICTIONARY> | t = <DICTIONARIES>
        | t = <DISTINCT> | t = <ELSE> | t = <END> | t = <EXCEPT> | t = <FROM> | t = <FINAL> | t = <FULL>
        | t = <FUNCTION> | t = <GLOBAL> | t = <GROUP> | t = <HAVING> | t = <IF> | t = <ILIKE> | t = <IN> | t = <INNER> 
        | t = <INTERVAL> | t = <INTO> | t = <IS> | t = <JOIN> | t = <LEFT> | t = <LIKE> | t = <LIMIT> | t = <LIVE> 
        | t = <MATERIALIZED> | t = <NOT> | t = <OFFSET> | t = <ON> | t = <OR> | t = <ORDER> | t = <POLICY>
        | t = <PREWHERE> | t = <PROFILE> | t = <QUOTA> | t = <REPLACE> | t = <RIGHT> | t = <ROLE> | t = <ROW> | t = <SAMPLE>
        | t = <SETTINGS> | t = <TEMPORARY> | t = <TABLE> | t = <TABLES> | t = <THEN> | t = <TIES> | t = <TIMESTAMP> 
        | t = <TOP> | t = <TOTALS> | t = <VALUES> | t = <VIEW> | t = <USER> | t = <UNION> | t = <USING> 
        | t = <WHEN> | t = <WHERE> | t = <WITH>
        // interval
        | t = <SECOND> | t = <MINUTE> | t = <HOUR> | t = <DAY> | t = <WEEK> | t = <MONTH> | t = <QUARTER> | t = <YEAR>
        // values
        | t = <INF> | t = <NAN> | t = <NULL>
    )
    { return t; }
}

Token keyword(): { Token t; } {
    (
        // leading keywords(except with)
        t = <ALTER> | t = <ATTACH> | t = <CHECK> | t = <CREATE> | t = <DELETE> | t = <DESC> | t = <DESCRIBE>
        | t = <DETACH> | t = <DROP> | t = <EXISTS> | t = <EXPLAIN> | t = <GRANT> | t = <INSERT> | t = <KILL>
        | t = <OPTIMIZE> | t = <RENAME> | t = <REVOKE> | t = <SELECT> | t = <SET> | t = <SHOW> | t = <SYSTEM>
        | t = <TRUNCATE> | t = <UPDATE> | t = <USE> | t = <WATCH>
        // others
        | t = <CASE> | t = <DATE> | t = <DATABASE> | t = <DATABASES> | t = <DICTIONARY>
        | t = <DICTIONARIES> | t = <DISTINCT> | t = <ELSE> | t = <END> | t = <EXCEPT>| t = <FUNCTION> 
        | t = <IF> | t = <INTERVAL> | t = <IS> | t = <LIVE>  | t = <MATERIALIZED> | t = <POLICY>
        | t = <PROFILE> | t = <QUOTA> | t = <REPLACE> | t = <ROLE> | t = <ROW> | t = <TEMPORARY> 
        | t = <TABLE> | t = <TABLES> | t = <THEN> | t = <TIES> | t = <TIMESTAMP> | t = <TOP> | t = <TOTALS> 
        | t = <VALUES> | t = <VIEW> | t = <WHEN> | t = <USER>
        // interval
        | t = <SECOND> | t = <MINUTE> | t = <HOUR> | t = <DAY> | t = <WEEK> | t = <MONTH> | t = <QUARTER> | t = <YEAR>
        // values
        | t = <INF> | t = <NAN> | t = <NULL>
    )
    { return t; }
}

// keywords
TOKEN: {
    <ALTER    : <A> <L> <T> <E> <R>>
    | <ATTACH   : <A> <T> <T> <A> <C> <H>        >
    | <CHECK    : <C> <H> <E> <C> <K>            >
    | <CREATE   : <C> <R> <E> <A> <T> <E>        >
    | <DELETE   : <D> <E> <L> <E> <T> <E>        >
    | <DESC     : <D> <E> <S> <C>                >
    | <DESCRIBE : <D> <E> <S> <C> <R> <I> <B> <E>>
    | <DETACH   : <D> <E> <T> <A> <C> <H>        >
    | <DROP     : <D> <R> <O> <P>                >
    | <EXISTS   : <E> <X> <I> <S> <T> <S>        >
    | <EXPLAIN  : <E> <X> <P> <L> <A> <I> <N>    >
    | <GRANT    : <G> <R> <A> <N> <T>            >
    | <INSERT   : <I> <N> <S> <E> <R> <T>        >
    | <KILL     : <K> <I> <L> <L>                >
    | <OPTIMIZE : <O> <P> <T> <I> <M> <I> <Z> <E>>
    | <RENAME   : <R> <E> <N> <A> <M> <E>        >
    | <REVOKE   : <R> <E> <V> <O> <K> <E>        >
    | <SELECT   : <S> <E> <L> <E> <C> <T>        >
    | <SET      : <S> <E> <T>                    >
    | <SHOW     : <S> <H> <O> <W>                >
    | <SYSTEM   : <S> <Y> <S> <T> <E> <M>        >
    | <TRUNCATE : <T> <R> <U> <N> <C> <A> <T> <E>>
    | <UPDATE   : <U> <P> <D> <A> <T> <E>        >
    | <USE      : <U> <S> <E>                    >
    | <WATCH    : <W> <A> <T> <C> <H>            >

    | <ALL      : <A> <L> <L>                        >
    | <AND      : <A> <N> <D>                        >
    | <APPLY    : <A> <P> <P> <L> <Y>                >
    | <ARRAY    : <A> <R> <R> <A> <Y>                >
    | <AS       : <A> <S>                            >
    | <ASOF     : <A> <S> <O> <F>                    >
    | <BETWEEN  : <B> <E> <T> <W> <E> <E> <N>        >
    | <CASE     : <C> <A> <S> <E>                    >
    | <DATE     : <D> <A> <T> <E>                    >
    | <DATABASE : <D> <A> <T> <A> <B> <A> <S> <E>    >
    | <DATABASES: <D> <A> <T> <A> <B> <A> <S> <E> <S>>
    | <DICTIONARY  : <D> <I> <C> <T> <I> <O> <N> <A> <R> <Y>        >
    | <DICTIONARIES: <D> <I> <C> <T> <I> <O> <N> <A> <R> <I> <E> <S>>
    | <DISTINCT : <D> <I> <S> <T> <I> <N> <C> <T>    >
    | <ELSE     : <E> <L> <S> <E>                    >
    | <END      : <E> <N> <D>                        >
    | <EXCEPT   : <E> <X> <C> <E> <P> <T>            >
    | <FINAL    : <F> <I> <N> <A> <L>                >
    | <FROM     : <F> <R> <O> <M>                    >
    | <FULL     : <F> <U> <L> <L>                    >
    | <FUNCTION : <F> <U> <N> <C> <T> <I> <O> <N>    >
    | <GLOBAL   : <G> <L> <O> <B> <A> <L>            >
    | <GROUP    : <G> <R> <O> <U> <P>                >
    | <HAVING   : <H> <A> <V> <I> <N> <G>            >
    | <IF       : <I> <F>                            >
    | <ILIKE    : <I> <L> <I> <K> <E>                >
    | <IN       : <I> <N>                            >
    | <INNER    : <I> <N> <N> <E> <R>                >
    | <INTERVAL : <I> <N> <T> <E> <R> <V> <A> <L>    >
    | <INTO     : <I> <N> <T> <O>                    >
    | <IS       : <I> <S>                            >
    | <JOIN     : <J> <O> <I> <N>                    >
    | <LEFT     : <L> <E> <F> <T>                    >
    | <LIKE     : <L> <I> <K> <E>                    >
    | <LIMIT    : <L> <I> <M> <I> <T>                >
    | <LIVE     : <L> <I> <V> <E>                    >
    | <MATERIALIZED: <M> <A> <T> <E> <R> <I> <A> <L> <I> <Z> <E> <D>>
    | <NOT      : <N> <O> <T>                        >
    | <OFFSET   : <O> <F> <F> <S> <E> <T>            >
    | <ON       : <O> <N>                            >
    | <OR       : <O> <R>                            >
    | <ORDER    : <O> <R> <D> <E> <R>                >
    | <POLICY   : <P> <O> <L> <I> <C> <Y>            >
    | <PREWHERE : <P> <R> <E> <W> <H> <E> <R> <E>    >
    | <PROFILE  : <P> <R> <O> <F> <I> <L> <E>        >
    | <QUOTA    : <Q> <U> <O> <T> <A>                >
    | <REPLACE  : <R> <E> <P> <L> <A> <C> <E>        >
    | <ROLE     : <R> <O> <L> <E>                    >
    | <ROW      : <R> <O> <W>                        >
    | <RIGHT    : <R> <I> <G> <H> <T>                >
    | <SAMPLE   : <S> <A> <M> <P> <L> <E>            >
    | <SETTINGS : <S> <E> <T> <T> <I> <N> <G> <S>    >
    | <TEMPORARY: <T> <E> <M> <P> <O> <R> <A> <R> <Y>>
    | <TABLE    : <T> <A> <B> <L> <E>                >
    | <TABLES   : <T> <A> <B> <L> <E> <S>            >
    | <THEN     : <T> <H> <E> <N>                    >
    | <TIES     : <T> <I> <E> <S>                    >
    | <TIMESTAMP: <T> <I> <M> <E> <S> <T> <A> <M> <P>>
    | <TOP      : <T> <O> <P>                        >
    | <TOTALS   : <T> <O> <T> <A> <L> <S>            >
    | <UNION    : <U> <N> <I> <O> <N>                >
    | <USER     : <U> <S> <E> <R>                    >
    | <USING    : <U> <S> <I> <N> <G>                >
    | <VALUES   : <V> <A> <L> <U> <E> <S>            >
    | <VIEW     : <V> <I> <E> <W>                    >
    | <WHEN     : <W> <H> <E> <N>                    >
    | <WHERE    : <W> <H> <E> <R> <E>                >
    | <WITH     : <W> <I> <T> <H>                    >

    | <SECOND   : <S> <E> <C> <O> <N> <D>     >
    | <MINUTE   : <M> <I> <N> <U> <T> <E>     >
    | <HOUR     : <H> <O> <U> <R>             >
    | <DAY      : <D> <A> <Y>                 >
    | <WEEK     : <W> <E> <E> <K>             >
    | <MONTH    : <M> <O> <N> <T> <H>         >
    | <QUARTER  : <Q> <U> <A> <R> <T> <E> <R> >
    | <YEAR     : <Y> <E> <A> <R>             >

    | <INF     : <I> <N> <F>                >
    | <NAN     : <N> <A> <N>                >
    | <NULL    : <N> <U> <L> <L>            >
}

// letters
TOKEN: {
    <#A: ["a", "A"]>
    | <#B: ["b", "B"]>
    | <#C: ["c", "C"]>
    | <#D: ["d", "D"]>
    | <#E: ["e", "E"]>
    | <#F: ["f", "F"]>
    | <#G: ["g", "G"]>
    | <#H: ["h", "H"]>
    | <#I: ["i", "I"]>
    | <#J: ["j", "J"]>
    | <#K: ["k", "K"]>
    | <#L: ["l", "L"]>
    | <#M: ["m", "M"]>
    | <#N: ["n", "N"]>
    | <#O: ["o", "O"]>
    | <#P: ["p", "P"]>
    | <#Q: ["q", "Q"]>
    | <#R: ["r", "R"]>
    | <#S: ["s", "S"]>
    | <#T: ["t", "T"]>
    | <#U: ["u", "U"]>
    | <#V: ["v", "V"]>
    | <#W: ["w", "W"]>
    | <#X: ["x", "X"]>
    | <#Y: ["y", "Y"]>
    | <#Z: ["z", "Z"]>

    | <#LETTER: ["a"-"z", "A"-"Z"]>
}

// numbers
TOKEN: {
    <#ZERO: "0">
    | <#DEC_DIGIT: ["0"-"9"]> // including octal digit
    | <#HEX_DIGIT: ["0"-"9", "a"-"f", "A"-"F"]>
}

// symbols
TOKEN: {
    <ARROW: "->">
    | <ASTERISK: "*">
    | <AT: "@">
    | <BACK_QUOTE: "`">
    | <BACK_SLASH: "\\">
    | <COLON: ":">
    | <COMMA: ",">
    | <CONCAT: "||">
    | <DASH: "-">
    | <DOLLAR: "$">
    | <DOT: ".">
    | <EQ_DOUBLE: "==">
    | <EQ_SINGLE: "=">
    | <GE: ">=">
    | <GT: ">">
    | <HASH: "#">
    | <LBRACE: "{">
    | <LBRACKET: "[">
    | <LE: "<=">
    | <LPAREN: "(">
    | <LT: "<">
    | <NOT_EQ: "!=" | "<>">
    | <PERCENT: "%">
    | <PLUS: "+">
    | <QUERY: "?">
    | <DOUBLE_QUOTE: "\"">
    | <SINGLE_QUOTE: "'">
    | <RBRACE: "}">
    | <RBRACKET: "]">
    | <RPAREN: ")">
    | <SEMICOLON: ";">
    | <SLASH: "/">
    | <#UNDERSCORE: "_">
}

// string literal
TOKEN: { 
    <STRING_LITERAL: <SINGLE_QUOTE> (<BACK_SLASH> ~[] | ~["'", "\\"] | "''")* <SINGLE_QUOTE>>
}

TOKEN: {
    <IDENTIFIER:
        (<LETTER> | <UNDERSCORE> | <DOLLAR>) (<LETTER> | <UNDERSCORE> | <DOLLAR> | <DEC_DIGIT>)*
        | (<DEC_DIGIT>)+ (
            <E> (<LETTER> | <UNDERSCORE>)*
            | (
                <A> | <B> | <C> | <D> | <F> | <G> | <H> | <I> | <J> | <K> | <L> | <M> | <N>
                | <O> | <P> | <Q> | <R> | <S> | <T> | <U> | <V> | <W> | <X> | <Y> | <Z>
                | <UNDERSCORE>
            ) (<LETTER> | <UNDERSCORE> | <DEC_DIGIT>)*
        )>
    | <BACK_QUOTED_NAME: <BACK_QUOTE> (<BACK_SLASH> ~[] | ~["`", "\\"] | "``")* <BACK_QUOTE>>
    | <DOUBLE_QUOTED_NAME: <DOUBLE_QUOTE> (<BACK_SLASH> ~[] | ~["\"", "\\"] | "\"\"")* <DOUBLE_QUOTE>>
}

TOKEN: {
    <FLOATING_LITERAL:
        <HEXADECIMAL_LITERAL> <DOT> (<HEXADECIMAL_LITERAL>)? (<P> | <E>) (<PLUS> | <DASH>)? <DECIMAL_LITERAL>
        | <HEXADECIMAL_LITERAL> (<P> | <E>) (<PLUS> | <DASH>)? <DECIMAL_LITERAL>
        | <DECIMAL_LITERAL> <DOT> (<DECIMAL_LITERAL>)? (<E> (<PLUS> | <DASH>)? <DECIMAL_LITERAL>)?
        | <DOT> <DECIMAL_LITERAL> (<E> (<PLUS> | <DASH>)? <DECIMAL_LITERAL>)?
        | <DECIMAL_LITERAL> <E> (<PLUS> | <DASH>)? <DECIMAL_LITERAL>>
}
TOKEN: { <DECIMAL_LITERAL: (<DEC_DIGIT>)+> }
TOKEN: { <HEXADECIMAL_LITERAL: <ZERO> <X> (<HEX_DIGIT>)+> }