/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

options
{
    STATIC = false;
    UNICODE_INPUT = true;
    // DEBUG_TOKEN_MANAGER = true;
    // DEBUG_PARSER = true;
}

PARSER_BEGIN(FMParser)

package freemarker.core;

import freemarker.template.*;
import freemarker.template.utility.*;
import java.io.*;
import java.util.*;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

/**
 * This class is generated by JavaCC from a grammar file.
 */
public class FMParser {

    private static final int ITERATOR_BLOCK_KIND_LIST = 0; 
    private static final int ITERATOR_BLOCK_KIND_FOREACH = 1; 
    private static final int ITERATOR_BLOCK_KIND_ITEMS = 2; 
    private static final int ITERATOR_BLOCK_KIND_USER_DIRECTIVE = 3; 

    private static class ParserIteratorBlockContext {
        private String loopVarName;
        private int kind;
    }

    private Template template;

    private boolean stripWhitespace, stripText;
    private int incompatibleImprovements;
    private OutputFormat outputFormat;
    private int autoEscapingPolicy;
    private boolean autoEscaping;
    private ParserConfiguration pCfg;

    /** Keeps track of #list and #foreach nesting. */
    private List/*<ParserIteratorBlockContext>*/ iteratorBlockContexts;
    
    /**
     * Keeps track of the nesting depth of directives that support #break.
     */
    private int breakableDirectiveNesting;
    
    /**
     * Keeps track of the flags of the innermost parent #list or #foreach directive.
     */
    private int parentListAndForeachFlags;
    
    private boolean inMacro, inFunction;
    private LinkedList escapes = new LinkedList();
    private int mixedContentNesting; // for stripText

    /**
     * Create an FM expression parser using a string.
     *
     * @Deprecated This is an internal API of FreeMarker; can be removed any time.
     */
    static public FMParser createExpressionParser(String s) {
        SimpleCharStream scs = new SimpleCharStream(new StringReader(s), 1, 1, s.length());
        FMParserTokenManager token_source = new FMParserTokenManager(scs);
        token_source.SwitchTo(FMParserConstants.FM_EXPRESSION);
        FMParser parser = new FMParser(token_source);
        token_source.setParser(parser);
        return parser;
    }

    /**
     * Constructs a new parser object.
     * 
     * @param template
     *            The template associated with this parser.
     * @param reader
     *            The character stream to use as input
     * @param strictEscapeSyntax
     *            Whether FreeMarker directives must start with a #
     *
     * @Deprecated This is an internal API of FreeMarker; will be removed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictEscapeSyntax, boolean stripWhitespace) {
        this(template, reader, strictEscapeSyntax, stripWhitespace, Configuration.AUTO_DETECT_TAG_SYNTAX);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictEscapeSyntax, boolean stripWhitespace, int tagSyntax) {
        this(template, reader, strictEscapeSyntax, stripWhitespace, tagSyntax,
                Configuration.PARSED_DEFAULT_INCOMPATIBLE_ENHANCEMENTS);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictEscapeSyntax, boolean stripWhitespace,
            int tagSyntax, int incompatibleImprovements) {
        this(template, reader, strictEscapeSyntax, stripWhitespace,
                tagSyntax, Configuration.AUTO_DETECT_NAMING_CONVENTION, incompatibleImprovements);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(String template) {
        this(dummyTemplate(),
                new StringReader(template), true, true);
    }

    private static Template dummyTemplate() {
        try {
            return new Template(null, new StringReader(""), Configuration.getDefaultConfiguration());
        } catch (IOException e) {
            throw new RuntimeException("Failed to create dummy template", e);
        }
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictSyntaxMode, boolean whitespaceStripping,
            int tagSyntax, int namingConvention, int incompatibleImprovements) {
        this(template, reader,
                new LegacyConstructorParserConfiguration(
                        strictSyntaxMode, whitespaceStripping,
                        tagSyntax, namingConvention,
                        template != null ? template.getParserConfiguration().getAutoEscapingPolicy()
                                : Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY,
                        template != null ? template.getParserConfiguration().getOutputFormat()
                                : null,
                        template != null ? template.getParserConfiguration().getRecognizeStandardFileExtensions()
                                : null,
                        new Version(incompatibleImprovements),
                        template != null ? template.getArithmeticEngine() : null));
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; don't call it from outside FreeMarker.
     * 
     * @since 2.3.24
     */
    public FMParser(Template template, Reader reader, ParserConfiguration pCfg) {
        this(template, true, readerToTokenManager(reader), pCfg);
    }

    private static FMParserTokenManager readerToTokenManager(Reader reader) {
        return new FMParserTokenManager(new SimpleCharStream(reader, 1, 1));
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; don't call it from outside FreeMarker.
     * 
     * @since 2.3.24
     */
    public FMParser(Template template, boolean newTemplate, FMParserTokenManager tkMan, ParserConfiguration pCfg) {
        this(tkMan);

        NullArgumentException.check(pCfg);
        this.pCfg = pCfg;

        NullArgumentException.check(template);
        this.template = template;

        // Hack due to legacy public constructors (removed in 2.4):
        if (pCfg instanceof LegacyConstructorParserConfiguration) {
            LegacyConstructorParserConfiguration lpCfg = (LegacyConstructorParserConfiguration) pCfg;
            lpCfg.setArithmeticEngineIfNotSet(template.getArithmeticEngine());
            lpCfg.setAutoEscapingPolicyIfNotSet(template.getConfiguration().getAutoEscapingPolicy());
            lpCfg.setOutputFormatIfNotSet(template.getOutputFormat());
            lpCfg.setRecognizeStandardFileExtensionsIfNotSet(
                    template.getParserConfiguration().getRecognizeStandardFileExtensions());
        }

        int incompatibleImprovements = pCfg.getIncompatibleImprovements().intValue();
        token_source.incompatibleImprovements = incompatibleImprovements;
        this.incompatibleImprovements = incompatibleImprovements;

        {
            OutputFormat outputFormatFromExt;
            if (!pCfg.getRecognizeStandardFileExtensions()
                    || (outputFormatFromExt = getFormatFromStdFileExt()) == null) {
                autoEscapingPolicy = pCfg.getAutoEscapingPolicy();
                outputFormat = pCfg.getOutputFormat();
            } else {
                // Override it
                autoEscapingPolicy = Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY;
                outputFormat = outputFormatFromExt;
            }
        }
        recalculateAutoEscapingField();

        token_source.setParser(this);

        token_source.strictEscapeSyntax = pCfg.getStrictSyntaxMode();

        int tagSyntax = pCfg.getTagSyntax();
        switch (tagSyntax) {
        case Configuration.AUTO_DETECT_TAG_SYNTAX:
            token_source.autodetectTagSyntax = true;
            break;
        case Configuration.ANGLE_BRACKET_TAG_SYNTAX:
            token_source.squBracTagSyntax = false;
            break;
        case Configuration.SQUARE_BRACKET_TAG_SYNTAX:
            token_source.squBracTagSyntax = true;
            break;
        default:
            throw new IllegalArgumentException("Illegal argument for tagSyntax: " + tagSyntax);
        }

        int namingConvention = pCfg.getNamingConvention();
        switch (namingConvention) {
        case Configuration.AUTO_DETECT_NAMING_CONVENTION:
        case Configuration.CAMEL_CASE_NAMING_CONVENTION:
        case Configuration.LEGACY_NAMING_CONVENTION:
            token_source.initialNamingConvention = namingConvention;
            token_source.namingConvention = namingConvention;
            break;
        default:
            throw new IllegalArgumentException("Illegal argument for namingConvention: " + namingConvention);
        }

        this.stripWhitespace = pCfg.getWhitespaceStripping();

        // If this is a Template under construction, we do the below.
        // If this is just the enclosing Template for ?eval or such, we must not modify it.
        if (newTemplate) {
            _TemplateAPI.setAutoEscaping(template, autoEscaping);
            _TemplateAPI.setOutputFormat(template, outputFormat);
        }
    }
    
    void setupStringLiteralMode(FMParserTokenManager parentTokenSource, OutputFormat outputFormat) {
        token_source.initialNamingConvention = parentTokenSource.initialNamingConvention;
        token_source.namingConvention = parentTokenSource.namingConvention;
        token_source.namingConventionEstabilisher = parentTokenSource.namingConventionEstabilisher;
        token_source.SwitchTo(NODIRECTIVE);
        
        this.outputFormat = outputFormat;
        recalculateAutoEscapingField();                                
        if (incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_24) {
            // Emulate bug, where the string literal parser haven't inherited the IcI:
            incompatibleImprovements = _TemplateAPI.VERSION_INT_2_3_0;
        }
    }

    void tearDownStringLiteralMode(FMParserTokenManager parentTokenSource) {
        parentTokenSource.namingConvention = token_source.namingConvention;
        parentTokenSource.namingConventionEstabilisher = token_source.namingConventionEstabilisher;
    }

    private OutputFormat getFormatFromStdFileExt() {
        String sourceName = template.getSourceName();
        if (sourceName == null) {
            return null; // Not possible anyway...
        }

        int ln = sourceName.length();
        if (ln < 5) return null;

        char c = sourceName.charAt(ln - 5);
        if (c != '.') return null;

        c = sourceName.charAt(ln - 4);
        if (c != 'f' && c != 'F') return null;

        c = sourceName.charAt(ln - 3);
        if (c != 't' && c != 'T') return null;

        c = sourceName.charAt(ln - 2);
        if (c != 'l' && c != 'L') return null;

        c = sourceName.charAt(ln - 1);
        try {
            // Note: We get the output formats by name, so that custom overrides take effect.
            if (c == 'h' || c == 'H') {
                return template.getConfiguration().getOutputFormat(HTMLOutputFormat.INSTANCE.getName());
                }
            if (c == 'x' || c == 'X') {
                return template.getConfiguration().getOutputFormat(XMLOutputFormat.INSTANCE.getName());
            }
        } catch (UnregisteredOutputFormatException e) {
            throw new BugException("Unregistered std format", e);
        }
        return null;
    }
    
    /**
     * Updates the {@link #autoEscaping} field based on the {@link #autoEscapingPolicy} and {@link #outputFormat} fields.
     */
    private void recalculateAutoEscapingField() {
        if (outputFormat instanceof MarkupOutputFormat) {
            if (autoEscapingPolicy == Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY) {
                autoEscaping = ((MarkupOutputFormat) outputFormat).isAutoEscapedByDefault();
            } else if (autoEscapingPolicy == Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY) {
                autoEscaping = true;
            } else if (autoEscapingPolicy == Configuration.DISABLE_AUTO_ESCAPING_POLICY) {
                autoEscaping = false;
            } else {
                throw new IllegalStateException("Unhandled autoEscaping enum: " + autoEscapingPolicy);
            }
        } else {
            autoEscaping = false;
        }
    }
    
    MarkupOutputFormat getMarkupOutputFormat() {
        return outputFormat instanceof MarkupOutputFormat ? (MarkupOutputFormat) outputFormat : null;
    }

    /**
     * Don't use it, unless you are developing FreeMarker itself.
     */
    public int _getLastTagSyntax() {
        return token_source.squBracTagSyntax
                ? Configuration.SQUARE_BRACKET_TAG_SYNTAX
                : Configuration.ANGLE_BRACKET_TAG_SYNTAX;
    }
    
    /**
     * Don't use it, unless you are developing FreeMarker itself.
     * The naming convention used by this template; if it couldn't be detected so far, it will be the most probable one.
     * This could be used for formatting error messages, but not for anything serious.
     */
    public int _getLastNamingConvention() {
        return token_source.namingConvention;
    }

    /**
     * Throw an exception if the expression passed in is a String Literal
     */
    private void notStringLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof StringLiteral) {
            throw new ParseException(
                    "Found string literal: " + exp + ". Expecting: " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a Number Literal
     */
    private void notNumberLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof NumberLiteral) {
            throw new ParseException(
                    "Found number literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a boolean Literal
     */
    private void notBooleanLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof BooleanLiteral) {
            throw new ParseException("Found: " + exp.getCanonicalForm() + ". Expecting " + expected, exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a Hash Literal
     */
    private void notHashLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof HashLiteral) {
            throw new ParseException(
                    "Found hash literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a List Literal
     */
    private void notListLiteral(Expression exp, String expected)
            throws ParseException
    {
        if (exp instanceof ListLiteral) {
            throw new ParseException(
                    "Found list literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a literal other than of the numerical type
     */
    private void numberLiteralOnly(Expression exp) throws ParseException {
        notStringLiteral(exp, "number");
        notListLiteral(exp, "number");
        notHashLiteral(exp, "number");
        notBooleanLiteral(exp, "number");
    }

    /**
     * Throw an exception if the expression passed in is not a string.
     */
    private void stringLiteralOnly(Expression exp) throws ParseException {
        notNumberLiteral(exp, "string");
        notListLiteral(exp, "string");
        notHashLiteral(exp, "string");
        notBooleanLiteral(exp, "string");
    }

    /**
     * Throw an exception if the expression passed in is a literal other than of the boolean type
     */
    private void booleanLiteralOnly(Expression exp) throws ParseException {
        notStringLiteral(exp, "boolean (true/false)");
        notListLiteral(exp, "boolean (true/false)");
        notHashLiteral(exp, "boolean (true/false)");
        notNumberLiteral(exp, "boolean (true/false)");
    }

    private Expression escapedExpression(Expression exp) {
        if (!escapes.isEmpty()) {
            return ((EscapeBlock) escapes.getFirst()).doEscape(exp);
        } else {
            return exp;
        }
    }

    private boolean getBoolean(Expression exp, boolean legacyCompat) throws ParseException {
        TemplateModel tm = null;
        try {
            tm = exp.eval(null);
        } catch (Exception e) {
            throw new ParseException(e.getMessage()
                    + "\nCould not evaluate expression: "
                    + exp.getCanonicalForm(),
                    exp,
                    e);
        }
        if (tm instanceof TemplateBooleanModel) {
            try {
                return ((TemplateBooleanModel) tm).getAsBoolean();
            } catch (TemplateModelException tme) {
            }
        }
        if (legacyCompat && tm instanceof TemplateScalarModel) {
            try {
                return StringUtil.getYesNo(((TemplateScalarModel) tm).getAsString());
            } catch (Exception e) {
                throw new ParseException(e.getMessage()
                        + "\nExpecting boolean (true/false), found: " + exp.getCanonicalForm(),
                        exp);
            }
        }
        throw new ParseException("Expecting boolean (true/false) parameter", exp);
    }
    
    void checkCurrentOutputFormatCanEscape(Token start) throws ParseException {
        if (!(outputFormat instanceof MarkupOutputFormat)) {
            throw new ParseException("The current output format can't do escaping: " + outputFormat,
                    template, start);
        }
    }    
    
    private ParserIteratorBlockContext pushIteratorBlockContext() {
        if (iteratorBlockContexts == null) {
            iteratorBlockContexts = new ArrayList(4);
        }
        ParserIteratorBlockContext newCtx = new ParserIteratorBlockContext();
        iteratorBlockContexts.add(newCtx);
        return newCtx;
    }
    
    private void popIteratorBlockContext() {
        iteratorBlockContexts.remove(iteratorBlockContexts.size() - 1);
    }
    
    private ParserIteratorBlockContext peekIteratorBlockContext() {
        int size = iteratorBlockContexts != null ? iteratorBlockContexts.size() : 0;
        return size != 0 ? (ParserIteratorBlockContext) iteratorBlockContexts.get(size - 1) : null; 
    }
    
    private void checkLoopVariableBuiltInLHO(String loopVarName, Expression lhoExp, Token biName)
            throws ParseException {
        int size = iteratorBlockContexts != null ? iteratorBlockContexts.size() : 0;
        for (int i = size - 1; i >= 0; i--) {
            ParserIteratorBlockContext ctx = (ParserIteratorBlockContext) iteratorBlockContexts.get(i);
            if (loopVarName.equals(ctx.loopVarName)) {
                if (ctx.kind == ITERATOR_BLOCK_KIND_USER_DIRECTIVE) {
			        throw new ParseException(
			                "The left hand operand of ?" + biName.image
			                + " can't be the loop variable of an user defined directive: "
			                +  loopVarName,
			                lhoExp);
                }
                return;  // success
            }
        }
        throw new ParseException(
                "The left hand operand of ?" + biName.image + " must be a loop variable, "
                + "but there's no loop variable in scope with this name: " + loopVarName,
                lhoExp);
    }
    
	private String forEachDirectiveSymbol() {    
	    // [2.4] Use camel case as the default
	    return token_source.namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION ? "#forEach" : "#foreach";
	}
    
}

PARSER_END(FMParser)

/**
 * The lexer portion defines 5 lexical states:
 * DEFAULT, FM_EXPRESSION, IN_PAREN, NO_PARSE, and EXPRESSION_COMMENT.
 * The DEFAULT state is when you are parsing
 * text but are not inside a FreeMarker expression.
 * FM_EXPRESSION is the state you are in
 * when the parser wants a FreeMarker expression.
 * IN_PAREN is almost identical really. The difference
 * is that you are in this state when you are within
 * FreeMarker expression and also within (...).
 * This is a necessary subtlety because the
 * ">" and ">=" symbols can only be used
 * within parentheses because otherwise, it would
 * be ambiguous with the end of a directive.
 * So, for example, you enter the FM_EXPRESSION state
 * right after a ${ and leave it after the matching }.
 * Or, you enter the FM_EXPRESSION state right after
 * an "<if" and then, when you hit the matching ">"
 * that ends the if directive,
 * you go back to DEFAULT lexical state.
 * If, within the FM_EXPRESSION state, you enter a
 * parenthetical expression, you enter the IN_PAREN
 * state.
 * Note that whitespace is ignored in the
 * FM_EXPRESSION and IN_PAREN states
 * but is passed through to the parser as PCDATA in the DEFAULT state.
 * NO_PARSE and EXPRESSION_COMMENT are extremely simple
 * lexical states. NO_PARSE is when you are in a comment
 * block and EXPRESSION_COMMENT is when you are in a comment
 * that is within an FTL expression.
 */
TOKEN_MGR_DECLS:
{

    private static final String PLANNED_DIRECTIVE_HINT
            = "(If you have seen this directive in use elsewhere, this was a planned directive, "
                + "so maybe you need to upgrade FreeMarker.)";

    /**
     * The noparseTag is set when we enter a block of text that the parser more or less ignores. These are <noparse> and
     * <comment>. This variable tells us what the closing tag should be, and when we hit that, we resume parsing. Note
     * that with this scheme, <comment> and <noparse> tags cannot nest recursively, but it is not clear how important
     * that is.
     */
    String noparseTag;

    /**
     * Keeps track of how deeply nested we have the hash literals. This is necessary since we need to be able to
     * distinguish the } used to close a hash literal and the one used to close a ${
     */
    private FMParser parser;
    private int postInterpolationLexState = -1;
    private int hashLiteralNesting;
    private int parenthesisNesting;
    private int bracketNesting;
    private boolean inFTLHeader;
    boolean strictEscapeSyntax,
            squBracTagSyntax,
            autodetectTagSyntax,
            directiveSyntaxEstablished,
            inInvocation;
    int initialNamingConvention;
    int namingConvention;
    Token namingConventionEstabilisher;
    int incompatibleImprovements;

    void setParser(FMParser parser) {
        this.parser = parser;
    }

    // This method checks if we are in a strict mode where all
    // FreeMarker directives must start with <#. It also handles
    // tag syntax detection. If you update this logic, take a look
    // at the UNKNOWN_DIRECTIVE token too.
    private void strictSyntaxCheck(Token tok, int tokenNamingConvention, int newLexState) {
        final String image = tok.image;
        
        // Non-strict syntax (deprecated) only supports legacy naming convention.
        // We didn't push this on the tokenizer because it made it slow, so we filter here.
        if (!strictEscapeSyntax
                && (tokenNamingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION)
                && !isStrictTag(image)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        char firstChar = image.charAt(0);
        if (autodetectTagSyntax && !directiveSyntaxEstablished) {
            squBracTagSyntax = (firstChar == '[');
        }
        if ((firstChar == '[' && !squBracTagSyntax) || (firstChar == '<' && squBracTagSyntax)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        if (!strictEscapeSyntax) {
            // Legacy feature (or bug?): Tag syntax never gets estabilished in non-strict mode.
            // We do establilish the naming convention though.
            checkNamingConvention(tok, tokenNamingConvention);
            SwitchTo(newLexState);
            return;
        }
        
        // For square bracket tags there's no non-strict token, so we are sure that it's an FTL tag.
        // But if it's an angle bracket tag, we have to check if it's just static text or and FTL tag, because the
        // tokenizer will emit the same kind of token for both.
        if (!squBracTagSyntax && !isStrictTag(image)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        // We only get here if this is a strict FTL tag.
        directiveSyntaxEstablished = true;
        
        checkNamingConvention(tok, tokenNamingConvention);
        
        SwitchTo(newLexState);
    }

    void checkNamingConvention(Token tok) {
        checkNamingConvention(tok, _CoreStringUtils.getIdentifierNamingConvention(tok.image)); 
    }
    
    void checkNamingConvention(Token tok, int tokenNamingConvention) {
        if (tokenNamingConvention != Configuration.AUTO_DETECT_NAMING_CONVENTION) {
	        if (namingConvention == Configuration.AUTO_DETECT_NAMING_CONVENTION) {
	            namingConvention = tokenNamingConvention;
	            namingConventionEstabilisher = tok;
	        } else if (namingConvention != tokenNamingConvention) {
                throw newNameConventionMismatchException(tok);
	        }
        }
    }
    
    private TokenMgrError newNameConventionMismatchException(Token tok) {
        return new TokenMgrError(
                "Naming convention mismatch. "
                + "Identifiers that are part of the template language (not the user specified ones) "
                + (initialNamingConvention == Configuration.AUTO_DETECT_NAMING_CONVENTION
                    ? "must consistently use the same naming convention within the same template. This template uses "
                    : "must use the configured naming convention, which is the ")
                + (namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION
                            ? "camel case naming convention (like: exampleName) "
                            : (namingConvention == Configuration.LEGACY_NAMING_CONVENTION
                                    ? "legacy naming convention (directive (tag) names are like examplename, " 
                                      + "everything else is like example_name) "
                                    : "??? (internal error)"
                                    ))
                + (namingConventionEstabilisher != null
                        ? "estabilished by auto-detection at "
                            + MessageUtil.formatPosition(
                                    namingConventionEstabilisher.beginLine, namingConventionEstabilisher.beginColumn)
                            + " by token " + StringUtil.jQuote(namingConventionEstabilisher.image.trim())
                        : "")
                + ", but the problematic token, " + StringUtil.jQuote(tok.image.trim())
                + ", uses a different convention.",
                TokenMgrError.LEXICAL_ERROR,
                tok.beginLine, tok.beginColumn, tok.endLine, tok.endColumn);
    }

    /**
     * Used for tags whose name isn't affected by naming convention.
     */
    private void strictSyntaxCheck(Token tok, int newLexState) {
        strictSyntaxCheck(tok, Configuration.AUTO_DETECT_NAMING_CONVENTION, newLexState);
    }
    
    private boolean isStrictTag(String image) {
        return image.length() > 2 && (image.charAt(1) == '#' || image.charAt(2) == '#');
    }
    
    /**
     * Detects the naming convention used, both in start- and end-tag tokens.
     *
     * @param charIdxInName
     *         The index of the deciding character relatively to the first letter of the name.
     */
    private static int getTagNamingConvention(Token tok, int charIdxInName) {
        return _CoreStringUtils.isUpperUSASCII(getTagNameCharAt(tok, charIdxInName))
                ? Configuration.CAMEL_CASE_NAMING_CONVENTION : Configuration.LEGACY_NAMING_CONVENTION;
    }

    static char getTagNameCharAt(Token tok, int charIdxInName) {
        final String image = tok.image;
        
        // Skip tag delimiter:
        int idx = 0;
        for (;;) {
            final char c = image.charAt(idx);
            if (c != '<' && c != '[' && c != '/' && c != '#') {
                break;
            }
            idx++;
        }

        return image.charAt(idx + charIdxInName);
    }

    private void unifiedCall(Token tok) {
        char firstChar = tok.image.charAt(0);
        if (autodetectTagSyntax && !directiveSyntaxEstablished) {
            squBracTagSyntax = (firstChar == '[');
        }
        if (squBracTagSyntax && firstChar == '<') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        if (!squBracTagSyntax && firstChar == '[') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        directiveSyntaxEstablished = true;
        SwitchTo(NO_SPACE_EXPRESSION);
    }

    private void unifiedCallEnd(Token tok) {
        char firstChar = tok.image.charAt(0);
        if (squBracTagSyntax && firstChar == '<') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        if (!squBracTagSyntax && firstChar == '[') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
    }

    private void closeBracket(Token tok) {
        if (bracketNesting > 0) {
            --bracketNesting;
        } else {
            tok.kind = DIRECTIVE_END;
            if (inFTLHeader) {
                eatNewline();
                inFTLHeader = false;
            }
            SwitchTo(DEFAULT);
        }
    }
    
    private void startInterpolation(Token tok) {
        if (postInterpolationLexState != -1) {
            char c = tok.image.charAt(0);
            throw new TokenMgrError(
                    "You can't start an interpolation (" + c + "{...}) here "
                    + "as you are inside another interpolation.)",
                    TokenMgrError.LEXICAL_ERROR,
                    tok.beginLine, tok.beginColumn,
                    tok.endLine, tok.endColumn);
        }
        postInterpolationLexState = curLexState;
        SwitchTo(FM_EXPRESSION);
    }

    /**
     * @param tok
     *         Assumed to be an '}', or something that is the closing pair of another "mirror image" character.
     */
    private void endInterpolation(Token tok) {
        if (postInterpolationLexState == -1) {
            char c = tok.image.charAt(0);
            throw new TokenMgrError(
                    "You can't have an \"" + c + "\" here, as there's nothing open that it could close.",
                    TokenMgrError.LEXICAL_ERROR,
                    tok.beginLine, tok.beginColumn,
                    tok.endLine, tok.endColumn);
        }
        SwitchTo(postInterpolationLexState);
        postInterpolationLexState = -1;
    }

    private void eatNewline() {
        int charsRead = 0;
        try {
            while (true) {
                char c = input_stream.readChar();
                ++charsRead;
                if (!Character.isWhitespace(c)) {
                    input_stream.backup(charsRead);
                    return;
                } else if (c == '\r') {
                    char next = input_stream.readChar();
                    ++charsRead;
                    if (next != '\n') {
                        input_stream.backup(1);
                    }
                    return;
                } else if (c == '\n') {
                    return;
                }
            }
        } catch (IOException ioe) {
            input_stream.backup(charsRead);
        }
    }

    private void ftlHeader(Token matchedToken) {
        if (!directiveSyntaxEstablished) {
            squBracTagSyntax = matchedToken.image.charAt(0) == '[';
            directiveSyntaxEstablished = true;
            autodetectTagSyntax = false;
        }
        String img = matchedToken.image;
        char firstChar = img.charAt(0);
        char lastChar = img.charAt(img.length() - 1);
        if ((firstChar == '[' && !squBracTagSyntax) || (firstChar == '<' && squBracTagSyntax)) {
            matchedToken.kind = STATIC_TEXT_NON_WS;
        }
        if (matchedToken.kind != STATIC_TEXT_NON_WS) {
            if (lastChar != '>' && lastChar != ']') {
                SwitchTo(FM_EXPRESSION);
                inFTLHeader = true;
            } else {
                eatNewline();
            }
        }
    }
}

TOKEN:
{
    <#BLANK : " " | "\t" | "\n" | "\r">
    |
    <#START_TAG : "<" | "<#" | "[#">
    |
    <#END_TAG : "</" | "</#" | "[/#">
    |
    <#CLOSE_TAG1 : (<BLANK>)* (">" | "]")>
    |
    <#CLOSE_TAG2 : (<BLANK>)* ("/")? (">" | "]")>
    |
    /*
     * ATTENTION: Update _CoreAPI.*_BUILT_IN_DIRECTIVE_NAMES if you add new directives!
     */
    <ATTEMPT : <START_TAG> "attempt" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <RECOVER : <START_TAG> "recover" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); } 
    |
    <IF : <START_TAG> "if" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <ELSE_IF : <START_TAG> "else" ("i" | "I") "f" <BLANK>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 4), FM_EXPRESSION);
    }
    |
    <LIST : <START_TAG> "list" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <ITEMS : <START_TAG> "items" (<BLANK>)+ <AS> <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <SEP : <START_TAG> "sep" <CLOSE_TAG1>>
    |
    <FOREACH : <START_TAG> "for" ("e" | "E") "ach" <BLANK>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 3), FM_EXPRESSION);
    }
    |
    <SWITCH : <START_TAG> "switch" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <CASE : <START_TAG> "case" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <ASSIGN : <START_TAG> "assign" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <GLOBALASSIGN : <START_TAG> "global" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <LOCALASSIGN : <START_TAG> "local" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <_INCLUDE : <START_TAG> "include" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <IMPORT : <START_TAG> "import" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <FUNCTION : <START_TAG> "function" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <MACRO : <START_TAG> "macro" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <TRANSFORM : <START_TAG> "transform" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <VISIT : <START_TAG> "visit" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <STOP : <START_TAG> "stop" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <RETURN : <START_TAG> "return" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <CALL : <START_TAG> "call" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <SETTING : <START_TAG> "setting" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <OUTPUTFORMAT : <START_TAG> "output" ("f"|"F") "ormat" <BLANK>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 6), FM_EXPRESSION);
    }
    |
    <AUTOESC : <START_TAG> "auto" ("e"|"E") "sc" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 4), DEFAULT);
    }
    |
    <NOAUTOESC : <START_TAG> "no" ("autoe"|"AutoE") "sc" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
    <COMPRESS : <START_TAG> "compress" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <COMMENT : <START_TAG> "comment" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, NO_PARSE); noparseTag = "comment";
    }
    |
    <TERSE_COMMENT : ("<" | "[") "#--" > { noparseTag = "-->"; strictSyntaxCheck(matchedToken, NO_PARSE); }
    |
    <NOPARSE: <START_TAG> "no" ("p" | "P") "arse" <CLOSE_TAG1>> {
        int tagNamingConvention = getTagNamingConvention(matchedToken, 2);
        strictSyntaxCheck(matchedToken, tagNamingConvention, NO_PARSE);
        noparseTag = tagNamingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION ? "noParse" : "noparse";
    }
    |
    <END_IF : <END_TAG> "if" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_LIST : <END_TAG> "list" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_ITEMS : <END_TAG> "items" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_SEP : <END_TAG> "sep" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_RECOVER : <END_TAG> "recover" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_ATTEMPT : <END_TAG> "attempt" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_FOREACH : <END_TAG> "for" ("e" | "E") "ach" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 3), DEFAULT);
    }
    |
    <END_LOCAL : <END_TAG> "local" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_GLOBAL : <END_TAG> "global" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_ASSIGN : <END_TAG> "assign" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_FUNCTION : <END_TAG> "function" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_MACRO : <END_TAG> "macro" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_OUTPUTFORMAT : <END_TAG> "output" ("f" | "F") "ormat" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 6), DEFAULT);
    }
    |
    <END_AUTOESC : <END_TAG> "auto" ("e" | "E") "sc" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 4), DEFAULT);
    }
    |
    <END_NOAUTOESC : <END_TAG> "no" ("autoe"|"AutoE") "sc" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
    <END_COMPRESS : <END_TAG> "compress" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_TRANSFORM : <END_TAG> "transform" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <END_SWITCH : <END_TAG> "switch" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <ELSE : <START_TAG> "else" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <BREAK : <START_TAG> "break" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <SIMPLE_RETURN : <START_TAG> "return" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <HALT : <START_TAG> "stop" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <FLUSH : <START_TAG> "flush" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <TRIM : <START_TAG> "t" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <LTRIM : <START_TAG> "lt" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <RTRIM : <START_TAG> "rt" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <NOTRIM : <START_TAG> "nt" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <DEFAUL : <START_TAG> "default" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <SIMPLE_NESTED : <START_TAG> "nested" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <NESTED : <START_TAG> "nested" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <SIMPLE_RECURSE : <START_TAG> "recurse" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <RECURSE : <START_TAG> "recurse" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <FALLBACK : <START_TAG> "fallback" <CLOSE_TAG2>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <ESCAPE : <START_TAG> "escape" <BLANK>> { strictSyntaxCheck(matchedToken, FM_EXPRESSION); }
    |
    <END_ESCAPE : <END_TAG> "escape" <CLOSE_TAG1>> { strictSyntaxCheck(matchedToken, DEFAULT); }
    |
    <NOESCAPE : <START_TAG> "no" ("e" | "E") "scape" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
    <END_NOESCAPE : <END_TAG> "no" ("e" | "E") "scape" <CLOSE_TAG1>> {
        strictSyntaxCheck(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
    <UNIFIED_CALL : "<@" | "[@" > { unifiedCall(matchedToken); }
    |
    <UNIFIED_CALL_END : ("<" | "[") "/@" ((<ID>) ("."<ID>)*)? <CLOSE_TAG1>> { unifiedCallEnd(matchedToken); }
    |
    <FTL_HEADER : ("<#ftl" | "[#ftl") <BLANK>> { ftlHeader(matchedToken); }
    |
    <TRIVIAL_FTL_HEADER : ("<#ftl" | "[#ftl") ("/")? (">" | "]")> { ftlHeader(matchedToken); }
    |
    /*
     * ATTENTION: Update _CoreAPI.*_BUILT_IN_DIRECTIVE_NAMES if you add new directives!
     */
    <UNKNOWN_DIRECTIVE : ("[#" | "[/#" | "<#" | "</#") (["a"-"z", "A"-"Z", "_"])+>
    {
        if (!directiveSyntaxEstablished && incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_19) {
            matchedToken.kind = STATIC_TEXT_NON_WS;
        } else {
            char firstChar = matchedToken.image.charAt(0);

            if (!directiveSyntaxEstablished && autodetectTagSyntax) {
                squBracTagSyntax = (firstChar == '[');
                directiveSyntaxEstablished = true;
            }

            if (firstChar == '<' && squBracTagSyntax) {
                matchedToken.kind = STATIC_TEXT_NON_WS;
            } else if (firstChar == '[' && !squBracTagSyntax) {
                matchedToken.kind = STATIC_TEXT_NON_WS;
            } else if (strictEscapeSyntax) {
                String dn = matchedToken.image;
                int index = dn.indexOf('#');
                dn = dn.substring(index + 1);

                // Until the tokenizer/parser is reworked, we have this quirk where something like <#list>
                // doesn't match any directive starter tokens, because that token requires whitespace after the
                // name as it should be followed by parameters. For now we work this around so we don't report
                // unknown directive:
                if (_CoreAPI.ALL_BUILT_IN_DIRECTIVE_NAMES.contains(dn)) {
                    throw new TokenMgrError(
                            "#" + dn + " is an existing directive, but the tag is malformed. " 
                            + " (See FreeMarker Manual / Directive Reference.)",
                            TokenMgrError.LEXICAL_ERROR,
                            matchedToken.beginLine, matchedToken.beginColumn + 1,
                            matchedToken.endLine, matchedToken.endColumn);
                }

                String tip = null;
                if (dn.equals("set") || dn.equals("var")) {
                    tip = "Use #assign or #local or #global, depending on the intented scope "
                          + "(#assign is template-scope). " + PLANNED_DIRECTIVE_HINT;
                } else if (dn.equals("else_if") || dn.equals("elif")) {
                	tip = "Use #elseif.";
                } else if (dn.equals("no_escape")) {
                	tip = "Use #noescape instead.";
                } else if (dn.equals("method")) {
                	tip = "Use #function instead.";
                } else if (dn.equals("head") || dn.equals("template") || dn.equals("fm")) {
                	tip = "You may meant #ftl.";
                } else if (dn.equals("try") || dn.equals("atempt")) {
                	tip = "You may meant #attempt.";
                } else if (dn.equals("for") || dn.equals("each") || dn.equals("iterate") || dn.equals("iterator")) {
                    tip = "You may meant #list (http://freemarker.org/docs/ref_directive_list.html).";
                } else if (dn.equals("prefix")) {
                    tip = "You may meant #import. " + PLANNED_DIRECTIVE_HINT;
                } else if (dn.equals("item") | dn.equals("row") | dn.equals("rows")) {
                    tip = "You may meant #items.";
                } else if (dn.equals("separator") | dn.equals("separate") | dn.equals("separ")) {
                    tip = "You may meant #sep.";
                } else {
                    tip = "Help (latest version): http://freemarker.org/docs/ref_directive_alphaidx.html; "
                            + "you're using FreeMarker " + Configuration.getVersion() + ".";
                }
                throw new TokenMgrError(
                        "Unknown directive: #" + dn + (tip != null ? ". " + tip : ""),
                        TokenMgrError.LEXICAL_ERROR,
                        matchedToken.beginLine, matchedToken.beginColumn + 1,
                        matchedToken.endLine, matchedToken.endColumn);
            }
        }
    }
}

<DEFAULT, NODIRECTIVE> TOKEN :
{
    <STATIC_TEXT_WS : ("\n" | "\r" | "\t" | " ")+>
    |
    <STATIC_TEXT_NON_WS : (~["$", "<", "#", "[", "{", "\n", "\r", "\t", " "])+>
    |
    <STATIC_TEXT_FALSE_ALARM : "$" | "#" | "<" | "[" | "{"> // to handle a lone dollar sign or "<" or "# or <@ with whitespace after"
    |
    <DOLLAR_INTERPOLATION_OPENING : "${"> { startInterpolation(matchedToken); }
    |
    <HASH_INTERPOLATION_OPENING : "#{"> { startInterpolation(matchedToken); }
}

<FM_EXPRESSION, IN_PAREN, NAMED_PARAMETER_EXPRESSION> SKIP :
{
    < ( " " | "\t" | "\n" | "\r" )+ >
    |
    < ("<" | "[") ("#" | "!") "--"> : EXPRESSION_COMMENT
}

<EXPRESSION_COMMENT> SKIP:
{
    < (~["-", ">", "]"])+ >
    |
    < ">">
    |
    < "]">
    |
    < "-">
    |
    < "-->" | "--]">
    {
        if (parenthesisNesting > 0) SwitchTo(IN_PAREN);
        else if (inInvocation) SwitchTo(NAMED_PARAMETER_EXPRESSION);
        else SwitchTo(FM_EXPRESSION);
    }
}

<FM_EXPRESSION, IN_PAREN, NO_SPACE_EXPRESSION, NAMED_PARAMETER_EXPRESSION> TOKEN :
{
    <#ESCAPED_CHAR :
        "\\"
        (
            ("n" | "t" | "r" | "f" | "b" | "g" | "l" | "a" | "\\" | "'" | "\"" | "$" | "{")
            |
            ("x" ["0"-"9", "A"-"F", "a"-"f"])
        )
    >
    | 
    <STRING_LITERAL :
        (
            "\""
            ((~["\"", "\\"]) | <ESCAPED_CHAR>)*
            "\""
        )
        |
        (
            "'"
            ((~["'", "\\"]) | <ESCAPED_CHAR>)*
            "'"
        )
    >
    |
    <RAW_STRING : "r" (("\"" (~["\""])* "\"") | ("'" (~["'"])* "'"))>
    |
    <FALSE : "false">
    |
    <TRUE : "true">
    |
    <INTEGER : (["0"-"9"])+>
    |
    <DECIMAL : <INTEGER> "." <INTEGER>>
    |
    <DOT : ".">
    |
    <DOT_DOT : "..">
    |
    <DOT_DOT_LESS : "..<" | "..!" >
    |
    <DOT_DOT_ASTERISK : "..*" >
    |
    <BUILT_IN : "?">
    |
    <EXISTS : "??">
    |
    <EQUALS : "=">
    |
    <DOUBLE_EQUALS : "==">
    |
    <NOT_EQUALS : "!=">
    |
    <PLUS_EQUALS : "+=">
    |
    <MINUS_EQUALS : "-=">
    |
    <TIMES_EQUALS : "*=">
    |
    <DIV_EQUALS : "/=">
    |
    <MOD_EQUALS : "%=">
    |
    <PLUS_PLUS : "++">
    |
    <MINUS_MINUS : "--">
    |
    <LESS_THAN : "lt" | "\\lt" | "<" | "&lt;">
    |
    <LESS_THAN_EQUALS : "lte" | "\\lte" | "<=" | "&lt;=">
    |
    <ESCAPED_GT: "gt" | "\\gt" |  "&gt;">
    |
    <ESCAPED_GTE : "gte" | "\\gte" | "&gt;=">
    |
    <PLUS : "+">
    |
    <MINUS : "-">
    |
    <TIMES : "*">
    |
    <DOUBLE_STAR : "**">
    |
    <ELLIPSIS : "...">
    |
    <DIVIDE : "/">
    |
    <PERCENT : "%">
    |
    <AND : "&" | "&&" >
    |
    <OR : "|" | "||">
    |
    <EXCLAM : "!">
    |
    <COMMA : ",">
    |
    <SEMICOLON : ";">
    |
    <COLON : ":">
    |
    <OPEN_BRACKET : "[">
    {
        ++bracketNesting;
    }
    |
    <CLOSE_BRACKET : "]">
    {
        closeBracket(matchedToken);
    }
    |
    <OPEN_PAREN : "(">
    {
        ++parenthesisNesting;
        if (parenthesisNesting == 1) SwitchTo(IN_PAREN);
    }
    |
    <CLOSE_PAREN : ")">
    {
        --parenthesisNesting;
        if (parenthesisNesting == 0) {
            if (inInvocation) SwitchTo(NAMED_PARAMETER_EXPRESSION);
            else SwitchTo(FM_EXPRESSION);
        }
    }
    |
    <OPENING_CURLY_BRACKET : "{">
    {
        ++hashLiteralNesting;
    }
    |
    <CLOSING_CURLY_BRACKET : "}">
    {
        if (hashLiteralNesting == 0) endInterpolation(matchedToken);
        else --hashLiteralNesting;
    }
    |
    <IN : "in">
    |
    <AS : "as">
    |
    <USING : "using">
    |
    <ID: <ID_START_CHAR> (<ID_START_CHAR>|<ASCII_DIGIT>)*> {
        // Remove backslashes from Token.image:
        final String s = matchedToken.image;
        if (s.indexOf('\\') != -1) {
            final int srcLn = s.length(); 
            final char[] newS = new char[srcLn - 1];
            int dstIdx = 0;
            for (int srcIdx = 0; srcIdx < srcLn; srcIdx++) {
                final char c = s.charAt(srcIdx);
                if (c != '\\') {
                    newS[dstIdx++] = c;
                }
            }
            matchedToken.image = new String(newS, 0, dstIdx);
        }
    }
    |
    <OPEN_MISPLACED_INTERPOLATION : "${" | "#{">
    {
        if ("".length() == 0) {  // prevents unreachabe "break" compilation error in generated Java
            char c = matchedToken.image.charAt(0);
            throw new TokenMgrError(
                    "You can't use \"" + c + "{\" here as you are already in FreeMarker-expression-mode. Thus, instead "
                    + "of " + c + "{myExpression}, just write myExpression. "
                    + "(" + c + "{...} is only needed where otherwise static text is expected, i.e, outside " 
                    + "FreeMarker tags and ${...}-s.)",
                    TokenMgrError.LEXICAL_ERROR,
                    matchedToken.beginLine, matchedToken.beginColumn,
                    matchedToken.endLine, matchedToken.endColumn);
        }
    }
    |
    <#NON_ESCAPED_ID_START_CHAR:
        [
            // This was generated on JDK 1.8.0_20 Win64 with src/main/misc/identifierChars/IdentifierCharGenerator.java
			"$", 
			"@" - "Z", 
			"_", 
			"a" - "z", 
			"\u00AA", 
			"\u00B5", 
			"\u00BA", 
			"\u00C0" - "\u00D6", 
			"\u00D8" - "\u00F6", 
			"\u00F8" - "\u1FFF", 
			"\u2071", 
			"\u207F", 
			"\u2090" - "\u209C", 
			"\u2102", 
			"\u2107", 
			"\u210A" - "\u2113", 
			"\u2115", 
			"\u2119" - "\u211D", 
			"\u2124", 
			"\u2126", 
			"\u2128", 
			"\u212A" - "\u212D", 
			"\u212F" - "\u2139", 
			"\u213C" - "\u213F", 
			"\u2145" - "\u2149", 
			"\u214E", 
			"\u2183" - "\u2184", 
			"\u2C00" - "\u2C2E", 
			"\u2C30" - "\u2C5E", 
			"\u2C60" - "\u2CE4", 
			"\u2CEB" - "\u2CEE", 
			"\u2CF2" - "\u2CF3", 
			"\u2D00" - "\u2D25", 
			"\u2D27", 
			"\u2D2D", 
			"\u2D30" - "\u2D67", 
			"\u2D6F", 
			"\u2D80" - "\u2D96", 
			"\u2DA0" - "\u2DA6", 
			"\u2DA8" - "\u2DAE", 
			"\u2DB0" - "\u2DB6", 
			"\u2DB8" - "\u2DBE", 
			"\u2DC0" - "\u2DC6", 
			"\u2DC8" - "\u2DCE", 
			"\u2DD0" - "\u2DD6", 
			"\u2DD8" - "\u2DDE", 
			"\u2E2F", 
			"\u3005" - "\u3006", 
			"\u3031" - "\u3035", 
			"\u303B" - "\u303C", 
			"\u3040" - "\u318F", 
			"\u31A0" - "\u31BA", 
			"\u31F0" - "\u31FF", 
			"\u3300" - "\u337F", 
			"\u3400" - "\u4DB5", 
			"\u4E00" - "\uA48C", 
			"\uA4D0" - "\uA4FD", 
			"\uA500" - "\uA60C", 
			"\uA610" - "\uA62B", 
			"\uA640" - "\uA66E", 
			"\uA67F" - "\uA697", 
			"\uA6A0" - "\uA6E5", 
			"\uA717" - "\uA71F", 
			"\uA722" - "\uA788", 
			"\uA78B" - "\uA78E", 
			"\uA790" - "\uA793", 
			"\uA7A0" - "\uA7AA", 
			"\uA7F8" - "\uA801", 
			"\uA803" - "\uA805", 
			"\uA807" - "\uA80A", 
			"\uA80C" - "\uA822", 
			"\uA840" - "\uA873", 
			"\uA882" - "\uA8B3", 
			"\uA8D0" - "\uA8D9", 
			"\uA8F2" - "\uA8F7", 
			"\uA8FB", 
			"\uA900" - "\uA925", 
			"\uA930" - "\uA946", 
			"\uA960" - "\uA97C", 
			"\uA984" - "\uA9B2", 
			"\uA9CF" - "\uA9D9", 
			"\uAA00" - "\uAA28", 
			"\uAA40" - "\uAA42", 
			"\uAA44" - "\uAA4B", 
			"\uAA50" - "\uAA59", 
			"\uAA60" - "\uAA76", 
			"\uAA7A", 
			"\uAA80" - "\uAAAF", 
			"\uAAB1", 
			"\uAAB5" - "\uAAB6", 
			"\uAAB9" - "\uAABD", 
			"\uAAC0", 
			"\uAAC2", 
			"\uAADB" - "\uAADD", 
			"\uAAE0" - "\uAAEA", 
			"\uAAF2" - "\uAAF4", 
			"\uAB01" - "\uAB06", 
			"\uAB09" - "\uAB0E", 
			"\uAB11" - "\uAB16", 
			"\uAB20" - "\uAB26", 
			"\uAB28" - "\uAB2E", 
			"\uABC0" - "\uABE2", 
			"\uABF0" - "\uABF9", 
			"\uAC00" - "\uD7A3", 
			"\uD7B0" - "\uD7C6", 
			"\uD7CB" - "\uD7FB", 
			"\uF900" - "\uFB06", 
			"\uFB13" - "\uFB17", 
			"\uFB1D", 
			"\uFB1F" - "\uFB28", 
			"\uFB2A" - "\uFB36", 
			"\uFB38" - "\uFB3C", 
			"\uFB3E", 
			"\uFB40" - "\uFB41", 
			"\uFB43" - "\uFB44", 
			"\uFB46" - "\uFBB1", 
			"\uFBD3" - "\uFD3D", 
			"\uFD50" - "\uFD8F", 
			"\uFD92" - "\uFDC7", 
			"\uFDF0" - "\uFDFB", 
			"\uFE70" - "\uFE74", 
			"\uFE76" - "\uFEFC", 
			"\uFF10" - "\uFF19", 
			"\uFF21" - "\uFF3A", 
			"\uFF41" - "\uFF5A", 
			"\uFF66" - "\uFFBE", 
			"\uFFC2" - "\uFFC7", 
			"\uFFCA" - "\uFFCF", 
			"\uFFD2" - "\uFFD7", 
			"\uFFDA" - "\uFFDC" 
        ]
    >
    |
    <#ESCAPED_ID_CHAR: "\\" ("-" | "." | ":")>
    |
    <#ID_START_CHAR: <NON_ESCAPED_ID_START_CHAR>|<ESCAPED_ID_CHAR>>
    |
    <#ASCII_DIGIT: ["0" - "9"]>
}

<FM_EXPRESSION, NO_SPACE_EXPRESSION, NAMED_PARAMETER_EXPRESSION> TOKEN :
{
    <DIRECTIVE_END : ">">
    {
        if (inFTLHeader) eatNewline();
        inFTLHeader = false;
        if (squBracTagSyntax) {
            matchedToken.kind = NATURAL_GT;
        } else {
            SwitchTo(DEFAULT);
        }
    }
    |
    <EMPTY_DIRECTIVE_END : "/>" | "/]">
    {
        if (inFTLHeader) eatNewline();
        inFTLHeader = false;
        SwitchTo(DEFAULT);
    }
}

<IN_PAREN> TOKEN :
{
    <NATURAL_GT : ">">
    |
    <NATURAL_GTE : ">=">
}

<NO_SPACE_EXPRESSION> TOKEN :
{
    <TERMINATING_WHITESPACE :  (["\n", "\r", "\t", " "])+> : FM_EXPRESSION
}

<NAMED_PARAMETER_EXPRESSION> TOKEN :
{
    <TERMINATING_EXCLAM : "!" (["\n", "\r", "\t", " "])+> : FM_EXPRESSION
}

<NO_PARSE> TOKEN :
{
    <TERSE_COMMENT_END : "-->" | "--]">
    {
        if (noparseTag.equals("-->")) {
            boolean squareBracket = matchedToken.image.endsWith("]");
            if ((squBracTagSyntax && squareBracket) || (!squBracTagSyntax && !squareBracket)) {
                matchedToken.image = matchedToken.image + ";"; 
                SwitchTo(DEFAULT);
            }
        }
    }
    |
    <MAYBE_END :
        ("<" | "[")
        "/"
        ("#")?
        (["a"-"z", "A"-"Z"])+
        ( " " | "\t" | "\n" | "\r" )*
        (">" | "]")
    >
    {
        StringTokenizer st = new StringTokenizer(image.toString(), " \t\n\r<>[]/#", false);
        if (st.nextToken().equals(noparseTag)) {
            matchedToken.image = matchedToken.image + ";"; 
            SwitchTo(DEFAULT);
        }
    }
    |
    <KEEP_GOING : (~["<", "[", "-"])+>
    |
    <LONE_LESS_THAN_OR_DASH : ["<", "[", "-"]>
}

// Now the actual parsing code, starting
// with the productions for FreeMarker's
// expression syntax.

/**
 * This is the same as OrExpression, since
 * the OR is the operator with the lowest
 * precedence.
 */
Expression Expression() :
{
    Expression exp;
}
{
    exp = OrExpression()
    {
        return exp;
    }
}

/**
 * Lowest level expression, a literal, a variable,
 * or a possibly more complex expression bounded
 * by parentheses.
 */
Expression PrimaryExpression() :
{
    Expression exp;
}
{
    (
        exp = NumberLiteral()
        |   
        exp = HashLiteral()
        |   
        exp = StringLiteral(true)
        |   
        exp = BooleanLiteral()
        |   
        exp = ListLiteral()
        |   
        exp = Identifier()
        |   
        exp = Parenthesis()
        |   
        exp = BuiltinVariable()
    )
    (
        LOOKAHEAD(<DOT> | <OPEN_BRACKET> |<OPEN_PAREN> | <BUILT_IN> | <EXCLAM> | <TERMINATING_EXCLAM> | <EXISTS>)
        exp = AddSubExpression(exp)
    )*
    {
        return exp;
    }
}

Expression Parenthesis() :
{
    Expression exp, result;
    Token start, end;
}
{
    start = <OPEN_PAREN>
    exp = Expression()
    end = <CLOSE_PAREN>
    {
        result = new ParentheticalExpression(exp);
        result.setLocation(template, start, end);
        return result;
    }
}

/**
 * A primary expression preceded by zero or
 * more unary operators. (The only unary operator we
 * currently have is the NOT.)
 */
Expression UnaryExpression() :
{
    Expression exp, result;
    boolean haveNot = false;
    Token t = null, start = null;
}
{
    (
        result = UnaryPlusMinusExpression()
        |
        result = NotExpression()
        |
        result = PrimaryExpression()
    )
    {
        return result;
    }
}

Expression NotExpression() : 
{
    Token t;
    Expression exp, result = null;
    ArrayList nots = new ArrayList();
}
{
    (
        t = <EXCLAM> { nots.add(t); }
    )+
    exp = PrimaryExpression()
    {
        for (int i = 0; i < nots.size(); i++) {
            result = new NotExpression(exp);
            Token tok = (Token) nots.get(nots.size() -i -1);
            result.setLocation(template, tok, exp);
            exp = result;
        }
        return result;
    }
}

Expression UnaryPlusMinusExpression() :
{
    Expression exp, result;
    boolean isMinus = false;
    Token t;
}
{
    (
        t = <PLUS>
        |
        t = <MINUS> { isMinus = true; }
    )
    exp = PrimaryExpression()
    {
        result = new UnaryPlusMinusExpression(exp, isMinus);  
        result.setLocation(template, t, exp);
        return result;
    }
}

Expression AdditiveExpression() :
{
    Expression lhs, rhs, result;
    boolean plus;
}
{
    lhs = MultiplicativeExpression() { result = lhs; }
    (
        LOOKAHEAD(<PLUS>|<MINUS>)
        (
            (
                <PLUS> { plus = true; }
                |
                <MINUS> { plus = false; }
            )
        )
        rhs = MultiplicativeExpression()
        {
            if (plus) {
	            // plus is treated separately, since it is also
	            // used for concatenation.
                result = new AddConcatExpression(lhs, rhs);
            } else {
                numberLiteralOnly(lhs);
                numberLiteralOnly(rhs);
                result = new ArithmeticExpression(lhs, rhs, ArithmeticExpression.TYPE_SUBSTRACTION);
            }
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

/**
 * A unary expression followed by zero or more
 * unary expressions with operators in between.
 */
Expression MultiplicativeExpression() :
{
    Expression lhs, rhs, result;
    int operation = ArithmeticExpression.TYPE_MULTIPLICATION;
}
{
    lhs = UnaryExpression() { result = lhs; }
    (
        LOOKAHEAD(<TIMES>|<DIVIDE>|<PERCENT>)
        (
            (
                <TIMES> { operation = ArithmeticExpression.TYPE_MULTIPLICATION; }
                |
                <DIVIDE> { operation = ArithmeticExpression.TYPE_DIVISION; }
                |
                <PERCENT> {operation = ArithmeticExpression.TYPE_MODULO; }
            )
        )
        rhs = UnaryExpression()
        {
            numberLiteralOnly(lhs);
            numberLiteralOnly(rhs);
            result = new ArithmeticExpression(lhs, rhs, operation);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}


Expression EqualityExpression() :
{
    Expression lhs, rhs, result;
    Token t;
}
{
    lhs = RelationalExpression() { result = lhs; }
    [
        LOOKAHEAD(<NOT_EQUALS>|<EQUALS>|<DOUBLE_EQUALS>)
        (
            t = <NOT_EQUALS> 
            |
            t = <EQUALS> 
            |
            t = <DOUBLE_EQUALS>
        )
        rhs = RelationalExpression()
        {
	        notHashLiteral(lhs, "scalar");
	        notHashLiteral(rhs, "scalar");
	        notListLiteral(lhs, "scalar");
	        notListLiteral(rhs, "scalar");
	        result = new ComparisonExpression(lhs, rhs, t.image);
	        result.setLocation(template, lhs, rhs);
        }
    ]
    {
        return result;
    }
}

Expression RelationalExpression() :
{
    Expression lhs, rhs, result;
    Token t;
}
{
    lhs = RangeExpression() { result = lhs; }
    [
        LOOKAHEAD(<NATURAL_GTE>|<ESCAPED_GTE>|<NATURAL_GT>|<ESCAPED_GT>|<LESS_THAN_EQUALS>|<LESS_THAN_EQUALS>|<LESS_THAN>)
        (
            t = <NATURAL_GTE>
            |
            t = <ESCAPED_GTE>
            |
            t = <NATURAL_GT>
            |
            t = <ESCAPED_GT>
            |
            t = <LESS_THAN_EQUALS>
            |
            t = <LESS_THAN>
        )
        rhs = RangeExpression()
        {
            notHashLiteral(lhs, "scalar");
            notHashLiteral(rhs, "scalar");
            notListLiteral(lhs, "scalar");
            notListLiteral(rhs, "scalar");
            notStringLiteral(lhs, "number");
            notStringLiteral(rhs, "number");
            result = new ComparisonExpression(lhs, rhs, t.image);
            result.setLocation(template, lhs, rhs);
        }
    ]
    {
        return result;
    }
}

Expression RangeExpression() :
{
    Expression lhs, rhs = null, result;
    int endType;
    Token dotDot = null;
}
{
    lhs = AdditiveExpression() { result = lhs; }
    [
        LOOKAHEAD(1)  // To suppress warning
        (
            (
                (
                    <DOT_DOT_LESS> { endType = Range.END_EXCLUSIVE; }
                    |
                    <DOT_DOT_ASTERISK> { endType = Range.END_SIZE_LIMITED; }
                )
                rhs = AdditiveExpression()
            )
            | 
            (
                dotDot = <DOT_DOT> { endType = Range.END_UNBOUND; }
                [
                    LOOKAHEAD(AdditiveExpression())
                    rhs = AdditiveExpression()
                    {
                        endType = Range.END_INCLUSIVE;
                    }
                ]
            )
        )
        {
            numberLiteralOnly(lhs);
            if (rhs != null) {
                numberLiteralOnly(rhs);
            }
           
            Range range = new Range(lhs, rhs, endType);
            if (rhs != null) {
                range.setLocation(template, lhs, rhs);
            } else {
                range.setLocation(template, lhs, dotDot);
            }
            result = range;
        }
    ]
    {
        return result;
    }
}


Expression AndExpression() :
{
    Expression lhs, rhs, result;
}
{
    lhs = EqualityExpression() { result = lhs; }
    (
        LOOKAHEAD(<AND>)
        <AND>
        rhs = EqualityExpression()
        {
            booleanLiteralOnly(lhs);
            booleanLiteralOnly(rhs);
            result = new AndExpression(lhs, rhs);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

Expression OrExpression() :
{
    Expression lhs, rhs, result;
}
{
    lhs = AndExpression() { result = lhs; }
    (
        LOOKAHEAD(<OR>)
        <OR>
        rhs = AndExpression()
        {
            booleanLiteralOnly(lhs);
            booleanLiteralOnly(rhs);
            result = new OrExpression(lhs, rhs);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

ListLiteral ListLiteral() :
{
    ArrayList values = new ArrayList();
    Token begin, end;
}
{
    begin = <OPEN_BRACKET>
    values = PositionalArgs()
    end = <CLOSE_BRACKET>
    {
        ListLiteral result = new ListLiteral(values);
        result.setLocation(template, begin, end);
        return result;
    }
}

Expression NumberLiteral() :
{
    Token op = null, t;
}
{
    (
        t = <INTEGER>
        |
        t = <DECIMAL>
    )
    {
        String s = t.image;
        Expression result = new NumberLiteral(pCfg.getArithmeticEngine().toNumber(s));
        Token startToken = (op != null) ? op : t;
        result.setLocation(template, startToken, t);
        return result;
    }
}

Identifier Identifier() :
{
    Token t;
}
{
    t = <ID>
    {
        Identifier id = new Identifier(t.image);
        id.setLocation(template, t, t);
        return id;
    }
}

Expression IdentifierOrStringLiteral() :
{
    Expression exp;
}
{
    (
        exp = Identifier()
        |
        exp = StringLiteral(false)
    )
    {
        return exp;
    }   
}

BuiltinVariable BuiltinVariable() :
{
    Token dot, name;
}
{
    dot = <DOT>
    name = <ID>
    {
        BuiltinVariable result = null;
        token_source.checkNamingConvention(name);

        TemplateModel parseTimeValue;
        String nameStr = name.image;
        if (nameStr.equals(BuiltinVariable.OUTPUT_FORMAT) || nameStr.equals(BuiltinVariable.OUTPUT_FORMAT_CC)) {
            parseTimeValue = new SimpleScalar(outputFormat.getName());
        } else if (nameStr.equals(BuiltinVariable.AUTO_ESC) || nameStr.equals(BuiltinVariable.AUTO_ESC_CC)) {
            parseTimeValue = autoEscaping ? TemplateBooleanModel.TRUE : TemplateBooleanModel.FALSE;
        } else {            parseTimeValue = null;
        }        
        result = new BuiltinVariable(name, token_source, parseTimeValue);
        
        result.setLocation(template, dot, name);
        return result;
    }
}

/**
 * Production that builds up an expression
 * using the dot or dynamic key name
 * or the args list if this is a method invocation.
 */
Expression AddSubExpression(Expression exp) :
{
    Expression result = null;
}
{
    (
        result = DotVariable(exp)
        |
        result = DynamicKey(exp)
        |
        result = MethodArgs(exp)
        |
        result = BuiltIn(exp)
        |
        result = DefaultTo(exp)
        |
        result = Exists(exp)
    )
    {
        return result;
    }
}

Expression DefaultTo(Expression exp) :
{
    Expression rhs = null;
    Token t;
}
{
    (
        t = <TERMINATING_EXCLAM>
        |
        (
            t = <EXCLAM>
            [
                LOOKAHEAD(Expression())
                rhs = Expression()
            ]
        )
    )
    {
        DefaultToExpression result = new DefaultToExpression(exp, rhs);
        if (rhs == null) {
            result.setLocation(template, exp, t);
        } else {
            result.setLocation(template, exp, rhs);
        }
        return result;
    }
}

Expression Exists(Expression exp) :
{
    Token t;
}
{
    t = <EXISTS>
    {
        ExistsExpression result = new ExistsExpression(exp);
        result.setLocation(template, exp, t);
        return result;
    }
}

Expression BuiltIn(Expression lhoExp) :
{
    Token t = null;
    BuiltIn result;
    ArrayList/*<Expression>*/ args = null;
    Token openParen;
    Token closeParen;
}
{
    <BUILT_IN>
    t = <ID>
    {
        token_source.checkNamingConvention(t);
        result = BuiltIn.newBuiltIn(incompatibleImprovements, lhoExp, t, token_source);
        result.setLocation(template, lhoExp, t);
        
        if (!(result instanceof SpecialBuiltIn)) {
            return result;
        }

        if (result instanceof BuiltInForLoopVariable) {
            if (!(lhoExp instanceof Identifier)) {
                throw new ParseException(
                        "Expression used as the left hand operand of ?" + t.image
                        + " must be a simple loop variable name.", lhoExp);
            }
            String loopVarName = ((Identifier) lhoExp).getName();
            checkLoopVariableBuiltInLHO(loopVarName, lhoExp, t);
            ((BuiltInForLoopVariable) result).bindToLoopVariable(loopVarName);
            
            return result;
        }
        
        if (result instanceof BuiltInBannedWhenAutoEscaping) {
	        if (outputFormat instanceof MarkupOutputFormat && autoEscaping) {
	            throw new ParseException(
	                    "Using ?" + t.image + " (legacy escaping) is not allowed when auto-escaping is on with "
	                    + "a markup output format (" + outputFormat.getName() + "), to avoid double-escaping mistakes.",
	                    template, t);
	        }
            
            return result;
        }

        if (result instanceof MarkupOutputFormatBoundBuiltIn) {
            if (!(outputFormat instanceof MarkupOutputFormat)) {
                throw new ParseException(
                        "?" + t.image + " can't be used here, as the current output format isn't a markup (escaping) "
                        + "format: " + outputFormat, template, t);
            }
            ((MarkupOutputFormatBoundBuiltIn) result).bindToMarkupOutputFormat((MarkupOutputFormat) outputFormat);
            
            return result;
        }

        if (result instanceof OutputFormatBoundBuiltIn) {
            ((OutputFormatBoundBuiltIn) result).bindToOutputFormat(outputFormat, autoEscapingPolicy);
            
            return result;
        }
    }
    [
        LOOKAHEAD({ result instanceof BuiltInWithParseTimeParameters  })
        openParen = <OPEN_PAREN>
        args = PositionalArgs()
        closeParen = <CLOSE_PAREN> {
            result.setLocation(template, lhoExp, closeParen);
            ((BuiltInWithParseTimeParameters) result).bindToParameters(args, openParen, closeParen);
            
            return result;
        }
    ]
    {
        // Should have already return-ed
        throw new AssertionError("Unhandled " + SpecialBuiltIn.class.getName() + " subclass: " + result.getClass());
    }
}


/**
 * production for when a key is specified by <DOT> + keyname
 */
Expression DotVariable(Expression exp) :
{
    Token t;
}
{
        <DOT>
        (
            t = <ID> | t = <TIMES> | t = <DOUBLE_STAR> 
            |
            (
                t = <LESS_THAN>
                |
                t = <LESS_THAN_EQUALS>
                |
                t = <ESCAPED_GT>
                |
                t = <ESCAPED_GTE>
                |
                t = <FALSE>
                |
                t = <TRUE>
                |
                t = <IN>
                |
                t = <AS>
                |
                t = <USING>
            )
            {
                if (!Character.isLetter(t.image.charAt(0))) {
                    throw new ParseException(t.image + " is not a valid identifier.", template, t);
                }
            }
        )
        {
            notListLiteral(exp, "hash");
            notStringLiteral(exp, "hash");
            notBooleanLiteral(exp, "hash");
            Dot dot = new Dot(exp, t.image);
            dot.setLocation(template, exp, t);
            return dot;
        }
}

/**
 * production for when the key is specified
 * in brackets.
 */
Expression DynamicKey(Expression exp) :
{
    Expression arg;
    Token t;
}
{
    <OPEN_BRACKET>
    arg = Expression()
    t = <CLOSE_BRACKET>
    {
        notBooleanLiteral(exp, "list or hash");
        notNumberLiteral(exp, "list or hash");
        DynamicKeyName dkn = new DynamicKeyName(exp, arg);
        dkn.setLocation(template, exp, t);
        return dkn;
    }
}

/**
 * production for an arglist part of a method invocation.
 */
MethodCall MethodArgs(Expression exp) :
{
        ArrayList args = new ArrayList();
        Token end;
}
{
        <OPEN_PAREN>
        args = PositionalArgs()
        end = <CLOSE_PAREN>
        {
            args.trimToSize();
            MethodCall result = new MethodCall(exp, args);
            result.setLocation(template, exp, end);
            return result;
        }
}

StringLiteral StringLiteral(boolean interpolate) :
{
    Token t;
    boolean raw = false;
}
{
    (
        t = <STRING_LITERAL>
        |
        t = <RAW_STRING> { raw = true; }
    )
    {
        String s;
        // Get rid of the quotes.
        if (raw) {
            s = t.image.substring(2, t.image.length() -1);
        } else {
	        try {
	            s = StringUtil.FTLStringLiteralDec(t.image.substring(1, t.image.length() -1));
	        } catch (ParseException pe) {
	            pe.lineNumber = t.beginLine;
	            pe.columnNumber = t.beginColumn;
	            pe.endLineNumber = t.endLine;
	            pe.endColumnNumber = t.endColumn;
	            throw pe;
	        }
        }
        StringLiteral result = new StringLiteral(s);
        result.setLocation(template, t, t);
        if (interpolate && !raw) {
            // TODO: This logic is broken. It can't handle literals that contains both ${...} and $\{...}. 
            if (t.image.indexOf("${") >= 0 || t.image.indexOf("#{") >= 0) result.parseValue(token_source, outputFormat);
        }
        return result;
    }
}

Expression BooleanLiteral() :
{
    Token t;
    Expression result;
}
{
    (
        t = <FALSE> { result = new BooleanLiteral(false); }
        |
        t = <TRUE> { result = new BooleanLiteral(true); }
    )
    {
        result.setLocation(template, t, t);
        return result;
    }
}


HashLiteral HashLiteral() :
{
    Token begin, end;
    Expression key, value;
    ArrayList keys = new ArrayList();
    ArrayList values = new ArrayList();
}
{
    begin = <OPENING_CURLY_BRACKET>
    [
        key = Expression()
        (<COMMA>|<COLON>)
        value = Expression()
        {
            stringLiteralOnly(key);
            keys.add(key);
            values.add(value);
        }
        (
            <COMMA>
            key = Expression()
            (<COMMA>|<COLON>)
            value = Expression()
            {
                stringLiteralOnly(key);
                keys.add(key);
                values.add(value);
            }
        )*
    ]
    end = <CLOSING_CURLY_BRACKET>
    {
        HashLiteral result = new HashLiteral(keys, values);
        result.setLocation(template, begin, end);
        return result;
    }
}

/**
 * A production representing the ${...}
 * that outputs a variable.
 */
DollarVariable StringOutput() :
{
    Expression exp;
    Token begin, end;
}
{
    begin = <DOLLAR_INTERPOLATION_OPENING>
    exp = Expression()
    {
        notHashLiteral(exp, NonStringException.STRING_COERCABLE_TYPES_DESC);
        notListLiteral(exp, NonStringException.STRING_COERCABLE_TYPES_DESC);
    }
    end = <CLOSING_CURLY_BRACKET>
    {
        DollarVariable result = new DollarVariable(
                exp, escapedExpression(exp),
                outputFormat,
                autoEscaping);
        result.setLocation(template, begin, end);
        return result;
    }
}

NumericalOutput NumericalOutput() :
{
    Expression exp;
    Token fmt = null, begin, end;
}
{
    begin = <HASH_INTERPOLATION_OPENING>
    exp = Expression() { numberLiteralOnly(exp); }
    [
        <SEMICOLON>
        fmt = <ID>
    ]
    end = <CLOSING_CURLY_BRACKET>
    {
        MarkupOutputFormat<?> autoEscOF = autoEscaping && outputFormat instanceof MarkupOutputFormat
                ? (MarkupOutputFormat<?>) outputFormat : null;
    
        NumericalOutput result;
        if (fmt != null) {
            int minFrac = -1;  // -1 indicates that the value has not been set
            int maxFrac = -1;

            StringTokenizer st = new StringTokenizer(fmt.image, "mM", true);
            char type = '-';
            while (st.hasMoreTokens()) {
                String token = st.nextToken();
                try {
	                if (type != '-') {
	                    switch (type) {
	                    case 'm':
	                        if (minFrac != -1) throw new ParseException("Invalid formatting string", template, fmt);
	                        minFrac = Integer.parseInt(token);
	                        break;
	                    case 'M':
	                        if (maxFrac != -1) throw new ParseException("Invalid formatting string", template, fmt);
	                        maxFrac = Integer.parseInt(token);
	                        break;
	                    default:
	                        throw new ParseException("Invalid formatting string", template, fmt);
	                    }
	                    type = '-';
	                } else if (token.equals("m")) {
	                    type = 'm';
	                } else if (token.equals("M")) {
	                    type = 'M';
	                } else {
	                    throw new ParseException();
	                }
                } catch (ParseException e) {
                	throw new ParseException("Invalid format specifier " + fmt.image, template, fmt);
                } catch (NumberFormatException e) {
                	throw new ParseException("Invalid number in the format specifier " + fmt.image, template, fmt);
                }
            }

            if (maxFrac == -1) {
	            if (minFrac == -1) {
	                throw new ParseException(
	                		"Invalid format specification, at least one of m and M must be specified!", template, fmt);
	            }
            	maxFrac = minFrac;
            } else if (minFrac == -1) {
            	minFrac = 0;
            }
            if (minFrac > maxFrac) {
            	throw new ParseException(
            			"Invalid format specification, min cannot be greater than max!", template, fmt);
            }
            if (minFrac > 50 || maxFrac > 50) {// sanity check
                throw new ParseException("Cannot specify more than 50 fraction digits", template, fmt);
            }
            result = new NumericalOutput(exp, minFrac, maxFrac, autoEscOF);
        } else {  // if format != null
            result = new NumericalOutput(exp, autoEscOF);
        }
        result.setLocation(template, begin, end);
        return result;
    }
}

TemplateElement If() :
{
    Token start, end, t;
    Expression condition;
    TemplateElements children;
    IfBlock ifBlock;
    ConditionalBlock cblock;
}
{
    start = <IF>
    condition = Expression()
    end = <DIRECTIVE_END>
    children = MixedContentElements()
    {
        cblock = new ConditionalBlock(condition, children, ConditionalBlock.TYPE_IF);
        cblock.setLocation(template, start, end, children);
        ifBlock = new IfBlock(cblock);
    }
    (
        t = <ELSE_IF>
        condition = Expression()
        end = LooseDirectiveEnd()
        children = MixedContentElements()
        {
            cblock = new ConditionalBlock(condition, children, ConditionalBlock.TYPE_ELSE_IF);
            cblock.setLocation(template, t, end, children);
            ifBlock.addBlock(cblock);
        }
    )*
    [
            t = <ELSE>
            children = MixedContentElements()
            {
                cblock = new ConditionalBlock(null, children, ConditionalBlock.TYPE_ELSE);
                cblock.setLocation(template, t, t, children);
                ifBlock.addBlock(cblock);
            }
    ]
    end = <END_IF>
    {
        ifBlock.setLocation(template, start, end);
        return ifBlock;
    }
}

AttemptBlock Attempt() :
{
    Token start, end;
    TemplateElements children;
    RecoveryBlock recoveryBlock;
}
{
    start = <ATTEMPT>
    children = MixedContentElements()
    recoveryBlock = Recover()
    (
        end = <END_RECOVER>
        |
        end = <END_ATTEMPT>
    )
    {
        AttemptBlock result = new AttemptBlock(children, recoveryBlock);
        result.setLocation(template, start, end);
        return result;
    }
}

RecoveryBlock Recover() : 
{
    Token start;
    TemplateElements children;
}
{
    start = <RECOVER>
    children = MixedContentElements()
    {
        RecoveryBlock result = new RecoveryBlock(children);
        result.setLocation(template, start, start, children);
        return result;
    }
}

TemplateElement List() :
{
    Expression exp;
    Token loopVar = null, start, end;
    TemplateElements childrendBeforeElse;
    ElseOfList elseOfList = null;
    ParserIteratorBlockContext iterCtx;
}
{
    start = <LIST>
    exp = Expression()
    [
        <AS>
        loopVar = <ID>
    ]
    <DIRECTIVE_END>
    {
        iterCtx = pushIteratorBlockContext();
        if (loopVar != null) {
            iterCtx.loopVarName = loopVar.image;
            breakableDirectiveNesting++;
        }
    }
    
    childrendBeforeElse = MixedContentElements()
    {
        if (loopVar != null) {
            breakableDirectiveNesting--;
        } else if (iterCtx.kind != ITERATOR_BLOCK_KIND_ITEMS) {
            throw new ParseException(
                    "#list must have either \"as loopVar\" parameter or nested #items that belongs to it.",
                    template, start);
        }
        popIteratorBlockContext();
    }
    
    [
        elseOfList = ElseOfList()
    ]
    
    end = <END_LIST>
    {
        IteratorBlock list = new IteratorBlock(exp, loopVar != null ? loopVar.image : null, childrendBeforeElse, false);
        list.setLocation(template, start, end);

        TemplateElement result;
        if (elseOfList == null) {
            result = list;
        } else {
            result = new ListElseContainer(list, elseOfList);
            result.setLocation(template, start, end);
        }
        return result;
    }
}

ElseOfList ElseOfList() :
{
    Token start;
    TemplateElements children;
}
{
        start = <ELSE>
        children = MixedContentElements()
        {
            ElseOfList result = new ElseOfList(children);
	        result.setLocation(template, start, start, children);
	        return result;
        }
}

IteratorBlock ForEach() :
{
    Expression exp;
    Token loopVar, start, end;
    TemplateElements children;
}
{
    start = <FOREACH>
    loopVar = <ID>
    <IN>
    exp = Expression()
    <DIRECTIVE_END>
    {
        ParserIteratorBlockContext iterCtx = pushIteratorBlockContext();
        iterCtx.loopVarName = loopVar.image;
        iterCtx.kind = ITERATOR_BLOCK_KIND_FOREACH;
        breakableDirectiveNesting++;
    }
    
    children = MixedContentElements()
    
    end = <END_FOREACH>
    {
        breakableDirectiveNesting--;
        popIteratorBlockContext();
                
        IteratorBlock result = new IteratorBlock(exp, loopVar.image, children, true);
        result.setLocation(template, start, end);
        return result;
    }
}

Items Items() :
{
    Token loopVar, start, end;
    TemplateElements children;
    ParserIteratorBlockContext iterCtx;
}
{
    start = <ITEMS>
    loopVar = <ID>
    <DIRECTIVE_END>
    {
        iterCtx = peekIteratorBlockContext();
        if (iterCtx == null) {
            throw new ParseException("#items must be inside a #list block.", template, start);
        }
        if (iterCtx.loopVarName != null) {
            String msg;
	        if (iterCtx.kind == ITERATOR_BLOCK_KIND_FOREACH) {
	            msg = forEachDirectiveSymbol() + " doesn't support nested #items.";
	        } else if (iterCtx.kind == ITERATOR_BLOCK_KIND_ITEMS) {
                msg = "Can't nest #items into each other that belong to the same #list.";
	        } else {
	            msg = "The parent #list of the #items must not have \"as loopVar\" parameter.";
            }
            throw new ParseException(msg, template, start);
        }
        iterCtx.kind = ITERATOR_BLOCK_KIND_ITEMS;
        iterCtx.loopVarName = loopVar.image;
    
        breakableDirectiveNesting++;
    }
    
    children = MixedContentElements()
    
    end = <END_ITEMS>
    {
        breakableDirectiveNesting--;
        iterCtx.loopVarName = null;
        
        Items result = new Items(loopVar.image, children);
        result.setLocation(template, start, end);
        return result;
    }
}

Sep Sep() :
{
    Token loopVar, start, end = null;
    TemplateElements children;
}
{
    start = <SEP>
    {
        if (peekIteratorBlockContext() == null) {
            throw new ParseException(
                    "#sep must be inside a #list (or " + forEachDirectiveSymbol() + ") block.",
                    template, start);
        }
    }
    children = MixedContentElements()
    [
        LOOKAHEAD(1)
        end = <END_SEP>
    ]
    {
        Sep result = new Sep(children);
        if (end != null) {
            result.setLocation(template, start, end);
        } else {
            result.setLocation(template, start, start, children);
        }
        return result;
    }
}

VisitNode Visit() :
{
    Token start, end;
    Expression targetNode, namespaces = null;
}
{
    start = <VISIT>
    targetNode = Expression()
    [
        <USING>
        namespaces = Expression()
    ]
    end = LooseDirectiveEnd()
    {
        VisitNode result = new VisitNode(targetNode, namespaces);
        result.setLocation(template, start, end);
        return result;
    }
}

RecurseNode Recurse() :
{
    Token start, end = null;
    Expression node = null, namespaces = null;
}
{
    (
        start = <SIMPLE_RECURSE>
        |
        (
            start = <RECURSE>
            [
                node = Expression()
            ]
            [
                <USING>
                namespaces = Expression()
            ]
            end = LooseDirectiveEnd()
        )
    )
    {
        if (end == null) end = start;
        RecurseNode result = new RecurseNode(node, namespaces);
        result.setLocation(template, start, end);
        return result;
    }
}

FallbackInstruction FallBack() :
{
    Token tok;
}
{
    tok = <FALLBACK>
    {
        if (!inMacro) {
            throw new ParseException("Cannot fall back outside a macro.", template, tok);
        }
        FallbackInstruction result = new FallbackInstruction();
        result.setLocation(template, tok, tok);
        return result;
    }
}

/**
 * Production used to break out of a loop or a switch block.
 */
BreakInstruction Break() :
{
    Token start;
}
{
    start = <BREAK>
    {
        if (breakableDirectiveNesting < 1) {
            throw new ParseException(start.image + " must be nested inside a directive that supports it: " 
                    + " #list with \"as\", #items, #switch (or the deprecated " + forEachDirectiveSymbol() + ")",
                    template, start);
        }
        BreakInstruction result = new BreakInstruction();
        result.setLocation(template, start, start);
        return result;
    }
}

/**
 * Production used to jump out of a macro.
 * The stop instruction terminates the rendering of the template.
 */
ReturnInstruction Return() :
{
    Token start, end = null;
    Expression exp = null;
}
{
    (
        start = <SIMPLE_RETURN> { end = start; }
        |
        start = <RETURN> exp = Expression() end = LooseDirectiveEnd()
    )
    {
        if (inMacro) {
            if (exp != null) {
            	throw new ParseException("A macro cannot return a value", template, start);
            }
        } else if (inFunction) {
            if (exp == null) {
            	throw new ParseException("A function must return a value", template, start);
            }
        } else {
            if (exp == null) {
            	throw new ParseException(
            			"A return instruction can only occur inside a macro or function", template, start);
            }
        }
        ReturnInstruction result = new ReturnInstruction(exp);
        result.setLocation(template, start, end);
        return result;
    }
}

StopInstruction Stop() :
{
    Token start = null;
    Expression exp = null;
}
{
    (
        start = <HALT>
        |
        start = <STOP> exp = Expression() LooseDirectiveEnd()
    )
    {
        StopInstruction result = new StopInstruction(exp);
        result.setLocation(template, start, start);
        return result;
    }
}

TemplateElement Nested() :
{
    Token t, end;
    ArrayList bodyParameters;
    BodyInstruction result = null;
}
{
    (
        (
            t = <SIMPLE_NESTED>
            {
                result = new BodyInstruction(null);
                result.setLocation(template, t, t);
            }
        )
        |
        (
            t = <NESTED>
            bodyParameters = PositionalArgs()
            end = LooseDirectiveEnd()
            {
                result = new BodyInstruction(bodyParameters);
                result.setLocation(template, t, end);
            }
        )
    )
    {
        if (!inMacro) {
            throw new ParseException("Cannot use a " + t.image + " instruction outside a macro.", template, t);
        }
        return result;
    }
}

TemplateElement Flush() :
{
    Token t;
}
{
    t = <FLUSH>
    {
        FlushInstruction result = new FlushInstruction();
        result.setLocation(template, t, t);
        return result;
    }
}

TemplateElement Trim() :
{
    Token t;
    TrimInstruction result = null;
}
{
    (
        t = <TRIM> { result = new TrimInstruction(true, true); }
        |
        t = <LTRIM> { result = new TrimInstruction(true, false); }
        |
        t = <RTRIM> { result = new TrimInstruction(false, true); }
        |
        t = <NOTRIM> { result = new TrimInstruction(false, false); }
    )
    {
        result.setLocation(template, t, t);
        return result;
    }
}


TemplateElement Assign() :
{
    Token start, end;
    int scope;
    Token id = null;
    Token equalsOp;
    Expression nameExp, exp, nsExp = null;
    String varName;
    ArrayList assignments = new ArrayList();
    Assignment ass;
    TemplateElements children;
}
{
    (
        start = <ASSIGN> { scope = Assignment.NAMESPACE; }
        |
        start = <GLOBALASSIGN> { scope = Assignment.GLOBAL; }
        |
        start = <LOCALASSIGN> { scope = Assignment.LOCAL; }
        {
            scope = Assignment.LOCAL;
            if (!inMacro && !inFunction) {
                throw new ParseException("Local variable assigned outside a macro.", template, start);
            }
        }
    )
    nameExp = IdentifierOrStringLiteral()
    {
        varName = (nameExp instanceof StringLiteral)
                ? ((StringLiteral) nameExp).getAsString()
                : ((Identifier) nameExp).getName();
    }
    (
    	(
            (
	    	    (
			        (<EQUALS>|<PLUS_EQUALS>|<MINUS_EQUALS>|<TIMES_EQUALS>|<DIV_EQUALS>|<MOD_EQUALS>)
			        {
			           equalsOp = token;
			        }
			        exp = Expression()
		        )
		        |
		        (
	                (<PLUS_PLUS>|<MINUS_MINUS>)
	                {
	                   equalsOp = token;
	                   exp = null;
	                }
		        )
	        )
	        {
	            ass = new Assignment(varName, equalsOp.kind, exp, scope);
                if (exp != null) {
                   ass.setLocation(template, nameExp, exp);
                } else {
                   ass.setLocation(template, nameExp, equalsOp);
                }
	            assignments.add(ass);
	        }
	        (
	            LOOKAHEAD(
	               [<COMMA>]
	               (<ID>|<STRING_LITERAL>)
	               (<EQUALS>|<PLUS_EQUALS>|<MINUS_EQUALS>|<TIMES_EQUALS>|<DIV_EQUALS>|<MOD_EQUALS>
	                       |<PLUS_PLUS>|<MINUS_MINUS>)
	            )
	            [<COMMA>]
	            nameExp = IdentifierOrStringLiteral()
	            {
	                varName = (nameExp instanceof StringLiteral)
	                		? ((StringLiteral) nameExp).getAsString()
	                		: ((Identifier) nameExp).getName();
	            }
	            (
	                (
	                    (<EQUALS>|<PLUS_EQUALS>|<MINUS_EQUALS>|<TIMES_EQUALS>|<DIV_EQUALS>|<MOD_EQUALS>)
	                    {
	                       equalsOp = token;
	                    }
	                    exp = Expression()
	                )
	                |
	                (
	                    (<PLUS_PLUS>|<MINUS_MINUS>)
	                    {
	                       equalsOp = token;
	                       exp = null;
	                    }
	                )
	            )
	            {
	                ass = new Assignment(varName, equalsOp.kind, exp, scope);
	                if (exp != null) {
	                   ass.setLocation(template, nameExp, exp);
	                } else {
                       ass.setLocation(template, nameExp, equalsOp);
	                }
	                assignments.add(ass);
	            } 
	        )*
	        [
	            id = <IN>
	            nsExp = Expression()
	            {
	                if (scope != Assignment.NAMESPACE) {
	                	throw new ParseException("Cannot assign to namespace here.", template, id);
                	}
	            }
	        ]
	        end = LooseDirectiveEnd()
	        {
                if (assignments.size() == 1) {
                    Assignment a = (Assignment) assignments.get(0);
                    a.setNamespaceExp(nsExp);
                    a.setLocation(template, start, end);
                    return a;
                } else {
		            AssignmentInstruction ai = new AssignmentInstruction(scope);
		            for (int i = 0; i< assignments.size(); i++) {
		                ai.addAssignment((Assignment) assignments.get(i));
		            }
		            ai.setNamespaceExp(nsExp);
		            ai.setLocation(template, start, end);
		            return ai;
	            }
	        }
	    )
	    |
	    (
	        [
	            id = <IN>
	            nsExp = Expression()
	            {
	                if (scope != Assignment.NAMESPACE) {
	                	throw new ParseException("Cannot assign to namespace here.", template, id);
	            	}
	            }
	        ]
	        <DIRECTIVE_END>
	        children = MixedContentElements()
	        (
	            end = <END_LOCAL>
	            {
	            	if (scope != Assignment.LOCAL) {
	            		throw new ParseException("Mismatched assignment tags.", template, end);
	        		}
	        	}
	            |
	            end = <END_ASSIGN>
	            {
	            	if (scope != Assignment.NAMESPACE) {
	            		throw new ParseException("Mismatched assignment tags.", template, end);
	        		}
	        	}
	            |
	            end = <END_GLOBAL>
	            {
	            	if (scope != Assignment.GLOBAL) throw new ParseException(
	            			"Mismatched assignment tags", template, end);
            	}
	        )
	        {
	            BlockAssignment ba = new BlockAssignment(
	                   children, varName, scope, nsExp,
	                   getMarkupOutputFormat());
	            ba.setLocation(template, start, end);
	            return ba;
	        }
	    )
    )
}

Include Include() :
{
    Expression nameExp;
    Token att, start, end;
    Expression exp, parseExp = null, encodingExp = null, ignoreMissingExp = null;
}
{
    start = <_INCLUDE>
    nameExp = Expression()
    [<SEMICOLON>]
    (
        att = <ID>
        <EQUALS>
        exp = Expression()
        {
            String attString = att.image;
            if (attString.equalsIgnoreCase("parse")) {
        	    parseExp = exp;
            } else if (attString.equalsIgnoreCase("encoding")) {
            	encodingExp = exp;
            } else if (attString.equalsIgnoreCase("ignore_missing") || attString.equals("ignoreMissing")) {
                token_source.checkNamingConvention(att);
            	ignoreMissingExp = exp;
            } else {
                String correctedName = attString.equals("ignoreMissing") ? "ignore_missing" : null;
                throw new ParseException(
                		"Unsupported named #include parameter: \"" + attString + "\". Supported parameters are: "
                		+ "\"parse\", \"encoding\", \"ignore_missing\"."
                		+ (correctedName == null
                		      ? ""
                		      : " Supporting camelCase parameter names is planned for FreeMarker 2.4.0; "
	                              + "check if an update is available, and if it indeed supports camel "
	                              + "case."),
                		template, att);
            }
        }
    )*
    end = LooseDirectiveEnd()
    {
        Include result = new Include(template, nameExp, encodingExp, parseExp, ignoreMissingExp);
        result.setLocation(template, start, end);
        return result;
    }
}

LibraryLoad Import() :
{
    Token start, end, ns;
    Expression nameExp;
}
{
    start = <IMPORT>
    nameExp = Expression()
    <AS>
    ns = <ID>
    end = LooseDirectiveEnd()
    {
        LibraryLoad result = new LibraryLoad(template, nameExp, ns.image);
        result.setLocation(template, start, end);
        template.addImport(result);
        return result;
    }
}

Macro Macro() :
{
    Token arg, start, end;
    Expression nameExp;
    String name;
    ArrayList argNames = new ArrayList();
    HashMap args = new HashMap();
    ArrayList defNames = new ArrayList();
    Expression defValue = null;
    List lastIteratorBlockContexts;
    int lastBreakableDirectiveNesting;
    TemplateElements children;
    boolean isFunction = false, hasDefaults = false;
    boolean isCatchAll = false;
    String catchAll = null;
}
{
    (
        start = <MACRO>
        |
        start = <FUNCTION> { isFunction = true; }
    )
    {
        if (inMacro || inFunction) {
            throw new ParseException("Macros cannot be nested.", template, start);
        }
        if (isFunction) inFunction = true; else inMacro = true;
    }
    nameExp = IdentifierOrStringLiteral()
    {
        name = (nameExp instanceof StringLiteral)
                ? ((StringLiteral) nameExp).getAsString()
                : ((Identifier) nameExp).getName();
    }
    [<OPEN_PAREN>]
    (
        arg = <ID> { defValue = null; }
        [
            <ELLIPSIS> { isCatchAll = true; }
        ]
        [
            <EQUALS>
            defValue = Expression()
            {
                defNames.add(arg.image);
                hasDefaults = true;
            }
        ]
        [<COMMA>]
        {
            if (catchAll != null) {
                throw new ParseException(
                "There may only be one \"catch-all\" parameter in a macro declaration, and it must be the last parameter.",
                template, arg);
            }
            if (isCatchAll) {
                if (defValue != null) {
                    throw new ParseException(
                    "\"Catch-all\" macro parameter may not have a default value.",
                    template, arg);
                }
                catchAll = arg.image;
            } else {
                argNames.add(arg.image);
                if (hasDefaults && defValue == null) {
                    throw new ParseException(
		                    "In a macro declaration, parameters without a default value "
		                    + "must all occur before the parameters with default values.",
                    template, arg);
                }
                args.put(arg.image, defValue);
            }
        }
    )*
    [<CLOSE_PAREN>]
    <DIRECTIVE_END>
    {
        // To prevent parser check loopholes like <#list ...><#macro ...><#break></#macro></#list>.
        lastIteratorBlockContexts = iteratorBlockContexts;
        iteratorBlockContexts = null;
        if (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_23) {
	        lastBreakableDirectiveNesting = breakableDirectiveNesting;
	        breakableDirectiveNesting = 0; 
        } else {
            lastBreakableDirectiveNesting = 0; // Just to prevent uninitialized local variable error later
        }
    }
    children = MixedContentElements()
    (
        end = <END_MACRO>
        {
        	if (isFunction) throw new ParseException("Expected function end tag here.", template, start);
    	}
        |
        end = <END_FUNCTION>
        {
    		if (!isFunction) throw new ParseException("Expected macro end tag here.", template, start);
    	}
    )
    {
        iteratorBlockContexts = lastIteratorBlockContexts;
        if (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_23) {
            breakableDirectiveNesting = lastBreakableDirectiveNesting;
        }
        
        inMacro = inFunction = false;
        Macro result = new Macro(name, argNames, args, catchAll, isFunction, children);
        result.setLocation(template, start, end);
        template.addMacro(result);
        return result;
    }
}

CompressedBlock Compress() :
{
    TemplateElements children;
    Token start, end;
}
{
    start = <COMPRESS>
    children = MixedContentElements()
    end = <END_COMPRESS>
    {
        CompressedBlock cb = new CompressedBlock(children);
        cb.setLocation(template, start, end);
        return cb;
    }
}

TemplateElement UnifiedMacroTransform() :
{
    Token start = null, end, t;
    HashMap namedArgs = null;
    ArrayList positionalArgs = null, bodyParameters = null;
    Expression startTagNameExp;
    TemplateElements children;
    Expression exp;
    int pushedCtxCount = 0;
}
{
    start = <UNIFIED_CALL>
    exp = Expression()
    {
        if (exp instanceof Identifier || (exp instanceof Dot && ((Dot) exp).onlyHasIdentifiers())) {
            startTagNameExp = exp;
        } else {
            startTagNameExp = null;
        }
    }
    [<TERMINATING_WHITESPACE>]
    (
        LOOKAHEAD(<ID><EQUALS>)
        namedArgs = NamedArgs()
        |
        positionalArgs = PositionalArgs()
    )
    [
        <SEMICOLON>{bodyParameters = new ArrayList(4); }
        [
            [<TERMINATING_WHITESPACE>] t = <ID> { bodyParameters.add(t.image); }
            (
                [<TERMINATING_WHITESPACE>] <COMMA>
                [<TERMINATING_WHITESPACE>] t = <ID>{bodyParameters.add(t.image); }
            )*
        ]
    ]
    (
        end = <EMPTY_DIRECTIVE_END> { children = TemplateElements.EMPTY; }
        |
        (
            <DIRECTIVE_END> {
                if (bodyParameters != null && iteratorBlockContexts != null && !iteratorBlockContexts.isEmpty()) {
                    // It's possible that we shadow a #list/#items loop variable, in which case that must be noted.
                    int ctxsLen = iteratorBlockContexts.size();
                    int bodyParsLen = bodyParameters.size();
	                for (int bodyParIdx = 0; bodyParIdx < bodyParsLen; bodyParIdx++) {
                        String bodyParName = (String) bodyParameters.get(bodyParIdx);
                        walkCtxSack: for (int ctxIdx = ctxsLen - 1; ctxIdx >= 0; ctxIdx--) {
                            ParserIteratorBlockContext ctx
                                    = (ParserIteratorBlockContext) iteratorBlockContexts.get(ctxIdx);
                            if (ctx.loopVarName != null && ctx.loopVarName.equals(bodyParName)) {
                                // If it wasn't already shadowed, shadow it:
                                if (ctx.kind != ITERATOR_BLOCK_KIND_USER_DIRECTIVE) {
                                    ParserIteratorBlockContext shadowingCtx = pushIteratorBlockContext();
                                    shadowingCtx.loopVarName = bodyParName;
                                    shadowingCtx.kind = ITERATOR_BLOCK_KIND_USER_DIRECTIVE;
                                    pushedCtxCount++;
                                }
                                break walkCtxSack;
                            }
                        }
                   }
                }
            }
            children = MixedContentElements()
            end = <UNIFIED_CALL_END>
            {
                for (int i = 0; i < pushedCtxCount; i++) {
                    popIteratorBlockContext();
                }
            
                String endTagName = end.image.substring(3, end.image.length() - 1).trim();
                if (endTagName.length() > 0) {
                    if (startTagNameExp == null) {
                        throw new ParseException("Expecting </@>", template, end);
                    } else {
                        String startTagName = startTagNameExp.getCanonicalForm();
                        if (!endTagName.equals(startTagName)) {
                            throw new ParseException("Expecting </@> or </@" + startTagName + ">", template, end);
                        }
                    }
                }
            }
        )
    )
    {
        TemplateElement result = (positionalArgs != null)
        		? new UnifiedCall(exp, positionalArgs, children, bodyParameters)
	            : new UnifiedCall(exp, namedArgs, children, bodyParameters);
        result.setLocation(template, start, end);
        return result;
    }
}

TemplateElement Call() :
{
    Token start, end, id;
    HashMap namedArgs = null;
    ArrayList positionalArgs = null;
    String macroName= null;
}
{
    start = <CALL>
    id = <ID> { macroName = id.image; }
    (
        LOOKAHEAD(<ID><EQUALS>)
        namedArgs = NamedArgs()
        |
        (
            [
            LOOKAHEAD(<OPEN_PAREN>)
                <OPEN_PAREN>
            ]
            positionalArgs = PositionalArgs()
            [<CLOSE_PAREN>]
        )
    )
    end = LooseDirectiveEnd()
    {
        UnifiedCall result = null;
        if (positionalArgs != null) {
            result = new UnifiedCall(new Identifier(macroName), positionalArgs, TemplateElements.EMPTY, null);
        } else {
            result = new UnifiedCall(new Identifier(macroName), namedArgs, TemplateElements.EMPTY, null);
        }
        result.legacySyntax = true;
        result.setLocation(template, start, end);
        return result;
    }
}

HashMap NamedArgs() :
{
    HashMap result = new HashMap();
    Token t;
    Expression exp;
}
{
    (
        t = <ID>
        <EQUALS>
        {
            token_source.SwitchTo(token_source.NAMED_PARAMETER_EXPRESSION);
            token_source.inInvocation = true;
        }             
        exp = Expression()
        {
            result.put(t.image, exp);
        }
    )+
    {
        token_source.inInvocation = false;
        return result;
    }
}

ArrayList PositionalArgs() :
{
    ArrayList result = new ArrayList();
    Expression arg;
}
{
    [
        arg = Expression() { result.add(arg); }
        (
            [<COMMA>]
            arg = Expression() { result.add(arg); }
        )*
    ]
    {
        return result;
    }
}


Comment Comment() :
{
    Token start, end;
    StringBuilder buf = new StringBuilder();
}
{
    (
        start = <COMMENT>
        |
        start = <TERSE_COMMENT>
    )
    end = UnparsedContent(start, buf)
    {
        Comment result = new Comment(buf.toString());
        result.setLocation(template, start, end);
        return result;
    }
}

TextBlock NoParse() :
{
    Token start, end;
    StringBuilder buf = new StringBuilder();
}
{
    start = <NOPARSE>
    end = UnparsedContent(start, buf)
    {
        TextBlock result = new TextBlock(buf.toString(), true);
        result.setLocation(template, start, end);
        return result;
    }
}

TransformBlock Transform() :
{
    Token start, end, argName;
    Expression exp, argExp;
    TemplateElements children = null;
    HashMap args = null;
}
{
    start = <TRANSFORM>
    exp = Expression()
    [<SEMICOLON>]
    (
        argName = <ID>
        <EQUALS>
        argExp = Expression()
        {
            if (args == null) args = new HashMap();
            args.put(argName.image, argExp);
        }
    )*
    (
        end = <EMPTY_DIRECTIVE_END>
        |
        (
            <DIRECTIVE_END>
            children = MixedContentElements()
            end = <END_TRANSFORM>
        )
    )
    {
        TransformBlock result = new TransformBlock(exp, args, children);
        result.setLocation(template, start, end);
        return result;
    }
}

SwitchBlock Switch() :
{
    SwitchBlock switchBlock;
    Case caseIns;
    Expression switchExp;
    Token start, end;
    boolean defaultFound = false;
}
{
    start = <SWITCH>
    switchExp = Expression()
    <DIRECTIVE_END>
    {
        breakableDirectiveNesting++;
        switchBlock = new SwitchBlock(switchExp);
    }
    (
        LOOKAHEAD(2)
        caseIns = Case()
        {
            if (caseIns.condition == null) {
                if (defaultFound) {
                    throw new ParseException(
                    "You can only have one default case in a switch statement", template, start);
                }
                defaultFound = true;
            }
            switchBlock.addCase(caseIns);
        }
    )*
    [<STATIC_TEXT_WS>]
    end = <END_SWITCH>
    {
        breakableDirectiveNesting--;
        switchBlock.setLocation(template, start, end);
        return switchBlock;
    }
}

Case Case() :
{
    Expression exp;
    TemplateElements children;
    Token start;
}
{
    [<STATIC_TEXT_WS>]
    (
        start = <CASE> exp = Expression() <DIRECTIVE_END>
        |
        start = <DEFAUL> { exp = null; }
    )
    children = MixedContentElements()
    {
        Case result = new Case(exp, children);
        result.setLocation(template, start, start, children);
        return result;
    }
}

EscapeBlock Escape() :
{
    Token variable, start, end;
    Expression escapeExpr;
    TemplateElements children;
}
{
    start = <ESCAPE>
    {
        if (outputFormat instanceof MarkupOutputFormat && autoEscaping) {
            throw new ParseException(
                    "Using the \"escape\" directive (legacy escaping) is not allowed when auto-escaping is on with "
                    + "a markup output format (" + outputFormat.getName()
                    + "), to avoid confusion and double-escaping mistakes.",
                    template, start);
        }
    }
    variable = <ID>
    <AS>
    escapeExpr = Expression()
    <DIRECTIVE_END>
    {
        EscapeBlock result = new EscapeBlock(variable.image, escapeExpr, escapedExpression(escapeExpr));
        escapes.addFirst(result);
    }
    children = MixedContentElements()
    {
        result.setContent(children);
        escapes.removeFirst();
    }
    end = <END_ESCAPE>
    {
        result.setLocation(template, start, end);
        return result;
    }
}

NoEscapeBlock NoEscape() :
{
    Token start, end;
    TemplateElements children;
}
{
    start = <NOESCAPE>
    {
        if (escapes.isEmpty()) {
            throw new ParseException("#noescape with no matching #escape encountered.", template, start);
        }
        Object escape = escapes.removeFirst();
    }
    children = MixedContentElements()
    end = <END_NOESCAPE>
    {
        escapes.addFirst(escape);
        NoEscapeBlock result = new NoEscapeBlock(children);
        result.setLocation(template, start, end);
        return result;
    }
}

OutputFormatBlock OutputFormat() :
{
    Token start, end;
    Expression paramExp;
    TemplateElements children;
    OutputFormat lastOutputFormat;
}
{
    start = <OUTPUTFORMAT>
    paramExp = Expression()
    <DIRECTIVE_END>
    {
        if (!paramExp.isLiteral()) {
            throw new ParseException(
                    "Parameter expression must be parse-time evaluable (constant): "
                    + paramExp.getCanonicalForm(),
                    paramExp);
        }
    
        TemplateModel paramTM;
        try {
            paramTM = paramExp.eval(null);
        } catch (Exception e) {
            throw new ParseException(
                    "Could not evaluate expression (on parse-time): " + paramExp.getCanonicalForm()
                    + "\nUnderlying cause: " +  e,
                    paramExp, e);
        }
        String paramStr;
        if (paramTM instanceof TemplateScalarModel) {
            try {
                paramStr = ((TemplateScalarModel) paramTM).getAsString();
            } catch (TemplateModelException e) {
	            throw new ParseException(
	                    "Could not evaluate expression (on parse-time): " + paramExp.getCanonicalForm()
	                    + "\nUnderlying cause: " +  e,
	                    paramExp, e);
            }
        } else {
            throw new ParseException(
                    "Parameter must be a string, but was: " + ClassUtil.getFTLTypeDescription(paramTM),
                    paramExp);
        }
        
        lastOutputFormat = outputFormat;
        try { 
            if (paramStr.startsWith("{")) {
                if (!paramStr.endsWith("}")) {
                    throw new ParseException("Output format name that starts with '{' must end with '}': " + paramStr,
                            template, start);
                }
                OutputFormat innerOutputFormat = template.getConfiguration().getOutputFormat(
                        paramStr.substring(1, paramStr.length() - 1));
                if (!(innerOutputFormat instanceof MarkupOutputFormat)) {
                    throw new ParseException(
                            "The output format inside the {...} must be a markup format, but was: "
                            + innerOutputFormat,
                            template, start);
                }
                if (!(outputFormat instanceof MarkupOutputFormat)) {
                    throw new ParseException(
                            "The current output format must be a markup format when using {...}, but was: "
                            + outputFormat,
                            template, start);
                }
                outputFormat = new CombinedMarkupOutputFormat(
                        (MarkupOutputFormat) outputFormat, (MarkupOutputFormat) innerOutputFormat);
            } else {
                outputFormat = template.getConfiguration().getOutputFormat(paramStr);
            }
            recalculateAutoEscapingField();
        } catch (IllegalArgumentException e) {
            throw new ParseException("Invalid format name: " + e.getMessage(), template, start, e.getCause());
        } catch (UnregisteredOutputFormatException e) {
            throw new ParseException(e.getMessage(), template, start, e.getCause());
        }
    }
    children = MixedContentElements()
    end = <END_OUTPUTFORMAT>
    {
        OutputFormatBlock result = new OutputFormatBlock(children, paramExp);
        result.setLocation(template, start, end);
        
        outputFormat = lastOutputFormat;
        recalculateAutoEscapingField();         
        return result;
    }
}

AutoEscBlock AutoEsc() :
{
    Token start, end;
    TemplateElements children;
    int lastAutoEscapingPolicy;
}
{
    start = <AUTOESC>
    {
        checkCurrentOutputFormatCanEscape(start);
        lastAutoEscapingPolicy = autoEscapingPolicy;
        autoEscapingPolicy = Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY;
        recalculateAutoEscapingField();
    }
    children = MixedContentElements()
    end = <END_AUTOESC>
    {
        AutoEscBlock result = new AutoEscBlock(children);
        result.setLocation(template, start, end);
        
        autoEscapingPolicy = lastAutoEscapingPolicy; 
        recalculateAutoEscapingField();
        return result;
    }
}

NoAutoEscBlock NoAutoEsc() :
{
    Token start, end;
    TemplateElements children;
    int lastAutoEscapingPolicy;
}
{
    start = <NOAUTOESC>
    {
        lastAutoEscapingPolicy = autoEscapingPolicy;
        autoEscapingPolicy = Configuration.DISABLE_AUTO_ESCAPING_POLICY;
        recalculateAutoEscapingField();
    }
    children = MixedContentElements()
    end = <END_NOAUTOESC>
    {
        NoAutoEscBlock result = new NoAutoEscBlock(children);
        result.setLocation(template, start, end);
        
        autoEscapingPolicy = lastAutoEscapingPolicy;
        recalculateAutoEscapingField(); 
        return result;
    }
}

/**
 * Production to terminate potentially empty elements. Either a ">" or "/>"
 */
Token LooseDirectiveEnd() :
{
    Token t;
}
{
    (
        t = <DIRECTIVE_END>
        |
        t = <EMPTY_DIRECTIVE_END>
    )
    {
        return t;
    }
}

PropertySetting Setting() :
{
    Token start, end, key;
    Expression value;
}
{
    start = <SETTING>
    key = <ID>
    <EQUALS>
    value = Expression()
    end = LooseDirectiveEnd()
    {
        token_source.checkNamingConvention(key);
        PropertySetting result = new PropertySetting(key, token_source, value, template.getConfiguration());
        result.setLocation(template, start, end);
        return result;
    }
}

/**
 * A production for FreeMarker directives.
 */
TemplateElement FreemarkerDirective() :
{
    TemplateElement tp;
}
{
    // Note that this doesn't include elements like "else", "recover", etc., because those indicate the end
    // of the MixedContentElements of "if", "attempt", etc.
    (
        tp = If()
        |
        tp = List()
        |
        tp = ForEach()
        |
        tp = Assign()
        |
        tp = Include()
        |
        tp = Import()
        |
        tp = Macro()
        |
        tp = Compress()
        |
        tp = UnifiedMacroTransform()
        |
        tp = Items()
        |
        tp = Sep()
        |
        tp = Call()
        |
        tp = Comment()
        |
        tp = NoParse()
        |
        tp = Transform()
        |
        tp = Switch()
        |
        tp = Setting()
        |
        tp = Break()
        |
        tp = Return()
        |
        tp = Stop()
        |
        tp = Flush()
        |
        tp = Trim()
        |
        tp = Nested()
        |
        tp = Escape()
        |
        tp = NoEscape()
        |
        tp = Visit()
        |
        tp = Recurse()
        |
        tp = FallBack()
        |
        tp = Attempt()
        |
        tp = OutputFormat()
        |
        tp = AutoEsc()
        |
        tp = NoAutoEsc()
    )
    {
        return tp;
    }
}

/**
 * Production for a block of raw text
 * i.e. text that contains no
 * FreeMarker directives.
 */
TextBlock PCData() :
{
    StringBuilder buf = new StringBuilder();
    Token t = null, start = null, prevToken = null;
}
{
    (
        (
            t = <STATIC_TEXT_WS>
            |
            t = <STATIC_TEXT_NON_WS>
            |
            t = <STATIC_TEXT_FALSE_ALARM>
        )
        {
            buf.append(t.image);
            if (start == null) start = t;
            if (prevToken != null) prevToken.next = null;
            prevToken = t;
        }
    )+
    {
        if (stripText && mixedContentNesting == 1) return null;

        TextBlock result = new TextBlock(buf.toString(), false);
        result.setLocation(template, start, t);
        return result;
    }
}

/**
 * Production for dealing with unparsed content,
 * i.e. what is inside a comment or noparse tag.
 * It returns the ending token. The content
 * of the tag is put in buf.
 */
Token UnparsedContent(Token start, StringBuilder buf) :
{
    Token t;
}
{
    (
        (t = <KEEP_GOING> | t = <MAYBE_END> | t = <TERSE_COMMENT_END> | t = <LONE_LESS_THAN_OR_DASH>)
        {
            buf.append(t.image);
        }
    )+
    {
        buf.setLength(buf.length() - t.image.length());
        if (!t.image.endsWith(";")
                && _TemplateAPI.getTemplateLanguageVersionAsInt(template) >= _TemplateAPI.VERSION_INT_2_3_21) {
            throw new ParseException("Unclosed \"" + start.image + "\"", template, start);
        }
        return t;
    }
}

TemplateElements MixedContentElements() :
{
    TemplateElement[] childBuffer = null;
    int childCount = 0;
    TemplateElement elem;
    mixedContentNesting++;
}
{
    (
        LOOKAHEAD(1) // Just tells javacc that we know what we're doing.
        (
            elem = PCData()
            |
            elem = StringOutput()
            |
            elem = NumericalOutput()
            |
            elem = FreemarkerDirective()
        )
        {
            // Note: elem == null when it's was top-level PCData removed by stripText
            if (elem != null) {
	            childCount++;
	            if (childBuffer == null) {
	                childBuffer = new TemplateElement[16]; 
	            } else if (childBuffer.length < childCount) {
	                TemplateElement[] newChildBuffer = new TemplateElement[childCount * 2];
	                for (int i = 0; i < childBuffer.length; i++) {
	                    newChildBuffer[i] = childBuffer[i];
	                }
	                childBuffer = newChildBuffer;
	            }
	            childBuffer[childCount - 1] = elem;
            }
        }
    )*
    {
        mixedContentNesting--;
        return childBuffer != null ? new TemplateElements(childBuffer, childCount) : TemplateElements.EMPTY;
    }
}

/**
 * Not used anymore; kept for backward compatibility.
 *
 * @deprecated Use {@link #MixedContentElements} instead.
 */
MixedContent MixedContent() :
{
    MixedContent mixedContent = new MixedContent();
    TemplateElement elem, begin = null;
    mixedContentNesting++;
}
{
    (
        LOOKAHEAD(1) // Just tells javacc that we know what we're doing.
        (
            elem = PCData()
            |
            elem = StringOutput()
            |
            elem = NumericalOutput()
            |
            elem = FreemarkerDirective()
        )
        {
            if (begin == null) {
                begin = elem;
            }
            mixedContent.addElement(elem);
        }
    )+
    {
        mixedContentNesting--;
        mixedContent.setLocation(template, begin, elem);
        return mixedContent;
    }
}

/**
 * Not used anymore; kept for backward compatibility.
 *
 * <p>A production for a block of optional content.
 * Returns an empty Text block if there is no
 * content.
 *
 * @deprecated Use {@link #MixedContentElements} instead.
 */
TemplateElement OptionalBlock() :
{
    TemplateElement tp = null;
}
{
    [
        LOOKAHEAD(1) // has no effect but to get rid of a spurious warning.
        tp = MixedContent()
    ]
    {
        return tp != null ? tp : new TextBlock(CollectionUtils.EMPTY_CHAR_ARRAY, false);
    }
}

/**
 * A production freemarker text that may contain
 * ${...} and #{...} but no directives.
 */
TemplateElement FreeMarkerText() :
{
    MixedContent nodes = new MixedContent();
    TemplateElement elem, begin = null;
}
{
    (
        (
            elem = PCData()
            |
            elem = StringOutput()
            |
            elem = NumericalOutput()
        )
        {
            if (begin == null) {
            	begin = elem;
            }
            nodes.addChild(elem);
        }
    )+
    {
        nodes.setLocation(template, begin, elem);
        return nodes;
    }
}

void HeaderElement() :
{
    Token key;
    Expression exp = null;
    Token autoEscRequester = null;
}
{
    [<STATIC_TEXT_WS>]
    (
        <TRIVIAL_FTL_HEADER>
        |
        (
            <FTL_HEADER>
            (
                key = <ID>
                <EQUALS>
                exp = Expression()
                {
                    token_source.checkNamingConvention(key);
                
                    String ks = key.image;
                    TemplateModel value = null;
                    try {
                        value = exp.eval(null);
                    } catch (Exception e) {
                        throw new ParseException(
                        		"Could not evaluate expression (on parse-time): " + exp.getCanonicalForm()
                        		+ " \nUnderlying cause: " +  e,
                                exp, e);
                    }
                    String vs = null;
                    if (value instanceof TemplateScalarModel) {
                        try {
                            vs = ((TemplateScalarModel) exp).getAsString();
                        } catch (TemplateModelException tme) {}
                    }
                    if (template != null) {
                        if (ks.equalsIgnoreCase("encoding")) {
                            if (vs == null) {
                                throw new ParseException("Expected a string constant for \"" + ks + "\".", exp);
                            }
                            String encoding = template.getEncoding();
                            if (encoding != null && !encoding.equalsIgnoreCase(vs)) {
                                throw new Template.WrongEncodingException(vs, encoding);
                            }
                        } else if (ks.equalsIgnoreCase("STRIP_WHITESPACE") || ks.equals("stripWhitespace")) {
                            this.stripWhitespace = getBoolean(exp, true);
                        } else if (ks.equalsIgnoreCase("STRIP_TEXT") || ks.equals("stripText")) {
                            this.stripText = getBoolean(exp, true);
                        } else if (ks.equalsIgnoreCase("STRICT_SYNTAX") || ks.equals("strictSyntax")) {
                            this.token_source.strictEscapeSyntax = getBoolean(exp, true);
                        } else if (ks.equalsIgnoreCase("auto_esc") || ks.equals("autoEsc")) {
                            if (getBoolean(exp, false)) {
                                autoEscRequester = key;
                                autoEscapingPolicy = Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY;
                            } else {
                                autoEscapingPolicy = Configuration.DISABLE_AUTO_ESCAPING_POLICY;
                            }
                            recalculateAutoEscapingField();
                            _TemplateAPI.setAutoEscaping(template, autoEscaping);
                        } else if (ks.equalsIgnoreCase("output_format") || ks.equals("outputFormat")) {
                            if (vs == null) {
                                throw new ParseException("Expected a string constant for \"" + ks + "\".", exp);
                            }
                            try {
                                outputFormat = template.getConfiguration().getOutputFormat(vs);
					        } catch (IllegalArgumentException e) {
					            throw new ParseException("Invalid format name: " + e.getMessage(), exp, e.getCause());
					        } catch (UnregisteredOutputFormatException e) {
					            throw new ParseException(e.getMessage(), exp, e.getCause());
					        }
                            recalculateAutoEscapingField();                                
                            _TemplateAPI.setOutputFormat(template, outputFormat);
                            _TemplateAPI.setAutoEscaping(template, autoEscaping);
                        } else if (ks.equalsIgnoreCase("ns_prefixes") || ks.equals("nsPrefixes")) {
                            if (!(value instanceof TemplateHashModelEx)) {
                                throw new ParseException("Expecting a hash of prefixes to namespace URI's.", exp);
                            }
                            TemplateHashModelEx prefixMap = (TemplateHashModelEx) value;
                            try {
                                TemplateCollectionModel keys = prefixMap.keys();
                                for (TemplateModelIterator it = keys.iterator(); it.hasNext();) {
                                    String prefix = ((TemplateScalarModel) it.next()).getAsString();
                                    TemplateModel valueModel = prefixMap.get(prefix);
                                    if (!(valueModel instanceof TemplateScalarModel)) {
                                        throw new ParseException("Non-string value in prefix to namespace hash.", exp);
                                    }
                                    String nsURI = ((TemplateScalarModel) valueModel).getAsString();
                                    try {
                                        template.addPrefixNSMapping(prefix, nsURI);
                                    } catch (IllegalArgumentException iae) {
                                        throw new ParseException(iae.getMessage(), exp);
                                    }
                                }
                            } catch (TemplateModelException tme) {
                            }
                        } else if (ks.equalsIgnoreCase("attributes")) {
                            if (!(value instanceof TemplateHashModelEx)) {
                                throw new ParseException("Expecting a hash of attribute names to values.", exp);
                            }
                            TemplateHashModelEx attributeMap = (TemplateHashModelEx) value;
                            try {
                                TemplateCollectionModel keys = attributeMap.keys();
                                for (TemplateModelIterator it = keys.iterator(); it.hasNext();) {
                                        String attName = ((TemplateScalarModel) it.next()).getAsString();
                                        Object attValue = DeepUnwrap.unwrap(attributeMap.get(attName));
                                        template.setCustomAttribute(attName, attValue);
                                }
                            } catch (TemplateModelException tme) {
                            }
                        } else {
                            String correctName;
	                        if (ks.equals("charset")) {
	                            correctName = "encoding";
	                        } else if (ks.equals("xmlns")) {
	                            // [2.4] If camel case will be the default, update this
                                correctName
                                        = token_source.namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION
                                                ? "nsPrefixes" : "ns_prefixes";
                            } else if (ks.equals("auto_escape") || ks.equals("auto_escaping") || ks.equals("autoesc")) {
                                correctName = "auto_esc";
                            } else if (ks.equals("autoEscape") || ks.equals("autoEscaping")) {
                                correctName = "autoEsc";
	                        } else {
                                correctName = null;
	                        }
                            throw new ParseException(
                                    "Unknown FTL header parameter: " + key.image
                                    + (correctName == null ? "" : ". You may meant: " + correctName),
                                    template, key);
                        }
                    }
                }
            )*
        )
        {
            if (autoEscRequester != null) {
                checkCurrentOutputFormatCanEscape(autoEscRequester);
            }        
        }
        LooseDirectiveEnd()
    )
}

Map ParamList() :
{
    Identifier id;
    Expression exp;
    Map result = new HashMap();
}
{
    (
        id = Identifier()
        <EQUALS>
        exp = Expression() { result.put(id.toString(), exp); }
        [<COMMA>]
    )+
    {
        return result;
    }
}

/**
 * Parses the already un-escaped content of a string literal (input must not include the quotation marks).
 *
 * @return A {@link List} of {@link String}-s and {@link Interpolation}-s. 
 */
List<Object> StaticTextAndInterpolations() :
{
    Token t;
    Interpolation interpolation;
    StringBuilder staticTextCollector = null;
    ArrayList<Object> parts = new ArrayList<Object>();
}
{
    (
	    (
		    t = <STATIC_TEXT_WS>
		    |
		    t = <STATIC_TEXT_NON_WS>
		    |
		    t = <STATIC_TEXT_FALSE_ALARM>
	    )
	    {
	       String s = t.image;
	       if (s.length() != 0) {
	           if (staticTextCollector == null) {
	               staticTextCollector = new StringBuilder(t.image);
	           } else {
	               staticTextCollector.append(t.image);
	           }
	       }
	    }
	    |
	    (
	        LOOKAHEAD(<DOLLAR_INTERPOLATION_OPENING>)
		    (
		        interpolation = StringOutput()
	        )
		    |
            LOOKAHEAD(<HASH_INTERPOLATION_OPENING>)
		    (
                interpolation = NumericalOutput()
		    )
	    )
	    {
            if (staticTextCollector != null) {
                parts.add(staticTextCollector.toString());
                staticTextCollector.setLength(0);
            }
            parts.add(interpolation);
	    }
    )*
    {
        if (staticTextCollector != null && staticTextCollector.length() != 0) {
            parts.add(staticTextCollector.toString());
        }
        parts.trimToSize();
        return parts;
    }
}

/**
 * Root production to be used when parsing
 * an entire file.
 */
TemplateElement Root() :
{
    TemplateElements children;
}
{
    [
        LOOKAHEAD([<STATIC_TEXT_WS>](<TRIVIAL_FTL_HEADER>|<FTL_HEADER>))
        HeaderElement()
    ]
    children = MixedContentElements()
    <EOF>
    {
        TemplateElement root = children.asSingleElement(); 
        root.setFieldsForRootElement();
        root = root.postParseCleanup(stripWhitespace);
        // The cleanup result is possibly an element from deeper:
        root.setFieldsForRootElement();
        return root;
    }
}