| // ## Token types |
| |
| // The assignment of fine-grained, information-carrying type objects |
| // allows the tokenizer to store the information it has about a |
| // token in a way that is very cheap for the parser to look up. |
| |
| // All token type variables start with an underscore, to make them |
| // easy to recognize. |
| |
| // The `beforeExpr` property is used to disambiguate between regular |
| // expressions and divisions. It is set on all token types that can |
| // be followed by an expression (thus, a slash after them would be a |
| // regular expression). |
| // |
| // The `startsExpr` property is used to check if the token ends a |
| // `yield` expression. It is set on all token types that either can |
| // directly start an expression (like a quotation mark) or can |
| // continue an expression (like the body of a string). |
| // |
| // `isLoop` marks a keyword as starting a loop, which is important |
| // to know when parsing a label, in order to allow or disallow |
| // continue jumps to that label. |
| |
| export class TokenType { |
| constructor(label, conf = {}) { |
| this.label = label |
| this.keyword = conf.keyword |
| this.beforeExpr = !!conf.beforeExpr |
| this.startsExpr = !!conf.startsExpr |
| this.isLoop = !!conf.isLoop |
| this.isAssign = !!conf.isAssign |
| this.prefix = !!conf.prefix |
| this.postfix = !!conf.postfix |
| this.binop = conf.binop || null |
| this.updateContext = null |
| } |
| } |
| |
| function binop(name, prec) { |
| return new TokenType(name, {beforeExpr: true, binop: prec}) |
| } |
| const beforeExpr = {beforeExpr: true}, startsExpr = {startsExpr: true} |
| |
| // Map keyword names to token types. |
| |
| export const keywords = {} |
| |
| // Succinct definitions of keyword token types |
| function kw(name, options = {}) { |
| options.keyword = name |
| return keywords[name] = new TokenType(name, options) |
| } |
| |
| export const types = { |
| num: new TokenType("num", startsExpr), |
| regexp: new TokenType("regexp", startsExpr), |
| string: new TokenType("string", startsExpr), |
| name: new TokenType("name", startsExpr), |
| eof: new TokenType("eof"), |
| |
| // Punctuation token types. |
| bracketL: new TokenType("[", {beforeExpr: true, startsExpr: true}), |
| bracketR: new TokenType("]"), |
| braceL: new TokenType("{", {beforeExpr: true, startsExpr: true}), |
| braceR: new TokenType("}"), |
| parenL: new TokenType("(", {beforeExpr: true, startsExpr: true}), |
| parenR: new TokenType(")"), |
| comma: new TokenType(",", beforeExpr), |
| semi: new TokenType(";", beforeExpr), |
| colon: new TokenType(":", beforeExpr), |
| dot: new TokenType("."), |
| question: new TokenType("?", beforeExpr), |
| arrow: new TokenType("=>", beforeExpr), |
| template: new TokenType("template"), |
| ellipsis: new TokenType("...", beforeExpr), |
| backQuote: new TokenType("`", startsExpr), |
| dollarBraceL: new TokenType("${", {beforeExpr: true, startsExpr: true}), |
| |
| // Operators. These carry several kinds of properties to help the |
| // parser use them properly (the presence of these properties is |
| // what categorizes them as operators). |
| // |
| // `binop`, when present, specifies that this operator is a binary |
| // operator, and will refer to its precedence. |
| // |
| // `prefix` and `postfix` mark the operator as a prefix or postfix |
| // unary operator. |
| // |
| // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as |
| // binary operators with a very low precedence, that should result |
| // in AssignmentExpression nodes. |
| |
| eq: new TokenType("=", {beforeExpr: true, isAssign: true}), |
| assign: new TokenType("_=", {beforeExpr: true, isAssign: true}), |
| incDec: new TokenType("++/--", {prefix: true, postfix: true, startsExpr: true}), |
| prefix: new TokenType("prefix", {beforeExpr: true, prefix: true, startsExpr: true}), |
| logicalOR: binop("||", 1), |
| logicalAND: binop("&&", 2), |
| bitwiseOR: binop("|", 3), |
| bitwiseXOR: binop("^", 4), |
| bitwiseAND: binop("&", 5), |
| equality: binop("==/!=", 6), |
| relational: binop("</>", 7), |
| bitShift: binop("<</>>", 8), |
| plusMin: new TokenType("+/-", {beforeExpr: true, binop: 9, prefix: true, startsExpr: true}), |
| modulo: binop("%", 10), |
| star: binop("*", 10), |
| slash: binop("/", 10), |
| starstar: new TokenType("**", {beforeExpr: true}), |
| |
| // Keyword token types. |
| _break: kw("break"), |
| _case: kw("case", beforeExpr), |
| _catch: kw("catch"), |
| _continue: kw("continue"), |
| _debugger: kw("debugger"), |
| _default: kw("default", beforeExpr), |
| _do: kw("do", {isLoop: true, beforeExpr: true}), |
| _else: kw("else", beforeExpr), |
| _finally: kw("finally"), |
| _for: kw("for", {isLoop: true}), |
| _function: kw("function", startsExpr), |
| _if: kw("if"), |
| _return: kw("return", beforeExpr), |
| _switch: kw("switch"), |
| _throw: kw("throw", beforeExpr), |
| _try: kw("try"), |
| _var: kw("var"), |
| _const: kw("const"), |
| _while: kw("while", {isLoop: true}), |
| _with: kw("with"), |
| _new: kw("new", {beforeExpr: true, startsExpr: true}), |
| _this: kw("this", startsExpr), |
| _super: kw("super", startsExpr), |
| _class: kw("class"), |
| _extends: kw("extends", beforeExpr), |
| _export: kw("export"), |
| _import: kw("import"), |
| _null: kw("null", startsExpr), |
| _true: kw("true", startsExpr), |
| _false: kw("false", startsExpr), |
| _in: kw("in", {beforeExpr: true, binop: 7}), |
| _instanceof: kw("instanceof", {beforeExpr: true, binop: 7}), |
| _typeof: kw("typeof", {beforeExpr: true, prefix: true, startsExpr: true}), |
| _void: kw("void", {beforeExpr: true, prefix: true, startsExpr: true}), |
| _delete: kw("delete", {beforeExpr: true, prefix: true, startsExpr: true}) |
| } |