| 'use strict'; |
| |
| // token types (note: value shouldn't intersect with used char codes) |
| var WHITESPACE = 1; |
| var IDENTIFIER = 2; |
| var NUMBER = 3; |
| var STRING = 4; |
| var COMMENT = 5; |
| var PUNCTUATOR = 6; |
| var CDO = 7; |
| var CDC = 8; |
| var ATKEYWORD = 14; |
| var FUNCTION = 15; |
| var URL = 16; |
| var RAW = 17; |
| |
| var TAB = 9; |
| var N = 10; |
| var F = 12; |
| var R = 13; |
| var SPACE = 32; |
| |
| var TYPE = { |
| WhiteSpace: WHITESPACE, |
| Identifier: IDENTIFIER, |
| Number: NUMBER, |
| String: STRING, |
| Comment: COMMENT, |
| Punctuator: PUNCTUATOR, |
| CDO: CDO, |
| CDC: CDC, |
| AtKeyword: ATKEYWORD, |
| Function: FUNCTION, |
| Url: URL, |
| Raw: RAW, |
| |
| ExclamationMark: 33, // ! |
| QuotationMark: 34, // " |
| NumberSign: 35, // # |
| DollarSign: 36, // $ |
| PercentSign: 37, // % |
| Ampersand: 38, // & |
| Apostrophe: 39, // ' |
| LeftParenthesis: 40, // ( |
| RightParenthesis: 41, // ) |
| Asterisk: 42, // * |
| PlusSign: 43, // + |
| Comma: 44, // , |
| HyphenMinus: 45, // - |
| FullStop: 46, // . |
| Solidus: 47, // / |
| Colon: 58, // : |
| Semicolon: 59, // ; |
| LessThanSign: 60, // < |
| EqualsSign: 61, // = |
| GreaterThanSign: 62, // > |
| QuestionMark: 63, // ? |
| CommercialAt: 64, // @ |
| LeftSquareBracket: 91, // [ |
| Backslash: 92, // \ |
| RightSquareBracket: 93, // ] |
| CircumflexAccent: 94, // ^ |
| LowLine: 95, // _ |
| GraveAccent: 96, // ` |
| LeftCurlyBracket: 123, // { |
| VerticalLine: 124, // | |
| RightCurlyBracket: 125, // } |
| Tilde: 126 // ~ |
| }; |
| |
| var NAME = Object.keys(TYPE).reduce(function(result, key) { |
| result[TYPE[key]] = key; |
| return result; |
| }, {}); |
| |
| // https://drafts.csswg.org/css-syntax/#tokenizer-definitions |
| // > non-ASCII code point |
| // > A code point with a value equal to or greater than U+0080 <control> |
| // > name-start code point |
| // > A letter, a non-ASCII code point, or U+005F LOW LINE (_). |
| // > name code point |
| // > A name-start code point, a digit, or U+002D HYPHEN-MINUS (-) |
| // That means only ASCII code points has a special meaning and we a maps for 0..127 codes only |
| var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported |
| var SYMBOL_TYPE = new SafeUint32Array(0x80); |
| var PUNCTUATION = new SafeUint32Array(0x80); |
| var STOP_URL_RAW = new SafeUint32Array(0x80); |
| |
| for (var i = 0; i < SYMBOL_TYPE.length; i++) { |
| SYMBOL_TYPE[i] = IDENTIFIER; |
| } |
| |
| // fill categories |
| [ |
| TYPE.ExclamationMark, // ! |
| TYPE.QuotationMark, // " |
| TYPE.NumberSign, // # |
| TYPE.DollarSign, // $ |
| TYPE.PercentSign, // % |
| TYPE.Ampersand, // & |
| TYPE.Apostrophe, // ' |
| TYPE.LeftParenthesis, // ( |
| TYPE.RightParenthesis, // ) |
| TYPE.Asterisk, // * |
| TYPE.PlusSign, // + |
| TYPE.Comma, // , |
| TYPE.HyphenMinus, // - |
| TYPE.FullStop, // . |
| TYPE.Solidus, // / |
| TYPE.Colon, // : |
| TYPE.Semicolon, // ; |
| TYPE.LessThanSign, // < |
| TYPE.EqualsSign, // = |
| TYPE.GreaterThanSign, // > |
| TYPE.QuestionMark, // ? |
| TYPE.CommercialAt, // @ |
| TYPE.LeftSquareBracket, // [ |
| // TYPE.Backslash, // \ |
| TYPE.RightSquareBracket, // ] |
| TYPE.CircumflexAccent, // ^ |
| // TYPE.LowLine, // _ |
| TYPE.GraveAccent, // ` |
| TYPE.LeftCurlyBracket, // { |
| TYPE.VerticalLine, // | |
| TYPE.RightCurlyBracket, // } |
| TYPE.Tilde // ~ |
| ].forEach(function(key) { |
| SYMBOL_TYPE[Number(key)] = PUNCTUATOR; |
| PUNCTUATION[Number(key)] = PUNCTUATOR; |
| }); |
| |
| for (var i = 48; i <= 57; i++) { |
| SYMBOL_TYPE[i] = NUMBER; |
| } |
| |
| SYMBOL_TYPE[SPACE] = WHITESPACE; |
| SYMBOL_TYPE[TAB] = WHITESPACE; |
| SYMBOL_TYPE[N] = WHITESPACE; |
| SYMBOL_TYPE[R] = WHITESPACE; |
| SYMBOL_TYPE[F] = WHITESPACE; |
| |
| SYMBOL_TYPE[TYPE.Apostrophe] = STRING; |
| SYMBOL_TYPE[TYPE.QuotationMark] = STRING; |
| |
| STOP_URL_RAW[SPACE] = 1; |
| STOP_URL_RAW[TAB] = 1; |
| STOP_URL_RAW[N] = 1; |
| STOP_URL_RAW[R] = 1; |
| STOP_URL_RAW[F] = 1; |
| STOP_URL_RAW[TYPE.Apostrophe] = 1; |
| STOP_URL_RAW[TYPE.QuotationMark] = 1; |
| STOP_URL_RAW[TYPE.LeftParenthesis] = 1; |
| STOP_URL_RAW[TYPE.RightParenthesis] = 1; |
| |
| // whitespace is punctuation ... |
| PUNCTUATION[SPACE] = PUNCTUATOR; |
| PUNCTUATION[TAB] = PUNCTUATOR; |
| PUNCTUATION[N] = PUNCTUATOR; |
| PUNCTUATION[R] = PUNCTUATOR; |
| PUNCTUATION[F] = PUNCTUATOR; |
| // ... hyper minus is not |
| PUNCTUATION[TYPE.HyphenMinus] = 0; |
| |
| module.exports = { |
| TYPE: TYPE, |
| NAME: NAME, |
| |
| SYMBOL_TYPE: SYMBOL_TYPE, |
| PUNCTUATION: PUNCTUATION, |
| STOP_URL_RAW: STOP_URL_RAW |
| }; |