| module.exports = Tokenizer; |
| |
| var decodeCodePoint = require("entities/lib/decode_codepoint.js"); |
| var entityMap = require("entities/maps/entities.json"); |
| var legacyMap = require("entities/maps/legacy.json"); |
| var xmlMap = require("entities/maps/xml.json"); |
| |
| var i = 0; |
| |
| var TEXT = i++; |
| var BEFORE_TAG_NAME = i++; //after < |
| var IN_TAG_NAME = i++; |
| var IN_SELF_CLOSING_TAG = i++; |
| var BEFORE_CLOSING_TAG_NAME = i++; |
| var IN_CLOSING_TAG_NAME = i++; |
| var AFTER_CLOSING_TAG_NAME = i++; |
| |
| //attributes |
| var BEFORE_ATTRIBUTE_NAME = i++; |
| var IN_ATTRIBUTE_NAME = i++; |
| var AFTER_ATTRIBUTE_NAME = i++; |
| var BEFORE_ATTRIBUTE_VALUE = i++; |
| var IN_ATTRIBUTE_VALUE_DQ = i++; // " |
| var IN_ATTRIBUTE_VALUE_SQ = i++; // ' |
| var IN_ATTRIBUTE_VALUE_NQ = i++; |
| |
| //declarations |
| var BEFORE_DECLARATION = i++; // ! |
| var IN_DECLARATION = i++; |
| |
| //processing instructions |
| var IN_PROCESSING_INSTRUCTION = i++; // ? |
| |
| //comments |
| var BEFORE_COMMENT = i++; |
| var IN_COMMENT = i++; |
| var AFTER_COMMENT_1 = i++; |
| var AFTER_COMMENT_2 = i++; |
| |
| //cdata |
| var BEFORE_CDATA_1 = i++; // [ |
| var BEFORE_CDATA_2 = i++; // C |
| var BEFORE_CDATA_3 = i++; // D |
| var BEFORE_CDATA_4 = i++; // A |
| var BEFORE_CDATA_5 = i++; // T |
| var BEFORE_CDATA_6 = i++; // A |
| var IN_CDATA = i++; // [ |
| var AFTER_CDATA_1 = i++; // ] |
| var AFTER_CDATA_2 = i++; // ] |
| |
| //special tags |
| var BEFORE_SPECIAL = i++; //S |
| var BEFORE_SPECIAL_END = i++; //S |
| |
| var BEFORE_SCRIPT_1 = i++; //C |
| var BEFORE_SCRIPT_2 = i++; //R |
| var BEFORE_SCRIPT_3 = i++; //I |
| var BEFORE_SCRIPT_4 = i++; //P |
| var BEFORE_SCRIPT_5 = i++; //T |
| var AFTER_SCRIPT_1 = i++; //C |
| var AFTER_SCRIPT_2 = i++; //R |
| var AFTER_SCRIPT_3 = i++; //I |
| var AFTER_SCRIPT_4 = i++; //P |
| var AFTER_SCRIPT_5 = i++; //T |
| |
| var BEFORE_STYLE_1 = i++; //T |
| var BEFORE_STYLE_2 = i++; //Y |
| var BEFORE_STYLE_3 = i++; //L |
| var BEFORE_STYLE_4 = i++; //E |
| var AFTER_STYLE_1 = i++; //T |
| var AFTER_STYLE_2 = i++; //Y |
| var AFTER_STYLE_3 = i++; //L |
| var AFTER_STYLE_4 = i++; //E |
| |
| var BEFORE_ENTITY = i++; //& |
| var BEFORE_NUMERIC_ENTITY = i++; //# |
| var IN_NAMED_ENTITY = i++; |
| var IN_NUMERIC_ENTITY = i++; |
| var IN_HEX_ENTITY = i++; //X |
| |
| var j = 0; |
| |
| var SPECIAL_NONE = j++; |
| var SPECIAL_SCRIPT = j++; |
| var SPECIAL_STYLE = j++; |
| |
| function whitespace(c) { |
| return c === " " || c === "\n" || c === "\t" || c === "\f" || c === "\r"; |
| } |
| |
| function ifElseState(upper, SUCCESS, FAILURE) { |
| var lower = upper.toLowerCase(); |
| |
| if (upper === lower) { |
| return function(c) { |
| if (c === lower) { |
| this._state = SUCCESS; |
| } else { |
| this._state = FAILURE; |
| this._index--; |
| } |
| }; |
| } else { |
| return function(c) { |
| if (c === lower || c === upper) { |
| this._state = SUCCESS; |
| } else { |
| this._state = FAILURE; |
| this._index--; |
| } |
| }; |
| } |
| } |
| |
| function consumeSpecialNameChar(upper, NEXT_STATE) { |
| var lower = upper.toLowerCase(); |
| |
| return function(c) { |
| if (c === lower || c === upper) { |
| this._state = NEXT_STATE; |
| } else { |
| this._state = IN_TAG_NAME; |
| this._index--; //consume the token again |
| } |
| }; |
| } |
| |
| function Tokenizer(options, cbs) { |
| this._state = TEXT; |
| this._buffer = ""; |
| this._sectionStart = 0; |
| this._index = 0; |
| this._bufferOffset = 0; //chars removed from _buffer |
| this._baseState = TEXT; |
| this._special = SPECIAL_NONE; |
| this._cbs = cbs; |
| this._running = true; |
| this._ended = false; |
| this._xmlMode = !!(options && options.xmlMode); |
| this._decodeEntities = !!(options && options.decodeEntities); |
| } |
| |
| Tokenizer.prototype._stateText = function(c) { |
| if (c === "<") { |
| if (this._index > this._sectionStart) { |
| this._cbs.ontext(this._getSection()); |
| } |
| this._state = BEFORE_TAG_NAME; |
| this._sectionStart = this._index; |
| } else if ( |
| this._decodeEntities && |
| this._special === SPECIAL_NONE && |
| c === "&" |
| ) { |
| if (this._index > this._sectionStart) { |
| this._cbs.ontext(this._getSection()); |
| } |
| this._baseState = TEXT; |
| this._state = BEFORE_ENTITY; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeTagName = function(c) { |
| if (c === "/") { |
| this._state = BEFORE_CLOSING_TAG_NAME; |
| } else if (c === "<") { |
| this._cbs.ontext(this._getSection()); |
| this._sectionStart = this._index; |
| } else if (c === ">" || this._special !== SPECIAL_NONE || whitespace(c)) { |
| this._state = TEXT; |
| } else if (c === "!") { |
| this._state = BEFORE_DECLARATION; |
| this._sectionStart = this._index + 1; |
| } else if (c === "?") { |
| this._state = IN_PROCESSING_INSTRUCTION; |
| this._sectionStart = this._index + 1; |
| } else { |
| this._state = |
| !this._xmlMode && (c === "s" || c === "S") |
| ? BEFORE_SPECIAL |
| : IN_TAG_NAME; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInTagName = function(c) { |
| if (c === "/" || c === ">" || whitespace(c)) { |
| this._emitToken("onopentagname"); |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeCloseingTagName = function(c) { |
| if (whitespace(c)); |
| else if (c === ">") { |
| this._state = TEXT; |
| } else if (this._special !== SPECIAL_NONE) { |
| if (c === "s" || c === "S") { |
| this._state = BEFORE_SPECIAL_END; |
| } else { |
| this._state = TEXT; |
| this._index--; |
| } |
| } else { |
| this._state = IN_CLOSING_TAG_NAME; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInCloseingTagName = function(c) { |
| if (c === ">" || whitespace(c)) { |
| this._emitToken("onclosetag"); |
| this._state = AFTER_CLOSING_TAG_NAME; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateAfterCloseingTagName = function(c) { |
| //skip everything until ">" |
| if (c === ">") { |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeAttributeName = function(c) { |
| if (c === ">") { |
| this._cbs.onopentagend(); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } else if (c === "/") { |
| this._state = IN_SELF_CLOSING_TAG; |
| } else if (!whitespace(c)) { |
| this._state = IN_ATTRIBUTE_NAME; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInSelfClosingTag = function(c) { |
| if (c === ">") { |
| this._cbs.onselfclosingtag(); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } else if (!whitespace(c)) { |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInAttributeName = function(c) { |
| if (c === "=" || c === "/" || c === ">" || whitespace(c)) { |
| this._cbs.onattribname(this._getSection()); |
| this._sectionStart = -1; |
| this._state = AFTER_ATTRIBUTE_NAME; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateAfterAttributeName = function(c) { |
| if (c === "=") { |
| this._state = BEFORE_ATTRIBUTE_VALUE; |
| } else if (c === "/" || c === ">") { |
| this._cbs.onattribend(); |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| this._index--; |
| } else if (!whitespace(c)) { |
| this._cbs.onattribend(); |
| this._state = IN_ATTRIBUTE_NAME; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeAttributeValue = function(c) { |
| if (c === '"') { |
| this._state = IN_ATTRIBUTE_VALUE_DQ; |
| this._sectionStart = this._index + 1; |
| } else if (c === "'") { |
| this._state = IN_ATTRIBUTE_VALUE_SQ; |
| this._sectionStart = this._index + 1; |
| } else if (!whitespace(c)) { |
| this._state = IN_ATTRIBUTE_VALUE_NQ; |
| this._sectionStart = this._index; |
| this._index--; //reconsume token |
| } |
| }; |
| |
| Tokenizer.prototype._stateInAttributeValueDoubleQuotes = function(c) { |
| if (c === '"') { |
| this._emitToken("onattribdata"); |
| this._cbs.onattribend(); |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| } else if (this._decodeEntities && c === "&") { |
| this._emitToken("onattribdata"); |
| this._baseState = this._state; |
| this._state = BEFORE_ENTITY; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInAttributeValueSingleQuotes = function(c) { |
| if (c === "'") { |
| this._emitToken("onattribdata"); |
| this._cbs.onattribend(); |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| } else if (this._decodeEntities && c === "&") { |
| this._emitToken("onattribdata"); |
| this._baseState = this._state; |
| this._state = BEFORE_ENTITY; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInAttributeValueNoQuotes = function(c) { |
| if (whitespace(c) || c === ">") { |
| this._emitToken("onattribdata"); |
| this._cbs.onattribend(); |
| this._state = BEFORE_ATTRIBUTE_NAME; |
| this._index--; |
| } else if (this._decodeEntities && c === "&") { |
| this._emitToken("onattribdata"); |
| this._baseState = this._state; |
| this._state = BEFORE_ENTITY; |
| this._sectionStart = this._index; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeDeclaration = function(c) { |
| this._state = |
| c === "[" |
| ? BEFORE_CDATA_1 |
| : c === "-" |
| ? BEFORE_COMMENT |
| : IN_DECLARATION; |
| }; |
| |
| Tokenizer.prototype._stateInDeclaration = function(c) { |
| if (c === ">") { |
| this._cbs.ondeclaration(this._getSection()); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInProcessingInstruction = function(c) { |
| if (c === ">") { |
| this._cbs.onprocessinginstruction(this._getSection()); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeComment = function(c) { |
| if (c === "-") { |
| this._state = IN_COMMENT; |
| this._sectionStart = this._index + 1; |
| } else { |
| this._state = IN_DECLARATION; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInComment = function(c) { |
| if (c === "-") this._state = AFTER_COMMENT_1; |
| }; |
| |
| Tokenizer.prototype._stateAfterComment1 = function(c) { |
| if (c === "-") { |
| this._state = AFTER_COMMENT_2; |
| } else { |
| this._state = IN_COMMENT; |
| } |
| }; |
| |
| Tokenizer.prototype._stateAfterComment2 = function(c) { |
| if (c === ">") { |
| //remove 2 trailing chars |
| this._cbs.oncomment( |
| this._buffer.substring(this._sectionStart, this._index - 2) |
| ); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } else if (c !== "-") { |
| this._state = IN_COMMENT; |
| } |
| // else: stay in AFTER_COMMENT_2 (`--->`) |
| }; |
| |
| Tokenizer.prototype._stateBeforeCdata1 = ifElseState( |
| "C", |
| BEFORE_CDATA_2, |
| IN_DECLARATION |
| ); |
| Tokenizer.prototype._stateBeforeCdata2 = ifElseState( |
| "D", |
| BEFORE_CDATA_3, |
| IN_DECLARATION |
| ); |
| Tokenizer.prototype._stateBeforeCdata3 = ifElseState( |
| "A", |
| BEFORE_CDATA_4, |
| IN_DECLARATION |
| ); |
| Tokenizer.prototype._stateBeforeCdata4 = ifElseState( |
| "T", |
| BEFORE_CDATA_5, |
| IN_DECLARATION |
| ); |
| Tokenizer.prototype._stateBeforeCdata5 = ifElseState( |
| "A", |
| BEFORE_CDATA_6, |
| IN_DECLARATION |
| ); |
| |
| Tokenizer.prototype._stateBeforeCdata6 = function(c) { |
| if (c === "[") { |
| this._state = IN_CDATA; |
| this._sectionStart = this._index + 1; |
| } else { |
| this._state = IN_DECLARATION; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInCdata = function(c) { |
| if (c === "]") this._state = AFTER_CDATA_1; |
| }; |
| |
| Tokenizer.prototype._stateAfterCdata1 = function(c) { |
| if (c === "]") this._state = AFTER_CDATA_2; |
| else this._state = IN_CDATA; |
| }; |
| |
| Tokenizer.prototype._stateAfterCdata2 = function(c) { |
| if (c === ">") { |
| //remove 2 trailing chars |
| this._cbs.oncdata( |
| this._buffer.substring(this._sectionStart, this._index - 2) |
| ); |
| this._state = TEXT; |
| this._sectionStart = this._index + 1; |
| } else if (c !== "]") { |
| this._state = IN_CDATA; |
| } |
| //else: stay in AFTER_CDATA_2 (`]]]>`) |
| }; |
| |
| Tokenizer.prototype._stateBeforeSpecial = function(c) { |
| if (c === "c" || c === "C") { |
| this._state = BEFORE_SCRIPT_1; |
| } else if (c === "t" || c === "T") { |
| this._state = BEFORE_STYLE_1; |
| } else { |
| this._state = IN_TAG_NAME; |
| this._index--; //consume the token again |
| } |
| }; |
| |
| Tokenizer.prototype._stateBeforeSpecialEnd = function(c) { |
| if (this._special === SPECIAL_SCRIPT && (c === "c" || c === "C")) { |
| this._state = AFTER_SCRIPT_1; |
| } else if (this._special === SPECIAL_STYLE && (c === "t" || c === "T")) { |
| this._state = AFTER_STYLE_1; |
| } else this._state = TEXT; |
| }; |
| |
| Tokenizer.prototype._stateBeforeScript1 = consumeSpecialNameChar( |
| "R", |
| BEFORE_SCRIPT_2 |
| ); |
| Tokenizer.prototype._stateBeforeScript2 = consumeSpecialNameChar( |
| "I", |
| BEFORE_SCRIPT_3 |
| ); |
| Tokenizer.prototype._stateBeforeScript3 = consumeSpecialNameChar( |
| "P", |
| BEFORE_SCRIPT_4 |
| ); |
| Tokenizer.prototype._stateBeforeScript4 = consumeSpecialNameChar( |
| "T", |
| BEFORE_SCRIPT_5 |
| ); |
| |
| Tokenizer.prototype._stateBeforeScript5 = function(c) { |
| if (c === "/" || c === ">" || whitespace(c)) { |
| this._special = SPECIAL_SCRIPT; |
| } |
| this._state = IN_TAG_NAME; |
| this._index--; //consume the token again |
| }; |
| |
| Tokenizer.prototype._stateAfterScript1 = ifElseState("R", AFTER_SCRIPT_2, TEXT); |
| Tokenizer.prototype._stateAfterScript2 = ifElseState("I", AFTER_SCRIPT_3, TEXT); |
| Tokenizer.prototype._stateAfterScript3 = ifElseState("P", AFTER_SCRIPT_4, TEXT); |
| Tokenizer.prototype._stateAfterScript4 = ifElseState("T", AFTER_SCRIPT_5, TEXT); |
| |
| Tokenizer.prototype._stateAfterScript5 = function(c) { |
| if (c === ">" || whitespace(c)) { |
| this._special = SPECIAL_NONE; |
| this._state = IN_CLOSING_TAG_NAME; |
| this._sectionStart = this._index - 6; |
| this._index--; //reconsume the token |
| } else this._state = TEXT; |
| }; |
| |
| Tokenizer.prototype._stateBeforeStyle1 = consumeSpecialNameChar( |
| "Y", |
| BEFORE_STYLE_2 |
| ); |
| Tokenizer.prototype._stateBeforeStyle2 = consumeSpecialNameChar( |
| "L", |
| BEFORE_STYLE_3 |
| ); |
| Tokenizer.prototype._stateBeforeStyle3 = consumeSpecialNameChar( |
| "E", |
| BEFORE_STYLE_4 |
| ); |
| |
| Tokenizer.prototype._stateBeforeStyle4 = function(c) { |
| if (c === "/" || c === ">" || whitespace(c)) { |
| this._special = SPECIAL_STYLE; |
| } |
| this._state = IN_TAG_NAME; |
| this._index--; //consume the token again |
| }; |
| |
| Tokenizer.prototype._stateAfterStyle1 = ifElseState("Y", AFTER_STYLE_2, TEXT); |
| Tokenizer.prototype._stateAfterStyle2 = ifElseState("L", AFTER_STYLE_3, TEXT); |
| Tokenizer.prototype._stateAfterStyle3 = ifElseState("E", AFTER_STYLE_4, TEXT); |
| |
| Tokenizer.prototype._stateAfterStyle4 = function(c) { |
| if (c === ">" || whitespace(c)) { |
| this._special = SPECIAL_NONE; |
| this._state = IN_CLOSING_TAG_NAME; |
| this._sectionStart = this._index - 5; |
| this._index--; //reconsume the token |
| } else this._state = TEXT; |
| }; |
| |
| Tokenizer.prototype._stateBeforeEntity = ifElseState( |
| "#", |
| BEFORE_NUMERIC_ENTITY, |
| IN_NAMED_ENTITY |
| ); |
| Tokenizer.prototype._stateBeforeNumericEntity = ifElseState( |
| "X", |
| IN_HEX_ENTITY, |
| IN_NUMERIC_ENTITY |
| ); |
| |
| //for entities terminated with a semicolon |
| Tokenizer.prototype._parseNamedEntityStrict = function() { |
| //offset = 1 |
| if (this._sectionStart + 1 < this._index) { |
| var entity = this._buffer.substring( |
| this._sectionStart + 1, |
| this._index |
| ), |
| map = this._xmlMode ? xmlMap : entityMap; |
| |
| if (map.hasOwnProperty(entity)) { |
| this._emitPartial(map[entity]); |
| this._sectionStart = this._index + 1; |
| } |
| } |
| }; |
| |
| //parses legacy entities (without trailing semicolon) |
| Tokenizer.prototype._parseLegacyEntity = function() { |
| var start = this._sectionStart + 1, |
| limit = this._index - start; |
| |
| if (limit > 6) limit = 6; //the max length of legacy entities is 6 |
| |
| while (limit >= 2) { |
| //the min length of legacy entities is 2 |
| var entity = this._buffer.substr(start, limit); |
| |
| if (legacyMap.hasOwnProperty(entity)) { |
| this._emitPartial(legacyMap[entity]); |
| this._sectionStart += limit + 1; |
| return; |
| } else { |
| limit--; |
| } |
| } |
| }; |
| |
| Tokenizer.prototype._stateInNamedEntity = function(c) { |
| if (c === ";") { |
| this._parseNamedEntityStrict(); |
| if (this._sectionStart + 1 < this._index && !this._xmlMode) { |
| this._parseLegacyEntity(); |
| } |
| this._state = this._baseState; |
| } else if ( |
| (c < "a" || c > "z") && |
| (c < "A" || c > "Z") && |
| (c < "0" || c > "9") |
| ) { |
| if (this._xmlMode); |
| else if (this._sectionStart + 1 === this._index); |
| else if (this._baseState !== TEXT) { |
| if (c !== "=") { |
| this._parseNamedEntityStrict(); |
| } |
| } else { |
| this._parseLegacyEntity(); |
| } |
| |
| this._state = this._baseState; |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._decodeNumericEntity = function(offset, base) { |
| var sectionStart = this._sectionStart + offset; |
| |
| if (sectionStart !== this._index) { |
| //parse entity |
| var entity = this._buffer.substring(sectionStart, this._index); |
| var parsed = parseInt(entity, base); |
| |
| this._emitPartial(decodeCodePoint(parsed)); |
| this._sectionStart = this._index; |
| } else { |
| this._sectionStart--; |
| } |
| |
| this._state = this._baseState; |
| }; |
| |
| Tokenizer.prototype._stateInNumericEntity = function(c) { |
| if (c === ";") { |
| this._decodeNumericEntity(2, 10); |
| this._sectionStart++; |
| } else if (c < "0" || c > "9") { |
| if (!this._xmlMode) { |
| this._decodeNumericEntity(2, 10); |
| } else { |
| this._state = this._baseState; |
| } |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._stateInHexEntity = function(c) { |
| if (c === ";") { |
| this._decodeNumericEntity(3, 16); |
| this._sectionStart++; |
| } else if ( |
| (c < "a" || c > "f") && |
| (c < "A" || c > "F") && |
| (c < "0" || c > "9") |
| ) { |
| if (!this._xmlMode) { |
| this._decodeNumericEntity(3, 16); |
| } else { |
| this._state = this._baseState; |
| } |
| this._index--; |
| } |
| }; |
| |
| Tokenizer.prototype._cleanup = function() { |
| if (this._sectionStart < 0) { |
| this._buffer = ""; |
| this._bufferOffset += this._index; |
| this._index = 0; |
| } else if (this._running) { |
| if (this._state === TEXT) { |
| if (this._sectionStart !== this._index) { |
| this._cbs.ontext(this._buffer.substr(this._sectionStart)); |
| } |
| this._buffer = ""; |
| this._bufferOffset += this._index; |
| this._index = 0; |
| } else if (this._sectionStart === this._index) { |
| //the section just started |
| this._buffer = ""; |
| this._bufferOffset += this._index; |
| this._index = 0; |
| } else { |
| //remove everything unnecessary |
| this._buffer = this._buffer.substr(this._sectionStart); |
| this._index -= this._sectionStart; |
| this._bufferOffset += this._sectionStart; |
| } |
| |
| this._sectionStart = 0; |
| } |
| }; |
| |
| //TODO make events conditional |
| Tokenizer.prototype.write = function(chunk) { |
| if (this._ended) this._cbs.onerror(Error(".write() after done!")); |
| |
| this._buffer += chunk; |
| this._parse(); |
| }; |
| |
| Tokenizer.prototype._parse = function() { |
| while (this._index < this._buffer.length && this._running) { |
| var c = this._buffer.charAt(this._index); |
| if (this._state === TEXT) { |
| this._stateText(c); |
| } else if (this._state === BEFORE_TAG_NAME) { |
| this._stateBeforeTagName(c); |
| } else if (this._state === IN_TAG_NAME) { |
| this._stateInTagName(c); |
| } else if (this._state === BEFORE_CLOSING_TAG_NAME) { |
| this._stateBeforeCloseingTagName(c); |
| } else if (this._state === IN_CLOSING_TAG_NAME) { |
| this._stateInCloseingTagName(c); |
| } else if (this._state === AFTER_CLOSING_TAG_NAME) { |
| this._stateAfterCloseingTagName(c); |
| } else if (this._state === IN_SELF_CLOSING_TAG) { |
| this._stateInSelfClosingTag(c); |
| } else if (this._state === BEFORE_ATTRIBUTE_NAME) { |
| |
| /* |
| * attributes |
| */ |
| this._stateBeforeAttributeName(c); |
| } else if (this._state === IN_ATTRIBUTE_NAME) { |
| this._stateInAttributeName(c); |
| } else if (this._state === AFTER_ATTRIBUTE_NAME) { |
| this._stateAfterAttributeName(c); |
| } else if (this._state === BEFORE_ATTRIBUTE_VALUE) { |
| this._stateBeforeAttributeValue(c); |
| } else if (this._state === IN_ATTRIBUTE_VALUE_DQ) { |
| this._stateInAttributeValueDoubleQuotes(c); |
| } else if (this._state === IN_ATTRIBUTE_VALUE_SQ) { |
| this._stateInAttributeValueSingleQuotes(c); |
| } else if (this._state === IN_ATTRIBUTE_VALUE_NQ) { |
| this._stateInAttributeValueNoQuotes(c); |
| } else if (this._state === BEFORE_DECLARATION) { |
| |
| /* |
| * declarations |
| */ |
| this._stateBeforeDeclaration(c); |
| } else if (this._state === IN_DECLARATION) { |
| this._stateInDeclaration(c); |
| } else if (this._state === IN_PROCESSING_INSTRUCTION) { |
| |
| /* |
| * processing instructions |
| */ |
| this._stateInProcessingInstruction(c); |
| } else if (this._state === BEFORE_COMMENT) { |
| |
| /* |
| * comments |
| */ |
| this._stateBeforeComment(c); |
| } else if (this._state === IN_COMMENT) { |
| this._stateInComment(c); |
| } else if (this._state === AFTER_COMMENT_1) { |
| this._stateAfterComment1(c); |
| } else if (this._state === AFTER_COMMENT_2) { |
| this._stateAfterComment2(c); |
| } else if (this._state === BEFORE_CDATA_1) { |
| |
| /* |
| * cdata |
| */ |
| this._stateBeforeCdata1(c); |
| } else if (this._state === BEFORE_CDATA_2) { |
| this._stateBeforeCdata2(c); |
| } else if (this._state === BEFORE_CDATA_3) { |
| this._stateBeforeCdata3(c); |
| } else if (this._state === BEFORE_CDATA_4) { |
| this._stateBeforeCdata4(c); |
| } else if (this._state === BEFORE_CDATA_5) { |
| this._stateBeforeCdata5(c); |
| } else if (this._state === BEFORE_CDATA_6) { |
| this._stateBeforeCdata6(c); |
| } else if (this._state === IN_CDATA) { |
| this._stateInCdata(c); |
| } else if (this._state === AFTER_CDATA_1) { |
| this._stateAfterCdata1(c); |
| } else if (this._state === AFTER_CDATA_2) { |
| this._stateAfterCdata2(c); |
| } else if (this._state === BEFORE_SPECIAL) { |
| |
| /* |
| * special tags |
| */ |
| this._stateBeforeSpecial(c); |
| } else if (this._state === BEFORE_SPECIAL_END) { |
| this._stateBeforeSpecialEnd(c); |
| } else if (this._state === BEFORE_SCRIPT_1) { |
| |
| /* |
| * script |
| */ |
| this._stateBeforeScript1(c); |
| } else if (this._state === BEFORE_SCRIPT_2) { |
| this._stateBeforeScript2(c); |
| } else if (this._state === BEFORE_SCRIPT_3) { |
| this._stateBeforeScript3(c); |
| } else if (this._state === BEFORE_SCRIPT_4) { |
| this._stateBeforeScript4(c); |
| } else if (this._state === BEFORE_SCRIPT_5) { |
| this._stateBeforeScript5(c); |
| } else if (this._state === AFTER_SCRIPT_1) { |
| this._stateAfterScript1(c); |
| } else if (this._state === AFTER_SCRIPT_2) { |
| this._stateAfterScript2(c); |
| } else if (this._state === AFTER_SCRIPT_3) { |
| this._stateAfterScript3(c); |
| } else if (this._state === AFTER_SCRIPT_4) { |
| this._stateAfterScript4(c); |
| } else if (this._state === AFTER_SCRIPT_5) { |
| this._stateAfterScript5(c); |
| } else if (this._state === BEFORE_STYLE_1) { |
| |
| /* |
| * style |
| */ |
| this._stateBeforeStyle1(c); |
| } else if (this._state === BEFORE_STYLE_2) { |
| this._stateBeforeStyle2(c); |
| } else if (this._state === BEFORE_STYLE_3) { |
| this._stateBeforeStyle3(c); |
| } else if (this._state === BEFORE_STYLE_4) { |
| this._stateBeforeStyle4(c); |
| } else if (this._state === AFTER_STYLE_1) { |
| this._stateAfterStyle1(c); |
| } else if (this._state === AFTER_STYLE_2) { |
| this._stateAfterStyle2(c); |
| } else if (this._state === AFTER_STYLE_3) { |
| this._stateAfterStyle3(c); |
| } else if (this._state === AFTER_STYLE_4) { |
| this._stateAfterStyle4(c); |
| } else if (this._state === BEFORE_ENTITY) { |
| |
| /* |
| * entities |
| */ |
| this._stateBeforeEntity(c); |
| } else if (this._state === BEFORE_NUMERIC_ENTITY) { |
| this._stateBeforeNumericEntity(c); |
| } else if (this._state === IN_NAMED_ENTITY) { |
| this._stateInNamedEntity(c); |
| } else if (this._state === IN_NUMERIC_ENTITY) { |
| this._stateInNumericEntity(c); |
| } else if (this._state === IN_HEX_ENTITY) { |
| this._stateInHexEntity(c); |
| } else { |
| this._cbs.onerror(Error("unknown _state"), this._state); |
| } |
| |
| this._index++; |
| } |
| |
| this._cleanup(); |
| }; |
| |
| Tokenizer.prototype.pause = function() { |
| this._running = false; |
| }; |
| Tokenizer.prototype.resume = function() { |
| this._running = true; |
| |
| if (this._index < this._buffer.length) { |
| this._parse(); |
| } |
| if (this._ended) { |
| this._finish(); |
| } |
| }; |
| |
| Tokenizer.prototype.end = function(chunk) { |
| if (this._ended) this._cbs.onerror(Error(".end() after done!")); |
| if (chunk) this.write(chunk); |
| |
| this._ended = true; |
| |
| if (this._running) this._finish(); |
| }; |
| |
| Tokenizer.prototype._finish = function() { |
| //if there is remaining data, emit it in a reasonable way |
| if (this._sectionStart < this._index) { |
| this._handleTrailingData(); |
| } |
| |
| this._cbs.onend(); |
| }; |
| |
| Tokenizer.prototype._handleTrailingData = function() { |
| var data = this._buffer.substr(this._sectionStart); |
| |
| if ( |
| this._state === IN_CDATA || |
| this._state === AFTER_CDATA_1 || |
| this._state === AFTER_CDATA_2 |
| ) { |
| this._cbs.oncdata(data); |
| } else if ( |
| this._state === IN_COMMENT || |
| this._state === AFTER_COMMENT_1 || |
| this._state === AFTER_COMMENT_2 |
| ) { |
| this._cbs.oncomment(data); |
| } else if (this._state === IN_NAMED_ENTITY && !this._xmlMode) { |
| this._parseLegacyEntity(); |
| if (this._sectionStart < this._index) { |
| this._state = this._baseState; |
| this._handleTrailingData(); |
| } |
| } else if (this._state === IN_NUMERIC_ENTITY && !this._xmlMode) { |
| this._decodeNumericEntity(2, 10); |
| if (this._sectionStart < this._index) { |
| this._state = this._baseState; |
| this._handleTrailingData(); |
| } |
| } else if (this._state === IN_HEX_ENTITY && !this._xmlMode) { |
| this._decodeNumericEntity(3, 16); |
| if (this._sectionStart < this._index) { |
| this._state = this._baseState; |
| this._handleTrailingData(); |
| } |
| } else if ( |
| this._state !== IN_TAG_NAME && |
| this._state !== BEFORE_ATTRIBUTE_NAME && |
| this._state !== BEFORE_ATTRIBUTE_VALUE && |
| this._state !== AFTER_ATTRIBUTE_NAME && |
| this._state !== IN_ATTRIBUTE_NAME && |
| this._state !== IN_ATTRIBUTE_VALUE_SQ && |
| this._state !== IN_ATTRIBUTE_VALUE_DQ && |
| this._state !== IN_ATTRIBUTE_VALUE_NQ && |
| this._state !== IN_CLOSING_TAG_NAME |
| ) { |
| this._cbs.ontext(data); |
| } |
| //else, ignore remaining data |
| //TODO add a way to remove current tag |
| }; |
| |
| Tokenizer.prototype.reset = function() { |
| Tokenizer.call( |
| this, |
| { xmlMode: this._xmlMode, decodeEntities: this._decodeEntities }, |
| this._cbs |
| ); |
| }; |
| |
| Tokenizer.prototype.getAbsoluteIndex = function() { |
| return this._bufferOffset + this._index; |
| }; |
| |
| Tokenizer.prototype._getSection = function() { |
| return this._buffer.substring(this._sectionStart, this._index); |
| }; |
| |
| Tokenizer.prototype._emitToken = function(name) { |
| this._cbs[name](this._getSection()); |
| this._sectionStart = -1; |
| }; |
| |
| Tokenizer.prototype._emitPartial = function(value) { |
| if (this._baseState !== TEXT) { |
| this._cbs.onattribdata(value); //TODO implement the new event |
| } else { |
| this._cbs.ontext(value); |
| } |
| }; |
