blob: 7283fe7ca289ab66e529f68637ac4c1024a40b91 [file] [log] [blame]
* Copyright (C) 1999-2000 Harri Porten (
* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2011, 2012, 2013 Apple Inc. All rights reserved.
* Copyright (C) 2010 Zoltan Herczeg (
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* Library General Public License for more details.
* You should have received a copy of the GNU Library General Public License
* along with this library; see the file COPYING.LIB. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
#pragma once
#include "Lookup.h"
#include "ParserArena.h"
#include "ParserTokens.h"
#include "SourceCode.h"
#include <wtf/ASCIICType.h>
#include <wtf/Vector.h>
namespace JSC {
enum LexerFlags {
LexerFlagsIgnoreReservedWords = 1,
LexerFlagsDontBuildStrings = 2,
LexexFlagsDontBuildKeywords = 4
enum class LexerEscapeParseMode { Template, String };
struct ParsedUnicodeEscapeValue;
bool isLexerKeyword(const Identifier&);
template <typename T>
class Lexer {
Lexer(VM*, JSParserBuiltinMode, JSParserScriptMode);
// Character manipulation functions.
static bool isWhiteSpace(T character);
static bool isLineTerminator(T character);
static unsigned char convertHex(int c1, int c2);
static UChar convertUnicode(int c1, int c2, int c3, int c4);
// Functions to set up parsing.
void setCode(const SourceCode&, ParserArena*);
void setIsReparsingFunction() { m_isReparsingFunction = true; }
bool isReparsingFunction() const { return m_isReparsingFunction; }
JSTokenType lex(JSToken*, unsigned, bool strictMode);
bool nextTokenIsColon();
int lineNumber() const { return m_lineNumber; }
ALWAYS_INLINE int currentOffset() const { return offsetFromSourcePtr(m_code); }
ALWAYS_INLINE int currentLineStartOffset() const { return offsetFromSourcePtr(m_lineStart); }
ALWAYS_INLINE JSTextPosition currentPosition() const
return JSTextPosition(m_lineNumber, currentOffset(), currentLineStartOffset());
JSTextPosition positionBeforeLastNewline() const { return m_positionBeforeLastNewline; }
JSTokenLocation lastTokenLocation() const { return m_lastTokenLocation; }
void setLastLineNumber(int lastLineNumber) { m_lastLineNumber = lastLineNumber; }
int lastLineNumber() const { return m_lastLineNumber; }
bool prevTerminator() const { return m_terminator; }
JSTokenType scanRegExp(JSToken*, UChar patternPrefix = 0);
enum class RawStringsBuildMode { BuildRawStrings, DontBuildRawStrings };
JSTokenType scanTemplateString(JSToken*, RawStringsBuildMode);
// Functions for use after parsing.
bool sawError() const { return m_error; }
void setSawError(bool sawError) { m_error = sawError; }
String getErrorMessage() const { return m_lexErrorMessage; }
void setErrorMessage(const String& errorMessage) { m_lexErrorMessage = errorMessage; }
String sourceURL() const { return m_sourceURLDirective; }
String sourceMappingURL() const { return m_sourceMappingURLDirective; }
void clear();
void setOffset(int offset, int lineStartOffset)
m_error = 0;
m_lexErrorMessage = String();
m_code = sourcePtrFromOffset(offset);
m_lineStart = sourcePtrFromOffset(lineStartOffset);
ASSERT(currentOffset() >= currentLineStartOffset());
if (LIKELY(m_code < m_codeEnd))
m_current = *m_code;
m_current = 0;
void setLineNumber(int line)
m_lineNumber = line;
void setTerminator(bool terminator)
m_terminator = terminator;
JSTokenType lexExpectIdentifier(JSToken*, unsigned, bool strictMode);
ALWAYS_INLINE StringView getToken(const JSToken& token)
SourceProvider* sourceProvider = m_source->provider();
ASSERT_WITH_MESSAGE(token.m_location.startOffset <= token.m_location.endOffset, "Calling this function with the baked token.");
return sourceProvider->getRange(token.m_location.startOffset, token.m_location.endOffset);
void record8(int);
void append8(const T*, size_t);
void record16(int);
void record16(T);
void recordUnicodeCodePoint(UChar32);
void append16(const LChar*, size_t);
void append16(const UChar* characters, size_t length) { m_buffer16.append(characters, length); }
ALWAYS_INLINE void shift();
ALWAYS_INLINE bool atEnd() const;
ALWAYS_INLINE T peek(int offset) const;
ParsedUnicodeEscapeValue parseUnicodeEscape();
void shiftLineTerminator();
ALWAYS_INLINE int offsetFromSourcePtr(const T* ptr) const { return ptr - m_codeStart; }
ALWAYS_INLINE const T* sourcePtrFromOffset(int offset) const { return m_codeStart + offset; }
String invalidCharacterMessage() const;
ALWAYS_INLINE const T* currentSourcePtr() const;
ALWAYS_INLINE void setOffsetFromSourcePtr(const T* sourcePtr, unsigned lineStartOffset) { setOffset(offsetFromSourcePtr(sourcePtr), lineStartOffset); }
ALWAYS_INLINE void setCodeStart(const StringView&);
ALWAYS_INLINE const Identifier* makeIdentifier(const LChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeIdentifier(const UChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeLCharIdentifier(const LChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeLCharIdentifier(const UChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeRightSizedIdentifier(const UChar* characters, size_t length, UChar orAllChars);
ALWAYS_INLINE const Identifier* makeIdentifierLCharFromUChar(const UChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeEmptyIdentifier();
ALWAYS_INLINE bool lastTokenWasRestrKeyword() const;
ALWAYS_INLINE void skipWhitespace();
template <int shiftAmount> void internalShift();
template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType parseKeyword(JSTokenData*);
template <bool shouldBuildIdentifiers> ALWAYS_INLINE JSTokenType parseIdentifier(JSTokenData*, unsigned lexerFlags, bool strictMode);
template <bool shouldBuildIdentifiers> NEVER_INLINE JSTokenType parseIdentifierSlowCase(JSTokenData*, unsigned lexerFlags, bool strictMode);
enum StringParseResult {
template <bool shouldBuildStrings> ALWAYS_INLINE StringParseResult parseString(JSTokenData*, bool strictMode);
template <bool shouldBuildStrings> NEVER_INLINE StringParseResult parseStringSlowCase(JSTokenData*, bool strictMode);
template <bool shouldBuildStrings, LexerEscapeParseMode escapeParseMode> ALWAYS_INLINE StringParseResult parseComplexEscape(bool strictMode, T stringQuoteCharacter);
ALWAYS_INLINE StringParseResult parseTemplateLiteral(JSTokenData*, RawStringsBuildMode);
ALWAYS_INLINE void parseHex(double& returnValue);
ALWAYS_INLINE bool parseBinary(double& returnValue);
ALWAYS_INLINE bool parseOctal(double& returnValue);
ALWAYS_INLINE bool parseDecimal(double& returnValue);
ALWAYS_INLINE void parseNumberAfterDecimalPoint();
ALWAYS_INLINE bool parseNumberAfterExponentIndicator();
ALWAYS_INLINE bool parseMultilineComment();
ALWAYS_INLINE void parseCommentDirective();
ALWAYS_INLINE String parseCommentDirectiveValue();
template <unsigned length>
ALWAYS_INLINE bool consume(const char (&input)[length]);
void fillTokenInfo(JSToken*, JSTokenType, int lineNumber, int endOffset, int lineStartOffset, JSTextPosition endPosition);
static const size_t initialReadBufferCapacity = 32;
int m_lineNumber;
int m_lastLineNumber;
Vector<LChar> m_buffer8;
Vector<UChar> m_buffer16;
Vector<UChar> m_bufferForRawTemplateString16;
bool m_terminator;
int m_lastToken;
const SourceCode* m_source;
unsigned m_sourceOffset;
const T* m_code;
const T* m_codeStart;
const T* m_codeEnd;
const T* m_codeStartPlusOffset;
const T* m_lineStart;
JSTextPosition m_positionBeforeLastNewline;
JSTokenLocation m_lastTokenLocation;
bool m_isReparsingFunction;
bool m_atLineStart;
bool m_error;
String m_lexErrorMessage;
String m_sourceURLDirective;
String m_sourceMappingURLDirective;
T m_current;
IdentifierArena* m_arena;
VM* m_vm;
bool m_parsingBuiltinFunction;
JSParserScriptMode m_scriptMode;
template <>
ALWAYS_INLINE bool Lexer<LChar>::isWhiteSpace(LChar ch)
return ch == ' ' || ch == '\t' || ch == 0xB || ch == 0xC || ch == 0xA0;
template <>
ALWAYS_INLINE bool Lexer<UChar>::isWhiteSpace(UChar ch)
// 0x180E used to be in Zs category before Unicode 6.3, and EcmaScript says that we should keep treating it as such.
return (ch < 256) ? Lexer<LChar>::isWhiteSpace(static_cast<LChar>(ch)) : (u_charType(ch) == U_SPACE_SEPARATOR || ch == 0x180E || ch == 0xFEFF);
template <>
ALWAYS_INLINE bool Lexer<LChar>::isLineTerminator(LChar ch)
return ch == '\r' || ch == '\n';
template <>
ALWAYS_INLINE bool Lexer<UChar>::isLineTerminator(UChar ch)
return ch == '\r' || ch == '\n' || (ch & ~1) == 0x2028;
template <typename T>
inline unsigned char Lexer<T>::convertHex(int c1, int c2)
return (toASCIIHexValue(c1) << 4) | toASCIIHexValue(c2);
template <typename T>
inline UChar Lexer<T>::convertUnicode(int c1, int c2, int c3, int c4)
return (convertHex(c1, c2) << 8) | convertHex(c3, c4);
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifier(const LChar* characters, size_t length)
return &m_arena->makeIdentifier(m_vm, characters, length);
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifier(const UChar* characters, size_t length)
return &m_arena->makeIdentifier(m_vm, characters, length);
template <>
ALWAYS_INLINE const Identifier* Lexer<LChar>::makeRightSizedIdentifier(const UChar* characters, size_t length, UChar)
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
template <>
ALWAYS_INLINE const Identifier* Lexer<UChar>::makeRightSizedIdentifier(const UChar* characters, size_t length, UChar orAllChars)
if (!(orAllChars & ~0xff))
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
return &m_arena->makeIdentifier(m_vm, characters, length);
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeEmptyIdentifier()
return &m_arena->makeEmptyIdentifier(m_vm);
template <>
ALWAYS_INLINE void Lexer<LChar>::setCodeStart(const StringView& sourceString)
m_codeStart = sourceString.characters8();
template <>
ALWAYS_INLINE void Lexer<UChar>::setCodeStart(const StringView& sourceString)
m_codeStart = sourceString.characters16();
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifierLCharFromUChar(const UChar* characters, size_t length)
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeLCharIdentifier(const LChar* characters, size_t length)
return &m_arena->makeIdentifier(m_vm, characters, length);
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeLCharIdentifier(const UChar* characters, size_t length)
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
ALWAYS_INLINE bool isSafeBuiltinIdentifier(VM&, const Identifier*) { return true; }
bool isSafeBuiltinIdentifier(VM&, const Identifier*);
template <typename T>
ALWAYS_INLINE JSTokenType Lexer<T>::lexExpectIdentifier(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
JSTokenData* tokenData = &tokenRecord->m_data;
JSTokenLocation* tokenLocation = &tokenRecord->m_location;
ASSERT((lexerFlags & LexerFlagsIgnoreReservedWords));
const T* start = m_code;
const T* ptr = start;
const T* end = m_codeEnd;
JSTextPosition startPosition = currentPosition();
if (ptr >= end) {
ASSERT(ptr == end);
goto slowCase;
if (!WTF::isASCIIAlpha(*ptr))
goto slowCase;
while (ptr < end) {
if (!WTF::isASCIIAlphanumeric(*ptr))
// Here's the shift
if (ptr < end) {
if ((!WTF::isASCII(*ptr)) || (*ptr == '\\') || (*ptr == '_') || (*ptr == '$'))
goto slowCase;
m_current = *ptr;
} else
m_current = 0;
m_code = ptr;
ASSERT(currentOffset() >= currentLineStartOffset());
// Create the identifier if needed
if (lexerFlags & LexexFlagsDontBuildKeywords
&& !m_parsingBuiltinFunction
tokenData->ident = 0;
tokenData->ident = makeLCharIdentifier(start, ptr - start);
tokenLocation->line = m_lineNumber;
tokenLocation->lineStartOffset = currentLineStartOffset();
tokenLocation->startOffset = offsetFromSourcePtr(start);
tokenLocation->endOffset = currentOffset();
ASSERT(tokenLocation->startOffset >= tokenLocation->lineStartOffset);
tokenRecord->m_startPosition = startPosition;
tokenRecord->m_endPosition = currentPosition();
if (m_parsingBuiltinFunction) {
if (!isSafeBuiltinIdentifier(*m_vm, tokenData->ident))
return ERRORTOK;
m_lastToken = IDENT;
return IDENT;
return lex(tokenRecord, lexerFlags, strictMode);
} // namespace JSC