| // Copyright 2014 Google Inc. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "pagespeed/kernel/js/js_tokenizer.h" |
| |
| #include <utility> |
| |
| #include "pagespeed/kernel/base/google_message_handler.h" |
| #include "pagespeed/kernel/base/gtest.h" |
| #include "pagespeed/kernel/base/scoped_ptr.h" |
| #include "pagespeed/kernel/base/stdio_file_system.h" |
| #include "pagespeed/kernel/base/string_util.h" |
| #include "pagespeed/kernel/js/js_keywords.h" |
| |
| using pagespeed::JsKeywords; |
| using pagespeed::js::JsTokenizer; |
| using pagespeed::js::JsTokenizerPatterns; |
| |
| namespace { |
| |
| const char kTestRootDir[] = "/pagespeed/kernel/js/testdata/third_party/"; |
| |
| class JsTokenizerTest : public testing::Test { |
| protected: |
| void BeginTokenizing(StringPiece input) { |
| tokenizer_.reset(new JsTokenizer(&patterns_, input)); |
| } |
| |
| void ExpectParseStack(StringPiece expected_parse_stack) { |
| EXPECT_STREQ(expected_parse_stack, tokenizer_->ParseStackForTest()); |
| } |
| |
| void ExpectToken(JsKeywords::Type expected_type, |
| StringPiece expected_token) { |
| StringPiece actual_token; |
| const JsKeywords::Type actual_type = tokenizer_->NextToken(&actual_token); |
| EXPECT_EQ(make_pair(expected_type, expected_token), |
| make_pair(actual_type, actual_token)); |
| EXPECT_FALSE(tokenizer_->has_error()); |
| } |
| |
| void ExpectToken(JsKeywords::Type expected_type, StringPiece expected_token, |
| StringPiece expected_parse_stack) { |
| ExpectToken(expected_type, expected_token); |
| ExpectParseStack(expected_parse_stack); |
| } |
| |
| void ExpectEndOfInput() { |
| ExpectToken(JsKeywords::kEndOfInput, ""); |
| ExpectParseStack(""); |
| } |
| |
| void ExpectError(StringPiece expected_token) { |
| StringPiece actual_token; |
| const JsKeywords::Type actual_type = tokenizer_->NextToken(&actual_token); |
| EXPECT_EQ(make_pair(JsKeywords::kError, expected_token), |
| make_pair(actual_type, actual_token)); |
| EXPECT_TRUE(tokenizer_->has_error()); |
| } |
| |
| void ExpectTokenizeFileSuccessfully(StringPiece filename) { |
| // Read in the JavaScript file. |
| GoogleString original; |
| { |
| net_instaweb::StdioFileSystem file_system; |
| const GoogleString filepath = net_instaweb::StrCat( |
| net_instaweb::GTestSrcDir(), kTestRootDir, filename); |
| net_instaweb::GoogleMessageHandler message_handler; |
| ASSERT_TRUE(file_system.ReadFile( |
| filepath.c_str(), &original, &message_handler)); |
| } |
| // Tokenize the JavaScript, appending each token onto the output string. |
| // There should be no tokenizer errors. |
| GoogleString output; |
| { |
| output.reserve(original.size()); |
| JsTokenizer tokenizer(&patterns_, original); |
| StringPiece token; |
| while (tokenizer.NextToken(&token) != |
| JsKeywords::kEndOfInput) { |
| ASSERT_FALSE(tokenizer.has_error()) |
| << "Error at: " << token.substr(0, 150) |
| << "\nWith stack: " << tokenizer.ParseStackForTest(); |
| token.AppendToString(&output); |
| } |
| } |
| // The concatenation of all tokens should exactly reproduce the input. |
| EXPECT_STREQ(original, output); |
| } |
| |
| private: |
| JsTokenizerPatterns patterns_; |
| net_instaweb::scoped_ptr<JsTokenizer> tokenizer_; |
| }; |
| |
| TEST_F(JsTokenizerTest, EmptyInput) { |
| BeginTokenizing(""); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Blocks) { |
| BeginTokenizing("if (foo){\n" |
| "}else if(bar){\n" |
| "}else baz;"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kIf, "if", "Start BkKwd"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "(", "Start BkKwd ("); |
| ExpectToken(JsKeywords::kIdentifier, "foo", "Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")", "Start BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "}", "Start"); |
| ExpectToken(JsKeywords::kElse, "else", "Start BkHdr"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIf, "if", "Start BkHdr BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "(", "Start BkHdr BkKwd ("); |
| ExpectToken(JsKeywords::kIdentifier, "bar", "Start BkHdr BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")", "Start BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "}", "Start"); |
| ExpectToken(JsKeywords::kElse, "else", "Start BkHdr"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "baz", "Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, ";", "Start"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Functions) { |
| BeginTokenizing("function foo(){return 1}\n" |
| "bar=function(){return 2};\n" |
| "(function(){window=5})();"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kFunction, "function"); |
| ExpectParseStack("Start BkKwd"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectParseStack("Start BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start BkHdr {"); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectParseStack("Start BkHdr { RetTh"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectParseStack("Start BkHdr { RetTh Expr"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectParseStack("Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kFunction, "function"); |
| ExpectParseStack("Start Expr Oper BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start Expr Oper BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start Expr Oper BkHdr {"); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectParseStack("Start Expr Oper BkHdr { RetTh"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNumber, "2"); |
| ExpectParseStack("Start Expr Oper BkHdr { RetTh Expr"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start Expr"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start ("); |
| ExpectToken(JsKeywords::kFunction, "function"); |
| ExpectParseStack("Start ( BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start ( BkKwd ("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start ( BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start ( BkHdr {"); |
| ExpectToken(JsKeywords::kIdentifier, "window"); |
| ExpectParseStack("Start ( BkHdr { Expr"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectParseStack("Start ( BkHdr { Expr Oper"); |
| ExpectToken(JsKeywords::kNumber, "5"); |
| ExpectParseStack("Start ( BkHdr { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start Expr ("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start Expr"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ForLoop) { |
| BeginTokenizing("loop:for(var x=0;x<5;++x){\n" |
| " break loop;\n" |
| "}"); |
| ExpectToken(JsKeywords::kIdentifier, "loop"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kFor, "for"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kNumber, "0"); |
| ExpectParseStack("Start BkKwd ( Other Expr"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start BkKwd ("); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "<"); |
| ExpectToken(JsKeywords::kNumber, "5"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectParseStack("Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kBreak, "break"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "loop"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Keywords) { |
| // A piece of code that contains every keyword at least once. |
| BeginTokenizing("(function(){\n" |
| " var\n" |
| " x=typeof null\n" |
| " const y=void false\n" |
| " delete x\n" |
| " if(this instanceof String){\n" |
| " debugger\n" |
| " for(z in this)\n" |
| " continue\n" |
| " do break;while(true)\n" |
| " switch(y){\n" |
| " case 0:\n" |
| " default:\n" |
| " try{\n" |
| " with(this){\n" |
| " throw new Object()\n" |
| " }\n" |
| " }catch(e){\n" |
| " return\n" |
| " }finally{}\n" |
| " }\n" |
| " }else return\n" |
| "})"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kFunction, "function"); |
| ExpectParseStack("Start ( BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr {"); |
| |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectParseStack("Start ( BkHdr { Other"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kTypeof, "typeof"); |
| ExpectParseStack("Start ( BkHdr { Other Expr Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNull, "null"); |
| ExpectParseStack("Start ( BkHdr { Other Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n "); |
| ExpectParseStack("Start ( BkHdr {"); |
| |
| ExpectToken(JsKeywords::kConst, "const"); |
| ExpectParseStack("Start ( BkHdr { Other"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kVoid, "void"); |
| ExpectParseStack("Start ( BkHdr { Other Expr Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kFalse, "false"); |
| ExpectParseStack("Start ( BkHdr { Other Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n "); |
| ExpectParseStack("Start ( BkHdr {"); |
| |
| ExpectToken(JsKeywords::kDelete, "delete"); |
| ExpectParseStack("Start ( BkHdr { Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n "); |
| ExpectParseStack("Start ( BkHdr {"); |
| |
| ExpectToken(JsKeywords::kIf, "if"); |
| ExpectParseStack("Start ( BkHdr { BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kThis, "this"); |
| ExpectParseStack("Start ( BkHdr { BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kInstanceof, "instanceof"); |
| ExpectParseStack("Start ( BkHdr { BkKwd ( Expr Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "String"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kDebugger, "debugger"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { Jump"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kFor, "for"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kIdentifier, "z"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIn, "in"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkKwd ( Expr Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kThis, "this"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr"); |
| |
| ExpectToken(JsKeywords::kContinue, "continue"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr Jump"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kDo, "do"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kBreak, "break"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr Jump"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start ( BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kWhile, "while"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kTrue, "true"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kSwitch, "switch"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kCase, "case"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNumber, "0"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kDefault, "default"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { Other"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kTry, "try"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kWith, "with"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kThis, "this"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kThrow, "throw"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkHdr { RetTh"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNew, "new"); |
| ExpectParseStack( |
| "Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkHdr { RetTh Oper"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "Object"); |
| ExpectParseStack( |
| "Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkHdr { RetTh Expr"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack( |
| "Start ( BkHdr { BkHdr { BkHdr { BkHdr { BkHdr { RetTh Expr"); |
| |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kCatch, "catch"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kIdentifier, "e"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr { RetTh"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kFinally, "finally"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr { BkHdr {"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectParseStack("Start ( BkHdr { BkHdr { BkHdr {"); |
| |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start ( BkHdr {"); |
| ExpectToken(JsKeywords::kElse, "else"); |
| ExpectParseStack("Start ( BkHdr { BkHdr"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectParseStack("Start ( BkHdr { BkHdr RetTh"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectParseStack("Start ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start Expr"); |
| ExpectEndOfInput(); |
| ExpectParseStack(""); |
| } |
| |
| TEST_F(JsTokenizerTest, StrictModeReservedWords) { |
| // These names are reserved words in strict mode, but otherwise they're legal |
| // identifiers. |
| // TODO(mdsteele): At some point we may want to implement strict mode error |
| // checking, at which point we should add a test with "use strict". |
| BeginTokenizing("var implements,interface,let,package\n" |
| " ,private,protected,public,static,yield;"); |
| ExpectToken(JsKeywords::kVar, "var", "Start Other"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "implements", "Start Other Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Other"); |
| ExpectToken(JsKeywords::kIdentifier, "interface"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "let"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "package"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "private"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "protected"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "public"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "static"); |
| ExpectToken(JsKeywords::kOperator, ","); |
| ExpectToken(JsKeywords::kIdentifier, "yield"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ReservedWordsAsFieldNames1) { |
| // Reserved words may be used as identifiers in certain contexts, such as |
| // field access. |
| // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar#Reserved_word_usage |
| BeginTokenizing("document.true=true;"); |
| ExpectToken(JsKeywords::kIdentifier, "document"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "true"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kTrue, "true"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ReservedWordsAsFieldNames2) { |
| // Reserved words may be used as identifiers in certain contexts, such as |
| // object literal property names. |
| // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar#Reserved_word_usage |
| BeginTokenizing("x={if:false,class:3};"); |
| ExpectToken(JsKeywords::kIdentifier, "x", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start Expr Oper {"); |
| ExpectToken(JsKeywords::kIdentifier, "if", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":", "Start Expr Oper { Oper"); |
| ExpectToken(JsKeywords::kFalse, "false", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Expr Oper {"); |
| ExpectToken(JsKeywords::kIdentifier, "class", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":", "Start Expr Oper { Oper"); |
| ExpectToken(JsKeywords::kNumber, "3", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, ";", "Start"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ReservedWordsAsLabelNames) { |
| // Reserved words may NOT be used as label identifiers. We should |
| // distinguish this case from the object literal case above. |
| BeginTokenizing("{if:false,class:3};"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start {"); |
| ExpectToken(JsKeywords::kIf, "if", "Start { BkKwd"); |
| ExpectError(":false,class:3};"); |
| } |
| |
| TEST_F(JsTokenizerTest, Comments) { |
| BeginTokenizing("x=1; // hello\n" |
| "y=/* world */2;\n" |
| "z<!--sgml\n" // <!-- is a line comment, but |
| "foo-->bar;\n" // --> is a line comment only at |
| " --> sgml\n"); // the start of the line. |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kComment, "// hello"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kComment, "/* world */"); |
| ExpectToken(JsKeywords::kNumber, "2"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "z"); |
| ExpectToken(JsKeywords::kComment, "<!--sgml"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, "--"); |
| ExpectToken(JsKeywords::kOperator, ">"); |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| |
| ExpectToken(JsKeywords::kComment, "--> sgml"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, LineComments) { |
| BeginTokenizing("1//foo\r" |
| "+2//bar\xE2\x80\xA8" // U+2028 LINE SEPARATOR" |
| "+3//baz\xE2\x80\xA9" // U+2029 PARAGRAPH SEPARATOR |
| "4//quux\xE2\x81\x9F" // U+205F MEDIUM MATHEMATICAL SPACE |
| "+5//hello, world!\n" |
| "6"); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectToken(JsKeywords::kComment, "//foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\r"); |
| |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kNumber, "2"); |
| ExpectToken(JsKeywords::kComment, "//bar"); |
| ExpectToken(JsKeywords::kLineSeparator, "\xE2\x80\xA8"); |
| |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kNumber, "3"); |
| ExpectToken(JsKeywords::kComment, "//baz"); |
| ExpectToken(JsKeywords::kSemiInsert, "\xE2\x80\xA9"); |
| |
| ExpectToken(JsKeywords::kNumber, "4"); |
| ExpectToken(JsKeywords::kComment, "//quux\xE2\x81\x9F+5//hello, world!"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kNumber, "6"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, SgmlComments) { |
| BeginTokenizing("4+<!--foo\n" |
| "3+\n" |
| "/*foo*/ --> foo\n" |
| "x --> foo\n"); |
| ExpectToken(JsKeywords::kNumber, "4"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kComment, "<!--foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kNumber, "3"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kComment, "/*foo*/"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kComment, "--> foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "--"); |
| ExpectToken(JsKeywords::kOperator, ">"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Whitespace) { |
| BeginTokenizing("\xEF\xBB\xBF" // U+FEFF BYTE ORDER MARK |
| "a\f=\t1\v+ 2\r" |
| "5\xC2\xA0" // U+00A0 NO-BREAK SPACE |
| "\xE2\x81\x9F" // U+205F MEDIUM MATHEMATICAL SPACE |
| "+\xE1\x9A\x80" // U+1680 OGHAM SPACE MARK |
| "7\xE2\x80\xA9" // U+2029 PARAGRAPH SEPARATOR |
| ";\xE2\x80\xA8"); // U+2028 LINE SEPARATOR |
| ExpectToken(JsKeywords::kWhitespace, "\xEF\xBB\xBF"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kWhitespace, "\f"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kWhitespace, "\t"); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectToken(JsKeywords::kWhitespace, "\v"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNumber, "2"); |
| ExpectToken(JsKeywords::kSemiInsert, "\r"); |
| |
| ExpectToken(JsKeywords::kNumber, "5"); |
| ExpectToken(JsKeywords::kWhitespace, "\xC2\xA0\xE2\x81\x9F"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, "\xE1\x9A\x80"); |
| ExpectToken(JsKeywords::kNumber, "7"); |
| ExpectToken(JsKeywords::kLineSeparator, "\xE2\x80\xA9"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\xE2\x80\xA8"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, NumericLiterals) { |
| BeginTokenizing("var foo =\n .85 - 0191.e+3+0171.toString();\n" |
| "1..property"); |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kLineSeparator, "\n "); |
| ExpectToken(JsKeywords::kNumber, ".85"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kNumber, "0191.e+3"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kNumber, "0171"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "toString"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kNumber, "1."); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "property"); |
| ExpectEndOfInput(); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, RegexLiterals) { |
| BeginTokenizing("foo=/quux/;\n" |
| "bar=/quux/ig;\n" |
| "baz=a/quux/ig;"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kRegex, "/quux/"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kRegex, "/quux/ig"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "baz"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "quux"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "ig"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, RegexVsSlash) { |
| BeginTokenizing("if(a*(b+c)/d<e)/d<e/.exec('\\'');\n" |
| "else/x/.exec(\"\");"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kIf, "if"); |
| ExpectParseStack("Start BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start BkKwd ("); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectParseStack("Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "*"); |
| ExpectParseStack("Start BkKwd ( Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start BkKwd ( Expr Oper ("); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectParseStack("Start BkKwd ( Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectParseStack("Start BkKwd ( Expr Oper ( Expr Oper"); |
| ExpectToken(JsKeywords::kIdentifier, "c"); |
| ExpectParseStack("Start BkKwd ( Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectParseStack("Start BkKwd ( Expr Oper"); |
| ExpectToken(JsKeywords::kIdentifier, "d"); |
| ExpectParseStack("Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "<"); |
| ExpectParseStack("Start BkKwd ( Expr Oper"); |
| ExpectToken(JsKeywords::kIdentifier, "e"); |
| ExpectParseStack("Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkHdr"); |
| ExpectToken(JsKeywords::kRegex, "/d<e/"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectParseStack("Start BkHdr Expr ."); |
| ExpectToken(JsKeywords::kIdentifier, "exec"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start BkHdr Expr ("); |
| ExpectToken(JsKeywords::kStringLiteral, "'\\''"); |
| ExpectParseStack("Start BkHdr Expr ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kElse, "else"); |
| ExpectParseStack("Start BkHdr"); |
| ExpectToken(JsKeywords::kRegex, "/x/"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectParseStack("Start BkHdr Expr ."); |
| ExpectToken(JsKeywords::kIdentifier, "exec"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectParseStack("Start BkHdr Expr ("); |
| ExpectToken(JsKeywords::kStringLiteral, "\"\""); |
| ExpectParseStack("Start BkHdr Expr ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectParseStack("Start BkHdr Expr"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectParseStack("Start"); |
| ExpectEndOfInput(); |
| ExpectParseStack(""); |
| } |
| |
| TEST_F(JsTokenizerTest, Operators1) { |
| BeginTokenizing("foo /= bar+++baz;\n" |
| "a=b==c&&d===e;\n" |
| "a>>>=b>=c?d>>_:$;"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "/="); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "baz"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kOperator, "=="); |
| ExpectToken(JsKeywords::kIdentifier, "c"); |
| ExpectToken(JsKeywords::kOperator, "&&"); |
| ExpectToken(JsKeywords::kIdentifier, "d"); |
| ExpectToken(JsKeywords::kOperator, "==="); |
| ExpectToken(JsKeywords::kIdentifier, "e"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, ">>>="); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kOperator, ">="); |
| ExpectToken(JsKeywords::kIdentifier, "c"); |
| ExpectToken(JsKeywords::kOperator, "?"); |
| ExpectToken(JsKeywords::kIdentifier, "d"); |
| ExpectToken(JsKeywords::kOperator, ">>"); |
| ExpectToken(JsKeywords::kIdentifier, "_"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kIdentifier, "$"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Operators2) { |
| BeginTokenizing("a+++b\n" |
| "a+ ++b\n" |
| "a+ +b\n" |
| "a---b\n" |
| "a-++b\n" |
| "!!b\n"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "--"); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kOperator, "!"); |
| ExpectToken(JsKeywords::kOperator, "!"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Colons1) { |
| // Each of the three lines below contains the substring ":{}/x/i". However, |
| // in the first two lines that's a label colon, followed by an empty block, |
| // followed by a regex literal, while in the third line it's an ternary |
| // operator, followed by an empty object literal, followed by division. |
| BeginTokenizing("switch(x){default:{}/x/i}\n" |
| "foobar:{}/x/i;\n" |
| "a?b:{}/x/i"); |
| ExpectToken(JsKeywords::kSwitch, "switch"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kDefault, "default"); |
| ExpectParseStack("Start BkHdr { Other"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectParseStack("Start BkHdr {"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kRegex, "/x/i"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "foobar"); |
| ExpectParseStack("Start Expr"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kRegex, "/x/i"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "?"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectParseStack("Start Expr ? Expr"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectParseStack("Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "i"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Colons2) { |
| // Each of the three lines below contains the substring "{foo:{}/x/i}". In |
| // the first line the outer braces are a block, so the inside is a label, |
| // followed by an empty block, followed by a regex literal. in the second |
| // line the outer braces are an object literal, so the inside is an object |
| // property equal to an empty object literal divided by some other values. |
| // In the third line the outer braces are again a block (even though they're |
| // part of an expression), so the inside is again a label/block/regex. |
| BeginTokenizing("{foo:{}/x/i}\n" |
| "y={foo:{}/x/i}\n" |
| "z=function(){foo:{}/x/i}"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kRegex, "/x/i"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "i"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "z"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kFunction, "function"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kRegex, "/x/i"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ObjectLiteralsArray) { |
| BeginTokenizing("[{a:42},{a:32}]"); |
| ExpectToken(JsKeywords::kOperator, "[", "Start ["); |
| ExpectToken(JsKeywords::kOperator, "{", "Start [ {"); |
| ExpectToken(JsKeywords::kIdentifier, "a", "Start [ { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":", "Start [ { Oper"); |
| ExpectToken(JsKeywords::kNumber, "42", "Start [ { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start [ Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start ["); |
| ExpectToken(JsKeywords::kOperator, "{", "Start [ {"); |
| ExpectToken(JsKeywords::kIdentifier, "a", "Start [ { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":", "Start [ { Oper"); |
| ExpectToken(JsKeywords::kNumber, "32", "Start [ { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start [ Expr"); |
| ExpectToken(JsKeywords::kOperator, "]", "Start Expr"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, TrailingCommas) { |
| BeginTokenizing("x={a:1,}\n" |
| "y=[,,]"); |
| ExpectToken(JsKeywords::kIdentifier, "x", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start Expr Oper {"); |
| ExpectToken(JsKeywords::kIdentifier, "a", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ":", "Start Expr Oper { Oper"); |
| ExpectToken(JsKeywords::kNumber, "1", "Start Expr Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Expr Oper {"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n", "Start"); |
| ExpectToken(JsKeywords::kIdentifier, "y", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "[", "Start Expr Oper ["); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Expr Oper ["); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Expr Oper ["); |
| ExpectToken(JsKeywords::kOperator, "]", "Start Expr"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CommaOperator) { |
| BeginTokenizing("x=(y++,y*5)"); |
| ExpectToken(JsKeywords::kIdentifier, "x", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "(", "Start Expr Oper ("); |
| ExpectToken(JsKeywords::kIdentifier, "y", "Start Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "++", "Start Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Expr Oper ( Expr Oper"); |
| ExpectToken(JsKeywords::kIdentifier, "y", "Start Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, "*", "Start Expr Oper ( Expr Oper"); |
| ExpectToken(JsKeywords::kNumber, "5", "Start Expr Oper ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")", "Start Expr"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ObjectLiteralRegexLiteral) { |
| // On the first line, this looks like it should be an object literal divided |
| // by x divided by i, but nope, that's a block with a labelled expression |
| // statement, followed by a regex literal. The second line, on the other |
| // hand, _is_ an object literal, followed by division. |
| BeginTokenizing("{foo:1} / x/i;\n" |
| "x={foo:1} / x/i;"); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kRegex, "/ x/i"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kOperator, "{"); |
| ExpectToken(JsKeywords::kIdentifier, "foo"); |
| ExpectToken(JsKeywords::kOperator, ":"); |
| ExpectToken(JsKeywords::kNumber, "1"); |
| ExpectToken(JsKeywords::kOperator, "}"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "i"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, EmptyBlockRegexLiteral) { |
| BeginTokenizing("if(true){{}/foo/}"); |
| ExpectToken(JsKeywords::kIf, "if", "Start BkKwd"); |
| ExpectToken(JsKeywords::kOperator, "(", "Start BkKwd ("); |
| ExpectToken(JsKeywords::kTrue, "true", "Start BkKwd ( Expr"); |
| ExpectToken(JsKeywords::kOperator, ")", "Start BkHdr"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start BkHdr {"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start BkHdr { {"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start BkHdr {"); |
| ExpectToken(JsKeywords::kRegex, "/foo/", "Start BkHdr { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, TrickyRegexLiteral) { |
| BeginTokenizing("var x=a[0] / b /i;\n" |
| "var y=a[0]+/ b /i;"); |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "["); |
| ExpectToken(JsKeywords::kNumber, "0"); |
| ExpectToken(JsKeywords::kOperator, "]"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kIdentifier, "i"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "["); |
| ExpectToken(JsKeywords::kNumber, "0"); |
| ExpectToken(JsKeywords::kOperator, "]"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kRegex, "/ b /i"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, RegexLiteralsWithBrackets) { |
| BeginTokenizing("/http:\\/\\/[^/]+\\// / /z[\\]/ ]/"); |
| // The / in [^/] doesn't end the regex. |
| ExpectToken(JsKeywords::kRegex, "/http:\\/\\/[^/]+\\//"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "/"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| // The first ] is escaped and doesn't close the [, so the following / doesn't |
| // close the regex. |
| ExpectToken(JsKeywords::kRegex, "/z[\\]/ ]/"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ReturnRegex) { |
| // Make sure we understand that this is not division; "return" is not an |
| // identifier! |
| BeginTokenizing("return / x /g;\n" |
| "return/#.+/.test(\n'#24' );"); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kRegex, "/ x /g"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectToken(JsKeywords::kRegex, "/#.+/"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "test"); |
| ExpectToken(JsKeywords::kOperator, "("); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kStringLiteral, "'#24'"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, ")"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ReturnRegex2) { |
| // Make sure we understand that this is not division; "return" is not an |
| // identifier! |
| BeginTokenizing("return / x /g;"); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kRegex, "/ x /g"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, ThrowRegex) { |
| // Make sure we understand that this is not division; "throw" is not an |
| // identifier! (And yes, in JS you're allowed to throw a regex.) |
| BeginTokenizing("throw / x /g;"); |
| ExpectToken(JsKeywords::kThrow, "throw"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kRegex, "/ x /g"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, UnicodeRegexFlags) { |
| // The Look Of Disapproval emoticon is probably not a semantically valid |
| // regex flag, but it is lexically valid, so we should be able to tokenize |
| // it. |
| BeginTokenizing("/\xE2\x98\x83/\xE0\xB2\xA0_\xE0\xB2\xA0\xE2\x80\xA9;"); |
| ExpectToken(JsKeywords::kRegex, "/\xE2\x98\x83/\xE0\xB2\xA0_\xE0\xB2\xA0"); |
| ExpectToken(JsKeywords::kLineSeparator, "\xE2\x80\xA9"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, SemicolonInsertion1) { |
| BeginTokenizing("3\n" // Semicolon is not inserted here. |
| "// foo\n" |
| "--> foo\n" |
| "-5\n" // Semicolon is inserted here. |
| "// bar\n" |
| "6"); |
| ExpectToken(JsKeywords::kNumber, "3"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kComment, "// foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kComment, "--> foo"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kNumber, "5"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| ExpectToken(JsKeywords::kComment, "// bar"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kNumber, "6"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, SemicolonInsertion2) { |
| BeginTokenizing("w\n" // Semicolon is inserted here... |
| "++\n" // ...but not here. |
| "x\n" // Semicolon inserted here again. |
| "y++\n" // And here again. |
| "z"); |
| ExpectToken(JsKeywords::kIdentifier, "w", "Start Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n", "Start"); |
| ExpectToken(JsKeywords::kOperator, "++", "Start Oper"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n", "Start Oper"); |
| ExpectToken(JsKeywords::kIdentifier, "x", "Start Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n", "Start"); |
| ExpectToken(JsKeywords::kIdentifier, "y", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "++", "Start Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n", "Start"); |
| ExpectToken(JsKeywords::kIdentifier, "z", "Start Expr"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, SemicolonInsertion3) { |
| BeginTokenizing("x\nin\xE1\x9A\x80y;\n" // U+1680 OGHAM SPACE MARK |
| "x\nin\\u0063\ny;\n" |
| "x\ninstanceof\ny;\n" |
| "x\ninstanceof\xE0\xB2\xA0_\n" // U+0CA0 KANNADA LETTER TTHA |
| "y;"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kIn, "in"); |
| ExpectToken(JsKeywords::kWhitespace, "\xE1\x9A\x80"); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| ExpectToken(JsKeywords::kIdentifier, "in\\u0063"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kInstanceof, "instanceof"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| ExpectToken(JsKeywords::kIdentifier, "instanceof\xE0\xB2\xA0_"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n"); |
| ExpectToken(JsKeywords::kIdentifier, "y"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, SemicolonInsertion4) { |
| BeginTokenizing("x={}\n" |
| "{debugger}"); |
| ExpectToken(JsKeywords::kIdentifier, "x", "Start Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Expr Oper"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start Expr Oper {"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start Expr"); |
| ExpectToken(JsKeywords::kSemiInsert, "\n", "Start"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start {"); |
| ExpectToken(JsKeywords::kDebugger, "debugger", "Start { Jump"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, UnclosedBlockComment) { |
| BeginTokenizing("debugger; /* foo"); |
| ExpectToken(JsKeywords::kDebugger, "debugger"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectError("/* foo"); |
| } |
| |
| TEST_F(JsTokenizerTest, UnclosedRegexLiteral) { |
| BeginTokenizing("var bar=/quux;"); |
| ExpectParseStack("Start"); |
| ExpectToken(JsKeywords::kVar, "var", "Start Other"); |
| ExpectToken(JsKeywords::kWhitespace, " ", "Start Other"); |
| ExpectToken(JsKeywords::kIdentifier, "bar", "Start Other Expr"); |
| ExpectToken(JsKeywords::kOperator, "=", "Start Other Expr Oper"); |
| ExpectError("/quux;"); |
| ExpectParseStack("Start Other Expr Oper"); |
| } |
| |
| TEST_F(JsTokenizerTest, LinebreakInRegex) { |
| // Regexes cannot contain linebreaks. |
| BeginTokenizing("x=/foo\nquux/;"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectError("/foo\nquux/;"); |
| |
| // They can contain Unicode characters, but not Unicode linebreaks. |
| BeginTokenizing("x=/foo\xE2\x98\x83quux/+" // U+2603 SNOWMAN |
| "/foo\xE2\x80\xA9quux/;"); // U+2029 PARAGRAPH SEPARATOR |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kRegex, "/foo\xE2\x98\x83quux/"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectError("/foo\xE2\x80\xA9quux/;"); |
| |
| // Unlike in strings, newlines in regexes cannot be escaped by backslashes. |
| BeginTokenizing("x=/foo\\\nquux/;"); |
| ExpectToken(JsKeywords::kIdentifier, "x"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectError("/foo\\\nquux/;"); |
| } |
| |
| TEST_F(JsTokenizerTest, LinebreakInStringLiteral) { |
| // Strings can contain linebreaks only if escaped by a backslash. |
| BeginTokenizing("'foo\\\nquux'+" |
| // A CRLF counts as one linebreak. |
| "\"foo\\\r\nquux\"+" |
| // Apparently, so does LFCR, for some reason. |
| "\"foo\\\n\rquux\"+" |
| // But two LFs are two linebreaks! |
| "'foo\\\n\nquux';"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\\\nquux'"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"foo\\\r\nquux\""); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"foo\\\n\rquux\""); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectError("'foo\\\n\nquux';"); |
| |
| // Strings can contain Unicode characters. |
| BeginTokenizing("'foo\xE2\x98\x83quux'+" // U+2603 SNOWMAN |
| // Unicode linebreaks are allowed if backslash-escaped. |
| "'foo\\\xE2\x80\xA8quux'+" // U+2028 LINE SEPARATOR |
| // But it's an error if the Unicode linebreak isn't escaped! |
| "'foo\xE2\x80\xA8quux';"); // U+2028 LINE SEPARATOR |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\xE2\x98\x83quux'"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\\\xE2\x80\xA8quux'"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectError("'foo\xE2\x80\xA8quux';"); |
| } |
| |
| TEST_F(JsTokenizerTest, EscapedQuotesInStringLiteral1) { |
| BeginTokenizing("'foo\\\\\\'bar';"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\\\\\\'bar'"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, EscapedQuotesInStringLiteral2) { |
| BeginTokenizing("\"baz\"+'foo\\\\\\'ba\"r';"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"baz\""); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\\\\\\'ba\"r'"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, EscapedQuotesInStringLiteral3) { |
| BeginTokenizing("'b\\\\az'+'foo\\'bar';"); |
| ExpectToken(JsKeywords::kStringLiteral, "'b\\\\az'"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foo\\'bar'"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, EscapedQuotesInStringLiteral4) { |
| BeginTokenizing("'b\\'az'+'foobar';"); |
| ExpectToken(JsKeywords::kStringLiteral, "'b\\'az'"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kStringLiteral, "'foobar'"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, EscapedQuotesInStringLiteral5) { |
| BeginTokenizing("\"f\xFFoo\\\\\\\"bar\";"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"f\xFFoo\\\\\\\"bar\""); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, UnclosedStringLiteral) { |
| BeginTokenizing("bar='quux;"); |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectError("'quux;"); |
| } |
| |
| TEST_F(JsTokenizerTest, UnmatchedCloseParen) { |
| BeginTokenizing("bar='quux');"); |
| ExpectToken(JsKeywords::kIdentifier, "bar"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kStringLiteral, "'quux'"); |
| ExpectError(");"); |
| } |
| |
| TEST_F(JsTokenizerTest, BogusInputCharacter) { |
| BeginTokenizing("var #foo;"); |
| ExpectToken(JsKeywords::kVar, "var"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectError("#foo;"); |
| } |
| |
| TEST_F(JsTokenizerTest, BackslashesInIdentifier) { |
| BeginTokenizing("a\\u03c0b"); |
| ExpectToken(JsKeywords::kIdentifier, "a\\u03c0b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, BackslashesInString) { |
| BeginTokenizing("\"a\\\"b\""); |
| ExpectToken(JsKeywords::kStringLiteral, "\"a\\\"b\""); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombinePluses) { |
| BeginTokenizing("a+++b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombinePluses2) { |
| BeginTokenizing("a+ ++b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombinePlusesSpace) { |
| BeginTokenizing("a+ +b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombineMinuses) { |
| BeginTokenizing("a---b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "--"); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombineMixed) { |
| BeginTokenizing("a--+b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "--"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| |
| } |
| |
| TEST_F(JsTokenizerTest, CombineMixed2) { |
| BeginTokenizing("a-++b"); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "-"); |
| ExpectToken(JsKeywords::kOperator, "++"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, CombineBangs) { |
| BeginTokenizing("!!b"); |
| ExpectToken(JsKeywords::kOperator, "!"); |
| ExpectToken(JsKeywords::kOperator, "!"); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, NumbersAndDotsAndIdentifiersAndKeywords) { |
| BeginTokenizing("return a.b+5.3"); |
| ExpectToken(JsKeywords::kReturn, "return"); |
| ExpectToken(JsKeywords::kWhitespace, " "); |
| ExpectToken(JsKeywords::kIdentifier, "a"); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "b"); |
| ExpectToken(JsKeywords::kOperator, "+"); |
| ExpectToken(JsKeywords::kNumber, "5.3"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, NumberProperty) { |
| BeginTokenizing("1..property"); |
| ExpectToken(JsKeywords::kNumber, "1."); |
| ExpectToken(JsKeywords::kOperator, "."); |
| ExpectToken(JsKeywords::kIdentifier, "property"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, LineCommentAtEndOfInput) { |
| BeginTokenizing("hello//world"); |
| ExpectToken(JsKeywords::kIdentifier, "hello"); |
| ExpectToken(JsKeywords::kComment, "//world"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Latin1BlockComment) { |
| // Try to tokenize input that is Latin-1 encoded. This is not valid UTF-8, |
| // but we should be able to proceed gracefully (in most cases) if the |
| // non-ascii characters only ever appear in string literals and comments. |
| BeginTokenizing("/* qu\xE9 pasa */\n"); |
| ExpectToken(JsKeywords::kComment, "/* qu\xE9 pasa */"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Latin1LineComment) { |
| // Try to tokenize input that is Latin-1 encoded. This is not valid UTF-8, |
| // but we should be able to proceed gracefully (in most cases) if the |
| // non-ascii characters only ever appear in string literals and comments. |
| BeginTokenizing("// qu\xE9 pasa\n"); |
| ExpectToken(JsKeywords::kComment, "// qu\xE9 pasa"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Latin1StringLiteral) { |
| // Try to tokenize input that is Latin-1 encoded. This is not valid UTF-8, |
| // but we should be able to proceed gracefully (in most cases) if the |
| // non-ascii characters only ever appear in string literals and comments. |
| BeginTokenizing("\"qu\xE9 pasa\"\n"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"qu\xE9 pasa\""); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| |
| // An example with more complicated escaping: |
| BeginTokenizing("'\xAA\\'\xBB\\\r\n\xCC'\n"); |
| ExpectToken(JsKeywords::kStringLiteral, "'\xAA\\'\xBB\\\r\n\xCC'"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, Latin1Input) { |
| // Try to tokenize input that is Latin-1 encoded. This is not valid UTF-8, |
| // but we should be able to proceed gracefully (in most cases) if the |
| // non-ascii characters only ever appear in string literals and comments. |
| BeginTokenizing("str='Qu\xE9 pasa';// 'qu\xE9' means 'what'\n" |
| "cents=/* 73\xA2 is $0.73 */73;"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "str"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kStringLiteral, "'Qu\xE9 pasa'"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectToken(JsKeywords::kComment, "// 'qu\xE9' means 'what'"); |
| ExpectToken(JsKeywords::kLineSeparator, "\n"); |
| |
| ExpectToken(JsKeywords::kIdentifier, "cents"); |
| ExpectToken(JsKeywords::kOperator, "="); |
| ExpectToken(JsKeywords::kComment, "/* 73\xA2 is $0.73 */"); |
| ExpectToken(JsKeywords::kNumber, "73"); |
| ExpectToken(JsKeywords::kOperator, ";"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, JsonHeuristic) { |
| // Sometimes we put JSON data through the JavaScript tokenizer. Most JSON |
| // will parse just fine, but JSON object literals, without parse context to |
| // mark them as expressions (rather than code blocks) will generally not |
| // parse as JavaScript code. Therefore, the tokenizer has a heuristic to |
| // recognize input consisting of a JSON object literal and alter the parse |
| // state to handle it. |
| BeginTokenizing("{\"foo\":{\"bar\":1},\"baz\":2}"); |
| // At first, we assume this is JS code, as usual: |
| ExpectToken(JsKeywords::kOperator, "{", "Start {"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"foo\"", "Start { Expr"); |
| // Once we see the colon, we know this is actually a JSON object literal |
| // rather than a code block (a string literal followed by a colon isn't valid |
| // syntax at start-of-statement). Adding a synthetic Oper parse state in |
| // front of the { state allows us to treat this as an object literal rather |
| // than a code block. From here we can proceed as normal. |
| ExpectToken(JsKeywords::kOperator, ":", "Start Oper { Oper"); |
| ExpectToken(JsKeywords::kOperator, "{", "Start Oper { Oper {"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"bar\""); |
| ExpectToken(JsKeywords::kOperator, ":", "Start Oper { Oper { Oper"); |
| ExpectToken(JsKeywords::kNumber, "1", "Start Oper { Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, ",", "Start Oper {"); |
| ExpectToken(JsKeywords::kStringLiteral, "\"baz\""); |
| ExpectToken(JsKeywords::kOperator, ":", "Start Oper { Oper"); |
| ExpectToken(JsKeywords::kNumber, "2", "Start Oper { Expr"); |
| ExpectToken(JsKeywords::kOperator, "}", "Start Expr"); |
| ExpectEndOfInput(); |
| } |
| |
| TEST_F(JsTokenizerTest, TokenizeAngular) { |
| ExpectTokenizeFileSuccessfully("angular.original"); |
| } |
| |
| TEST_F(JsTokenizerTest, TokenizeJQuery) { |
| ExpectTokenizeFileSuccessfully("jquery.original"); |
| } |
| |
| TEST_F(JsTokenizerTest, TokenizePrototype) { |
| ExpectTokenizeFileSuccessfully("prototype.original"); |
| } |
| |
| } // namespace |
