| /* |
| * Copyright 2010 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| // Author: jmaessen@google.com (Jan Maessen) |
| |
| #include "net/instaweb/rewriter/public/javascript_code_block.h" |
| |
| #include "net/instaweb/rewriter/public/javascript_library_identification.h" |
| #include "pagespeed/kernel/base/google_message_handler.h" |
| #include "pagespeed/kernel/base/gtest.h" |
| #include "pagespeed/kernel/base/md5_hasher.h" |
| #include "pagespeed/kernel/base/scoped_ptr.h" |
| #include "pagespeed/kernel/base/statistics.h" |
| #include "pagespeed/kernel/base/string_util.h" |
| #include "pagespeed/kernel/base/thread_system.h" |
| #include "pagespeed/kernel/js/js_tokenizer.h" |
| #include "pagespeed/kernel/util/platform.h" |
| #include "pagespeed/kernel/util/simple_stats.h" |
| |
| namespace net_instaweb { |
| |
| namespace { |
| |
| // This sample code comes from Douglas Crockford's jsmin example. |
| // The same code is used to test jsminify in pagespeed. |
| // We've added some leading and trailing whitespace here just to |
| // test our treatment of those cases (we used to erase this stuff |
| // even if the file wasn't minifiable). |
| const char kBeforeCompilation[] = |
| " \n" |
| "// is.js\n" |
| "\n" |
| "// (c) 2001 Douglas Crockford\n" |
| "// 2001 June 3\n" |
| "\n" |
| "\n" |
| "// is\n" |
| "\n" |
| "// The -is- object is used to identify the browser. " |
| "Every browser edition\n" |
| "// identifies itself, but there is no standard way of doing it, " |
| "and some of\n" |
| "// the identification is deceptive. This is because the authors of web\n" |
| "// browsers are liars. For example, Microsoft's IE browsers claim to be\n" |
| "// Mozilla 4. Netscape 6 claims to be version 5.\n" |
| "\n" |
| "var is = {\n" |
| " ie: navigator.appName == 'Microsoft Internet Explorer',\n" |
| " java: navigator.javaEnabled(),\n" |
| " ns: navigator.appName == 'Netscape',\n" |
| " ua: navigator.userAgent.toLowerCase(),\n" |
| " version: parseFloat(navigator.appVersion.substr(21)) ||\n" |
| " parseFloat(navigator.appVersion),\n" |
| " win: navigator.platform == 'Win32'\n" |
| "}\n" |
| "is.mac = is.ua.indexOf('mac') >= 0;\n" |
| "if (is.ua.indexOf('opera') >= 0) {\n" |
| " is.ie = is.ns = false;\n" |
| " is.opera = true;\n" |
| "}\n" |
| "if (is.ua.indexOf('gecko') >= 0) {\n" |
| " is.ie = is.ns = false;\n" |
| " is.gecko = true;\n" |
| "}\n" |
| " \n"; |
| |
| const char kLibraryUrl[] = "//example.com/test_library.js"; |
| |
| const char kTruncatedComment[] = |
| "// is.js\n" |
| "\n" |
| "// (c) 2001 Douglas Crockford\n" |
| "// 2001 June 3\n" |
| "\n" |
| "\n" |
| "// is\n" |
| "\n" |
| "/* The -is- object is used to identify the browser. " |
| "Every browser edition\n" |
| " identifies itself, but there is no standard way of doing it, " |
| "and some of\n"; |
| |
| // Again we add some leading whitespace here to check for handling of this issue |
| // in otherwise non-minifiable code. We've elected not to strip the whitespace. |
| const char kTruncatedString[] = |
| " \n" |
| "var is = {\n" |
| " ie: navigator.appName == 'Microsoft Internet Explo"; |
| |
| const char kAfterCompilationOld[] = |
| "var is={ie:navigator.appName=='Microsoft Internet Explorer'," |
| "java:navigator.javaEnabled(),ns:navigator.appName=='Netscape'," |
| "ua:navigator.userAgent.toLowerCase(),version:parseFloat(" |
| "navigator.appVersion.substr(21))||parseFloat(navigator.appVersion)" |
| ",win:navigator.platform=='Win32'}\n" |
| "is.mac=is.ua.indexOf('mac')>=0;if(is.ua.indexOf('opera')>=0){" |
| "is.ie=is.ns=false;is.opera=true;}\n" // Note trailing \n |
| "if(is.ua.indexOf('gecko')>=0){is.ie=is.ns=false;is.gecko=true;}"; |
| |
| const char kAfterCompilationNew[] = |
| "var is={ie:navigator.appName=='Microsoft Internet Explorer'," |
| "java:navigator.javaEnabled(),ns:navigator.appName=='Netscape'," |
| "ua:navigator.userAgent.toLowerCase(),version:parseFloat(" |
| "navigator.appVersion.substr(21))||parseFloat(navigator.appVersion)" |
| ",win:navigator.platform=='Win32'}\n" |
| "is.mac=is.ua.indexOf('mac')>=0;if(is.ua.indexOf('opera')>=0){" |
| "is.ie=is.ns=false;is.opera=true;}" // Note lack of trailing \n |
| "if(is.ua.indexOf('gecko')>=0){is.ie=is.ns=false;is.gecko=true;}"; |
| |
| const char kJsWithGetElementsByTagNameScript[] = |
| "// this shouldn't be altered" |
| " var scripts = document.getElementsByTagName('script')," |
| " script = scripts[scripts.length - 1];" |
| " var some_url = document.createElement(\"a\");"; |
| |
| const char kJsWithJQueryScriptElementSelection[] = |
| "// this shouldn't be altered either" |
| " var scripts = $(\"script\")," |
| " script = scripts[scripts.length - 1];" |
| " var some_url = document.createElement(\"a\");"; |
| |
| const char kBogusLibraryMD5[] = "ltVVzzYxo0"; |
| |
| const char kBogusLibraryUrl[] = |
| "//www.example.com/js/bogus_library.js"; |
| |
| // Sample JSON code from http://json.org/example with tons of whitespace. |
| // Modified to include even more whitespace between special characters and |
| // in string values/keys. |
| const char kJsonBeforeCompilation[] = |
| "\n\n{\n" |
| " \"glossary \": {\n" |
| " \"title\": 'example glossary',\n" |
| "\t\t \"GlossDiv\": {\n" |
| " \"title\": \"S\",\n" |
| "\t\t\t\"GlossList\" : {\n" |
| " \"GlossEntry\": {\n" |
| " \"ID\": \"SGML\" ,\t\n" |
| "\t\t\t\t\t\t\"SortAs\": \"SGML\",\n" |
| "\t\t\t\t\t\t\t\t\"GlossTerm\": \"Standard Generalized Markup Language\",\n" |
| "\t\t\t\t\t\t\t\t\t\t\t \t \t\t \t \"Acronym\": \"SGML\",\n" |
| "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \"Abbrev\": \"ISO 8879:1986\",\n" |
| "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \"GlossDef\": {\n" |
| " \"para\": \"A meta-markup language, used to create" |
| " markup languages such as DocBook.\",\n" |
| "\t\t\t\t \t \t\t \"GlossSeeAlso\": [\"GML\", \"XML\"]\n" |
| " },\n" |
| "\t\t\t\t\t\t\"GlossSee\": \"markup\"\n" |
| " }\n" |
| " }\n" |
| " }\n" |
| " }\n" |
| "}\n\n\n"; |
| |
| const char kJsonAfterCompilation[] = |
| "{\"glossary \":{\"title\":'example glossary',\"GlossDiv\":{\"title\":" |
| "\"S\",\"GlossList\":{\"GlossEntry\":{\"ID\":\"SGML\",\"SortAs\":\"SGML\"," |
| "\"GlossTerm\":\"Standard Generalized Markup Language\",\"Acronym\":" |
| "\"SGML\",\"Abbrev\":\"ISO 8879:1986\",\"GlossDef\":{\"para\":\"A " |
| "meta-markup language, used to create markup languages such as DocBook.\"," |
| "\"GlossSeeAlso\":[\"GML\",\"XML\"]},\"GlossSee\":\"markup\"}}}}}"; |
| |
| class JsCodeBlockTest : public ::testing::Test, |
| public ::testing::WithParamInterface<bool> { |
| protected: |
| JsCodeBlockTest() |
| : thread_system_(Platform::CreateThreadSystem()), |
| stats_(thread_system_.get()), |
| use_experimental_minifier_(GetParam()), |
| after_compilation_(use_experimental_minifier_ |
| ? kAfterCompilationNew |
| : kAfterCompilationOld) { |
| JavascriptRewriteConfig::InitStats(&stats_); |
| config_.reset(new JavascriptRewriteConfig( |
| &stats_, true, use_experimental_minifier_, &libraries_, |
| &js_tokenizer_patterns_)); |
| // Register a bogus library with a made-up md5 and plausible canonical url |
| // that doesn't occur in our tests, but has the same size as our canonical |
| // test case. |
| EXPECT_TRUE(libraries_.RegisterLibrary(strlen(after_compilation_), |
| kBogusLibraryMD5, kBogusLibraryUrl)); |
| } |
| |
| void ExpectStats(int blocks_minified, int minification_failures, |
| int total_bytes_saved, int total_original_bytes, |
| int num_reducing_uses) { |
| EXPECT_EQ(blocks_minified, config_->blocks_minified()->Get()); |
| EXPECT_EQ(minification_failures, config_->minification_failures()->Get()); |
| EXPECT_EQ(total_bytes_saved, config_->total_bytes_saved()->Get()); |
| EXPECT_EQ(total_original_bytes, config_->total_original_bytes()->Get()); |
| EXPECT_EQ(num_reducing_uses, config_->num_reducing_uses()->Get()); |
| // Note: We cannot compare num_uses() because we only use it in |
| // javascript_filter.cc, not javascript_code_block.cc. |
| } |
| |
| void DisableMinification() { |
| config_.reset(new JavascriptRewriteConfig( |
| &stats_, false, use_experimental_minifier_, &libraries_, |
| &js_tokenizer_patterns_)); |
| } |
| |
| // Must be called after DisableMinification if we call both. |
| void DisableLibraryIdentification() { |
| config_.reset(new JavascriptRewriteConfig( |
| &stats_, config_->minify(), use_experimental_minifier_, NULL, |
| &js_tokenizer_patterns_)); |
| } |
| |
| void RegisterLibrariesIn(JavascriptLibraryIdentification* libs) { |
| MD5Hasher md5(JavascriptLibraryIdentification::kNumHashChars); |
| GoogleString after_md5 = md5.Hash(after_compilation_); |
| EXPECT_TRUE(libs->RegisterLibrary(strlen(after_compilation_), |
| after_md5, kLibraryUrl)); |
| EXPECT_EQ(JavascriptLibraryIdentification::kNumHashChars, |
| after_md5.size()); |
| } |
| |
| void RegisterLibraries() { |
| RegisterLibrariesIn(&libraries_); |
| } |
| |
| JavascriptCodeBlock* TestBlock(StringPiece code) { |
| return new JavascriptCodeBlock(code, config_.get(), "Test", &handler_); |
| } |
| |
| void SingleBlockRewriteTest(const char* before_compilation, |
| const char* after_compilation) { |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(before_compilation)); |
| EXPECT_TRUE(block->Rewrite()); |
| EXPECT_TRUE(block->successfully_rewritten()); |
| EXPECT_EQ(after_compilation, block->rewritten_code()); |
| ExpectStats(1, 0, |
| strlen(before_compilation) - strlen(after_compilation), |
| strlen(before_compilation), 1); |
| } |
| |
| GoogleMessageHandler handler_; |
| scoped_ptr<ThreadSystem> thread_system_; |
| SimpleStats stats_; |
| JavascriptLibraryIdentification libraries_; |
| const pagespeed::js::JsTokenizerPatterns js_tokenizer_patterns_; |
| scoped_ptr<JavascriptRewriteConfig> config_; |
| |
| const bool use_experimental_minifier_; |
| const char* after_compilation_; |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(JsCodeBlockTest); |
| }; |
| |
| TEST_P(JsCodeBlockTest, Config) { |
| EXPECT_TRUE(config_->minify()); |
| ExpectStats(0, 0, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, Rewrite) { |
| SingleBlockRewriteTest(kBeforeCompilation, after_compilation_); |
| } |
| |
| TEST_P(JsCodeBlockTest, RewriteNoIdentification) { |
| // Make sure library identification setting doesn't change minification. |
| DisableLibraryIdentification(); |
| SingleBlockRewriteTest(kBeforeCompilation, after_compilation_); |
| } |
| |
| TEST_P(JsCodeBlockTest, UnsafeToRename) { |
| EXPECT_TRUE(JavascriptCodeBlock::UnsafeToRename( |
| kJsWithGetElementsByTagNameScript)); |
| EXPECT_TRUE(JavascriptCodeBlock::UnsafeToRename( |
| kJsWithJQueryScriptElementSelection)); |
| EXPECT_FALSE(JavascriptCodeBlock::UnsafeToRename( |
| kBeforeCompilation)); |
| } |
| |
| TEST_P(JsCodeBlockTest, NoRewrite) { |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(after_compilation_)); |
| EXPECT_FALSE(block->Rewrite()); |
| // Note: Minifier succeeded, but no minification was applied and thus |
| // no bytes saved (nor original bytes marked). |
| ExpectStats(1, 0, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, TruncatedComment) { |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kTruncatedComment)); |
| EXPECT_FALSE(block->Rewrite()); |
| ExpectStats(0, 1, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, TruncatedString) { |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kTruncatedString)); |
| EXPECT_FALSE(block->Rewrite()); |
| ExpectStats(0, 1, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, NoMinification) { |
| DisableMinification(); |
| DisableLibraryIdentification(); |
| EXPECT_FALSE(config_->minify()); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kBeforeCompilation)); |
| EXPECT_FALSE(block->Rewrite()); |
| ExpectStats(0, 0, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, DealWithSgmlComment) { |
| // Based on actual code seen in the wild; the surprising part is this works at |
| // all (due to xhtml in the source document)! |
| static const char kOriginal[] = " <!-- \nvar x = 1;\n //--> "; |
| static const char kExpected[] = "var x=1;"; |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kOriginal)); |
| EXPECT_TRUE(block->Rewrite()); |
| EXPECT_EQ(kExpected, block->rewritten_code()); |
| ExpectStats(1, 0, |
| STATIC_STRLEN(kOriginal) - STATIC_STRLEN(kExpected), |
| STATIC_STRLEN(kOriginal), 1); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyUnminified) { |
| RegisterLibraries(); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kBeforeCompilation)); |
| block->Rewrite(); |
| EXPECT_EQ(kLibraryUrl, block->ComputeJavascriptLibrary()); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyMerged) { |
| JavascriptLibraryIdentification other_libraries; |
| RegisterLibrariesIn(&other_libraries); |
| libraries_.Merge(other_libraries); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kBeforeCompilation)); |
| block->Rewrite(); |
| EXPECT_EQ(kLibraryUrl, block->ComputeJavascriptLibrary()); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyMergedDuplicate) { |
| RegisterLibraries(); |
| JavascriptLibraryIdentification other_libraries; |
| RegisterLibrariesIn(&other_libraries); |
| libraries_.Merge(other_libraries); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kBeforeCompilation)); |
| block->Rewrite(); |
| EXPECT_EQ(kLibraryUrl, block->ComputeJavascriptLibrary()); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyMinified) { |
| RegisterLibraries(); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(after_compilation_)); |
| block->Rewrite(); |
| EXPECT_EQ(kLibraryUrl, block->ComputeJavascriptLibrary()); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyNoMinification) { |
| DisableMinification(); |
| RegisterLibraries(); |
| scoped_ptr<JavascriptCodeBlock> block(TestBlock(kBeforeCompilation)); |
| block->Rewrite(); |
| EXPECT_EQ(kLibraryUrl, block->ComputeJavascriptLibrary()); |
| EXPECT_FALSE(block->successfully_rewritten()); |
| ExpectStats(1, 0, 0, 0, 0); |
| } |
| |
| TEST_P(JsCodeBlockTest, IdentifyNoMatch) { |
| RegisterLibraries(); |
| scoped_ptr<JavascriptCodeBlock> block( |
| TestBlock(kJsWithGetElementsByTagNameScript)); |
| block->Rewrite(); |
| EXPECT_EQ("", block->ComputeJavascriptLibrary()); |
| } |
| |
| TEST_P(JsCodeBlockTest, LibrarySignature) { |
| RegisterLibraries(); |
| GoogleString signature; |
| libraries_.AppendSignature(&signature); |
| MD5Hasher md5(JavascriptLibraryIdentification::kNumHashChars); |
| GoogleString after_md5 = md5.Hash(after_compilation_); |
| GoogleString expected_signature = |
| StrCat("S:", Integer64ToString(strlen(after_compilation_)), |
| "_H:", after_md5, "_J:", kLibraryUrl, |
| StrCat("_H:", kBogusLibraryMD5, "_J:", kBogusLibraryUrl)); |
| EXPECT_EQ(expected_signature, signature); |
| } |
| |
| TEST_P(JsCodeBlockTest, RewriteJson) { |
| SingleBlockRewriteTest(kJsonBeforeCompilation, kJsonAfterCompilation); |
| } |
| |
| TEST_P(JsCodeBlockTest, InvalidJsonValidJs) { |
| // The JS minifier cannot detect invalid JSON which is also valid JS, so we |
| // expect this to work. |
| SingleBlockRewriteTest( |
| "{'foo': bar, baz :}", |
| "{'foo':bar,baz:}"); |
| } |
| |
| TEST_P(JsCodeBlockTest, BogusLibraryRegistration) { |
| RegisterLibraries(); |
| // Try to register a library with a bad md5 string. |
| EXPECT_FALSE(libraries_.RegisterLibrary(73, "@$%@^#&#$^!%@#$", |
| "//www.example.com/test.js")); |
| // Try to register a library with a bad url. |
| EXPECT_FALSE(libraries_.RegisterLibrary(47, kBogusLibraryMD5, |
| "totally://bogus.protocol/")); |
| EXPECT_FALSE(libraries_.RegisterLibrary(74, kBogusLibraryMD5, |
| "totally:bogus.protocol")); |
| |
| // Don't allow non-standard protocols either. |
| EXPECT_FALSE(libraries_.RegisterLibrary(138, kBogusLibraryMD5, |
| "mailto:johndoe@example.com")); |
| EXPECT_FALSE(libraries_.RegisterLibrary(150, kBogusLibraryMD5, |
| "ftp://www.example.com/test.js")); |
| EXPECT_FALSE(libraries_.RegisterLibrary(222, kBogusLibraryMD5, |
| "file:///etc/passwd")); |
| EXPECT_FALSE(libraries_.RegisterLibrary(234, kBogusLibraryMD5, |
| "data:text/plain,Hello-world")); |
| } |
| |
| // We test with use_experimental_minifier == GetParam() as both true and false. |
| INSTANTIATE_TEST_CASE_P(JsCodeBlockTestInstance, JsCodeBlockTest, |
| ::testing::Bool()); |
| |
| } // namespace |
| |
| } // namespace net_instaweb |