Google Git
Sign in
apache / incubator-pagespeed-debian / refs/heads/upstream / . / src / net / instaweb / rewriter / rewrite_driver_test.cc
blob: 430ffe2e0af1ea70c3a8e2fbcc2200b22f88687f [file] [log] [blame]
/*
* Copyright 2010 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jmarantz@google.com (Joshua D. Marantz)
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/counting_url_async_fetcher.h"
#include "net/instaweb/http/public/log_record.h"
#include "net/instaweb/http/public/logging_proto_impl.h"
#include "net/instaweb/http/public/mock_url_fetcher.h"
#include "net/instaweb/http/public/wait_url_async_fetcher.h"
#include "net/instaweb/rewriter/public/domain_lawyer.h"
#include "net/instaweb/rewriter/public/file_load_policy.h"
#include "net/instaweb/rewriter/public/mock_resource_callback.h"
#include "net/instaweb/rewriter/public/output_resource_kind.h"
#include "net/instaweb/rewriter/public/request_properties.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/rewriter/public/resource_slot.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/rewrite_test_base.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/single_rewrite_context.h"
#include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/test_url_namer.h"
#include "pagespeed/kernel/base/abstract_mutex.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/gtest.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/mock_message_handler.h"
#include "pagespeed/kernel/base/mock_timer.h"
#include "pagespeed/kernel/base/null_mutex.h"
#include "pagespeed/kernel/base/sha1_signature.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/cache/lru_cache.h"
#include "pagespeed/kernel/html/empty_html_filter.h"
#include "pagespeed/kernel/html/html_name.h"
#include "pagespeed/kernel/html/html_parse_test_base.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/http_options.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/semantic_type.h"
#include "pagespeed/kernel/http/user_agent_matcher_test_base.h"
#include "pagespeed/kernel/thread/worker_test_base.h"
namespace net_instaweb {
class RewriteFilter;
class RewriteDriverTest : public RewriteTestBase {
protected:
RewriteDriverTest() {}
bool CanDecodeUrl(const StringPiece& url) {
GoogleUrl gurl(url);
RewriteFilter* filter;
OutputResourcePtr resource(
rewrite_driver()->DecodeOutputResource(gurl, &filter));
return (resource.get() != NULL);
}
GoogleString BaseUrlSpec() {
return rewrite_driver()->base_url().Spec().as_string();
}
// A helper to call ComputeCurrentFlushWindowRewriteDelayMs() that allows
// us to keep it private.
int64 GetFlushTimeout() {
return rewrite_driver()->ComputeCurrentFlushWindowRewriteDelayMs();
}
bool IsDone(RewriteDriver::WaitMode wait_mode, bool deadline_reached) {
ScopedMutex lock(rewrite_driver()->rewrite_mutex());
return rewrite_driver()->IsDone(wait_mode, deadline_reached);
}
void IncrementAsyncEventsCount() {
rewrite_driver()->IncrementAsyncEventsCount();
}
void DecrementAsyncEventsCount() {
rewrite_driver()->DecrementAsyncEventsCount();
}
void SetupResponsesForDownstreamCacheTesting() {
// Setup responses for the resources.
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
SetResponseWithDefaultHeaders("test/b.css", kContentTypeCss, kCss, 100);
// Setup a fake response for the expected purge path.
SetResponseWithDefaultHeaders("http://localhost:1234/purge/",
kContentTypeCss, "", 100);
}
void ProcessHtmlForDownstreamCacheTesting() {
GoogleString input_html(
StrCat(CssLinkHref("a.css"), " ", CssLinkHref("test/b.css")));
ParseUrl(kTestDomain, input_html);
}
void TestBlockingRewrite(RequestHeaders* request_headers,
bool expected_blocking_rewrite,
bool expected_fast_blocking_rewrite) {
rewrite_driver()->EnableBlockingRewrite(request_headers);
EXPECT_EQ(expected_blocking_rewrite,
rewrite_driver()->fully_rewrite_on_flush());
EXPECT_EQ(expected_fast_blocking_rewrite,
rewrite_driver()->fast_blocking_rewrite());
// Reset the flags to their default values after the test.
rewrite_driver()->set_fully_rewrite_on_flush(false);
rewrite_driver()->set_fast_blocking_rewrite(true);
EXPECT_FALSE(request_headers->Has(
HttpAttributes::kXPsaBlockingRewrite));
EXPECT_FALSE(request_headers->Has(
HttpAttributes::kXPsaBlockingRewriteMode));
}
void TestPendingEventsIsDone(bool wait_for_completion) {
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForShutDown, false));
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, false));
IncrementAsyncEventsCount();
EXPECT_FALSE(IsDone(RewriteDriver::kWaitForShutDown, false));
EXPECT_EQ(wait_for_completion,
IsDone(RewriteDriver::kWaitForCompletion, false));
DecrementAsyncEventsCount();
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForShutDown, false));
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, false));
}
void TestPendingEventsDriverCleanup(bool blocking_rewrite,
bool fast_blocking_rewrite) {
RewriteDriver* other_driver =
server_context()->NewRewriteDriver(CreateRequestContext());
other_driver->set_fully_rewrite_on_flush(blocking_rewrite);
other_driver->set_fast_blocking_rewrite(fast_blocking_rewrite);
other_driver->IncrementAsyncEventsCount();
other_driver->Cleanup();
other_driver->DecrementAsyncEventsCount();
EXPECT_EQ(0, server_context()->num_active_rewrite_drivers());
}
private:
DISALLOW_COPY_AND_ASSIGN(RewriteDriverTest);
};
namespace {
const char kBikePngFile[] = "BikeCrashIcn.png";
const char kNonRewrittenCachableHtml[] =
"<html>\n<link rel=stylesheet href=a.css> "
"<link rel=stylesheet href=test/b.css></html>";
const char kRewrittenCachableHtmlWithCacheExtension[] =
"<html>\n"
"<link rel=stylesheet href=a.css.pagespeed.ce.0.css> "
"<link rel=stylesheet href=test/b.css.pagespeed.ce.0.css>"
"</html>";
const char kRewrittenCachableHtmlWithCollapseWhitespace[] =
"<html>\n<link rel=stylesheet href=a.css> "
"<link rel=stylesheet href=test/b.css></html>";
TEST_F(RewriteDriverTest, NoChanges) {
ValidateNoChanges("no_changes",
"<head><script src=\"foo.js\"></script></head>"
"<body><form method=\"post\">"
"<input type=\"checkbox\" checked>"
"</form></body>");
}
TEST_F(RewriteDriverTest, CloneMarksNested) {
RequestHeaders request_headers;
request_headers.Add(HttpAttributes::kAccept, "image/webp");
request_headers.Add("a", "b");
request_headers.Add(HttpAttributes::kVia, "1.1 google");
rewrite_driver()->SetRequestHeaders(request_headers);
RewriteDriver* clone1 = rewrite_driver()->Clone();
EXPECT_TRUE(clone1->is_nested());
EXPECT_TRUE(clone1->request_properties()->SupportsWebpRewrittenUrls());
EXPECT_TRUE(rewrite_driver()->request_headers()->HasValue("a", "b"));
EXPECT_TRUE(clone1->request_headers()->HasValue("a", "b"));
EXPECT_TRUE(rewrite_driver()->request_headers()->HasValue(
HttpAttributes::kVia, "1.1 google"));
EXPECT_FALSE(clone1->request_headers()->HasValue(
HttpAttributes::kVia, "1.1 google"));
clone1->Cleanup();
RewriteDriver* parent2 =
server_context()->NewRewriteDriver(CreateRequestContext());
parent2->SetRequestHeaders(request_headers);
RewriteDriver* clone2 = parent2->Clone();
EXPECT_TRUE(clone2->is_nested());
clone2->Cleanup();
parent2->Cleanup();
}
TEST_F(RewriteDriverTest, TestLegacyUrl) {
GoogleString hash(32, '0');
rewrite_driver()->AddFilters();
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/jm." + hash + ".orig"))
<< "not enough dots";
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/jm.0.orig.js"))
<< "hash too short";
EXPECT_TRUE(CanDecodeUrl("http://example.com/dir/123/jm."+hash+".orig.js"));
EXPECT_TRUE(CanDecodeUrl(
"http://x.com/dir/123/jm.0123456789abcdef0123456789ABCDEF.orig.js"));
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/xx."+hash+".orig.js"))
<< "invalid filter xx";
GoogleString bad_hash(32, 'z');
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/jm." + bad_hash +
".orig.js"))
<< "invalid hash code -- not hex";
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/jm.ab.orig.js"))
<< "invalid hash code -- not 32 chars";
EXPECT_FALSE(CanDecodeUrl("http://example.com/dir/123/jm."
+ hash + ".orig.x"))
<< "invalid extension";
}
TEST_F(RewriteDriverTest, TestValidUrlSignatures) {
StringPiece key("helloworld");
options()->set_url_signing_key(key);
EXPECT_EQ(10, options()->sha1signature()->SignatureSizeInChars());
rewrite_driver()->AddFilters();
EXPECT_TRUE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.UQ_aP9rObnq.css"))
<< "valid signature";
EXPECT_FALSE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.UAAAAAAAAAA.css"))
<< "invalid signature";
EXPECT_FALSE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.U.css"))
<< "no signature";
}
TEST_F(RewriteDriverTest, TestIgnoringUrlSignatures) {
options()->set_url_signing_key("helloworld");
options()->set_accept_invalid_signatures(true);
EXPECT_EQ(10, options()->sha1signature()->SignatureSizeInChars());
rewrite_driver()->AddFilters();
EXPECT_TRUE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.UQ_aP9rObnq.css"))
<< "valid signature, ignored";
EXPECT_TRUE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.UAAAAAAAAAA.css"))
<< "invalid signature, ignored";
EXPECT_TRUE(CanDecodeUrl(
"http://signed-urls.example.com/mod_pagespeed_example/styles/"
"A.all_styles.css.pagespeed.cf.U.css"))
<< "no signature, ignored";
}
TEST_F(RewriteDriverTest, PagespeedObliviousPositiveTest) {
RewriteOptions* ops = options();
ops->set_oblivious_pagespeed_urls(false); // Decode Pagespeed URL.
rewrite_driver()->AddFilters();
EXPECT_TRUE(CanDecodeUrl(
"http://www.example.com/foresee-trigger.js.pagespeed.jm.0D45DpKAeI.js"));
}
TEST_F(RewriteDriverTest, PagespeedObliviousNegativeTest) {
RewriteOptions* ops = options();
ops->set_oblivious_pagespeed_urls(true); // Don't decode Pagespeed URL.
rewrite_driver()->AddFilters();
EXPECT_FALSE(CanDecodeUrl(
"http://www.example.com/foresee-trigger.js.pagespeed.jm.0D45DpKAeI.js"));
}
TEST_F(RewriteDriverTest, TestModernUrl) {
rewrite_driver()->AddFilters();
// Sanity-check on a valid one
EXPECT_TRUE(CanDecodeUrl(
Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg")));
// Query is OK, too.
EXPECT_TRUE(CanDecodeUrl(
StrCat(Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg"),
"?s=ok")));
// Invalid filter code
EXPECT_FALSE(CanDecodeUrl(
Encode("http://example.com/", "NOFILTER", "HASH", "Puzzle.jpg", "jpg")));
// Nonsense extension -- we will just ignore it these days.
EXPECT_TRUE(CanDecodeUrl(
Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpgif")));
// No hash
GoogleString encoded_url(Encode("http://example.com/", "ce", "123456789",
"Puzzle.jpg", "jpg"));
GlobalReplaceSubstring("123456789", "", &encoded_url);
EXPECT_FALSE(CanDecodeUrl(encoded_url));
}
class RewriteDriverTestUrlNamer : public RewriteDriverTest {
public:
RewriteDriverTestUrlNamer() {
SetUseTestUrlNamer(true);
}
};
TEST_F(RewriteDriverTestUrlNamer, TestEncodedUrls) {
rewrite_driver()->AddFilters();
// Sanity-check on a valid one
EXPECT_TRUE(CanDecodeUrl(
Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg")));
// Query is OK, too.
EXPECT_TRUE(CanDecodeUrl(
StrCat(Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg"),
"?s=ok")));
// Invalid filter code
EXPECT_FALSE(CanDecodeUrl(
Encode("http://example.com/", "NOFILTER", "HASH", "Puzzle.jpg", "jpg")));
// Nonsense extension -- we will just ignore it these days.
EXPECT_TRUE(CanDecodeUrl(
Encode("http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpgif")));
// No hash
GoogleString encoded_url(Encode("http://example.com/", "ce", "123456789",
"Puzzle.jpg", "jpg"));
GlobalReplaceSubstring("123456789", "", &encoded_url);
EXPECT_FALSE(CanDecodeUrl(encoded_url));
// Valid proxy domain but invalid decoded URL.
encoded_url = Encode("http://example.com/", "ce", "0", "Puzzle.jpg", "jpg");
GlobalReplaceSubstring("example.com/",
"example.comWYTHQ000JRJFCAAKYU1EMA6VUBDTS4DESLRWIPMS"
"KKMQH0XYN1FURDBBSQ9AYXVX3TZDKZEIJNLRHU05ATHBAWWAG2+"
"ADDCXPWGGP1VTHJIYU13IIFQYSYMGKIMSFIEBM+HCAACVNGO8CX"
"XO%81%9F%F1m/", &encoded_url);
// By default TestUrlNamer doesn't proxy but we need it to for this test.
TestUrlNamer::SetProxyMode(true);
EXPECT_FALSE(CanDecodeUrl(encoded_url));
}
TEST_F(RewriteDriverTestUrlNamer, TestDecodeUrls) {
// Sanity-check on a valid one
GoogleUrl gurl_good(Encode(
"http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg"));
rewrite_driver()->AddFilters();
StringVector urls;
TestUrlNamer::SetProxyMode(true);
EXPECT_TRUE(rewrite_driver()->DecodeUrl(gurl_good, &urls));
EXPECT_EQ(1, urls.size());
EXPECT_EQ("http://example.com/Puzzle.jpg", urls[0]);
// Invalid filter code
urls.clear();
GoogleUrl gurl_bad(Encode(
"http://example.com/", "NOFILTER", "HASH", "Puzzle.jpg", "jpgif"));
EXPECT_FALSE(rewrite_driver()->DecodeUrl(gurl_bad, &urls));
// Combine filters
urls.clear();
GoogleUrl gurl_multi(Encode(
"http://example.com/", "cc", "HASH", MultiUrl("a.css", "b.css"), "css"));
EXPECT_TRUE(rewrite_driver()->DecodeUrl(gurl_multi, &urls));
EXPECT_EQ(2, urls.size());
EXPECT_EQ("http://example.com/a.css", urls[0]);
EXPECT_EQ("http://example.com/b.css", urls[1]);
// Invalid Url.
urls.clear();
GoogleUrl gurl_invalid("invalid url");
EXPECT_FALSE(rewrite_driver()->DecodeUrl(gurl_invalid, &urls));
EXPECT_EQ(0, urls.size());
// ProxyMode off
urls.clear();
TestUrlNamer::SetProxyMode(false);
SetUseTestUrlNamer(false);
gurl_good.Reset(Encode(
"http://example.com/", "ce", "HASH", "Puzzle.jpg", "jpg"));
EXPECT_TRUE(rewrite_driver()->DecodeUrl(gurl_good, &urls));
EXPECT_EQ(1, urls.size());
EXPECT_EQ("http://example.com/Puzzle.jpg", urls[0]);
urls.clear();
gurl_multi.Reset(Encode(
"http://example.com/", "cc", "HASH", MultiUrl("a.css", "b.css"), "css"));
EXPECT_TRUE(rewrite_driver()->DecodeUrl(gurl_multi, &urls));
EXPECT_EQ(2, urls.size());
EXPECT_EQ("http://example.com/a.css", urls[0]);
EXPECT_EQ("http://example.com/b.css", urls[1]);
}
// Test to make sure we do not put in extra things into the cache.
// This is using the CSS rewriter, which caches the output.
TEST_F(RewriteDriverTest, TestCacheUse) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
}
// Test to make sure when we fetch a with a Via header, "public"
// is added to the Cache-Control.
TEST_F(RewriteDriverTest, ViaPublicPageSpeedResource) {
RequestHeaders request_headers;
request_headers.Add(HttpAttributes::kVia, "1.1 google");
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString url = Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
ResponseHeaders response;
GoogleString contents;
ASSERT_TRUE(FetchResourceUrl(url, &request_headers, &contents, &response));
EXPECT_TRUE(response.HasValue(HttpAttributes::kCacheControl, "public"));
// Warm load.
response.Clear();
ASSERT_TRUE(FetchResourceUrl(url, &request_headers, &contents, &response));
EXPECT_TRUE(response.HasValue(HttpAttributes::kCacheControl, "public"));
}
// Extension of above with cache invalidation.
TEST_F(RewriteDriverTest, TestCacheUseWithInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
// Set cache invalidation timestamp (to now, so that response date header is
// in the "past") and load. Should get inserted again.
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
options()->UpdateCacheInvalidationTimestampMs(now_ms);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We expect: identical input a new rname entry (its version # changed),
// and the output which may not may not auto-advance due to MockTimer
// black magic.
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(2, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, TestCacheUseWithUrlPatternAllInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
// Set cache invalidation (to now) for all URLs with "a.css" and also
// invalidate all metadata (the last 'false' argument below).
options()->AddUrlCacheInvalidationEntry("*a.css*", now_ms, false);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We expect: identical input, a new rewrite entry (its version # changed),
// and the output which may not may not auto-advance due to MockTimer black
// magic.
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(2, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, TestCacheUseWithUrlPatternOnlyInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
// Set cache invalidation (to now) for all URLs with "a.css". Does not
// invalidate any metadata (the last 'true' argument below).
options()->AddUrlCacheInvalidationEntry("*a.css*", now_ms, true);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
// The output rewritten URL is invalidated, the input is also invalidated, and
// fetched again. The rewrite entry does not change, and gets reinserted.
// Thus, we have identical input, rname entry, and the output.
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(3, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, TestCacheUseWithRewrittenUrlAllInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
// Set a URL cache invalidation entry for output URL. Original input URL is
// not affected. Also invalidate all metadata (the
// ignores_metadata_and_pcache argument being false below).
options()->AddUrlCacheInvalidationEntry(
css_minified_url, now_ms, false /* ignores_metadata_and_pcache */);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We expect: a new rewrite entry (its version # changed), and identical
// output.
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(1, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, TestCacheUseWithRewrittenUrlOnlyInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
// Set cache invalidation (to now) for output URL. Original input URL is not
// affected. Does not invalidate any metadata (the last 'true' argument
// below).
options()->AddUrlCacheInvalidationEntry(css_minified_url, now_ms, true);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We expect: identical rewrite entry and output.
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(2, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, TestCacheUseWithOriginalUrlInvalidation) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
const char kMinCss[] = "*{display:none}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString css_minified_url =
Encode(kTestDomain, RewriteOptions::kCssFilterId,
hasher()->Hash(kMinCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_minified_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result.
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
// Warm load. This one should not change the number of inserts at all
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
// Set cache invalidation (to now) for input URL. Rewritten output URL is not
// affected. So there will be no cache inserts or reinserts.
// Note: Whether we invalidate all metadata (the last argument below) is
// immaterial in this test.
options()->AddUrlCacheInvalidationEntry("http://test.com/a.css", now_ms,
false);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(css_minified_url));
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
}
// Similar to TestCacheUse, but with cache-extender which reconstructs on the
// fly.
TEST_F(RewriteDriverTest, TestCacheUseOnTheFly) {
AddFilter(RewriteOptions::kExtendCacheCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString cache_extended_url =
Encode(kTestDomain, RewriteOptions::kCacheExtenderId,
hasher()->Hash(kCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(cache_extended_url));
// We should have 2 things inserted:
// 1) the source data
// 2) the rname entry for the result (only in sync)
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(2, cold_num_inserts);
// Warm load. This one does a read-check to avoid a re-inserts in the
// rname entry.
EXPECT_TRUE(TryFetchResource(cache_extended_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
}
// Verifies that the computed rewrite delay agrees with expectations
// depending on the configuration of constituent delay variables.
TEST_F(RewriteDriverTest, TestComputeCurrentFlushWindowRewriteDelayMs) {
options()->set_rewrite_deadline_ms(1000);
// "Start" a parse to configure the start time in the driver.
ASSERT_TRUE(rewrite_driver()->StartParseId("http://site.com/",
"compute_flush_window_test",
kContentTypeHtml));
// The per-page deadline is initially unconfigured.
EXPECT_EQ(1000, GetFlushTimeout());
// If the per-page deadline is less than the per-flush window timeout,
// the per-page deadline is returned.
rewrite_driver()->set_max_page_processing_delay_ms(500);
EXPECT_EQ(500, GetFlushTimeout());
// If the per-page deadline exceeds the per-flush window timeout, the flush
// timeout is returned.
rewrite_driver()->set_max_page_processing_delay_ms(1750);
EXPECT_EQ(1000, GetFlushTimeout());
// If we advance mock time to leave less than a flush window timeout remaining
// against the page deadline, the appropriate page deadline difference is
// returned.
SetTimeMs(start_time_ms() + 1000);
EXPECT_EQ(750, GetFlushTimeout()); // 1750 - 1000
// If we advance mock time beyond the per-page limit, a value of 1 is
// returned. (This is required since values <= 0 are interpreted by internal
// timeout functions as unlimited.)
SetTimeMs(start_time_ms() + 2000);
EXPECT_EQ(1, GetFlushTimeout());
rewrite_driver()->FinishParse();
}
// Extension of above with cache invalidation.
TEST_F(RewriteDriverTest, TestCacheUseOnTheFlyWithInvalidation) {
AddFilter(RewriteOptions::kExtendCacheCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
GoogleString cache_extended_url =
Encode(kTestDomain, RewriteOptions::kCacheExtenderId,
hasher()->Hash(kCss), "a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(cache_extended_url));
// We should have 2 things inserted:
// 1) the source data
// 2) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(2, cold_num_inserts);
// Warm load. This one does a read-check to avoid a re-inserts in the
// rname entry.
EXPECT_TRUE(TryFetchResource(cache_extended_url));
EXPECT_EQ(cold_num_inserts, lru_cache()->num_inserts());
EXPECT_EQ(0, lru_cache()->num_identical_reinserts());
EXPECT_EQ(2, lru_cache()->num_hits());
// Set cache invalidation timestamp (to now, so that response date header is
// in the "past") and load.
ClearStats();
int64 now_ms = timer()->NowMs();
options()->ClearSignatureForTesting();
options()->UpdateCacheInvalidationTimestampMs(now_ms);
options()->ComputeSignature();
EXPECT_TRUE(TryFetchResource(cache_extended_url));
// We expect: input re-insert, new metadata key
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(1, lru_cache()->num_identical_reinserts());
}
TEST_F(RewriteDriverTest, BaseTags) {
// Starting the parse, the base-tag will be derived from the html url.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
rewrite_driver()->Flush();
EXPECT_EQ("http://example.com/index.html", BaseUrlSpec());
// If we then encounter a base tag, that will become the new base.
rewrite_driver()->ParseText("<base href='http://new.example.com/subdir/'>");
rewrite_driver()->Flush();
EXPECT_EQ(0, message_handler()->TotalMessages());
EXPECT_EQ("http://new.example.com/subdir/", BaseUrlSpec());
// A second base tag will be ignored, and an info message will be printed.
rewrite_driver()->ParseText("<base href=http://second.example.com/subdir2>");
rewrite_driver()->Flush();
EXPECT_EQ(1, message_handler()->TotalMessages());
EXPECT_EQ("http://new.example.com/subdir/", BaseUrlSpec());
// Restart the parse with a new URL and we start fresh.
rewrite_driver()->FinishParse();
ASSERT_TRUE(rewrite_driver()->StartParse(
"http://restart.example.com/index.html"));
rewrite_driver()->Flush();
EXPECT_EQ("http://restart.example.com/index.html", BaseUrlSpec());
// We should be able to reset again.
rewrite_driver()->ParseText("<base href='http://new.example.com/subdir/'>");
rewrite_driver()->Flush();
EXPECT_EQ(1, message_handler()->TotalMessages());
EXPECT_EQ("http://new.example.com/subdir/", BaseUrlSpec());
}
TEST_F(RewriteDriverTest, RelativeBaseTag) {
// Starting the parse, the base-tag will be derived from the html url.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
rewrite_driver()->ParseText("<base href='subdir/'>");
rewrite_driver()->Flush();
EXPECT_EQ(0, message_handler()->TotalMessages());
EXPECT_EQ("http://example.com/subdir/", BaseUrlSpec());
}
TEST_F(RewriteDriverTest, InvalidBaseTag) {
// Encountering an invalid base tag should be ignored (except info message).
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// Note: Even nonsensical protocols must be accepted as base URLs.
rewrite_driver()->ParseText("<base href='slwly:example.com/subdir'>");
rewrite_driver()->Flush();
EXPECT_EQ(0, message_handler()->TotalMessages());
EXPECT_EQ("slwly:example.com/subdir", BaseUrlSpec());
// Reasonable base URLs following that do not change it.
rewrite_driver()->ParseText("<base href='http://example.com/absolute/'>");
rewrite_driver()->Flush();
EXPECT_EQ("slwly:example.com/subdir", BaseUrlSpec());
}
// The TestUrlNamer produces a url like below which is too long.
// http://cdn.com/http/base.example.com/http/unmapped.example.com/dir/test.jpg.pagespeed.xy.#. NOLINT
TEST_F(RewriteDriverTest, CreateOutputResourceTooLongSeparateBase) {
SetUseTestUrlNamer(true);
OutputResourcePtr resource;
GoogleString failure_reason;
options()->set_max_url_size(94);
resource.reset(rewrite_driver()->CreateOutputResourceWithPath(
"http://mapped.example.com/dir/",
"http://unmapped.example.com/dir/",
"http://base.example.com/dir/",
"xy",
"test.jpg",
kRewrittenResource,
&failure_reason));
EXPECT_TRUE(NULL == resource.get());
EXPECT_EQ("Rewritten URL too long: http://cdn.com/http/base.example.com/"
"http/unmapped.example.com/dir/test.jpg.pagespeed.xy.#.",
failure_reason);
failure_reason = "";
options()->set_max_url_size(95);
resource.reset(rewrite_driver()->CreateOutputResourceWithPath(
"http://mapped.example.com/dir/",
"http://unmapped.example.com/dir/",
"http://base.example.com/dir/",
"xy",
"test.jpg",
kRewrittenResource,
&failure_reason));
EXPECT_TRUE(NULL != resource.get());
EXPECT_EQ("", failure_reason);
}
TEST_F(RewriteDriverTest, CreateOutputResourceTooLong) {
const OutputResourceKind resource_kinds[] = {
kRewrittenResource,
kOnTheFlyResource,
kOutlinedResource,
};
// short_path.size() < options()->max_url_size() < long_path.size()
GoogleString short_path = "http://www.example.com/dir/";
GoogleString long_path = short_path;
for (int i = 0; 2 * i < options()->max_url_size(); ++i) {
long_path += "z/";
}
// short_name.size() < options()->max_url_segment_size() < long_name.size()
GoogleString short_name = "foo.css";
GoogleString long_name =
StrCat("foo.css?",
GoogleString(options()->max_url_segment_size() + 1, 'z'));
GoogleString dummy_filter_id = "xy";
OutputResourcePtr resource;
GoogleString failure_reason;
for (int k = 0; k < arraysize(resource_kinds); ++k) {
failure_reason = "";
// Short name should always succeed at creating new resource.
resource.reset(rewrite_driver()->CreateOutputResourceWithPath(
short_path, dummy_filter_id, short_name, resource_kinds[k],
&failure_reason));
EXPECT_TRUE(NULL != resource.get());
EXPECT_EQ("", failure_reason);
failure_reason = "";
// Long leaf-name should always fail at creating new resource.
resource.reset(rewrite_driver()->CreateOutputResourceWithPath(
short_path, dummy_filter_id, long_name, resource_kinds[k],
&failure_reason));
EXPECT_TRUE(NULL == resource.get());
EXPECT_EQ("Rewritten URL segment too long.", failure_reason);
failure_reason = "";
// Long total URL length should always fail at creating new resource.
resource.reset(rewrite_driver()->CreateOutputResourceWithPath(
long_path, dummy_filter_id, short_name, resource_kinds[k],
&failure_reason));
EXPECT_TRUE(NULL == resource.get());
EXPECT_EQ(StrCat("Rewritten URL too long: ", long_path, short_name,
".pagespeed.xy.#."),
failure_reason);
}
}
TEST_F(RewriteDriverTest, MultipleDomains) {
rewrite_driver()->AddFilters();
// Make sure we authorize domains for resources properly. This is a regression
// test for where loading things from a domain would prevent loads from an
// another domain from the same RewriteDriver.
const char kCss[] = "* { display: none; }";
const char kAltDomain[] = "http://www.example.co.uk/";
SetResponseWithDefaultHeaders(StrCat(kTestDomain, "a.css"), kContentTypeCss,
kCss, 100);
SetResponseWithDefaultHeaders(StrCat(kAltDomain, "b.css"), kContentTypeCss,
kCss, 100);
GoogleString rewritten1 = Encode(kTestDomain,
RewriteOptions::kCacheExtenderId,
hasher()->Hash(kCss), "a.css", "css");
GoogleString rewritten2 = Encode(kAltDomain, RewriteOptions::kCacheExtenderId,
hasher()->Hash(kCss), "b.css", "css");
EXPECT_TRUE(TryFetchResource(rewritten1));
ClearRewriteDriver();
EXPECT_TRUE(TryFetchResource(rewritten2));
}
TEST_F(RewriteDriverTest, ResourceCharset) {
// Make sure we properly pick up the charset into a resource on read.
const char kUrl[] = "http://www.example.com/foo.css";
ResponseHeaders resource_headers;
SetDefaultLongCacheHeaders(&kContentTypeCss, &resource_headers);
resource_headers.Replace(HttpAttributes::kContentType,
"text/css; charset=koi8-r");
const char kContents[] = "\xF5\xD2\xC1!"; // Ура!
SetFetchResponse(kUrl, resource_headers, kContents);
// We do this twice to make sure the cached version is OK, too.
for (int round = 0; round < 2; ++round) {
ResourcePtr resource(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
kUrl));
MockResourceCallback mock_callback(resource, factory()->thread_system());
ASSERT_TRUE(resource.get() != NULL);
resource->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback);
EXPECT_TRUE(mock_callback.done());
EXPECT_TRUE(mock_callback.success());
EXPECT_EQ(kContents, resource->ExtractUncompressedContents());
ASSERT_TRUE(resource->type() != NULL);
EXPECT_EQ(ContentType::kCss, resource->type()->type());
EXPECT_STREQ("koi8-r", resource->charset());
}
}
// Test caching behavior for normal UrlInputResources.
// This is the base case that LoadResourcesFromFiles below contrasts with.
TEST_F(RewriteDriverTest, LoadResourcesFromTheWeb) {
rewrite_driver()->AddFilters();
const char kStaticUrlPrefix[] = "http://www.example.com/";
const char kResourceName[ ]= "foo.css";
GoogleString resource_url = StrCat(kStaticUrlPrefix, kResourceName);
const char kResourceContents1[] = "body { background: red; }";
const char kResourceContents2[] = "body { background: blue; }";
ResponseHeaders resource_headers;
// This sets 1 year cache lifetime :/ TODO(sligocki): Shorten this.
SetDefaultLongCacheHeaders(&kContentTypeCss, &resource_headers);
// Clear the Etag and Last-Modified headers since this
// SetDefaultLongCacheHeaders sets their value to constants which don't change
// when their value is updated.
resource_headers.RemoveAll(HttpAttributes::kEtag);
resource_headers.RemoveAll(HttpAttributes::kLastModified);
// Set the fetch value.
SetFetchResponse(resource_url, resource_headers, kResourceContents1);
// Make sure file can be loaded. Note this cannot be loaded through the
// mock_url_fetcher, because it has not been set in that fetcher.
ResourcePtr resource(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
resource_url));
MockResourceCallback mock_callback(resource, factory()->thread_system());
ASSERT_TRUE(resource.get() != NULL);
resource->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback);
EXPECT_TRUE(mock_callback.done());
EXPECT_TRUE(mock_callback.success());
EXPECT_EQ(kResourceContents1, resource->ExtractUncompressedContents());
// TODO(sligocki): Check it was cached.
// Change the fetch value.
SetFetchResponse(resource_url, resource_headers, kResourceContents2);
// Check that the resource loads cached.
ResourcePtr resource2(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
resource_url));
MockResourceCallback mock_callback2(resource2, factory()->thread_system());
ASSERT_TRUE(resource2.get() != NULL);
resource2->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback2);
EXPECT_TRUE(mock_callback2.done());
EXPECT_TRUE(mock_callback2.success());
EXPECT_EQ(kResourceContents1, resource2->ExtractUncompressedContents());
// Advance timer and check that the resource loads updated.
AdvanceTimeMs(10 * Timer::kYearMs);
// Check that the resource loads updated.
ResourcePtr resource3(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
resource_url));
MockResourceCallback mock_callback3(resource3, factory()->thread_system());
ASSERT_TRUE(resource3.get() != NULL);
resource3->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback3);
EXPECT_TRUE(mock_callback3.done());
EXPECT_EQ(kResourceContents2, resource3->ExtractUncompressedContents());
}
// Test that we successfully load specified resources from files and that
// file resources have the appropriate properties, such as being loaded from
// file every time they are fetched (not being cached).
TEST_F(RewriteDriverTest, LoadResourcesFromFiles) {
rewrite_driver()->AddFilters();
const char kStaticUrlPrefix[] = "http://www.example.com/static/";
const char kStaticFilenamePrefix[] = "/htmlcontent/static/";
const char kResourceName[ ]= "foo.css";
GoogleString resource_filename = StrCat(kStaticFilenamePrefix, kResourceName);
GoogleString resource_url = StrCat(kStaticUrlPrefix, kResourceName);
const char kResourceContents1[] = "body { background: red; }";
const char kResourceContents2[] = "body { background: blue; }";
// Tell RewriteDriver to associate static URLs with filenames.
options()->file_load_policy()->Associate(kStaticUrlPrefix,
kStaticFilenamePrefix);
// Write a file.
WriteFile(resource_filename.c_str(), kResourceContents1);
// Make sure file can be loaded. Note this cannot be loaded through the
// mock_url_fetcher, because it has not been set in that fetcher.
ResourcePtr resource(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
resource_url));
ASSERT_TRUE(resource.get() != NULL);
EXPECT_EQ(&kContentTypeCss, resource->type());
MockResourceCallback mock_callback(resource, factory()->thread_system());
resource->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback);
EXPECT_TRUE(mock_callback.done());
EXPECT_TRUE(mock_callback.success());
EXPECT_EQ(kResourceContents1, resource->ExtractUncompressedContents());
// TODO(sligocki): Check it wasn't cached.
// Change the file.
WriteFile(resource_filename.c_str(), kResourceContents2);
// Make sure the resource loads updated.
ResourcePtr resource2(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
resource_url));
ASSERT_TRUE(resource2.get() != NULL);
EXPECT_EQ(&kContentTypeCss, resource2->type());
MockResourceCallback mock_callback2(resource2, factory()->thread_system());
resource2->LoadAsync(Resource::kReportFailureIfNotCacheable,
rewrite_driver()->request_context(),
&mock_callback2);
EXPECT_TRUE(mock_callback2.done());
EXPECT_TRUE(mock_callback2.success());
EXPECT_EQ(kResourceContents2, resource2->ExtractUncompressedContents());
}
// Make sure the content-type is set correctly, even for URLs with queries.
// http://code.google.com/p/modpagespeed/issues/detail?id=405
TEST_F(RewriteDriverTest, LoadResourcesContentType) {
rewrite_driver()->AddFilters();
// Tell RewriteDriver to associate static URLs with filenames.
options()->file_load_policy()->Associate("http://www.example.com/static/",
"/htmlcontent/static/");
// Write file with readable extension.
WriteFile("/htmlcontent/foo.js", "");
// Load the file with a query param (add .css at the end of the param just
// for optimal trickyness).
ResourcePtr resource(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
"http://www.example.com/static/foo.js?version=2.css"));
EXPECT_TRUE(resource.get() != NULL);
EXPECT_EQ(&kContentTypeJavascript, resource->type());
// Write file with bogus extension.
WriteFile("/htmlcontent/bar.bogus", "");
// Load it normally.
ResourcePtr resource2(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
"http://www.example.com/static/bar.bogus"));
EXPECT_TRUE(resource2.get() != NULL);
EXPECT_TRUE(NULL == resource2->type());
}
TEST_F(RewriteDriverTest, ResolveAnchorUrl) {
rewrite_driver()->AddFilters();
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
GoogleUrl resolved(rewrite_driver()->base_url(), "#anchor");
EXPECT_EQ("http://example.com/index.html#anchor", resolved.Spec());
rewrite_driver()->FinishParse();
}
// A rewrite context that's not actually capable of rewriting -- we just need
// one to pass in to InfoAt in test below.
class MockRewriteContext : public SingleRewriteContext {
public:
explicit MockRewriteContext(RewriteDriver* driver) :
SingleRewriteContext(driver, NULL, NULL) {}
virtual void RewriteSingle(const ResourcePtr& input,
const OutputResourcePtr& output) {}
virtual const char* id() const { return "mock"; }
virtual OutputResourceKind kind() const { return kOnTheFlyResource; }
};
TEST_F(RewriteDriverTest, DiagnosticsWithPercent) {
// Regression test for crash in InfoAt where location has %stuff in it.
// (make sure it actually shows up first, though).
int prev_log_level = logging::GetMinLogLevel();
logging::SetMinLogLevel(logging::LOG_INFO);
rewrite_driver()->AddFilters();
MockRewriteContext context(rewrite_driver());
ResourcePtr resource(
rewrite_driver()->CreateInputResourceAbsoluteUncheckedForTestsOnly(
"http://www.example.com/%s%s%s%d%f"));
ResourceSlotPtr slot(new FetchResourceSlot(resource));
context.AddSlot(slot);
rewrite_driver()->InfoAt(&context, "Just a test");
logging::SetMinLogLevel(prev_log_level);
}
// Tests that we reject https URLs quickly.
TEST_F(RewriteDriverTest, RejectHttpsQuickly) {
// Need to expressly authorize https even though we don't support it.
options()->WriteableDomainLawyer()->AddDomain("https://*/",
message_handler());
AddFilter(RewriteOptions::kRewriteJavascriptExternal);
// When we don't support https then we fail quickly and cleanly.
factory()->mock_url_async_fetcher()->set_fetcher_supports_https(false);
ValidateNoChanges("reject_https_quickly",
"<script src='https://example.com/a.js'></script>");
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
// When we do support https the fetcher fails to find the resource.
factory()->mock_url_async_fetcher()->set_fetcher_supports_https(true);
SetFetchResponse404("https://example.com/a.js");
ValidateNoChanges("reject_https_quickly",
"<script src='https://example.com/a.js'></script>");
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, counting_url_async_fetcher()->failure_count());
}
// Test that CreateInputResource doesn't crash when handed a data url.
// This was causing a query of death in some circumstances.
TEST_F(RewriteDriverTest, RejectDataResourceGracefully) {
MockRewriteContext context(rewrite_driver());
GoogleUrl dataUrl("data:");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(dataUrl,
&is_authorized));
EXPECT_TRUE(resource.get() == NULL);
EXPECT_TRUE(is_authorized);
}
// Test that when inline_unauthorized_resources is set to false (the default
// case), no resources are created for unauthorized resources, but authorized
// ones are created with the right cache-key.
TEST_F(RewriteDriverTest, NoCreateInputResourceUnauthorized) {
MockRewriteContext context(rewrite_driver());
// Call StartParseUrl so that the base_url gets set to a non-empty string.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// Test that an unauthorized resource is not allowed to be created.
GoogleUrl unauthorized_url("http://unauthorized.domain.com/a.js");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(unauthorized_url,
&is_authorized));
EXPECT_TRUE(resource.get() == NULL);
EXPECT_FALSE(is_authorized);
// Test that an authorized resource is created with the right cache key even
// if the filter allows unauthorized domains.
GoogleUrl authorized_url("http://example.com/a.js");
ResourcePtr resource2(rewrite_driver()->CreateInputResource(
authorized_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource2.get() != NULL);
EXPECT_TRUE(is_authorized);
EXPECT_STREQ(authorized_url.spec_c_str(), resource2->url());
EXPECT_STREQ(authorized_url.spec_c_str(), resource2->cache_key());
}
// Test that when inline_unauthorized_resources is set to true, resources
// are created for unauthorized resources with the correctly prefixed keys, and
// the authorized resources continue to get created with the right cache-keys.
TEST_F(RewriteDriverTest, CreateInputResourceUnauthorized) {
options()->AddInlineUnauthorizedResourceType(semantic_type::kScript);
MockRewriteContext context(rewrite_driver());
// Call StartParseUrl so that the base_url gets set to a non-empty string.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// Test that an unauthorized resource is created with the right cache key.
GoogleUrl unauthorized_url("http://unauthorized.domain.com/a.js");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(
unauthorized_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource.get() != NULL);
EXPECT_FALSE(is_authorized);
EXPECT_STREQ(unauthorized_url.spec_c_str(), resource->url());
EXPECT_STREQ("unauth://unauthorized.domain.com/a.js", resource->cache_key());
// Test that an authorized resource continues to be created with the right
// cache key.
GoogleUrl authorized_url("http://example.com/a.js");
ResourcePtr resource2(rewrite_driver()->CreateInputResource(
authorized_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource2.get() != NULL);
EXPECT_TRUE(is_authorized);
EXPECT_STREQ(authorized_url.spec_c_str(), resource2->url());
EXPECT_STREQ(authorized_url.spec_c_str(), resource2->cache_key());
// Test that an unauthorized resource is not created if
// allow_unauthorized_domain is false.
ResourcePtr resource3(rewrite_driver()->CreateInputResource(
unauthorized_url,
RewriteDriver::kInlineOnlyAuthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource3.get() == NULL);
EXPECT_FALSE(is_authorized);
// Test that an unauthorized resource is not created with the default
// CreateInputResource call.
ResourcePtr resource4(
rewrite_driver()->CreateInputResource(unauthorized_url, &is_authorized));
EXPECT_TRUE(resource4.get() == NULL);
EXPECT_FALSE(is_authorized);
}
// Test that when inline_unauthorized_resources is set to true, unauthorized
// resources continue to be not created when they match a disallowed pattern.
TEST_F(RewriteDriverTest, CreateInputResourceUnauthorizedWithDisallow) {
options()->AddInlineUnauthorizedResourceType(semantic_type::kScript);
options()->Disallow("http://unauthorized.domain.com/*");
MockRewriteContext context(rewrite_driver());
// Call StartParseUrl so that the base_url gets set to a non-empty string.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// Test that an unauthorized resource is not created when it is disallowed.
GoogleUrl unauthorized_url("http://unauthorized.domain.com/a.js");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(
unauthorized_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource.get() == NULL);
EXPECT_FALSE(is_authorized);
}
// Test AllowWhenInlining overrides Disallow when inlining.
TEST_F(RewriteDriverTest, AllowWhenInliningOverridesDisallow) {
options()->AllowOnlyWhenInlining("*a.js*");
MockRewriteContext context(rewrite_driver());
// Call StartParseUrl so that the base_url gets set to a non-empty string.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// This resource would normally not be created because it is disallowed,
// except that we explicitly allowed it with AllowWhenInlining.
GoogleUrl js_url("http://example.com/a.js");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(
js_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForInlining,
&is_authorized));
EXPECT_FALSE(resource.get() == NULL);
EXPECT_TRUE(is_authorized);
}
// Test AllowWhenInlining fails to overrides Disallow when not inlining.
TEST_F(RewriteDriverTest, AllowWhenInliningDoesntOverrideDisallow) {
options()->AllowOnlyWhenInlining("*a.js*");
MockRewriteContext context(rewrite_driver());
// Call StartParseUrl so that the base_url gets set to a non-empty string.
ASSERT_TRUE(rewrite_driver()->StartParse("http://example.com/index.html"));
// This resource would normally not be created because it is disallowed, and
// AllowWhenInlining doesn't apply because we're not inlining.
GoogleUrl js_url("http://example.com/a.js");
bool is_authorized;
ResourcePtr resource(rewrite_driver()->CreateInputResource(
js_url,
RewriteDriver::kInlineUnauthorizedResources,
RewriteDriver::kIntendedForGeneral,
&is_authorized));
EXPECT_TRUE(resource.get() == NULL);
EXPECT_FALSE(is_authorized);
}
class ResponseHeadersCheckingFilter : public EmptyHtmlFilter {
public:
explicit ResponseHeadersCheckingFilter(RewriteDriver* driver)
: driver_(driver),
flush_occurred_(false) {
}
void CheckAccess() {
EXPECT_TRUE(driver_->response_headers() != NULL);
if (flush_occurred_) {
EXPECT_TRUE(driver_->mutable_response_headers() == NULL);
} else {
EXPECT_EQ(driver_->mutable_response_headers(),
driver_->response_headers());
}
}
virtual void StartDocument() {
flush_occurred_ = false;
CheckAccess();
}
virtual void Flush() {
CheckAccess(); // We still can access the mutable headers during Flush.
flush_occurred_ = true;
}
virtual void StartElement(HtmlElement* element) { CheckAccess(); }
virtual void EndElement(HtmlElement* element) { CheckAccess(); }
virtual void EndDocument() { CheckAccess(); }
virtual const char* Name() const { return "ResponseHeadersCheckingFilter"; }
private:
RewriteDriver* driver_;
bool flush_occurred_;
};
class DetermineEnabledCheckingFilter : public EmptyHtmlFilter {
public:
DetermineEnabledCheckingFilter() :
start_document_called_(false),
enabled_value_(false) {}
virtual void StartDocument() {
start_document_called_ = true;
}
virtual void DetermineEnabled(GoogleString* disabled_reason) {
set_is_enabled(enabled_value_);
}
void SetEnabled(bool enabled_value) {
enabled_value_ = enabled_value;
}
bool start_document_called() {
return start_document_called_;
}
virtual const char* Name() const { return "DetermineEnabledCheckingFilter"; }
private:
bool start_document_called_;
bool enabled_value_;
};
TEST_F(RewriteDriverTest, DetermineEnabledTest) {
RewriteDriver* driver = rewrite_driver();
DetermineEnabledCheckingFilter* filter =
new DetermineEnabledCheckingFilter();
driver->AddOwnedEarlyPreRenderFilter(filter);
driver->StartParse("http://example.com/index.html");
rewrite_driver()->ParseText("<div>");
driver->Flush();
EXPECT_FALSE(filter->start_document_called());
rewrite_driver()->ParseText("</div>");
driver->FinishParse();
filter = new DetermineEnabledCheckingFilter();
filter->SetEnabled(true);
driver->AddOwnedEarlyPreRenderFilter(filter);
driver->StartParse("http://example.com/index.html");
rewrite_driver()->ParseText("<div>");
driver->Flush();
EXPECT_TRUE(filter->start_document_called());
rewrite_driver()->ParseText("</div>");
driver->FinishParse();
}
// Tests that we access driver->response_headers() before/after Flush(),
// and driver->mutable_response_headers() at only before Flush().
TEST_F(RewriteDriverTest, ResponseHeadersAccess) {
RewriteDriver* driver = rewrite_driver();
ResponseHeaders headers;
driver->set_response_headers_ptr(&headers);
driver->AddOwnedEarlyPreRenderFilter(new ResponseHeadersCheckingFilter(
driver));
driver->AddOwnedPostRenderFilter(new ResponseHeadersCheckingFilter(driver));
// Starting the parse, the base-tag will be derived from the html url.
ASSERT_TRUE(driver->StartParse("http://example.com/index.html"));
rewrite_driver()->ParseText("<div>");
driver->Flush();
rewrite_driver()->ParseText("</div>");
driver->FinishParse();
}
TEST_F(RewriteDriverTest, SetSessionFetcherTest) {
AddFilter(RewriteOptions::kExtendCacheCss);
const char kFetcher1Css[] = "Fetcher #1";
const char kFetcher2Css[] = "Fetcher #2";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kFetcher1Css, 100);
GoogleString url = Encode(kTestDomain, RewriteOptions::kCacheExtenderId,
hasher()->Hash(kFetcher1Css), "a.css", "css");
// Fetch from default.
GoogleString output;
ResponseHeaders response_headers;
EXPECT_TRUE(FetchResourceUrl(url, &output, &response_headers));
EXPECT_STREQ(kFetcher1Css, output);
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
// Load up a different file into a second fetcher.
// We misappropriate the response_headers from previous fetch for simplicity.
scoped_ptr<MockUrlFetcher> mock2(new MockUrlFetcher);
mock2->SetResponse(AbsolutifyUrl("a.css"), response_headers, kFetcher2Css);
// Switch over to new fetcher, making sure to set two of them to exercise
// memory management. Note the synchronous mock fetcher we still have to
// manage ourselves (as the RewriteDriver API is for async ones only).
RewriteDriver* driver = rewrite_driver();
driver->SetSessionFetcher(mock2.release());
CountingUrlAsyncFetcher* counter =
new CountingUrlAsyncFetcher(driver->async_fetcher());
driver->SetSessionFetcher(counter);
EXPECT_EQ(counter, driver->async_fetcher());
// Note that FetchResourceUrl will call driver->Clear() so we cannot
// access 'counter' past this point.
lru_cache()->Clear(); // get rid of cached version of input
EXPECT_TRUE(FetchResourceUrl(url, &output, &response_headers));
EXPECT_STREQ(kFetcher2Css, output);
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
// As FetchResourceUrl has cleared the driver, further fetcher should
// grab fetcher 1 version.
lru_cache()->Clear(); // get rid of cached version of input
EXPECT_TRUE(FetchResourceUrl(url, &output, &response_headers));
EXPECT_STREQ(kFetcher1Css, output);
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
}
class WaitAsyncFetch : public StringAsyncFetch {
public:
WaitAsyncFetch(const RequestContextPtr& req, GoogleString* content,
ThreadSystem* thread_system)
: StringAsyncFetch(req, content),
sync_(thread_system) {
}
virtual ~WaitAsyncFetch() {}
virtual void HandleDone(bool status) {
StringAsyncFetch::HandleDone(status);
sync_.Notify();
}
void Wait() { sync_.Wait(); }
private:
WorkerTestBase::SyncPoint sync_;
};
class InPlaceTest : public RewriteTestBase {
protected:
InPlaceTest() {}
virtual ~InPlaceTest() {}
bool FetchInPlaceResource(const StringPiece& url,
bool proxy_mode,
GoogleString* content,
ResponseHeaders* response) {
GoogleUrl gurl(url);
content->clear();
WaitAsyncFetch async_fetch(CreateRequestContext(), content,
server_context()->thread_system());
async_fetch.set_response_headers(response);
rewrite_driver_->SetRequestHeaders(*async_fetch.request_headers());
rewrite_driver_->FetchInPlaceResource(gurl, proxy_mode,
&async_fetch);
async_fetch.Wait();
// Make sure we let the rewrite complete, and also wait for the driver to be
// idle so we can reuse it safely.
rewrite_driver_->WaitForShutDown();
ClearRewriteDriver(); // makes sure to re-create the request context.
EXPECT_TRUE(async_fetch.done());
return async_fetch.done() && async_fetch.success();
}
bool TryFetchInPlaceResource(const StringPiece& url,
bool proxy_mode) {
GoogleString contents;
ResponseHeaders response;
return FetchInPlaceResource(url, proxy_mode, &contents, &response);
}
private:
DISALLOW_COPY_AND_ASSIGN(InPlaceTest);
};
TEST_F(InPlaceTest, FetchInPlaceResource) {
AddFilter(RewriteOptions::kRewriteCss);
GoogleString url = "http://example.com/foo.css";
SetResponseWithDefaultHeaders(url, kContentTypeCss,
".a { color: red; }", 100);
GoogleString html_url = "http://example.com/foo.html";
SetResponseWithDefaultHeaders(html_url, kContentTypeHtml,
"<b>Bold!</b>", 100);
// This will fail because cache is empty and we are not allowing HTTP fetch.
EXPECT_FALSE(TryFetchInPlaceResource(url, false /* proxy_mode */));
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
ClearStats();
// Now we allow HTTP fetches and we expect success.
EXPECT_TRUE(TryFetchInPlaceResource(url, true /* proxy_mode */));
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
// We insert both original and rewritten resources.
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
ClearStats();
// Now that we've loaded the resource into cache, we expect success.
EXPECT_TRUE(TryFetchInPlaceResource(url, false /* proxy_mode */));
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
ClearStats();
// In proxy mode, we should successfully pass through HTML.
EXPECT_TRUE(TryFetchInPlaceResource(html_url, true /* proxy_mode */));
// In non-proxy mode producing HTML should fail; it's expected the origin
// server would produce things we aren't optimizing through its usual
// code paths. Note that this needs to happen after the previous call so that
// we get the resource into cache.
EXPECT_FALSE(TryFetchInPlaceResource(html_url, false /* proxy_mode */));
}
TEST_F(InPlaceTest, InPlaceCssDebug) {
// Regression test: ipro + debug would crash when a debug message was
// produced.
options()->EnableFilter(RewriteOptions::kDebug);
options()->EnableFilter(RewriteOptions::kFlattenCssImports);
AddFilter(RewriteOptions::kRewriteCss);
GoogleString url = "http://example.com/foo.css";
SetResponseWithDefaultHeaders(url, kContentTypeCss,
"@import \"weird://foo\"; .a { color: red; }",
100);
EXPECT_TRUE(TryFetchInPlaceResource(url, true /* proxy_mode */));
}
TEST_F(RewriteDriverTest, DebugModeTest) {
// Verify that DebugMode() corresponds to RewriteOptions::kDebug as expected
EXPECT_FALSE(rewrite_driver()->DebugMode());
options()->EnableFilter(RewriteOptions::kDebug);
EXPECT_TRUE(rewrite_driver()->DebugMode());
options()->DisableFilter(RewriteOptions::kDebug);
EXPECT_FALSE(rewrite_driver()->DebugMode());
}
TEST_F(RewriteDriverTest, CachePollutionWithWrongEncodingCharacter) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("dir/a.css", kContentTypeCss, kCss, 100);
GoogleString css_wrong_url =
"http://test.com/dir/B.a.css.pagespeed.cf.0.css";
GoogleString correct_url = Encode(
"dir/", RewriteOptions::kCssFilterId, hasher()->Hash(kCss),
"a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_wrong_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
EXPECT_EQ(kFoundResult,
HttpBlockingFindStatus(StrCat(kTestDomain, correct_url),
http_cache()));
GoogleString input_html(CssLinkHref("dir/a.css"));
GoogleString output_html(CssLinkHref(correct_url));
ValidateExpected("wrong_encoding", input_html, output_html);
}
TEST_F(RewriteDriverTest, CachePollutionWithLowerCasedncodingCharacter) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("dir/a.css", kContentTypeCss, kCss, 100);
GoogleString css_wrong_url =
"http://test.com/dir/a.a.css.pagespeed.cf.0.css";
GoogleString correct_url = Encode(
"dir/", RewriteOptions::kCssFilterId, hasher()->Hash(kCss),
"a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_wrong_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
EXPECT_EQ(kFoundResult,
HttpBlockingFindStatus(StrCat(kTestDomain, correct_url),
http_cache()));
GoogleString input_html(CssLinkHref("dir/a.css"));
GoogleString output_html(CssLinkHref(correct_url));
ValidateExpected("wrong_encoding", input_html, output_html);
}
TEST_F(RewriteDriverTest, CachePollutionWithExperimentId) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("dir/a.css", kContentTypeCss, kCss, 100);
GoogleString css_wrong_url =
"http://test.com/dir/A.a.css.pagespeed.b.cf.0.css";
GoogleString correct_url = Encode(
"dir/", RewriteOptions::kCssFilterId, hasher()->Hash(kCss),
"a.css", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_wrong_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
EXPECT_EQ(kFoundResult,
HttpBlockingFindStatus(StrCat(kTestDomain, correct_url),
http_cache()));
GoogleString input_html(CssLinkHref("dir/a.css"));
GoogleString output_html(CssLinkHref(correct_url));
ValidateExpected("wrong_encoding", input_html, output_html);
}
TEST_F(RewriteDriverTest, CachePollutionWithQueryParams) {
AddFilter(RewriteOptions::kRewriteCss);
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("dir/a.css?ver=3", kContentTypeCss, kCss, 100);
GoogleString css_wrong_url =
"http://test.com/dir/A.a.css,qver%3D3.pagespeed.cf.0.css";
GoogleString correct_url = Encode(
"dir/", RewriteOptions::kCssFilterId, hasher()->Hash(kCss),
"a.css?ver=3", "css");
// Cold load.
EXPECT_TRUE(TryFetchResource(css_wrong_url));
// We should have 3 things inserted:
// 1) the source data
// 2) the result
// 3) the rname entry for the result
int cold_num_inserts = lru_cache()->num_inserts();
EXPECT_EQ(3, cold_num_inserts);
EXPECT_EQ(kFoundResult,
HttpBlockingFindStatus(StrCat(kTestDomain, correct_url),
http_cache()));
GoogleString input_html(CssLinkHref("dir/a.css?ver=3"));
GoogleString output_html(CssLinkHref(correct_url));
ValidateExpected("wrong_encoding", input_html, output_html);
}
TEST_F(RewriteDriverTest, NoLoggingForImagesRewrittenInsideCss) {
options()->set_image_inline_max_bytes(100000);
options()->EnableFilter(RewriteOptions::kExtendCacheCss);
options()->EnableFilter(RewriteOptions::kRewriteCss);
options()->EnableFilter(RewriteOptions::kExtendCacheImages);
options()->EnableFilter(RewriteOptions::kRecompressPng);
options()->set_always_rewrite_css(true);
rewrite_driver_->AddFilters();
GoogleString contents = "#a {background:url(1.png) ;}";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, contents, 100);
AddFileToMockFetcher(StrCat(kTestDomain, "1.png"), kBikePngFile,
kContentTypePng, 100);
GoogleString correct_url = Encode(
"", RewriteOptions::kCssFilterId, hasher()->Hash(contents),
"a.css", "css");
GoogleString input_html(CssLinkHref("a.css"));
GoogleString output_html(CssLinkHref(correct_url));
ValidateExpected("no_logging_images_inside_css", input_html, output_html);
LoggingInfo* logging_info = rewrite_driver_->log_record()->logging_info();
ASSERT_EQ(1, logging_info->rewriter_info_size());
EXPECT_EQ("cf", logging_info->rewriter_info(0).id());
}
TEST_F(RewriteDriverTest, DecodeMultiUrlsEncodesCorrectly) {
options()->EnableFilter(RewriteOptions::kRewriteCss);
options()->EnableFilter(RewriteOptions::kCombineCss);
rewrite_driver()->AddFilters();
const char kCss[] = "* { display: none; }";
SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCss, 100);
SetResponseWithDefaultHeaders("test/b.css", kContentTypeCss, kCss, 100);
// Combine filters
GoogleString multi_url = Encode(
"", RewriteOptions::kCssFilterId, hasher()->Hash(kCss),
"a.css+test,_b.css.pagespeed.cc.0.css", "css");
EXPECT_TRUE(TryFetchResource(StrCat(kTestDomain, multi_url)));
GoogleString input_html(
StrCat(CssLinkHref("a.css"), CssLinkHref("test/b.css")));
ParseUrl(kTestDomain, input_html);
StringVector css_urls;
CollectCssLinks("multi", output_buffer_, &css_urls);
EXPECT_EQ(1UL, css_urls.size());
EXPECT_EQ(multi_url, css_urls[0]);
}
// Records URL of the last img element it sees at point of RenderDone().
class RenderDoneCheckingFilter : public EmptyHtmlFilter {
public:
RenderDoneCheckingFilter() : element_(NULL) {}
virtual ~RenderDoneCheckingFilter() {}
const GoogleString& src() const { return src_; }
protected:
virtual void StartElement(HtmlElement* element) {
if (element->keyword() == HtmlName::kImg) {
element_ = element;
}
}
virtual void RenderDone() {
if (element_ != NULL) {
const char* val = element_->AttributeValue(HtmlName::kSrc);
src_ = (val != NULL ? val : "");
}
}
virtual const char* Name() const { return "RenderDoneCheckingFilter"; }
private:
HtmlElement* element_;
GoogleString src_;
DISALLOW_COPY_AND_ASSIGN(RenderDoneCheckingFilter);
};
TEST_F(RewriteDriverTest, RenderDoneTest) {
// Test to make sure RenderDone sees output of a pre-render filter.
RewriteDriver* driver = rewrite_driver();
RenderDoneCheckingFilter* filter =
new RenderDoneCheckingFilter();
driver->AddOwnedEarlyPreRenderFilter(filter);
SetResponseWithDefaultHeaders("a.png", kContentTypePng, "PNGkinda", 100);
AddFilter(RewriteOptions::kExtendCacheImages);
driver->StartParse(kTestDomain);
rewrite_driver()->ParseText("<img src=\"a.png\">");
driver->FinishParse();
EXPECT_EQ(Encode("", RewriteOptions::kCacheExtenderId, "0", "a.png", "png"),
filter->src());
}
TEST_F(RewriteDriverTest, BlockingRewriteFlagTest) {
RequestHeaders request_headers;
RewriteDriver* driver = rewrite_driver();
options()->ClearSignatureForTesting();
options()->set_blocking_rewrite_key("blocking");
options()->ComputeSignature();
// case 1.
TestBlockingRewrite(&request_headers, false, true);
// case 2.
request_headers.Add(HttpAttributes::kXPsaBlockingRewrite, "not-blocking");
TestBlockingRewrite(&request_headers, false, true);
// case 3.
request_headers.Add(HttpAttributes::kXPsaBlockingRewrite, "blocking");
TestBlockingRewrite(&request_headers, true, true);
// case 4.
request_headers.Add(HttpAttributes::kXPsaBlockingRewrite, "blocking");
request_headers.Add(HttpAttributes::kXPsaBlockingRewriteMode, "junk");
TestBlockingRewrite(&request_headers, true, true);
// case 5.
request_headers.Add(HttpAttributes::kXPsaBlockingRewrite, "blocking");
request_headers.Add(HttpAttributes::kXPsaBlockingRewriteMode, "slow");
TestBlockingRewrite(&request_headers, true, false);
options()->ClearSignatureForTesting();
options()->EnableBlockingRewriteForRefererUrlPattern("http://example.com");
options()->ComputeSignature();
// case 6.
request_headers.Add(HttpAttributes::kReferer, "http://junk.com/");
driver->EnableBlockingRewrite(&request_headers);
TestBlockingRewrite(&request_headers, false, true);
// case 7.
request_headers.RemoveAll(HttpAttributes::kReferer);
request_headers.Add(HttpAttributes::kReferer, "http://example.com");
request_headers.Add(HttpAttributes::kXPsaBlockingRewriteMode, "junk");
TestBlockingRewrite(&request_headers, true, true);
// case 8.
request_headers.RemoveAll(HttpAttributes::kReferer);
request_headers.Add(HttpAttributes::kReferer, "http://example.com");
request_headers.Add(HttpAttributes::kXPsaBlockingRewriteMode, "slow");
TestBlockingRewrite(&request_headers, true, false);
}
TEST_F(RewriteDriverTest, PendingAsyncEventsTest) {
RewriteDriver* driver = rewrite_driver();
driver->set_fully_rewrite_on_flush(true);
driver->set_fast_blocking_rewrite(true);
TestPendingEventsIsDone(true);
// Only when we are doing a slow blocking rewrite (waiting for async events),
// IsDone() returns false for kWaitForCompletion.
driver->set_fully_rewrite_on_flush(true);
driver->set_fast_blocking_rewrite(false);
TestPendingEventsIsDone(false);
driver->set_fully_rewrite_on_flush(false);
driver->set_fast_blocking_rewrite(true);
TestPendingEventsIsDone(true);
driver->set_fully_rewrite_on_flush(false);
driver->set_fast_blocking_rewrite(false);
TestPendingEventsIsDone(true);
// Make sure we properly cleanup as well.
TestPendingEventsDriverCleanup(false, false);
TestPendingEventsDriverCleanup(false, true);
TestPendingEventsDriverCleanup(true, false);
TestPendingEventsDriverCleanup(true, true);
}
TEST_F(RewriteDriverTest, PendingRenderBlockingAsyncEventsTest) {
RewriteDriver* driver = rewrite_driver();
driver->set_fully_rewrite_on_flush(false);
// Plain async event doesn't prevent completion.
driver->IncrementAsyncEventsCount();
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, false));
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, true));
// Render blocking one does, however.
driver->IncrementRenderBlockingAsyncEventsCount();
EXPECT_FALSE(IsDone(RewriteDriver::kWaitForCompletion, false));
EXPECT_FALSE(IsDone(RewriteDriver::kWaitForCompletion, true));
driver->DecrementAsyncEventsCount();
// Still does when regular async removed.
EXPECT_FALSE(IsDone(RewriteDriver::kWaitForCompletion, false));
EXPECT_FALSE(IsDone(RewriteDriver::kWaitForCompletion, true));
// Once all counts are gone it's now Done, though.
driver->DecrementRenderBlockingAsyncEventsCount();
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, false));
EXPECT_TRUE(IsDone(RewriteDriver::kWaitForCompletion, true));
}
TEST_F(RewriteDriverTest, ValidateCacheResponseRewrittenWebp) {
const StringPiece kWebpMimeType = kContentTypeWebp.mime_type();
RequestContextPtr request_context(new RequestContext(
kDefaultHttpOptionsForTests, new NullMutex, timer()));
options()->ClearSignatureForTesting();
ResponseHeaders response_headers;
response_headers.Add(HttpAttributes::kContentType, kWebpMimeType);
response_headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms,
300 * Timer::kSecondMs, "");
response_headers.ComputeCaching();
const char kOriginUrl[] = "foo.webp";
// No vary:accept, accepts_webp false. Note that we ignore the lack of
// browser capability to display webp and send it anyway.
request_context->SetAcceptsWebp(false);
options()->set_serve_rewritten_webp_urls_to_any_agent(true);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
options()->set_serve_rewritten_webp_urls_to_any_agent(false);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
// no vary:accept, accepts_webp true.
request_context->SetAcceptsWebp(true);
options()->set_serve_rewritten_webp_urls_to_any_agent(true);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
options()->set_serve_rewritten_webp_urls_to_any_agent(false);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
// Now add a Vary: Accept and we'll start paying attention to the
// browser capabilities.
response_headers.Add(HttpAttributes::kVary, HttpAttributes::kAccept);
response_headers.ComputeCaching();
request_context->SetAcceptsWebp(false);
// vary:accept, accepts_webp false.
options()->set_serve_rewritten_webp_urls_to_any_agent(true);
EXPECT_FALSE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
options()->set_serve_rewritten_webp_urls_to_any_agent(false);
EXPECT_FALSE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
// vary:accept, accepts_webp true.
request_context->SetAcceptsWebp(true);
options()->set_serve_rewritten_webp_urls_to_any_agent(true);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
options()->set_serve_rewritten_webp_urls_to_any_agent(false);
EXPECT_TRUE(OptionsAwareHTTPCacheCallback::IsCacheValid(
kOriginUrl, *options(), request_context, response_headers));
}
TEST_F(RewriteDriverTest, SetRequestHeadersPopulatesWebpAccept) {
RequestHeaders headers;
headers.Add(HttpAttributes::kAccept, "image/webp");
headers.Add(HttpAttributes::kUserAgent,
UserAgentMatcherTestBase::kChrome42UserAgent);
rewrite_driver()->SetRequestHeaders(headers);
const RequestProperties* request_properties =
rewrite_driver()->request_properties();
EXPECT_TRUE(request_properties->SupportsWebpInPlace());
EXPECT_TRUE(request_properties->SupportsWebpRewrittenUrls());
EXPECT_TRUE(request_properties->SupportsWebpLosslessAlpha());
}
TEST_F(RewriteDriverTest, SetRequestHeadersPopulatesWebpNoAccept) {
RequestHeaders headers;
headers.Add(HttpAttributes::kUserAgent,
UserAgentMatcherTestBase::kAndroidICSUserAgent);
rewrite_driver()->SetRequestHeaders(headers);
const RequestProperties* request_properties =
rewrite_driver()->request_properties();
EXPECT_FALSE(request_properties->SupportsWebpInPlace());
EXPECT_TRUE(request_properties->SupportsWebpRewrittenUrls());
EXPECT_FALSE(request_properties->SupportsWebpLosslessAlpha());
}
// Test classes created for using a managed rewrite driver, so that downstream
// caching behavior (especially cache purging) can be tested. Since managed
// rewrite drivers need their filters to be setup before the custom rewrite
// driver is constructed, we need these classes with specific SetUp methods
// for configuring the options.
// This class has ExtendCacheCss filter enabled and has possibility of
// purge requests for the html because of the resources not being
// rewritten in the very first go.
class DownstreamCacheWithPossiblePurgeTest : public RewriteDriverTest {
protected:
void SetUp() {
options()->EnableFilter(RewriteOptions::kExtendCacheCss);
SetUseManagedRewriteDrivers(true);
RewriteDriverTest::SetUp();
}
};
// This class has CollapseWhitespace filter enabled and has no possibility of
// purge requests for the html because the html will always get fully rewritten
// in the very first go.
class DownstreamCacheWithNoPossiblePurgeTest : public RewriteDriverTest {
protected:
void SetUp() {
options()->EnableFilter(RewriteOptions::kCollapseWhitespace);
SetUseManagedRewriteDrivers(true);
RewriteDriverTest::SetUp();
}
};
TEST_F(DownstreamCacheWithPossiblePurgeTest, DownstreamCacheEnabled) {
SetDownstreamCacheDirectives("GET", "http://localhost:1234/purge", "");
// Use a wait fetcher so that the response does not get a chance to get
// rewritten.
SetupWaitFetcher();
// Since we want to call both FinishParse() and WaitForCompletion() (it's
// inside CallFetcherCallbacksForDriver) on a managed rewrite driver,
// we have to pin it, since otherwise FinishParse will drop our last
// reference.
rewrite_driver()->AddUserReference();
SetupResponsesForDownstreamCacheTesting();
// Setup request headers since the subsequent purge request needs this.
RequestHeaders request_headers;
rewrite_driver()->SetRequestHeaders(request_headers);
ProcessHtmlForDownstreamCacheTesting();
EXPECT_STREQ(kNonRewrittenCachableHtml, output_buffer_);
// Since the response would now have been generated (without any rewriting,
// because neither of the 2 resource fetches for a.css and b.css
// would have completed), we allow the fetches to complete now.
factory()->CallFetcherCallbacksForDriver(rewrite_driver());
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
// Now we want to permit fetches to go ahead once we let purge happen
factory()->wait_url_async_fetcher()->SetPassThroughMode(true);
rewrite_driver()->Cleanup(); // Drop our ref, to let purge go ahead.
// We can actually check the result of flush already because
// our fetcher is immediate.
EXPECT_EQ(3, counting_url_async_fetcher()->fetch_count());
EXPECT_STREQ("http://localhost:1234/purge/",
counting_url_async_fetcher()->most_recent_fetched_url());
EXPECT_EQ(1, factory()->rewrite_stats()->
downstream_cache_purge_attempts()->Get());
}
TEST_F(DownstreamCacheWithPossiblePurgeTest, DownstreamCacheDisabled) {
SetDownstreamCacheDirectives("GET", "", "");
// Use a wait fetcher so that the response does not get a chance to get
// rewritten.
SetupWaitFetcher();
// Since we want to call both FinishParse() and WaitForCompletion() (it's
// inside CallFetcherCallbacksForDriver) on a managed rewrite driver,
// we have to pin it, since otherwise FinishParse will drop our last
// reference.
rewrite_driver()->AddUserReference();
SetupResponsesForDownstreamCacheTesting();
// Setup request headers since the subsequent purge request needs this.
RequestHeaders request_headers;
rewrite_driver()->SetRequestHeaders(request_headers);
ProcessHtmlForDownstreamCacheTesting();
EXPECT_STREQ(kNonRewrittenCachableHtml, output_buffer_);
// Since the response would now have been generated (without any rewriting,
// because neither of the 2 resource fetches for a.css and b.css
// would have completed), we allow the fetches to complete now.
factory()->CallFetcherCallbacksForDriver(rewrite_driver());
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
// The purge-request-fetch can be allowed to complete without any waiting.
// Hence, we set the pass-through-mode to true.
factory()->wait_url_async_fetcher()->SetPassThroughMode(true);
rewrite_driver()->Cleanup(); // Drop our ref, to let any purge go ahead.
// We expect no purges in this flow.
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
EXPECT_STREQ("http://test.com/test/b.css",
counting_url_async_fetcher()->most_recent_fetched_url());
EXPECT_EQ(0, factory()->rewrite_stats()->
downstream_cache_purge_attempts()->Get());
}
TEST_F(DownstreamCacheWithPossiblePurgeTest,
DownstreamCache100PercentRewritten) {
SetDownstreamCacheDirectives("GET", "http://localhost:1234/purge", "");
// Do not use a wait fetcher here because we want both the fetches (for a.css
// and b.css) and their rewrites to finish before the response is served out.
SetupResponsesForDownstreamCacheTesting();
// Setup request headers since the subsequent purge request needs this.
RequestHeaders request_headers;
rewrite_driver()->SetRequestHeaders(request_headers);
ProcessHtmlForDownstreamCacheTesting();
EXPECT_STREQ(kRewrittenCachableHtmlWithCacheExtension, output_buffer_);
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
// We expect no purges in this flow.
EXPECT_EQ(2, counting_url_async_fetcher()->fetch_count());
EXPECT_STREQ("http://test.com/test/b.css",
counting_url_async_fetcher()->most_recent_fetched_url());
EXPECT_EQ(0, factory()->rewrite_stats()->
downstream_cache_purge_attempts()->Get());
}
TEST_F(DownstreamCacheWithNoPossiblePurgeTest, DownstreamCacheNoInitRewrites) {
SetDownstreamCacheDirectives("GET", "http://localhost:1234/purge", "");
// Use a wait fetcher so that the response does not get a chance to get
// rewritten.
SetupWaitFetcher();
rewrite_driver()->AddUserReference();
SetupResponsesForDownstreamCacheTesting();
// Setup request headers since the subsequent purge request needs this.
RequestHeaders request_headers;
rewrite_driver()->SetRequestHeaders(request_headers);
ProcessHtmlForDownstreamCacheTesting();
EXPECT_STREQ(kRewrittenCachableHtmlWithCollapseWhitespace, output_buffer_);
// Since only collapse-whitespace is enabled in this test, we do not expect
// any fetches (or fetch callbacks for the wait fetcher) here.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
// Release RewriteDriver and trigger any purge.
rewrite_driver()->Cleanup();
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, factory()->rewrite_stats()->
downstream_cache_purge_attempts()->Get());
}
} // namespace
} // namespace net_instaweb