| /* |
| * Copyright 2011 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // Author: morlovich@google.com (Maksim Orlovich) |
| |
| // Unit-tests for ProxyInterface. |
| |
| #include "pagespeed/automatic/proxy_interface.h" |
| |
| #include "net/instaweb/http/public/async_fetch.h" |
| #include "net/instaweb/http/public/http_cache.h" |
| #include "net/instaweb/http/public/logging_proto_impl.h" |
| #include "net/instaweb/http/public/mock_callback.h" |
| #include "net/instaweb/http/public/mock_url_fetcher.h" |
| #include "net/instaweb/http/public/reflecting_test_fetcher.h" |
| #include "net/instaweb/http/public/request_context.h" |
| #include "net/instaweb/http/public/request_timing_info.h" |
| #include "net/instaweb/rewriter/public/blink_util.h" |
| #include "net/instaweb/rewriter/public/domain_lawyer.h" |
| #include "net/instaweb/rewriter/public/experiment_util.h" |
| #include "net/instaweb/rewriter/public/rewrite_driver.h" |
| #include "net/instaweb/rewriter/public/rewrite_options.h" |
| #include "net/instaweb/rewriter/public/rewrite_test_base.h" |
| #include "net/instaweb/rewriter/public/server_context.h" |
| #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" |
| #include "net/instaweb/util/public/fallback_property_page.h" |
| #include "net/instaweb/util/public/mock_property_page.h" |
| #include "net/instaweb/util/public/property_cache.h" |
| #include "pagespeed/automatic/proxy_fetch.h" |
| #include "pagespeed/automatic/proxy_interface_test_base.h" |
| #include "pagespeed/kernel/base/abstract_mutex.h" |
| #include "pagespeed/kernel/base/basictypes.h" |
| #include "pagespeed/kernel/base/function.h" |
| #include "pagespeed/kernel/base/gtest.h" |
| #include "pagespeed/kernel/base/mock_message_handler.h" |
| #include "pagespeed/kernel/base/mock_timer.h" |
| #include "pagespeed/kernel/base/null_message_handler.h" |
| #include "pagespeed/kernel/base/scoped_ptr.h" |
| #include "pagespeed/kernel/base/statistics.h" |
| #include "pagespeed/kernel/base/string.h" |
| #include "pagespeed/kernel/base/string_util.h" |
| #include "pagespeed/kernel/base/thread_system.h" |
| #include "pagespeed/kernel/base/time_util.h" |
| #include "pagespeed/kernel/base/timer.h" |
| #include "pagespeed/kernel/cache/lru_cache.h" |
| #include "pagespeed/kernel/html/html_parse_test_base.h" |
| #include "pagespeed/kernel/http/content_type.h" |
| #include "pagespeed/kernel/http/google_url.h" |
| #include "pagespeed/kernel/http/http_names.h" |
| #include "pagespeed/kernel/http/request_headers.h" |
| #include "pagespeed/kernel/http/response_headers.h" |
| #include "pagespeed/kernel/http/semantic_type.h" |
| #include "pagespeed/kernel/http/user_agent_matcher.h" |
| #include "pagespeed/kernel/thread/mock_scheduler.h" |
| #include "pagespeed/kernel/thread/queued_worker_pool.h" |
| #include "pagespeed/kernel/thread/thread_synchronizer.h" |
| #include "pagespeed/kernel/thread/worker_test_base.h" |
| |
| namespace net_instaweb { |
| |
| namespace { |
| |
| // This jpeg file lacks a .jpg or .jpeg extension. So we initiate |
| // a property-cache read prior to getting the response-headers back, |
| // but will never go into the ProxyFetch flow that blocks waiting |
| // for the cache lookup to come back. |
| const char kImageFilenameLackingExt[] = "jpg_file_lacks_ext"; |
| const char kHttpsPageUrl[] = "https://www.test.com/page.html"; |
| const char kHttpsCssUrl[] = "https://www.test.com/style.css"; |
| |
| const char kCssContent[] = "* { display: none; }"; |
| const char kMinimizedCssContent[] = "*{display:none}"; |
| |
| } // namespace |
| |
| class ProxyInterfaceTest : public ProxyInterfaceTestBase { |
| protected: |
| static const int kHtmlCacheTimeSec = 5000; |
| |
| ProxyInterfaceTest() |
| : start_time_ms_(0), |
| max_age_300_("max-age=300") { |
| ConvertTimeToString(MockTimer::kApr_5_2010_ms, &start_time_string_); |
| ConvertTimeToString(MockTimer::kApr_5_2010_ms + 5 * Timer::kMinuteMs, |
| &start_time_plus_300s_string_); |
| ConvertTimeToString(MockTimer::kApr_5_2010_ms - 2 * Timer::kDayMs, |
| &old_time_string_); |
| } |
| virtual ~ProxyInterfaceTest() {} |
| |
| virtual void SetUp() { |
| ThreadSynchronizer* sync = server_context()->thread_synchronizer(); |
| sync->EnableForPrefix(ProxyFetch::kCollectorDoneFinish); |
| sync->EnableForPrefix(ProxyFetch::kCollectorDetachFinish); |
| sync->EnableForPrefix(ProxyFetch::kCollectorConnectProxyFetchFinish); |
| server_context_->set_enable_property_cache(true); |
| const PropertyCache::Cohort* dom_cohort = |
| SetupCohort(server_context_->page_property_cache(), |
| RewriteDriver::kDomCohort); |
| const PropertyCache::Cohort* blink_cohort = |
| SetupCohort(server_context_->page_property_cache(), |
| BlinkUtil::kBlinkCohort); |
| server_context()->set_dom_cohort(dom_cohort); |
| server_context()->set_blink_cohort(blink_cohort); |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->EnableFilter(RewriteOptions::kRewriteCss); |
| options->set_max_html_cache_time_ms(kHtmlCacheTimeSec * Timer::kSecondMs); |
| options->set_in_place_rewriting_enabled(true); |
| options->Disallow("*blacklist*"); |
| server_context()->ComputeSignature(options); |
| ProxyInterfaceTestBase::SetUp(); |
| // The original url_async_fetcher() is still owned by RewriteDriverFactory. |
| background_fetch_fetcher_.reset(new BackgroundFetchCheckingUrlAsyncFetcher( |
| factory()->ComputeUrlAsyncFetcher())); |
| server_context()->set_default_system_fetcher( |
| background_fetch_fetcher_.get()); |
| |
| start_time_ms_ = timer()->NowMs(); |
| |
| SetResponseWithDefaultHeaders(kImageFilenameLackingExt, kContentTypeJpeg, |
| "image data", 300); |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| "<div><p></p></div>", 0); |
| } |
| |
| void CheckHeaders(const ResponseHeaders& headers, |
| const ContentType& expect_type) { |
| ASSERT_TRUE(headers.has_status_code()); |
| EXPECT_EQ(HttpStatus::kOK, headers.status_code()); |
| EXPECT_STREQ(expect_type.mime_type(), |
| headers.Lookup1(HttpAttributes::kContentType)); |
| } |
| |
| void CheckBackgroundFetch(const ResponseHeaders& headers, |
| bool is_background_fetch) { |
| EXPECT_STREQ(is_background_fetch ? "1" : "0", |
| headers.Lookup1(kBackgroundFetchHeader)); |
| } |
| |
| void CheckNumBackgroundFetches(int num) { |
| EXPECT_EQ(num, background_fetch_fetcher_->num_background_fetches()); |
| } |
| |
| virtual void ClearStats() { |
| RewriteTestBase::ClearStats(); |
| background_fetch_fetcher_->clear_num_background_fetches(); |
| } |
| |
| // Serve a trivial HTML page with initial Cache-Control header set to |
| // input_cache_control and return the Cache-Control header after running |
| // through ProxyInterface. |
| // |
| // A unique id must be set to assure different websites are requested. |
| // id is put in a URL, so it probably shouldn't have spaces and other |
| // special chars. |
| GoogleString RewriteHtmlCacheHeader(const StringPiece& id, |
| const StringPiece& input_cache_control) { |
| GoogleString url = StrCat("http://www.example.com/", id, ".html"); |
| ResponseHeaders input_headers; |
| DefaultResponseHeaders(kContentTypeHtml, 100, &input_headers); |
| input_headers.Replace(HttpAttributes::kCacheControl, input_cache_control); |
| SetFetchResponse(url, input_headers, "<body>Foo</body>"); |
| |
| GoogleString body; |
| ResponseHeaders output_headers; |
| FetchFromProxy(url, true, &body, &output_headers); |
| return output_headers.LookupJoined(HttpAttributes::kCacheControl); |
| } |
| |
| int GetStatusCodeInPropertyCache(const GoogleString& url) { |
| PropertyCache* pcache = page_property_cache(); |
| scoped_ptr<MockPropertyPage> page(NewMockPage( |
| url, |
| server_context()->GetRewriteOptionsSignatureHash( |
| server_context()->global_options()), |
| UserAgentMatcher::kDesktop)); |
| const PropertyCache::Cohort* cohort = pcache->GetCohort( |
| RewriteDriver::kDomCohort); |
| PropertyValue* value; |
| pcache->Read(page.get()); |
| value = page->GetProperty( |
| cohort, RewriteDriver::kStatusCodePropertyName); |
| int status_code; |
| EXPECT_TRUE(StringToInt(value->value(), &status_code)); |
| return status_code; |
| } |
| |
| void DisableAjax() { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_in_place_rewriting_enabled(false); |
| server_context()->ComputeSignature(options); |
| } |
| |
| void RejectBlacklisted() { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_reject_blacklisted(true); |
| options->set_reject_blacklisted_status_code(HttpStatus::kImATeapot); |
| server_context()->ComputeSignature(options); |
| } |
| |
| // This function is primarily meant to enable writes to the dom cohort of the |
| // property cache. Writes to this cohort are predicated on a filter that uses |
| // that cohort being enabled, which includes the insert dns prefetch filter. |
| void EnableDomCohortWritesWithDnsPrefetch() { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->EnableFilter(RewriteOptions::kInsertDnsPrefetch); |
| server_context()->ComputeSignature(options); |
| } |
| |
| void TestFallbackPageProperties( |
| const GoogleString& url, const GoogleString& fallback_url) { |
| GoogleUrl gurl(url); |
| GoogleString kPropertyName("prop"); |
| GoogleString kValue("value"); |
| options()->set_use_fallback_property_cache_values(true); |
| // No fallback value is present. |
| const PropertyCache::Cohort* cohort = |
| page_property_cache()->GetCohort(RewriteDriver::kDomCohort); |
| StringAsyncFetch callback( |
| RequestContext::NewTestRequestContext( |
| server_context()->thread_system())); |
| RequestHeaders request_headers; |
| callback.set_request_headers(&request_headers); |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> callback_collector( |
| proxy_interface_->InitiatePropertyCacheLookup( |
| false, gurl, options(), &callback, false)); |
| |
| FallbackPropertyPage* fallback_page = |
| callback_collector->fallback_property_page(); |
| fallback_page->UpdateValue(cohort, kPropertyName, kValue); |
| fallback_page->WriteCohort(cohort); |
| |
| // Read from fallback value. |
| GoogleUrl new_gurl(fallback_url); |
| callback_collector.reset(proxy_interface_->InitiatePropertyCacheLookup( |
| false, new_gurl, options(), &callback, false)); |
| fallback_page = callback_collector->fallback_property_page(); |
| EXPECT_FALSE(fallback_page->actual_property_page()->GetProperty( |
| cohort, kPropertyName)->has_value()); |
| EXPECT_EQ(kValue, |
| fallback_page->GetProperty(cohort, kPropertyName)->value()); |
| |
| // If use_fallback_property_cache_values option is set to false, fallback |
| // values will not be used. |
| options()->ClearSignatureForTesting(); |
| options()->set_use_fallback_property_cache_values(false); |
| callback_collector.reset(proxy_interface_->InitiatePropertyCacheLookup( |
| false, new_gurl, options(), &callback, false)); |
| EXPECT_FALSE(callback_collector->fallback_property_page()->GetProperty( |
| cohort, kPropertyName)->has_value()); |
| } |
| |
| void TestQueryParameters(StringPiece original_domain, |
| StringPiece redirect_domain, |
| bool add_params_to_redirect, |
| bool add_params_to_location) { |
| // Test to check if we re-add our query params when we get a redirect. |
| const char kParams[] = "PageSpeedCssInlineMaxBytes=99"; |
| GoogleString original_url = StrCat(original_domain, "a/?x=y"); |
| GoogleString redirect_url = StrCat(redirect_domain, "b/", |
| add_params_to_redirect ? "?x=y" : ""); |
| GoogleString set_text, get_text; |
| RequestHeaders request_headers; |
| ResponseHeaders set_headers, get_headers; |
| NullMessageHandler handler; |
| GoogleString expected_location(redirect_url); |
| if (add_params_to_location) { |
| StrAppend(&expected_location, |
| add_params_to_redirect ? "&" : "?", kParams); |
| } |
| |
| set_headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| set_headers.Add(HttpAttributes::kLocation, redirect_url); |
| set_headers.SetStatusAndReason(HttpStatus::kFound); |
| set_text = "<html></html>"; |
| mock_url_fetcher_.SetResponse(original_url, set_headers, set_text); |
| FetchFromProxy(StrCat(original_url, "&", kParams), request_headers, true, |
| &get_text, &get_headers); |
| |
| EXPECT_STREQ(StrCat("HTTP/1.0 302 Found\r\n" |
| "Content-Type: text/html\r\n" |
| "Location: ", expected_location, "\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "Date: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Expires: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Cache-Control: max-age=0, private\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n"), |
| get_headers.ToString()); |
| } |
| |
| scoped_ptr<BackgroundFetchCheckingUrlAsyncFetcher> background_fetch_fetcher_; |
| int64 start_time_ms_; |
| GoogleString start_time_string_; |
| GoogleString start_time_plus_300s_string_; |
| GoogleString old_time_string_; |
| const GoogleString max_age_300_; |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(ProxyInterfaceTest); |
| }; |
| |
| TEST_F(ProxyInterfaceTest, LoggingInfo) { |
| GoogleString url = "http://www.example.com/"; |
| GoogleString text; |
| RequestHeaders request_headers; |
| ResponseHeaders headers; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| |
| // Fetch HTML content. |
| mock_url_fetcher_.SetResponse(url, headers, "<html></html>"); |
| FetchFromProxy(url, request_headers, true, &text, &headers); |
| |
| CheckBackgroundFetch(headers, false); |
| CheckNumBackgroundFetches(0); |
| const RequestTimingInfo& rti = timing_info(); |
| int64 latency_ms; |
| ASSERT_TRUE(rti.GetHTTPCacheLatencyMs(&latency_ms)); |
| EXPECT_EQ(0, latency_ms); |
| EXPECT_FALSE(rti.GetL2HTTPCacheLatencyMs(&latency_ms)); |
| |
| EXPECT_FALSE(rti.GetFetchHeaderLatencyMs(&latency_ms)); |
| EXPECT_FALSE(rti.GetFetchLatencyMs(&latency_ms)); |
| EXPECT_TRUE(logging_info()->is_html_response()); |
| EXPECT_FALSE(logging_info()->is_url_disallowed()); |
| EXPECT_FALSE(logging_info()->is_request_disabled()); |
| EXPECT_FALSE(logging_info()->is_pagespeed_resource()); |
| |
| // Fetch non-HTML content. |
| logging_info()->Clear(); |
| mock_url_fetcher_.SetResponse(url, headers, "js"); |
| FetchFromProxy(url, request_headers, true, &text, &headers); |
| EXPECT_FALSE(logging_info()->is_html_response()); |
| EXPECT_FALSE(logging_info()->is_url_disallowed()); |
| EXPECT_FALSE(logging_info()->is_request_disabled()); |
| |
| // Fetch blacklisted url. |
| url = "http://www.blacklist.com/"; |
| logging_info()->Clear(); |
| mock_url_fetcher_.SetResponse(url, headers, "<html></html>"); |
| FetchFromProxy(url, |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(logging_info()->is_html_response()); |
| EXPECT_TRUE(logging_info()->is_url_disallowed()); |
| EXPECT_FALSE(logging_info()->is_request_disabled()); |
| |
| // Fetch disabled url. |
| url = "http://www.example.com/?PageSpeed=off"; |
| logging_info()->Clear(); |
| mock_url_fetcher_.SetResponse("http://www.example.com/", headers, |
| "<html></html>"); |
| FetchFromProxy(url, |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(logging_info()->is_html_response()); |
| EXPECT_FALSE(logging_info()->is_url_disallowed()); |
| EXPECT_TRUE(logging_info()->is_request_disabled()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, SkipPropertyCacheLookupIfOptionsNotEnabled) { |
| GoogleString url = "http://www.example.com/"; |
| GoogleString text; |
| RequestHeaders request_headers; |
| ResponseHeaders headers; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| |
| // Fetch disabled url. |
| url = "http://www.example.com/?PageSpeed=off"; |
| logging_info()->Clear(); |
| mock_url_fetcher_.SetResponse("http://www.example.com/", headers, |
| "<html></html>"); |
| FetchFromProxy(url, |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(logging_info()->is_html_response()); |
| EXPECT_FALSE(logging_info()->is_url_disallowed()); |
| EXPECT_TRUE(logging_info()->is_request_disabled()); |
| |
| // Only the HTTP response lookup is issued and it is not in the cache. |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, SkipPropertyCacheLookupIfUrlBlacklisted) { |
| GoogleString url = "http://www.blacklist.com/"; |
| RequestHeaders request_headers; |
| GoogleString text; |
| ResponseHeaders headers; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| |
| custom_options->AddRejectedUrlWildcard(AbsolutifyUrl("blacklist*")); |
| SetRewriteOptions(custom_options.get()); |
| |
| logging_info()->Clear(); |
| mock_url_fetcher_.SetResponse(url, headers, "<html></html>"); |
| FetchFromProxy(url, request_headers, true, &text, &headers, false); |
| EXPECT_TRUE(logging_info()->is_html_response()); |
| EXPECT_TRUE(logging_info()->is_url_disallowed()); |
| EXPECT_FALSE(logging_info()->is_request_disabled()); |
| |
| // Only the HTTP response lookup is issued and it is not in the cache. |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, HeadRequest) { |
| // Test to check if we are handling Head requests correctly. |
| GoogleString url = "http://www.example.com/"; |
| GoogleString set_text, get_text; |
| RequestHeaders request_headers; |
| ResponseHeaders set_headers, get_headers; |
| |
| set_headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| set_headers.SetStatusAndReason(HttpStatus::kOK); |
| |
| set_text = "<html><head/></html>"; |
| |
| mock_url_fetcher_.SetResponse(url, set_headers, set_text); |
| FetchFromProxy(url, request_headers, true, &get_text, &get_headers); |
| |
| // Headers and body are correct for a Get request. |
| EXPECT_EQ("HTTP/1.0 200 OK\r\n" |
| "Content-Type: text/html\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "Date: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Expires: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Cache-Control: max-age=0, private\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n", get_headers.ToString()); |
| EXPECT_EQ(set_text, get_text); |
| |
| // Remove from the cache so we can actually test a HEAD fetch. |
| http_cache()->Delete(url, rewrite_driver_->CacheFragment()); |
| |
| ClearStats(); |
| |
| // Headers and body are correct for a Head request. |
| request_headers.set_method(RequestHeaders::kHead); |
| FetchFromProxy(url, |
| request_headers, |
| true, /* expect_success */ |
| &get_text, |
| &get_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| |
| EXPECT_EQ("HTTP/1.0 200 OK\r\n" |
| "Content-Type: text/html\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n", get_headers.ToString()); |
| EXPECT_TRUE(get_text.empty()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RedirectRequestWhenDomainRewriterEnabled) { |
| // A MapRewriteDomain should not change the Location of a redirect. |
| |
| // Test to check if we are handling Head requests correctly. |
| GoogleString url = "http://www.example.com/"; |
| GoogleString set_text, get_text; |
| RequestHeaders request_headers; |
| ResponseHeaders set_headers, get_headers; |
| NullMessageHandler handler; |
| |
| set_headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| set_headers.Add(HttpAttributes::kLocation, "http://m.example.com"); |
| set_headers.SetStatusAndReason(HttpStatus::kFound); |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| custom_options->EnableFilter(RewriteOptions::kRewriteDomains); |
| custom_options->WriteableDomainLawyer()->AddTwoProtocolRewriteDomainMapping( |
| "www.example.com", "m.example.com", &handler); |
| SetRewriteOptions(custom_options.get()); |
| set_text = "<html></html>"; |
| mock_url_fetcher_.SetResponse(url, set_headers, set_text); |
| FetchFromProxy(url, request_headers, true, &get_text, &get_headers); |
| |
| // Headers and body are correct for a Get request. |
| EXPECT_STREQ("HTTP/1.0 302 Found\r\n" |
| "Content-Type: text/html\r\n" |
| "Location: http://m.example.com\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "Date: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Expires: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Cache-Control: max-age=0, private\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n", |
| get_headers.ToString()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RewriteDomainForRedirectsAndCookiesWithEmptyBody) { |
| GoogleString url = "http://www.example.com/"; |
| GoogleString get_text; |
| RequestHeaders request_headers; |
| ResponseHeaders set_headers, get_headers; |
| NullMessageHandler handler; |
| |
| set_headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| set_headers.Add(HttpAttributes::kLocation, "http://m.example.com"); |
| set_headers.Add(HttpAttributes::kSetCookie, |
| "a=b; domain=m.example.com; HttpOnly"); |
| set_headers.SetStatusAndReason(HttpStatus::kFound); |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| custom_options->EnableFilter(RewriteOptions::kRewriteDomains); |
| custom_options->WriteableDomainLawyer()->AddTwoProtocolRewriteDomainMapping( |
| "www.example.com", "m.example.com", &handler); |
| custom_options->set_domain_rewrite_hyperlinks(true); |
| custom_options->set_domain_rewrite_cookies(true); |
| SetRewriteOptions(custom_options.get()); |
| mock_url_fetcher_.SetResponse(url, set_headers, ""); |
| FetchFromProxy(url, request_headers, true, &get_text, &get_headers); |
| |
| // The ProxyFetch should change the Location of a redirect when |
| // domain_rewrite_hyperlinks is enabled. |
| const char* location = get_headers.Lookup1(HttpAttributes::kLocation); |
| EXPECT_STREQ("http://www.example.com/", location); |
| |
| // The ProxyFetch should change the domain of a Cookie when |
| // domain_rewrite_cookies is enabled. |
| const char* cookie = get_headers.Lookup1(HttpAttributes::kSetCookie); |
| EXPECT_STREQ("a=b; domain=www.example.com; HttpOnly", cookie); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RedirectRequestRetainsQueryParams) { |
| // "Location: http://www.example.com/b/?x=y&PageSpeedCssInlineMaxBytes=99" is |
| // what we expect to see - note the PageSpeed query parameter IS copied over. |
| TestQueryParameters(/* original_domain= */ "http://www.example.com/", |
| /* redirect_domain= */ "http://www.example.com/", |
| /* add_params_to_redirect= */ true, |
| /* add_params_to_location= */ true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RedirectRequestRetainsOnlyPagespeedQueryParams) { |
| // "Location: http://www.example.com/b/?PageSpeedCssInlineMaxBytes=99" is |
| // what we expect to see - note the PageSpeed query parameter IS copied over. |
| TestQueryParameters(/* original_domain= */ "http://www.example.com/", |
| /* redirect_domain= */ "http://www.example.com/", |
| /* add_params_to_redirect= */ false, |
| /* add_params_to_location= */ true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RedirectToDiffDomainDiscardsQueryParams) { |
| // "Location: http://test.com/?x=y" is what we expect to see - note the |
| // PageSpeed query parameter is NOT copied over. |
| TestQueryParameters(/* original_domain= */ "http://www.example.com/", |
| /* redirect_domain= */ kTestDomain, |
| /* add_params_to_redirect= */ true, |
| /* add_params_to_location= */ false); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RedirectToDiffDomainDiscardsPagespeedQueryParams) { |
| // "Location: http://test.com/" is what we expect to see - note the |
| // PageSpeed query parameter is NOT copied over. |
| TestQueryParameters(/* original_domain= */ "http://www.example.com/", |
| /* redirect_domain= */ kTestDomain, |
| /* add_params_to_redirect= */ false, |
| /* add_params_to_location= */ false); |
| } |
| |
| TEST_F(ProxyInterfaceTest, HeadResourceRequest) { |
| // Test to check if we are handling Head requests correctly in pagespeed |
| // resource flow. |
| const char kCssWithEmbeddedImage[] = "*{background-image:url(%s)}"; |
| const char kBackgroundImage[] = "1.png"; |
| |
| GoogleString text; |
| RequestHeaders request_headers; |
| ResponseHeaders response_headers; |
| GoogleString expected_response_headers_string = "HTTP/1.1 200 OK\r\n" |
| "Content-Type: text/css\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "X-Content-Type-Options: nosniff\r\n" |
| "Date: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Expires: Tue, 02 Feb 2010 18:56:26 GMT\r\n" |
| "Cache-Control: max-age=300,private\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n"; |
| |
| // We're not going to image-compress so we don't need our mock image |
| // to really be an image. |
| SetResponseWithDefaultHeaders(kBackgroundImage, kContentTypePng, "image", |
| kHtmlCacheTimeSec * 2); |
| GoogleString orig_css = StringPrintf(kCssWithEmbeddedImage, kBackgroundImage); |
| SetResponseWithDefaultHeaders("embedded.css", kContentTypeCss, |
| orig_css, kHtmlCacheTimeSec * 2); |
| |
| // By default, cache extension is off in the default options. |
| server_context()->global_options()->SetDefaultRewriteLevel( |
| RewriteOptions::kPassThrough); |
| |
| // Because cache-extension was turned off, the image in the CSS file |
| // will not be changed. |
| FetchFromProxy("I.embedded.css.pagespeed.cf.0.css", |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(logging_info()->is_pagespeed_resource()); |
| EXPECT_EQ(expected_response_headers_string, response_headers.ToString()); |
| EXPECT_EQ(orig_css, text); |
| // Headers and body are correct for a Head request. |
| request_headers.set_method(RequestHeaders::kHead); |
| FetchFromProxy("I.embedded.css.pagespeed.cf.0.css", |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| |
| // This leads to a conditional refresh of the original resource. |
| expected_response_headers_string = "HTTP/1.1 200 OK\r\n" |
| "Content-Type: text/css\r\n" |
| "X-Background-Fetch: 0\r\n" |
| "X-Content-Type-Options: nosniff\r\n" |
| "Date: Tue, 02 Feb 2010 18:51:26 GMT\r\n" |
| "Expires: Tue, 02 Feb 2010 18:56:26 GMT\r\n" |
| "Cache-Control: max-age=300,private\r\n" |
| "X-Page-Speed: \r\n" |
| "HeadersComplete: 1\r\n\r\n"; |
| |
| EXPECT_EQ(expected_response_headers_string, response_headers.ToString()); |
| EXPECT_TRUE(text.empty()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, FetchFailure) { |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| // We don't want fetcher to fail the test, merely the fetch. |
| SetFetchFailOnUnexpected(false); |
| FetchFromProxy("invalid", false, &text, &headers); |
| CheckBackgroundFetch(headers, false); |
| CheckNumBackgroundFetches(0); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ReturnUnavailableForBlockedUrls) { |
| GoogleString text; |
| ResponseHeaders response_headers; |
| response_headers.SetStatusAndReason(HttpStatus::kOK); |
| mock_url_fetcher_.SetResponse(AbsolutifyUrl("blocked"), response_headers, |
| "<html></html>"); |
| FetchFromProxy("blocked", true, |
| &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| |
| text.clear(); |
| response_headers.Clear(); |
| |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| |
| custom_options->AddRejectedUrlWildcard(AbsolutifyUrl("block*")); |
| SetRewriteOptions(custom_options.get()); |
| RequestHeaders request_headers; |
| FetchFromProxy( |
| "blocked", |
| request_headers, |
| false, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(HttpStatus::kProxyDeclinedRequest, response_headers.status_code()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ReturnUnavailableForBlockedHeaders) { |
| GoogleString text; |
| RequestHeaders request_headers; |
| ResponseHeaders response_headers; |
| response_headers.SetStatusAndReason(HttpStatus::kOK); |
| mock_url_fetcher_.SetResponse(kTestDomain, response_headers, "<html></html>"); |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| |
| custom_options->AddRejectedHeaderWildcard(HttpAttributes::kUserAgent, |
| "*Chrome*"); |
| custom_options->AddRejectedHeaderWildcard(HttpAttributes::kXForwardedFor, |
| "10.3.4.*"); |
| SetRewriteOptions(custom_options.get()); |
| |
| request_headers.Add(HttpAttributes::kUserAgent, "Firefox"); |
| request_headers.Add(HttpAttributes::kXForwardedFor, "10.0.0.11"); |
| FetchFromProxy(kTestDomain, request_headers, true, |
| &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| |
| request_headers.Clear(); |
| response_headers.Clear(); |
| |
| request_headers.Add(HttpAttributes::kUserAgent, "abc"); |
| request_headers.Add(HttpAttributes::kUserAgent, "xyz Chrome abc"); |
| FetchFromProxy(kTestDomain, |
| request_headers, |
| false, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(HttpStatus::kProxyDeclinedRequest, response_headers.status_code()); |
| |
| request_headers.Clear(); |
| response_headers.Clear(); |
| |
| request_headers.Add(HttpAttributes::kXForwardedFor, "10.3.4.32"); |
| FetchFromProxy(kTestDomain, |
| request_headers, |
| false, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(HttpStatus::kProxyDeclinedRequest, response_headers.status_code()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, InvalidUrl) { |
| ExpectStringAsyncFetch fetch(false, rewrite_driver()->request_context()); |
| proxy_interface_->Fetch("localhost:3141", message_handler(), &fetch); |
| EXPECT_TRUE(fetch.done()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PassThrough404) { |
| GoogleString text; |
| ResponseHeaders headers; |
| SetFetchResponse404("404"); |
| FetchFromProxy("404", true, &text, &headers); |
| ASSERT_TRUE(headers.has_status_code()); |
| EXPECT_EQ(HttpStatus::kNotFound, headers.status_code()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PassThroughResource) { |
| GoogleString text; |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| |
| SetResponseWithDefaultHeaders("text.txt", kContentTypeText, kContent, |
| kHtmlCacheTimeSec * 2); |
| FetchFromProxy("text.txt", true, &text, &headers); |
| CheckHeaders(headers, kContentTypeText); |
| CheckBackgroundFetch(headers, false); |
| CheckNumBackgroundFetches(0); |
| EXPECT_EQ(kContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PassThroughEmptyResource) { |
| ResponseHeaders headers; |
| const char kContent[] = ""; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_EQ(kContent, text); |
| // HTTP resource not found the first time. |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(kContent, text2); |
| // Empty resources are not remembered in this flow, so stats are exactly the |
| // same on second load. |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, SetCookieNotCached) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| headers.Add(HttpAttributes::kSetCookie, "cookie"); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| // The first response served by the fetcher has Set-Cookie headers. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_STREQ("cookie", response_headers.Lookup1(HttpAttributes::kSetCookie)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // The next response that is served from cache does not have any Set-Cookie |
| // headers. |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(NULL, response_headers2.Lookup1(HttpAttributes::kSetCookie)); |
| EXPECT_EQ(kContent, text2); |
| // The HTTP response is found but the ajax metadata is not found. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, SetCookie2NotCached) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| headers.Add(HttpAttributes::kSetCookie2, "cookie"); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| // The first response served by the fetcher has Set-Cookie headers. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_STREQ("cookie", response_headers.Lookup1(HttpAttributes::kSetCookie2)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // The next response that is served from cache does not have any Set-Cookie |
| // headers. |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(NULL, response_headers2.Lookup1(HttpAttributes::kSetCookie2)); |
| EXPECT_EQ(kContent, text2); |
| // The HTTP response is found but the ajax metadata is not found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, NotCachedIfAuthorizedAndNotPublic) { |
| // We should not cache things which are default cache-control if we |
| // are sending Authorization:. See RFC 2616, 14.8. |
| ReflectingTestFetcher reflect; |
| server_context()->set_default_system_fetcher(&reflect); |
| |
| RequestHeaders request_headers; |
| request_headers.Add("Was", "Here"); |
| request_headers.Add(HttpAttributes::kAuthorization, "Secret"); |
| // This will get reflected as well, and hence will determine whether |
| // cacheable or not. |
| request_headers.Replace(HttpAttributes::kCacheControl, "max-age=600000"); |
| |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| // Using .txt here so we don't try any AJAX rewriting. |
| FetchFromProxy("http://test.com/file.txt", |
| request_headers, true, &out_text, &out_headers); |
| // We should see the request headers we sent back as the response headers |
| // as we're using a ReflectingTestFetcher. |
| EXPECT_STREQ("Here", out_headers.Lookup1("Was")); |
| |
| // Not cross-domain, so should propagate out header. |
| EXPECT_TRUE(out_headers.Has(HttpAttributes::kAuthorization)); |
| |
| // Should not have written anything to cache, due to the authorization |
| // header. |
| EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); |
| |
| ClearStats(); |
| |
| // Now try again. This time no authorization header, different 'Was'. |
| request_headers.Replace("Was", "There"); |
| request_headers.RemoveAll(HttpAttributes::kAuthorization); |
| |
| FetchFromProxy("http://test.com/file.txt", |
| request_headers, true, &out_text, &out_headers); |
| // Should get different headers since we should not be cached. |
| EXPECT_STREQ("There", out_headers.Lookup1("Was")); |
| EXPECT_FALSE(out_headers.Has(HttpAttributes::kAuthorization)); |
| |
| // And should be a miss per stats. |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| |
| mock_scheduler()->AwaitQuiescence(); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CachedIfAuthorizedAndPublic) { |
| // This with Cache-Control: public should be cached even if |
| // we are sending Authorization:. See RFC 2616. |
| ReflectingTestFetcher reflect; |
| server_context()->set_default_system_fetcher(&reflect); |
| |
| RequestHeaders request_headers; |
| request_headers.Add("Was", "Here"); |
| request_headers.Add(HttpAttributes::kAuthorization, "Secret"); |
| // This will get reflected as well, and hence will determine whether |
| // cacheable or not. |
| request_headers.Replace(HttpAttributes::kCacheControl, "max-age=600000"); |
| request_headers.Add(HttpAttributes::kCacheControl, "public"); // unlike above |
| |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| // Using .txt here so we don't try any AJAX rewriting. |
| FetchFromProxy("http://test.com/file.txt", |
| request_headers, true, &out_text, &out_headers); |
| EXPECT_STREQ("Here", out_headers.Lookup1("Was")); |
| |
| // Not cross-domain, so should propagate out header. |
| EXPECT_TRUE(out_headers.Has(HttpAttributes::kAuthorization)); |
| |
| // Should have written the result to the cache, despite the request having |
| // Authorization: thanks to cache-control: public, |
| EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); |
| |
| ClearStats(); |
| |
| // Now try again. This time no authorization header, different 'Was'. |
| request_headers.Replace("Was", "There"); |
| request_headers.RemoveAll(HttpAttributes::kAuthorization); |
| |
| FetchFromProxy("http://test.com/file.txt", |
| request_headers, true, &out_text, &out_headers); |
| // Should get old headers, since original was cacheable. |
| EXPECT_STREQ("Here", out_headers.Lookup1("Was")); |
| |
| // ... of course hopefully a real server won't serve secrets on a |
| // cache-control: public page. |
| EXPECT_STREQ("Secret", out_headers.Lookup1(HttpAttributes::kAuthorization)); |
| |
| // And should be a hit per stats. |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| |
| mock_scheduler()->AwaitQuiescence(); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ImplicitCachingHeadersForCss) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kContent); |
| |
| // The first response served by the fetcher has caching headers. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata, one for the HTTP response and one by the css |
| // filter which looks up metadata while rewriting. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One hit for ajax metadata and one for the HTTP response. |
| EXPECT_EQ(2, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, MinCacheTtl) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_min_cache_ttl_ms(600 * Timer::kSecondMs); |
| server_context()->ComputeSignature(options); |
| |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms, 300 * Timer::kSecondMs); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kContent); |
| |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| GoogleString expiry; |
| ConvertTimeToString(MockTimer::kApr_5_2010_ms + 600 * Timer::kSecondMs, |
| &expiry); |
| EXPECT_STREQ("max-age=600", |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(expiry, response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata, one for the HTTP response and one by the css |
| // filter which looks up metadata while rewriting. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| ClearStats(); |
| |
| // Change the origin content and advance time. Fetch again from cache. |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, "new"); |
| text.clear(); |
| response_headers.Clear(); |
| AdvanceTimeMs(400 * 1000); |
| // Even though the max age set on the resource of 300 seconds, since the |
| // min caching is set to 600 seconds, we should get a cache hit. And since |
| // the content is fetched from cache, the old content is returned. |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| ConvertTimeToString(timer()->NowMs() + 600 * Timer::kSecondMs, &expiry); |
| EXPECT_STREQ("max-age=600", |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(expiry, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_EQ(kContent, text); |
| // One hit for ajax metadata and one for the HTTP response. |
| EXPECT_EQ(2, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| ClearStats(); |
| |
| // Advance time past min cache ttl and fetch again. This time the new content |
| // should be fetched. |
| text.clear(); |
| response_headers.Clear(); |
| AdvanceTimeMs(400 * 1000); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| ConvertTimeToString(timer()->NowMs() + 600 * Timer::kSecondMs, &expiry); |
| EXPECT_STREQ("max-age=600", |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(expiry, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_EQ("new", text); |
| // 1. ajax metadata 2. text.css 3. cf filter metadata lookup. |
| EXPECT_EQ(3, lru_cache()->num_hits()); |
| // http cache miss for text.css as it has expired. |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CacheableSize) { |
| // Test to check that we are not caching responses which have content length > |
| // max_cacheable_response_content_length. |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms, 300 * Timer::kSecondMs); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| |
| // Set the set_max_cacheable_response_content_length to 10 bytes. |
| http_cache()->set_max_cacheable_response_content_length(10); |
| |
| // Fetch once. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| // One lookup for ajax metadata, one for the HTTP response and one for the |
| // property cache entry. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| |
| // Fetch again. It has the same caching headers. |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| // None are found as the size is bigger than |
| // max_cacheable_response_content_length. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| // Set the set_max_cacheable_response_content_length to 1024 bytes. |
| http_cache()->set_max_cacheable_response_content_length(1024); |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| // None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(1, lru_cache()->num_inserts()); |
| |
| // Fetch again. |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| // One hit for the HTTP response as content is smaller than |
| // max_cacheable_response_content_length. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CacheableSizeAjax) { |
| // Test to check that we are not caching responses which have content length > |
| // max_cacheable_response_content_length in Ajax flow. |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kCssContent); |
| |
| http_cache()->set_max_cacheable_response_content_length(0); |
| // The first response served by the fetcher and is not rewritten. An ajax |
| // rewrite should not be triggered as the content length is greater than |
| // max_cacheable_response_content_length. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_EQ(kCssContent, text); |
| // One lookup for ajax metadata, one for the HTTP response. None are found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| |
| ClearStats(); |
| // Fetch again. Optimized version is not served. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_EQ(kCssContent, text); |
| // None are found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CacheableSizeResource) { |
| // Test to check that we are not caching responses which have content length > |
| // max_cacheable_response_content_length in resource flow. |
| GoogleString text; |
| RequestHeaders request_headers; |
| ResponseHeaders headers; |
| |
| // Fetching of a rewritten resource we did not just create |
| // after an HTML rewrite. |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| // Set the set_max_cacheable_response_content_length to 0 bytes. |
| http_cache()->set_max_cacheable_response_content_length(0); |
| // Fetch fails as original is not accessible. |
| |
| FetchFromProxy( |
| Encode("", "cf", "0", "a.css", "css"), |
| request_headers, |
| false, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| } |
| |
| TEST_F(ProxyInterfaceTest, InvalidationForCacheableHtml) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms, 300 * Timer::kSecondMs); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata, one for the HTTP response and one for the |
| // property cache entry. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One hit for the HTTP response. Misses for the property cache entry and the |
| // ajax metadata. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| |
| // Change the response. |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, "new"); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| // We continue to serve the previous response since we've cached it. |
| EXPECT_EQ(kContent, text); |
| // One hit for the HTTP response. Misses for the property cache entry and the |
| // ajax metadata. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| |
| // Invalidate the cache. |
| scoped_ptr<RewriteOptions> custom_options( |
| server_context()->global_options()->Clone()); |
| custom_options->UpdateCacheInvalidationTimestampMs(timer()->NowMs()); |
| SetRewriteOptions(custom_options.get()); |
| |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| // We get the new response since we've invalidated the cache. |
| EXPECT_EQ("new", text); |
| // The HTTP response is found in the LRU cache but counts as a miss in the |
| // HTTPCache since it has been invalidated. Also, cache misses for the ajax |
| // metadata and property cache entry. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, UrlInvalidationForCacheableHtml) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms, 300 * Timer::kSecondMs); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata, one for the HTTP response and one for the |
| // property cache entry. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One hit for the HTTP response. Misses for the property cache entry and the |
| // ajax metadata. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| |
| // Change the response. |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, "new"); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| // We continue to serve the previous response since we've cached it. |
| EXPECT_EQ(kContent, text); |
| // One hit for the HTTP response. Misses for the property cache entry and the |
| // ajax metadata. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| |
| |
| // Invalidate the cache for some URL other than 'text.html'. |
| scoped_ptr<RewriteOptions> custom_options_1( |
| server_context()->global_options()->Clone()); |
| custom_options_1->AddUrlCacheInvalidationEntry( |
| AbsolutifyUrl("foo.bar"), timer()->NowMs(), true); |
| SetRewriteOptions(custom_options_1.get()); |
| |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| // We continue to serve the previous response since we've cached it. |
| EXPECT_EQ(kContent, text); |
| // One hit for the HTTP response. Misses for the property cache entry and the |
| // ajax metadata. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| |
| |
| // Invalidate the cache. |
| scoped_ptr<RewriteOptions> custom_options_2( |
| server_context()->global_options()->Clone()); |
| // Strictness of URL cache invalidation entry (last argument below) does not |
| // matter in this test since there is nothing cached in metadata or property |
| // caches. |
| custom_options_2->AddUrlCacheInvalidationEntry( |
| AbsolutifyUrl("text.html"), timer()->NowMs(), true); |
| SetRewriteOptions(custom_options_2.get()); |
| |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| // We get the new response since we've invalidated the cache. |
| EXPECT_EQ("new", text); |
| // The HTTP response is found in the LRU cache but counts as a miss in the |
| // HTTPCache since it has been invalidated. Also, cache misses for the ajax |
| // metadata and property cache entry. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, NoImplicitCachingHeadersForHtml) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| |
| // The first response served by the fetcher does not have implicit caching |
| // headers. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| EXPECT_STREQ(NULL, response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // Lookups for: (1) ajax metadata (2) HTTP response (3) Property cache. |
| // None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // Fetch again. Not found in cache. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.html", true, &text, &response_headers); |
| EXPECT_EQ(NULL, response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // Lookups for: (1) ajax metadata (2) HTTP response (3) Property cache. |
| // None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ModifiedImplicitCachingHeadersForCss) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_implicit_cache_ttl_ms(500 * Timer::kSecondMs); |
| server_context()->ComputeSignature(options); |
| |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| // Do not call ComputeCaching before calling SetFetchResponse because it will |
| // add an explicit max-age=300 cache control header. We do not want that |
| // header in this test. |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kContent); |
| |
| // The first response served by the fetcher has caching headers. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| GoogleString max_age_500 = "max-age=500"; |
| GoogleString start_time_plus_500s_string; |
| ConvertTimeToString(MockTimer::kApr_5_2010_ms + 500 * Timer::kSecondMs, |
| &start_time_plus_500s_string); |
| |
| EXPECT_STREQ(max_age_500, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_500s_string, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata, one for the HTTP response and one by the css |
| // filter which looks up metadata while rewriting. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // Fetch again from cache. It has the same caching headers. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_500, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_500s_string, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kContent, text); |
| // One hit for ajax metadata and one for the HTTP response. |
| EXPECT_EQ(2, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, EtagsAddedWhenAbsent) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| headers.RemoveAll(HttpAttributes::kEtag); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| // The first response served by the fetcher has no Etag in the response. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| EXPECT_EQ(NULL, response_headers.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| ClearStats(); |
| |
| // An Etag is added before writing to cache. The next response is served from |
| // cache and has an Etag. |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(HttpStatus::kOK, response_headers2.status_code()); |
| EXPECT_STREQ(kEtag0, response_headers2.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(kContent, text2); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| ClearStats(); |
| |
| // The Etag matches and a 304 is served out. |
| GoogleString text3; |
| ResponseHeaders response_headers3; |
| RequestHeaders request_headers; |
| request_headers.Add(HttpAttributes::kIfNoneMatch, kEtag0); |
| FetchFromProxy("text.txt", request_headers, true, &text3, &response_headers3); |
| EXPECT_EQ(HttpStatus::kNotModified, response_headers3.status_code()); |
| EXPECT_STREQ(NULL, response_headers3.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ("", text3); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, EtagMatching) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| headers.Replace(HttpAttributes::kEtag, "etag"); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| // The first response served by the fetcher has an Etag in the response. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| EXPECT_STREQ("etag", response_headers.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| |
| ClearStats(); |
| // The next response is served from cache. |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(HttpStatus::kOK, response_headers2.status_code()); |
| EXPECT_STREQ("etag", response_headers2.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(kContent, text2); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| ClearStats(); |
| |
| // The Etag matches and a 304 is served out. |
| GoogleString text3; |
| ResponseHeaders response_headers3; |
| RequestHeaders request_headers; |
| request_headers.Add(HttpAttributes::kIfNoneMatch, "etag"); |
| FetchFromProxy("text.txt", request_headers, true, &text3, &response_headers3); |
| EXPECT_EQ(HttpStatus::kNotModified, response_headers3.status_code()); |
| EXPECT_STREQ(NULL, response_headers3.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ("", text3); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| |
| ClearStats(); |
| // The Etag doesn't match and the full response is returned. |
| GoogleString text4; |
| ResponseHeaders response_headers4; |
| request_headers.Replace(HttpAttributes::kIfNoneMatch, "mismatch"); |
| FetchFromProxy("text.txt", request_headers, true, &text4, &response_headers4); |
| EXPECT_EQ(HttpStatus::kOK, response_headers4.status_code()); |
| EXPECT_STREQ("etag", response_headers4.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(kContent, text4); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, LastModifiedMatch) { |
| ResponseHeaders headers; |
| const char kContent[] = "A very compelling article"; |
| SetDefaultLongCacheHeaders(&kContentTypeText, &headers); |
| headers.SetLastModified(MockTimer::kApr_5_2010_ms); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.txt"), headers, kContent); |
| |
| // The first response served by the fetcher has an Etag in the response. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.txt", true, &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kLastModified)); |
| EXPECT_EQ(kContent, text); |
| // One lookup for ajax metadata and one for the HTTP response. Neither are |
| // found. |
| EXPECT_EQ(2, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| |
| ClearStats(); |
| // The next response is served from cache. |
| GoogleString text2; |
| ResponseHeaders response_headers2; |
| FetchFromProxy("text.txt", true, &text2, &response_headers2); |
| EXPECT_EQ(HttpStatus::kOK, response_headers2.status_code()); |
| EXPECT_STREQ(start_time_string_, |
| response_headers2.Lookup1(HttpAttributes::kLastModified)); |
| EXPECT_EQ(kContent, text2); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| |
| ClearStats(); |
| // The last modified timestamp matches and a 304 is served out. |
| GoogleString text3; |
| ResponseHeaders response_headers3; |
| RequestHeaders request_headers; |
| request_headers.Add(HttpAttributes::kIfModifiedSince, start_time_string_); |
| FetchFromProxy("text.txt", request_headers, true, &text3, &response_headers3); |
| EXPECT_EQ(HttpStatus::kNotModified, response_headers3.status_code()); |
| EXPECT_STREQ(NULL, response_headers3.Lookup1(HttpAttributes::kLastModified)); |
| EXPECT_EQ("", text3); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| |
| ClearStats(); |
| // The last modified timestamp doesn't match and the full response is |
| // returned. |
| GoogleString text4; |
| ResponseHeaders response_headers4; |
| request_headers.Replace(HttpAttributes::kIfModifiedSince, |
| "Fri, 02 Apr 2010 18:51:26 GMT"); |
| FetchFromProxy("text.txt", request_headers, true, &text4, &response_headers4); |
| EXPECT_EQ(HttpStatus::kOK, response_headers4.status_code()); |
| EXPECT_STREQ(start_time_string_, |
| response_headers4.Lookup1(HttpAttributes::kLastModified)); |
| EXPECT_EQ(kContent, text4); |
| // One lookup for ajax metadata and one for the HTTP response. The metadata is |
| // not found but the HTTP response is found.` |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, AjaxRewritingForCss) { |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kCssContent); |
| |
| // The first response served by the fetcher and is not rewritten. An ajax |
| // rewrite is triggered. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kCssContent, text); |
| CheckBackgroundFetch(response_headers, false); |
| CheckNumBackgroundFetches(0); |
| // One lookup for ajax metadata, one for the HTTP response and one by the css |
| // filter which looks up metadata while rewriting. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // The rewrite is complete and the optimized version is served. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kMinimizedCssContent, text); |
| // One hit for ajax metadata and one for the rewritten HTTP response. |
| EXPECT_EQ(2, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| CheckNumBackgroundFetches(0); |
| |
| ClearStats(); |
| // Advance close to expiry. |
| AdvanceTimeUs(270 * Timer::kSecondUs); |
| // The rewrite is complete and the optimized version is served. A freshen is |
| // triggered to refresh the original CSS file. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ("max-age=30", |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ("Mon, 05 Apr 2010 18:55:56 GMT", |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kMinimizedCssContent, text); |
| // One hit for ajax metadata, one for the rewritten HTTP response and one for |
| // the original HTTP response while freshening. |
| EXPECT_EQ(3, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| // One background fetch is triggered while freshening. |
| CheckNumBackgroundFetches(1); |
| |
| // Disable ajax rewriting. We now received the response fetched while |
| // freshening. This response has kBackgroundFetchHeader set to 1. |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_in_place_rewriting_enabled(false); |
| server_context()->ComputeSignature(options); |
| |
| ClearStats(); |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ("Mon, 05 Apr 2010 19:00:56 GMT", |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ("Mon, 05 Apr 2010 18:55:56 GMT", |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kCssContent, text); |
| CheckNumBackgroundFetches(0); |
| CheckBackgroundFetch(response_headers, true); |
| // Done HTTP cache hit for the original response. |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, FallbackNoAcceptGzip) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_http_cache_compression_level(9); |
| options->set_serve_stale_while_revalidate_threshold_sec( |
| 1000 * Timer::kDayMs); |
| server_context()->ComputeSignature(options); |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kCssContent); |
| |
| // The first response served by the fetcher and is not rewritten. An ajax |
| // rewrite is triggered. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kCssContent, text); |
| CheckBackgroundFetch(response_headers, false); |
| CheckNumBackgroundFetches(0); |
| // One lookup for ajax metadata, one for the HTTP response and one by the css |
| // filter which looks up metadata while rewriting. None are found. |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // The rewrite is complete and the optimized version is served. |
| text.clear(); |
| response_headers.Clear(); |
| AdvanceTimeUs(400 * Timer::kSecondUs); // expired |
| FetchFromProxy("text.css", true, &text, &response_headers); |
| EXPECT_FALSE(response_headers.IsGzipped()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, NoAjaxRewritingWhenAuthorizationSent) { |
| // We should not do ajax rewriting when sending over an authorization |
| // header if the original isn't cache-control: public. |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kCssContent); |
| |
| // The first response served by the fetcher and is not rewritten. An ajax |
| // rewrite is triggered. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| RequestHeaders request_headers; |
| request_headers.Add(HttpAttributes::kAuthorization, "Paperwork"); |
| FetchFromProxy("text.css", request_headers, true, &text, &response_headers); |
| EXPECT_EQ(kCssContent, text); |
| |
| // The second version should still be unoptimized, since original wasn't |
| // cacheable. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", request_headers, true, &text, &response_headers); |
| EXPECT_EQ(kCssContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, AjaxRewritingWhenAuthorizationButPublic) { |
| // We should do ajax rewriting when sending over an authorization |
| // header if the original is cache-control: public. |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.Add(HttpAttributes::kCacheControl, "public, max-age=400"); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("text.css"), headers, kCssContent); |
| |
| // The first response served by the fetcher and is not rewritten. An ajax |
| // rewrite is triggered. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| RequestHeaders request_headers; |
| request_headers.Add(HttpAttributes::kAuthorization, "Paperwork"); |
| FetchFromProxy("text.css", request_headers, true, &text, &response_headers); |
| EXPECT_EQ(kCssContent, text); |
| |
| // The second version should be optimized in this case. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("text.css", request_headers, true, &text, &response_headers); |
| EXPECT_EQ(kMinimizedCssContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, AjaxRewritingDisabledByGlobalDisable) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_enabled(RewriteOptions::kEnabledOff); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("a.css", true, &text, &response_headers); |
| // First fetch will not get rewritten no matter what. |
| EXPECT_STREQ(kCssContent, text); |
| |
| // Second fetch would get minified if ajax rewriting were on; but |
| // it got disabled by the global toggle. |
| text.clear(); |
| FetchFromProxy("a.css", true, &text, &response_headers); |
| EXPECT_STREQ(kCssContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, AjaxRewritingSkippedIfBlacklisted) { |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("blacklist.css"), headers, kCssContent); |
| |
| // The first response is served by the fetcher. Since the url is blacklisted, |
| // no ajax rewriting happens. |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("blacklist.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kCssContent, text); |
| // Since no ajax rewriting happens, there is only a single cache lookup for |
| // the resource. |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| |
| ClearStats(); |
| // The same thing happens on the second request. |
| text.clear(); |
| response_headers.Clear(); |
| FetchFromProxy("blacklist.css", true, &text, &response_headers); |
| |
| EXPECT_STREQ(max_age_300_, |
| response_headers.Lookup1(HttpAttributes::kCacheControl)); |
| EXPECT_STREQ(start_time_plus_300s_string_, |
| response_headers.Lookup1(HttpAttributes::kExpires)); |
| EXPECT_STREQ(start_time_string_, |
| response_headers.Lookup1(HttpAttributes::kDate)); |
| EXPECT_EQ(kCssContent, text); |
| // The resource is found in cache this time. |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| EXPECT_EQ(0, http_cache()->cache_misses()->Get()); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_hits()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, AjaxRewritingBlacklistReject) { |
| // Makes sure that we honor reject_blacklisted() when ajax rewriting may |
| // have normally happened. |
| RejectBlacklisted(); |
| |
| ResponseHeaders headers; |
| SetTimeMs(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeCss.mime_type()); |
| headers.SetDate(MockTimer::kApr_5_2010_ms); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("blacklistCoffee.css"), headers, kCssContent); |
| SetFetchResponse(AbsolutifyUrl("tea.css"), headers, kCssContent); |
| |
| GoogleString text; |
| ResponseHeaders response_headers; |
| FetchFromProxy("blacklistCoffee.css", true, &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kImATeapot, response_headers.status_code()); |
| EXPECT_TRUE(text.empty()); |
| |
| // Non-blacklisted stuff works OK. |
| FetchFromProxy("tea.css", true, &text, &response_headers); |
| EXPECT_EQ(HttpStatus::kOK, response_headers.status_code()); |
| EXPECT_EQ(kCssContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, EatCookiesOnReconstructFailure) { |
| // Make sure we don't pass through a Set-Cookie[2] when reconstructing |
| // a resource on demand fails. |
| GoogleString abs_path = AbsolutifyUrl("a.css"); |
| ResponseHeaders response_headers; |
| SetDefaultLongCacheHeaders(&kContentTypeCss, &response_headers); |
| response_headers.Add(HttpAttributes::kSetCookie, "a cookie"); |
| response_headers.Add(HttpAttributes::kSetCookie2, "a weird old-time cookie"); |
| response_headers.ComputeCaching(); |
| SetFetchResponse(abs_path, response_headers, "broken_css{"); |
| |
| ResponseHeaders out_response_headers; |
| RequestHeaders request_headers; |
| GoogleString text; |
| FetchFromProxy(Encode("", "cf", "0", "a.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &out_response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(NULL, out_response_headers.Lookup1(HttpAttributes::kSetCookie)); |
| EXPECT_EQ(NULL, out_response_headers.Lookup1(HttpAttributes::kSetCookie2)); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RewriteHtml) { |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->SetRewriteLevel(RewriteOptions::kPassThrough); |
| options->EnableFilter(RewriteOptions::kRewriteCss); |
| server_context()->ComputeSignature(options); |
| |
| headers.Add(HttpAttributes::kEtag, "something"); |
| headers.SetDateAndCaching(MockTimer::kApr_5_2010_ms, |
| kHtmlCacheTimeSec * 2 * Timer::kSecondMs); |
| headers.SetLastModified(MockTimer::kApr_5_2010_ms); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), headers, CssLinkHref("a.css")); |
| |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| text.clear(); |
| headers.Clear(); |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| CheckBackgroundFetch(headers, false); |
| CheckNumBackgroundFetches(1); |
| CheckHeaders(headers, kContentTypeHtml); |
| EXPECT_EQ(CssLinkHref(Encode("", "cf", "0", "a.css", "css")), text); |
| headers.ComputeCaching(); |
| EXPECT_LE(start_time_ms_ + kHtmlCacheTimeSec * Timer::kSecondMs, |
| headers.CacheExpirationTimeMs()); |
| EXPECT_EQ(NULL, headers.Lookup1(HttpAttributes::kEtag)); |
| EXPECT_EQ(NULL, headers.Lookup1(HttpAttributes::kLastModified)); |
| EXPECT_STREQ("cf", AppliedRewriterStringFromLog()); |
| |
| // Fetch the rewritten resource as well. |
| text.clear(); |
| headers.Clear(); |
| ClearStats(); |
| RequestHeaders request_headers; |
| FetchFromProxy(Encode("", "cf", "0", "a.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| CheckHeaders(headers, kContentTypeCss); |
| // Note that the fetch for the original resource was triggered as a result of |
| // the initial HTML request. Hence, its headers indicate that it is a |
| // background request |
| // This response has kBackgroundFetchHeader set to 1 since a fetch was |
| // triggered for it in the background while rewriting the original html. |
| CheckBackgroundFetch(headers, true); |
| CheckNumBackgroundFetches(0); |
| headers.ComputeCaching(); |
| EXPECT_LE(start_time_ms_ + Timer::kYearMs, headers.CacheExpirationTimeMs()); |
| EXPECT_EQ(kMinimizedCssContent, text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, LogChainedResourceRewrites) { |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| SetResponseWithDefaultHeaders("1.js", kContentTypeJavascript, "var wxyz=1;", |
| kHtmlCacheTimeSec * 2); |
| SetResponseWithDefaultHeaders("2.js", kContentTypeJavascript, "var abcd=2;", |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString combined_js_url = Encode( |
| kTestDomain, "jc", "0", |
| "1.js.pagespeed.jm.0.jsX2.js.pagespeed.jm.0.js", "js"); |
| combined_js_url[combined_js_url.find('X')] = '+'; |
| RequestHeaders request_headers; |
| FetchFromProxy( |
| combined_js_url, |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_STREQ("jc,jm", AppliedRewriterStringFromLog()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, FlushHugeHtml) { |
| // Test the forced flushing of HTML controlled by flush_buffer_limit_bytes(). |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_flush_buffer_limit_bytes(8); // 2 self-closing tags ("<p/>") |
| options->set_flush_html(true); |
| options->DisableFilter(RewriteOptions::kAddHead); |
| rewrite_driver()->AddFilters(); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders("page.html", kContentTypeHtml, |
| "<a/><b/><c/><d/><e/><f/><g/><h/>", |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString out; |
| FetchFromProxyLoggingFlushes("page.html", true /*success*/, &out); |
| EXPECT_EQ( |
| "<a/><b/>|Flush|<c/><d/>|Flush|<e/><f/>|Flush|<g/><h/>|Flush||Flush|", |
| out); |
| |
| // Now tell to flush after 3 self-closing tags. |
| options->ClearSignatureForTesting(); |
| options->set_flush_buffer_limit_bytes(12); // 3 self-closing tags |
| server_context()->ComputeSignature(options); |
| |
| FetchFromProxyLoggingFlushes("page.html", true /*success*/, &out); |
| EXPECT_EQ( |
| "<a/><b/><c/>|Flush|<d/><e/><f/>|Flush|<g/><h/>|Flush|", out); |
| |
| // And now with 2.5. This means we will flush 2 (as that many are complete), |
| // then 5, and 7. |
| options->ClearSignatureForTesting(); |
| options->set_flush_buffer_limit_bytes(10); |
| server_context()->ComputeSignature(options); |
| |
| FetchFromProxyLoggingFlushes("page.html", true /*success*/, &out); |
| EXPECT_EQ( |
| "<a/><b/>|Flush|<c/><d/><e/>|Flush|<f/><g/>|Flush|<h/>|Flush|", out); |
| |
| // Now 9 bytes, e.g. 2 1/4 of a self-closing tag. Looks almost the same as |
| // every 2 self-closing tags (8 bytes), but we don't get an extra flush |
| // at the end. |
| options->ClearSignatureForTesting(); |
| options->set_flush_buffer_limit_bytes(9); |
| server_context()->ComputeSignature(options); |
| FetchFromProxyLoggingFlushes("page.html", true /*success*/, &out); |
| EXPECT_EQ( |
| "<a/><b/>|Flush|<c/><d/>|Flush|<e/><f/>|Flush|<g/><h/>|Flush|", |
| out); |
| } |
| |
| TEST_F(ProxyInterfaceTest, DontRewriteDisallowedHtml) { |
| // Blacklisted URL should not be rewritten. |
| SetResponseWithDefaultHeaders("blacklist.html", kContentTypeHtml, |
| CssLinkHref("a.css"), kHtmlCacheTimeSec * 2), |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString text; |
| ResponseHeaders headers; |
| FetchFromProxy("blacklist.html", true, &text, &headers); |
| CheckHeaders(headers, kContentTypeHtml); |
| EXPECT_EQ(CssLinkHref("a.css"), text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, DontRewriteDisallowedHtmlRejectMode) { |
| // If we're in reject_blacklisted mode, we should just respond with the |
| // configured status. |
| RejectBlacklisted(); |
| SetResponseWithDefaultHeaders("blacklistCoffee.html", kContentTypeHtml, |
| CssLinkHref("a.css"), kHtmlCacheTimeSec * 2), |
| SetResponseWithDefaultHeaders("tea.html", kContentTypeHtml, |
| "tasty", kHtmlCacheTimeSec * 2); |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString text; |
| ResponseHeaders headers; |
| FetchFromProxy("blacklistCoffee.html", true, &text, &headers); |
| EXPECT_EQ(HttpStatus::kImATeapot, headers.status_code()); |
| EXPECT_TRUE(text.empty()); |
| |
| // Fetching non-blacklisted one works fine. |
| FetchFromProxy("tea.html", true, &text, &headers); |
| EXPECT_EQ(HttpStatus::kOK, headers.status_code()); |
| EXPECT_STREQ("tasty", text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, DontRewriteMislabeledAsHtml) { |
| // Make sure we don't rewrite things that claim to be HTML, but aren't. |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| SetResponseWithDefaultHeaders("page.js", kContentTypeHtml, |
| StrCat("//", CssLinkHref("a.css")), |
| kHtmlCacheTimeSec * 2); |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| FetchFromProxy("page.js", true, &text, &headers); |
| CheckHeaders(headers, kContentTypeHtml); |
| EXPECT_EQ(StrCat("//", CssLinkHref("a.css")), text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ReconstructResource) { |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| // Fetching of a rewritten resource we did not just create |
| // after an HTML rewrite. |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| RequestHeaders request_headers; |
| FetchFromProxy(Encode("", "cf", "0", "a.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| CheckHeaders(headers, kContentTypeCss); |
| headers.ComputeCaching(); |
| CheckBackgroundFetch(headers, false); |
| EXPECT_LE(start_time_ms_ + Timer::kYearMs, headers.CacheExpirationTimeMs()); |
| EXPECT_EQ(kMinimizedCssContent, text); |
| EXPECT_STREQ("cf", AppliedRewriterStringFromLog()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ReconstructResourceCustomOptions) { |
| const char kCssWithEmbeddedImage[] = "*{background-image:url(%s)}"; |
| const char kBackgroundImage[] = "1.png"; |
| |
| GoogleString text; |
| ResponseHeaders headers; |
| |
| // We're not going to image-compress so we don't need our mock image |
| // to really be an image. |
| SetResponseWithDefaultHeaders(kBackgroundImage, kContentTypePng, "image", |
| kHtmlCacheTimeSec * 2); |
| GoogleString orig_css = StringPrintf(kCssWithEmbeddedImage, kBackgroundImage); |
| SetResponseWithDefaultHeaders("embedded.css", kContentTypeCss, |
| orig_css, kHtmlCacheTimeSec * 2); |
| |
| // By default, cache extension is off in the default options. |
| server_context()->global_options()->SetDefaultRewriteLevel( |
| RewriteOptions::kPassThrough); |
| ASSERT_FALSE(options()->Enabled(RewriteOptions::kExtendCacheCss)); |
| ASSERT_FALSE(options()->Enabled(RewriteOptions::kExtendCacheImages)); |
| ASSERT_FALSE(options()->Enabled(RewriteOptions::kExtendCacheScripts)); |
| ASSERT_FALSE(options()->Enabled(RewriteOptions::kExtendCachePdfs)); |
| ASSERT_EQ(RewriteOptions::kPassThrough, options()->level()); |
| |
| // Because cache-extension was turned off, the image in the CSS file |
| // will not be changed. |
| RequestHeaders request_headers; |
| FetchFromProxy("I.embedded.css.pagespeed.cf.0.css", |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(orig_css, text); |
| |
| // Now turn on cache-extension for custom options. Invalidate cache entries |
| // up to and including the current timestamp and advance by 1ms, otherwise |
| // the previously stored embedded.css.pagespeed.cf.0.css will get re-used. |
| scoped_ptr<RewriteOptions> custom_options(factory()->NewRewriteOptions()); |
| custom_options->EnableFilter(RewriteOptions::kExtendCacheCss); |
| custom_options->EnableFilter(RewriteOptions::kExtendCacheImages); |
| custom_options->EnableFilter(RewriteOptions::kExtendCacheScripts); |
| custom_options->EnableFilter(RewriteOptions::kExtendCachePdfs); |
| custom_options->UpdateCacheInvalidationTimestampMs(timer()->NowMs()); |
| AdvanceTimeUs(Timer::kMsUs); |
| |
| // Inject the custom options into the flow via a RewriteOptionsManager. |
| SetRewriteOptions(custom_options.get()); |
| |
| // Use EncodeNormal because it matches the logic used by ProxyUrlNamer. |
| const GoogleString kExtendedBackgroundImage = |
| EncodeNormal("", "ce", "0", kBackgroundImage, "png"); |
| |
| // Now when we fetch the options, we'll find the image in the CSS |
| // cache-extended. |
| text.clear(); |
| FetchFromProxy("I.embedded.css.pagespeed.cf.0.css", |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(StringPrintf(kCssWithEmbeddedImage, |
| kExtendedBackgroundImage.c_str()), |
| text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, MinResourceTimeZero) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->SetRewriteLevel(RewriteOptions::kPassThrough); |
| options->EnableFilter(RewriteOptions::kRewriteCss); |
| options->set_min_resource_cache_time_to_rewrite_ms( |
| kHtmlCacheTimeSec * Timer::kSecondMs); |
| SetRewriteOptions(options); |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| CssLinkHref("a.css"), kHtmlCacheTimeSec * 2); |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString text; |
| ResponseHeaders headers; |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| EXPECT_EQ(CssLinkHref(Encode("", "cf", "0", "a.css", "css")), text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, MinResourceTimeLarge) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->SetRewriteLevel(RewriteOptions::kPassThrough); |
| options->EnableFilter(RewriteOptions::kRewriteCss); |
| options->set_min_resource_cache_time_to_rewrite_ms( |
| 4 * kHtmlCacheTimeSec * Timer::kSecondMs); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| CssLinkHref("a.css"), kHtmlCacheTimeSec * 2); |
| SetResponseWithDefaultHeaders("a.css", kContentTypeCss, kCssContent, |
| kHtmlCacheTimeSec * 2); |
| |
| GoogleString text; |
| ResponseHeaders headers; |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| EXPECT_EQ(CssLinkHref("a.css"), text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CacheRequests) { |
| ResponseHeaders html_headers; |
| DefaultResponseHeaders(kContentTypeHtml, kHtmlCacheTimeSec, &html_headers); |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "1"); |
| ResponseHeaders resource_headers; |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "a"); |
| |
| GoogleString text; |
| ResponseHeaders actual_headers; |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("1", text); |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "2"); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "b"); |
| |
| // Original response is still cached in both cases, so we do not |
| // fetch the new values. |
| text.clear(); |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("1", text); |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| } |
| |
| // Verifies that we proxy uncacheable resources, but do not insert them in the |
| // cache. |
| TEST_F(ProxyInterfaceTest, UncacheableResourcesNotCachedOnProxy) { |
| ResponseHeaders resource_headers; |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| resource_headers.SetDateAndCaching(http_cache()->timer()->NowMs(), |
| 300 * Timer::kSecondMs, ", private"); |
| resource_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "a"); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| |
| // We should not cache while fetching via kProxyHost. |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/style.css"), |
| true, &out_text, &out_headers); |
| EXPECT_EQ("a", out_text); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); // mapping, input resource |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // input resource |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); |
| |
| // We should likewise not cache while fetching on the origin domain. |
| out_text.clear(); |
| ClearStats(); |
| FetchFromProxy("style.css", true, &out_text, &out_headers); |
| EXPECT_EQ("a", out_text); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); // mapping, input resource |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // input resource |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); |
| |
| // Since the original response is not cached, we should pick up changes in the |
| // input resource immediately. |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "b"); |
| out_text.clear(); |
| ClearStats(); |
| FetchFromProxy("style.css", true, &out_text, &out_headers); |
| EXPECT_EQ("b", out_text); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); // mapping, input resource |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // input resource |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); |
| } |
| |
| // Verifies that we retrieve and serve uncacheable resources, but do not insert |
| // them in the cache. |
| TEST_F(ProxyInterfaceTest, UncacheableResourcesNotCachedOnResourceFetch) { |
| ResponseHeaders resource_headers; |
| RequestHeaders request_headers; |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| resource_headers.SetDateAndCaching(http_cache()->timer()->NowMs(), |
| 300 * Timer::kSecondMs, ", private"); |
| resource_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "a"); |
| |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->SetRewriteLevel(RewriteOptions::kPassThrough); |
| options->EnableFilter(RewriteOptions::kRewriteCss); |
| server_context()->ComputeSignature(options); |
| |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| |
| // cf is not on-the-fly, and we can reconstruct it while keeping it private. |
| FetchFromProxy(Encode("", "cf", "0", "style.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(out_headers.HasValue(HttpAttributes::kCacheControl, "private")); |
| EXPECT_EQ("a", out_text); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(3, lru_cache()->num_misses()); // 2x output, metadata, input |
| EXPECT_EQ(2, http_cache()->cache_misses()->Get()); // 2x output, input |
| EXPECT_EQ(2, lru_cache()->num_inserts()); // mapping, uncacheable memo |
| EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); // uncacheable memo |
| |
| out_text.clear(); |
| ClearStats(); |
| // ce is on-the-fly, and we can recover even though style.css is private. |
| FetchFromProxy(Encode("", "ce", "0", "style.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(out_headers.HasValue(HttpAttributes::kCacheControl, "private")); |
| EXPECT_EQ("a", out_text); |
| EXPECT_EQ(1, lru_cache()->num_hits()); // input uncacheable memo |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // input uncacheable memo |
| EXPECT_EQ(1, lru_cache()->num_inserts()); // mapping |
| EXPECT_EQ(1, lru_cache()->num_identical_reinserts()); // uncacheable memo |
| EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); // uncacheable memo |
| |
| out_text.clear(); |
| ClearStats(); |
| FetchFromProxy(Encode("", "ce", "0", "style.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(out_headers.HasValue(HttpAttributes::kCacheControl, "private")); |
| EXPECT_EQ("a", out_text); |
| EXPECT_EQ(2, lru_cache()->num_hits()); // uncacheable memo & modified check |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // uncacheable memo |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_identical_reinserts()) << "uncacheable memo"; |
| EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); // uncacheable memo |
| |
| // Since the original response is not cached, we should pick up changes in the |
| // input resource immediately. |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "b"); |
| out_text.clear(); |
| ClearStats(); |
| FetchFromProxy(Encode("", "ce", "0", "style.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_TRUE(out_headers.HasValue(HttpAttributes::kCacheControl, "private")); |
| EXPECT_EQ("b", out_text); |
| EXPECT_EQ(2, lru_cache()->num_hits()); // uncacheable memo & modified-check |
| EXPECT_EQ(0, http_cache()->cache_hits()->Get()); |
| EXPECT_EQ(0, lru_cache()->num_misses()); |
| EXPECT_EQ(1, http_cache()->cache_misses()->Get()) << "uncacheable memo"; |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_identical_reinserts()) << "uncacheable memo"; |
| EXPECT_EQ(1, http_cache()->cache_inserts()->Get()) << "uncacheable memo"; |
| } |
| |
| // No matter what options->respect_vary() is set to we will respect HTML Vary |
| // headers. |
| TEST_F(ProxyInterfaceTest, NoCacheVaryHtml) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_respect_vary(false); |
| server_context()->ComputeSignature(options); |
| |
| ResponseHeaders html_headers; |
| DefaultResponseHeaders(kContentTypeHtml, kHtmlCacheTimeSec, &html_headers); |
| html_headers.Add(HttpAttributes::kVary, HttpAttributes::kUserAgent); |
| html_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "1"); |
| ResponseHeaders resource_headers; |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| resource_headers.Add(HttpAttributes::kVary, HttpAttributes::kUserAgent); |
| resource_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "a"); |
| |
| GoogleString text; |
| ResponseHeaders actual_headers; |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("1", text); |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "2"); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "b"); |
| |
| // HTML was not cached because of Vary: User-Agent header. |
| // So we do fetch the new value. |
| text.clear(); |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("2", text); |
| // Resource was cached because we have respect_vary == false. |
| // So we serve the old value. |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| } |
| |
| // Test https HTML responses are never cached, while https resources are cached. |
| TEST_F(ProxyInterfaceTest, NoCacheHttpsHtml) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_respect_vary(false); |
| server_context()->ComputeSignature(options); |
| http_cache()->set_disable_html_caching_on_https(true); |
| |
| ResponseHeaders html_headers(options->ComputeHttpOptions()); |
| DefaultResponseHeaders(kContentTypeHtml, kHtmlCacheTimeSec, &html_headers); |
| html_headers.ComputeCaching(); |
| SetFetchResponse(kHttpsPageUrl, html_headers, "1"); |
| ResponseHeaders resource_headers(options->ComputeHttpOptions()); |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| resource_headers.ComputeCaching(); |
| SetFetchResponse(kHttpsCssUrl, resource_headers, "a"); |
| |
| GoogleString text; |
| ResponseHeaders actual_headers(options->ComputeHttpOptions()); |
| FetchFromProxy(kHttpsPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("1", text); |
| text.clear(); |
| FetchFromProxy(kHttpsCssUrl, true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| |
| SetFetchResponse(kHttpsPageUrl, html_headers, "2"); |
| SetFetchResponse(kHttpsCssUrl, resource_headers, "b"); |
| |
| ClearStats(); |
| // HTML was not cached because it was via https. So we do fetch the new value. |
| text.clear(); |
| FetchFromProxy(kHttpsPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("2", text); |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| // Resource was cached, so we serve the old value. |
| text.clear(); |
| FetchFromProxy(kHttpsCssUrl, true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| EXPECT_EQ(1, http_cache()->cache_hits()->Get()); |
| } |
| |
| // Respect Vary for resources if options tell us to. |
| TEST_F(ProxyInterfaceTest, NoCacheVaryAll) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_respect_vary(true); |
| server_context()->ComputeSignature(options); |
| |
| ResponseHeaders html_headers(options->ComputeHttpOptions()); |
| DefaultResponseHeaders(kContentTypeHtml, kHtmlCacheTimeSec, &html_headers); |
| html_headers.Add(HttpAttributes::kVary, HttpAttributes::kUserAgent); |
| html_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "1"); |
| ResponseHeaders resource_headers(options->ComputeHttpOptions()); |
| DefaultResponseHeaders(kContentTypeCss, kHtmlCacheTimeSec, &resource_headers); |
| resource_headers.Add(HttpAttributes::kVary, HttpAttributes::kUserAgent); |
| resource_headers.ComputeCaching(); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "a"); |
| |
| GoogleString text; |
| ResponseHeaders actual_headers(options->ComputeHttpOptions()); |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("1", text); |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("a", text); |
| |
| SetFetchResponse(AbsolutifyUrl(kPageUrl), html_headers, "2"); |
| SetFetchResponse(AbsolutifyUrl("style.css"), resource_headers, "b"); |
| |
| // Original response was not cached in either case, so we do fetch the |
| // new value. |
| text.clear(); |
| FetchFromProxy(kPageUrl, true, &text, &actual_headers); |
| EXPECT_EQ("2", text); |
| text.clear(); |
| FetchFromProxy("style.css", true, &text, &actual_headers); |
| EXPECT_EQ("b", text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, Blacklist) { |
| const char content[] = |
| "<html>\n" |
| " <head/>\n" |
| " <body>\n" |
| " <script src='tiny_mce.js'></script>\n" |
| " </body>\n" |
| "</html>\n"; |
| SetResponseWithDefaultHeaders("tiny_mce.js", kContentTypeJavascript, "", 100); |
| ValidateNoChanges("blacklist", content); |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, content, 0); |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_STREQ(content, text_out); |
| } |
| |
| TEST_F(ProxyInterfaceTest, RepairMismappedResource) { |
| // Teach the mock fetcher to serve origin content for |
| // "http://test.com/foo.js". |
| const char kContent[] = "function f() {alert('foo');}"; |
| SetResponseWithDefaultHeaders("foo.js", kContentTypeHtml, kContent, |
| kHtmlCacheTimeSec * 2); |
| |
| // Set up a Mock Namer that will mutate output resources to |
| // be served on proxy_host.com, encoding the origin URL. |
| ProxyUrlNamer url_namer; |
| ResponseHeaders headers; |
| GoogleString text; |
| server_context()->set_url_namer(&url_namer); |
| |
| // Now fetch the origin content. This will simply hit the |
| // mock fetcher and always worked. |
| FetchFromProxy("foo.js", true, &text, &headers); |
| EXPECT_EQ(kContent, text); |
| |
| // Now make a weird URL encoding of the origin resource using the |
| // proxy host. This may happen via javascript that detects its |
| // own path and initiates a 'load()' of another js file from the |
| // same path. In this variant, the resource is served from the |
| // "source domain", so it is automatically whitelisted. |
| text.clear(); |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, "/test.com/test.com/foo.js"), |
| true, &text, &headers); |
| EXPECT_EQ(kContent, text); |
| |
| // In the next case, the resource is served from a different domain. This |
| // is an open-proxy vulnerability and thus should fail. |
| text.clear(); |
| url_namer.set_authorized(false); |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, "/test.com/evil.com/foo.js"), |
| false, &text, &headers); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainHeaders) { |
| // If we're serving content from test.com via kProxyHost URL, we need to make |
| // sure that cookies are not propagated, as evil.com could also be potentially |
| // proxied via kProxyHost. |
| const char kText[] = "* { pretty; }"; |
| |
| ResponseHeaders orig_headers; |
| DefaultResponseHeaders(kContentTypeCss, 100, &orig_headers); |
| orig_headers.Add(HttpAttributes::kSetCookie, "tasty"); |
| SetFetchResponse("http://test.com/file.css", orig_headers, kText); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/file.css"), |
| true, &out_text, &out_headers); |
| EXPECT_STREQ(kText, out_text); |
| EXPECT_STREQ(NULL, out_headers.Lookup1(HttpAttributes::kSetCookie)); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainRedirectIfBlacklisted) { |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test1.com/blacklist.css"), |
| false, &out_text, &out_headers); |
| EXPECT_STREQ("", out_text); |
| EXPECT_EQ(HttpStatus::kFound, out_headers.status_code()); |
| EXPECT_STREQ("http://test1.com/blacklist.css", |
| out_headers.Lookup1(HttpAttributes::kLocation)); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainAuthorization) { |
| // If we're serving content from evil.com via kProxyHostUrl, we need to make |
| // sure we don't propagate through any (non-proxy) authorization headers, as |
| // they may have been cached from good.com (as both would look like |
| // kProxyHost to the browser). |
| ReflectingTestFetcher reflect; |
| server_context()->set_default_system_fetcher(&reflect); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| |
| RequestHeaders request_headers; |
| request_headers.Add("Was", "Here"); |
| request_headers.Add(HttpAttributes::kAuthorization, "Secret"); |
| request_headers.Add(HttpAttributes::kProxyAuthorization, "OurSecret"); |
| |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| // Using .txt here so we don't try any AJAX rewriting. |
| FetchFromProxy(StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/file.txt"), |
| request_headers, true, &out_text, &out_headers); |
| EXPECT_STREQ("Here", out_headers.Lookup1("Was")); |
| EXPECT_FALSE(out_headers.Has(HttpAttributes::kAuthorization)); |
| EXPECT_FALSE(out_headers.Has(HttpAttributes::kProxyAuthorization)); |
| mock_scheduler()->AwaitQuiescence(); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainHeadersWithUncacheableResourceOnProxy) { |
| // Check that we do not propagate cookies from test.com via kProxyHost URL, |
| // as in CrossDomainHeaders above. Also check that we do propagate cache |
| // control. |
| const char kText[] = "* { pretty; }"; |
| |
| ResponseHeaders orig_headers; |
| DefaultResponseHeaders(kContentTypeCss, 100, &orig_headers); |
| orig_headers.Add(HttpAttributes::kSetCookie, "tasty"); |
| orig_headers.SetDateAndCaching(http_cache()->timer()->NowMs(), |
| 400 * Timer::kSecondMs, ", private"); |
| orig_headers.ComputeCaching(); |
| SetFetchResponse("http://test.com/file.css", orig_headers, kText); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/file.css"), |
| true, &out_text, &out_headers); |
| |
| // Check that we ate the cookies. |
| EXPECT_STREQ(kText, out_text); |
| ConstStringStarVector values; |
| out_headers.Lookup(HttpAttributes::kSetCookie, &values); |
| EXPECT_EQ(0, values.size()); |
| |
| // Check that the resource Cache-Control has been preserved. |
| values.clear(); |
| out_headers.Lookup(HttpAttributes::kCacheControl, &values); |
| ASSERT_EQ(2, values.size()); |
| EXPECT_STREQ("max-age=400", *values[0]); |
| EXPECT_STREQ("private", *values[1]); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainHeadersWithUncacheableResourceOnFetch) { |
| // Check that we do not propagate cookies from test.com via a resource fetch, |
| // as in CrossDomainHeaders above. Also check that we do propagate cache |
| // control, and that we run the filter specified in the resource fetch URL. |
| // Note that the running of filters at present can only happen if |
| // the filter is on the-fly. |
| const char kText[] = "* { pretty; }"; |
| |
| ResponseHeaders orig_headers; |
| DefaultResponseHeaders(kContentTypeCss, 100, &orig_headers); |
| orig_headers.Add(HttpAttributes::kSetCookie, "tasty"); |
| orig_headers.SetDateAndCaching(http_cache()->timer()->NowMs(), |
| 400 * Timer::kSecondMs, ", private"); |
| orig_headers.ComputeCaching(); |
| SetFetchResponse("http://test.com/file.css", orig_headers, kText); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| RequestHeaders request_headers; |
| FetchFromProxy(Encode("", "ce", "0", "file.css", "css"), |
| request_headers, |
| true, |
| &out_text, |
| &out_headers, |
| false); |
| |
| // Check that we passed through the CSS. |
| EXPECT_STREQ(kText, out_text); |
| // Check that we ate the cookies. |
| ConstStringStarVector values; |
| out_headers.Lookup(HttpAttributes::kSetCookie, &values); |
| EXPECT_EQ(0, values.size()); |
| |
| // Check that the resource Cache-Control has been preserved. |
| // max-age actually gets smaller, though, since this also triggers |
| // a rewrite failure. |
| values.clear(); |
| EXPECT_STREQ("max-age=300, private", out_headers.LookupJoined( |
| HttpAttributes::kCacheControl)); |
| } |
| |
| TEST_F(ProxyInterfaceTest, CrossDomainHeadersWithUncacheableResourceOnFetch2) { |
| // Variant of the above with a non-on-the-fly filter. |
| const char kText[] = "* { pretty; }"; |
| |
| ResponseHeaders orig_headers; |
| DefaultResponseHeaders(kContentTypeCss, 100, &orig_headers); |
| orig_headers.Add(HttpAttributes::kSetCookie, "tasty"); |
| orig_headers.SetDateAndCaching(http_cache()->timer()->NowMs(), |
| 400 * Timer::kSecondMs, ", private"); |
| orig_headers.ComputeCaching(); |
| SetFetchResponse("http://test.com/file.css", orig_headers, kText); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| RequestHeaders request_headers; |
| FetchFromProxy(Encode("", "cf", "0", "file.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| // Proper output |
| EXPECT_STREQ("*{pretty}", out_text); |
| |
| // Private. |
| EXPECT_STREQ("max-age=400, private", out_headers.LookupJoined( |
| HttpAttributes::kCacheControl)); |
| |
| // Check that we ate the cookies. |
| EXPECT_FALSE(out_headers.Has(HttpAttributes::kSetCookie)); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ProxyResourceQueryOnly) { |
| // At one point we had a bug where if we optimized a pagespeed resource |
| // whose original name was a bare query, we would loop infinitely when |
| // trying to fetch it from a separate-domain proxy. |
| const char kUrl[] = "?somestuff"; |
| SetResponseWithDefaultHeaders(kUrl, kContentTypeJavascript, |
| "var a = 2;// stuff", kHtmlCacheTimeSec * 2); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| RequestHeaders request_headers; |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/", |
| EncodeNormal("", "jm", "0", kUrl, "css")), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_STREQ("var a=2;", out_text); |
| CheckBackgroundFetch(out_headers, false); |
| } |
| |
| TEST_F(ProxyInterfaceTest, NoRehostIncompatMPS) { |
| // Make sure we don't try to interpret a URL from an incompatible |
| // mod_pagespeed version at our proxy host level. |
| |
| // This url will be rejected by CssUrlEncoder |
| const char kOldName[] = "style.css.pagespeed.cf.0.css"; |
| const char kContent[] = "* {}"; |
| SetResponseWithDefaultHeaders(kOldName, kContentTypeCss, kContent, 100); |
| |
| ProxyUrlNamer url_namer; |
| server_context()->set_url_namer(&url_namer); |
| ResponseHeaders out_headers; |
| GoogleString out_text; |
| RequestHeaders request_headers; |
| FetchFromProxy( |
| StrCat("http://", ProxyUrlNamer::kProxyHost, |
| "/test.com/test.com/", |
| EncodeNormal("", "ce", "0", kOldName, "css")), |
| request_headers, |
| true, /* expect_success */ |
| &out_text, |
| &out_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| EXPECT_EQ(HttpStatus::kOK, out_headers.status_code()); |
| EXPECT_STREQ(kContent, out_text); |
| } |
| |
| // Test that we serve "Cache-Control: no-store" only when original page did. |
| TEST_F(ProxyInterfaceTest, NoStore) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_max_html_cache_time_ms(0); |
| server_context()->ComputeSignature(options); |
| |
| // Most headers get converted to "no-cache, max-age=0". |
| EXPECT_STREQ("max-age=0, no-cache", |
| RewriteHtmlCacheHeader("empty", "")); |
| EXPECT_STREQ("max-age=0, no-cache", |
| RewriteHtmlCacheHeader("private", "private, max-age=100")); |
| EXPECT_STREQ("max-age=0, no-cache", |
| RewriteHtmlCacheHeader("no-cache", "no-cache")); |
| |
| // Headers with "no-store", preserve that header as well. |
| EXPECT_STREQ("max-age=0, no-cache, no-store", |
| RewriteHtmlCacheHeader("no-store", "no-cache, no-store")); |
| EXPECT_STREQ("max-age=0, no-cache, no-store", |
| RewriteHtmlCacheHeader("no-store2", "no-store, max-age=300")); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheFilter) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->DisableFilter(RewriteOptions::kAddHead); |
| server_context()->ComputeSignature(options); |
| |
| CreateFilterCallback create_filter_callback; |
| factory()->AddCreateFilterCallback(&create_filter_callback); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| "<div><p></p></div>", 0); |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ("<!-- --><div><p></p></div>", text_out); |
| |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ("<!-- 2 elements unstable --><div><p></p></div>", text_out); |
| |
| // How many refreshes should we require before it's stable? That |
| // tuning can be done in the PropertyCacheTest. For this |
| // system-test just do a hundred blind refreshes and check again for |
| // stability. |
| const int kFetchIterations = 100; |
| for (int i = 0; i < kFetchIterations; ++i) { |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| } |
| |
| // Must be stable by now! |
| EXPECT_EQ("<!-- 2 elements stable --><div><p></p></div>", text_out); |
| |
| // In this algorithm we will spend a property-cache-write per fetch. |
| // |
| // We'll also check that we do no cache writes when there are no properties |
| // to save. |
| EXPECT_EQ(2 + kFetchIterations, lru_cache()->num_inserts()); |
| |
| // Now change the HTML and watch the #elements change. |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| "<div><span><p></p></span></div>", 0); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ("<!-- 3 elements stable --><div><span><p></p></span></div>", |
| text_out); |
| |
| ClearStats(); |
| |
| // Finally, disable the property-cache and note that the element-count |
| // annotatation reverts to "unknown mode" |
| server_context_->set_enable_property_cache(false); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ("<!-- --><div><span><p></p></span></div>", text_out); |
| } |
| |
| TEST_F(ProxyInterfaceTest, DomCohortWritten) { |
| // Other than the write of DomCohort, there will be no properties added to |
| // the cache in this test because we have not enabled the filter with |
| // CreateFilterCallback create_filter_callback; |
| // factory()->AddCreateFilterCallback(&callback); |
| |
| DisableAjax(); |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| |
| // No writes should occur if no filter that uses the dom cohort is enabled. |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); // 1 property-cache + 1 http-cache |
| |
| // Enable a filter that uses the dom cohort and make sure property cache is |
| // updated. |
| ClearStats(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(1, lru_cache()->num_inserts()); |
| EXPECT_EQ(2, lru_cache()->num_misses()); // 1 property-cache + 1 http-cache |
| |
| ClearStats(); |
| server_context_->set_enable_property_cache(false); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); // http-cache only. |
| } |
| |
| TEST_F(ProxyInterfaceTest, StatusCodePropertyWritten) { |
| DisableAjax(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| |
| // Status code 404 gets written when page is not available. |
| SetFetchResponse404(kPageUrl); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(HttpStatus::kNotFound, |
| GetStatusCodeInPropertyCache(StrCat(kTestDomain, kPageUrl))); |
| |
| // Status code 200 gets written when page is available. |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, |
| "<html></html>", kHtmlCacheTimeSec); |
| lru_cache()->Clear(); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(HttpStatus::kOK, |
| GetStatusCodeInPropertyCache(StrCat(kTestDomain, kPageUrl))); |
| // Status code 301 gets written when it is a permanent redirect. |
| headers_out.Clear(); |
| text_out.clear(); |
| headers_out.SetStatusAndReason(HttpStatus::kMovedPermanently); |
| SetFetchResponse(StrCat(kTestDomain, kPageUrl), headers_out, text_out); |
| lru_cache()->Clear(); |
| FetchFromProxy(kPageUrl, true, &text_out, &headers_out); |
| EXPECT_EQ(HttpStatus::kMovedPermanently, |
| GetStatusCodeInPropertyCache(StrCat(kTestDomain, kPageUrl))); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheNoWritesIfHtmlEndsWithTxt) { |
| CreateFilterCallback create_filter_callback; |
| factory()->AddCreateFilterCallback(&create_filter_callback); |
| |
| // There will be no properties added to the cache set in this test because |
| // we have not enabled the filter with |
| // CreateFilterCallback create_filter_callback; |
| // factory()->AddCreateFilterCallback(&callback); |
| |
| DisableAjax(); |
| SetResponseWithDefaultHeaders("page.txt", kContentTypeHtml, |
| "<div><p></p></div>", 0); |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| |
| FetchFromProxy("page.txt", true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); // http-cache only |
| |
| ClearStats(); |
| server_context_->set_enable_property_cache(false); |
| FetchFromProxy("page.txt", true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); // http-cache only |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheNoWritesForNonGetRequests) { |
| CreateFilterCallback create_filter_callback; |
| factory()->AddCreateFilterCallback(&create_filter_callback); |
| |
| DisableAjax(); |
| SetResponseWithDefaultHeaders("page.txt", kContentTypeHtml, |
| "<div><p></p></div>", 0); |
| GoogleString text_out; |
| ResponseHeaders headers_out; |
| RequestHeaders request_headers; |
| request_headers.set_method(RequestHeaders::kPost); |
| |
| FetchFromProxy("page.txt", true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); // http-cache only |
| |
| ClearStats(); |
| server_context_->set_enable_property_cache(false); |
| FetchFromProxy("page.txt", true, &text_out, &headers_out); |
| EXPECT_EQ(0, lru_cache()->num_inserts()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); // http-cache only |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheNoWritesIfNonHtmlDelayedCache) { |
| DisableAjax(); |
| TestPropertyCache(kImageFilenameLackingExt, true, false, true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheNoWritesIfNonHtmlImmediateCache) { |
| // Tests rewriting a file that turns out to be a jpeg, but lacks an |
| // extension, where the property-cache lookup is delivered immediately. |
| DisableAjax(); |
| TestPropertyCache(kImageFilenameLackingExt, false, false, true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, PropCacheNoWritesIfNonHtmlThreadedCache) { |
| // Tests rewriting a file that turns out to be a jpeg, but lacks an |
| // extension, where the property-cache lookup is delivered in a |
| // separate thread. |
| DisableAjax(); |
| TestPropertyCache(kImageFilenameLackingExt, true, true, true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, StatusCodeUpdateRace) { |
| // Tests rewriting a file that turns out to be a jpeg, but lacks an |
| // extension, where the property-cache lookup is delivered in a |
| // separate thread. Use sync points to ensure that Done() deletes the |
| // collector just after the Detach() critical block is executed. |
| DisableAjax(); |
| TestPropertyCache(kImageFilenameLackingExt, false, true, true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ThreadedHtml) { |
| // Tests rewriting HTML resource where property-cache lookup is delivered |
| // in a separate thread. |
| DisableAjax(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| TestPropertyCache(kPageUrl, true, true, true); |
| } |
| |
| TEST_F(ProxyInterfaceTest, ThreadedHtmlFetcherFailure) { |
| // Tests rewriting HTML resource where property-cache lookup is delivered |
| // in a separate thread, but the HTML lookup fails after emitting the |
| // body. |
| DisableAjax(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| mock_url_fetcher()->SetResponseFailure(AbsolutifyUrl(kPageUrl)); |
| TestPropertyCache(kPageUrl, true, true, false); |
| } |
| |
| TEST_F(ProxyInterfaceTest, HtmlFetcherFailure) { |
| // Tests rewriting HTML resource where property-cache lookup is |
| // delivered in a blocking fashion, and the HTML lookup fails after |
| // emitting the body. |
| DisableAjax(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| mock_url_fetcher()->SetResponseFailure(AbsolutifyUrl(kPageUrl)); |
| TestPropertyCache(kPageUrl, false, false, false); |
| } |
| |
| TEST_F(ProxyInterfaceTest, HeadersSetupRace) { |
| // |
| // This crash occured where an Idle-callback is used to flush HTML. |
| // In this bug, we were connecting the property-cache callback to |
| // the ProxyFetch and then mutating response-headers. The property-cache |
| // callback was waking up the QueuedWorkerPool::Sequence used by |
| // the ProxyFetch, which was waking up and calling HeadersComplete. |
| // If the implementation of HeadersComplete mutated headers itself, |
| // we'd have a deadly race. |
| // |
| // This test uses the ThreadSynchronizer class to induce the desired |
| // race, with strategically placed calls to Signal and Wait. |
| // |
| // Note that the fix for the race means that one of the Signals does |
| // not occur at all, so we have to declare it as "Sloppy" so the |
| // ThreadSynchronizer class doesn't vomit on destruction. |
| const int kIdleCallbackTimeoutMs = 10; |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_idle_flush_time_ms(kIdleCallbackTimeoutMs); |
| options->set_flush_html(true); |
| server_context()->ComputeSignature(options); |
| DisableAjax(); |
| EnableDomCohortWritesWithDnsPrefetch(); |
| ThreadSynchronizer* sync = server_context()->thread_synchronizer(); |
| sync->EnableForPrefix(ProxyFetch::kHeadersSetupRacePrefix); |
| ThreadSystem* thread_system = server_context()->thread_system(); |
| QueuedWorkerPool pool(1, "test", thread_system); |
| QueuedWorkerPool::Sequence* sequence = pool.NewSequence(); |
| WorkerTestBase::SyncPoint sync_point(thread_system); |
| sequence->Add(MakeFunction(static_cast<ProxyInterfaceTestBase*>(this), |
| &ProxyInterfaceTest::TestHeadersSetupRace)); |
| sequence->Add(new WorkerTestBase::NotifyRunFunction(&sync_point)); |
| sync->TimedWait(ProxyFetch::kHeadersSetupRaceAlarmQueued, |
| ProxyFetch::kTestSignalTimeoutMs); |
| { |
| // Trigger the idle-callback, if it has been queued. |
| ScopedMutex lock(mock_scheduler()->mutex()); |
| mock_scheduler()->ProcessAlarmsOrWaitUs( |
| kIdleCallbackTimeoutMs * Timer::kMsUs); |
| } |
| sync->Wait(ProxyFetch::kHeadersSetupRaceDone); |
| sync_point.Wait(); |
| pool.ShutDown(); |
| sync->AllowSloppyTermination(ProxyFetch::kHeadersSetupRaceAlarmQueued); |
| } |
| |
| // TODO(jmarantz): add a test with a simulated slow cache to see what happens |
| // when the rest of the system must block, buffering up incoming HTML text, |
| // waiting for the property-cache lookups to complete. |
| |
| // Test that we set the Experiment cookie up appropriately. |
| TEST_F(ProxyInterfaceTest, ExperimentTest) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_ga_id("123-455-2341"); |
| options->set_running_experiment(true); |
| NullMessageHandler handler; |
| options->AddExperimentSpec("id=2;enable=extend_cache;percent=100", &handler); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders("example.jpg", kContentTypeJpeg, |
| "image data", 300); |
| |
| ResponseHeaders headers; |
| const char kContent[] = "<html><head></head><body>A very compelling " |
| "article with an image: <img src=example.jpg></body></html>"; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| headers.Clear(); |
| |
| GoogleString text; |
| FetchFromProxy("text.html", true, &text, &headers); |
| // Assign all visitors to an experiment_spec. |
| EXPECT_TRUE(headers.Has(HttpAttributes::kSetCookie)); |
| ConstStringStarVector values; |
| headers.Lookup(HttpAttributes::kSetCookie, &values); |
| bool found = false; |
| for (int i = 0, n = values.size(); i < n; ++i) { |
| if (values[i]->find(experiment::kExperimentCookie) == 0) { |
| found = true; |
| break; |
| } |
| } |
| EXPECT_TRUE(found); |
| // Image cache-extended and including experiment_spec 'a'. |
| EXPECT_TRUE(text.find("example.jpg.pagespeed.a.ce") != GoogleString::npos); |
| |
| headers.Clear(); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| SetFetchResponse(AbsolutifyUrl("text2.html"), headers, kContent); |
| headers.Clear(); |
| text.clear(); |
| |
| RequestHeaders req_headers; |
| req_headers.Add(HttpAttributes::kCookie, "PageSpeedExperiment=2"); |
| |
| FetchFromProxy("text2.html", req_headers, true, &text, &headers); |
| // Visitor already has cookie with id=2; don't give them a new one. |
| EXPECT_FALSE(headers.Has(HttpAttributes::kSetCookie)); |
| // Image cache-extended and including experiment_spec 'a'. |
| EXPECT_TRUE(text.find("example.jpg.pagespeed.a.ce") != GoogleString::npos); |
| |
| // Check that we don't include an experiment_spec index in urls for the "no |
| // experiment" group (id=0). |
| headers.Clear(); |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| SetFetchResponse(AbsolutifyUrl("text3.html"), headers, kContent); |
| headers.Clear(); |
| text.clear(); |
| |
| RequestHeaders req_headers2; |
| req_headers2.Add(HttpAttributes::kCookie, "PageSpeedExperiment=0"); |
| |
| FetchFromProxy("text3.html", req_headers2, true, &text, &headers); |
| EXPECT_FALSE(headers.Has(HttpAttributes::kSetCookie)); |
| EXPECT_TRUE(text.find("example.jpg.pagespeed.ce") != GoogleString::npos); |
| } |
| |
| TEST_F(ProxyInterfaceTest, UrlAttributeTest) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->EnableFilter(RewriteOptions::kRewriteDomains); |
| options->set_domain_rewrite_hyperlinks(true); |
| NullMessageHandler handler; |
| options->WriteableDomainLawyer()->AddRewriteDomainMapping( |
| "http://dst.example.com", "http://src.example.com", &handler); |
| options->AddUrlValuedAttribute( |
| "span", "src", semantic_type::kHyperlink); |
| options->AddUrlValuedAttribute("hr", "imgsrc", semantic_type::kImage); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders( |
| "http://src.example.com/null", kContentTypeHtml, "", 0); |
| ResponseHeaders headers; |
| const char kContent[] = "<html><head></head><body>" |
| "<img src=\"http://src.example.com/null\">" |
| "<hr imgsrc=\"http://src.example.com/null\">" |
| "<span src=\"http://src.example.com/null\"></span>" |
| "<other src=\"http://src.example.com/null\"></other></body></html>"; |
| headers.Add(HttpAttributes::kContentType, kContentTypeHtml.mime_type()); |
| headers.SetStatusAndReason(HttpStatus::kOK); |
| SetFetchResponse(AbsolutifyUrl("text.html"), headers, kContent); |
| headers.Clear(); |
| GoogleString text; |
| FetchFromProxy("text.html", true, &text, &headers); |
| |
| // img.src, hr.imgsrc, and span.src are all rewritten |
| EXPECT_TRUE(text.find("<img src=\"http://dst.example.com/null\"") != |
| GoogleString::npos); |
| EXPECT_TRUE(text.find("<hr imgsrc=\"http://dst.example.com/null\"") != |
| GoogleString::npos); |
| EXPECT_TRUE(text.find("<span src=\"http://dst.example.com/null\"") != |
| GoogleString::npos); |
| // other.src not rewritten |
| EXPECT_TRUE(text.find("<other src=\"http://src.example.com/null\"") != |
| GoogleString::npos); |
| } |
| |
| TEST_F(ProxyInterfaceTest, TestFallbackPropertiesUsageWithQueryParams) { |
| GoogleString url("http://www.test.com/a/b.html?withquery=some"); |
| GoogleString fallback_url("http://www.test.com/a/b.html?withquery=different"); |
| TestFallbackPageProperties(url, fallback_url); |
| } |
| |
| TEST_F(ProxyInterfaceTest, TestFallbackPropertiesUsageWithLeafNode) { |
| GoogleString url("http://www.test.com/a/b.html"); |
| GoogleString fallback_url("http://www.test.com/a/c.html"); |
| TestFallbackPageProperties(url, fallback_url); |
| } |
| |
| TEST_F(ProxyInterfaceTest, |
| TestFallbackPropertiesUsageWithLeafNodeHavingTrailingSlash) { |
| GoogleString url("http://www.test.com/a/b/"); |
| GoogleString fallback_url("http://www.test.com/a/c/"); |
| TestFallbackPageProperties(url, fallback_url); |
| } |
| |
| TEST_F(ProxyInterfaceTest, TestNoFallbackCallWithNoLeaf) { |
| GoogleUrl gurl("http://www.test.com/"); |
| options()->set_use_fallback_property_cache_values(true); |
| StringAsyncFetch callback( |
| RequestContext::NewTestRequestContext( |
| server_context()->thread_system())); |
| RequestHeaders request_headers; |
| callback.set_request_headers(&request_headers); |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> callback_collector( |
| proxy_interface_->InitiatePropertyCacheLookup( |
| false, gurl, options(), &callback, false)); |
| |
| PropertyPage* fallback_page = callback_collector->fallback_property_page() |
| ->property_page_with_fallback_values(); |
| // No PropertyPage with fallback values. |
| EXPECT_EQ(NULL, fallback_page); |
| } |
| |
| TEST_F(ProxyInterfaceTest, TestSkipBlinkCohortLookUp) { |
| GoogleUrl gurl("http://www.test.com/"); |
| StringAsyncFetch callback( |
| RequestContext::NewTestRequestContext(server_context()->thread_system())); |
| RequestHeaders request_headers; |
| callback.set_request_headers(&request_headers); |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> callback_collector( |
| proxy_interface_->InitiatePropertyCacheLookup( |
| false, gurl, options(), &callback, false)); |
| |
| // Cache lookup only for dom cohort. |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(1, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, TestSkipBlinkCohortLookUpInFallbackPage) { |
| GoogleUrl gurl("http://www.test.com/1.html?a=b"); |
| options()->set_use_fallback_property_cache_values(true); |
| StringAsyncFetch callback( |
| RequestContext::NewTestRequestContext(server_context()->thread_system())); |
| RequestHeaders request_headers; |
| callback.set_request_headers(&request_headers); |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> callback_collector( |
| proxy_interface_->InitiatePropertyCacheLookup( |
| false, gurl, options(), &callback, true)); |
| |
| // Cache lookup for: |
| // dom and blink cohort for actual property page. |
| // dom cohort for fallback property page. |
| EXPECT_EQ(0, lru_cache()->num_hits()); |
| EXPECT_EQ(3, lru_cache()->num_misses()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, BailOutOfParsing) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->EnableExtendCacheFilters(); |
| options->set_max_html_parse_bytes(60); |
| options->DisableFilter(RewriteOptions::kAddHead); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders(StrCat(kTestDomain, "1.jpg"), kContentTypeJpeg, |
| "image", kHtmlCacheTimeSec * 2); |
| |
| // This is larger than 60 bytes. |
| const char kContent[] = "<html><head></head><body>" |
| "<img src=\"1.jpg\">" |
| "<p>Some very long and very boring text</p>" |
| "</body></html>"; |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, kContent, 0); |
| ResponseHeaders headers; |
| GoogleString text; |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| // For the first request, we bail out of parsing and insert the redirect. We |
| // also update the pcache. |
| EXPECT_STREQ( |
| "<html><script type=\"text/javascript\">" |
| "window.location=\"http://test.com/page.html?PageSpeed=off\";" |
| "</script></html>", |
| text); |
| |
| headers.Clear(); |
| text.clear(); |
| // We look up the pcache and find that we should skip parsing. Hence, we just |
| // pass the bytes through. |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| EXPECT_EQ(kContent, text); |
| |
| // This is smaller than 60 bytes. |
| const char kNewContent[] = "<html><head></head><body>" |
| "<img src=\"1.jpg\"></body></html>"; |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, kNewContent, 0); |
| headers.Clear(); |
| text.clear(); |
| // We still remember that we should skip parsing. Hence, we pass the bytes |
| // through. However, after this request, we update the pcache to indicate that |
| // we should no longer skip parsing. |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| EXPECT_EQ(kNewContent, text); |
| |
| headers.Clear(); |
| text.clear(); |
| // This request is rewritten. |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| EXPECT_EQ("<html><head></head><body>" |
| "<img src=\"1.jpg.pagespeed.ce.0.jpg\">" |
| "</body></html>", text); |
| } |
| |
| TEST_F(ProxyInterfaceTest, LoggingInfoRewriteInfoMaxSize) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->set_max_rewrite_info_log_size(10); |
| server_context()->ComputeSignature(options); |
| |
| SetResponseWithDefaultHeaders(StrCat(kTestDomain, "1.jpg"), kContentTypeJpeg, |
| "image", kHtmlCacheTimeSec * 2); |
| |
| GoogleString content = "<html><head></head><body>"; |
| for (int i = 0; i < 50; ++i) { |
| StrAppend(&content, "<img src=\"1.jpg\">"); |
| } |
| StrAppend(&content, "</body></html>"); |
| |
| SetResponseWithDefaultHeaders(kPageUrl, kContentTypeHtml, content, 0); |
| ResponseHeaders headers; |
| GoogleString text; |
| FetchFromProxy(kPageUrl, true, &text, &headers); |
| |
| GoogleString expected_response(content); |
| GlobalReplaceSubstring("1.jpg", "1.jpg.pagespeed.ce.0.jpg", |
| &expected_response); |
| EXPECT_STREQ(expected_response, text); |
| EXPECT_EQ(10, logging_info()->rewriter_info_size()); |
| EXPECT_TRUE(logging_info()->rewriter_info_size_limit_exceeded()); |
| } |
| |
| TEST_F(ProxyInterfaceTest, WebpImageReconstruction) { |
| RewriteOptions* options = server_context()->global_options(); |
| options->ClearSignatureForTesting(); |
| options->EnableFilter(RewriteOptions::kConvertJpegToWebp); |
| server_context()->ComputeSignature(options); |
| |
| AddFileToMockFetcher(StrCat(kTestDomain, "1.jpg"), "Puzzle.jpg", |
| kContentTypeJpeg, 100); |
| ResponseHeaders response_headers; |
| GoogleString text; |
| RequestHeaders request_headers; |
| request_headers.Replace(HttpAttributes::kUserAgent, "webp"); |
| request_headers.Replace(HttpAttributes::kAccept, "image/webp"); |
| |
| const GoogleString kWebpUrl = Encode("", "ic", "0", "1.jpg", "webp"); |
| |
| FetchFromProxy( |
| kWebpUrl, |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| response_headers.ComputeCaching(); |
| EXPECT_STREQ(kContentTypeWebp.mime_type(), |
| response_headers.Lookup1(HttpAttributes::kContentType)); |
| EXPECT_EQ(ServerContext::kGeneratedMaxAgeMs, response_headers.cache_ttl_ms()); |
| |
| const char kCssWithEmbeddedImage[] = "*{background-image:url(%s)}"; |
| SetResponseWithDefaultHeaders( |
| "embedded.css", kContentTypeCss, |
| StringPrintf(kCssWithEmbeddedImage, "1.jpg"), kHtmlCacheTimeSec * 2); |
| |
| FetchFromProxy(Encode("", "cf", "0", "embedded.css", "css"), |
| request_headers, |
| true, /* expect_success */ |
| &text, |
| &response_headers, |
| false /* proxy_fetch_property_callback_collector_created */); |
| response_headers.ComputeCaching(); |
| EXPECT_EQ(ServerContext::kGeneratedMaxAgeMs, response_headers.cache_ttl_ms()); |
| EXPECT_EQ(StringPrintf(kCssWithEmbeddedImage, kWebpUrl.c_str()), text); |
| } |
| |
| class ProxyInterfaceOriginPropertyPageTest : public ProxyInterfaceTest { |
| protected: |
| class PerOriginPageReaderFilter : public EmptyHtmlFilter { |
| public: |
| explicit PerOriginPageReaderFilter(RewriteDriver* driver) |
| : driver_(driver) {} |
| |
| virtual void StartDocument() { |
| // This keeps a little per-site visits counter in per-origin pcache |
| // and dumps it into a comment. |
| PropertyPage* sitewide_page = driver_->origin_property_page(); |
| const PropertyCache::Cohort* dom_cohort = |
| driver_->server_context()->dom_cohort(); |
| const char kVisits[] = "visits"; |
| PropertyValue* val = sitewide_page->GetProperty(dom_cohort, kVisits); |
| |
| int count = 0; |
| if (val->has_value()) { |
| EXPECT_TRUE(StringToInt(val->value(), &count)); |
| } |
| |
| driver_->InsertComment(StrCat("Site visit:", IntegerToString(count))); |
| |
| // Update counter. |
| sitewide_page->UpdateValue(dom_cohort, kVisits, |
| IntegerToString(count + 1)); |
| sitewide_page->WriteCohort(dom_cohort); |
| } |
| |
| virtual void StartElement(HtmlElement* element) {} |
| virtual void EndDocument() {} |
| virtual const char* Name() const { return "PerOriginPageReaderFilter"; } |
| |
| private: |
| RewriteDriver* driver_; |
| DISALLOW_COPY_AND_ASSIGN(PerOriginPageReaderFilter); |
| }; |
| |
| class PerOriginPageReaderFilterCreator |
| : public TestRewriteDriverFactory::CreateFilterCallback { |
| public: |
| PerOriginPageReaderFilterCreator() {} |
| virtual ~PerOriginPageReaderFilterCreator() {} |
| |
| virtual HtmlFilter* Done(RewriteDriver* driver) { |
| return new PerOriginPageReaderFilter(driver); |
| } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(PerOriginPageReaderFilterCreator); |
| }; |
| |
| virtual void SetUp() { |
| RewriteOptions* options = server_context()->global_options(); |
| // mobilizer turns on the per-domain page. |
| options->ClearSignatureForTesting(); |
| options->EnableFilter(RewriteOptions::kMobilize); |
| ProxyInterfaceTest::SetUp(); |
| } |
| }; |
| |
| TEST_F(ProxyInterfaceOriginPropertyPageTest, Basic) { |
| PerOriginPageReaderFilterCreator filter_creator; |
| factory()->AddCreateFilterCallback(&filter_creator); |
| |
| ResponseHeaders headers; |
| GoogleString body; |
| FetchFromProxy(kPageUrl, true, &body, &headers); |
| EXPECT_TRUE(HasPrefixString(body, "<!--Site visit:0-->")) << body; |
| |
| FetchFromProxy(kPageUrl, true, &body, &headers); |
| EXPECT_TRUE(HasPrefixString(body, "<!--Site visit:1-->")) << body; |
| |
| // Count increases on a different page, too. |
| GoogleString other_page = StrCat("totally/different/from/", kPageUrl); |
| SetResponseWithDefaultHeaders(other_page, kContentTypeHtml, |
| "<div><p></p></div>", 0); |
| |
| FetchFromProxy(other_page, true, &body, &headers); |
| EXPECT_TRUE(HasPrefixString(body, "<!--Site visit:2-->")) << body; |
| } |
| |
| TEST_F(ProxyInterfaceOriginPropertyPageTest, PostWithDelayCache) { |
| // Test for not crashing. |
| PerOriginPageReaderFilterCreator filter_creator; |
| factory()->AddCreateFilterCallback(&filter_creator); |
| |
| RequestHeaders request_headers; |
| ResponseHeaders headers; |
| request_headers.set_method(RequestHeaders::kPost); |
| |
| SetCacheDelayUs(100); |
| FetchFromProxyNoWait(kPageUrl, request_headers, true /*expect_success*/, |
| false /*log_flush*/, &headers); |
| mock_scheduler()->AdvanceTimeUs(200); |
| WaitForFetch(true); |
| } |
| |
| } // namespace net_instaweb |
