Google Git
Sign in
apache / incubator-pagespeed-debian / 16b49815e99b996f06e51fb4c4930437e6ccfc4c / . / src / net / instaweb / rewriter / in_place_rewrite_context_test.cc
blob: c435f49c1f86e5c90425aaea1fe2681ed35db486 [file] [log] [blame]
/*
* Copyright 2011 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: nikhilmadan@google.com (Nikhil Madan)
#include "net/instaweb/rewriter/public/in_place_rewrite_context.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/counting_url_async_fetcher.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/mock_url_fetcher.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/public/fake_filter.h"
#include "net/instaweb/rewriter/public/file_load_policy.h"
#include "net/instaweb/rewriter/public/output_resource.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_test_base.h"
#include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
#include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
#include "pagespeed/kernel/base/gtest.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/mock_message_handler.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/cache/lru_cache.h"
#include "pagespeed/kernel/html/html_parse_test_base.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/image_types.pb.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/semantic_type.h"
#include "pagespeed/kernel/http/user_agent_matcher.h"
#include "pagespeed/kernel/http/user_agent_matcher_test_base.h"
#include "pagespeed/kernel/thread/mock_scheduler.h"
#include "pagespeed/kernel/thread/worker_test_base.h"
namespace net_instaweb {
namespace {
class FakeFetch : public AsyncFetch {
public:
FakeFetch(const RequestContextPtr& request_context,
RewriteOptions* options,
const GoogleString& url,
WorkerTestBase::SyncPoint* sync,
ResponseHeaders* response_headers)
: AsyncFetch(request_context),
done_(false),
success_(false),
options_(options),
url_(url),
sync_(sync) {
set_response_headers(response_headers);
}
virtual ~FakeFetch() {}
virtual void HandleHeadersComplete() {}
virtual bool HandleWrite(const StringPiece& content,
MessageHandler* handler) {
content.AppendToString(&content_);
return true;
}
virtual bool HandleFlush(MessageHandler* handler) {
return true;
}
virtual void HandleDone(bool success) {
response_headers()->ComputeCaching();
done_ = true;
success_ = success;
sync_->Notify();
}
StringPiece content() { return content_; }
bool done() { return done_; }
bool success() { return success_; }
bool IsCachedResultValid(const ResponseHeaders& headers) {
return OptionsAwareHTTPCacheCallback::IsCacheValid(
url_, *options_, request_context(), headers);
}
private:
GoogleString content_;
bool done_;
bool success_;
const RewriteOptions* options_;
GoogleString url_;
WorkerTestBase::SyncPoint* sync_;
DISALLOW_COPY_AND_ASSIGN(FakeFetch);
};
class FakeImageFilter : public FakeFilter {
public:
class Context : public FakeFilter::Context {
public:
Context(FakeImageFilter* filter, RewriteDriver* driver,
RewriteContext* parent, ResourceContext* resource_context)
: FakeFilter::Context(filter, driver, parent, resource_context),
filter_(filter) { }
virtual void DoRewriteSingle(const ResourcePtr input,
OutputResourcePtr output) {
CachedResult* cached = output->EnsureCachedResultCreated();
FakeFilter::Context::DoRewriteSingle(input, output);
cached->set_optimized_image_type(filter_->optimized_image_type());
}
private:
FakeImageFilter* filter_;
DISALLOW_COPY_AND_ASSIGN(Context);
};
explicit FakeImageFilter(RewriteDriver* rewrite_driver)
: FakeFilter(RewriteOptions::kImageCompressionId,
rewrite_driver, semantic_type::kImage),
optimized_image_type_(IMAGE_WEBP) { }
void set_optimized_image_type(ImageType type) {
optimized_image_type_ = type;
}
ImageType optimized_image_type() const {
return optimized_image_type_;
}
RewriteContext* MakeFakeContext(
RewriteDriver* driver, RewriteContext* parent,
ResourceContext* resource_context) {
return new FakeImageFilter::Context(this, driver, parent, resource_context);
}
private:
ImageType optimized_image_type_;
DISALLOW_COPY_AND_ASSIGN(FakeImageFilter);
};
class InPlaceRewriteContextTest : public RewriteTestBase {
protected:
static const bool kWriteToCache = true;
static const bool kNoWriteToCache = false;
static const bool kNoTransform = true;
static const bool kTransform = false;
InPlaceRewriteContextTest()
: img_filter_(NULL),
other_img_filter_(NULL),
js_filter_(NULL),
css_filter_(NULL),
cache_html_url_("http://www.example.com/cacheable.html"),
cache_jpg_url_("http://www.example.com/cacheable.jpg"),
cache_jpg_no_extension_url_("http://www.example.com/cacheable_jpg"),
cache_jpg_notransform_url_("http://www.example.com/notransform.jpg"),
cache_jpg_vary_star_url_("http://www.example.com/vary_star.jpg"),
cache_jpg_vary_ua_url_("http://www.example.com/vary_ua.jpg"),
cache_jpg_vary_origin_url_("http://www.example.com/vary_origin.jpg"),
cache_png_url_("http://www.example.com/cacheable.png"),
cache_gif_url_("http://www.example.com/cacheable.gif"),
cache_webp_url_("http://www.example.com/cacheable.webp"),
cache_js_url_("http://www.example.com/cacheable.js"),
cache_js_jpg_extension_url_("http://www.example.com/cacheable_js.jpg"),
cache_css_url_("http://www.example.com/cacheable.css"),
nocache_html_url_("http://www.example.com/nocacheable.html"),
nocache_js_url_("http://www.example.com/nocacheable.js"),
private_cache_js_url_("http://www.example.com/privatecacheable.js"),
cache_js_no_max_age_url_("http://www.example.com/cacheablemod.js"),
bad_url_("http://www.example.com/bad.url"),
redirect_url_("http://www.example.com/redir.url"),
rewritten_jpg_url_(
"http://www.example.com/cacheable.jpg.pagespeed.ic.0.jpg"),
json_js_type_url_("http://www.example.com/cacheable_js_type.json"),
json_json_type_url_("http://www.example.com/cacheable_json_type.json"),
json_json_type_synonym_url_(
"http://www.example.com/cacheable_json_synonym_type.json"),
cache_body_("good"), nocache_body_("bad"),
bad_body_("ugly"),
redirect_body_("Location: http://www.example.com/final.url"),
ttl_ms_(Timer::kHourMs), etag_("W/\"PSA-aj-0\""),
original_etag_("original_etag"),
exceed_deadline_(false), optimize_for_browser_(false),
oversized_stream_(NULL), in_place_uncacheable_rewrites_(NULL),
distributed_rewrite_failures_(NULL),
distributed_rewrite_successes_(NULL) {}
virtual void Init() {
SetTimeMs(start_time_ms());
mock_url_fetcher()->set_fail_on_unexpected(false);
const StringPiece kNoVary("");
// Set fetcher result and headers.
AddResponse(cache_html_url_, kContentTypeHtml, cache_body_, start_time_ms(),
ttl_ms_, original_etag_, kNoVary, kNoWriteToCache, kTransform);
AddResponse(cache_jpg_url_, kContentTypeJpeg, cache_body_, start_time_ms(),
ttl_ms_, "", kNoVary, kNoWriteToCache, kTransform);
AddResponse(cache_jpg_no_extension_url_, kContentTypeJpeg, cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
AddResponse(cache_jpg_notransform_url_, kContentTypeJpeg, cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kNoTransform);
AddResponse(cache_jpg_vary_star_url_, kContentTypeJpeg, cache_body_,
start_time_ms(), ttl_ms_, "", /* Vary: */ "*",
kNoWriteToCache, kTransform);
AddResponse(cache_jpg_vary_ua_url_, kContentTypeJpeg, cache_body_,
start_time_ms(), ttl_ms_, "", /* Vary: */ "User-Agent",
kNoWriteToCache, kTransform);
AddResponse(cache_jpg_vary_origin_url_, kContentTypeJpeg, cache_body_,
start_time_ms(), ttl_ms_, "", /* Vary: */ "Origin",
kNoWriteToCache, kTransform);
AddResponse(cache_png_url_, kContentTypePng, cache_body_, start_time_ms(),
ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
AddResponse(cache_gif_url_, kContentTypeGif, cache_body_, start_time_ms(),
ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
AddResponse(cache_webp_url_, kContentTypeWebp, cache_body_, start_time_ms(),
ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
AddResponse(cache_js_url_, kContentTypeJavascript, cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
AddResponse(cache_js_jpg_extension_url_, kContentTypeJavascript,
cache_body_, start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
AddResponse(cache_css_url_, kContentTypeCss, cache_body_, start_time_ms(),
ttl_ms_, "", kNoVary, kNoWriteToCache, kTransform);
AddResponse(nocache_html_url_, kContentTypeHtml, nocache_body_,
start_time_ms(), -1, "", kNoVary, kNoWriteToCache, kTransform);
AddResponse(nocache_js_url_, kContentTypeJavascript, cache_body_,
start_time_ms(), -1 /*ttl*/, "" /*etag*/, kNoVary,
kNoWriteToCache, kTransform);
AddResponse(cache_js_no_max_age_url_, kContentTypeJavascript, cache_body_,
start_time_ms(), 0, "", kNoVary, kNoWriteToCache, kTransform);
AddResponseStrContentType(
json_js_type_url_, "application/javascript", cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
AddResponseStrContentType(
json_json_type_url_, "application/json", cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
AddResponseStrContentType(
json_json_type_synonym_url_, "application/x-json", cache_body_,
start_time_ms(), ttl_ms_, "", kNoVary,
kNoWriteToCache, kTransform);
ResponseHeaders private_headers;
SetDefaultHeaders(kContentTypeJavascript.mime_type(), &private_headers);
private_headers.SetDateAndCaching(start_time_ms(), 1200 /*ttl*/,
",private");
mock_url_fetcher()->SetResponse(
private_cache_js_url_, private_headers, cache_body_);
ResponseHeaders bad_headers;
bad_headers.set_first_line(1, 1, 404, "Not Found");
bad_headers.SetDate(start_time_ms());
mock_url_fetcher()->SetResponse(bad_url_, bad_headers, bad_body_);
// Add a response for permanent redirect.
ResponseHeaders redirect_headers;
redirect_headers.set_first_line(1, 1, 301, "Moved Permanently");
redirect_headers.ComputeCaching();
redirect_headers.SetCacheControlMaxAge(36000);
redirect_headers.Add(HttpAttributes::kCacheControl, "public");
redirect_headers.Add(HttpAttributes::kContentType, "image/jpeg");
mock_url_fetcher()->SetResponse(
redirect_url_, redirect_headers, redirect_body_);
img_filter_ = new FakeImageFilter(rewrite_driver());
js_filter_ = new FakeFilter(RewriteOptions::kJavascriptMinId,
rewrite_driver(), semantic_type::kScript);
css_filter_ = new FakeFilter(RewriteOptions::kCssFilterId, rewrite_driver(),
semantic_type::kStylesheet);
rewrite_driver()->AppendRewriteFilter(img_filter_);
rewrite_driver()->AppendRewriteFilter(js_filter_);
rewrite_driver()->AppendRewriteFilter(css_filter_);
options()->ClearSignatureForTesting();
AddRecompressImageFilters();
options()->EnableFilter(RewriteOptions::kRewriteJavascriptExternal);
options()->EnableFilter(RewriteOptions::kRewriteJavascriptInline);
options()->EnableFilter(RewriteOptions::kRewriteCss);
if (optimize_for_browser()) {
options()->EnableFilter(RewriteOptions::kInPlaceOptimizeForBrowser);
options()->EnableFilter(RewriteOptions::kConvertJpegToWebp);
}
options()->set_in_place_rewriting_enabled(true);
// Only allow to vary on "Accept" header.
RewriteOptions::AllowVaryOn allow_vary_on;
EXPECT_TRUE(RewriteOptions::ParseFromString("accept", &allow_vary_on));
options()->set_allow_vary_on(allow_vary_on);
server_context()->ComputeSignature(options());
// Clear stats since we may have added something to the cache.
ClearStats();
oversized_stream_ = statistics()->GetVariable(
InPlaceRewriteContext::kInPlaceOversizedOptStream);
in_place_uncacheable_rewrites_ = statistics()->GetVariable(
InPlaceRewriteContext::kInPlaceUncacheableRewrites);
distributed_rewrite_failures_ = statistics()->GetVariable(
RewriteContext::kNumDistributedRewriteFailures);
distributed_rewrite_successes_ = statistics()->GetVariable(
RewriteContext::kNumDistributedRewriteSuccesses);
}
void AddResponseStrContentType(
const GoogleString& url, const GoogleString& content_type,
const GoogleString& body, int64 now_ms, int64 ttl_ms,
const GoogleString& etag, const StringPiece& vary,
bool write_to_cache, bool no_transform) {
ResponseHeaders response_headers;
SetDefaultHeaders(content_type, &response_headers);
if (ttl_ms > 0) {
response_headers.SetDateAndCaching(now_ms, ttl_ms);
} else {
response_headers.SetDate(now_ms);
if (ttl_ms < 0) {
response_headers.Replace(HttpAttributes::kCacheControl, "no-cache");
} else {
response_headers.Replace(HttpAttributes::kCacheControl, "public");
}
}
if (!vary.empty()) {
response_headers.Replace(HttpAttributes::kVary, vary);
}
if (no_transform) {
response_headers.Replace(HttpAttributes::kCacheControl, "no-transform");
}
if (!etag.empty()) {
response_headers.Add(HttpAttributes::kEtag, etag);
}
mock_url_fetcher()->SetResponse(url, response_headers, body);
if (write_to_cache) {
response_headers.ComputeCaching();
http_cache()->Put(
url, rewrite_driver_->CacheFragment(),
RequestHeaders::Properties(),
ResponseHeaders::GetVaryOption(options()->respect_vary()),
&response_headers, body, message_handler());
}
}
void AddResponse(const GoogleString& url, const ContentType& content_type,
const GoogleString& body, int64 now_ms, int64 ttl_ms,
const GoogleString& etag, const StringPiece& vary,
bool write_to_cache, bool no_transform) {
AddResponseStrContentType(url, content_type.mime_type(), body, now_ms,
ttl_ms, etag, vary, write_to_cache, no_transform);
}
void SetDefaultHeaders(const GoogleString& content_type,
ResponseHeaders* header) const {
header->set_major_version(1);
header->set_minor_version(1);
header->SetStatusAndReason(HttpStatus::kOK);
header->Replace(HttpAttributes::kContentType, content_type);
}
void ResetUserAgent(StringPiece user_agent) {
ClearRewriteDriver();
SetCurrentUserAgent(user_agent);
}
void SetAcceptWebp() {
AddRequestAttribute(HttpAttributes::kAccept, "image/webp");
}
void FetchAndCheckResponse(const GoogleString& url,
const GoogleString& expected_body,
bool expected_success,
int64 expected_ttl,
const char* etag,
int64 date_ms) {
js_filter_->set_exceed_deadline(exceed_deadline_);
img_filter_->set_exceed_deadline(exceed_deadline_);
if (other_img_filter_ != NULL) {
other_img_filter_->set_exceed_deadline(exceed_deadline_);
}
css_filter_->set_exceed_deadline(exceed_deadline_);
WorkerTestBase::SyncPoint sync(server_context()->thread_system());
RequestContextPtr request_context(RequestContext::NewTestRequestContext(
server_context()->thread_system()));
FakeFetch mock_fetch(request_context, options(), url, &sync,
&response_headers_);
const RequestHeaders* driver_request_headers =
rewrite_driver()->request_headers();
if (driver_request_headers != NULL) {
mock_fetch.request_headers()->CopyFrom(*driver_request_headers);
}
rewrite_driver()->FetchResource(url, &mock_fetch);
// If we're testing if the rewrite takes too long, we need to push
// time forward here.
if (exceed_deadline()) {
rewrite_driver()->BoundedWaitFor(
RewriteDriver::kWaitForCompletion,
rewrite_driver()->rewrite_deadline_ms());
}
sync.Wait();
rewrite_driver()->WaitForShutDown();
mock_scheduler()->AwaitQuiescence(); // needed for cache puts to finish.
EXPECT_TRUE(mock_fetch.done());
EXPECT_EQ(expected_success, mock_fetch.success()) << url;
EXPECT_EQ(expected_body, mock_fetch.content()) << url;
EXPECT_EQ(expected_ttl, response_headers_.cache_ttl_ms()) << url;
EXPECT_STREQ(etag, response_headers_.Lookup1(HttpAttributes::kEtag)) << url;
EXPECT_EQ(date_ms, response_headers_.date_ms()) << url;
}
void ResetHeadersAndStats() {
response_headers_.Clear();
img_filter_->ClearStats();
if (other_img_filter_ != NULL) {
other_img_filter_->ClearStats();
}
js_filter_->ClearStats();
css_filter_->ClearStats();
RewriteTestBase::ClearStats();
ClearRewriteDriver();
}
void CheckWarmCache(StringPiece id) {
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count()) << id;
EXPECT_EQ(1, http_cache()->cache_hits()->Get()) << id;
EXPECT_EQ(0, http_cache()->cache_misses()->Get()) << id;
EXPECT_EQ(0, http_cache()->cache_inserts()->Get()) << id;
EXPECT_EQ(2, lru_cache()->num_hits()) << id;
EXPECT_EQ(0, lru_cache()->num_misses()) << id;
EXPECT_EQ(0, lru_cache()->num_inserts()) << id;
EXPECT_EQ(0, img_filter_->num_rewrites()) << id;
EXPECT_EQ(0, js_filter_->num_rewrites()) << id;
EXPECT_EQ(0, css_filter_->num_rewrites()) << id;
EXPECT_EQ(0, oversized_stream_->Get()) << id;
}
void SetupDistributedTest(const StringPiece& distributed_filter) {
SetupSharedCache();
other_img_filter_ = new FakeImageFilter(other_rewrite_driver());
other_rewrite_driver()->AppendRewriteFilter(other_img_filter_);
options()->ClearSignatureForTesting();
other_options()->ClearSignatureForTesting();
AddRecompressImageFilters();
options()->EnableFilter(RewriteOptions::kRewriteJavascriptExternal);
options()->EnableFilter(RewriteOptions::kRewriteCss);
options_->DistributeFilter(distributed_filter);
options_->set_distributed_rewrite_servers("example.com:80");
options_->set_distributed_rewrite_key("1234123");
Init();
other_options()->Merge(*options());
other_server_context()->ComputeSignature(other_options());
}
void CheckDistributedFetch(int distributed_fetch_success_count,
int distributed_fetch_failure_count,
int local_fetch_required, int rewritten) {
EXPECT_EQ(distributed_fetch_success_count + distributed_fetch_failure_count,
counting_distributed_fetcher()->fetch_count());
EXPECT_EQ(local_fetch_required,
counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(
0, other_factory_->counting_distributed_async_fetcher()->fetch_count());
EXPECT_EQ(distributed_fetch_success_count,
distributed_rewrite_successes_->Get());
EXPECT_EQ(distributed_fetch_failure_count,
distributed_rewrite_failures_->Get());
EXPECT_EQ(rewritten,
img_filter_->num_rewrites() + other_img_filter_->num_rewrites());
}
void ExpectInPlaceImageSuccessFlow(const GoogleString& url) {
FetchAndCheckResponse(url, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms());
// First fetch misses initial metadata cache lookup, finds original in
// cache; the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
FetchAndCheckResponse(url, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is
// served out.
CheckWarmCache("second_fetch_1");
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
FetchAndCheckResponse(url, cache_body_, true, ttl_ms_,
original_etag_, timer()->NowMs());
// The metadata and cache entry is stale now. Fetch the content and serve
// out the original. The background rewrite work then revalidates
// the response and updates metadata.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits()); // (expired) orig., aj, ic metadata
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
void CheckCachingAndContentType(const GoogleString& url,
const GoogleString& expected_mime_type,
const GoogleString& cache_body,
const GoogleString& filter_prefix) {
FetchAndCheckResponse(url, cache_body, true, ttl_ms_,
NULL, start_time_ms());
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
// Make sure the content type is unmodified.
EXPECT_STREQ(expected_mime_type,
response_headers_.Lookup1(HttpAttributes::kContentType));
// Try a second fetch and ensure we get a cache hit.
ResetHeadersAndStats();
FetchAndCheckResponse(url, StrCat(cache_body, ":", filter_prefix), true,
ttl_ms_, etag_, start_time_ms());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_STREQ(expected_mime_type,
response_headers_.Lookup1(HttpAttributes::kContentType));
}
bool exceed_deadline() { return exceed_deadline_; }
void set_exceed_deadline(bool x) { exceed_deadline_ = x; }
bool optimize_for_browser() { return optimize_for_browser_; }
void set_optimize_for_browser(bool x) { optimize_for_browser_ = x; }
FakeImageFilter* img_filter_;
FakeImageFilter* other_img_filter_;
FakeFilter* js_filter_;
FakeFilter* css_filter_;
ResponseHeaders response_headers_;
const GoogleString cache_html_url_;
const GoogleString cache_jpg_url_;
const GoogleString cache_jpg_no_extension_url_;
const GoogleString cache_jpg_notransform_url_;
const GoogleString cache_jpg_vary_star_url_;
const GoogleString cache_jpg_vary_ua_url_;
const GoogleString cache_jpg_vary_origin_url_;
const GoogleString cache_png_url_;
const GoogleString cache_gif_url_;
const GoogleString cache_webp_url_;
const GoogleString cache_js_url_;
const GoogleString cache_js_jpg_extension_url_;
const GoogleString cache_css_url_;
const GoogleString nocache_html_url_;
const GoogleString nocache_js_url_;
const GoogleString private_cache_js_url_;
const GoogleString cache_js_no_max_age_url_;
const GoogleString bad_url_;
const GoogleString redirect_url_;
const GoogleString rewritten_jpg_url_;
const GoogleString json_js_type_url_;
const GoogleString json_json_type_url_;
const GoogleString json_json_type_synonym_url_;
const GoogleString cache_body_;
const GoogleString nocache_body_;
const GoogleString bad_body_;
const GoogleString redirect_body_;
const int ttl_ms_;
const char* etag_;
const char* original_etag_;
bool exceed_deadline_;
bool optimize_for_browser_;
Variable* oversized_stream_;
Variable* in_place_uncacheable_rewrites_;
Variable* distributed_rewrite_failures_;
Variable* distributed_rewrite_successes_;
};
TEST_F(InPlaceRewriteContextTest, IngressDistributedRewrite) {
// Distribute an image rewrite (the response of the rewrite task is mocked).
SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
start_time_ms());
CheckDistributedFetch(1, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
1); // number of rewrites
// Ingress task distributes and streams back the result.
// Rewrite task misses metadata, misses http, fetches, writes original,
// misses ic metadata, writes optimized, and writes metadata for ipro and ic.
EXPECT_EQ(1, counting_distributed_fetcher()->fetch_count());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
// Ingress task distributes, distributed task hits and returns.
EXPECT_EQ(1, counting_distributed_fetcher()->fetch_count());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteImage) {
// Distribute the nested image task instead of the in_place_rewrite filter.
SetupDistributedTest(RewriteOptions::kImageCompressionId);
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
start_time_ms());
CheckDistributedFetch(1, // successful distributed fetches
0, // unsuccessful distributed fetches
1, // number of ingress fetches
1); // number of rewrites
// Ingress task: misses IPRO metadata, fetches the resource (miss and insert)
// misses ic metadata, and distributes, eventually writing IPRO metadata.
// Rewrite task: misses ic metadata, hits http, writes optimized, and ic.
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(4, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_ / 2);
// Ingress task hits ipro metadata and associated http resource.
EXPECT_EQ(0, counting_distributed_fetcher()->fetch_count());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, IngressDistributedNestedWaitForOptimized) {
// Like IngressDistributedNested but this time we want to wait for the
// optimized result, which causes a distributed GET request and the returned
// content is fed into the nested context's output_ for IPRO's Harvest.
options_->set_in_place_wait_for_optimized(true);
SetupDistributedTest(RewriteOptions::kImageCompressionId);
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
// Ingress task fetches in IPRO and then distributes the nested image rewrite.
CheckDistributedFetch(1, // successful distributed fetches
0, // unsuccessful distributed fetches
1, // number of ingress fetches
1); // number of rewrites
// Ingress task: IPRO misses metadata and http original resource then fetches
// and inserts http original resource. Ingress then starts nested image
// rewriter which skips metadata check (force_rewrite) and distributes. IPRO
// writes its metadata.
// Distributed task: Misses image metadata, hits the original resource, and
// stores the new metadata and optimized resource.
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_ / 2);
CheckDistributedFetch(0, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
// Ingress task hits ipro metadata and associated http resource.
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest,
IngressDistributedNestedWaitForOptimizedFail) {
// Wait for an optimized result but the fetcher breaks after the headers are
// written. Fall back to the original resource.
options_->set_in_place_wait_for_optimized(true);
SetupDistributedTest(RewriteOptions::kImageCompressionId);
test_distributed_fetcher()->set_fail_after_headers(true);
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
start_time_ms());
// Ingress task fetches in IPRO and then distributes the nested image rewrite.
CheckDistributedFetch(0, // successful distributed fetches
1, // unsuccessful distributed fetches
1, // number of ingress fetches
1); // number of rewrites
// Ingress task: IPRO misses metadata and http original resource,
// fetches and inserts http original resource. Then starts nested image
// rewriter which skips the metadata lookup (force_rewrite) and distributes.
// Distributed task: Misses image metadata, hits the original resource, and
// stores the new metadata and optimized resource.
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
start_time_ms());
CheckDistributedFetch(0, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
// Ingress task hits ipro metadata and associated http resource.
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteNotFound) {
// If the distributed fetcher returns a 404 then that's what should be
// returned.
SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);
GoogleString orig_url = StrCat(kTestDomain, "fourofour.png");
SetFetchResponse404(orig_url);
FetchAndCheckResponse(orig_url, "", true, ServerContext::kGeneratedMaxAgeMs,
ServerContext::kResourceEtagValue, start_time_ms());
EXPECT_EQ(HttpStatus::kNotFound, response_headers_.status_code());
// The distributed fetcher should have run once on the ingress task and the
// url fetcher should have run once on the rewrite task. The result goes to
// shared cache.
CheckDistributedFetch(1, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
// Ingress task distributes and returns the 404 it gets back.
// Rewrite task misses metadata and http. The 404 isn't cacheable, so it's
// not written.
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
// Ingress task distributes.
// Rewrite task misses ipro metadata + http, and returns that.
ResetHeadersAndStats();
FetchAndCheckResponse(orig_url, "", true, ServerContext::kGeneratedMaxAgeMs,
ServerContext::kResourceEtagValue, start_time_ms());
CheckDistributedFetch(1, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteFailFallback) {
// If the distributed fetch fails mid-stream then we fail.
SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);
test_distributed_fetcher()->set_fail_after_headers(true);
FetchAndCheckResponse(cache_jpg_url_, "", false, ttl_ms_, NULL,
start_time_ms());
// Note that we didn't need to fetch the original resource at the ingress task
// because the distributed task already fetched it and put it in shared cache.
CheckDistributedFetch(0, // successful distributed fetches
1, // unsuccessful distributed fetches
0, // number of ingress fetches
1); // number of rewrites
// Ingress task: Distributes and streams back a failed stream.
// Rewrite task: Misses ipro metadata, misses http, fetches and inserts http,
// misses image compression metadata, writes optimized http and 2 metadata,
// returns.
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
// Fetch again but we'll hit the cache this time.
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_ / 2));
FetchAndCheckResponse(cache_jpg_url_, "", false, ttl_ms_ / 2, etag_,
start_time_ms() + ttl_ms_ / 2);
CheckDistributedFetch(0, // successful distributed fetches
1, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteFailFallbackImage) {
// If the distributed fetch fails mid-stream then the unoptimized resource
// should be returned. This time image compression is distributed but not IPRO
// itself.
SetupDistributedTest(RewriteOptions::kImageCompressionId);
// Simulate distributed fetch failure and ensure that we fall back to the
// original.
test_distributed_fetcher()->set_fail_after_headers(true);
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
start_time_ms());
// Note that we didn't need to fetch the original resource at the ingress task
// because the distributed task already fetched it and put it in shared cache.
CheckDistributedFetch(0, // successful distributed fetches
1, // unsuccessful distributed fetches
1, // number of ingress fetches
1); // number of rewrites
// Ingress task: Misses ipro metadata, fetches and inserts input resource,
// misses ic metadata, distributes. Upon distributed failure, aborts nested
// rewrite and IPRO records the failure.
// Rewrite task: Misses ic metadata, hits input resources, writes optimized
// resource and ic metadata, then returns.
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(4, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
// On the first attempt IPRO failed to rewrite the image due to its nested
// task's distribution failure. On the second attempt, IPRO remembers that
// the first attempt failed and returns the original resource.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
start_time_ms());
CheckDistributedFetch(0, // successful distributed fetches
0, // unsuccessful distributed fetches
0, // number of ingress fetches
0); // number of rewrites
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}
TEST_F(InPlaceRewriteContextTest, CacheableHtmlUrlNoRewriting) {
// All these entries find no in-place rewrite metadata and no rewriting
// happens.
Init();
FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses()); // metadata + html
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms());
// Second fetch hits initial cache lookup and no extra fetches are needed.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(1, lru_cache()->num_misses()); // metadata
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms() + 2 * ttl_ms_);
// Cache entry is stale, so we must fetch again.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, lru_cache()->num_hits()); // HTML is in LRU cache, just expired.
EXPECT_EQ(1, lru_cache()->num_misses());
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, WaitForOptimizedFirstRequest) {
// By setting this flag we should get an optimized response on the first
// request unless we hit a rewrite timeout but in this test it will complete
// in time.
options()->set_in_place_wait_for_optimized(true);
Init();
// The optimized content from the fake rewriter has ":ic" appended to original
// content.
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache. The optimized version should be
// returned.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(0, oversized_stream_->Get());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is
// served out.
CheckWarmCache("second_fetch_2");
}
TEST_F(InPlaceRewriteContextTest, WaitForOptimizeWithDisabledFilter) {
// Wait for optimized but if the resource fails to optimize we should get
// back the original resource.
options()->set_in_place_wait_for_optimized(true);
// We'll also test that the hash values we get are legitimate and not
// hard-coded 0s
UseMd5Hasher();
Init();
// Turn off optimization. The filter will still run but return false in
// rewrite.
img_filter_->set_enabled(false);
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Failure to rewrite means original should be returned.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); // original only
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(0, oversized_stream_->Get());
ResetHeadersAndStats();
// The second time we get the cached original, which should have an md5'd
// etag.
// TODO(jkarlin): Note that if we advance time here, we'd expect the TTL of
// the cached resource to decrease on the second fetch, but that doesn't
// happen. That should be fixed.
GoogleString expected_etag = StrCat("W/\"PSA-", hasher()->Hash(cache_body_),
"\"");
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
expected_etag.c_str(), start_time_ms());
// Second fetch hits the metadata cache, sees that the rewrite failed and
// fetches and serves the original resource from cache.
CheckWarmCache("second_fetch_3");
}
TEST_F(InPlaceRewriteContextTest, WaitForOptimizeNoTransform) {
// Confirm that when cache-control:no-transform is present in the response
// headers that the in-place optimizer does not optimize the resource.
options()->set_in_place_wait_for_optimized(true);
Init();
// Don't rewrite since it's no-transform.
FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
ttl_ms_, NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(2, lru_cache()->num_inserts()); // original + ipro metadata
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_TRUE(response_headers_.HasValue(HttpAttributes::kCacheControl,
"no-transform"));
ResetHeadersAndStats();
// Don't rewrite since it's no-transform.
FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
ttl_ms_, kEtag0.c_str(), start_time_ms());
// The second fetch should return the cached original after seeing that it
// can't be rewritten.
CheckWarmCache("second_fetch_4");
}
TEST_F(InPlaceRewriteContextTest, OptimizeOnNoTransformIfOptionFalse) {
options()->set_disable_rewrite_on_no_transform(false);
Init();
FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
ttl_ms_, NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// into cache. Also the resource gets rewritten and the rewritten resource
// gets inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_notransform_url_, "good:ic", true,
ttl_ms_/2, etag_, start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
CheckWarmCache("second_fetch_notransform");
}
TEST_F(InPlaceRewriteContextTest, WaitForOptimizeTimeout) {
// Confirm that rewrite deadlines cause the original resource to be returned
// (but caches the optimized) even if in_place_wait_for_optimize is on.
options()->set_in_place_wait_for_optimized(true);
Init();
// Tells the optimizing filter to slow down.
exceed_deadline_ = true;
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Rewrite succeeds but is slow so original returned.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(0, oversized_stream_->Get());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
CheckWarmCache("second_fetch_5");
}
TEST_F(InPlaceRewriteContextTest, WaitForOptimizeResourceTooBig) {
// Wait for optimized but if it's larger than the RecordingFetch can handle
// make sure we piece together the original resource properly.
options()->set_in_place_wait_for_optimized(true);
Init();
// To make this more interesting there should be something in the cache to
// recover when we fail. Let's split the url_fetch from 'good' into 'go' and
// 'od' writes.
mock_url_fetcher()->set_split_writes(true);
// By setting cache max to 2, the second write ('od') will cause an overflow.
// Test that we recover.
http_cache()->set_max_cacheable_response_content_length(2);
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch but resource too
// big for cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(1, oversized_stream_->Get());
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
NULL, start_time_ms());
// Second fetch should also completely miss because the first fetch was too
// big to stuff in the cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(1, oversized_stream_->Get());
}
TEST_F(InPlaceRewriteContextTest, CacheableJpgUrlRewritingSucceeds) {
Init();
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_, NULL,
start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
CheckWarmCache("second_fetch_6");
ResetHeadersAndStats();
// We get a 304 if we send a request with an If-None-Match matching the hash
// of the rewritten resource.
AddRequestAttribute(HttpAttributes::kIfNoneMatch, etag_);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "", true, ttl_ms_/2, NULL, 0);
EXPECT_EQ(HttpStatus::kNotModified, response_headers_.status_code());
// We hit the metadata cache and find that the etag matches the hash of the
// rewritten resource.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
// The etag doesn't match and hence we serve the full response.
AddRequestAttribute(HttpAttributes::kIfNoneMatch, "no-match");
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
EXPECT_EQ(HttpStatus::kOK, response_headers_.status_code());
// We hit the metadata cache, but the etag doesn't match so we fetch the
// rewritten resource from the HTTPCache and serve it out.
CheckWarmCache("etag_mismatch");
// Delete the rewritten resource from cache to check if reconstruction works.
lru_cache()->Delete(HttpCacheKey(rewritten_jpg_url_));
ResetHeadersAndStats();
// Original resource is served with the date set to start time.
// The ETag we check for here is the ETag HTTPCache synthesized for
// the original resource.
FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
start_time_ms());
// We find the metadata in cache, but don't find the rewritten resource.
// Hence, we reconstruct the resource and insert it into cache. We see 2
// identical reinserts - one for the image rewrite filter metadata and one for
// the in-place metadata.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(1, lru_cache()->num_misses());
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(2, lru_cache()->num_identical_reinserts());
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
// For only the next request, update the date header so that freshening
// succeeds.
FetcherUpdateDateHeaders();
ResetHeadersAndStats();
int64 time_ms = start_time_ms() + ttl_ms_ - 2 * Timer::kMinuteMs;
SetTimeMs(time_ms);
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, 2 * Timer::kMinuteMs,
etag_, time_ms);
// This fetch hits the metadata cache and the rewritten resource is served
// out. Freshening is triggered here and we insert the freshened response and
// metadata into the cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(2, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
mock_url_fetcher()->set_update_date_headers(false);
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_ * 5/4));
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true,
ttl_ms_ * 3/4 - 2 * Timer::kMinuteMs, etag_,
start_time_ms() + ttl_ms_ * 5/4);
// Since the previous request freshened the metadata, this fetch hits the
// metadata cache and the rewritten resource is served out. Note that no
// freshening needs to be triggered here.
CheckWarmCache("freshened_metadata");
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_, NULL,
timer()->NowMs());
// The metadata and cache entry is stale now. Fetch the content and serve out
// the original. We will however notice that the contents did not
// actually change and update the metadata cache promptly, without
// rewriting.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingSucceeds) {
Init();
ExpectInPlaceImageSuccessFlow(cache_png_url_);
}
TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingSucceedsWithShards) {
Init();
const char kShard1[] = "http://s1.example.com/";
const char kShard2[] = "http://s2.example.com/";
AddShard("http://www.example.com", StrCat(kShard1, ",", kShard2));
ExpectInPlaceImageSuccessFlow(cache_png_url_);
}
TEST_F(InPlaceRewriteContextTest, CacheableiGifUrlRewritingSucceeds) {
Init();
ExpectInPlaceImageSuccessFlow(cache_gif_url_);
}
TEST_F(InPlaceRewriteContextTest, CacheableWebpUrlRewritingSucceeds) {
Init();
ExpectInPlaceImageSuccessFlow(cache_webp_url_);
}
TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingFails) {
// Setup the image filter to fail at rewriting.
Init();
img_filter_->set_enabled(false);
FetchAndCheckResponse(cache_png_url_, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms());
// First fetch misses initial metadata lookup, finds original in cache.
// The rewrite fails and metadata is inserted into the
// cache indicating that the rewriting didn't succeed.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(1, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(2, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
FetchAndCheckResponse(cache_png_url_, cache_body_, true, ttl_ms_,
original_etag_, start_time_ms());
// Second fetch hits the metadata cache, sees that the rewrite failed and
// fetches and serves the original resource from cache.
CheckWarmCache("second_fetch_7");
}
TEST_F(InPlaceRewriteContextTest, CacheableJsUrlRewritingSucceeds) {
Init();
FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
CheckWarmCache("second_fetch_8");
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
timer()->NowMs());
// The metadata and cache entry is stale now. Fetch the content and serve it
// out without rewriting. The background rewrite will then revalidate
// a previous rewrite's result and reuse it.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, CacheableJsUrlRewritingWithStaleServing) {
Init();
options()->ClearSignatureForTesting();
options()->set_metadata_cache_staleness_threshold_ms(ttl_ms_);
server_context()->ComputeSignature(options());
FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
CheckWarmCache("second_fetch_9");
SetTimeMs(start_time_ms() + (3 * ttl_ms_) / 2);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "good:jm", true,
RewriteOptions::kDefaultImplicitCacheTtlMs, etag_,
start_time_ms() + (3 * ttl_ms_) / 2);
// The metadata and cache entry is stale now. We serve the rewritten resource
// here, but trigger a fetch and rewrite to update the metadata.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(2, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, CacheableJsUrlModifiedImplicitCacheTtl) {
Init();
response_headers_.set_implicit_cache_ttl_ms(500 * Timer::kSecondMs);
FetchAndCheckResponse(cache_js_no_max_age_url_, cache_body_,
/* expected_success= */ true,
/* expected_ttl= */ 500 * Timer::kSecondMs,
/* etag= */ NULL,
/* date_ms= */ start_time_ms());
}
TEST_F(InPlaceRewriteContextTest, CacheableCssUrlIfCssRewritingDisabled) {
Init();
options()->ClearSignatureForTesting();
options()->DisableFilter(RewriteOptions::kRewriteCss);
server_context()->ComputeSignature(options());
FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
start_time_ms());
// First fetch succeeds at the fetcher, no rewriting happens since the css
// filter is disabled, and metadata indicating a rewriting failure gets
// inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(2, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
ResetHeadersAndStats();
// The ETag we check for here is the ETag HTTPCache synthesized for
// the original resource.
FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_,
kEtag0.c_str(), start_time_ms());
// Second fetch hits the metadata cache, finds that the result is not
// optimizable. It then looks up cache for the original and finds it.
CheckWarmCache("second_fetch_10");
}
TEST_F(InPlaceRewriteContextTest, CacheableCssUrlRewritingSucceeds) {
Init();
EnableCachePurge();
FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(3, lru_cache()->num_misses());
EXPECT_EQ(4, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(1, css_filter_->num_rewrites());
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
FetchAndCheckResponse(cache_css_url_, "good:cf", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
int64 date_of_css_ms = timer()->NowMs();
FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
date_of_css_ms);
// The metadata and cache entry is stale now. Fetch the content and serve it
// out without rewriting. The background rewrite attempt will end up reusing
// the old result due to revalidation, however.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(3, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
mock_url_fetcher()->set_timer(timer());
mock_url_fetcher()->set_update_date_headers(true);
SetCacheInvalidationTimestamp();
date_of_css_ms = timer()->NowMs();
// Having flushed cache, we are now back to serving the origin content.
FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_,
NULL, date_of_css_ms);
// Next time we'll serve optimized content.
AdvanceTimeMs(ttl_ms_/2);
ResetHeadersAndStats();
int64 expected_ttl_ms = ttl_ms_ - (timer()->NowMs() - date_of_css_ms);
FetchAndCheckResponse(cache_css_url_, "good:cf", true, expected_ttl_ms,
etag_, timer()->NowMs());
}
TEST_F(InPlaceRewriteContextTest, NonCacheableUrlNoRewriting) {
Init();
FetchAndCheckResponse(nocache_html_url_, nocache_body_, true, 0, NULL,
timer()->NowMs());
// First fetch misses initial cache lookup, succeeds at fetch and we don't
// insert into cache because it's not cacheable. Don't attempt to rewrite
// this since its not cacheable.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
// Tests that with correct flags set, the uncacheable resource will be
// rewritten. Also checks, that resource will not be inserted.
TEST_F(InPlaceRewriteContextTest, NonCacheableUrlRewriting) {
Init();
// Modify options for our test.
options()->ClearSignatureForTesting();
options()->set_in_place_wait_for_optimized(true);
options()->set_rewrite_uncacheable_resources(true);
server_context()->ComputeSignature(options());
// The ttl is just a value in proto, actual cacheable values will be checked
// below.
FetchAndCheckResponse(nocache_js_url_, StrCat(cache_body_, ":", "jm"),
true /* success */,
Timer::kYearMs /* ttl (ms) */,
etag_ /* etag */,
timer()->NowMs());
// Shouldn't be cacheable at all.
EXPECT_FALSE(response_headers_.IsBrowserCacheable());
EXPECT_FALSE(response_headers_.IsProxyCacheable());
// First fetch misses initial cache lookup, succeeds at fetch and we don't
// insert into cache because it's not cacheable. But since flags are set to
// rewrite uncacheable resources, JS rewriting should occur.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
// Should have been rewritten.
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(1, in_place_uncacheable_rewrites_->Get());
}
// Tests, that with correct flags set the private cacheable resource will be
// rewritten. Also checks, that the resource will not be cached.
TEST_F(InPlaceRewriteContextTest, PrivateCacheableUrlRewriting) {
Init();
// Modify options for our test.
options()->ClearSignatureForTesting();
options()->set_in_place_wait_for_optimized(true);
options()->set_rewrite_uncacheable_resources(true);
server_context()->ComputeSignature(options());
// The ttl is just a value in proto, actual cacheable values will be checked
// below.
FetchAndCheckResponse(private_cache_js_url_, StrCat(cache_body_, ":", "jm"),
true /* success */,
1000 /* ttl (s) */,
etag_ /* etag */,
timer()->NowMs());
// Should be cacheable.
EXPECT_TRUE(response_headers_.IsBrowserCacheable());
// But only in a private way.
EXPECT_FALSE(response_headers_.IsProxyCacheable());
// First fetch misses initial cache lookup, succeeds at fetch and we don't
// insert into cache because it's not cacheable. But since flags are set to
// rewrite uncacheable resources, JS rewriting should occur.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
// Should have been rewritten.
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(1, in_place_uncacheable_rewrites_->Get());
}
TEST_F(InPlaceRewriteContextTest, BadUrlNoRewriting) {
Init();
FetchAndCheckResponse(bad_url_, bad_body_, true, 0, NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and we don't
// insert into cache because it's not cacheable. Don't attempt to rewrite
// this since its not cacheable.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, PermanentRedirectNoRewriting) {
options()->set_in_place_wait_for_optimized(true);
Init();
FetchAndCheckResponse(
redirect_url_, redirect_body_, true /* expected_success */,
36000 /* ttl (s) */, NULL /* etag */, start_time_ms());
// Don't attempt to rewrite this since it's not a 200 response.
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(1, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, FetchFailedNoRewriting) {
Init();
FetchAndCheckResponse("http://www.notincache.com", "", false, 0, NULL,
start_time_ms());
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
}
TEST_F(InPlaceRewriteContextTest, HandleResourceCreationFailure) {
// Regression test. Trying to in-place optimize https resources with
// a fetcher that didn't support https would fail to invoke the callbacks
// and leak the rewrite driver.
Init();
factory()->mock_url_async_fetcher()->set_fetcher_supports_https(false);
FetchAndCheckResponse("https://www.example.com", "", false, 0, NULL, 0);
}
TEST_F(InPlaceRewriteContextTest, ResponseHeaderMimeTypeUpdate) {
options()->set_in_place_wait_for_optimized(true);
Init();
// We are going to rewrite a PNG image below. Assume it will be converted
// to a JPEG.
img_filter_->set_output_content_type(&kContentTypeJpeg);
FetchAndCheckResponse(cache_png_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_STREQ(kContentTypeJpeg.mime_type(),
response_headers_.Lookup1(HttpAttributes::kContentType));
}
TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserEncodingAndHeader) {
options()->set_in_place_wait_for_optimized(true);
set_optimize_for_browser(true);
Init();
// Image with correct extension in URL.
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_EQ(0, css_filter_->num_encode_user_agent());
EXPECT_EQ(1, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// Image with no extension in URL.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_jpg_no_extension_url_, "good:ic", true, ttl_ms_,
etag_, start_time_ms());
EXPECT_EQ(1, css_filter_->num_encode_user_agent());
EXPECT_EQ(1, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// CSS with correct extension in URL.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_css_url_, "good:cf", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_EQ(1, css_filter_->num_encode_user_agent());
EXPECT_EQ(0, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_STREQ(HttpAttributes::kUserAgent,
response_headers_.Lookup1(HttpAttributes::kVary));
// HTML with correct extension in URL.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_html_url_, "good", true, ttl_ms_,
original_etag_, start_time_ms());
EXPECT_EQ(0, css_filter_->num_encode_user_agent());
EXPECT_EQ(0, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
// Javascript with correct extension in URL.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_EQ(0, css_filter_->num_encode_user_agent());
EXPECT_EQ(0, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
// Javascript with jpeg extension in URL.
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_jpg_extension_url_, "good:jm", true, ttl_ms_,
etag_, start_time_ms());
EXPECT_EQ(0, css_filter_->num_encode_user_agent());
EXPECT_EQ(1, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
// Bad content with unknown extension.
ResetHeadersAndStats();
FetchAndCheckResponse(bad_url_, bad_body_, true, 0, NULL, start_time_ms());
EXPECT_EQ(1, css_filter_->num_encode_user_agent());
EXPECT_EQ(1, img_filter_->num_encode_user_agent());
EXPECT_EQ(0, js_filter_->num_encode_user_agent());
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
}
TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserRewriting) {
// When in_place_wait_for_optimized is true, force_rewrite is set to true and
// the nested RewriteContext will not check for rewritten content if input
// is ready. Keep that in mind when checking lru_cache hits/misses.
options()->set_in_place_wait_for_optimized(true);
options()->set_private_not_vary_for_ie(true);
set_optimize_for_browser(true);
Init();
// First fetch with kTestUserAgentWebP. This will miss everything (metadata
// lookup, original content, and rewritten content).
// Vary: Accept header should be added.
ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
SetAcceptWebp();
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(1, http_cache()->cache_misses()->Get()); // original
EXPECT_EQ(2, http_cache()->cache_inserts()->Get()); // rewritten + original
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses()); // + ipro-md
EXPECT_EQ(4, lru_cache()->num_inserts()); // + ipro-md + md
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(0, oversized_stream_->Get());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// The second fetch uses a different user agent, kTestUserAgentNoWebP.
// This will miss the metadata cache so it will start fetch input (cache hit)
// and rewrite content (cache miss).
// Vary: Accept header should be be added.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get()); // original
EXPECT_EQ(0, http_cache()->cache_misses()->Get()); // rewritten
EXPECT_EQ(1, http_cache()->cache_inserts()->Get()); // rewritten
EXPECT_EQ(1, lru_cache()->num_hits()); // original
EXPECT_EQ(1, lru_cache()->num_misses()); // ipro-md
EXPECT_EQ(3, lru_cache()->num_inserts()); // + ipro-md + md
EXPECT_EQ(1, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
EXPECT_EQ(0, oversized_stream_->Get());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// The third fetch uses an IE 9 user agent string, which should result in a
// Cache-Control: private resource and no Vary header.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
ResetUserAgent(UserAgentMatcherTestBase::kIe9UserAgent);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
CheckWarmCache("no_webp_to_ie");
EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));
ConstStringStarVector cache_controls;
EXPECT_TRUE(response_headers_.Lookup(
HttpAttributes::kCacheControl, &cache_controls));
ASSERT_EQ(2, cache_controls.size());
EXPECT_STREQ(HttpAttributes::kPrivate, *cache_controls[1]);
// Fetch again still with kTestUserAgentWebP, but omits the Accept:webp
// header. Metadata cache hits. No input fetch and rewriting.
// Vary: Accept header should be be added.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
CheckWarmCache("no_webp_without_accept");
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// Fetch another time, switching to just sending Accept: webp and using
// kTestUserAgentNoWebP. Metadata cache hits. No input fetch and rewriting.
// Vary: User-Agent header should be added.
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
SetAcceptWebp();
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
CheckWarmCache("back_to_webp");
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
}
TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserNoPrivateForIE) {
// Similar to test above, but set private_not_vary_for_ie to false and omit
// detailed checking of cache hit statistics, focusing just on a behavioral
// test.
options()->set_in_place_wait_for_optimized(true);
options()->set_private_not_vary_for_ie(false);
set_optimize_for_browser(true);
Init();
// First fetch with kTestUserAgentWebP.
// Vary: Accept header should be added.
ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
SetAcceptWebp();
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// The second fetch uses a different user agent, kTestUserAgentNoWebP.
// Vary: Accept header should be be added.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// The third fetch uses an IE 9 user agent string, which should *also* have a
// Vary: Accept header since private_not_vary_for_ie == false.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
ResetUserAgent(UserAgentMatcherTestBase::kIe9UserAgent);
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
}
TEST_F(InPlaceRewriteContextTest, AcceptHeaderMerging) {
options()->set_in_place_wait_for_optimized(true);
set_optimize_for_browser(true);
Init();
SetAcceptWebp();
SetDriverRequestHeaders();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_STREQ(HttpAttributes::kAccept,
response_headers_.Lookup1(HttpAttributes::kVary));
// We don't actually optimize the Vary: * resource. See
// CachingHeaders::HasExplicitNoCacheDirective(). Inexplicably (?), we also
// change its ttl to 0 in spite of incoming ttl headers.
FetchAndCheckResponse(cache_jpg_vary_star_url_, "good", true, 0,
NULL, start_time_ms());
EXPECT_STREQ("*", response_headers_.Lookup1(HttpAttributes::kVary));
// TODO(jmaessen): Right now we're not properly passing through Vary: headers
// from the fetched resource. When jmarantz's pending change lands, we will
// do so, and these tests should be re-enabled accordingly. Note that I've
// verified in gdb that we're actually handling pre-existing headers properly
// (due to a duplicate call; luckily we're idempotent!).
// FetchAndCheckResponse(cache_jpg_vary_ua_url_, "good:ic", true, ttl_ms_,
// etag_, start_time_ms());
// EXPECT_STREQ(HttpAttributes::kUserAgent,
// response_headers_.Lookup1(HttpAttributes::kVary));
// FetchAndCheckResponse(cache_jpg_vary_origin_url_, "good:ic", true, ttl_ms_,
// etag_, start_time_ms());
// ConstStringStarVector accepts;
// EXPECT_TRUE(response_headers_.Lookup(HttpAttributes::kVary, &accepts));
// ASSERT_EQ(2, accepts.size());
// EXPECT_STREQ("Origin", *accepts[0]);
// EXPECT_STREQ(HttpAttributes::kAccept, *accepts[1]);
}
TEST_F(InPlaceRewriteContextTest, NoAcceptHeaderForLosslessOrAnimated) {
// Make sure that InPlaceRewriteContext won't add "Vary: Accept" header to
// an image optimized to WebP lossless or WebP animated. Note that we're using
// FakeImageFilter in this test. If we use the real filter,
// ImageRewriteFilter, an image will never be converted to WebP lossless nor
// WebP animated, unless we're allowed to vary on user-agent.
options()->set_in_place_wait_for_optimized(true);
set_optimize_for_browser(true);
Init();
SetAcceptWebp();
// First check lossless case.
img_filter_->set_optimized_image_type(IMAGE_WEBP_LOSSLESS_OR_ALPHA);
FetchAndCheckResponse(cache_png_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));
// Now check animated case.
img_filter_->set_optimized_image_type(IMAGE_WEBP_ANIMATED);
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));
}
TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserNegative) {
options()->set_in_place_wait_for_optimized(true);
set_optimize_for_browser(false);
Init();
// Vary: User-Agent header should not be added no matter the user-agent.
ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
SetAcceptWebp();
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
start_time_ms());
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
ResetHeadersAndStats();
SetTimeMs((start_time_ms() + ttl_ms_/2));
ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
start_time_ms() + ttl_ms_/2);
EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
}
TEST_F(InPlaceRewriteContextTest, LoadFromFile) {
options()->file_load_policy()->Associate("http://www.example.com", "/test/");
WriteFile("/test/cacheable.js", cache_body_ /*" alert ( 'foo ') "*/);
Init();
// TODO(jmarantz): currently we will not have caching headers on
// file-input-resources so we default to the implicit cache TTL.
// We should probably have a new config options for file-input
// TTL for use with in-place.
const int64 kIproFileTtl = RewriteOptions::kDefaultImplicitCacheTtlMs;
FetchAndCheckResponse(cache_js_url_, cache_body_, true,
kIproFileTtl, NULL, start_time_ms());
// First fetch misses initial cache lookup, succeeds at fetch and inserts
// result into cache. Also, the resource gets rewritten and the rewritten
// resource gets inserted into cache.
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(0, lru_cache()->num_hits());
EXPECT_EQ(2, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
// Note that without file-input resources, we would expect that our
// TTL would be reduced to ttl_ms_/2. But it doesn't work like that
// for files. The TTL stays the same.
ResetHeadersAndStats();
SetTimeMs(start_time_ms() + ttl_ms_/2);
FetchAndCheckResponse(cache_js_url_, "good:jm", true,
kIproFileTtl, etag_,
start_time_ms() + ttl_ms_/2);
// Second fetch hits the metadata cache and the rewritten resource is served
// out.
CheckWarmCache("second_fetch_11");
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(1, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits());
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(0, lru_cache()->num_inserts());
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(0, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
// Third fetch is the same exact deal. The file hasn't actually
// changed and the existing rewrite still is valid. The metadata
// cache does not go stale until the file is actually touched.
AdvanceTimeMs(2 * ttl_ms_);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "good:jm", true,
kIproFileTtl, etag_, timer()->NowMs());
CheckWarmCache("third_fetch");
// OK let's now move time forward a little and touch the file
// without changing it. This results in a total reset back to
// the original state. It seems like we could read the file
// and see if it's changed, but we wind up queuing up the asynchronous
// rewrite.
AdvanceTimeMs(1 * Timer::kSecondMs);
WriteFile("/test/cacheable.js", cache_body_ /*" alert ( 'foo ') "*/);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, cache_body_, true,
kIproFileTtl, NULL, timer()->NowMs());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits()); // ipro-metadata, metadata
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts()); // http, metadata, ipro-metadata
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
AdvanceTimeMs(1 * Timer::kSecondMs);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "good:jm", true,
kIproFileTtl, etag_, timer()->NowMs());
CheckWarmCache("second_fetch_after_touch");
// Now change the content.
AdvanceTimeMs(1 * Timer::kSecondMs);
WriteFile("/test/cacheable.js", "new_content");
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "new_content", true,
kIproFileTtl, NULL, timer()->NowMs());
EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
EXPECT_EQ(0, http_cache()->cache_hits()->Get());
EXPECT_EQ(0, http_cache()->cache_misses()->Get());
EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
EXPECT_EQ(2, lru_cache()->num_hits()); // ipro-metadata, metadata
EXPECT_EQ(0, lru_cache()->num_misses());
EXPECT_EQ(3, lru_cache()->num_inserts()); // http, metadata, ipro-metadata
EXPECT_EQ(0, img_filter_->num_rewrites());
EXPECT_EQ(1, js_filter_->num_rewrites());
EXPECT_EQ(0, css_filter_->num_rewrites());
AdvanceTimeMs(1 * Timer::kSecondMs);
ResetHeadersAndStats();
FetchAndCheckResponse(cache_js_url_, "new_content:jm", true,
kIproFileTtl, etag_, timer()->NowMs());
CheckWarmCache("second_fetch_after_mutation");
}
TEST_F(InPlaceRewriteContextTest, JsonWithJsContentTypeSucceeds) {
Init();
CheckCachingAndContentType(json_js_type_url_,
"application/javascript",
cache_body_,
RewriteOptions::kJavascriptMinId);
}
TEST_F(InPlaceRewriteContextTest, JsonWithJsonContentTypeSucceeds) {
Init();
CheckCachingAndContentType(json_json_type_url_,
"application/json",
cache_body_,
RewriteOptions::kJavascriptMinId);
}
TEST_F(InPlaceRewriteContextTest, JsonWithJsonContentTypeSynonymSucceeds) {
Init();
CheckCachingAndContentType(json_json_type_synonym_url_,
"application/x-json",
cache_body_,
RewriteOptions::kJavascriptMinId);
}
} // namespace
} // namespace net_instaweb