/*
 * Copyright 2011 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Author: nikhilmadan@google.com (Nikhil Madan)

#include "net/instaweb/rewriter/public/in_place_rewrite_context.h"

#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/counting_url_async_fetcher.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/mock_url_fetcher.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/public/fake_filter.h"
#include "net/instaweb/rewriter/public/file_load_policy.h"
#include "net/instaweb/rewriter/public/output_resource.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_test_base.h"
#include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
#include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
#include "pagespeed/kernel/base/gtest.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/mock_message_handler.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/cache/lru_cache.h"
#include "pagespeed/kernel/html/html_parse_test_base.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/image_types.pb.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/semantic_type.h"
#include "pagespeed/kernel/http/user_agent_matcher.h"
#include "pagespeed/kernel/http/user_agent_matcher_test_base.h"
#include "pagespeed/kernel/thread/mock_scheduler.h"
#include "pagespeed/kernel/thread/worker_test_base.h"

namespace net_instaweb {

namespace {

class FakeFetch : public AsyncFetch {
 public:
  FakeFetch(const RequestContextPtr& request_context,
            RewriteOptions* options,
            const GoogleString& url,
            WorkerTestBase::SyncPoint* sync,
            ResponseHeaders* response_headers)
      : AsyncFetch(request_context),
        done_(false),
        success_(false),
        options_(options),
        url_(url),
        sync_(sync) {
    set_response_headers(response_headers);
  }

  virtual ~FakeFetch() {}

  virtual void HandleHeadersComplete() {}
  virtual bool HandleWrite(const StringPiece& content,
                           MessageHandler* handler) {
    content.AppendToString(&content_);
    return true;
  }
  virtual bool HandleFlush(MessageHandler* handler) {
    return true;
  }
  virtual void HandleDone(bool success) {
    response_headers()->ComputeCaching();
    done_ = true;
    success_ = success;
    sync_->Notify();
  }
  StringPiece content() { return content_; }
  bool done() { return done_; }
  bool success() { return success_; }

  bool IsCachedResultValid(const ResponseHeaders& headers) {
    return OptionsAwareHTTPCacheCallback::IsCacheValid(
        url_, *options_, request_context(), headers);
  }

 private:
  GoogleString content_;
  bool done_;
  bool success_;
  const RewriteOptions* options_;
  GoogleString url_;
  WorkerTestBase::SyncPoint* sync_;

  DISALLOW_COPY_AND_ASSIGN(FakeFetch);
};

class FakeImageFilter : public FakeFilter {
 public:
  class Context : public FakeFilter::Context {
   public:
    Context(FakeImageFilter* filter, RewriteDriver* driver,
            RewriteContext* parent, ResourceContext* resource_context)
        : FakeFilter::Context(filter, driver, parent, resource_context),
          filter_(filter) { }

    virtual void DoRewriteSingle(const ResourcePtr input,
                                 OutputResourcePtr output) {
      CachedResult* cached = output->EnsureCachedResultCreated();
      FakeFilter::Context::DoRewriteSingle(input, output);
      cached->set_optimized_image_type(filter_->optimized_image_type());
    }

   private:
    FakeImageFilter* filter_;
    DISALLOW_COPY_AND_ASSIGN(Context);
  };

  explicit FakeImageFilter(RewriteDriver* rewrite_driver)
      : FakeFilter(RewriteOptions::kImageCompressionId,
                   rewrite_driver, semantic_type::kImage),
        optimized_image_type_(IMAGE_WEBP) { }

  void set_optimized_image_type(ImageType type) {
    optimized_image_type_ = type;
  }
  ImageType optimized_image_type() const {
    return optimized_image_type_;
  }

  RewriteContext* MakeFakeContext(
      RewriteDriver* driver, RewriteContext* parent,
      ResourceContext* resource_context) {
    return new FakeImageFilter::Context(this, driver, parent, resource_context);
  }

 private:
  ImageType optimized_image_type_;
  DISALLOW_COPY_AND_ASSIGN(FakeImageFilter);
};

class InPlaceRewriteContextTest : public RewriteTestBase {
 protected:
  static const bool kWriteToCache = true;
  static const bool kNoWriteToCache = false;
  static const bool kNoTransform = true;
  static const bool kTransform = false;
  InPlaceRewriteContextTest()
      : img_filter_(NULL),
        other_img_filter_(NULL),
        js_filter_(NULL),
        css_filter_(NULL),
        cache_html_url_("http://www.example.com/cacheable.html"),
        cache_jpg_url_("http://www.example.com/cacheable.jpg"),
        cache_jpg_no_extension_url_("http://www.example.com/cacheable_jpg"),
        cache_jpg_notransform_url_("http://www.example.com/notransform.jpg"),
        cache_jpg_vary_star_url_("http://www.example.com/vary_star.jpg"),
        cache_jpg_vary_ua_url_("http://www.example.com/vary_ua.jpg"),
        cache_jpg_vary_origin_url_("http://www.example.com/vary_origin.jpg"),
        cache_png_url_("http://www.example.com/cacheable.png"),
        cache_gif_url_("http://www.example.com/cacheable.gif"),
        cache_webp_url_("http://www.example.com/cacheable.webp"),
        cache_js_url_("http://www.example.com/cacheable.js"),
        cache_js_jpg_extension_url_("http://www.example.com/cacheable_js.jpg"),
        cache_css_url_("http://www.example.com/cacheable.css"),
        nocache_html_url_("http://www.example.com/nocacheable.html"),
        nocache_js_url_("http://www.example.com/nocacheable.js"),
        private_cache_js_url_("http://www.example.com/privatecacheable.js"),
        cache_js_no_max_age_url_("http://www.example.com/cacheablemod.js"),
        bad_url_("http://www.example.com/bad.url"),
        redirect_url_("http://www.example.com/redir.url"),
        rewritten_jpg_url_(
            "http://www.example.com/cacheable.jpg.pagespeed.ic.0.jpg"),
        json_js_type_url_("http://www.example.com/cacheable_js_type.json"),
        json_json_type_url_("http://www.example.com/cacheable_json_type.json"),
        json_json_type_synonym_url_(
            "http://www.example.com/cacheable_json_synonym_type.json"),
        cache_body_("good"), nocache_body_("bad"),
        bad_body_("ugly"),
        redirect_body_("Location: http://www.example.com/final.url"),
        ttl_ms_(Timer::kHourMs), etag_("W/\"PSA-aj-0\""),
        original_etag_("original_etag"),
        exceed_deadline_(false), optimize_for_browser_(false),
        oversized_stream_(NULL), in_place_uncacheable_rewrites_(NULL),
        distributed_rewrite_failures_(NULL),
        distributed_rewrite_successes_(NULL) {}

  virtual void Init() {
    SetTimeMs(start_time_ms());
    mock_url_fetcher()->set_fail_on_unexpected(false);

    const StringPiece kNoVary("");

    // Set fetcher result and headers.
    AddResponse(cache_html_url_, kContentTypeHtml, cache_body_, start_time_ms(),
                ttl_ms_, original_etag_, kNoVary, kNoWriteToCache, kTransform);
    AddResponse(cache_jpg_url_, kContentTypeJpeg, cache_body_, start_time_ms(),
                ttl_ms_, "", kNoVary, kNoWriteToCache, kTransform);
    AddResponse(cache_jpg_no_extension_url_, kContentTypeJpeg, cache_body_,
                start_time_ms(), ttl_ms_, "", kNoVary,
                kNoWriteToCache, kTransform);
    AddResponse(cache_jpg_notransform_url_, kContentTypeJpeg, cache_body_,
                start_time_ms(), ttl_ms_, "", kNoVary,
                kNoWriteToCache, kNoTransform);
    AddResponse(cache_jpg_vary_star_url_, kContentTypeJpeg, cache_body_,
                start_time_ms(), ttl_ms_, "", /* Vary: */ "*",
                kNoWriteToCache, kTransform);
    AddResponse(cache_jpg_vary_ua_url_, kContentTypeJpeg, cache_body_,
                start_time_ms(), ttl_ms_, "", /* Vary: */ "User-Agent",
                kNoWriteToCache, kTransform);
    AddResponse(cache_jpg_vary_origin_url_, kContentTypeJpeg, cache_body_,
                start_time_ms(), ttl_ms_, "", /* Vary: */ "Origin",
                kNoWriteToCache, kTransform);
    AddResponse(cache_png_url_, kContentTypePng, cache_body_, start_time_ms(),
                ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
    AddResponse(cache_gif_url_, kContentTypeGif, cache_body_, start_time_ms(),
                ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
    AddResponse(cache_webp_url_, kContentTypeWebp, cache_body_, start_time_ms(),
                ttl_ms_, original_etag_, kNoVary, kWriteToCache, kTransform);
    AddResponse(cache_js_url_, kContentTypeJavascript, cache_body_,
                start_time_ms(), ttl_ms_, "", kNoVary,
                kNoWriteToCache, kTransform);
    AddResponse(cache_js_jpg_extension_url_, kContentTypeJavascript,
                cache_body_, start_time_ms(), ttl_ms_, "", kNoVary,
                kNoWriteToCache, kTransform);
    AddResponse(cache_css_url_, kContentTypeCss, cache_body_, start_time_ms(),
                ttl_ms_, "", kNoVary, kNoWriteToCache, kTransform);
    AddResponse(nocache_html_url_, kContentTypeHtml, nocache_body_,
                start_time_ms(), -1, "", kNoVary, kNoWriteToCache, kTransform);
    AddResponse(nocache_js_url_, kContentTypeJavascript, cache_body_,
                start_time_ms(), -1 /*ttl*/, "" /*etag*/, kNoVary,
                kNoWriteToCache, kTransform);
    AddResponse(cache_js_no_max_age_url_, kContentTypeJavascript, cache_body_,
                start_time_ms(), 0, "", kNoVary, kNoWriteToCache, kTransform);
    AddResponseStrContentType(
        json_js_type_url_, "application/javascript", cache_body_,
        start_time_ms(), ttl_ms_, "", kNoVary,
        kNoWriteToCache, kTransform);
    AddResponseStrContentType(
        json_json_type_url_, "application/json", cache_body_,
        start_time_ms(), ttl_ms_, "", kNoVary,
        kNoWriteToCache, kTransform);
    AddResponseStrContentType(
        json_json_type_synonym_url_, "application/x-json", cache_body_,
        start_time_ms(), ttl_ms_, "", kNoVary,
        kNoWriteToCache, kTransform);

    ResponseHeaders private_headers;
    SetDefaultHeaders(kContentTypeJavascript.mime_type(), &private_headers);
    private_headers.SetDateAndCaching(start_time_ms(), 1200 /*ttl*/,
                                      ",private");
    mock_url_fetcher()->SetResponse(
        private_cache_js_url_, private_headers, cache_body_);

    ResponseHeaders bad_headers;
    bad_headers.set_first_line(1, 1, 404, "Not Found");
    bad_headers.SetDate(start_time_ms());
    mock_url_fetcher()->SetResponse(bad_url_, bad_headers, bad_body_);

    // Add a response for permanent redirect.
    ResponseHeaders redirect_headers;
    redirect_headers.set_first_line(1, 1, 301, "Moved Permanently");
    redirect_headers.ComputeCaching();
    redirect_headers.SetCacheControlMaxAge(36000);
    redirect_headers.Add(HttpAttributes::kCacheControl, "public");
    redirect_headers.Add(HttpAttributes::kContentType, "image/jpeg");
    mock_url_fetcher()->SetResponse(
        redirect_url_, redirect_headers, redirect_body_);

    img_filter_ = new FakeImageFilter(rewrite_driver());
    js_filter_ = new FakeFilter(RewriteOptions::kJavascriptMinId,
                                rewrite_driver(), semantic_type::kScript);
    css_filter_ = new FakeFilter(RewriteOptions::kCssFilterId, rewrite_driver(),
                                 semantic_type::kStylesheet);

    rewrite_driver()->AppendRewriteFilter(img_filter_);
    rewrite_driver()->AppendRewriteFilter(js_filter_);
    rewrite_driver()->AppendRewriteFilter(css_filter_);
    options()->ClearSignatureForTesting();
    AddRecompressImageFilters();
    options()->EnableFilter(RewriteOptions::kRewriteJavascriptExternal);
    options()->EnableFilter(RewriteOptions::kRewriteJavascriptInline);
    options()->EnableFilter(RewriteOptions::kRewriteCss);
    if (optimize_for_browser()) {
      options()->EnableFilter(RewriteOptions::kInPlaceOptimizeForBrowser);
      options()->EnableFilter(RewriteOptions::kConvertJpegToWebp);
    }
    options()->set_in_place_rewriting_enabled(true);

    // Only allow to vary on "Accept" header.
    RewriteOptions::AllowVaryOn allow_vary_on;
    EXPECT_TRUE(RewriteOptions::ParseFromString("accept", &allow_vary_on));
    options()->set_allow_vary_on(allow_vary_on);

    server_context()->ComputeSignature(options());
    // Clear stats since we may have added something to the cache.
    ClearStats();

    oversized_stream_ = statistics()->GetVariable(
        InPlaceRewriteContext::kInPlaceOversizedOptStream);
    in_place_uncacheable_rewrites_ = statistics()->GetVariable(
        InPlaceRewriteContext::kInPlaceUncacheableRewrites);
    distributed_rewrite_failures_ = statistics()->GetVariable(
        RewriteContext::kNumDistributedRewriteFailures);
    distributed_rewrite_successes_ = statistics()->GetVariable(
        RewriteContext::kNumDistributedRewriteSuccesses);
  }

  void AddResponseStrContentType(
      const GoogleString& url, const GoogleString& content_type,
      const GoogleString& body, int64 now_ms, int64 ttl_ms,
      const GoogleString& etag, const StringPiece& vary,
      bool write_to_cache, bool no_transform) {
    ResponseHeaders response_headers;
    SetDefaultHeaders(content_type, &response_headers);
    if (ttl_ms > 0) {
      response_headers.SetDateAndCaching(now_ms, ttl_ms);
    } else {
      response_headers.SetDate(now_ms);
      if (ttl_ms < 0) {
        response_headers.Replace(HttpAttributes::kCacheControl, "no-cache");
      } else {
        response_headers.Replace(HttpAttributes::kCacheControl, "public");
      }
    }
    if (!vary.empty()) {
      response_headers.Replace(HttpAttributes::kVary, vary);
    }
    if (no_transform) {
      response_headers.Replace(HttpAttributes::kCacheControl, "no-transform");
    }
    if (!etag.empty()) {
      response_headers.Add(HttpAttributes::kEtag, etag);
    }
    mock_url_fetcher()->SetResponse(url, response_headers, body);
    if (write_to_cache) {
      response_headers.ComputeCaching();
      http_cache()->Put(
          url, rewrite_driver_->CacheFragment(),
          RequestHeaders::Properties(),
          ResponseHeaders::GetVaryOption(options()->respect_vary()),
          &response_headers, body, message_handler());
    }
  }

  void AddResponse(const GoogleString& url, const ContentType& content_type,
                   const GoogleString& body, int64 now_ms, int64 ttl_ms,
                   const GoogleString& etag, const StringPiece& vary,
                   bool write_to_cache, bool no_transform) {
    AddResponseStrContentType(url, content_type.mime_type(), body, now_ms,
                              ttl_ms, etag, vary, write_to_cache, no_transform);
  }

  void SetDefaultHeaders(const GoogleString& content_type,
                         ResponseHeaders* header) const {
    header->set_major_version(1);
    header->set_minor_version(1);
    header->SetStatusAndReason(HttpStatus::kOK);
    header->Replace(HttpAttributes::kContentType, content_type);
  }

  void ResetUserAgent(StringPiece user_agent) {
    ClearRewriteDriver();
    SetCurrentUserAgent(user_agent);
  }

  void SetAcceptWebp() {
    AddRequestAttribute(HttpAttributes::kAccept, "image/webp");
  }

  void FetchAndCheckResponse(const GoogleString& url,
                             const GoogleString& expected_body,
                             bool expected_success,
                             int64 expected_ttl,
                             const char* etag,
                             int64 date_ms) {
    js_filter_->set_exceed_deadline(exceed_deadline_);
    img_filter_->set_exceed_deadline(exceed_deadline_);
    if (other_img_filter_ != NULL) {
      other_img_filter_->set_exceed_deadline(exceed_deadline_);
    }
    css_filter_->set_exceed_deadline(exceed_deadline_);

    WorkerTestBase::SyncPoint sync(server_context()->thread_system());
    RequestContextPtr request_context(RequestContext::NewTestRequestContext(
        server_context()->thread_system()));
    FakeFetch mock_fetch(request_context, options(), url, &sync,
                         &response_headers_);
    const RequestHeaders* driver_request_headers =
        rewrite_driver()->request_headers();
    if (driver_request_headers != NULL) {
      mock_fetch.request_headers()->CopyFrom(*driver_request_headers);
    }
    rewrite_driver()->FetchResource(url, &mock_fetch);

    // If we're testing if the rewrite takes too long, we need to push
    // time forward here.
    if (exceed_deadline()) {
      rewrite_driver()->BoundedWaitFor(
          RewriteDriver::kWaitForCompletion,
          rewrite_driver()->rewrite_deadline_ms());
    }

    sync.Wait();
    rewrite_driver()->WaitForShutDown();
    mock_scheduler()->AwaitQuiescence();  // needed for cache puts to finish.
    EXPECT_TRUE(mock_fetch.done());
    EXPECT_EQ(expected_success, mock_fetch.success()) << url;
    EXPECT_EQ(expected_body, mock_fetch.content()) << url;
    EXPECT_EQ(expected_ttl, response_headers_.cache_ttl_ms()) << url;
    EXPECT_STREQ(etag, response_headers_.Lookup1(HttpAttributes::kEtag)) << url;
    EXPECT_EQ(date_ms, response_headers_.date_ms()) << url;
  }

  void ResetHeadersAndStats() {
    response_headers_.Clear();
    img_filter_->ClearStats();
    if (other_img_filter_ != NULL) {
      other_img_filter_->ClearStats();
    }
    js_filter_->ClearStats();
    css_filter_->ClearStats();
    RewriteTestBase::ClearStats();
    ClearRewriteDriver();
  }

  void CheckWarmCache(StringPiece id) {
    EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count()) << id;
    EXPECT_EQ(1, http_cache()->cache_hits()->Get()) << id;
    EXPECT_EQ(0, http_cache()->cache_misses()->Get()) << id;
    EXPECT_EQ(0, http_cache()->cache_inserts()->Get()) << id;
    EXPECT_EQ(2, lru_cache()->num_hits()) << id;
    EXPECT_EQ(0, lru_cache()->num_misses()) << id;
    EXPECT_EQ(0, lru_cache()->num_inserts()) << id;
    EXPECT_EQ(0, img_filter_->num_rewrites()) << id;
    EXPECT_EQ(0, js_filter_->num_rewrites()) << id;
    EXPECT_EQ(0, css_filter_->num_rewrites()) << id;
    EXPECT_EQ(0, oversized_stream_->Get()) << id;
  }

  void SetupDistributedTest(const StringPiece& distributed_filter) {
    SetupSharedCache();
    other_img_filter_ = new FakeImageFilter(other_rewrite_driver());
    other_rewrite_driver()->AppendRewriteFilter(other_img_filter_);
    options()->ClearSignatureForTesting();
    other_options()->ClearSignatureForTesting();
    AddRecompressImageFilters();
    options()->EnableFilter(RewriteOptions::kRewriteJavascriptExternal);
    options()->EnableFilter(RewriteOptions::kRewriteCss);
    options_->DistributeFilter(distributed_filter);
    options_->set_distributed_rewrite_servers("example.com:80");
    options_->set_distributed_rewrite_key("1234123");
    Init();
    other_options()->Merge(*options());
    other_server_context()->ComputeSignature(other_options());
  }

  void CheckDistributedFetch(int distributed_fetch_success_count,
                             int distributed_fetch_failure_count,
                             int local_fetch_required, int rewritten) {
    EXPECT_EQ(distributed_fetch_success_count + distributed_fetch_failure_count,
              counting_distributed_fetcher()->fetch_count());
    EXPECT_EQ(local_fetch_required,
              counting_url_async_fetcher()->fetch_count());
    EXPECT_EQ(
        0, other_factory_->counting_distributed_async_fetcher()->fetch_count());
    EXPECT_EQ(distributed_fetch_success_count,
              distributed_rewrite_successes_->Get());
    EXPECT_EQ(distributed_fetch_failure_count,
              distributed_rewrite_failures_->Get());
    EXPECT_EQ(rewritten,
              img_filter_->num_rewrites() + other_img_filter_->num_rewrites());
  }

  void ExpectInPlaceImageSuccessFlow(const GoogleString& url) {
    FetchAndCheckResponse(url, cache_body_, true, ttl_ms_,
                          original_etag_, start_time_ms());

    // First fetch misses initial metadata cache lookup, finds original in
    // cache; the resource gets rewritten and the rewritten
    // resource gets inserted into cache.
    EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
    EXPECT_EQ(1, http_cache()->cache_hits()->Get());
    EXPECT_EQ(0, http_cache()->cache_misses()->Get());
    EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
    EXPECT_EQ(1, lru_cache()->num_hits());
    EXPECT_EQ(2, lru_cache()->num_misses());
    EXPECT_EQ(3, lru_cache()->num_inserts());
    EXPECT_EQ(1, img_filter_->num_rewrites());
    EXPECT_EQ(0, js_filter_->num_rewrites());
    EXPECT_EQ(0, css_filter_->num_rewrites());

    ResetHeadersAndStats();
    SetTimeMs(start_time_ms() + ttl_ms_/2);
    FetchAndCheckResponse(url, "good:ic", true, ttl_ms_/2, etag_,
                          start_time_ms() + ttl_ms_/2);
    // Second fetch hits the metadata cache and the rewritten resource is
    // served out.
    CheckWarmCache("second_fetch_1");

    AdvanceTimeMs(2 * ttl_ms_);
    ResetHeadersAndStats();
    FetchAndCheckResponse(url, cache_body_, true, ttl_ms_,
                          original_etag_, timer()->NowMs());
    // The metadata and cache entry is stale now. Fetch the content and serve
    // out the original. The background rewrite work then revalidates
    // the response and updates metadata.
    EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
    EXPECT_EQ(0, http_cache()->cache_hits()->Get());
    EXPECT_EQ(1, http_cache()->cache_misses()->Get());
    EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
    EXPECT_EQ(3, lru_cache()->num_hits());  // (expired) orig., aj, ic metadata
    EXPECT_EQ(0, lru_cache()->num_misses());
    EXPECT_EQ(3, lru_cache()->num_inserts());
    EXPECT_EQ(0, img_filter_->num_rewrites());
    EXPECT_EQ(0, js_filter_->num_rewrites());
    EXPECT_EQ(0, css_filter_->num_rewrites());
  }

  void CheckCachingAndContentType(const GoogleString& url,
                                  const GoogleString& expected_mime_type,
                                  const GoogleString& cache_body,
                                  const GoogleString& filter_prefix) {
    FetchAndCheckResponse(url, cache_body, true, ttl_ms_,
                          NULL, start_time_ms());
    EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
    EXPECT_EQ(0, http_cache()->cache_hits()->Get());
    EXPECT_EQ(1, http_cache()->cache_misses()->Get());
    EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
    EXPECT_EQ(0, lru_cache()->num_hits());
    EXPECT_EQ(3, lru_cache()->num_misses());
    EXPECT_EQ(4, lru_cache()->num_inserts());
    EXPECT_EQ(0, img_filter_->num_rewrites());
    EXPECT_EQ(1, js_filter_->num_rewrites());
    EXPECT_EQ(0, css_filter_->num_rewrites());

    // Make sure the content type is unmodified.
    EXPECT_STREQ(expected_mime_type,
                 response_headers_.Lookup1(HttpAttributes::kContentType));

    // Try a second fetch and ensure we get a cache hit.
    ResetHeadersAndStats();
    FetchAndCheckResponse(url, StrCat(cache_body, ":", filter_prefix), true,
                          ttl_ms_, etag_, start_time_ms());
    EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
    EXPECT_EQ(1, http_cache()->cache_hits()->Get());
    EXPECT_EQ(0, http_cache()->cache_misses()->Get());
    EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
    EXPECT_EQ(2, lru_cache()->num_hits());
    EXPECT_EQ(0, lru_cache()->num_misses());
    EXPECT_EQ(0, lru_cache()->num_inserts());
    EXPECT_EQ(0, img_filter_->num_rewrites());
    EXPECT_EQ(0, js_filter_->num_rewrites());
    EXPECT_EQ(0, css_filter_->num_rewrites());
    EXPECT_STREQ(expected_mime_type,
                 response_headers_.Lookup1(HttpAttributes::kContentType));
  }

  bool exceed_deadline() { return exceed_deadline_; }
  void set_exceed_deadline(bool x) { exceed_deadline_ = x; }

  bool optimize_for_browser() { return optimize_for_browser_; }
  void set_optimize_for_browser(bool x) { optimize_for_browser_ = x; }

  FakeImageFilter* img_filter_;
  FakeImageFilter* other_img_filter_;
  FakeFilter* js_filter_;
  FakeFilter* css_filter_;

  ResponseHeaders response_headers_;

  const GoogleString cache_html_url_;
  const GoogleString cache_jpg_url_;
  const GoogleString cache_jpg_no_extension_url_;
  const GoogleString cache_jpg_notransform_url_;
  const GoogleString cache_jpg_vary_star_url_;
  const GoogleString cache_jpg_vary_ua_url_;
  const GoogleString cache_jpg_vary_origin_url_;
  const GoogleString cache_png_url_;
  const GoogleString cache_gif_url_;
  const GoogleString cache_webp_url_;
  const GoogleString cache_js_url_;
  const GoogleString cache_js_jpg_extension_url_;
  const GoogleString cache_css_url_;
  const GoogleString nocache_html_url_;
  const GoogleString nocache_js_url_;
  const GoogleString private_cache_js_url_;
  const GoogleString cache_js_no_max_age_url_;
  const GoogleString bad_url_;
  const GoogleString redirect_url_;
  const GoogleString rewritten_jpg_url_;
  const GoogleString json_js_type_url_;
  const GoogleString json_json_type_url_;
  const GoogleString json_json_type_synonym_url_;

  const GoogleString cache_body_;
  const GoogleString nocache_body_;
  const GoogleString bad_body_;
  const GoogleString redirect_body_;

  const int ttl_ms_;
  const char* etag_;
  const char* original_etag_;
  bool exceed_deadline_;
  bool optimize_for_browser_;

  Variable* oversized_stream_;
  Variable* in_place_uncacheable_rewrites_;
  Variable* distributed_rewrite_failures_;
  Variable* distributed_rewrite_successes_;
};

TEST_F(InPlaceRewriteContextTest, IngressDistributedRewrite) {
  // Distribute an image rewrite (the response of the rewrite task is mocked).
  SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);

  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
                        start_time_ms());
  CheckDistributedFetch(1,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        1);     // number of rewrites

  // Ingress task distributes and streams back the result.
  // Rewrite task misses metadata, misses http, fetches, writes original,
  // misses ic metadata, writes optimized, and writes metadata for ipro and ic.
  EXPECT_EQ(1, counting_distributed_fetcher()->fetch_count());
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());

  // Ingress task distributes, distributed task hits and returns.
  EXPECT_EQ(1, counting_distributed_fetcher()->fetch_count());
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteImage) {
  // Distribute the nested image task instead of the in_place_rewrite filter.
  SetupDistributedTest(RewriteOptions::kImageCompressionId);

  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
                        start_time_ms());
  CheckDistributedFetch(1,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        1,      // number of ingress fetches
                        1);     // number of rewrites

  // Ingress task: misses IPRO metadata, fetches the resource (miss and insert)
  // misses ic metadata, and distributes, eventually writing IPRO metadata.
  // Rewrite task: misses ic metadata, hits http, writes optimized, and ic.
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(4, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_ / 2);

  // Ingress task hits ipro metadata and associated http resource.
  EXPECT_EQ(0, counting_distributed_fetcher()->fetch_count());
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, IngressDistributedNestedWaitForOptimized) {
  // Like IngressDistributedNested but this time we want to wait for the
  // optimized result, which causes a distributed GET request and the returned
  // content is fed into the nested context's output_ for IPRO's Harvest.
  options_->set_in_place_wait_for_optimized(true);
  SetupDistributedTest(RewriteOptions::kImageCompressionId);

  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());

  // Ingress task fetches in IPRO and then distributes the nested image rewrite.
  CheckDistributedFetch(1,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        1,      // number of ingress fetches
                        1);     // number of rewrites

  // Ingress task: IPRO misses metadata and http original resource then fetches
  // and inserts http original resource. Ingress then starts nested image
  // rewriter which skips metadata check (force_rewrite) and distributes. IPRO
  // writes its metadata.
  // Distributed task: Misses image metadata, hits the original resource, and
  // stores the new metadata and optimized resource.
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_ / 2);
  CheckDistributedFetch(0,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites
  // Ingress task hits ipro metadata and associated http resource.
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest,
       IngressDistributedNestedWaitForOptimizedFail) {
  // Wait for an optimized result but the fetcher breaks after the headers are
  // written. Fall back to the original resource.
  options_->set_in_place_wait_for_optimized(true);
  SetupDistributedTest(RewriteOptions::kImageCompressionId);
  test_distributed_fetcher()->set_fail_after_headers(true);

  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
                        start_time_ms());

  // Ingress task fetches in IPRO and then distributes the nested image rewrite.
  CheckDistributedFetch(0,      // successful distributed fetches
                        1,      // unsuccessful distributed fetches
                        1,      // number of ingress fetches
                        1);     // number of rewrites

  // Ingress task: IPRO misses metadata and http original resource,
  // fetches and inserts http original resource. Then starts nested image
  // rewriter which skips the metadata lookup (force_rewrite) and distributes.
  // Distributed task: Misses image metadata, hits the original resource, and
  // stores the new metadata and optimized resource.
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
                        start_time_ms());
  CheckDistributedFetch(0,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites
  // Ingress task hits ipro metadata and associated http resource.
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteNotFound) {
  // If the distributed fetcher returns a 404 then that's what should be
  // returned.
  SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);

  GoogleString orig_url = StrCat(kTestDomain, "fourofour.png");
  SetFetchResponse404(orig_url);

  FetchAndCheckResponse(orig_url, "", true, ServerContext::kGeneratedMaxAgeMs,
                        ServerContext::kResourceEtagValue, start_time_ms());
  EXPECT_EQ(HttpStatus::kNotFound, response_headers_.status_code());

  // The distributed fetcher should have run once on the ingress task and the
  // url fetcher should have run once on the rewrite task.  The result goes to
  // shared cache.
  CheckDistributedFetch(1,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites

  // Ingress task distributes and returns the 404 it gets back.
  // Rewrite task misses metadata and http. The 404 isn't cacheable, so it's
  // not written.
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());

  // Ingress task distributes.
  // Rewrite task misses ipro metadata + http, and returns that.
  ResetHeadersAndStats();
  FetchAndCheckResponse(orig_url, "", true, ServerContext::kGeneratedMaxAgeMs,
                        ServerContext::kResourceEtagValue, start_time_ms());
  CheckDistributedFetch(1,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteFailFallback) {
  // If the distributed fetch fails mid-stream then we fail.
  SetupDistributedTest(RewriteOptions::kInPlaceRewriteId);
  test_distributed_fetcher()->set_fail_after_headers(true);

  FetchAndCheckResponse(cache_jpg_url_, "", false, ttl_ms_, NULL,
                        start_time_ms());
  // Note that we didn't need to fetch the original resource at the ingress task
  // because the distributed task already fetched it and put it in shared cache.
  CheckDistributedFetch(0,      // successful distributed fetches
                        1,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        1);     // number of rewrites
  // Ingress task: Distributes and streams back a failed stream.
  // Rewrite task: Misses ipro metadata, misses http, fetches and inserts http,
  // misses image compression metadata, writes optimized http and 2 metadata,
  // returns.
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  // Fetch again but we'll hit the cache this time.
  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_ / 2));
  FetchAndCheckResponse(cache_jpg_url_, "", false, ttl_ms_ / 2, etag_,
                        start_time_ms() + ttl_ms_ / 2);
  CheckDistributedFetch(0,      // successful distributed fetches
                        1,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, IngressDistributedRewriteFailFallbackImage) {
  // If the distributed fetch fails mid-stream then the unoptimized resource
  // should be returned. This time image compression is distributed but not IPRO
  // itself.
  SetupDistributedTest(RewriteOptions::kImageCompressionId);

  // Simulate distributed fetch failure and ensure that we fall back to the
  // original.
  test_distributed_fetcher()->set_fail_after_headers(true);

  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, NULL,
                        start_time_ms());
  // Note that we didn't need to fetch the original resource at the ingress task
  // because the distributed task already fetched it and put it in shared cache.
  CheckDistributedFetch(0,      // successful distributed fetches
                        1,      // unsuccessful distributed fetches
                        1,      // number of ingress fetches
                        1);     // number of rewrites
  // Ingress task: Misses ipro metadata, fetches and inserts input resource,
  // misses ic metadata, distributes. Upon distributed failure, aborts nested
  // rewrite and IPRO records the failure.
  // Rewrite task: Misses ic metadata, hits input resources, writes optimized
  // resource and ic metadata, then returns.
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(4, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());

  // On the first attempt IPRO failed to rewrite the image due to its nested
  // task's distribution failure.  On the second attempt, IPRO remembers that
  // the first attempt failed and returns the original resource.

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
                        start_time_ms());
  CheckDistributedFetch(0,      // successful distributed fetches
                        0,      // unsuccessful distributed fetches
                        0,      // number of ingress fetches
                        0);     // number of rewrites
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
}

TEST_F(InPlaceRewriteContextTest, CacheableHtmlUrlNoRewriting) {
  // All these entries find no in-place rewrite metadata and no rewriting
  // happens.
  Init();
  FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
                        original_etag_, start_time_ms());
  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());  // metadata + html
  EXPECT_EQ(1, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
                        original_etag_, start_time_ms());
  // Second fetch hits initial cache lookup and no extra fetches are needed.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(1, lru_cache()->num_misses());  // metadata
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  AdvanceTimeMs(2 * ttl_ms_);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_html_url_, cache_body_, true, ttl_ms_,
                        original_etag_, start_time_ms() + 2 * ttl_ms_);
  // Cache entry is stale, so we must fetch again.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(1, lru_cache()->num_hits());  // HTML is in LRU cache, just expired.
  EXPECT_EQ(1, lru_cache()->num_misses());
  EXPECT_EQ(1, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, WaitForOptimizedFirstRequest) {
  // By setting this flag we should get an optimized response on the first
  // request unless we hit a rewrite timeout but in this test it will complete
  // in time.
  options()->set_in_place_wait_for_optimized(true);
  Init();

  // The optimized content from the fake rewriter has ":ic" appended to original
  // content.
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache. The optimized version should be
  // returned.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(0, oversized_stream_->Get());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is
  // served out.
  CheckWarmCache("second_fetch_2");
}

TEST_F(InPlaceRewriteContextTest, WaitForOptimizeWithDisabledFilter) {
  // Wait for optimized but if the resource fails to optimize we should get
  // back the original resource.
  options()->set_in_place_wait_for_optimized(true);
  // We'll also test that the hash values we get are legitimate and not
  // hard-coded 0s
  UseMd5Hasher();

  Init();

  // Turn off optimization. The filter will still run but return false in
  // rewrite.
  img_filter_->set_enabled(false);
  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
                        NULL, start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Failure to rewrite means original should be returned.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());  // original only
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(0, oversized_stream_->Get());

  ResetHeadersAndStats();
  // The second time we get the cached original, which should have an md5'd
  // etag.

  // TODO(jkarlin): Note that if we advance time here, we'd expect the TTL of
  // the cached resource to decrease on the second fetch, but that doesn't
  // happen. That should be fixed.
  GoogleString expected_etag = StrCat("W/\"PSA-", hasher()->Hash(cache_body_),
                                      "\"");
  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
                        expected_etag.c_str(), start_time_ms());
  // Second fetch hits the metadata cache, sees that the rewrite failed and
  // fetches and serves the original resource from cache.
  CheckWarmCache("second_fetch_3");
}

TEST_F(InPlaceRewriteContextTest, WaitForOptimizeNoTransform) {
  // Confirm that when cache-control:no-transform is present in the response
  // headers that the in-place optimizer does not optimize the resource.
  options()->set_in_place_wait_for_optimized(true);
  Init();

  // Don't rewrite since it's no-transform.
  FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
                        ttl_ms_, NULL, start_time_ms());
  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(2, lru_cache()->num_inserts());  // original + ipro metadata
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  EXPECT_TRUE(response_headers_.HasValue(HttpAttributes::kCacheControl,
                                         "no-transform"));

  ResetHeadersAndStats();

  // Don't rewrite since it's no-transform.
  FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
                        ttl_ms_, kEtag0.c_str(), start_time_ms());
  // The second fetch should return the cached original after seeing that it
  // can't be rewritten.
  CheckWarmCache("second_fetch_4");
}

TEST_F(InPlaceRewriteContextTest, OptimizeOnNoTransformIfOptionFalse) {
  options()->set_disable_rewrite_on_no_transform(false);
  Init();
  FetchAndCheckResponse(cache_jpg_notransform_url_, cache_body_, true,
                        ttl_ms_, NULL, start_time_ms());
  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // into cache. Also the resource gets rewritten and the rewritten resource
  // gets inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  FetchAndCheckResponse(cache_jpg_notransform_url_, "good:ic", true,
                        ttl_ms_/2, etag_, start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  CheckWarmCache("second_fetch_notransform");
}

TEST_F(InPlaceRewriteContextTest, WaitForOptimizeTimeout) {
  // Confirm that rewrite deadlines cause the original resource to be returned
  // (but caches the optimized) even if in_place_wait_for_optimize is on.
  options()->set_in_place_wait_for_optimized(true);
  Init();

  // Tells the optimizing filter to slow down.
  exceed_deadline_ = true;

  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
                        NULL, start_time_ms());
  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Rewrite succeeds but is slow so original returned.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(0, oversized_stream_->Get());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));

  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  CheckWarmCache("second_fetch_5");
}

TEST_F(InPlaceRewriteContextTest, WaitForOptimizeResourceTooBig) {
  // Wait for optimized but if it's larger than the RecordingFetch can handle
  // make sure we piece together the original resource properly.
  options()->set_in_place_wait_for_optimized(true);

  Init();

  // To make this more interesting there should be something in the cache to
  // recover when we fail.  Let's split the url_fetch from 'good' into 'go' and
  // 'od' writes.
  mock_url_fetcher()->set_split_writes(true);

  // By setting cache max to 2, the second write ('od') will cause an overflow.
  // Test that we recover.
  http_cache()->set_max_cacheable_response_content_length(2);

  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
                        NULL, start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch but resource too
  // big for cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(1, oversized_stream_->Get());

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_,
                        NULL, start_time_ms());
  // Second fetch should also completely miss because the first fetch was too
  // big to stuff in the cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(1, oversized_stream_->Get());
}

TEST_F(InPlaceRewriteContextTest, CacheableJpgUrlRewritingSucceeds) {
  Init();
  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_, NULL,
                        start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  CheckWarmCache("second_fetch_6");

  ResetHeadersAndStats();
  // We get a 304 if we send a request with an If-None-Match matching the hash
  // of the rewritten resource.
  AddRequestAttribute(HttpAttributes::kIfNoneMatch, etag_);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "", true, ttl_ms_/2, NULL, 0);
  EXPECT_EQ(HttpStatus::kNotModified, response_headers_.status_code());
  // We hit the metadata cache and find that the etag matches the hash of the
  // rewritten resource.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  // The etag doesn't match and hence we serve the full response.
  AddRequestAttribute(HttpAttributes::kIfNoneMatch, "no-match");
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  EXPECT_EQ(HttpStatus::kOK, response_headers_.status_code());
  // We hit the metadata cache, but the etag doesn't match so we fetch the
  // rewritten resource from the HTTPCache and serve it out.
  CheckWarmCache("etag_mismatch");

  // Delete the rewritten resource from cache to check if reconstruction works.
  lru_cache()->Delete(HttpCacheKey(rewritten_jpg_url_));

  ResetHeadersAndStats();
  // Original resource is served with the date set to start time.
  // The ETag we check for here is the ETag HTTPCache synthesized for
  // the original resource.
  FetchAndCheckResponse(cache_jpg_url_, "good", true, ttl_ms_, kEtag0.c_str(),
                        start_time_ms());
  // We find the metadata in cache, but don't find the rewritten resource.
  // Hence, we reconstruct the resource and insert it into cache. We see 2
  // identical reinserts - one for the image rewrite filter metadata and one for
  // the in-place metadata.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(1, lru_cache()->num_misses());
  EXPECT_EQ(1, lru_cache()->num_inserts());
  EXPECT_EQ(2, lru_cache()->num_identical_reinserts());
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  // For only the next request, update the date header so that freshening
  // succeeds.
  FetcherUpdateDateHeaders();
  ResetHeadersAndStats();
  int64 time_ms = start_time_ms() + ttl_ms_ - 2 * Timer::kMinuteMs;
  SetTimeMs(time_ms);
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, 2 * Timer::kMinuteMs,
                        etag_, time_ms);
  // This fetch hits the metadata cache and the rewritten resource is served
  // out. Freshening is triggered here and we insert the freshened response and
  // metadata into the cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(3, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(2, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  mock_url_fetcher()->set_update_date_headers(false);

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_ * 5/4));
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true,
                        ttl_ms_ * 3/4 - 2 * Timer::kMinuteMs, etag_,
                        start_time_ms() + ttl_ms_ * 5/4);
  // Since the previous request freshened the metadata, this fetch hits the
  // metadata cache and the rewritten resource is served out. Note that no
  // freshening needs to be triggered here.
  CheckWarmCache("freshened_metadata");

  AdvanceTimeMs(2 * ttl_ms_);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_url_, cache_body_, true, ttl_ms_, NULL,
                        timer()->NowMs());
  // The metadata and cache entry is stale now. Fetch the content and serve out
  // the original. We will however notice that the contents did not
  // actually change and update the metadata cache promptly, without
  // rewriting.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(3, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingSucceeds) {
  Init();
  ExpectInPlaceImageSuccessFlow(cache_png_url_);
}

TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingSucceedsWithShards) {
  Init();
  const char kShard1[] = "http://s1.example.com/";
  const char kShard2[] = "http://s2.example.com/";
  AddShard("http://www.example.com", StrCat(kShard1, ",", kShard2));
  ExpectInPlaceImageSuccessFlow(cache_png_url_);
}

TEST_F(InPlaceRewriteContextTest, CacheableiGifUrlRewritingSucceeds) {
  Init();
  ExpectInPlaceImageSuccessFlow(cache_gif_url_);
}

TEST_F(InPlaceRewriteContextTest, CacheableWebpUrlRewritingSucceeds) {
  Init();
  ExpectInPlaceImageSuccessFlow(cache_webp_url_);
}

TEST_F(InPlaceRewriteContextTest, CacheablePngUrlRewritingFails) {
  // Setup the image filter to fail at rewriting.
  Init();
  img_filter_->set_enabled(false);
  FetchAndCheckResponse(cache_png_url_, cache_body_, true, ttl_ms_,
                        original_etag_, start_time_ms());

  // First fetch misses initial metadata lookup, finds original in cache.
  // The rewrite fails and metadata is inserted into the
  // cache indicating that the rewriting didn't succeed.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(1, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(2, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_png_url_, cache_body_, true, ttl_ms_,
                        original_etag_, start_time_ms());
  // Second fetch hits the metadata cache, sees that the rewrite failed and
  // fetches and serves the original resource from cache.
  CheckWarmCache("second_fetch_7");
}

TEST_F(InPlaceRewriteContextTest, CacheableJsUrlRewritingSucceeds) {
  Init();
  FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
                        start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  CheckWarmCache("second_fetch_8");

  AdvanceTimeMs(2 * ttl_ms_);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
                        timer()->NowMs());
  // The metadata and cache entry is stale now. Fetch the content and serve it
  // out without rewriting. The background rewrite will then revalidate
  // a previous rewrite's result and reuse it.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(3, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, CacheableJsUrlRewritingWithStaleServing) {
  Init();
  options()->ClearSignatureForTesting();
  options()->set_metadata_cache_staleness_threshold_ms(ttl_ms_);
  server_context()->ComputeSignature(options());

  FetchAndCheckResponse(cache_js_url_, cache_body_, true, ttl_ms_, NULL,
                        start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  CheckWarmCache("second_fetch_9");

  SetTimeMs(start_time_ms() + (3 * ttl_ms_) / 2);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "good:jm", true,
                        RewriteOptions::kDefaultImplicitCacheTtlMs, etag_,
                        start_time_ms() + (3 * ttl_ms_) / 2);
  // The metadata and cache entry is stale now. We serve the rewritten resource
  // here, but trigger a fetch and rewrite to update the metadata.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(3, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(2, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, CacheableJsUrlModifiedImplicitCacheTtl) {
  Init();
  response_headers_.set_implicit_cache_ttl_ms(500 * Timer::kSecondMs);
  FetchAndCheckResponse(cache_js_no_max_age_url_, cache_body_,
                        /* expected_success= */ true,
                        /* expected_ttl= */ 500 * Timer::kSecondMs,
                        /* etag= */ NULL,
                        /* date_ms= */ start_time_ms());
}

TEST_F(InPlaceRewriteContextTest, CacheableCssUrlIfCssRewritingDisabled) {
  Init();
  options()->ClearSignatureForTesting();
  options()->DisableFilter(RewriteOptions::kRewriteCss);
  server_context()->ComputeSignature(options());
  FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
                        start_time_ms());

  // First fetch succeeds at the fetcher, no rewriting happens since the css
  // filter is disabled, and metadata indicating a rewriting failure gets
  // inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(2, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  ResetHeadersAndStats();

  // The ETag we check for here is the ETag HTTPCache synthesized for
  // the original resource.
  FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_,
                        kEtag0.c_str(), start_time_ms());

  // Second fetch hits the metadata cache, finds that the result is not
  // optimizable. It then looks up cache for the original and finds it.
  CheckWarmCache("second_fetch_10");
}

TEST_F(InPlaceRewriteContextTest, CacheableCssUrlRewritingSucceeds) {
  Init();
  EnableCachePurge();
  FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
                        start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(3, lru_cache()->num_misses());
  EXPECT_EQ(4, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(1, css_filter_->num_rewrites());

  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  FetchAndCheckResponse(cache_css_url_, "good:cf", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  AdvanceTimeMs(2 * ttl_ms_);
  ResetHeadersAndStats();
  int64 date_of_css_ms = timer()->NowMs();
  FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_, NULL,
                        date_of_css_ms);
  // The metadata and cache entry is stale now. Fetch the content and serve it
  // out without rewriting. The background rewrite attempt will end up reusing
  // the old result due to revalidation, however.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(3, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  mock_url_fetcher()->set_timer(timer());
  mock_url_fetcher()->set_update_date_headers(true);
  SetCacheInvalidationTimestamp();
  date_of_css_ms = timer()->NowMs();

  // Having flushed cache, we are now back to serving the origin content.
  FetchAndCheckResponse(cache_css_url_, cache_body_, true, ttl_ms_,
                        NULL, date_of_css_ms);

  // Next time we'll serve optimized content.
  AdvanceTimeMs(ttl_ms_/2);
  ResetHeadersAndStats();
  int64 expected_ttl_ms = ttl_ms_ - (timer()->NowMs() - date_of_css_ms);
  FetchAndCheckResponse(cache_css_url_, "good:cf", true, expected_ttl_ms,
                        etag_, timer()->NowMs());
}

TEST_F(InPlaceRewriteContextTest, NonCacheableUrlNoRewriting) {
  Init();
  FetchAndCheckResponse(nocache_html_url_, nocache_body_, true, 0, NULL,
                        timer()->NowMs());
  // First fetch misses initial cache lookup, succeeds at fetch and we don't
  // insert into cache because it's not cacheable. Don't attempt to rewrite
  // this since its not cacheable.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

// Tests that with correct flags set, the uncacheable resource will be
// rewritten. Also checks, that resource will not be inserted.
TEST_F(InPlaceRewriteContextTest, NonCacheableUrlRewriting) {
  Init();

  // Modify options for our test.
  options()->ClearSignatureForTesting();
  options()->set_in_place_wait_for_optimized(true);
  options()->set_rewrite_uncacheable_resources(true);
  server_context()->ComputeSignature(options());

  // The ttl is just a value in proto, actual cacheable values will be checked
  // below.
  FetchAndCheckResponse(nocache_js_url_, StrCat(cache_body_, ":", "jm"),
                        true /* success */,
                        Timer::kYearMs /* ttl (ms) */,
                        etag_ /* etag */,
                        timer()->NowMs());

  // Shouldn't be cacheable at all.
  EXPECT_FALSE(response_headers_.IsBrowserCacheable());
  EXPECT_FALSE(response_headers_.IsProxyCacheable());

  // First fetch misses initial cache lookup, succeeds at fetch and we don't
  // insert into cache because it's not cacheable. But since flags are set to
  // rewrite uncacheable resources, JS rewriting should occur.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  // Should have been rewritten.
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(1, in_place_uncacheable_rewrites_->Get());
}

// Tests, that with correct flags set the private cacheable resource will be
// rewritten. Also checks, that the resource will not be cached.
TEST_F(InPlaceRewriteContextTest, PrivateCacheableUrlRewriting) {
  Init();

  // Modify options for our test.
  options()->ClearSignatureForTesting();
  options()->set_in_place_wait_for_optimized(true);
  options()->set_rewrite_uncacheable_resources(true);
  server_context()->ComputeSignature(options());

  // The ttl is just a value in proto, actual cacheable values will be checked
  // below.
  FetchAndCheckResponse(private_cache_js_url_, StrCat(cache_body_, ":", "jm"),
                        true /* success */,
                        1000 /* ttl (s) */,
                        etag_ /* etag */,
                        timer()->NowMs());
  // Should be cacheable.
  EXPECT_TRUE(response_headers_.IsBrowserCacheable());

  // But only in a private way.
  EXPECT_FALSE(response_headers_.IsProxyCacheable());

  // First fetch misses initial cache lookup, succeeds at fetch and we don't
  // insert into cache because it's not cacheable. But since flags are set to
  // rewrite uncacheable resources, JS rewriting should occur.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  // Should have been rewritten.
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(1, in_place_uncacheable_rewrites_->Get());
}


TEST_F(InPlaceRewriteContextTest, BadUrlNoRewriting) {
  Init();
  FetchAndCheckResponse(bad_url_, bad_body_, true, 0, NULL, start_time_ms());
  // First fetch misses initial cache lookup, succeeds at fetch and we don't
  // insert into cache because it's not cacheable. Don't attempt to rewrite
  // this since its not cacheable.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, PermanentRedirectNoRewriting) {
  options()->set_in_place_wait_for_optimized(true);
  Init();
  FetchAndCheckResponse(
      redirect_url_, redirect_body_, true /* expected_success */,
      36000 /* ttl (s) */, NULL /* etag */, start_time_ms());

  // Don't attempt to rewrite this since it's not a 200 response.
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(1, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, FetchFailedNoRewriting) {
  Init();
  FetchAndCheckResponse("http://www.notincache.com", "", false, 0, NULL,
                        start_time_ms());
  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
}

TEST_F(InPlaceRewriteContextTest, HandleResourceCreationFailure) {
  // Regression test. Trying to in-place optimize https resources with
  // a fetcher that didn't support https would fail to invoke the callbacks
  // and leak the rewrite driver.
  Init();
  factory()->mock_url_async_fetcher()->set_fetcher_supports_https(false);
  FetchAndCheckResponse("https://www.example.com", "", false, 0, NULL, 0);
}

TEST_F(InPlaceRewriteContextTest, ResponseHeaderMimeTypeUpdate) {
  options()->set_in_place_wait_for_optimized(true);
  Init();
  // We are going to rewrite a PNG image below. Assume it will be converted
  // to a JPEG.
  img_filter_->set_output_content_type(&kContentTypeJpeg);
  FetchAndCheckResponse(cache_png_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_STREQ(kContentTypeJpeg.mime_type(),
               response_headers_.Lookup1(HttpAttributes::kContentType));
}

TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserEncodingAndHeader) {
  options()->set_in_place_wait_for_optimized(true);
  set_optimize_for_browser(true);
  Init();

  // Image with correct extension in URL.
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_EQ(0, css_filter_->num_encode_user_agent());
  EXPECT_EQ(1, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // Image with no extension in URL.
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_jpg_no_extension_url_, "good:ic", true, ttl_ms_,
                        etag_, start_time_ms());
  EXPECT_EQ(1, css_filter_->num_encode_user_agent());
  EXPECT_EQ(1, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // CSS with correct extension in URL.
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_css_url_, "good:cf", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_EQ(1, css_filter_->num_encode_user_agent());
  EXPECT_EQ(0, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_STREQ(HttpAttributes::kUserAgent,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // HTML with correct extension in URL.
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_html_url_, "good", true, ttl_ms_,
                        original_etag_, start_time_ms());
  EXPECT_EQ(0, css_filter_->num_encode_user_agent());
  EXPECT_EQ(0, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));

  // Javascript with correct extension in URL.
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "good:jm", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_EQ(0, css_filter_->num_encode_user_agent());
  EXPECT_EQ(0, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));

  // Javascript with jpeg extension in URL.
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_jpg_extension_url_, "good:jm", true, ttl_ms_,
                        etag_, start_time_ms());
  EXPECT_EQ(0, css_filter_->num_encode_user_agent());
  EXPECT_EQ(1, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));

  // Bad content with unknown extension.
  ResetHeadersAndStats();
  FetchAndCheckResponse(bad_url_, bad_body_, true, 0, NULL, start_time_ms());
  EXPECT_EQ(1, css_filter_->num_encode_user_agent());
  EXPECT_EQ(1, img_filter_->num_encode_user_agent());
  EXPECT_EQ(0, js_filter_->num_encode_user_agent());
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
}

TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserRewriting) {
  // When in_place_wait_for_optimized is true, force_rewrite is set to true and
  // the nested RewriteContext will not check for rewritten content if input
  // is ready. Keep that in mind when checking lru_cache hits/misses.
  options()->set_in_place_wait_for_optimized(true);
  options()->set_private_not_vary_for_ie(true);
  set_optimize_for_browser(true);
  Init();

  // First fetch with kTestUserAgentWebP. This will miss everything (metadata
  // lookup, original content, and rewritten content).
  // Vary: Accept header should be added.
  ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
  SetAcceptWebp();
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());

  EXPECT_EQ(1, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(1, http_cache()->cache_misses()->Get());  // original
  EXPECT_EQ(2, http_cache()->cache_inserts()->Get());  // rewritten + original
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());  // + ipro-md
  EXPECT_EQ(4, lru_cache()->num_inserts());  // + ipro-md + md
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(0, oversized_stream_->Get());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // The second fetch uses a different user agent, kTestUserAgentNoWebP.
  // This will miss the metadata cache so it will start fetch input (cache hit)
  // and rewrite content (cache miss).
  // Vary: Accept header should be be added.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());  // original
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());  // rewritten
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());  // rewritten
  EXPECT_EQ(1, lru_cache()->num_hits());  // original
  EXPECT_EQ(1, lru_cache()->num_misses());  // ipro-md
  EXPECT_EQ(3, lru_cache()->num_inserts());  // + ipro-md + md
  EXPECT_EQ(1, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());
  EXPECT_EQ(0, oversized_stream_->Get());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // The third fetch uses an IE 9 user agent string, which should result in a
  // Cache-Control: private resource and no Vary header.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  ResetUserAgent(UserAgentMatcherTestBase::kIe9UserAgent);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  CheckWarmCache("no_webp_to_ie");
  EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));
  ConstStringStarVector cache_controls;
  EXPECT_TRUE(response_headers_.Lookup(
      HttpAttributes::kCacheControl, &cache_controls));
  ASSERT_EQ(2, cache_controls.size());
  EXPECT_STREQ(HttpAttributes::kPrivate, *cache_controls[1]);

  // Fetch again still with kTestUserAgentWebP, but omits the Accept:webp
  // header.  Metadata cache hits.  No input fetch and rewriting.
  // Vary: Accept header should be be added.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  CheckWarmCache("no_webp_without_accept");
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // Fetch another time, switching to just sending Accept: webp and using
  // kTestUserAgentNoWebP.  Metadata cache hits. No input fetch and rewriting.
  // Vary: User-Agent header should be added.
  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
  SetAcceptWebp();
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  CheckWarmCache("back_to_webp");
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));
}

TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserNoPrivateForIE) {
  // Similar to test above, but set private_not_vary_for_ie to false and omit
  // detailed checking of cache hit statistics, focusing just on a behavioral
  // test.
  options()->set_in_place_wait_for_optimized(true);
  options()->set_private_not_vary_for_ie(false);
  set_optimize_for_browser(true);
  Init();

  // First fetch with kTestUserAgentWebP.
  // Vary: Accept header should be added.
  ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
  SetAcceptWebp();
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // The second fetch uses a different user agent, kTestUserAgentNoWebP.
  // Vary: Accept header should be be added.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // The third fetch uses an IE 9 user agent string, which should *also* have a
  // Vary: Accept header since private_not_vary_for_ie == false.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  ResetUserAgent(UserAgentMatcherTestBase::kIe9UserAgent);
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));
}

TEST_F(InPlaceRewriteContextTest, AcceptHeaderMerging) {
  options()->set_in_place_wait_for_optimized(true);
  set_optimize_for_browser(true);
  Init();
  SetAcceptWebp();
  SetDriverRequestHeaders();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_STREQ(HttpAttributes::kAccept,
               response_headers_.Lookup1(HttpAttributes::kVary));

  // We don't actually optimize the Vary: * resource.  See
  // CachingHeaders::HasExplicitNoCacheDirective().  Inexplicably (?), we also
  // change its ttl to 0 in spite of incoming ttl headers.
  FetchAndCheckResponse(cache_jpg_vary_star_url_, "good", true, 0,
                        NULL, start_time_ms());
  EXPECT_STREQ("*", response_headers_.Lookup1(HttpAttributes::kVary));

  // TODO(jmaessen): Right now we're not properly passing through Vary: headers
  // from the fetched resource.  When jmarantz's pending change lands, we will
  // do so, and these tests should be re-enabled accordingly.  Note that I've
  // verified in gdb that we're actually handling pre-existing headers properly
  // (due to a duplicate call; luckily we're idempotent!).

  // FetchAndCheckResponse(cache_jpg_vary_ua_url_, "good:ic", true, ttl_ms_,
  //                       etag_, start_time_ms());
  // EXPECT_STREQ(HttpAttributes::kUserAgent,
  //              response_headers_.Lookup1(HttpAttributes::kVary));

  // FetchAndCheckResponse(cache_jpg_vary_origin_url_, "good:ic", true, ttl_ms_,
  //                       etag_, start_time_ms());
  // ConstStringStarVector accepts;
  // EXPECT_TRUE(response_headers_.Lookup(HttpAttributes::kVary, &accepts));
  // ASSERT_EQ(2, accepts.size());
  // EXPECT_STREQ("Origin", *accepts[0]);
  // EXPECT_STREQ(HttpAttributes::kAccept, *accepts[1]);
}

TEST_F(InPlaceRewriteContextTest, NoAcceptHeaderForLosslessOrAnimated) {
  // Make sure that InPlaceRewriteContext won't add "Vary: Accept" header to
  // an image optimized to WebP lossless or WebP animated. Note that we're using
  // FakeImageFilter in this test. If we use the real filter,
  // ImageRewriteFilter, an image will never be converted to WebP lossless nor
  // WebP animated, unless we're allowed to vary on user-agent.
  options()->set_in_place_wait_for_optimized(true);
  set_optimize_for_browser(true);
  Init();
  SetAcceptWebp();

  // First check lossless case.
  img_filter_->set_optimized_image_type(IMAGE_WEBP_LOSSLESS_OR_ALPHA);

  FetchAndCheckResponse(cache_png_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));

  // Now check animated case.
  img_filter_->set_optimized_image_type(IMAGE_WEBP_ANIMATED);
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_FALSE(response_headers_.Has(HttpAttributes::kVary));
}

TEST_F(InPlaceRewriteContextTest, OptimizeForBrowserNegative) {
  options()->set_in_place_wait_for_optimized(true);
  set_optimize_for_browser(false);
  Init();

  // Vary: User-Agent header should not be added no matter the user-agent.
  ResetUserAgent(UserAgentMatcher::kTestUserAgentWebP);
  SetAcceptWebp();
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_, etag_,
                        start_time_ms());
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));

  ResetHeadersAndStats();
  SetTimeMs((start_time_ms() + ttl_ms_/2));
  ResetUserAgent(UserAgentMatcher::kTestUserAgentNoWebP);
  FetchAndCheckResponse(cache_jpg_url_, "good:ic", true, ttl_ms_/2, etag_,
                        start_time_ms() + ttl_ms_/2);
  EXPECT_EQ(NULL, response_headers_.Lookup1(HttpAttributes::kVary));
}

TEST_F(InPlaceRewriteContextTest, LoadFromFile) {
  options()->file_load_policy()->Associate("http://www.example.com", "/test/");
  WriteFile("/test/cacheable.js", cache_body_ /*"   alert ( 'foo ')   "*/);

  Init();

  // TODO(jmarantz): currently we will not have caching headers on
  // file-input-resources so we default to the implicit cache TTL.
  // We should probably have a new config options for file-input
  // TTL for use with in-place.
  const int64 kIproFileTtl = RewriteOptions::kDefaultImplicitCacheTtlMs;
  FetchAndCheckResponse(cache_js_url_, cache_body_, true,
                        kIproFileTtl, NULL, start_time_ms());

  // First fetch misses initial cache lookup, succeeds at fetch and inserts
  // result into cache. Also, the resource gets rewritten and the rewritten
  // resource gets inserted into cache.
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(0, lru_cache()->num_hits());
  EXPECT_EQ(2, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  // Note that without file-input resources, we would expect that our
  // TTL would be reduced to ttl_ms_/2.  But it doesn't work like that
  // for files.  The TTL stays the same.
  ResetHeadersAndStats();
  SetTimeMs(start_time_ms() + ttl_ms_/2);
  FetchAndCheckResponse(cache_js_url_, "good:jm", true,
                        kIproFileTtl, etag_,
                        start_time_ms() + ttl_ms_/2);
  // Second fetch hits the metadata cache and the rewritten resource is served
  // out.
  CheckWarmCache("second_fetch_11");
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(1, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(0, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(2, lru_cache()->num_hits());
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(0, lru_cache()->num_inserts());
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(0, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  // Third fetch is the same exact deal.  The file hasn't actually
  // changed and the existing rewrite still is valid.  The metadata
  // cache does not go stale until the file is actually touched.
  AdvanceTimeMs(2 * ttl_ms_);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "good:jm", true,
                        kIproFileTtl, etag_, timer()->NowMs());
  CheckWarmCache("third_fetch");

  // OK let's now move time forward a little and touch the file
  // without changing it.  This results in a total reset back to
  // the original state.  It seems like we could read the file
  // and see if it's changed, but we wind up queuing up the asynchronous
  // rewrite.
  AdvanceTimeMs(1 * Timer::kSecondMs);
  WriteFile("/test/cacheable.js", cache_body_ /*"   alert ( 'foo ')   "*/);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, cache_body_, true,
                        kIproFileTtl, NULL, timer()->NowMs());
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(2, lru_cache()->num_hits());  // ipro-metadata, metadata
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());  // http, metadata, ipro-metadata
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  AdvanceTimeMs(1 * Timer::kSecondMs);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "good:jm", true,
                        kIproFileTtl, etag_, timer()->NowMs());
  CheckWarmCache("second_fetch_after_touch");

  // Now change the content.
  AdvanceTimeMs(1 * Timer::kSecondMs);
  WriteFile("/test/cacheable.js", "new_content");
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "new_content", true,
                        kIproFileTtl, NULL, timer()->NowMs());
  EXPECT_EQ(0, counting_url_async_fetcher()->fetch_count());
  EXPECT_EQ(0, http_cache()->cache_hits()->Get());
  EXPECT_EQ(0, http_cache()->cache_misses()->Get());
  EXPECT_EQ(1, http_cache()->cache_inserts()->Get());
  EXPECT_EQ(2, lru_cache()->num_hits());  // ipro-metadata, metadata
  EXPECT_EQ(0, lru_cache()->num_misses());
  EXPECT_EQ(3, lru_cache()->num_inserts());  // http, metadata, ipro-metadata
  EXPECT_EQ(0, img_filter_->num_rewrites());
  EXPECT_EQ(1, js_filter_->num_rewrites());
  EXPECT_EQ(0, css_filter_->num_rewrites());

  AdvanceTimeMs(1 * Timer::kSecondMs);
  ResetHeadersAndStats();
  FetchAndCheckResponse(cache_js_url_, "new_content:jm", true,
                        kIproFileTtl, etag_, timer()->NowMs());
  CheckWarmCache("second_fetch_after_mutation");
}

TEST_F(InPlaceRewriteContextTest, JsonWithJsContentTypeSucceeds) {
  Init();
  CheckCachingAndContentType(json_js_type_url_,
                             "application/javascript",
                             cache_body_,
                             RewriteOptions::kJavascriptMinId);
}

TEST_F(InPlaceRewriteContextTest, JsonWithJsonContentTypeSucceeds) {
  Init();
  CheckCachingAndContentType(json_json_type_url_,
                             "application/json",
                             cache_body_,
                             RewriteOptions::kJavascriptMinId);
}

TEST_F(InPlaceRewriteContextTest, JsonWithJsonContentTypeSynonymSucceeds) {
  Init();
  CheckCachingAndContentType(json_json_type_synonym_url_,
                             "application/x-json",
                             cache_body_,
                             RewriteOptions::kJavascriptMinId);
}

}  // namespace

}  // namespace net_instaweb