src/pagespeed/system/system_caches_test.cc

/*
 * Copyright 2013 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Author: morlovich@google.com (Maksim Orlovich)
// Unit tests for common cache configuration code.

#include "pagespeed/system/system_caches.h"

#include <cstdlib>
#include <vector>
#include <sys/socket.h>

#include "apr_poll.h"
#include "apr_version.h"
#include "apr_pools.h"
#include "apr_thread_proc.h"
#include "base/logging.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/http_value.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/rewriter/public/custom_rewrite_test_base.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_test_base.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
#include "pagespeed/system/admin_site.h"
#include "pagespeed/system/apr_mem_cache.h"
#include "pagespeed/system/system_cache_path.h"
#include "pagespeed/system/system_rewrite_options.h"
#include "pagespeed/system/system_server_context.h"
#include "net/instaweb/util/public/cache_property_store.h"
#include "net/instaweb/util/public/property_cache.h"
#include "net/instaweb/util/public/property_store.h"
#include "pagespeed/kernel/base/abstract_shared_mem.h"
#include "pagespeed/kernel/base/gtest.h"
#include "pagespeed/kernel/base/md5_hasher.h"
#include "pagespeed/kernel/base/mem_file_system.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/mock_message_handler.h"
#include "pagespeed/kernel/base/named_lock_manager.h"
#include "pagespeed/kernel/base/named_lock_tester.h"
#include "pagespeed/kernel/base/null_shared_mem.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/shared_string.h"
#include "pagespeed/kernel/base/stack_buffer.h"
#include "pagespeed/kernel/base/stl_util.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/cache/async_cache.h"
#include "pagespeed/kernel/cache/cache_batcher.h"
#include "pagespeed/kernel/cache/cache_interface.h"
#include "pagespeed/kernel/cache/cache_spammer.h"
#include "pagespeed/kernel/cache/cache_stats.h"
#include "pagespeed/kernel/cache/cache_test_base.h"
#include "pagespeed/kernel/cache/compressed_cache.h"
#include "pagespeed/kernel/cache/fallback_cache.h"
#include "pagespeed/kernel/cache/file_cache.h"
#include "pagespeed/kernel/cache/lru_cache.h"
#include "pagespeed/kernel/cache/threadsafe_cache.h"
#include "pagespeed/kernel/cache/write_through_cache.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/sharedmem/inprocess_shared_mem.h"
#include "pagespeed/kernel/sharedmem/shared_mem_lock_manager.h"
#include "pagespeed/kernel/thread/scheduler_based_abstract_lock.h"
#include "pagespeed/kernel/thread/worker_test_base.h"
#include "pagespeed/kernel/util/file_system_lock_manager.h"
#include "pagespeed/kernel/util/platform.h"
#include "pagespeed/kernel/util/simple_random.h"

namespace net_instaweb {

namespace {

const char kCachePath[] = "/mem/path/";
const char kAltCachePath[] = "/mem/path_alt/";
const char kAltCachePath2[] = "/mem/path_alt2/";
const char kUrl1[] = "http://example.com/a.css";
const char kUrl2[] = "http://example.com/b.css";

class SystemServerContextNoProxyHtml : public SystemServerContext {
 public:
  explicit SystemServerContextNoProxyHtml(RewriteDriverFactory* factory)
      : SystemServerContext(factory, "fake_hostname", 80 /* fake port */) {
  }

  virtual bool ProxiesHtml() const { return false; }

 private:
  DISALLOW_COPY_AND_ASSIGN(SystemServerContextNoProxyHtml);
};

class SystemCachesTest : public CustomRewriteTestBase<SystemRewriteOptions> {
 public:
  void PurgeDone(bool success) {
    purge_done_ = true;
    purge_success_ = success;
  }

  void MemCacheStressTestHelper(bool do_deletes) {
    options_->set_file_cache_path(kCachePath);
    options_->set_use_shared_mem_locking(false);
    options_->set_lru_cache_kb_per_process(0);
    options_->set_memcached_servers(MemCachedServerSpec());
    options_->set_memcached_threads(1);
    options_->set_default_shared_memory_cache_kb(0);
    options_->set_compress_metadata_cache(false);
    PrepareWithConfig(options_.get());
    scoped_ptr<ServerContext> server_context(
        SetupServerContext(options_.release()));
    CacheInterface* cache = server_context->metadata_cache();
    SimpleRandom random(new NullMutex);
    GoogleString value = random.GenerateHighEntropyString(20000);
    CacheSpammer::RunTests(4 /* num_threads */,
                           200 /* iters */,
                           200 /* inserts */,
                           false /* expecting_evictions */,
                           do_deletes,
                           value.c_str(),
                           cache,
                           thread_system_.get());
  }

  apr_size_t get_fake_memcached_port() {
    return fake_memcached_port_;
  }

  void set_fake_memcached_port(apr_size_t port) {
    fake_memcached_port_ = port;
  }

 protected:
  static const int kThreadLimit = 3;
  static const int kUsableMetadataCacheSize = 8 * 1024;
  apr_size_t fake_memcached_port_;

  // Helper that blocks for async cache lookups.
  class BlockingCallback : public CacheInterface::Callback {
   public:
    explicit BlockingCallback(ThreadSystem* threads)
        : sync_(threads), result_(CacheInterface::kNotFound) {}

    CacheInterface::KeyState result() const { return result_; }
    GoogleString value() const { return value_; }

    void Block() {
      sync_.Wait();
    }

   protected:
    virtual void Done(CacheInterface::KeyState state) {
      result_ = state;
      CacheInterface::Callback::value()->Value().CopyToString(&value_);
      sync_.Notify();
    }

   private:
    WorkerTestBase::SyncPoint sync_;
    CacheInterface::KeyState result_;
    GoogleString value_;
  };

  // Helper that blocks for async HTTP cache lookups.
  class HTTPBlockingCallback : public HTTPCache::Callback {
   public:
    explicit HTTPBlockingCallback(ThreadSystem* threads)
        : Callback(RequestContext::NewTestRequestContext(threads)),
          sync_(threads) {}

    HTTPCache::FindResult result() const { return result_; }
    GoogleString value() const { return value_; }

    void Block() {
      sync_.Wait();
    }

    // RespectVary not relevant in this context.
    virtual ResponseHeaders::VaryOption RespectVaryOnResources() const {
      return ResponseHeaders::kRespectVaryOnResources;
    }

   protected:
    virtual void Done(HTTPCache::FindResult state) {
      result_ = state;
      if (state.status == HTTPCache::kFound) {
        StringPiece contents;
        http_value()->ExtractContents(&contents);
        contents.CopyToString(&value_);
      }
      sync_.Notify();
    }

    virtual bool IsCacheValid(const GoogleString& key,
                              const ResponseHeaders& headers) {
      return true;
    }

   private:
    WorkerTestBase::SyncPoint sync_;
    HTTPCache::FindResult result_;
    GoogleString value_;
  };

  class FakeMemcacheServerThread : public ThreadSystem::Thread {
   public:
    FakeMemcacheServerThread(SystemCachesTest* owner)
        : Thread(owner->thread_system_.get(), "fake_memcache",
                 ThreadSystem::kJoinable),
          owner_(owner),
          mutex_(owner->thread_system_->NewMutex()),
          notify_port_(mutex_->NewCondvar()) {}
    virtual void Run() {
      static const char kMessage[] = "blah\n";
      apr_size_t message_size = STATIC_STRLEN(kMessage);
      char buf[kStackBufferSize];
      apr_size_t size = sizeof(buf) - 1;
      apr_socket_t* accepted_socket = NULL;
      apr_status_t status = apr_pool_create(&pool_, NULL);
      ASSERT_EQ(APR_SUCCESS, status);
      CreateSocketHelper();
      // Bind to an open port randomly.
      // Blocks on connection from main thread.
      apr_socket_bind(listening_socket_, sock_addr_);
      apr_socket_listen(listening_socket_, SOMAXCONN);
      apr_socket_accept(&accepted_socket, listening_socket_, pool_);
      apr_socket_recv(accepted_socket, buf, &size);
      apr_socket_send(accepted_socket, kMessage, &message_size);
      apr_pool_destroy(pool_);
    }

    void WaitForReady() {
      ScopedMutex hold_lock(mutex_.get());
      while (owner_->get_fake_memcached_port() == 0) {
        notify_port_->Wait();
      }
    }

   private:
    void CreateSocketHelper() {
      ScopedMutex hold_lock(mutex_.get());
      SimpleRandom simple_random(owner_->thread_system_.get()->NewMutex());
      apr_size_t port_num;
      const apr_int32_t kFamily = APR_INET;
      sock_addr_ = NULL;
      listening_socket_ = NULL;
      apr_status_t status;
      // Bind to an open port randomly.
      do {
        // Set port_num to 1024 - 65535
        port_num = (simple_random.Next() % 64511) + 1024;
        status =
            apr_sockaddr_info_get(&sock_addr_, 0, kFamily, port_num, 0, pool_);
        if (status == APR_SUCCESS) {
          status = apr_socket_create(&listening_socket_, sock_addr_->family,
                                      SOCK_STREAM, APR_PROTO_TCP, pool_);
        }
      } while (status != APR_SUCCESS);
      owner_->set_fake_memcached_port(port_num);
      notify_port_->Signal();
    }

    SystemCachesTest* owner_;
    scoped_ptr<ThreadSystem::CondvarCapableMutex> mutex_;
    scoped_ptr<ThreadSystem::Condvar> notify_port_;
    apr_pool_t* pool_;
    apr_sockaddr_t* sock_addr_;
    apr_socket_t* listening_socket_;
  };

  SystemCachesTest()
      : thread_system_(Platform::CreateThreadSystem()),
        options_(new SystemRewriteOptions(thread_system_.get())),
        purge_done_(false),
        purge_success_(false) {
    shared_mem_.reset(new InProcessSharedMem(thread_system_.get()));
    factory_->set_hasher(new MD5Hasher());
    Statistics* stats = factory()->statistics();
    SystemCaches::InitStats(stats);
    SystemServerContext::InitStats(stats);
    CacheStats::InitStats(
        PropertyCache::GetStatsPrefix(RewriteDriver::kBeaconCohort),
        stats);
    CacheStats::InitStats(
        PropertyCache::GetStatsPrefix(RewriteDriver::kDomCohort),
        stats);
  }

  virtual void SetUp() {
    // TODO(jcrowell) factor out apr_initialize/terminate to share in static
    // constructor similar to rewrite_test_base.cc.
    apr_initialize();
    atexit(apr_terminate);
    system_caches_.reset(
        new SystemCaches(factory(), shared_mem_.get(), kThreadLimit));
    CustomRewriteTestBase<SystemRewriteOptions>::SetUp();
  }

  virtual void TearDown() {
    system_caches_->StopCacheActivity();
    RewriteTestBase::TearDown();
    system_caches_->ShutDown(factory()->message_handler());
  }

  void PrepareWithConfig(SystemRewriteOptions* config) {
    system_caches_->RegisterConfig(config);
    system_caches_->RootInit();
    // pretend we fork here.
    system_caches_->ChildInit();
  }

  // Takes ownership of config.
  SystemServerContext* SetupServerContext(SystemRewriteOptions* config) {
    scoped_ptr<SystemServerContext> server_context(
        new SystemServerContextNoProxyHtml(factory()));
    server_context->reset_global_options(config);
    server_context->set_statistics(factory()->statistics());
    server_context->set_timer(factory()->timer());
    system_caches_->SetupCaches(server_context.get(),
                                true /* enable_property_cache */);

    // Sanity-check that the two caches work.
    TestPut(server_context->metadata_cache(), "a", "b");
    TestGet(server_context->metadata_cache(), "a",
            CacheInterface::kAvailable, "b");

    TestHttpPut(server_context->http_cache(), "http://www.example.com",
                "fragment", "a");
    TestHttpGet(server_context->http_cache(), "http://www.example.com",
                "fragment", kFoundResult, "a");
    return server_context.release();
  }

  void TestPut(CacheInterface* cache, StringPiece key, StringPiece value) {
    GoogleString value_copy;
    value.CopyToString(&value_copy);
    SharedString shared_value;
    shared_value.SwapWithString(&value_copy);
    cache->Put(key.as_string(), &shared_value);
  }

  void TestGet(CacheInterface* cache, StringPiece key,
               CacheInterface::KeyState expected_result,
               StringPiece expected_value) {
    BlockingCallback callback(thread_system_.get());
    cache->Get(key.as_string(), &callback);
    callback.Block();
    EXPECT_EQ(expected_result, callback.result());
    EXPECT_EQ(expected_value, callback.value());
  }

  void TestHttpPut(HTTPCache* cache, StringPiece key,
                   StringPiece fragment, StringPiece value) {
    ResponseHeaders headers;
    SetDefaultLongCacheHeaders(&kContentTypeText, &headers);
    cache->Put(key.as_string(), fragment.as_string(),
               RequestHeaders::Properties(),
               ResponseHeaders::kRespectVaryOnResources, &headers, value,
               factory()->message_handler());
  }

  void TestHttpGet(HTTPCache* cache, StringPiece key, StringPiece fragment,
                   HTTPCache::FindResult expected_state,
                   StringPiece expected_value) {
    HTTPBlockingCallback callback(thread_system_.get());
    cache->Find(key.as_string(), fragment.as_string(),
                factory()->message_handler(), &callback);
    callback.Block();
    EXPECT_EQ(expected_state, callback.result());
    EXPECT_EQ(expected_value, callback.value());
  }

  // Returns empty string if not enabled. Tests should exit in that case.
  GoogleString MemCachedServerSpec() {
    if (server_spec_.empty()) {
      // This matches the logic in apr_mem_cache_test.
      const char* port_string = getenv("MEMCACHED_PORT");
      if (port_string == NULL) {
        LOG(ERROR) << "AprMemCache tests are skipped because env var "
                   << "$MEMCACHED_PORT is not set.  Set that to the port "
                   << "number where memcached is running to enable the "
                   << "tests.  See install/run_program_with_memcached.sh";
        // Does not fail the test.
        return "";
      }
      server_spec_ = StrCat("localhost:", port_string);
    }
    return server_spec_;
  }

  // Unwraps any wrapper cache objects.
  CacheInterface* SkipWrappers(CacheInterface* in) {
    CacheInterface* backend = in->Backend();
    if (backend != in) {
      return SkipWrappers(backend);
    }
    return in;
  }

  void TestBasicMemCacheAndNoLru(int num_threads_specified,
                                 int num_threads_expected) {
    if (MemCachedServerSpec().empty()) {
      return;
    }

    options_->set_file_cache_path(kCachePath);
    options_->set_use_shared_mem_locking(false);
    options_->set_lru_cache_kb_per_process(0);
    options_->set_memcached_servers(MemCachedServerSpec());
    options_->set_memcached_threads(num_threads_specified);
    options_->set_default_shared_memory_cache_kb(0);
    PrepareWithConfig(options_.get());

    scoped_ptr<ServerContext> server_context(
        SetupServerContext(options_.release()));

    GoogleString mem_cache;
    if (num_threads_expected == 0) {
      mem_cache = Batcher(Stats(SystemCaches::kMemcachedAsync,
                                AprMemCache::FormatName()),
                          1, 1000);
    } else {
      mem_cache = Batcher(AsyncMemCacheWithStats(), num_threads_expected, 1000);
    }

    EXPECT_STREQ(
        Compressed(Fallback(mem_cache, Stats("file_cache", FileCacheName()))),
        server_context->metadata_cache()->Name());
    EXPECT_STREQ(
        HttpCache(
            Fallback(mem_cache, Stats("file_cache", FileCacheName()))),
        server_context->http_cache()->Name());
    ASSERT_TRUE(server_context->filesystem_metadata_cache() != NULL);

    // That the code that queries the FSMDC from the validator in RewriteContext
    // does a Get and needs the response to be available inline.
    EXPECT_TRUE(server_context->filesystem_metadata_cache()->IsBlocking());
    EXPECT_STREQ(
        Fallback(BlockingMemCacheWithStats(), FileCacheWithStats()),
        server_context->filesystem_metadata_cache()->Name());
  }

  // Wrapper functions to format expected cache descriptor strings with
  // concise function calls exposing the cache structure via normal code
  // indentation.
  GoogleString WriteThrough(StringPiece l1, StringPiece l2) {
    return WriteThroughCache::FormatName(l1, l2);
  }

  GoogleString HttpCache(StringPiece cache) {
    return HTTPCache::FormatName(cache);
  }

  GoogleString Fallback(StringPiece small, StringPiece large) {
    return FallbackCache::FormatName(small, large);
  }

  GoogleString Batcher(StringPiece cache, int parallel, int max) {
    return CacheBatcher::FormatName(cache, parallel, max);
  }

  GoogleString Stats(StringPiece prefix, StringPiece cache) {
    return CacheStats::FormatName(prefix, cache);
  }

  GoogleString ThreadsafeLRU() {
    return ThreadsafeCache::FormatName(LRUCache::FormatName());
  }

  GoogleString FileCacheName() { return FileCache::FormatName(); }

  GoogleString AsyncMemCacheWithStats() {
    return Stats(SystemCaches::kMemcachedAsync,
                 AsyncCache::FormatName(AprMemCache::FormatName()));
  }

  GoogleString BlockingMemCacheWithStats() {
    return Stats(SystemCaches::kMemcachedBlocking, AprMemCache::FormatName());
  }

  GoogleString FileCacheWithStats() {
    return Stats("file_cache", FileCacheName());
  }

  GoogleString Pcache(StringPiece cache) {
    return CachePropertyStore::FormatName2(
        RewriteDriver::kBeaconCohort,
        Stats(PropertyCache::GetStatsPrefix(RewriteDriver::kBeaconCohort),
              cache),
        RewriteDriver::kDomCohort,
        Stats(PropertyCache::GetStatsPrefix(RewriteDriver::kDomCohort),
              cache));
  }

  GoogleString Compressed(StringPiece cache) {
    return CompressedCache::FormatName(cache);
  }

  SystemServerContext* PopulateCacheForPurgeTest() {
    options_->set_file_cache_path(kCachePath);
    SystemRewriteOptions* options = options_.get();
    PrepareWithConfig(options);
    system_server_context_.reset(SetupServerContext(options_.release()));
    HTTPCache* http_cache = system_server_context_->http_cache();
    MessageHandler* handler = message_handler();
    system_server_context_->set_message_handler(handler);
    ResponseHeaders headers;
    SetDefaultLongCacheHeaders(&kContentTypeText, &headers);
    headers.ComputeCaching();
    RequestHeaders::Properties req_properties;
    http_cache->Put(kUrl1, rewrite_driver_->CacheFragment(), req_properties,
                    ResponseHeaders::kRespectVaryOnResources,
                    &headers, "a value", handler);
    http_cache->Put(kUrl2, rewrite_driver_->CacheFragment(), req_properties,
                    ResponseHeaders::kRespectVaryOnResources,
                    &headers, "b value", handler);
    AdvanceTimeMs(1000);
    HTTPValue value;

    // As expected, both kUrl1 and kUrl2 are valid after Put.
    EXPECT_EQ(kFoundResult, HttpBlockingFindWithOptions(
        options, kUrl1, http_cache, &value, &headers));
    EXPECT_EQ(kFoundResult, HttpBlockingFindWithOptions(
        options, kUrl2, http_cache, &value, &headers));
    AdvanceTimeMs(1000);
    return system_server_context_.get();
  }

  scoped_ptr<ThreadSystem> thread_system_;
  scoped_ptr<AbstractSharedMem> shared_mem_;
  scoped_ptr<SystemCaches> system_caches_;
  scoped_ptr<SystemRewriteOptions> options_;
  scoped_ptr<SystemServerContext> system_server_context_;
  bool purge_done_;
  bool purge_success_;

 private:
  GoogleString server_spec_;  // Set lazily by MemCachedServerSpec()
};

TEST_F(SystemCachesTest, BasicFileAndLruCache) {
  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(Compressed(WriteThrough(Stats("lru_cache", ThreadsafeLRU()),
                                       FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(
          WriteThrough(
              Stats("lru_cache", ThreadsafeLRU()),
              FileCacheWithStats())),
      server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, BasicFileOnlyCache) {
  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(Compressed(FileCacheWithStats()),
               server_context->metadata_cache()->Name());
  EXPECT_STREQ(HttpCache(FileCacheWithStats()),
               server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, UnusableShmAndLru) {
  // Test that we properly fallback when we can't create the shm cache
  // due to too small a size given.
  GoogleString error_msg;
  EXPECT_FALSE(
      system_caches_->CreateShmMetadataCache(kCachePath, 10, &error_msg));
  EXPECT_STREQ("Shared memory cache unusably small.", error_msg);

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(Compressed(WriteThrough(Stats("lru_cache", ThreadsafeLRU()),
                                       FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          FileCacheWithStats())),
      server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, BasicShmAndLru) {
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // We don't use the LRU when shm cache is on.
  EXPECT_STREQ(Compressed(Fallback(Stats("shm_cache", "SharedMemCache<64>"),
                                   FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  // HTTP cache is unaffected.
  EXPECT_STREQ(
      HttpCache(WriteThrough(Stats("lru_cache", ThreadsafeLRU()),
                             FileCacheWithStats())),
      server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, BasicShmAndNoLru) {
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // We don't use the LRU when shm cache is on.
  EXPECT_STREQ(Compressed(Fallback(Stats("shm_cache", "SharedMemCache<64>"),
                                   FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  // HTTP cache is unaffected.
  EXPECT_STREQ(HttpCache(FileCacheWithStats()),
               server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, DoubleShmCreate) {
  // Proper error message on two creation attempts for the same name.
  GoogleString error_msg;
  EXPECT_TRUE(
      system_caches_->CreateShmMetadataCache(kCachePath,
                                             kUsableMetadataCacheSize,
                                             &error_msg));
  EXPECT_FALSE(
      system_caches_->CreateShmMetadataCache(kCachePath,
                                             kUsableMetadataCacheSize,
                                             &error_msg));
  EXPECT_STREQ(StrCat("Cache named ", kCachePath, " already exists."),
               error_msg);

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // We don't use the LRU when shm cache is on.
  EXPECT_STREQ(Compressed(Fallback(Stats("shm_cache", "SharedMemCache<64>"),
                                   FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  // HTTP cache is unaffected.
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          FileCacheWithStats())),
      server_context->http_cache()->Name());
  EXPECT_TRUE(server_context->filesystem_metadata_cache() == NULL);
}

TEST_F(SystemCachesTest, BasicMemCachedAndLru) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  options_->set_memcached_servers(MemCachedServerSpec());
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(Compressed(
      WriteThrough(Stats("lru_cache", ThreadsafeLRU()),
                   Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                            FileCacheWithStats()))),
               server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                           FileCacheWithStats()))),
      server_context->http_cache()->Name());
  ASSERT_TRUE(server_context->filesystem_metadata_cache() != NULL);
  EXPECT_TRUE(server_context->filesystem_metadata_cache()->IsBlocking());
  EXPECT_STREQ(
      Fallback(BlockingMemCacheWithStats(), FileCacheWithStats()),
      server_context->filesystem_metadata_cache()->Name());
}

TEST_F(SystemCachesTest, BasicMemCachedAndNoLru_0_Threads) {
  TestBasicMemCacheAndNoLru(0, 0);
}

TEST_F(SystemCachesTest, BasicMemCachedAndNoLru_1_Thread) {
  TestBasicMemCacheAndNoLru(1, 1);
}

TEST_F(SystemCachesTest, BasicMemCachedAndNoLru_2_Threads) {
  TestBasicMemCacheAndNoLru(2, 1);  // Clamp to 1.
}

TEST_F(SystemCachesTest, BasicMemCachedLruShm) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  options_->set_memcached_servers(MemCachedServerSpec());
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // For metadata, we fallback to memcached behind shmcache.
  EXPECT_STREQ(
      Compressed(WriteThrough(
          Stats("shm_cache", SharedMemCache<64>::FormatName()),
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                   FileCacheWithStats()))),
      server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                             FileCacheWithStats()))),
      server_context->http_cache()->Name());
}

TEST_F(SystemCachesTest, BasicMemCachedShmNoLru) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  options_->set_memcached_servers(MemCachedServerSpec());
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(
      Compressed(
          WriteThrough(
              Stats("shm_cache", "SharedMemCache<64>"),
              Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                       FileCacheWithStats()))),
      server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                   FileCacheWithStats())),
      server_context->http_cache()->Name());
  ASSERT_TRUE(server_context->filesystem_metadata_cache() != NULL);
  EXPECT_TRUE(server_context->filesystem_metadata_cache()->IsBlocking());
  EXPECT_STREQ(
      Stats("shm_cache", "SharedMemCache<64>"),
      server_context->filesystem_metadata_cache()->Name());
}

TEST_F(SystemCachesTest, BasicFileLockManager) {
  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());
  NamedLockManager* named_locks = system_caches_->GetLockManager(
      options_.get());
  EXPECT_TRUE(named_locks != NULL);
  EXPECT_TRUE(dynamic_cast<FileSystemLockManager*>(named_locks) != NULL);
}

TEST_F(SystemCachesTest, BasicShmLockManager) {
  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(true);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());
  NamedLockManager* named_locks = system_caches_->GetLockManager(
      options_.get());
  EXPECT_TRUE(named_locks != NULL);
  EXPECT_TRUE(dynamic_cast<SharedMemLockManager*>(named_locks) != NULL);
}

TEST_F(SystemCachesTest, FileShare) {
  // [0], [1], share path, [2] doesn't.
  std::vector<SystemRewriteOptions*> configs;
  for (int i = 0; i < 3; ++i) {
    SystemRewriteOptions* config = options_->NewOptions();
    config->set_file_cache_path((i == 2) ? kCachePath : kAltCachePath);
    config->set_default_shared_memory_cache_kb(0);
    system_caches_->RegisterConfig(config);
    configs.push_back(config);
  }
  system_caches_->RootInit();
  // pretend we fork here.
  system_caches_->ChildInit();

  std::vector<ServerContext*> servers;
  for (int i = 0; i < 3; ++i) {
    servers.push_back(SetupServerContext(configs[i]));
  }

  TestPut(servers[0]->metadata_cache(), "b", "value");
  TestGet(servers[0]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[1]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[2]->metadata_cache(), "b", CacheInterface::kNotFound, "");

  TestHttpPut(servers[0]->http_cache(), "http://b.org", "fragment", "value");
  TestHttpGet(servers[0]->http_cache(), "http://b.org", "fragment",
              kFoundResult, "value");
  TestHttpGet(servers[1]->http_cache(), "http://b.org", "fragment",
              kFoundResult, "value");
  TestHttpGet(servers[2]->http_cache(), "http://b.org", "fragment",
              kNotFoundResult, "");

  // Lock managers have similar sharing semantics
  scoped_ptr<NamedLock> lock0(
      system_caches_->GetLockManager(configs[0])->CreateNamedLock("a"));
  scoped_ptr<NamedLock> lock1(
      system_caches_->GetLockManager(configs[1])->CreateNamedLock("a"));
  scoped_ptr<NamedLock> lock2(
      system_caches_->GetLockManager(configs[2])->CreateNamedLock("a"));
  NamedLockTester tester(thread_system_.get());
  EXPECT_TRUE(tester.TryLock(lock0.get()));
  EXPECT_FALSE(tester.TryLock(lock1.get()));
  EXPECT_TRUE(tester.TryLock(lock2.get()));
  lock0->Unlock();
  EXPECT_TRUE(tester.TryLock(lock1.get()));

  STLDeleteElements(&servers);
}

TEST_F(SystemCachesTest, ShmShare) {
  // For SHM metadata cache, sharing is based on explicit segment names/
  // [0], [1], share, [2] doesn't.
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kAltCachePath, kUsableMetadataCacheSize, &error_msg));

  std::vector<SystemRewriteOptions*> configs;
  for (int i = 0; i < 3; ++i) {
    SystemRewriteOptions* config = options_->NewOptions();
    config->set_file_cache_path((i == 2) ? kAltCachePath : kCachePath);
    system_caches_->RegisterConfig(config);
    configs.push_back(config);
  }

  system_caches_->RootInit();
  // pretend we fork here.
  system_caches_->ChildInit();

  std::vector<ServerContext*> servers;
  for (int i = 0; i < 3; ++i) {
    servers.push_back(SetupServerContext(configs[i]));
    EXPECT_STREQ(Compressed(Fallback(Stats("shm_cache", "SharedMemCache<64>"),
                                     FileCacheWithStats())),
                 servers[i]->metadata_cache()->Name());
  }

  // This is only about metadata cache.
  TestPut(servers[0]->metadata_cache(), "b", "value");
  TestGet(servers[0]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[1]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[2]->metadata_cache(), "b", CacheInterface::kNotFound, "");

  STLDeleteElements(&servers);
}

TEST_F(SystemCachesTest, ShmDefault) {
  // Unless a cache is explicitly defined or the default is disabled with
  // set_default_shared_memory_cache_kb(0), use the default.  Unlike explicitly
  // configured shared memory caches, default ones write through to an L2 (file
  // or memcache).
  //
  // [0] and [1] share the default, [2] has one separately configured.  All
  // three have different file cache paths.
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kAltCachePath2, kUsableMetadataCacheSize, &error_msg));

  std::vector<SystemRewriteOptions*> configs;
  const char* kPaths[] = {kCachePath, kAltCachePath, kAltCachePath2};
  for (int i = 0; i < 3; ++i) {
    SystemRewriteOptions* config = options_->NewOptions();
    config->set_file_cache_path(kPaths[i]);
    system_caches_->RegisterConfig(config);
    configs.push_back(config);
  }

  // No shm metadata cache was created for [0]'s kCachePath or [1]'s
  // kAltCachePath, only [2]'s kAltCachePath2.  So [0] and [1] will share the
  // default.

  system_caches_->RootInit();
  // pretend we fork here.
  system_caches_->ChildInit();

  std::vector<ServerContext*> servers;
  for (int i = 0; i < 3; ++i) {
    servers.push_back(SetupServerContext(configs[i]));
  }
  EXPECT_STREQ(Compressed(WriteThrough(Stats("shm_cache", "SharedMemCache<64>"),
                                       FileCacheWithStats())),
               servers[0]->metadata_cache()->Name());
  EXPECT_STREQ(Compressed(WriteThrough(Stats("shm_cache", "SharedMemCache<64>"),
                                       FileCacheWithStats())),
               servers[1]->metadata_cache()->Name());
  EXPECT_STREQ(Compressed(Fallback(Stats("shm_cache", "SharedMemCache<64>"),
                                   FileCacheWithStats())),
               servers[2]->metadata_cache()->Name());


  // This is only about metadata cache.
  TestPut(servers[0]->metadata_cache(), "b", "value");
  TestGet(servers[0]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[1]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[2]->metadata_cache(), "b", CacheInterface::kNotFound, "");

  STLDeleteElements(&servers);
}

TEST_F(SystemCachesTest, MemCachedShare) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  // Just share 3 memcached clients for the same server (so we don't
  // need 2 servers for the test)

  std::vector<SystemRewriteOptions*> configs;
  for (int i = 0; i < 3; ++i) {
    SystemRewriteOptions* config = options_->NewOptions();
    config->set_file_cache_path(kCachePath);
    config->set_memcached_servers(MemCachedServerSpec());
    config->set_default_shared_memory_cache_kb(0);
    system_caches_->RegisterConfig(config);
    configs.push_back(config);
  }

  system_caches_->RootInit();
  // pretend we fork here.
  system_caches_->ChildInit();

  std::vector<ServerContext*> servers;
  for (int i = 0; i < 3; ++i) {
    servers.push_back(SetupServerContext(configs[i]));
    EXPECT_STREQ(
        Compressed(Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                            FileCacheWithStats())),
        servers[i]->metadata_cache()->Name());

    EXPECT_STREQ(Pcache(Compressed(Fallback(BlockingMemCacheWithStats(),
                                            FileCacheWithStats()))),
                 servers[i]->page_property_cache()->property_store()->Name());
  }

  // Metadata + HTTP cache will end up shared
  TestPut(servers[0]->metadata_cache(), "b", "value");
  TestGet(servers[0]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[1]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");
  TestGet(servers[2]->metadata_cache(), "b",
          CacheInterface::kAvailable, "value");

  TestHttpPut(servers[0]->http_cache(), "http://b.org", "fragment", "value");
  TestHttpGet(servers[0]->http_cache(), "http://b.org", "fragment",
              kFoundResult, "value");
  TestHttpGet(servers[1]->http_cache(), "http://b.org", "fragment",
              kFoundResult, "value");
  TestHttpGet(servers[2]->http_cache(), "http://b.org", "fragment",
              kFoundResult, "value");

  STLDeleteElements(&servers);
}

TEST_F(SystemCachesTest, FileCacheSettings) {
  // Make sure we apply the various file cache settings right.
  options_->set_file_cache_path(kCachePath);
  options_->set_file_cache_clean_interval_ms(3 * Timer::kHourMs);
  options_->set_file_cache_clean_size_kb(1024);
  options_->set_file_cache_clean_inode_limit(50000);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(Compressed(FileCacheWithStats()),
               server_context->metadata_cache()->Name());
  EXPECT_STREQ(HttpCache(FileCacheWithStats()),
               server_context->http_cache()->Name());
  EXPECT_STREQ(Pcache(Compressed(FileCacheWithStats())),
               server_context->page_property_cache()->property_store()->Name());

  FileCache* file_cache = dynamic_cast<FileCache*>(
      SkipWrappers(server_context->metadata_cache()));
  ASSERT_TRUE(file_cache != NULL);
  EXPECT_EQ(kCachePath, file_cache->path());
  EXPECT_EQ(3 * Timer::kHourMs, file_cache->cache_policy()->clean_interval_ms);
  // Note: this is in bytes, the setting is in kb.
  EXPECT_EQ(1024*1024, file_cache->cache_policy()->target_size_bytes);
  EXPECT_EQ(50000, file_cache->cache_policy()->target_inode_count);
  EXPECT_TRUE(file_cache->worker() != NULL);
}

TEST_F(SystemCachesTest, LruCacheSettings) {
  // Test that we apply LRU cache settings right.
  options_->set_file_cache_path(kCachePath);
  options_->set_lru_cache_kb_per_process(1024);
  options_->set_lru_cache_byte_limit(500);
  options_->set_default_shared_memory_cache_kb(0);
  PrepareWithConfig(options_.get());
  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));

  WriteThroughCache* write_through = dynamic_cast<WriteThroughCache*>(
      SkipWrappers(server_context->metadata_cache()));
  ASSERT_TRUE(write_through != NULL);
  EXPECT_EQ(500, write_through->cache1_limit());

  LRUCache* lru_cache = dynamic_cast<LRUCache*>(
      SkipWrappers(write_through->cache1()));
  ASSERT_TRUE(lru_cache != NULL);
  EXPECT_EQ(1024*1024, lru_cache->max_bytes_in_cache());

  // Also on the HTTP cache
  WriteThroughCache* http_write_through =
      dynamic_cast<WriteThroughCache*>(server_context->http_cache()->cache());
  ASSERT_TRUE(http_write_through != NULL);
  EXPECT_EQ(500, http_write_through->cache1_limit());
}

TEST_F(SystemCachesTest, HangingMultigetTest) {
  // Test that we do not hang in the case of corrupted responses from memcached,
  // as seen in bug report 1048
  // https://github.com/pagespeed/mod_pagespeed/issues/1048
  set_fake_memcached_port(0);
  scoped_ptr<FakeMemcacheServerThread> thread(
      new FakeMemcacheServerThread(this));
  ASSERT_TRUE(thread->Start());
  thread->WaitForReady();
  ASSERT_NE(get_fake_memcached_port(), 0);
  GoogleString apr_str =
      StrCat("localhost:", Integer64ToString(get_fake_memcached_port()));
  AprMemCache* cache = system_caches_->NewAprMemCache(apr_str);
  static const char k1[] = "hello";
  static const char k2[] = "hi";
  BlockingCallback cb1(thread_system_.get());
  BlockingCallback cb2(thread_system_.get());
  CacheInterface::MultiGetRequest* request =
      new CacheInterface::MultiGetRequest;
  request->push_back(CacheInterface::KeyCallback(k1, &cb1));
  request->push_back(CacheInterface::KeyCallback(k2, &cb2));
  cache->Connect();
  // Capture stderr, make sure we get the proper string.
  // This test depends on a custom fprintf in apr_memcache2.
  // TODO(jcrowell) do this more nicely, don't depend on the print from
  // multiget, as the real test is that this should not hang.
  int stderr_backup;
  char buffer[4096];
  fflush(stderr);
  int err_pipe[2];
  stderr_backup = dup(STDERR_FILENO);
  ASSERT_NE(-1, stderr_backup);
  ASSERT_EQ(0, pipe(err_pipe));
  ASSERT_NE(-1, dup2(err_pipe[1], STDERR_FILENO));
  close(err_pipe[1]);
  // Make the multiget request.
  cache->MultiGet(request);
  thread->Join();
  fflush(stderr);
  int bytes_read = read(err_pipe[0], buffer, sizeof(buffer));
  ASSERT_NE(-1, bytes_read);
  // And give back stderr.
  ASSERT_NE(-1, dup2(stderr_backup, STDERR_FILENO));
  // Now check to make sure that we had the proper output.
  StringPiece output(buffer, bytes_read);
  EXPECT_TRUE(
      output.starts_with("Caught potential spin in apr_memcache multiget!"));
}

TEST_F(SystemCachesTest, StatsStringMinimal) {
  // The format is rather dependent on the implementation so we don't check it,
  // but we do care that it at least doesn't crash.
  GoogleString out;

  if (MemCachedServerSpec().empty()) {
    return;
  }

  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  options_->set_memcached_servers(MemCachedServerSpec());
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));

  system_caches_->PrintCacheStats(
      static_cast<SystemCaches::StatFlags>(SystemCaches::kGlobalView |
                                           SystemCaches::kIncludeMemcached),
      &out);
}

TEST_F(SystemCachesTest, ShareIdenticalNoPurge) {
  options_->set_file_cache_path(kCachePath);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_EQ(path1, path2);
}

TEST_F(SystemCachesTest, ShareIdenticalPurge) {
  options_->set_file_cache_path(kCachePath);
  options_->set_enable_cache_purge(true);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_enable_cache_purge(true);
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_EQ(path1, path2);
}

TEST_F(SystemCachesTest, NoSharePurgeFlush) {
  options_->set_file_cache_path(kCachePath);
  options_->set_enable_cache_purge(true);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_NE(path1, path2);
}

TEST_F(SystemCachesTest, ShareIdenticalPurgeCustomPath) {
  options_->set_file_cache_path(kCachePath);
  options_->set_cache_flush_filename("f1");
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_cache_flush_filename("f1");
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_EQ(path1, path2);
}

TEST_F(SystemCachesTest, NoShareVaryingPurgeCustomPath) {
  options_->set_file_cache_path(kCachePath);
  options_->set_cache_flush_filename("f1");
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_cache_flush_filename("f2");
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_NE(path1, path2);
}

TEST_F(SystemCachesTest, ShareOnOff) {
  options_->set_file_cache_path(kCachePath);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_enabled(RewriteOptions::kEnabledOff);
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_EQ(path1, path2);
}

TEST_F(SystemCachesTest, NoShareOnUnplugged) {
  options_->set_file_cache_path("/a");
  options_->set_enabled(RewriteOptions::kEnabledUnplugged);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path("/b");
  options2.set_enabled(RewriteOptions::kEnabledUnplugged);
  options2.set_cache_flush_filename("f2");
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_EQ(path1, path2);  // All 'unplugged' caches are the same.
}

TEST_F(SystemCachesTest, ShareUnpluggedWithOtherMismatches) {
  options_->set_file_cache_path(kCachePath);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_enabled(RewriteOptions::kEnabledUnplugged);
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_NE(path1, path2);
}

TEST_F(SystemCachesTest, FileCacheNoConflictTwoPaths) {
  options_->set_file_cache_path(kCachePath);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  EXPECT_NE(path1, path2);
  EXPECT_EQ(0, message_handler()->MessagesOfType(kWarning));
}

TEST_F(SystemCachesTest, FileCacheFullConflictTwoPaths) {
  options_->set_file_cache_path(kCachePath);
  options_->set_file_cache_clean_size_kb(10);
  options_->set_file_cache_clean_inode_limit(20);
  options_->set_file_cache_clean_interval_ms(1000);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_file_cache_clean_size_kb(11);        // wins
  options2.set_file_cache_clean_inode_limit(19);    // loses
  options2.set_file_cache_clean_interval_ms(999);   // wins
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  ASSERT_EQ(path1, path2);
  FileCache* file_cache = path1->file_cache_backend();
  const FileCache::CachePolicy* policy = file_cache->cache_policy();
  EXPECT_EQ(11*1024, policy->target_size_bytes);
  EXPECT_EQ(20, policy->target_inode_count);
  EXPECT_EQ(999, policy->clean_interval_ms);
  EXPECT_EQ(3, message_handler()->MessagesOfType(kWarning));
}

TEST_F(SystemCachesTest, FileCacheNoConflictOnDefaults) {
  options_->set_file_cache_path(kCachePath);
  options_->set_file_cache_clean_inode_limit(20);
  options_->set_file_cache_clean_interval_ms(1000);
  SystemCachePath* path1 = system_caches_->GetCache(options_.get());
  SystemRewriteOptions options2(thread_system_.get());
  options2.set_file_cache_path(kCachePath);
  options2.set_file_cache_clean_size_kb(11);        // wins
  SystemCachePath* path2 = system_caches_->GetCache(&options2);
  ASSERT_EQ(path1, path2);
  FileCache* file_cache = path1->file_cache_backend();
  const FileCache::CachePolicy* policy = file_cache->cache_policy();
  EXPECT_EQ(11*1024, policy->target_size_bytes);
  EXPECT_EQ(20, policy->target_inode_count);
  EXPECT_EQ(1000, policy->clean_interval_ms);
  EXPECT_EQ(0, message_handler()->MessagesOfType(kWarning));
}

TEST_F(SystemCachesTest, PurgeUrl) {
  options_->set_enable_cache_purge(true);
  SystemServerContext* server_context = PopulateCacheForPurgeTest();
  server_context->PostInitHook();
  SystemRewriteOptions* options =
      server_context->global_system_rewrite_options();
  RequestContextPtr request_context(
      RequestContext::NewTestRequestContext(thread_system_.get()));
  StringAsyncFetch fetch(request_context);

  // Invalidate kUrl1 but leave kUrl2 intact.
  AdminSite* admin_site = server_context->admin_site();
  admin_site->PurgeHandler(kUrl1, server_context->cache_path(), &fetch);
  ASSERT_TRUE(fetch.done());
  ASSERT_TRUE(fetch.success());
  server_context->FlushCacheIfNecessary();

  // Make sure we can no longer fetch kUrl1, but we can still fetch kUrl2.
  ResponseHeaders headers;
  HTTPValue value;
  EXPECT_EQ(kNotFoundResult, HttpBlockingFindWithOptions(
      options, kUrl1, server_context->http_cache(), &value, &headers));
  EXPECT_EQ(kFoundResult, HttpBlockingFindWithOptions(
      options, kUrl2, server_context->http_cache(), &value, &headers));

  // Now set the global invalidation timestamp, and kUrl2 will now be invalid
  // as well.
  AdvanceTimeMs(1);
  fetch.Reset();
  admin_site->PurgeHandler("http://example.com/*", server_context->cache_path(),
                           &fetch);
  server_context->FlushCacheIfNecessary();
  AdvanceTimeMs(1);
  EXPECT_EQ(kNotFoundResult, HttpBlockingFindWithOptions(
      options, kUrl2, server_context->http_cache(), &value, &headers));
}

TEST_F(SystemCachesTest, InvalidateWithPurgeDisabled) {
  options_->set_enable_cache_purge(false);
  SystemServerContext* server_context = PopulateCacheForPurgeTest();
  SystemRewriteOptions* options =
      server_context->global_system_rewrite_options();

  // touch cache.flush
  file_system()->WriteFile(StrCat(kCachePath, "/cache.flush").c_str(),
                           "", message_handler());
  AdvanceTimeMs(1000);
  server_context->FlushCacheIfNecessary();

  // Make sure both kUrl1 and kUrl2 are invalidated from touching the file.
  ResponseHeaders headers;
  HTTPValue value;
  EXPECT_EQ(kNotFoundResult, HttpBlockingFindWithOptions(
      options, kUrl1, server_context->http_cache(), &value, &headers));
  EXPECT_EQ(kNotFoundResult, HttpBlockingFindWithOptions(
      options, kUrl2, server_context->http_cache(), &value, &headers));
}

TEST_F(SystemCachesTest, StressTest) {
  MemCacheStressTestHelper(false);
}

TEST_F(SystemCachesTest, StressTestWithDeletions) {
  MemCacheStressTestHelper(true);
}

// Tests for how we fallback when SHM setup ops fail.
class BrokenShmSystemCachesTest : public SystemCachesTest {
 protected:
  BrokenShmSystemCachesTest() {
    shared_mem_.reset(new NullSharedMem());
  }
};

TEST_F(BrokenShmSystemCachesTest, FallbackShmLockManager) {
  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(true);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());
  NamedLockManager* named_locks = system_caches_->GetLockManager(
      options_.get());
  EXPECT_TRUE(named_locks != NULL);

  // Actually file system based here, due to fallback.
  EXPECT_TRUE(dynamic_cast<FileSystemLockManager*>(named_locks) != NULL);
}

TEST_F(BrokenShmSystemCachesTest, FallbackShmAndLru) {
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // We don't use the LRU when shm cache is on.
  EXPECT_STREQ(Compressed(WriteThrough(Stats("lru_cache", ThreadsafeLRU()),
                                       FileCacheWithStats())),
               server_context->metadata_cache()->Name());
  // HTTP cache is unaffected.
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          FileCacheWithStats())),
      server_context->http_cache()->Name());
}

TEST_F(BrokenShmSystemCachesTest, FallbackShmAndNoLru) {
  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // We don't use the LRU when shm cache is on.
  EXPECT_STREQ(Compressed(FileCacheWithStats()),
               server_context->metadata_cache()->Name());
  // HTTP cache is unaffected.
  EXPECT_STREQ(HttpCache(FileCacheWithStats()),
               server_context->http_cache()->Name());
}

TEST_F(BrokenShmSystemCachesTest, FallbackMemCachedLruShm) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(100);
  options_->set_memcached_servers(MemCachedServerSpec());
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  // For metadata, we fallback to memcached behind shmcache.
  EXPECT_STREQ(
      Compressed(
          WriteThrough(
              Stats("lru_cache", ThreadsafeLRU()),
              Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                       FileCacheWithStats()))),
      server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(WriteThrough(
          Stats("lru_cache", ThreadsafeLRU()),
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                   FileCacheWithStats()))),
      server_context->http_cache()->Name());
  EXPECT_STREQ(Pcache(Compressed(Fallback(BlockingMemCacheWithStats(),
                                          FileCacheWithStats()))),
               server_context->page_property_cache()->property_store()->Name());
}

TEST_F(BrokenShmSystemCachesTest, FallbackMemCachedShmNoLru) {
  if (MemCachedServerSpec().empty()) {
    return;
  }

  GoogleString error_msg;
  EXPECT_TRUE(system_caches_->CreateShmMetadataCache(
      kCachePath, kUsableMetadataCacheSize, &error_msg));

  options_->set_file_cache_path(kCachePath);
  options_->set_use_shared_mem_locking(false);
  options_->set_lru_cache_kb_per_process(0);
  options_->set_memcached_servers(MemCachedServerSpec());
  PrepareWithConfig(options_.get());

  scoped_ptr<ServerContext> server_context(
      SetupServerContext(options_.release()));
  EXPECT_STREQ(
      Compressed(Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                          FileCacheWithStats())),
      server_context->metadata_cache()->Name());
  EXPECT_STREQ(
      HttpCache(
          Fallback(Batcher(AsyncMemCacheWithStats(), 1, 1000),
                   FileCacheWithStats())),
      server_context->http_cache()->Name());
}

}  // namespace

}  // namespace net_instaweb