Google Git
Sign in
apache / incubator-pagespeed-debian / refs/heads/upstream / . / src / net / instaweb / rewriter / rewrite_driver.cc
blob: f30e7cf8f3b18c834c8572eefdeab91599d68756 [file] [log] [blame]
/*
* Copyright 2010 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jmarantz@google.com (Joshua Marantz)
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include <algorithm>
#include <cstdarg>
#include <cstdio>
#include <list>
#include <map>
#include <set>
#include <utility> // for std::pair
#include <vector>
#include "base/logging.h"
#include "net/instaweb/config/rewrite_options_manager.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/http_value.h"
#include "net/instaweb/http/public/log_record.h"
#include "net/instaweb/http/public/logging_proto_impl.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/critical_css.pb.h"
#include "net/instaweb/rewriter/critical_keys.pb.h"
#include "net/instaweb/rewriter/critical_line_info.pb.h"
#include "net/instaweb/rewriter/flush_early.pb.h"
#include "net/instaweb/rewriter/public/add_head_filter.h"
#include "net/instaweb/rewriter/public/add_ids_filter.h"
#include "net/instaweb/rewriter/public/add_instrumentation_filter.h"
#include "net/instaweb/rewriter/public/base_tag_filter.h"
#include "net/instaweb/rewriter/public/cache_extender.h"
#include "net/instaweb/rewriter/public/cache_html_filter.h"
#include "net/instaweb/rewriter/public/collect_flush_early_content_filter.h"
#include "net/instaweb/rewriter/public/compute_visible_text_filter.h"
#include "net/instaweb/rewriter/public/critical_css_beacon_filter.h"
#include "net/instaweb/rewriter/public/critical_css_filter.h"
#include "net/instaweb/rewriter/public/critical_images_beacon_filter.h"
#include "net/instaweb/rewriter/public/critical_selector_filter.h"
#include "net/instaweb/rewriter/public/critical_selector_finder.h"
#include "net/instaweb/rewriter/public/css_combine_filter.h"
#include "net/instaweb/rewriter/public/css_filter.h"
#include "net/instaweb/rewriter/public/css_inline_filter.h"
#include "net/instaweb/rewriter/public/css_inline_import_to_link_filter.h"
#include "net/instaweb/rewriter/public/css_move_to_head_filter.h"
#include "net/instaweb/rewriter/public/css_outline_filter.h"
#include "net/instaweb/rewriter/public/css_summarizer_base.h"
#include "net/instaweb/rewriter/public/css_tag_scanner.h"
#include "net/instaweb/rewriter/public/data_url_input_resource.h"
#include "net/instaweb/rewriter/public/debug_filter.h"
#include "net/instaweb/rewriter/public/decode_rewritten_urls_filter.h"
#include "net/instaweb/rewriter/public/dedup_inlined_images_filter.h"
#include "net/instaweb/rewriter/public/defer_iframe_filter.h"
#include "net/instaweb/rewriter/public/delay_images_filter.h"
#include "net/instaweb/rewriter/public/deterministic_js_filter.h"
#include "net/instaweb/rewriter/public/dom_stats_filter.h"
#include "net/instaweb/rewriter/public/domain_lawyer.h"
#include "net/instaweb/rewriter/public/domain_rewrite_filter.h"
#include "net/instaweb/rewriter/public/downstream_cache_purger.h"
#include "net/instaweb/rewriter/public/file_input_resource.h"
#include "net/instaweb/rewriter/public/file_load_policy.h"
#include "net/instaweb/rewriter/public/fix_reflow_filter.h"
#include "net/instaweb/rewriter/public/flush_early_content_writer_filter.h"
#include "net/instaweb/rewriter/public/flush_html_filter.h"
#include "net/instaweb/rewriter/public/google_analytics_filter.h"
#include "net/instaweb/rewriter/public/google_font_css_inline_filter.h"
#include "net/instaweb/rewriter/public/handle_noscript_redirect_filter.h"
#include "net/instaweb/rewriter/public/iframe_fetcher.h"
#include "net/instaweb/rewriter/public/image_combine_filter.h"
#include "net/instaweb/rewriter/public/image_rewrite_filter.h"
#include "net/instaweb/rewriter/public/in_place_rewrite_context.h"
#include "net/instaweb/rewriter/public/insert_dns_prefetch_filter.h"
#include "net/instaweb/rewriter/public/insert_ga_filter.h"
#include "net/instaweb/rewriter/public/javascript_filter.h"
#include "net/instaweb/rewriter/public/js_combine_filter.h"
#include "net/instaweb/rewriter/public/js_defer_disabled_filter.h"
#include "net/instaweb/rewriter/public/js_disable_filter.h"
#include "net/instaweb/rewriter/public/js_inline_filter.h"
#include "net/instaweb/rewriter/public/js_outline_filter.h"
#include "net/instaweb/rewriter/public/lazyload_images_filter.h"
#include "net/instaweb/rewriter/public/local_storage_cache_filter.h"
#include "net/instaweb/rewriter/public/make_show_ads_async_filter.h"
#include "net/instaweb/rewriter/public/meta_tag_filter.h"
#include "net/instaweb/rewriter/public/mobilize_label_filter.h"
#include "net/instaweb/rewriter/public/mobilize_menu_filter.h"
#include "net/instaweb/rewriter/public/mobilize_menu_render_filter.h"
#include "net/instaweb/rewriter/public/mobilize_rewrite_filter.h"
#include "net/instaweb/rewriter/public/output_resource.h"
#include "net/instaweb/rewriter/public/output_resource_kind.h"
#include "net/instaweb/rewriter/public/pedantic_filter.h"
#include "net/instaweb/rewriter/public/property_cache_util.h"
#include "net/instaweb/rewriter/public/redirect_on_size_limit_filter.h"
#include "net/instaweb/rewriter/public/request_properties.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/rewriter/public/resource_namer.h"
#include "net/instaweb/rewriter/public/resource_slot.h"
#include "net/instaweb/rewriter/public/responsive_image_filter.h"
#include "net/instaweb/rewriter/public/rewrite_context.h"
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/rewrite_filter.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_query.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/rewritten_content_scanning_filter.h"
#include "net/instaweb/rewriter/public/scan_filter.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/split_html_beacon_filter.h"
#include "net/instaweb/rewriter/public/split_html_config.h"
#include "net/instaweb/rewriter/public/split_html_filter.h"
#include "net/instaweb/rewriter/public/split_html_helper_filter.h"
#include "net/instaweb/rewriter/public/strip_non_cacheable_filter.h"
#include "net/instaweb/rewriter/public/strip_scripts_filter.h"
#include "net/instaweb/rewriter/public/strip_subresource_hints_filter.h"
#include "net/instaweb/rewriter/public/support_noscript_filter.h"
#include "net/instaweb/rewriter/public/suppress_prehead_filter.h"
#include "net/instaweb/rewriter/public/url_input_resource.h"
#include "net/instaweb/rewriter/public/url_left_trim_filter.h"
#include "net/instaweb/rewriter/public/url_namer.h"
#include "net/instaweb/util/public/fallback_property_page.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/callback.h"
#include "pagespeed/kernel/base/file_system.h"
#include "pagespeed/kernel/base/function.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/request_trace.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/sha1_signature.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/stl_util.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/base/writer.h"
#include "pagespeed/kernel/cache/cache_interface.h"
#include "pagespeed/kernel/html/collapse_whitespace_filter.h"
#include "pagespeed/kernel/html/elide_attributes_filter.h"
#include "pagespeed/kernel/html/html_attribute_quote_removal.h"
#include "pagespeed/kernel/html/html_element.h"
#include "pagespeed/kernel/html/html_filter.h"
#include "pagespeed/kernel/html/html_keywords.h"
#include "pagespeed/kernel/html/html_node.h"
#include "pagespeed/kernel/html/html_parse.h"
#include "pagespeed/kernel/html/html_writer_filter.h"
#include "pagespeed/kernel/html/remove_comments_filter.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/thread/scheduler.h"
#include "pagespeed/kernel/util/statistics_logger.h"
namespace net_instaweb {
class CriticalCssFinder;
class RewriteDriverPool;
namespace {
const int kTestTimeoutMs = 10000;
const char kDeadlineExceeded[] = "deadline_exceeded";
// Implementation of RemoveCommentsFilter::OptionsInterface that wraps
// a RewriteOptions instance.
class RemoveCommentsFilterOptions
: public RemoveCommentsFilter::OptionsInterface {
public:
explicit RemoveCommentsFilterOptions(const RewriteOptions* options)
: options_(options) {
}
virtual bool IsRetainedComment(const StringPiece& comment) const {
return options_->IsRetainedComment(comment);
}
private:
const RewriteOptions* options_;
DISALLOW_COPY_AND_ASSIGN(RemoveCommentsFilterOptions);
};
// Provides hook to CacheUrlAsyncFetcher to protect the lifetime of the
// RewriteDriver which owns fetcher, otherwise, fetcher may be deleted
// by the time background fetch completes.
class RewriteDriverCacheUrlAsyncFetcherAsyncOpHooks
: public CacheUrlAsyncFetcher::AsyncOpHooks {
public:
explicit RewriteDriverCacheUrlAsyncFetcherAsyncOpHooks(
RewriteDriver* rewrite_driver) : rewrite_driver_(rewrite_driver) {
}
virtual ~RewriteDriverCacheUrlAsyncFetcherAsyncOpHooks() {
}
// TODO(pulkitg): Remove session fetchers, so that fetcher can live as long
// server is alive and there is no need of
// {Increment/Decrement}AsyncEventsCount().
virtual void StartAsyncOp() {
// Increment async_events_counts so that driver will be alive as long as
// background fetch happens in CacheUrlAsyncFetcher.
rewrite_driver_->IncrementAsyncEventsCount();
}
virtual void FinishAsyncOp() {
rewrite_driver_->DecrementAsyncEventsCount();
}
private:
RewriteDriver* rewrite_driver_;
DISALLOW_COPY_AND_ASSIGN(RewriteDriverCacheUrlAsyncFetcherAsyncOpHooks);
};
} // namespace
const char RewriteDriver::kDomCohort[] = "dom";
const char RewriteDriver::kBeaconCohort[] = "beacon_cohort";
const char RewriteDriver::kSubresourcesPropertyName[] = "subresources";
const char RewriteDriver::kStatusCodePropertyName[] = "status_code";
const char RewriteDriver::kLastRequestTimestamp[] = "last_request_timestamp";
const char RewriteDriver::kParseSizeLimitExceeded[] =
"parse_size_limit_exceeded";
int RewriteDriver::initialized_count_ = 0;
RewriteDriver::RewriteDriver(MessageHandler* message_handler,
FileSystem* file_system,
UrlAsyncFetcher* url_async_fetcher)
: HtmlParse(message_handler),
base_was_set_(false),
refs_before_base_(false),
filters_added_(false),
externally_managed_(false),
ref_counts_(this),
release_driver_(false),
waiting_(kNoWait),
waiting_deadline_reached_(false),
fully_rewrite_on_flush_(false),
fast_blocking_rewrite_(true),
flush_requested_(false),
flush_occurred_(false),
flushed_cached_html_(false),
flushing_cached_html_(false),
flushed_early_(false),
flushing_early_(false),
is_lazyload_script_flushed_(false),
write_property_cache_dom_cohort_(false),
should_skip_parsing_(kNotSet),
response_headers_(NULL),
status_code_(HttpStatus::kUnknownStatusCode),
max_page_processing_delay_ms_(-1),
num_initiated_rewrites_(0),
num_detached_rewrites_(0),
possibly_quick_rewrites_(0),
file_system_(file_system),
server_context_(NULL),
scheduler_(NULL),
default_url_async_fetcher_(url_async_fetcher),
url_async_fetcher_(default_url_async_fetcher_),
distributed_async_fetcher_(NULL),
dom_stats_filter_(NULL),
scan_filter_(this),
controlling_pool_(NULL),
cache_url_async_fetcher_async_op_hooks_(
new RewriteDriverCacheUrlAsyncFetcherAsyncOpHooks(this)),
html_worker_(NULL),
rewrite_worker_(NULL),
low_priority_rewrite_worker_(NULL),
writer_(NULL),
fallback_property_page_(NULL),
owns_property_page_(false),
device_type_(UserAgentMatcher::kDesktop),
xhtml_mimetype_computed_(false),
xhtml_status_(kXhtmlUnknown),
num_inline_preview_images_(0),
num_flushed_early_pagespeed_resources_(0),
num_bytes_in_(0),
debug_filter_(NULL),
can_rewrite_resources_(true),
is_nested_(false),
request_context_(NULL),
start_time_ms_(0),
tried_to_distribute_fetch_(false),
defer_instrumentation_script_(false),
downstream_cache_purger_(this)
// NOTE: Be sure to clear per-request member variables in Clear()
{ // NOLINT -- I want the initializer-list to end with that comment.
// The Scan filter always goes first so it can find base-tags.
early_pre_render_filters_.push_back(&scan_filter_);
}
void RewriteDriver::PopulateRequestContext() {
if ((request_context_.get() != NULL && (request_headers_ != NULL))) {
request_context_->SetAcceptsWebp(
request_properties_->SupportsWebpRewrittenUrls());
request_context_->SetAcceptsGzip(request_properties_->AcceptsGzip());
request_context_->Freeze();
}
}
void RewriteDriver::SetRequestHeaders(const RequestHeaders& headers) {
DCHECK(request_headers_.get() == NULL);
RequestHeaders* new_request_headers = new RequestHeaders();
new_request_headers->CopyFrom(headers);
new_request_headers->PopulateLazyCaches();
request_headers_.reset(new_request_headers);
ClearRequestProperties();
const char* user_agent = request_headers_->Lookup1(
HttpAttributes::kUserAgent);
if (user_agent != NULL) {
user_agent_ = user_agent;
request_properties_->SetUserAgent(user_agent_);
}
request_properties_->ParseRequestHeaders(*request_headers_);
PopulateRequestContext();
}
void RewriteDriver::set_request_context(const RequestContextPtr& x) {
// Ideally, we would have a CHECK(x.get() != NULL) here, since all "real"
// RewriteDrivers should have a valid request context.
//
// However, one use-case currently prevent this --
// ServerContext::InitWorkersAndDecodingDriver() creates a new driver
// to decode options. This creation, via NewUnmanagedRewriteDriver(), invokes
// this method with the provided request context, which really should be NULL
// because it is not associated with a request.
//
// In lieu of the significant refactor required to move option decoding out
// of RewriteDriver or synthesizing a context, we allow NULL here, and opt
// to instead CHECK aggressively on code paths that really should have a
// request context; i.e., those necessarily associated with page serving
// rather than option decoding.
request_context_.reset(x);
if (request_context_.get() != NULL) {
request_context_->log_record()->SetRewriterInfoMaxSize(
options()->max_rewrite_info_log_size());
request_context_->log_record()->SetAllowLoggingUrls(
options()->allow_logging_urls_in_log_record());
request_context_->log_record()->SetLogUrlIndices(
options()->log_url_indices());
PopulateRequestContext();
}
}
AbstractLogRecord* RewriteDriver::log_record() {
CHECK(request_context_.get() != NULL);
return request_context_->log_record();
}
RewriteDriver::~RewriteDriver() {
if (rewrite_worker_ != NULL) {
scheduler_->UnregisterWorker(rewrite_worker_);
server_context_->rewrite_workers()->FreeSequence(rewrite_worker_);
}
if (html_worker_ != NULL) {
scheduler_->UnregisterWorker(html_worker_);
server_context_->html_workers()->FreeSequence(html_worker_);
}
if (low_priority_rewrite_worker_ != NULL) {
scheduler_->UnregisterWorker(low_priority_rewrite_worker_);
server_context_->low_priority_rewrite_workers()->FreeSequence(
low_priority_rewrite_worker_);
}
Clear();
STLDeleteElements(&filters_to_delete_);
STLDeleteElements(&resource_claimants_);
}
RewriteDriver* RewriteDriver::Clone() {
RewriteDriver* result;
RewriteDriverPool* pool = controlling_pool();
if (pool == NULL) {
// TODO(jmarantz): when used with SetParent, it should not be
// necessary to clone the options here. Once we set the child's
// parent to this, the child will reference this->options() and
// ignores its self_options_. To exploit that, we'd need to
// make a different entry-point for CloneAndSetParent.
RewriteOptions* options_copy = options()->Clone();
options_copy->ComputeSignature();
result =
server_context_->NewCustomRewriteDriver(options_copy, request_context_);
} else {
result = server_context_->NewRewriteDriverFromPool(pool, request_context_);
}
result->is_nested_ = true;
// Remove any Via headers for the nested driver. This is intended for
// removing "Via:1.1 google", so that nested drivers don't wind up
// adding cc:public into intermediate cached results.
//
// Note that we *do* want to propagate http/2 detection to nested drivers.
// This is OK because it gets captured in the RequestContext, which is
// shared, and is not reconstructed from request-headers.
RequestHeaders headers;
headers.CopyFrom(*request_headers_);
headers.RemoveAll(HttpAttributes::kVia);
result->SetRequestHeaders(headers);
return result;
}
void RewriteDriver::Clear() NO_THREAD_SAFETY_ANALYSIS {
HtmlParse::Clear();
// If this was a fetch, fetch_rewrites_ may still hold a reference to a
// RewriteContext.
STLDeleteElements(&fetch_rewrites_);
DCHECK(!flush_requested_);
release_driver_ = false;
downstream_cache_purger_.Clear();
write_property_cache_dom_cohort_ = false;
base_url_.Clear();
DCHECK(!base_url_.IsAnyValid());
decoded_base_url_.Clear();
fetch_url_.clear();
if (!server_context_->shutting_down()) {
if (!externally_managed_) {
ref_counts_.DCheckAllCountsZero();
}
DCHECK(primary_rewrite_context_map_.empty());
DCHECK(initiated_rewrites_.empty());
DCHECK(detached_rewrites_.empty());
DCHECK(rewrites_.empty());
DCHECK_EQ(0, possibly_quick_rewrites_);
}
xhtml_mimetype_computed_ = false;
xhtml_status_ = kXhtmlUnknown;
should_skip_parsing_ = kNotSet;
max_page_processing_delay_ms_ = -1;
request_headers_.reset(NULL);
response_headers_ = NULL;
status_code_ = 0;
flush_requested_ = false;
flush_occurred_ = false;
flushed_cached_html_ = false;
flushing_cached_html_ = false;
flushed_early_ = false;
flushing_early_ = false;
tried_to_distribute_fetch_ = false;
defer_instrumentation_script_ = false;
is_lazyload_script_flushed_ = false;
base_was_set_ = false;
refs_before_base_ = false;
containing_charset_.clear();
fully_rewrite_on_flush_ = false;
fast_blocking_rewrite_ = true;
num_inline_preview_images_ = 0;
num_flushed_early_pagespeed_resources_ = 0;
num_bytes_in_ = 0;
flush_early_info_.reset(NULL);
flush_early_render_info_.reset(NULL);
can_rewrite_resources_ = true;
is_nested_ = false;
num_initiated_rewrites_ = 0;
num_detached_rewrites_ = 0;
if (request_context_.get() != NULL) {
request_context_->WriteBackgroundRewriteLog();
request_context_.reset(NULL);
}
start_time_ms_ = 0;
critical_css_result_.reset(NULL);
critical_images_info_.reset(NULL);
critical_line_info_.reset(NULL);
beacon_critical_line_info_.reset(NULL);
critical_selector_info_.reset(NULL);
if (owns_property_page_) {
delete fallback_property_page_;
}
fallback_property_page_ = NULL;
origin_property_page_.reset();
owns_property_page_ = false;
device_type_ = UserAgentMatcher::kDesktop;
pagespeed_query_params_.clear();
pagespeed_option_cookies_.clear();
// Reset to the default fetcher from any session fetcher
// (as the request is over).
url_async_fetcher_ = default_url_async_fetcher_;
STLDeleteElements(&owned_url_async_fetchers_);
ClearRequestProperties();
user_agent_.clear();
}
// Must be called with rewrite_mutex() held.
bool RewriteDriver::RewritesComplete() const {
// 3 kinds of rewrites triggered from HTML:
bool no_pending_rewrites =
(ref_counts_.QueryCountMutexHeld(kRefPendingRewrites) == 0);
bool no_deleting_rewrites =
(ref_counts_.QueryCountMutexHeld(kRefDeletingRewrites) == 0);
bool no_detached_rewrites = detached_rewrites_.empty();
DCHECK_EQ(static_cast<int>(detached_rewrites_.size()),
ref_counts_.QueryCountMutexHeld(kRefDetachedRewrites));
// And also user-facing fetches. Note that background fetches are handled
// by IsDone separately.
bool no_user_facing_fetch =
(ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing) == 0);
return no_pending_rewrites && no_deleting_rewrites && no_detached_rewrites &&
no_user_facing_fetch;
}
void RewriteDriver::WaitForCompletion() {
BoundedWaitFor(kWaitForCompletion, -1);
}
void RewriteDriver::WaitForShutDown() {
BoundedWaitFor(kWaitForShutDown, -1);
}
void RewriteDriver::BoundedWaitFor(WaitMode mode, int64 timeout_ms) {
SchedulerBlockingFunction wait(scheduler_);
{
ScopedMutex lock(rewrite_mutex());
CheckForCompletionAsync(mode, timeout_ms, &wait);
}
wait.Block();
#ifndef NDEBUG
{
ScopedMutex lock(rewrite_mutex());
CHECK_EQ(waiting_, kNoWait);
}
#endif
}
void RewriteDriver::CheckForCompletionAsync(WaitMode wait_mode,
int64 timeout_ms,
Function* done) {
scheduler_->DCheckLocked();
DCHECK_NE(kNoWait, wait_mode);
DCHECK_EQ(kNoWait, waiting_);
waiting_ = wait_mode;
waiting_deadline_reached_ = false;
int64 end_time_ms;
if (timeout_ms <= 0) {
end_time_ms = -1; // Encodes unlimited
} else {
end_time_ms = server_context()->timer()->NowMs() + timeout_ms;
}
TryCheckForCompletion(wait_mode, end_time_ms, done);
}
void RewriteDriver::TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
Function* done)
NO_THREAD_SAFETY_ANALYSIS {
scheduler_->DCheckLocked();
int64 now_ms = server_context_->timer()->NowMs();
int64 sleep_ms;
if (end_time_ms < 0) {
waiting_deadline_reached_ = false; // Unlimited wait..
sleep_ms = kTestTimeoutMs;
} else {
waiting_deadline_reached_ = (now_ms >= end_time_ms);
if (waiting_deadline_reached_) {
// If deadline is already reached if we keep going we will want to use
// long sleeps since we expect to be woken up based on conditions.
sleep_ms = kTestTimeoutMs;
} else {
sleep_ms = end_time_ms - now_ms;
}
}
// Note that we may end up going past the deadline in order to make sure
// that at least the metadata cache lookups have a chance to come in.
if (!IsDone(wait_mode, waiting_deadline_reached_)) {
scheduler_->TimedWaitMs(
sleep_ms,
MakeFunction(this, &RewriteDriver::TryCheckForCompletion,
wait_mode, end_time_ms, done));
} else {
// Done. Note that we may get deleted by our callback, so we have to
// make sure to save the mutex pointer. The thread annotation can't deal
// with this aliasing, hence the need for NO_THREAD_SAFETY_ANALYSIS above.
AbstractMutex* mutex = rewrite_mutex();
waiting_ = kNoWait;
mutex->Unlock();
done->CallRun();
mutex->Lock();
}
}
bool RewriteDriver::IsDone(WaitMode wait_mode, bool deadline_reached) {
int async_events = ref_counts_.QueryCountMutexHeld(kRefAsyncEvents);
if (async_events > 0 && WaitForPendingAsyncEvents(wait_mode)) {
return false;
}
int render_blocking_async_events =
ref_counts_.QueryCountMutexHeld(kRefRenderBlockingAsyncEvents);
if (render_blocking_async_events > 0) {
return false;
}
// Before deadline, we're happy only if we're 100% done.
if (!deadline_reached) {
bool have_background_fetch =
(ref_counts_.QueryCountMutexHeld(kRefFetchBackground) != 0);
return RewritesComplete() &&
!((wait_mode == kWaitForShutDown) && have_background_fetch);
} else {
// When we've reached the deadline, if we're Render()'ing
// we also give the jobs we can serve from cache a chance to finish
// (so they always render).
// We do not have to worry about possibly_quick_rewrites_ not being
// incremented yet as jobs are only initiated from the HTML parse thread.
if (wait_mode == kWaitForCachedRender) {
return (possibly_quick_rewrites_ == 0);
} else {
return true;
}
}
}
void RewriteDriver::ExecuteFlushIfRequested() {
if (flush_requested_) {
Flush();
}
}
void RewriteDriver::ExecuteFlushIfRequestedAsync(Function* callback) {
if (flush_requested_) {
FlushAsync(callback);
} else {
callback->CallRun();
}
}
void RewriteDriver::Flush() {
SchedulerBlockingFunction wait(scheduler_);
FlushAsync(&wait);
wait.Block();
flush_requested_ = false;
}
void RewriteDriver::FlushAsync(Function* callback) {
DCHECK(request_context_.get() != NULL);
TraceLiteral("RewriteDriver::FlushAsync()");
if (debug_filter_ != NULL) {
debug_filter_->StartRender();
}
flush_requested_ = false;
// Figure out which filters should be enabled and whether any enabled filter
// can modify urls.
DetermineFiltersBehavior();
for (FilterList::iterator it = early_pre_render_filters_.begin();
it != early_pre_render_filters_.end(); ++it) {
HtmlFilter* filter = *it;
if (filter->is_enabled()) {
ApplyFilter(filter);
}
}
for (FilterList::iterator it = pre_render_filters_.begin();
it != pre_render_filters_.end(); ++it) {
HtmlFilter* filter = *it;
if (filter->is_enabled()) {
ApplyFilter(filter);
}
}
int num_rewrites = rewrites_.size();
// Copy all of the RewriteContext* into the initiated_rewrites_ set
// *before* initiating them, as we are doing this before we lock.
// The RewriteThread can start mutating the initiated_rewrites_
// set as soon as one is initiated.
{
// If not locked, this WRITE to initiated_rewrites_ can race with
// locked READs of initiated_rewrites_ in RewriteComplete which
// runs in the Rewrite thread. Note that the DCHECK above, of
// initiated_rewrites_.empty(), is a READ and it's OK to have
// concurrent READs.
ScopedMutex lock(rewrite_mutex());
// Note that no actual resource Rewriting can occur until this point
// is reached, where we initiate all the RewriteContexts.
DCHECK(initiated_rewrites_.empty());
DCHECK_EQ(ref_counts_.QueryCountMutexHeld(kRefPendingRewrites),
num_rewrites);
initiated_rewrites_.insert(rewrites_.begin(), rewrites_.end());
num_initiated_rewrites_ += num_rewrites;
// We must also start tasks while holding the lock, as otherwise a
// successor task may complete and delete itself before we see if we
// are the ones to start it.
for (int i = 0; i < num_rewrites; ++i) {
RewriteContext* rewrite_context = rewrites_[i];
if (!rewrite_context->chained()) {
rewrite_context->Initiate();
}
}
}
rewrites_.clear();
{
ScopedMutex lock(rewrite_mutex());
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchBackground));
Function* flush_async_done =
MakeFunction(this, &RewriteDriver::QueueFlushAsyncDone,
num_rewrites, callback);
if (fully_rewrite_on_flush_) {
CheckForCompletionAsync(kWaitForCompletion, -1, flush_async_done);
} else {
int64 deadline = ComputeCurrentFlushWindowRewriteDelayMs();
CheckForCompletionAsync(kWaitForCachedRender, deadline, flush_async_done);
}
}
}
int64 RewriteDriver::ComputeCurrentFlushWindowRewriteDelayMs() {
int64 deadline = rewrite_deadline_ms();
// If we've configured a max processing delay for the entire page, enforce
// that limit here.
if (max_page_processing_delay_ms_ > 0) {
int64 ms_since_start =
server_context_->timer()->NowMs() - start_time_ms_;
int64 ms_remaining = max_page_processing_delay_ms_ - ms_since_start;
// If the deadline for the current flush window (deadline) is less
// than the overall time remaining (ms_remaining), we enforce the
// per-flush window deadline. Otherwise, we wait for the overall
// page deadline.
//
// In any case, we require a minimum value of 1 millisecond since
// a value <= 0 implies an unlimited wait.
deadline =
std::max(std::min(ms_remaining, deadline), static_cast<int64>(1));
}
return deadline;
}
void RewriteDriver::QueueFlushAsyncDone(int num_rewrites, Function* callback) {
html_worker_->Add(MakeFunction(this, &RewriteDriver::FlushAsyncDone,
num_rewrites, callback));
}
void RewriteDriver::FlushAsyncDone(int num_rewrites, Function* callback) {
DCHECK(request_context_.get() != NULL);
TraceLiteral("RewriteDriver::FlushAsyncDone()");
{
ScopedMutex lock(rewrite_mutex());
DCHECK_EQ(0, possibly_quick_rewrites_);
int still_pending_rewrites =
ref_counts_.QueryCountMutexHeld(kRefPendingRewrites);
int completed_rewrites = num_rewrites - still_pending_rewrites;
// If the output cache lookup came as a HIT in after the deadline, that
// means that (a) we can't use the result and (b) we don't need
// to re-initiate the rewrite since it was in fact in cache. Hopefully
// the cache system will respond to HIT by making the next HIT faster
// so it meets our deadline. In either case we will track with stats.
//
RewriteStats* stats = server_context_->rewrite_stats();
stats->cached_output_hits()->Add(completed_rewrites);
stats->cached_output_missed_deadline()->Add(still_pending_rewrites);
{
// Add completed_rewrites (from this flush window) to the logged value.
ScopedMutex lock(log_record()->mutex());
MetadataCacheInfo* metadata_log_info =
log_record()->logging_info()->mutable_metadata_cache_info();
metadata_log_info->set_num_rewrites_completed(
metadata_log_info->num_rewrites_completed() + completed_rewrites);
}
// Detach all rewrites that are still outstanding, by moving them from
// initiated_rewrites_ to detached_rewrites_; also notify them that they
// will not be rendered.
for (RewriteContextSet::iterator p = initiated_rewrites_.begin(),
e = initiated_rewrites_.end(); p != e; ++p) {
RewriteContext* rewrite_context = *p;
// If debugging is enabled, annotate that we have missed our rewrite
// deadline.
if (options()->Enabled(RewriteOptions::kDebug)) {
for (int i = 0, n = rewrite_context->num_slots(); i < n; ++i) {
ResourceSlotPtr slot = rewrite_context->slot(i);
GoogleString suffix;
const char* id = rewrite_context->id();
StringFilterMap::const_iterator p = resource_filter_map_.find(id);
if (p != resource_filter_map_.end()) {
RewriteFilter* filter = p->second;
InsertDebugComment(DeadlineExceededMessage(filter->Name()),
slot->element());
} else {
InsertDebugComment(kDeadlineExceeded, slot->element());
}
}
}
rewrite_context->WillNotRender();
detached_rewrites_.insert(rewrite_context);
++num_detached_rewrites_;
ref_counts_.AddRefMutexHeld(kRefDetachedRewrites);
ref_counts_.ReleaseRefMutexHeld(kRefPendingRewrites);
}
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefPendingRewrites));
initiated_rewrites_.clear();
slots_.clear();
inline_slots_.clear();
inline_attribute_slots_.clear();
}
// Notify all enabled pre-render filters that rendering is done.
if (debug_filter_ != NULL) {
debug_filter_->RenderDone();
}
for (FilterList::iterator it = early_pre_render_filters_.begin();
it != early_pre_render_filters_.end(); ++it) {
HtmlFilter* filter = *it;
if (filter->is_enabled()) {
filter->RenderDone();
}
}
for (FilterList::iterator it = pre_render_filters_.begin();
it != pre_render_filters_.end(); ++it) {
HtmlFilter* filter = *it;
if (filter->is_enabled()) {
filter->RenderDone();
}
}
// Run all the post-render filters, and clear the event queue.
HtmlParse::Flush();
flush_occurred_ = true;
callback->CallRun();
}
GoogleString RewriteDriver::DeadlineExceededMessage(StringPiece filter_name) {
return StrCat(kDeadlineExceeded, " for filter ", filter_name);
}
void RewriteDriver::Initialize() {
++initialized_count_;
if (initialized_count_ == 1) {
RewriteOptions::Initialize();
ImageRewriteFilter::Initialize();
CssFilter::Initialize();
SplitHtmlConfig::Initialize();
}
}
void RewriteDriver::InitStats(Statistics* statistics) {
AddInstrumentationFilter::InitStats(statistics);
CacheExtender::InitStats(statistics);
CriticalCssBeaconFilter::InitStats(statistics);
CriticalImagesBeaconFilter::InitStats(statistics);
CssCombineFilter::InitStats(statistics);
CssFilter::InitStats(statistics);
CssInlineFilter::InitStats(statistics);
CssInlineImportToLinkFilter::InitStats(statistics);
CssMoveToHeadFilter::InitStats(statistics);
CssSummarizerBase::InitStats(statistics);
DedupInlinedImagesFilter::InitStats(statistics);
DomainRewriteFilter::InitStats(statistics);
GoogleAnalyticsFilter::InitStats(statistics);
GoogleFontCssInlineFilter::InitStats(statistics);
ImageCombineFilter::InitStats(statistics);
ImageRewriteFilter::InitStats(statistics);
InPlaceRewriteContext::InitStats(statistics);
InsertGAFilter::InitStats(statistics);
JavascriptFilter::InitStats(statistics);
JsCombineFilter::InitStats(statistics);
JsInlineFilter::InitStats(statistics);
LocalStorageCacheFilter::InitStats(statistics);
MakeShowAdsAsyncFilter::InitStats(statistics);
MetaTagFilter::InitStats(statistics);
MobilizeLabelFilter::InitStats(statistics);
MobilizeMenuFilter::InitStats(statistics);
MobilizeMenuRenderFilter::InitStats(statistics);
MobilizeRewriteFilter::InitStats(statistics);
SplitHtmlBeaconFilter::InitStats(statistics);
RewriteContext::InitStats(statistics);
UrlInputResource::InitStats(statistics);
UrlLeftTrimFilter::InitStats(statistics);
}
void RewriteDriver::Terminate() {
// Clean up statics.
--initialized_count_;
if (initialized_count_ == 0) {
CssFilter::Terminate();
ImageRewriteFilter::Terminate();
RewriteOptions::Terminate();
SplitHtmlConfig::Terminate();
}
}
void RewriteDriver::SetServerContext(ServerContext* server_context)
NO_THREAD_SAFETY_ANALYSIS {
DCHECK(server_context_ == NULL);
server_context_ = server_context;
scheduler_ = server_context_->scheduler();
ref_counts_.set_mutex(rewrite_mutex());
set_timer(server_context->timer());
rewrite_worker_ = server_context_->rewrite_workers()->NewSequence();
html_worker_ = server_context_->html_workers()->NewSequence();
low_priority_rewrite_worker_ =
server_context_->low_priority_rewrite_workers()->NewSequence();
scheduler_->RegisterWorker(rewrite_worker_);
scheduler_->RegisterWorker(html_worker_);
scheduler_->RegisterWorker(low_priority_rewrite_worker_);
DCHECK(resource_filter_map_.empty());
// Add the rewriting filters to the map unconditionally -- we may
// need them to process resource requests due to a query-specific
// 'rewriters' specification. We still use the passed-in options
// to determine whether they get added to the html parse filter chain.
// Note: RegisterRewriteFilter takes ownership of these filters.
CacheExtender* cache_extender = new CacheExtender(this);
ImageCombineFilter* image_combiner = new ImageCombineFilter(this);
ImageRewriteFilter* image_rewriter = new ImageRewriteFilter(this);
RegisterRewriteFilter(new CssCombineFilter(this));
RegisterRewriteFilter(
new CssFilter(this, cache_extender, image_rewriter, image_combiner));
RegisterRewriteFilter(new JavascriptFilter(this));
RegisterRewriteFilter(new JsCombineFilter(this));
RegisterRewriteFilter(image_rewriter);
RegisterRewriteFilter(cache_extender);
RegisterRewriteFilter(image_combiner);
RegisterRewriteFilter(new LocalStorageCacheFilter(this));
RegisterRewriteFilter(new JavascriptSourceMapFilter(this));
// These filters are needed to rewrite and trim urls in modified CSS files.
domain_rewriter_.reset(new DomainRewriteFilter(this, statistics()));
url_trim_filter_.reset(new UrlLeftTrimFilter(this, statistics()));
}
RequestTrace* RewriteDriver::trace_context() {
return request_context_.get() == NULL ? NULL :
request_context_->root_trace_context();
}
void RewriteDriver::TracePrintf(const char* fmt, ...) {
if (trace_context() == NULL || !trace_context()->tracing_enabled()) {
return;
}
va_list argp;
va_start(argp, fmt);
trace_context()->TraceVPrintf(fmt, argp);
va_end(argp);
}
void RewriteDriver::TraceLiteral(const char* literal) {
if (trace_context() == NULL || !trace_context()->tracing_enabled()) {
return;
}
trace_context()->TraceLiteral(literal);
}
void RewriteDriver::TraceString(const GoogleString& s) {
if (trace_context() == NULL || !trace_context()->tracing_enabled()) {
return;
}
trace_context()->TraceString(s);
}
void RewriteDriver::AddFilters() {
CHECK(html_writer_filter_ == NULL);
CHECK(!filters_added_);
server_context_->ComputeSignature(options_.get());
filters_added_ = true;
AddPreRenderFilters();
AddPostRenderFilters();
}
void RewriteDriver::AddPreRenderFilters() {
// This function defines the order that filters are run. We document
// in pagespeed.conf.template that the order specified in the conf
// file does not matter, but we give the filters there in the order
// they are actually applied, for the benefit of the understanding
// of the site owner. So if you change that here, change it in
// install/common/pagespeed.conf.template as well.
//
// Also be sure to update the doc in net/instaweb/doc/docs/config_filters.ezt.
//
// Now process boolean options, which may include propagating non-boolean
// and boolean parameter settings to filters.
const RewriteOptions* rewrite_options = options();
if (rewrite_options->flush_html()) {
// Note that this does not get hooked into the normal html-parse
// filter-chain as it gets run immediately after every call to
// ParseText, possibly inducing the system to trigger a Flush
// based on the content it sees.
add_event_listener(new FlushHtmlFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kComputeStatistics)) {
dom_stats_filter_ = new DomStatsFilter(this);
AddOwnedEarlyPreRenderFilter(dom_stats_filter_);
}
if (!rewrite_options->preserve_subresource_hints()) {
AddOwnedEarlyPreRenderFilter(new StripSubresourceHintsFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kDecodeRewrittenUrls)) {
AddOwnedEarlyPreRenderFilter(new DecodeRewrittenUrlsFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kSplitHtmlHelper)) {
AddOwnedEarlyPreRenderFilter(new SplitHtmlHelperFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kResponsiveImages) &&
rewrite_options->Enabled(RewriteOptions::kResizeImages)) {
ResponsiveImageFirstFilter* resp_filter1 =
new ResponsiveImageFirstFilter(this);
AddOwnedEarlyPreRenderFilter(resp_filter1);
ResponsiveImageSecondFilter* resp_filter2 =
new ResponsiveImageSecondFilter(this, resp_filter1);
AddOwnedPostRenderFilter(resp_filter2);
}
// We disable combine_css and combine_javascript when flush_subresources is
// enabled, since the way CSS and JS is combined is not deterministic.
// However, we do not disable combine_javascript when defer_javascript is
// enabled since in this case, flush_subresources does not flush JS resources.
bool flush_subresources_enabled = rewrite_options->Enabled(
RewriteOptions::kFlushSubresources);
if (rewrite_options->Enabled(RewriteOptions::kAddBaseTag) ||
rewrite_options->Enabled(RewriteOptions::kAddHead) ||
rewrite_options->Enabled(RewriteOptions::kAddInstrumentation) ||
rewrite_options->Enabled(RewriteOptions::kCombineHeads) ||
rewrite_options->Enabled(RewriteOptions::kDeterministicJs) ||
rewrite_options->Enabled(RewriteOptions::kHandleNoscriptRedirect) ||
rewrite_options->Enabled(RewriteOptions::kMakeGoogleAnalyticsAsync) ||
rewrite_options->Enabled(RewriteOptions::kMobilize) ||
rewrite_options->Enabled(RewriteOptions::kMoveCssAboveScripts) ||
rewrite_options->Enabled(RewriteOptions::kMoveCssToHead) ||
flush_subresources_enabled) {
// Adds a filter that adds a 'head' section to html documents if
// none found prior to the body.
AddOwnedEarlyPreRenderFilter(new AddHeadFilter(
this, rewrite_options->Enabled(RewriteOptions::kCombineHeads)));
}
if (rewrite_options->Enabled(RewriteOptions::kAddBaseTag)) {
AddOwnedEarlyPreRenderFilter(new BaseTagFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kAddIds) ||
rewrite_options->MobUseLabelFilter()) {
AddOwnedEarlyPreRenderFilter(new AddIdsFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kStripScripts)) {
// Experimental filter that blindly strips all scripts from a page.
AppendOwnedPreRenderFilter(new StripScriptsFilter(this));
}
if (is_critical_images_beacon_enabled()) {
// This filter should be enabled early, at least before image rewriting,
// because it depends on seeing the original image URLs.
AppendOwnedPreRenderFilter(new CriticalImagesBeaconFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kMakeShowAdsAsync)) {
// We want this filter early in case we ever inline the loader JS.
AppendOwnedPreRenderFilter(new MakeShowAdsAsyncFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kSplitHtml) &&
server_context()->factory()->UseBeaconResultsInFilters()) {
AppendOwnedPreRenderFilter(new SplitHtmlBeaconFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kInlineImportToLink) ||
(!rewrite_options->Forbidden(RewriteOptions::kInlineImportToLink) &&
(CriticalSelectorsEnabled() ||
rewrite_options->Enabled(RewriteOptions::kComputeCriticalCss)))) {
// If we're converting simple embedded CSS @imports into a href link
// then we need to do that before any other CSS processing.
AppendOwnedPreRenderFilter(new CssInlineImportToLinkFilter(this,
statistics()));
}
if (rewrite_options->Enabled(RewriteOptions::kPrioritizeCriticalCss)) {
// If we're inlining styles that resolved initially, skip outlining
// css since that works against this.
// TODO(slamm): Figure out if move_css_to_head needs to be disabled.
CriticalCssFinder* finder = server_context()->critical_css_finder();
if (finder != NULL && !CriticalSelectorsEnabled()) {
AppendOwnedPreRenderFilter(new CriticalCssFilter(this, finder));
}
} else if (rewrite_options->Enabled(RewriteOptions::kOutlineCss)) {
// Cut out inlined styles and make them into external resources.
// This can only be called once and requires a server_context_ to be set.
CHECK(server_context_ != NULL);
AppendOwnedPreRenderFilter(new CssOutlineFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kInlineGoogleFontCss)) {
// Inline small Google Font Service CSS files.
// Do this before MoveCssToHead / MoveCssAboveScripts.
AppendOwnedPreRenderFilter(new GoogleFontCssInlineFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kMoveCssToHead) ||
rewrite_options->Enabled(RewriteOptions::kMoveCssAboveScripts)) {
// It's good to move CSS links to the head prior to running CSS combine,
// which only combines CSS links that are already in the head.
AppendOwnedPreRenderFilter(new CssMoveToHeadFilter(this));
}
if (!flush_subresources_enabled &&
rewrite_options->Enabled(RewriteOptions::kCombineCss)) {
// Combine external CSS resources after we've outlined them.
// CSS files in html document. This can only be called
// once and requires a server_context_ to be set.
EnableRewriteFilter(RewriteOptions::kCssCombinerId);
}
if (rewrite_options->Enabled(RewriteOptions::kRewriteCss) ||
(!rewrite_options->Forbidden(RewriteOptions::kRewriteCss) &&
FlattenCssImportsEnabled())) {
// Since AddFilters only applies to the HTML rewrite path, we check here
// if IPRO preemptive rewrites are disabled and skip the filter if so.
if (!rewrite_options->css_preserve_urls() ||
rewrite_options->in_place_preemptive_rewrite_css()) {
EnableRewriteFilter(RewriteOptions::kCssFilterId);
}
}
if ((rewrite_options->Enabled(RewriteOptions::kPrioritizeCriticalCss) &&
server_context()->factory()->UseBeaconResultsInFilters()) ||
(rewrite_options->Enabled(RewriteOptions::kComputeCriticalCss) &&
rewrite_options->use_selectors_for_critical_css())) {
// Add the critical selector instrumentation before the rewriting filter.
AppendOwnedPreRenderFilter(new CriticalCssBeaconFilter(this));
}
if (CriticalSelectorsEnabled()) {
AppendOwnedPreRenderFilter(new CriticalSelectorFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kInlineCss)) {
// Inline small CSS files. Give CSS minification and flattening a chance to
// run before we decide what counts as "small".
CHECK(server_context_ != NULL);
AppendOwnedPreRenderFilter(new CssInlineFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kOutlineJavascript)) {
// Cut out inlined scripts and make them into external resources.
// This can only be called once and requires a server_context_ to be set.
CHECK(server_context_ != NULL);
AppendOwnedPreRenderFilter(new JsOutlineFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kMakeGoogleAnalyticsAsync)) {
// Converts sync loads of Google Analytics javascript to async loads.
// This needs to be listed before rewrite_javascript because it injects
// javascript that has comments and extra whitespace.
AppendOwnedPreRenderFilter(new GoogleAnalyticsFilter(this, statistics()));
}
if ((rewrite_options->Enabled(RewriteOptions::kInsertGA) ||
rewrite_options->running_experiment()) &&
rewrite_options->ga_id() != "") {
// Like MakeGoogleAnalyticsAsync, InsertGA should be before js rewriting.
AppendOwnedPreRenderFilter(new InsertGAFilter(this));
}
if (!flush_subresources_enabled &&
rewrite_options->Enabled(RewriteOptions::kCombineJavascript)) {
// Combine external JS resources. Done after minification and analytics
// detection, as it converts script sources into string literals, making
// them opaque to analysis.
EnableRewriteFilter(RewriteOptions::kJavascriptCombinerId);
}
if (rewrite_options->Enabled(RewriteOptions::kRewriteJavascriptExternal) ||
rewrite_options->Enabled(RewriteOptions::kRewriteJavascriptInline) ||
rewrite_options->Enabled(
RewriteOptions::kCanonicalizeJavascriptLibraries)) {
// Since AddFilters only applies to the HTML rewrite path, we check here
// if IPRO preemptive rewrites are disabled and skip the filter if so.
//
// Note that we minify before we inline, so if you enable
// rewrite_javascript_inline but not rewrite_javascript_external, we
// will only minify the already-inlined JavaScript, and we will not
// minify external JS that we decided later to inline. It seems unlikely
// that someone would want to enable inline_javascript and not enable
// rewrite_javascript_external though.
if (!rewrite_options->js_preserve_urls() ||
rewrite_options->in_place_preemptive_rewrite_javascript() ||
rewrite_options->Enabled(RewriteOptions::kRewriteJavascriptInline)) {
// Rewrite (minify etc.) JavaScript code to reduce time to first
// interaction.
EnableRewriteFilter(RewriteOptions::kJavascriptMinId);
}
}
// Disable incomplete and unsupported moblization filters.
//
// Mobilize after JS-rewrite and before JS-inline. We don't
// want to use the PageSpeed minifier to re-optimize the
// closure-compiled mobilization code because (a) it won't
// do much good (b) it would rename the URL to something we won't
// find on /mod_pagespeed_static and (c) we certainly don't want
// source-maps for the compiled code. However, we do want
// the inliner to work on the small compiled xhr.js.
// if (rewrite_options->Enabled(RewriteOptions::kMobilize)) {
// if (rewrite_options->MobUseLabelFilter()) {
// AppendOwnedPreRenderFilter(
// new MobilizeLabelFilter(false /* not menu request */, this));
// }
// if (rewrite_options->MobRenderServerSideMenus()) {
// AppendOwnedPreRenderFilter(new MobilizeMenuRenderFilter(this));
// }
// AppendOwnedPreRenderFilter(new MobilizeRewriteFilter(this));
// }
if (rewrite_options->Enabled(RewriteOptions::kInlineJavascript)) {
// Inline small Javascript files. Give JS minification a chance to run
// before we decide what counts as "small".
CHECK(server_context_ != NULL);
AppendOwnedPreRenderFilter(new JsInlineFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kConvertJpegToProgressive) ||
rewrite_options->ImageOptimizationEnabled() ||
rewrite_options->Enabled(RewriteOptions::kResizeImages) ||
rewrite_options->Enabled(
RewriteOptions::kResizeToRenderedImageDimensions) ||
rewrite_options->Enabled(RewriteOptions::kInlineImages) ||
rewrite_options->Enabled(RewriteOptions::kInsertImageDimensions) ||
rewrite_options->Enabled(RewriteOptions::kJpegSubsampling) ||
rewrite_options->Enabled(RewriteOptions::kStripImageColorProfile) ||
rewrite_options->Enabled(RewriteOptions::kStripImageMetaData) ||
rewrite_options->Enabled(RewriteOptions::kDelayImages)) {
// Since AddFilters only applies to the HTML rewrite path, we check here
// if IPRO preemptive rewrites are disabled and skip the filter if so.
if (!rewrite_options->image_preserve_urls() ||
rewrite_options->in_place_preemptive_rewrite_images()) {
EnableRewriteFilter(RewriteOptions::kImageCompressionId);
}
}
if (rewrite_options->Enabled(RewriteOptions::kRemoveComments)) {
AppendOwnedPreRenderFilter(new RemoveCommentsFilter(
this, new RemoveCommentsFilterOptions(rewrite_options)));
}
if (rewrite_options->Enabled(RewriteOptions::kElideAttributes)) {
// Remove HTML element attribute values where
// http://www.w3.org/TR/html4/loose.dtd says that the name is all
// that's necessary
AppendOwnedPreRenderFilter(new ElideAttributesFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kExtendCacheCss) ||
rewrite_options->Enabled(RewriteOptions::kExtendCacheImages) ||
rewrite_options->Enabled(RewriteOptions::kExtendCachePdfs) ||
rewrite_options->Enabled(RewriteOptions::kExtendCacheScripts)) {
// Extend the cache lifetime of resources.
EnableRewriteFilter(RewriteOptions::kCacheExtenderId);
}
if (rewrite_options->Enabled(RewriteOptions::kSpriteImages)) {
EnableRewriteFilter(RewriteOptions::kImageCombineId);
}
if (rewrite_options->Enabled(RewriteOptions::kLocalStorageCache)) {
EnableRewriteFilter(RewriteOptions::kLocalStorageCacheId);
}
// Enable Flush subresources early filter to extract the subresources from
// head. This should be the last prerender filter.
if (flush_subresources_enabled) {
AppendOwnedPreRenderFilter(new
CollectFlushEarlyContentFilter(this));
}
}
void RewriteDriver::AddPostRenderFilters() {
const RewriteOptions* rewrite_options = options();
if (rewrite_options->domain_lawyer()->can_rewrite_domains() &&
(rewrite_options->Enabled(RewriteOptions::kRewriteDomains) ||
rewrite_options->mob_iframe())) {
// Rewrite mapped domains and shard any resources not otherwise rewritten.
// We want do do this after all the content-changing rewrites, because they
// will map & shard as part of their execution.
//
// TODO(jmarantz): Consider removing all the domain-mapping functionality
// from other rewrites and do it exclusively in this filter. Before we
// do that we'll need to validate this filter so we can turn it on by
// default.
//
// Note that the "domain_lawyer" filter controls whether we rewrite
// domains for resources in HTML files. However, when we cache-extend
// CSS files, we rewrite the domains in them whether this filter is
// specified or not.
AddUnownedPostRenderFilter(domain_rewriter_.get());
}
if (rewrite_options->Enabled(RewriteOptions::kLeftTrimUrls)) {
// Trim extraneous prefixes from urls in attribute values.
// Happens before RemoveQuotes but after everything else. Note:
// we Must left trim urls BEFORE quote removal.
AddUnownedPostRenderFilter(url_trim_filter_.get());
}
if (rewrite_options->Enabled(RewriteOptions::kFlushSubresources) &&
!options()->pre_connect_url().empty()) {
AddOwnedPostRenderFilter(new RewrittenContentScanningFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kInsertDnsPrefetch)) {
InsertDnsPrefetchFilter* insert_dns_prefetch_filter =
new InsertDnsPrefetchFilter(this);
AddOwnedPostRenderFilter(insert_dns_prefetch_filter);
}
if (rewrite_options->Enabled(RewriteOptions::kAddInstrumentation)) {
// Inject javascript to instrument loading-time. This should run before
// defer js so that its onload handler can fire before JS starts executing.
AddOwnedPostRenderFilter(new AddInstrumentationFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kSplitHtml)) {
AddOwnedPostRenderFilter(new DeferIframeFilter(this));
AddOwnedPostRenderFilter(new JsDisableFilter(this));
} else if (rewrite_options->Enabled(RewriteOptions::kDeferJavascript) ||
rewrite_options->Enabled(RewriteOptions::kCachePartialHtml)) {
// Defers javascript download and execution to post onload. This filter
// should be applied before JsDisableFilter and JsDeferFilter.
// kDeferIframe filter should never be turned on when either defer_js
// or disable_js is enabled.
AddOwnedPostRenderFilter(new DeferIframeFilter(this));
AddOwnedPostRenderFilter(new JsDisableFilter(this));
// Though we are adding JsDeferDisabledFilter here, if we are flushing
// cached html or we have flushed cached html, this filter will disable
// itself.
AddOwnedPostRenderFilter(new JsDeferDisabledFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kFixReflows)) {
AddOwnedPostRenderFilter(new FixReflowFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kDeterministicJs)) {
AddOwnedPostRenderFilter(new DeterministicJsFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kConvertMetaTags)) {
AddOwnedPostRenderFilter(new MetaTagFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kDisableJavascript)) {
// kDeferIframe filter should never be turned on when either defer_js
// or disable_js is enabled.
AddOwnedPostRenderFilter(new DeferIframeFilter(this));
AddOwnedPostRenderFilter(new JsDisableFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kDelayImages)) {
// kInsertImageDimensions should be enabled to avoid drastic reflows.
AddOwnedPostRenderFilter(new DelayImagesFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kDedupInlinedImages)) {
AddOwnedPostRenderFilter(new DedupInlinedImagesFilter(this));
}
// TODO(nikhilmadan): Should we disable this for bots?
// LazyLoadImagesFilter should be applied after DelayImagesFilter.
if (rewrite_options->Enabled(RewriteOptions::kLazyloadImages)) {
AddOwnedPostRenderFilter(new LazyloadImagesFilter(this));
}
if (rewrite_options->support_noscript_enabled()) {
AddOwnedPostRenderFilter(new SupportNoscriptFilter(this));
}
if (rewrite_options->Enabled(RewriteOptions::kHandleNoscriptRedirect)) {
AddOwnedPostRenderFilter(new HandleNoscriptRedirectFilter(this));
}
if (rewrite_options->max_html_parse_bytes() > 0) {
AddOwnedPostRenderFilter(new RedirectOnSizeLimitFilter(this));
set_size_limit(rewrite_options->max_html_parse_bytes());
}
if (rewrite_options->Enabled(RewriteOptions::kStripNonCacheable)) {
StripNonCacheableFilter* filter = new StripNonCacheableFilter(this);
AddOwnedPostRenderFilter(filter);
}
if (rewrite_options->Enabled(RewriteOptions::kComputeVisibleText)) {
ComputeVisibleTextFilter* filter = new ComputeVisibleTextFilter(this);
AddOwnedPostRenderFilter(filter);
}
if (rewrite_options->Enabled(RewriteOptions::kPedantic)) {
// Add HTML type attributes where HTML4 says that it's necessary.
PedanticFilter* filter = new PedanticFilter(this);
AddOwnedPostRenderFilter(filter);
}
// Remove quotes and collapse whitespace at the very end for maximum effect.
if (rewrite_options->Enabled(RewriteOptions::kRemoveQuotes)) {
// Remove extraneous quotes from html attributes.
AddOwnedPostRenderFilter(new HtmlAttributeQuoteRemoval(this));
}
if (rewrite_options->Enabled(RewriteOptions::kCollapseWhitespace)) {
// Remove excess whitespace in HTML.
AddOwnedPostRenderFilter(new CollapseWhitespaceFilter(this));
}
if (DebugMode()) {
debug_filter_ = new DebugFilter(this);
AddOwnedPostRenderFilter(debug_filter_);
}
// NOTE(abliss): Adding a new filter? Does it export any statistics? If it
// doesn't, it probably should. If it does, be sure to add it to the
// InitStats() function above or it will break under Apache!
}
void RewriteDriver::AddOwnedEarlyPreRenderFilter(HtmlFilter* filter) {
filters_to_delete_.push_back(filter);
early_pre_render_filters_.push_back(filter);
}
void RewriteDriver::PrependOwnedPreRenderFilter(HtmlFilter* filter) {
filters_to_delete_.push_back(filter);
pre_render_filters_.push_front(filter);
}
void RewriteDriver::AppendOwnedPreRenderFilter(HtmlFilter* filter) {
filters_to_delete_.push_back(filter);
pre_render_filters_.push_back(filter);
}
void RewriteDriver::AppendUnownedPreRenderFilter(HtmlFilter* filter) {
pre_render_filters_.push_back(filter);
}
void RewriteDriver::AddOwnedPostRenderFilter(HtmlFilter* filter) {
filters_to_delete_.push_back(filter);
AddUnownedPostRenderFilter(filter);
}
void RewriteDriver::AddUnownedPostRenderFilter(HtmlFilter* filter) {
HtmlParse::AddFilter(filter);
}
void RewriteDriver::AppendRewriteFilter(RewriteFilter* filter) {
CHECK(filter != NULL);
RegisterRewriteFilter(filter);
pre_render_filters_.push_back(filter);
}
void RewriteDriver::PrependRewriteFilter(RewriteFilter* filter) {
CHECK(filter != NULL);
RegisterRewriteFilter(filter);
pre_render_filters_.push_front(filter);
}
void RewriteDriver::AddResourceUrlClaimant(ResourceUrlClaimant* claimant) {
CHECK(claimant != NULL);
resource_claimants_.push_back(claimant);
}
void RewriteDriver::EnableRewriteFilter(const char* id) {
RewriteFilter* filter = resource_filter_map_[id];
CHECK(filter != NULL);
pre_render_filters_.push_back(filter);
}
void RewriteDriver::RegisterRewriteFilter(RewriteFilter* filter) {
// Track resource_fetches if we care about statistics. Note that
// the statistics are owned by the server context, which generally
// should be set up prior to the rewrite_driver.
//
// TODO(sligocki): It'd be nice to get this into the constructor.
resource_filter_map_[filter->id()] = filter;
filters_to_delete_.push_back(filter);
}
void RewriteDriver::SetWriter(Writer* writer) {
writer_ = writer;
if (html_writer_filter_ == NULL) {
if (options()->Enabled(RewriteOptions::kCachePartialHtml) &&
flushed_cached_html_) {
html_writer_filter_.reset(new CacheHtmlFilter(this));
} else if (options()->Enabled(RewriteOptions::kFlushSubresources) &&
flushing_early_) {
// If we are flushing early using this RewriteDriver object, we use the
// FlushEarlyContentWriterFilter.
html_writer_filter_.reset(new FlushEarlyContentWriterFilter(this));
} else if (options()->Enabled(RewriteOptions::kSplitHtml)) {
html_writer_filter_.reset(new SplitHtmlFilter(this));
} else if (options()->Enabled(RewriteOptions::kFlushSubresources)) {
html_writer_filter_.reset(new SuppressPreheadFilter(this));
} else {
html_writer_filter_.reset(new HtmlWriterFilter(this));
}
html_writer_filter_->set_case_fold(options()->lowercase_html_names());
if (options()->Enabled(RewriteOptions::kHtmlWriterFilter)) {
HtmlParse::AddFilter(html_writer_filter_.get());
}
}
html_writer_filter_->set_writer(writer);
}
Statistics* RewriteDriver::statistics() const {
return (server_context_ == NULL) ? NULL : server_context_->statistics();
}
void RewriteDriver::SetSessionFetcher(UrlAsyncFetcher* f) {
url_async_fetcher_ = f;
owned_url_async_fetchers_.push_back(f);
}
CacheUrlAsyncFetcher* RewriteDriver::CreateCustomCacheFetcher(
UrlAsyncFetcher* base_fetcher) {
return server_context()->CreateCustomCacheFetcher(
options(), CacheFragment(), cache_url_async_fetcher_async_op_hooks_.get(),
base_fetcher);
}
CacheUrlAsyncFetcher* RewriteDriver::CreateCacheFetcher() {
if (options()->mob_iframe() && !options()->mob_config()) {
IframeFetcher* ifetcher = new IframeFetcher(
options(), server_context_->user_agent_matcher(),
url_async_fetcher_);
CacheUrlAsyncFetcher* cache_fetcher = CreateCustomCacheFetcher(ifetcher);
cache_fetcher->set_own_fetcher(true);
return cache_fetcher;
}
return CreateCustomCacheFetcher(url_async_fetcher_);
}
CacheUrlAsyncFetcher* RewriteDriver::CreateCacheOnlyFetcher() {
return CreateCustomCacheFetcher(NULL);
}
bool RewriteDriver::Decode(StringPiece leaf,
ResourceNamer* resource_namer) const {
return resource_namer->Decode(
leaf, server_context()->hasher()->HashSizeInChars(), SignatureLength());
}
int RewriteDriver::SignatureLength() const {
return options()->url_signing_key().empty()
? 0
: options()->sha1signature()->SignatureSizeInChars();
}
bool RewriteDriver::DecodeOutputResourceNameHelper(
const GoogleUrl& gurl,
const RewriteOptions* options_to_use,
const UrlNamer* url_namer,
ResourceNamer* namer_out,
OutputResourceKind* kind_out,
RewriteFilter** filter_out,
GoogleString* url_base,
StringVector* urls) const {
// In forward proxy in preserve-URLs mode we want to fetch .pagespeed.
// resource, i.e. do not decode and and do not fetch original (especially
// that encoded one will never be cached internally).
if (options_to_use != NULL && options_to_use->oblivious_pagespeed_urls()) {
return false;
}
// First, we can't handle anything that's not a valid URL nor is named
// properly as our resource.
if (!gurl.IsWebValid()) {
return false;
}
StringPiece name = gurl.LeafSansQuery();
if (!Decode(name, namer_out)) {
return false;
}
// URLs without any hash are rejected as well, as they do not produce
// OutputResources with a computable URL. (We do accept 'wrong' hashes since
// they could come up legitimately under some asynchrony scenarios)
if (namer_out->hash().empty()) {
return false;
}
GoogleString decoded_url;
// If we are running in proxy mode we need to ignore URLs where the leaf is
// encoded but the URL as a whole isn't proxy encoded, since that can happen
// when proxying from a server using mod_pagespeed.
//
// This is also important for XSS avoidance when running in proxy mode with
// a relaxed lawyer, as it ensures that resources will only ever go under
// the low-privilege proxy domain and not the trusted site domain.
//
// If we are running in proxy mode and the URL is in the proxy domain, we
// also need to ensure that the URL decodes correctly as otherwise we end
// up with an invalid decoded base URL, which ultimately leads to inability
// to rewrite the URL.
if (url_namer->ProxyMode()) {
if (!url_namer->IsProxyEncoded(gurl)) {
message_handler()->Message(kInfo,
"Decoding of resource name %s failed because "
"it is not proxy encoded.",
gurl.spec_c_str());
return false;
} else if (!url_namer->Decode(gurl, options_to_use, NULL, &decoded_url)) {
message_handler()->Message(kInfo,
"Decoding of resource name %s failed because "
" the URL namer cannot decode it.",
gurl.spec_c_str());
return false;
}
GoogleUrl decoded_gurl(decoded_url);
if (decoded_gurl.IsWebValid()) {
*url_base = (decoded_gurl.AllExceptLeaf()).as_string();
} else {
return false;
}
} else {
*url_base = (gurl.AllExceptLeaf()).as_string();
}
// Now let's reject as mal-formed if the id string is not
// in the rewrite drivers. Also figure out the filter's preferred
// resource kind.
StringPiece id = namer_out->id();
GoogleString id_str(id.data(), id.size());
*kind_out = kRewrittenResource;
StringFilterMap::const_iterator p = resource_filter_map_.find(id_str);
if (p != resource_filter_map_.end()) {
*filter_out = p->second;
if ((*filter_out)->ComputeOnTheFly()) {
*kind_out = kOnTheFlyResource;
}
} else if ((id == CssOutlineFilter::kFilterId) ||
(id == JsOutlineFilter::kFilterId)) {
// OutlineFilter is special because it's not a RewriteFilter -- it's
// just an HtmlFilter, but it does encode rewritten resources that
// must be served from the cache.
//
// TODO(jmarantz): figure out a better way to refactor this.
// TODO(jmarantz): add a unit-test to show serving outline-filter resources.
*kind_out = kOutlinedResource;
*filter_out = NULL;
} else {
message_handler()->Message(kInfo,
"Decoding of resource name %s failed because "
" there is no filter with id %s.",
gurl.spec_c_str(), id_str.c_str());
return false;
}
// Check if filter-specific decoding works as well.
// TODO(morlovich): This is doing some redundant work.
if (*filter_out != NULL) {
ResourceContext resource_context;
if (!(*filter_out)->encoder()->Decode(
namer_out->name(), urls, &resource_context, message_handler())) {
message_handler()->Message(kInfo,
"Decoding of resource name %s failed because "
" filter %s cannot decode the URL.",
gurl.spec_c_str(), (*filter_out)->Name());
return false;
}
}
// Check if the id string's filter is forbidden and reject the URL if so.
if (options_to_use->Forbidden(id_str)) {
message_handler()->Message(kInfo,
"Decoding of resource name %s failed because "
" filter_id %s is forbidden.",
gurl.spec_c_str(), id_str.c_str());
return false;
}
return true;
}
bool RewriteDriver::DecodeOutputResourceName(
const GoogleUrl& gurl,
const RewriteOptions* options_to_use,
const UrlNamer* url_namer,
ResourceNamer* namer_out,
OutputResourceKind* kind_out,
RewriteFilter** filter_out) const {
StringVector urls;
GoogleString url_base;
return DecodeOutputResourceNameHelper(
gurl, options_to_use, url_namer, namer_out, kind_out,
filter_out, &url_base, &urls);
}
bool RewriteDriver::DecodeUrl(const GoogleUrl& url,
StringVector* decoded_urls) const {
return DecodeUrlGivenOptions(url, options(),
server_context()->url_namer(), decoded_urls);
}
bool RewriteDriver::DecodeUrlGivenOptions(
const GoogleUrl& url,
const RewriteOptions* options,
const UrlNamer* url_namer,
StringVector* decoded_urls) const {
ResourceNamer namer;
OutputResourceKind kind;
RewriteFilter* filter = NULL;
GoogleString url_base;
bool is_decoded = DecodeOutputResourceNameHelper(
url, options, url_namer, &namer, &kind, &filter, &url_base, decoded_urls);
if (is_decoded) {
GoogleUrl gurl_base(url_base);
for (int i = 0, n = decoded_urls->size(); i < n; ++i) {
GoogleUrl full_url(gurl_base, (*decoded_urls)[i]);
(*decoded_urls)[i] = full_url.Spec().as_string();
}
}
return is_decoded;
}
OutputResourcePtr RewriteDriver::DecodeOutputResource(
const GoogleUrl& gurl,
RewriteFilter** filter) const {
ResourceNamer namer;
OutputResourceKind kind;
if (!DecodeOutputResourceName(gurl, options(), server_context()->url_namer(),
&namer, &kind, filter)) {
return OutputResourcePtr();
}
StringPiece base = gurl.AllExceptLeaf();
OutputResourcePtr output_resource(
new OutputResource(this, base, base, base, namer, kind));
if (!output_resource.get()->CheckSignature()) {
output_resource.clear();
}
return output_resource;
}
namespace {
class FilterFetch : public SharedAsyncFetch {
public:
FilterFetch(RewriteDriver* driver, AsyncFetch* async_fetch)
: SharedAsyncFetch(async_fetch),
driver_(driver) {
}
virtual ~FilterFetch() {}
static bool Start(RewriteFilter* filter,
const OutputResourcePtr& output_resource,
AsyncFetch* async_fetch,
MessageHandler* handler) {
RewriteDriver* driver = filter->driver();
FilterFetch* filter_fetch = new FilterFetch(driver, async_fetch);
bool queued = false;
RewriteContext* context = filter->MakeRewriteContext();
DCHECK(context != NULL);
if (context != NULL) {
queued = context->Fetch(output_resource, filter_fetch, handler);
}
if (!queued) {
RewriteStats* stats = driver->server_context()->rewrite_stats();
stats->failed_filter_resource_fetches()->Add(1);
async_fetch->Done(false);
driver->FetchComplete();
delete filter_fetch;
}
return queued;
}
protected:
virtual void HandleDone(bool success) {
RewriteStats* stats = driver_->server_context()->rewrite_stats();
if (success) {
stats->succeeded_filter_resource_fetches()->Add(1);
} else {
stats->failed_filter_resource_fetches()->Add(1);
}
SharedAsyncFetch::HandleDone(success);
driver_->FetchComplete();
delete this;
}
private:
RewriteDriver* driver_;
};
class CacheCallback : public OptionsAwareHTTPCacheCallback {
public:
CacheCallback(RewriteDriver* driver,
RewriteFilter* filter,
const OutputResourcePtr& output_resource,
AsyncFetch* async_fetch,
MessageHandler* handler)
: OptionsAwareHTTPCacheCallback(driver->options(),
async_fetch->request_context()),
driver_(driver),
filter_(filter),
output_resource_(output_resource),
async_fetch_(async_fetch),
handler_(handler) {
// Canonicalize the URL before looking it up. Applies
// rewrite-domain mappings, and reverses any sharding. E.g.
// if you have
// ModPagespeedMapRewriteDomain master alias
// ModPagespeedShardDomain master shard1,shard2
// then this will convert:
// http://alias/foo --> http://master/foo
// http://shard1/foo --> http://master/foo
// http://shard2/foo --> http://master/foo
// http://master/foo --> http://master/foo
canonical_url_ = output_resource_->HttpCacheKey();
}
virtual ~CacheCallback() {}
void Find() {
ServerContext* server_context = driver_->server_context();
HTTPCache* http_cache = server_context->http_cache();
http_cache->Find(canonical_url_, driver_->CacheFragment(), handler_, this);
}
bool IsCacheValid(const GoogleString& key, const ResponseHeaders& headers) {
// If the user cares, don't try to send a rewritten .pagespeed. webp
// resources to a browser that can't handle it.
if (!driver_->options()->serve_rewritten_webp_urls_to_any_agent() &&
(headers.DetermineContentType() == &kContentTypeWebp) &&
!async_fetch_->request_context()->accepts_webp()) {
return false;
}
return OptionsAwareHTTPCacheCallback::IsCacheValid(key, headers);
}
virtual void Done(HTTPCache::FindResult find_result) {
StringPiece content;
ResponseHeaders* response_headers = async_fetch_->response_headers();
if (find_result.status == HTTPCache::kFound) {
RewriteStats* stats = driver_->server_context()->rewrite_stats();
stats->cached_resource_fetches()->Add(1);
HTTPValue* value = http_value();
bool success = (value->ExtractContents(&content) &&
value->ExtractHeaders(response_headers, handler_));
if (success) {
output_resource_->Link(value, handler_);
output_resource_->SetWritten(true);
async_fetch_->set_content_length(content.size());
async_fetch_->FixCacheControlForGoogleCache();
async_fetch_->HeadersComplete();
success = async_fetch_->Write(content, handler_);
}
async_fetch_->Done(success);
driver_->FetchComplete();
delete this;
} else {
if (output_resource_->IsWritten()) {
// OutputResources can also be loaded while not in cache if
// FetchOutputResource() somehow got called on an already written
// resource object (while the cache somehow decided not to store it).
content = output_resource_->ExtractUncompressedContents();
response_headers->CopyFrom(*output_resource_->response_headers());
ServerContext* server_context = driver_->server_context();
HTTPCache* http_cache = server_context->http_cache();
http_cache->Put(canonical_url_, driver_->CacheFragment(),
RequestHeaders::Properties(),
(ResponseHeaders::GetVaryOption(
driver_->options()->respect_vary())),
response_headers, content, handler_);
async_fetch_->Done(async_fetch_->Write(content, handler_));
driver_->FetchComplete();
} else {
// Use the filter to reconstruct.
if (filter_ != NULL) {
FilterFetch::Start(filter_, output_resource_, async_fetch_, handler_);
} else {
response_headers->SetStatusAndReason(HttpStatus::kNotFound);
async_fetch_->Done(false);
driver_->FetchComplete();
}
}
delete this;
}
}
private:
RewriteDriver* driver_;
RewriteFilter* filter_;
OutputResourcePtr output_resource_;
AsyncFetch* async_fetch_;
MessageHandler* handler_;
GoogleString canonical_url_;
};
// A fetch that writes back to the base fetch, takes care of a few stats,
// and can recover from an early (before HeadersComplete) fetcher error by
// ignoring subsequent writes and calling FetchResources() on the driver once
// Done is called.
class DistributedFetchResourceFetch : public SharedAsyncFetch {
public:
// Increments the driver's async_events_count to ensure that it survives
// as long as the fetch does.
explicit DistributedFetchResourceFetch(AsyncFetch* base_fetch,
RewriteDriver* driver)
: SharedAsyncFetch(base_fetch),
driver_(driver),
early_failure_(false),
driver_fetch_(base_fetch),
url_(driver->fetch_url().as_string()) {
RequestHeaders* new_req_headers = new RequestHeaders();
new_req_headers->CopyFrom(*driver_->request_headers());
SetRequestHeadersTakingOwnership(new_req_headers);
}
virtual ~DistributedFetchResourceFetch() {}
// Subclasses should override HandleHeadersComplete and set early_failure_
// to true if a recoverable failure is detected.
virtual void HandleHeadersComplete() {
if (response_headers()->status_code() >=
HttpStatus::kProxyPublisherFailure) {
// Was it an instaweb failure? If so, we'll make note of that and try
// again locally.
early_failure_ = true;
} else {
SharedAsyncFetch::HandleHeadersComplete();
}
}
virtual void HandleDone(bool success) {
// Bump the stats.
if (success) {
driver_->statistics()
->GetVariable(RewriteContext::kNumDistributedRewriteSuccesses)
->Add(1);
} else {
driver_->statistics()
->GetVariable(RewriteContext::kNumDistributedRewriteFailures)->Add(1);
}
if (early_failure_) {
// Perhaps an RPC error? We can recover from this state since we haven't
// written anything to the base fetch yet. Tell the driver to try again
// but this time don't distribute the request because
// tried_to_distribute_fetch_ is true.
driver_->FetchResource(url_, driver_fetch_);
} else {
SharedAsyncFetch::HandleDone(success);
}
driver_->DecrementAsyncEventsCount();
delete this;
}
virtual bool HandleWrite(const StringPiece& content,
MessageHandler* handler) {
if (early_failure_) {
return true;
} else {
return SharedAsyncFetch::HandleWrite(content, handler);
}
}
void DispatchFetch() {
StringPiece distributed_key = driver_->options()->distributed_rewrite_key();
request_headers()->Add(HttpAttributes::kXPsaDistributedRewriteFetch,
distributed_key);
// Nested driver fetches are not supposed to use deadlines, so block the
// distributed rewrite.
if (driver_->is_nested()) {
request_headers()->Add(HttpAttributes::kXPsaDistributedRewriteBlock,
distributed_key);
}
RewriteOptionsManager* rewrite_options_manager =
driver_->server_context()->rewrite_options_manager();
GoogleString url = driver_->fetch_url().as_string();
driver_->IncrementAsyncEventsCount();
rewrite_options_manager->PrepareRequest(
driver_->options(), driver_->request_context(), &url, request_headers(),
NewCallback(this, &DistributedFetchResourceFetch::StartFetch));
}
void StartFetch(bool success) {
if (success) {
driver_->distributed_fetcher()->Fetch(driver_->fetch_url().as_string(),
driver_->message_handler(), this);
} else {
// We failed. Try fetching again, but this time we won't distribute
// because tried_to_distribute_fetch_ is true.
driver_->FetchResource(driver_->fetch_url(), driver_fetch_);
driver_->DecrementAsyncEventsCount();
delete this;
}
}
private:
// This class increments the asynchronous event count on the RewriteDriver to
// ensure that it stays alive as long as the fetch does.
RewriteDriver* driver_;
bool early_failure_;
AsyncFetch* driver_fetch_; // This is owned externally.
GoogleString url_;
DISALLOW_COPY_AND_ASSIGN(DistributedFetchResourceFetch);
};
} // namespace
bool RewriteDriver::ShouldDistributeFetch(const StringPiece& filter_id) {
// TODO(jkarlin): There is also a RewriteContext::ShouldDistributeFetch
// intended for the HTML-path but not the fetch paths. Consolidate the code if
// reasonable.
if (distributed_fetcher() == NULL ||
!options()->Distributable(filter_id) ||
!options()->distribute_fetches() ||
tried_to_distribute_fetch_ ||
options()->distributed_rewrite_key().empty() ||
options()->distributed_rewrite_servers().empty()) {
return false;
}
// Don't redistribute an already distributed rewrite. Note: We don't verify
// the distributed rewrite key because we want to be conservative about loop
// detection.
DCHECK(request_headers() != NULL);
if (request_headers() != NULL) {
if (request_headers()->Has(HttpAttributes::kXPsaDistributedRewriteFetch) ||
request_headers()->Has(
HttpAttributes::kXPsaDistributedRewriteForHtml)) {
return false;
}
}
return true;
}
bool RewriteDriver::DistributeFetch(const StringPiece& url,
const StringPiece& filter_id,
AsyncFetch* async_fetch) {
if (!ShouldDistributeFetch(filter_id)) {
return false;
}
DistributedFetchResourceFetch* dist_fetch =
new DistributedFetchResourceFetch(async_fetch, this);
tried_to_distribute_fetch_ = true;
// The following line might delete 'this' and clean up the RewriteDriver if it
// finishes fast enough so don't touch those things afterwards.
dist_fetch->DispatchFetch();
return true;
}
bool RewriteDriver::FetchResource(const StringPiece& url,
AsyncFetch* async_fetch) {
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchBackground));
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefParsing));
bool handled = false;
fetch_url_ = url.as_string();
// Set the request headers if they haven't been yet.
if (request_headers_ == NULL && async_fetch->request_headers() != NULL) {
SetRequestHeaders(*async_fetch->request_headers());
}
// Note that this does permission checking and parsing of the url, but doesn't
// actually fetch any data until we specifically ask it to.
RewriteFilter* filter = NULL;
GoogleUrl gurl(url);
OutputResourcePtr output_resource(DecodeOutputResource(gurl, &filter));
if (output_resource.get() != NULL) {
handled = true;
if (filter != NULL) {
// TODO(marq): This is a gross generalization. Remove this and properly
// log the application of each rewrite filter.
filter->LogFilterModifiedContent();
}
FetchOutputResource(output_resource, filter, async_fetch);
} else if (options()->in_place_rewriting_enabled()) {
// TODO(jcrowell): Make URLs with signatures take this path so they will 403
// instead of 404.
// This is an ajax resource.
handled = true;
// TODO(sligocki): Get rid of this fallback and make all callers call
// FetchInPlaceResource when that is what they want.
FetchInPlaceResource(gurl, true /* proxy_mode */, async_fetch);
}
// Note: "this" may have been deleted by this point. It is not safe to
// reference data members.
return handled;
}
void RewriteDriver::FetchInPlaceResource(const GoogleUrl& gurl,
bool proxy_mode,
AsyncFetch* async_fetch) {
CHECK(gurl.IsWebValid()) << "Invalid URL " << gurl.spec_c_str();
CHECK(request_headers_.get() != NULL);
gurl.Spec().CopyToString(&fetch_url_);
StringPiece base = gurl.AllExceptLeaf();
ResourceNamer namer;
OutputResourcePtr output_resource(
new OutputResource(this, base, base, base, namer, kRewrittenResource));
SetBaseUrlForFetch(gurl.Spec());
// Set the request headers if they haven't been yet.
if (request_headers_ == NULL && async_fetch->request_headers() != NULL) {
SetRequestHeaders(*async_fetch->request_headers());
}
if (DistributeFetch(fetch_url_, RewriteOptions::kInPlaceRewriteId,
async_fetch)) {
return;
}
ref_counts_.AddRef(kRefFetchUserFacing);
InPlaceRewriteContext* context = new InPlaceRewriteContext(this, gurl.Spec());
context->set_proxy_mode(proxy_mode);
// Save pointer to stats_logger before "this" is deleted.
StatisticsLogger* stats_logger =
server_context_->statistics()->console_logger();
if (!context->Fetch(output_resource, async_fetch, message_handler())) {
// RewriteContext::Fetch can fail if the input URLs are undecodeable
// or unfetchable. There is no decoding in this case, but unfetchability
// is possible if we're given an https URL but have a fetcher that
// can't do it. In that case, the only thing we can do is fail
// and cleanup.
async_fetch->Done(false);
FetchComplete();
}
// Note: "this" may have been deleted by this point. It is not safe to
// reference data members.
// Update statistics log.
if (stats_logger != NULL) {
stats_logger->UpdateAndDumpIfRequired();
}
}
bool RewriteDriver::FetchOutputResource(
const OutputResourcePtr& output_resource,
RewriteFilter* filter,
AsyncFetch* async_fetch) {
if (DistributeFetch(output_resource->url(), output_resource->filter_prefix(),
async_fetch)) {
// TODO(jkarlin): This doesn't fill in the output_resource with the result
// of the fetch. Right now I believe the only thing expecting data to be in
// the output_resource is a nested_driver fetch in
// RewriteContext::FetchInputs (which calls FetchResource) but it currently
// copies from the fetch into the OutputResource anyway so nothing is broken
// yet. One option is to change the first parameter of FetchOutputResource
// to a URL instead of an OutputResourcePtr.
return true;
}
// None of our resources ever change -- the hash of the content is embedded
// in the filename. This is why we serve them with very long cache
// lifetimes. However, when the user presses Reload, the browser may
// attempt to validate that the cached copy is still fresh by sending a GET
// with an If-Modified-Since header. If this header is present, we should
// return a 304 Not Modified, since any representation of the resource
// that's in the browser's cache must be correct.
bool queued = false;
ConstStringStarVector values;
// Save pointer to stats_logger before "this" is deleted.
StatisticsLogger* stats_logger =
server_context_->statistics()->console_logger();
if (async_fetch->request_headers()->Lookup(HttpAttributes::kIfModifiedSince,
&values)) {
async_fetch->response_headers()->SetStatusAndReason(
HttpStatus::kNotModified);
async_fetch->HeadersComplete();
async_fetch->Done(true);
queued = false;
} else {
SetBaseUrlForFetch(output_resource->url());
ref_counts_.AddRef(kRefFetchUserFacing);
if (output_resource->kind() == kOnTheFlyResource ||
MetadataRequested(*async_fetch->request_headers())) {
// Don't bother to look up the resource in the cache: ask the filter. If
// metadata is requested we need to skip the initial http cache lookup
// because we can't return until we've done a metadata lookup first.
if (filter != NULL) {
queued = FilterFetch::Start(filter, output_resource, async_fetch,
message_handler());
}
} else {
CacheCallback* cache_callback = new CacheCallback(
this, filter, output_resource, async_fetch, message_handler());
cache_callback->Find();
queued = true;
}
}
// Update statistics log.
if (stats_logger != NULL) {
stats_logger->UpdateAndDumpIfRequired();
}
return queued;
}
void RewriteDriver::FetchComplete() {
DropReference(kRefFetchUserFacing);
}
void RewriteDriver::DetachFetch() {
ScopedMutex lock(rewrite_mutex());
CHECK_EQ(1, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
CHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchBackground));
ref_counts_.AddRefMutexHeld(kRefFetchBackground);
}
void RewriteDriver::DetachedFetchComplete() {
DropReference(kRefFetchBackground);
}
bool RewriteDriver::MayRewriteUrl(
const GoogleUrl& domain_url,
const GoogleUrl& input_url,
InlineAuthorizationPolicy inline_authorization_policy,
IntendedFor intended_for,
bool* is_authorized_domain) const {
*is_authorized_domain = false;
if (domain_url.IsWebValid()) {
if (options()->IsAllowed(input_url.Spec()) ||
(intended_for == kIntendedForInlining &&
options()->IsAllowedWhenInlining(input_url.Spec()))) {
*is_authorized_domain = options()->domain_lawyer()->IsDomainAuthorized(
domain_url, input_url);
if (!*is_authorized_domain &&
inline_authorization_policy == kInlineUnauthorizedResources) {
// We decide that this URL can be rewritten (true) but
// is_authorized_domain will be retained as false to allow creation of
// the Resource object in the correct cache key space.
return true;
}
}
}
return *is_authorized_domain;
}
bool RewriteDriver::MatchesBaseUrl(const GoogleUrl& input_url) const {
return (decoded_base_url_.IsWebValid() &&
options()->IsAllowed(input_url.Spec()) &&
decoded_base_url_.Origin() == input_url.Origin());
}
ResourcePtr RewriteDriver::CreateInputResource(const GoogleUrl& input_url,
bool* is_authorized) {
return CreateInputResource(
input_url, kInlineOnlyAuthorizedResources, kIntendedForGeneral,
is_authorized);
}
ResourcePtr RewriteDriver::CreateInputResource(
const GoogleUrl& input_url,
InlineAuthorizationPolicy inline_authorization_policy,
IntendedFor intended_for,
bool* is_authorized) {
*is_authorized = true; // Must be false iff we fail b/c of authorization.
ResourcePtr resource;
bool may_rewrite = false;
if (input_url.SchemeIs("data")) {
// Skip and silently ignore; don't log a failure.
// For the moment we assume data: urls are small enough to not be worth
// optimizing. We have optimized them in the past, but that code is likely
// to have bit-rotted since it was disabled.
return resource;
} else if (decoded_base_url_.IsAnyValid()) {
may_rewrite = MayRewriteUrl(decoded_base_url_, input_url,
inline_authorization_policy,
intended_for,
is_authorized);
// In the case where we are proxying and we have resources that have been
// rewritten multiple times, input_url will still have the encoded domain,
// and we can rewrite that, so test again but against the encoded base url.
if (!may_rewrite) {
UrlNamer* namer = server_context()->url_namer();
GoogleString decoded_input;
if (namer->Decode(input_url, options(), NULL, &decoded_input)) {
GoogleUrl decoded_url(decoded_input);
may_rewrite = MayRewriteUrl(decoded_base_url_, decoded_url,
inline_authorization_policy,
intended_for,
is_authorized);
}
}
} else {
// Shouldn't happen?
message_handler()->Message(
kFatal, "invalid decoded_base_url_ for '%s'", input_url.spec_c_str());
LOG(DFATAL);
}
RewriteStats* stats = server_context_->rewrite_stats();
if (may_rewrite) {
// *is_authorized may be true or false (if inlining an unauth'd URL).
resource = CreateInputResourceUnchecked(input_url, *is_authorized);
stats->resource_url_domain_acceptances()->Add(1);
} else {
DCHECK(!*is_authorized);
message_handler()->Message(kInfo, "No permission to rewrite '%s'",
input_url.spec_c_str());
stats->resource_url_domain_rejections()->Add(1);
}
return resource;
}
ResourcePtr RewriteDriver::CreateInputResourceAbsoluteUncheckedForTestsOnly(
const StringPiece& absolute_url) {
GoogleUrl url(absolute_url);
if (!url.IsWebOrDataValid()) {
// Note: Bad user-content can leave us here. But it's really hard
// to concatenate a valid protocol and domain onto an arbitrary string
// and end up with an invalid GURL.
message_handler()->Message(kInfo, "Invalid resource url '%s'",
url.spec_c_str());
return ResourcePtr();
}
return CreateInputResourceUnchecked(url, true);
}
ResourcePtr RewriteDriver::CreateInputResourceUnchecked(
const GoogleUrl& url,
bool is_authorized_domain) {
StringPiece url_string = url.Spec();
ResourcePtr resource;
if (IsResourceUrlClaimed(url)) {
return resource;
}
if (url.SchemeIs("data")) {
resource = DataUrlInputResource::Make(url_string, this);
if (resource.get() == NULL) {
// Note: Bad user-content can leave us here.
message_handler()->Message(kWarning, "Badly formatted data url '%s'",
url.spec_c_str());
}
} else if (url.SchemeIs("http") || url.SchemeIs("https")) {
// Note: type may be NULL if url has an unexpected or malformed extension.
const ContentType* type = NameExtensionToContentType(url.LeafSansQuery());
GoogleString filename;
if (options()->file_load_policy()->ShouldLoadFromFile(url, &filename)) {
resource.reset(
new FileInputResource(this, type, url_string, filename));
} else {
// If the scheme is https and the fetcher doesn't support https, map
// the URL to what will ultimately be fetched to see if that will be
// http, in which case the fetcher will be able to handle it.
GoogleString mapped_url;
GoogleString host_header;
bool is_proxy = false;
options()->domain_lawyer()->MapOriginUrl(url, &mapped_url,
&host_header, &is_proxy);
GoogleUrl mapped_gurl(mapped_url);
if (mapped_gurl.SchemeIs("http") ||
(mapped_gurl.SchemeIs("https") &&
url_async_fetcher_->SupportsHttps())) {
resource.reset(new UrlInputResource(this, type, url_string,
is_authorized_domain));
} else {
message_handler()->Message(
kInfo, "Cannot fetch url '%s': as %s is not supported",
url.spec_c_str(), mapped_gurl.Scheme().as_string().c_str());
}
}
} else {
// Note: Valid user-content can leave us here.
// Specifically, any URLs with scheme other than data: or http: or https:.
// TODO(sligocki): Is this true? Or will such URLs not make it this far?
message_handler()->Message(kWarning, "Unsupported scheme '%s' for url '%s'",
url.Scheme().as_string().c_str(),
url.spec_c_str());
}
return resource;
}
bool RewriteDriver::IsResourceUrlClaimed(const GoogleUrl& url) const {
for (int i = 0, n = resource_claimants_.size(); i < n; ++i) {
bool claims = false;
resource_claimants_[i]->Run(url, &claims);
if (claims) {
return true;
}
}
return false;
}
bool RewriteDriver::StartParseId(const StringPiece& url, const StringPiece& id,
const ContentType& content_type) {
if (response_headers_ != NULL) {
status_code_ = response_headers_->status_code();
}
start_time_ms_ = server_context_->timer()->NowMs();
set_log_rewrite_timing(options()->log_rewrite_timing());
if (debug_filter_ != NULL) {
debug_filter_->InitParse();
}
bool ret = HtmlParse::StartParseId(url, id, content_type);
if (ret) {
ScopedMutex lock(rewrite_mutex());
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefParsing));
ref_counts_.AddRefMutexHeld(kRefParsing);
}
if (ret) {
base_was_set_ = false;
if (is_url_valid()) {
base_url_.Reset(google_url());
SetDecodedUrlFromBase();
}
}
can_rewrite_resources_ = server_context_->metadata_cache()->IsHealthy();
return ret;
}
void RewriteDriver::ParseTextInternal(const char* content, int size) {
num_bytes_in_ += size;
if (ShouldSkipParsing()) {
writer()->Write(content, message_handler());
} else if (debug_filter_ != NULL) {
debug_filter_->StartParse();
HtmlParse::ParseTextInternal(content, size);
debug_filter_->EndParse();
} else {
HtmlParse::ParseTextInternal(content, size);
}
}
void RewriteDriver::SetDecodedUrlFromBase() {
UrlNamer* namer = server_context()->url_namer();
GoogleString decoded_base;
if (namer->Decode(base_url_, options(), NULL, &decoded_base)) {
decoded_base_url_.Reset(decoded_base);
} else {
decoded_base_url_.Reset(base_url_);
}
DCHECK(decoded_base_url_.IsAnyValid());
}
bool RewriteDriver::ShouldSkipParsing() {
if (should_skip_parsing_ == kNotSet) {
bool should_skip = false;
PropertyPage* page = property_page();
if (page != NULL) {
PropertyCache* pcache = server_context_->page_property_cache();
const PropertyCache::Cohort* dom_cohort = pcache->GetCohort(kDomCohort);
if (dom_cohort != NULL) {
PropertyValue* property_value = property_page()->GetProperty(
dom_cohort, kParseSizeLimitExceeded);
should_skip = property_value->has_value() &&
StringCaseEqual(property_value->value(), "1");
}
}
should_skip_parsing_ = should_skip ? kTrue : kFalse;
}
return (should_skip_parsing_ == kTrue);
}
bool RewriteDriver::PrepareShouldSignal() {
// Basically, we just save IsDone() from before state changes.
return IsDone(waiting_, waiting_deadline_reached_);
}
void RewriteDriver::SignalIfRequired(bool result_of_prepare_should_signal) {
// If we were already done before, or no one is waiting, no need to signal
if (result_of_prepare_should_signal || waiting_ == kNoWait) {
return;
}
if (IsDone(waiting_, waiting_deadline_reached_)) {
// If someone is waiting, refcount shouldn't be 0!
DCHECK(!release_driver_);
scheduler_->Signal();
}
}
void RewriteDriver::RewriteComplete(RewriteContext* rewrite_context,
bool permit_render) {
{
ScopedMutex lock(rewrite_mutex());
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
bool signal_cookie = PrepareShouldSignal();
bool attached = false;
// Rewrite transitions either pending -> deleting or detached -> deleting
ref_counts_.AddRefMutexHeld(kRefDeletingRewrites);
RewriteContextSet::iterator p = initiated_rewrites_.find(rewrite_context);
if (p != initiated_rewrites_.end()) {
if (rewrite_context->is_metadata_cache_miss()) {
// If the rewrite completed within the deadline and it actually involved
// and fetch rewrite (not a metadata hit or successful revalidate) then
// bump up the corresponding counter in log record.
ScopedMutex lock(log_record()->mutex());
MetadataCacheInfo* metadata_log_info =
log_record()->logging_info()->mutable_metadata_cache_info();
metadata_log_info->set_num_successful_rewrites_on_miss(
metadata_log_info->num_successful_rewrites_on_miss() + 1);
}
initiated_rewrites_.erase(p);
attached = true;
ref_counts_.ReleaseRefMutexHeld(kRefPendingRewrites);
if (!rewrite_context->slow()) {
--possibly_quick_rewrites_;
}
} else {
int erased = detached_rewrites_.erase(rewrite_context);
CHECK_EQ(1, erased) << " rewrite_context " << rewrite_context
<< " not in either detached_rewrites or "
<< "initiated_rewrites_";
ref_counts_.ReleaseRefMutexHeld(kRefDetachedRewrites);
}
// release_driver_ should be false since we moved a count between
// categories, and didn't change the total.
DCHECK(!release_driver_) << ref_counts_.DebugStringMutexHeld();
rewrite_context->Propagate(attached && permit_render);
SignalIfRequired(signal_cookie);
}
}
void RewriteDriver::ReportSlowRewrites(int num) {
ScopedMutex lock(rewrite_mutex());
bool signal_cookie = PrepareShouldSignal();
possibly_quick_rewrites_ -= num;
CHECK_LE(0, possibly_quick_rewrites_) << base_url_.Spec();
SignalIfRequired(signal_cookie);
}
void RewriteDriver::DeleteRewriteContext(RewriteContext* rewrite_context) {
delete rewrite_context;
DropReference(kRefDeletingRewrites);
}
void RewriteDriver::PossiblyPurgeCachedResponseAndReleaseDriver() {
DCHECK(!externally_managed_);
// We might temporarily (due to purging) revive the object here, so
// better clear the "we were told it's dead!" bit.
release_driver_ = false;
if (downstream_cache_purger_.MaybeIssuePurge(google_url())) {
return;
}
server_context_->ReleaseRewriteDriver(this);
}
RewriteContext* RewriteDriver::RegisterForPartitionKey(
const GoogleString& partition_key, RewriteContext* candidate) {
std::pair<PrimaryRewriteContextMap::iterator, bool> insert_result =
primary_rewrite_context_map_.insert(
std::make_pair(partition_key, candidate));
if (insert_result.second) {
// Our value is new, so just return NULL.
return NULL;
} else {
// Insert failed, return the old value.
return insert_result.first->second;
}
}
void RewriteDriver::DeregisterForPartitionKey(const GoogleString& partition_key,
RewriteContext* rewrite_context) {
// If the context being deleted is the primary for some cache key,
// deregister it.
PrimaryRewriteContextMap::iterator i =
primary_rewrite_context_map_.find(partition_key);
if ((i != primary_rewrite_context_map_.end()) &&
(i->second == rewrite_context)) {
primary_rewrite_context_map_.erase(i);
}
}
void RewriteDriver::WriteDomCohortIntoPropertyCache() {
// Only update the property cache if there is a filter or option enabled that
// actually makes use of it.
if (!(write_property_cache_dom_cohort_ ||
options()->max_html_parse_bytes() > 0)) {
return;
}
PropertyPage* page = property_page();
// Dont update property cache value if we are flushing early.
// TODO(jud): Is this the best place to check for shutting down? It might
// make more sense for this check to be done at the property cache or
// lower level.
if (server_context_->shutting_down() ||
page == NULL ||
!owns_property_page_) {
return;
}
// Update the timestamp of the last request in both actual property page
// and property page with fallback values.
UpdatePropertyValueInDomCohort(
fallback_property_page(),
kLastRequestTimestamp,
Integer64ToString(server_context()->timer()->NowMs()));
// Update the status code of the last request.
if (status_code_ != HttpStatus::kUnknownStatusCode) {
UpdatePropertyValueInDomCohort(
fallback_property_page(),
kStatusCodePropertyName, IntegerToString(status_code_));
}
if (options()->max_html_parse_bytes() > 0) {
// Update whether the page exceeded the html parse size limit.
UpdatePropertyValueInDomCohort(
page, kParseSizeLimitExceeded,
num_bytes_in_ > options()->max_html_parse_bytes() ? "1" : "0");
}
if (flush_early_info_.get() != NULL) {
GoogleString value;
flush_early_info_->SerializeToString(&value);
UpdatePropertyValueInDomCohort(
fallback_property_page(), kSubresourcesPropertyName, value);
}
// Write dom cohort for both actual property page and property page with
// fallback values.
fallback_property_page()->WriteCohort(server_context()->dom_cohort());
}
void RewriteDriver::UpdatePropertyValueInDomCohort(
AbstractPropertyPage* page,
StringPiece property_name,
StringPiece property_value) {
if (page == NULL || !owns_property_page_) {
return;
}
page->UpdateValue(
server_context()->dom_cohort(), property_name, property_value);
}
void RewriteDriver::Cleanup() {
{
// TODO(morlovich): Clean this up, it's a rather inappropriate place to
// do this.
ScopedMutex lock(log_record()->mutex());
if (!log_record()->logging_info()->has_experiment_id()) {
log_record()->logging_info()->set_experiment_id(
options()->experiment_id());
}
}
DropReference(kRefUser);
}
void RewriteDriver::AddUserReference() {
ref_counts_.AddRef(kRefUser);
}
namespace {
void AppendBool(GoogleString* out, const char* name, bool val) {
StrAppend(out, name, ": ", val ? "true\n": "false\n");
}
} // namespace
GoogleString RewriteDriver::ToString(bool show_detached_contexts) const {
GoogleString out;
{
ScopedMutex lock(rewrite_mutex());
StrAppend(&out, "URL: ", google_url().Spec(), "\n");
StrAppend(&out, "decoded_base: ", decoded_base_url().Spec(), "\n");
AppendBool(&out, "base_was_set", base_was_set_);
StrAppend(&out, "containing_charset: ", containing_charset_, "\n");
AppendBool(&out, "filters_added", filters_added_);
AppendBool(&out, "externally_managed", externally_managed_);
switch (waiting_) {
case kNoWait:
StrAppend(&out, "waiting: kNoWait\n");
break;
case kWaitForCompletion:
StrAppend(&out, "waiting: kWaitForCompletion\n");
break;
case kWaitForCachedRender:
StrAppend(&out, "waiting: kWaitForCachedRender\n");
break;
case kWaitForShutDown:
StrAppend(&out, "waiting: kWaitForShutDown\n");
break;
default:
StrAppend(&out, "waiting: ", IntegerToString(waiting_));
break;
}
AppendBool(&out, "waiting_deadline_reached", waiting_deadline_reached_);
StrAppend(&out, "detached_rewrites_.size(): ",
IntegerToString(detached_rewrites_.size()), "\n");
if (show_detached_contexts) {
for (RewriteContextSet::iterator p = detached_rewrites_.begin(),
e = detached_rewrites_.end(); p != e; ++p) {
RewriteContext* detached_rewrite = *p;
StrAppend(&out, " Detached Rewrite:\n",
detached_rewrite->ToStringWithPrefix(" "));
}
}
AppendBool(&out, "RewritesComplete()", RewritesComplete());
AppendBool(&out, "fully_rewrite_on_flush", fully_rewrite_on_flush_);
AppendBool(&out, "fast_blocking_rewrite", fast_blocking_rewrite_);
AppendBool(&out, "flush_requested", flush_requested_);
AppendBool(&out, "flush_occurred", flush_occurred_);
AppendBool(&out, "flushed_early", flushed_early_);
AppendBool(&out, "flushing_early", flushing_early_);
AppendBool(&out, "is_lazyload_script_flushed", is_lazyload_script_flushed_);
AppendBool(&out, "release_driver", release_driver_);
AppendBool(&out, "write_property_cache_dom_cohort",
write_property_cache_dom_cohort_);
AppendBool(&out, "using_spdy", using_spdy());
AppendBool(&out, "owns_property_page", owns_property_page_);
AppendBool(&out, "xhtml_mimetype_computed", xhtml_mimetype_computed_);
AppendBool(&out, "can_rewrite_resources", can_rewrite_resources_);
AppendBool(&out, "is_nested", is_nested());
StrAppend(&out, "ref counts:\n", ref_counts_.DebugStringMutexHeld());
}
return out;
}
void RewriteDriver::PrintState(bool show_detached_contexts) {
fputs(ToString(show_detached_contexts).c_str(), stderr);
fputc('\n', stderr);
}
void RewriteDriver::PrintStateToErrorLog(bool show_detached_contexts) {
message_handler()->MessageS(kError, ToString(show_detached_contexts));
}
void RewriteDriver::LogStats() {
if (dom_stats_filter_ != NULL && log_record() != NULL) {
log_record()->SetImageStats(dom_stats_filter_->num_img_tags(),
dom_stats_filter_->num_inlined_img_tags(),
dom_stats_filter_->num_critical_images_used());
log_record()->SetResourceCounts(dom_stats_filter_->num_external_css(),
dom_stats_filter_->num_scripts());
}
request_properties_->LogDeviceInfo(
log_record(), options()->enable_aggressive_rewriters_for_mobile());
bool is_xhr = request_headers() != NULL &&
request_headers()->IsXmlHttpRequest();
log_record()->LogIsXhr(is_xhr);
}
void RewriteDriver::FinishParse() {
SchedulerBlockingFunction wait(scheduler_);
FinishParseAsync(&wait);
wait.Block();
}
void RewriteDriver::FinishParseAsync(Function* callback) {
HtmlParse::BeginFinishParse();
FlushAsync(
MakeFunction(this, &RewriteDriver::QueueFinishParseAfterFlush, callback));
}
void RewriteDriver::QueueFinishParseAfterFlush(Function* user_callback) {
Function* finish_parse = MakeFunction(this,
&RewriteDriver::FinishParseAfterFlush,
user_callback);
html_worker_->Add(finish_parse);
}
void RewriteDriver::FinishParseAfterFlush(Function* user_callback) {
DCHECK_EQ(0U, GetEventQueueSize());
HtmlParse::EndFinishParse();
LogStats();
WriteDomCohortIntoPropertyCache();
// Update stats.
RewriteStats* stats = server_context_->rewrite_stats();
stats->rewrite_latency_histogram()->Add(
server_context_->timer()->NowMs() - start_time_ms_);
stats->total_rewrite_count()->IncBy(1);
// Update statistics log.
StatisticsLogger* stats_logger =
server_context_->statistics()->console_logger();
if (stats_logger != NULL) {
stats_logger->UpdateAndDumpIfRequired();
}
DropReference(kRefParsing);
Cleanup();
if (user_callback != NULL) {
user_callback->CallRun();
}
}
void RewriteDriver::InfoAt(const RewriteContext* context,
const char* msg, ...) {
va_list args;
va_start(args, msg);
if ((context == NULL) || (context->num_slots() == 0)) {
InfoHereV(msg, args);
} else {
GoogleString new_msg;
for (int c = 0; c < context->num_slots(); ++c) {
StrAppend(&new_msg, context->slot(c)->LocationString(),
((c == context->num_slots() - 1) ? ": " : " "));
}
StringAppendV(&new_msg, msg, args);
message_handler()->MessageS(kInfo, new_msg);
}
va_end(args);
}
// Constructs name and URL for the specified input resource and encoder.
bool RewriteDriver::GenerateOutputResourceNameAndUrl(
const UrlSegmentEncoder* encoder,
const ResourceContext* data,
const ResourcePtr& input_resource,
GoogleString* name,
GoogleUrl* mapped_gurl,
GoogleString* failure_reason) {
if (input_resource.get() == NULL) {
*failure_reason = "No input resource.";
return false;
}
// TODO(jmarantz): It would be more efficient to pass in the base
// document GURL or save that in the input resource.
GoogleUrl unmapped_gurl(input_resource->url());
GoogleString mapped_domain; // Unused. TODO(sligocki): Stop setting this?
// Get the domain and URL after any domain lawyer rewriting.
if (!options()->IsAllowed(unmapped_gurl.Spec())) {
*failure_reason = StrCat("Rewriting disallowed for ", unmapped_gurl.Spec());
return false;
}
if (!options()->domain_lawyer()->MapRequestToDomain(
unmapped_gurl, unmapped_gurl.Spec(), &mapped_domain, mapped_gurl,
server_context_->message_handler())) {
*failure_reason = StrCat("Domain not authorized for ",
unmapped_gurl.Spec());
return false;
}
StringVector v;
v.push_back(mapped_gurl->LeafWithQuery().as_string());
encoder->Encode(v, data, name);
return true;
}
// Constructs an output resource corresponding to the specified input resource
// and encoded using the provided encoder.
OutputResourcePtr RewriteDriver::CreateOutputResourceFromResource(
const char* filter_id,
const UrlSegmentEncoder* encoder,
const ResourceContext* data,
const ResourcePtr& input_resource,
OutputResourceKind kind,
GoogleString* failure_reason) {
OutputResourcePtr result;
GoogleString name;
GoogleUrl mapped_gurl;
if (!GenerateOutputResourceNameAndUrl(encoder, data, input_resource, &name,
&mapped_gurl, failure_reason)) {
return result;
}
// TODO(jmarantz): It would be more efficient to pass in the base
// document GURL or save that in the input resource.
GoogleUrl unmapped_gurl(input_resource->url());
result.reset(CreateOutputResourceWithMappedPath(
mapped_gurl.AllExceptLeaf(), unmapped_gurl.AllExceptLeaf(),
filter_id, name, kind, failure_reason));
CHECK(input_resource->is_authorized_domain());
return result;
}
void RewriteDriver::PopulateResourceNamer(
const StringPiece& filter_id,
const StringPiece& name,
ResourceNamer* full_name) {
full_name->set_id(filter_id);
full_name->set_name(name);
full_name->set_experiment(options()->GetExperimentStateStr());
// Note that we never populate ResourceNamer::options for in place resource
// rewrites.
if (filter_id != RewriteOptions::kInPlaceRewriteId &&
!full_name->has_experiment() && options()->add_options_to_urls()) {
GoogleString resource_option = RewriteQuery::GenerateResourceOption(
filter_id, this);
full_name->set_options(resource_option);
} else {
full_name->set_options("");
}
}
OutputResourcePtr RewriteDriver::CreateOutputResourceWithPath(
const StringPiece& mapped_path,
const StringPiece& unmapped_path,
const StringPiece& base_url,
const StringPiece& filter_id,
const StringPiece& name,
OutputResourceKind kind,
GoogleString* failure_reason) {
ResourceNamer full_name;
PopulateResourceNamer(filter_id, name, &full_name);
OutputResourcePtr resource;
int max_leaf_size =
full_name.EventualSize(*server_context_->hasher(), SignatureLength()) +
ContentType::MaxProducedExtensionLength();
if (max_leaf_size > options()->max_url_segment_size()) {
*failure_reason = "Rewritten URL segment too long.";
return resource;
}
bool no_hash = false;
int extra_len = 0;
Hasher* hasher = server_context()->hasher();
if (full_name.hash().empty()) {
// Content and content type are not present. So set some nonzero hash and
// assume largest possible extension.
no_hash = true;
full_name.set_hash(GoogleString(hasher->HashSizeInChars(), '#'));
extra_len = ContentType::MaxProducedExtensionLength();
}
resource.reset(new OutputResource(
this, mapped_path, unmapped_path, base_url, full_name, kind));
if (options()->max_url_size() <
(static_cast<int>(resource->url().size()) + extra_len)) {
*failure_reason = StrCat("Rewritten URL too long: ", resource->url());
resource.clear();
return resource;
}
if (no_hash) {
resource->clear_hash();
}
return resource;
}
OutputResourcePtr RewriteDriver::CreateOutputResourceWithUnmappedUrl(
const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
const StringPiece& name, OutputResourceKind kind,
GoogleString* failure_reason) {
OutputResourcePtr resource;
GoogleString mapped_domain; // Unused. TODO(sligocki): Stop setting this?
GoogleUrl mapped_gurl;
// Get the domain and URL after any domain lawyer rewriting.
if (!options()->IsAllowed(unmapped_gurl.Spec())) {
*failure_reason = StrCat("Rewriting disallowed for ", unmapped_gurl.Spec());
return resource;
}
if (!options()->domain_lawyer()->MapRequestToDomain(
unmapped_gurl, unmapped_gurl.Spec(), &mapped_domain, &mapped_gurl,
server_context_->message_handler())) {
*failure_reason = StrCat("Domain not authorized for ",
unmapped_gurl.Spec());
return resource;
}
resource.reset(CreateOutputResourceWithMappedPath(
mapped_gurl.AllExceptLeaf(), unmapped_gurl.AllExceptLeaf(),
filter_id, name, kind, failure_reason));
return resource;
}
void RewriteDriver::SetBaseUrlIfUnset(const StringPiece& new_base) {
// Base url is relative to the document URL in HTML5, but not in
// HTML4.01. FF3.x does it HTML4.01 way, Chrome, Opera 11 and FF4
// betas do it according to HTML5, as is our implementation here.
GoogleUrl new_base_url(base_url_, new_base);
if (new_base_url.IsAnyValid()) {
if (base_was_set_) {
if (new_base_url.Spec() != base_url_.Spec()) {
InfoHere("Conflicting base tags: %s and %s",
new_base_url.spec_c_str(),
base_url_.spec_c_str());
}
} else {
base_was_set_ = true;
base_url_.Swap(&new_base_url);
SetDecodedUrlFromBase();
}
} else {
InfoHere("Invalid base tag %s relative to %s",
new_base.as_string().c_str(),
base_url_.spec_c_str());
}
}
void RewriteDriver::SetBaseUrlForFetch(const StringPiece& url) {
// Set the base url for the resource fetch. This corresponds to where the
// fetched resource resides (which might or might not be where the original
// resource lived).
// TODO(jmaessen): we're re-constructing a GoogleUrl after having already
// done so (repeatedly over several calls) in DecodeOutputResource! Gah!
// We at least assume that base_url_ is valid since it was checked when
// output_resource was created.
base_url_.Reset(url);
DCHECK(base_url_.IsAnyValid());
SetDecodedUrlFromBase();
base_was_set_ = false;
}
RewriteFilter* RewriteDriver::FindFilter(const StringPiece& id) const {
RewriteFilter* filter = NULL;
StringFilterMap::const_iterator p = resource_filter_map_.find(id.as_string());
if (p != resource_filter_map_.end()) {
filter = p->second;
}
return filter;
}
HtmlResourceSlotPtr RewriteDriver::GetSlot(
const ResourcePtr& resource, HtmlElement* elt,
HtmlElement::Attribute* attr) {
HtmlResourceSlotPtr slot(new HtmlResourceSlot(resource, elt, attr, this));
std::pair<HtmlResourceSlotSet::iterator, bool> iter_inserted =
slots_.insert(slot);
if (!iter_inserted.second) {
// The slot was already in the set. Release the one we just
// allocated and use the one already in.
HtmlResourceSlotSet::iterator iter = iter_inserted.first;
slot.reset(*iter);
}
return slot;
}
InlineResourceSlotPtr RewriteDriver::GetInlineSlot(
const ResourcePtr& resource, HtmlCharactersNode* char_node) {
InlineResourceSlotPtr slot(
new InlineResourceSlot(resource, char_node, UrlLine()));
std::pair<InlineResourceSlotSet::iterator, bool> iter_inserted =
inline_slots_.insert(slot);
if (!iter_inserted.second) {
// The slot was already in the set. Release the one we just
// allocated and use the one already in.
InlineResourceSlotSet::iterator iter = iter_inserted.first;
slot.reset(*iter);
}
return slot;
}
InlineAttributeSlotPtr RewriteDriver::GetInlineAttributeSlot(
const ResourcePtr& resource, HtmlElement* element,
HtmlElement::Attribute* attribute) {
InlineAttributeSlotPtr slot(
new InlineAttributeSlot(resource, element, attribute, UrlLine()));
std::pair<InlineAttributeSlotSet::iterator, bool> iter_inserted =
inline_attribute_slots_.insert(slot);
if (!iter_inserted.second) {
// The slot was already in the set. Release the one we just
// allocated and use the one already in.
InlineAttributeSlotSet::iterator iter = iter_inserted.first;
slot.reset(*iter);
}
return slot;
}
bool RewriteDriver::InitiateRewrite(RewriteContext* rewrite_context) {
#ifndef NDEBUG
{
ScopedMutex lock(rewrite_mutex());
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
}
#endif
// Drop all rewrites if metadata_cache is unhealthy. This has
// got to be done 100% or not at all, otherwise we can wind up with
// a broken slot-context graph.
//
// Note that we strobe cache health at the beginning of request
// (StartParseId), so that we don't decide in the middle of an HTML
// rewrite that we won't be able to initialize the resource, thus leaving
// us with a partially constructed slot-graph.
if (!can_rewrite_resources_) {
if (rewrites_.empty()) {
rewrite_context->DetachSlots();
delete rewrite_context;
return false;
} else {
// A programming error has allowed a RewriteContext to be added
// despite not being able to rewrite resources. Log a fatal for
// debug builds, and otherwise fall through to keep the context-slot
// graph coherent.
LOG(DFATAL)
<< "Unexpected queued RewriteContext when cannot rewrite resources";
}
}
rewrites_.push_back(rewrite_context);
{
ScopedMutex lock(rewrite_mutex());
ref_counts_.AddRefMutexHeld(kRefPendingRewrites);
++possibly_quick_rewrites_;
}
return true;
}
void RewriteDriver::InitiateFetch(RewriteContext* rewrite_context) {
// TODO(jmarantz): consider setting a bit in the RewriteContext
// based on server_context_->metadata_cache()->IsHealthy() to tell
// the system not to perform any optimization on single resources,
// since the results would not wind up cached. Instead, just serve
// the origin resource as it's fetched. For combined resources, of
// course, we'll have to run the combiner logic on the fetched data
// after we collect it all in memory.
DCHECK_EQ(0, ref_counts_.QueryCountMutexHeld(kRefParsing));
DCHECK_EQ(1, ref_counts_.QueryCountMutexHeld(kRefFetchUserFacing));
fetch_rewrites_.push_back(rewrite_context);
}
bool RewriteDriver::MayCacheExtendCss() const {
return options()->Enabled(RewriteOptions::kExtendCacheCss);
}
bool RewriteDriver::MayCacheExtendImages() const {
return options()->Enabled(RewriteOptions::kExtendCacheImages);
}
bool RewriteDriver::MayCacheExtendPdfs() const {
return options()->Enabled(RewriteOptions::kExtendCachePdfs);
}
bool RewriteDriver::MayCacheExtendScripts() const {
return options()->Enabled(RewriteOptions::kExtendCacheScripts);
}
void RewriteDriver::AddRewriteTask(Function* task) {
rewrite_worker_->Add(task);
}
void RewriteDriver::AddLowPriorityRewriteTask(Function* task) {
low_priority_rewrite_worker_->Add(task);
}
OptionsAwareHTTPCacheCallback::OptionsAwareHTTPCacheCallback(
const RewriteOptions* rewrite_options, const RequestContextPtr& request_ctx)
: HTTPCache::Callback(request_ctx, RequestHeaders::Properties()),
rewrite_options_(rewrite_options) {
// We initialize the callback with a blank RequestHeaders::Properties,
// rather than extracing the actual request properties from
// request_ctx->request_headers(). This is because, with our domain
// mapping, we don't know for sure whether cookies should apply
// to Vary:Cacheable resources. So we pessimistically assume there
// are cookies by initializing a blank one.
response_headers()->set_implicit_cache_ttl_ms(
rewrite_options->implicit_cache_ttl_ms());
response_headers()->set_min_cache_ttl_ms(rewrite_options->min_cache_ttl_ms());
}
OptionsAwareHTTPCacheCallback::~OptionsAwareHTTPCacheCallback() {}
bool OptionsAwareHTTPCacheCallback::IsCacheValid(
const GoogleString& key, const ResponseHeaders& headers) {
return IsCacheValid(key, *rewrite_options_, request_context(), headers);
}
ResponseHeaders::VaryOption
OptionsAwareHTTPCacheCallback::RespectVaryOnResources() const {
return ResponseHeaders::GetVaryOption(rewrite_options_->respect_vary());
}
// static
bool OptionsAwareHTTPCacheCallback::IsCacheValid(
const GoogleString& url,
const RewriteOptions& rewrite_options,
const RequestContextPtr& request_ctx,
const ResponseHeaders& headers) {
if ((headers.DetermineContentType() == &kContentTypeWebp) &&
!request_ctx->accepts_webp() &&
headers.HasValue(HttpAttributes::kVary, HttpAttributes::kAccept)) {
return false;
}
return (headers.has_date_ms() &&
rewrite_options.IsUrlCacheValid(url, headers.date_ms(),
true /* search_wildcards */));
}
int64 OptionsAwareHTTPCacheCallback::OverrideCacheTtlMs(
const GoogleString& key) {
if (rewrite_options_->IsCacheTtlOverridden(key)) {
return rewrite_options_->override_caching_ttl_ms();
}
return -1;
}
RewriteDriver::CssResolutionStatus RewriteDriver::ResolveCssUrls(
const GoogleUrl& input_css_base,
const StringPiece& output_css_base,
const StringPiece& contents,
Writer* writer,
MessageHandler* handler) {
GoogleUrl output_base(output_css_base);
bool proxy_mode;
if (ShouldAbsolutifyUrl(input_css_base, output_base, &proxy_mode)) {
RewriteDomainTransformer transformer(&input_css_base, &output_base,
server_context(), options(),
message_handler());
if (proxy_mode) {
// If URLs are being rewritten to a proxy domain, then trimming
// them based purely on the domain-lawyer mappings is going to
// relativize them so that they cannot be resolved properly in
// their intended context.
//
// TODO(jmarantz): Consider merging the url_namer with DomainLawyer
// so that DomainLawyer::WillDomainChange will be accurate.
transformer.set_trim_urls(false);
}
if (CssTagScanner::TransformUrls(contents, writer, &transformer, handler)) {
return kSuccess;
} else {
return kWriteFailed;
}
}
return kNoResolutionNeeded;
}
bool RewriteDriver::ShouldAbsolutifyUrl(const GoogleUrl& input_base,
const GoogleUrl& output_base,
bool* proxy_mode) const {
bool result = false;
const UrlNamer* url_namer = server_context_->url_namer();
bool proxying = url_namer->ProxyMode();
if (proxying) {
result = true;
} else if (input_base.AllExceptLeaf() != output_base.AllExceptLeaf()) {
result = true;
} else {
const DomainLawyer* domain_lawyer = options()->domain_lawyer();
result = domain_lawyer->WillDomainChange(input_base);
}
if (proxy_mode != NULL) {
*proxy_mode = proxying;
}
return result;
}
PropertyPage* RewriteDriver::property_page() const {
return fallback_property_page_ == NULL ?
NULL : fallback_property_page_->actual_property_page();
}
PropertyPage* RewriteDriver::origin_property_page() const {
return origin_property_page_.get();
}
// This is in the .cc rather than the header to avoid the need to
// include property_cache.h in the header.
void RewriteDriver::set_property_page(PropertyPage* page) {
if (page == NULL) {
set_fallback_property_page(NULL);
return;
}
FallbackPropertyPage* fallback_page = new FallbackPropertyPage(page, NULL);
set_fallback_property_page(fallback_page);
}
void RewriteDriver::set_fallback_property_page(FallbackPropertyPage* page) {
if (owns_property_page_) {
delete fallback_property_page_;
}
fallback_property_page_ = page;
owns_property_page_ = true;
}
void RewriteDriver::set_unowned_fallback_property_page(
FallbackPropertyPage* page) {
if (owns_property_page_) {
delete fallback_property_page_;
}
fallback_property_page_ = page;
owns_property_page_ = false;
}
void RewriteDriver::set_origin_property_page(PropertyPage* page) {
origin_property_page_.reset(page);
}
bool RewriteDriver::CriticalSelectorsEnabled() const {
return (options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) &&
(server_context()->factory()->UseBeaconResultsInFilters() ||
options()->use_selectors_for_critical_css()));
}
void RewriteDriver::increment_num_inline_preview_images() {
++num_inline_preview_images_;
}
StringPiece RewriteDriver::RefCategoryName(RefCategory cat) {
switch (cat) {
case kRefUser:
return "User references";
case kRefParsing:
return "Parsing";
case kRefPendingRewrites:
return "Pending rewrites";
case kRefDetachedRewrites:
return "Detached rewrites";
case kRefDeletingRewrites:
return "Deleting rewrites";
case kRefFetchUserFacing:
return "User-facing fetch rewrite";
case kRefFetchBackground:
return "Background fetch rewrite";
case kRefAsyncEvents:
return "Misc async event";
case kRefRenderBlockingAsyncEvents:
return "Misc async event that's render-blocking";
case kNumRefCategories:
break;
}
LOG(DFATAL) << "Invalid argument to RefCategoryName" << cat;
return "";
}
void RewriteDriver::LastRefRemoved() {
if (!externally_managed_) {
release_driver_ = true;
} else {
ref_counts_.DCheckAllCountsZeroMutexHeld();
// In externally managed mode, we always keep at least one "user"
// reference to the driver for our bookkeeping purposes.
ref_counts_.AddRefMutexHeld(kRefUser);
}
}
void RewriteDriver::DropReference(RefCategory ref_cat) {
bool should_release = false;
{
ScopedMutex lock(rewrite_mutex());
bool signal_cookie = PrepareShouldSignal();
ref_counts_.ReleaseRefMutexHeld(ref_cat);
should_release = release_driver_;
SignalIfRequired(signal_cookie);
}
if (should_release) {
PossiblyPurgeCachedResponseAndReleaseDriver();
}
}
void RewriteDriver::IncrementAsyncEventsCount() {
ref_counts_.AddRef(kRefAsyncEvents);
}
void RewriteDriver::DecrementAsyncEventsCount() {
DropReference(kRefAsyncEvents);
}
void RewriteDriver::IncrementRenderBlockingAsyncEventsCount() {
ref_counts_.AddRef(kRefRenderBlockingAsyncEvents);
}
void RewriteDriver::DecrementRenderBlockingAsyncEventsCount() {
DropReference(kRefRenderBlockingAsyncEvents);
}
void RewriteDriver::EnableBlockingRewrite(RequestHeaders* request_headers) {
if (!options()->blocking_rewrite_key().empty()) {
const char* blocking_rewrite_key = request_headers->Lookup1(
HttpAttributes::kXPsaBlockingRewrite);
if (blocking_rewrite_key != NULL) {
if (options()->blocking_rewrite_key() == blocking_rewrite_key) {
set_fully_rewrite_on_flush(true);
}
// TODO(bharathbhushan): Allow for multiple PSAs on the request path by
// interpreting the value as a comma separated list of keys and avoid
// removing this header unconditionally.
request_headers->RemoveAll(HttpAttributes::kXPsaBlockingRewrite);
}
}
if (!fully_rewrite_on_flush() &&
options()->IsBlockingRewriteRefererUrlPatternPresent()) {
const char* referer = request_headers->Lookup1(
HttpAttributes::kReferer);
if (referer != NULL &&
options()->IsBlockingRewriteEnabledForReferer(referer)) {
set_fully_rewrite_on_flush(true);
}
}
if (fully_rewrite_on_flush()) {
const char* blocking_rewrite_mode(request_headers->Lookup1(
HttpAttributes::kXPsaBlockingRewriteMode));
if (blocking_rewrite_mode != NULL) {
StringPiece mode(HttpAttributes::kXPsaBlockingRewriteModeSlow);
if (blocking_rewrite_mode == mode) {
// Don't wait for async events.
set_fast_blocking_rewrite(false);
}
request_headers->RemoveAll(HttpAttributes::kXPsaBlockingRewriteMode);
}
}
}
RewriteDriver::XhtmlStatus RewriteDriver::MimeTypeXhtmlStatus() {
if (!xhtml_mimetype_computed_ &&
server_context_->response_headers_finalized() &&
(response_headers_ != NULL)) {
xhtml_mimetype_computed_ = true;
const ContentType* content_type = response_headers_->DetermineContentType();
if (content_type != NULL) {
if (content_type->IsXmlLike()) {
xhtml_status_ = kIsXhtml;
} else {
xhtml_status_ = kIsNotXhtml;
}
}
}
return xhtml_status_;
}
FlushEarlyInfo* RewriteDriver::flush_early_info() {
if (flush_early_info_.get() == NULL) {
PropertyCacheDecodeResult status;
flush_early_info_.reset(DecodeFromPropertyCache<FlushEarlyInfo>(
server_context()->page_property_cache(),
fallback_property_page(),
server_context()->dom_cohort(),
kSubresourcesPropertyName,
-1 /* no ttl checking*/,
&status));
if (status != kPropertyCacheDecodeOk) {
flush_early_info_.reset(new FlushEarlyInfo);
}
}
return flush_early_info_.get();
}
void RewriteDriver::InsertDebugComment(StringPiece unescaped,
HtmlNode* node) {
if (DebugMode() && node != NULL && IsRewritable(node)) {
GoogleString escaped;
HtmlKeywords::Escape(unescaped, &escaped);
HtmlNode* comment_node = NewCommentNode(node->parent(), escaped);
InsertNodeAfterNode(node, comment_node);
}
}
void RewriteDriver::InsertDebugComments(
const protobuf::RepeatedPtrField<GoogleString>& unescaped_messages,
HtmlElement* element) {
if (DebugMode() && element != NULL && IsRewritable(element)) {
HtmlNode* preceding_node = element;
for (protobuf::RepeatedPtrField<GoogleString>::const_iterator unescaped =
unescaped_messages.begin();
unescaped != unescaped_messages.end(); ++unescaped) {
GoogleString escaped;
HtmlKeywords::Escape(*unescaped, &escaped);
HtmlNode* comment_node =
NewCommentNode(preceding_node->parent(), escaped);
InsertNodeAfterNode(preceding_node, comment_node);
preceding_node = comment_node;
}
}
}
void RewriteDriver::InsertUnauthorizedDomainDebugComment(StringPiece url,
HtmlElement* element) {
if (DebugMode() && element != NULL && IsRewritable(element)) {
GoogleUrl gurl(url);
InsertNodeAfterNode(
element, NewCommentNode(element->parent(),
GenerateUnauthorizedDomainDebugComment(gurl)));
}
}
GoogleString RewriteDriver::GenerateUnauthorizedDomainDebugComment(
const GoogleUrl& gurl) {
GoogleString comment("The preceding resource was not rewritten because ");
// Note: this is all being defensive - at the time of writing I believe
// url will always be a valid URL.
if (gurl.IsWebValid()) {
StrAppend(&comment, "its domain (", gurl.Host(), ") is not authorized");
} else if (gurl.IsWebOrDataValid()) {
StrAppend(&comment, "it is a data URI");
} else {
StrAppend(&comment, "it is not authorized");
}
GoogleString escaped;
HtmlKeywords::Escape(comment, &escaped);
return escaped;
}
void RewriteDriver::SaveOriginalHeaders(const ResponseHeaders& headers) {
headers.GetSanitizedProto(flush_early_info()->mutable_response_headers());
}
const CriticalLineInfo* RewriteDriver::critical_line_info() const {
return critical_line_info_.get();
}
void RewriteDriver::set_critical_line_info(
CriticalLineInfo* critical_line_info) {
critical_line_info_.reset(critical_line_info);
}
CriticalKeys* RewriteDriver::beacon_critical_line_info() const {
return beacon_critical_line_info_.get();
}
void RewriteDriver::set_beacon_critical_line_info(
CriticalKeys* beacon_critical_line_info) {
beacon_critical_line_info_.reset(beacon_critical_line_info);
}
// The split html config is lazily constructed on first access. Since the
// split-html-filter and the split-html-helper-filter access this from the html
// parsing thread, the lazy construction does not need mutex protection.
const SplitHtmlConfig* RewriteDriver::split_html_config() {
if (split_html_config_ == NULL) {
split_html_config_.reset(new SplitHtmlConfig(this));
}
return split_html_config_.get();
}
CriticalCssResult* RewriteDriver::critical_css_result() const {
return critical_css_result_.get();
}
void RewriteDriver::set_critical_css_result(
CriticalCssResult* critical_css_rules) {
critical_css_result_.reset(critical_css_rules);
}
bool RewriteDriver::is_critical_images_beacon_enabled() {
return (options()->Enabled(RewriteOptions::kLazyloadImages) ||
options()->Enabled(RewriteOptions::kInlineImages) ||
options()->Enabled(RewriteOptions::kDelayImages) ||
options()->Enabled(
RewriteOptions::kResizeToRenderedImageDimensions)) &&
options()->critical_images_beacon_enabled() &&
server_context_->factory()->UseBeaconResultsInFilters() &&
server_context_->page_property_cache()->enabled();
}
FlushEarlyRenderInfo* RewriteDriver::flush_early_render_info() const {
return flush_early_render_info_.get();
}
void RewriteDriver::set_flush_early_render_info(
FlushEarlyRenderInfo* flush_early_render_info) {
flush_early_render_info_.reset(flush_early_render_info);
}
bool RewriteDriver::Write(const ResourceVector& inputs,
const StringPiece& contents,
const ContentType* type,
StringPiece charset,
OutputResource* output) {
output->SetType(type);
output->set_charset(charset);
ResponseHeaders* meta_data = output->response_headers();
bool clear_last_modified = false;
// Transfer Last-Modified from the input for single-input on-the-fly
// resources.
if ((inputs.size() == 1) && (output->kind() == kOnTheFlyResource)) {
const ResponseHeaders* input_headers = inputs[0]->response_headers();
const char* last_modified = input_headers->Lookup1(
HttpAttributes::kLastModified);
if (last_modified == NULL) {
clear_last_modified = true;
} else {
meta_data->Add(HttpAttributes::kLastModified, last_modified);
}
}
server_context_->SetDefaultLongCacheHeaders(
type, charset, output->cache_control_suffix(), meta_data);
if (clear_last_modified) {
meta_data->RemoveAll(HttpAttributes::kLastModified);
}
meta_data->SetStatusAndReason(HttpStatus::kOK);
server_context_->ApplyInputCacheControl(inputs, meta_data);
server_context_->AddOriginalContentLengthHeader(inputs, meta_data);
// The URL for any resource we will write includes the hash of contents,
// so it can can live, essentially, forever. So compute this hash,
// and cache the output using meta_data's default headers which are to cache
// forever.
MessageHandler* handler = message_handler();
Writer* writer = output->BeginWrite(handler);
bool ret = (writer != NULL);
if (ret) {
ret = writer->Write(contents, handler);
output->EndWrite(handler);
HTTPCache* http_cache = server_context_->http_cache();
if (output->kind() != kOnTheFlyResource &&
output->kind() != kInlineResource &&
(http_cache->force_caching() || meta_data->IsProxyCacheable())) {
// This URL should already be mapped to the canonical rewrite domain,
// But we should store its unsharded form in the cache.
http_cache->Put(output->HttpCacheKey(), CacheFragment(),
RequestHeaders::Properties(),
options()->ComputeHttpOptions(),
&output->value_, handler);
}
// If we're asked to, also save a debug dump
if (server_context_->store_outputs_in_file_system()) {
output->DumpToDisk(handler);
}
// If our URL is derived from some pre-existing URL (and not invented by
// us due to something like outlining), cache the mapping from original URL
// to the constructed one.
if (output->kind() == kRewrittenResource ||
output->kind() == kOnTheFlyResource) {
CachedResult* cached = output->EnsureCachedResultCreated();
cached->set_optimizable(true);
cached->set_url(output->url()); // Note: output->url() will be sharded.
}
} else {
// Note that we've already gotten a "could not open file" message;
// this just serves to explain why and suggest a remedy.
handler->Message(kInfo, "Could not create output resource"
" (bad filename prefix '%s'?)",
server_context_->filename_prefix().as_string().c_str());
}
return ret;
}
void RewriteDriver::DetermineFiltersBehaviorImpl() {
DetermineFilterListBehavior(early_pre_render_filters_);
DetermineFilterListBehavior(pre_render_filters_);
// Call parent to set up post render filters.
HtmlParse::DetermineFiltersBehaviorImpl();
}
void RewriteDriver::ClearRequestProperties() {
request_properties_.reset(new RequestProperties(
server_context_->user_agent_matcher()));
}
bool RewriteDriver::MetadataRequested(
const RequestHeaders& request_headers) const {
StringPiece expected_key = options_->distributed_rewrite_key();
// Empty keys don't count.
if (expected_key.empty()) {
return false;
}
return request_headers.HasValue(HttpAttributes::kXPsaRequestMetadata,
expected_key);
}
const GoogleString& RewriteDriver::CacheFragment() const {
CHECK(options_ != NULL);
const GoogleString& fragment = options_->cache_fragment();
if (!fragment.empty()) {
return fragment;
}
CHECK(request_context_.get() != NULL) << "NULL request context in "
<< "RewriteDriver::CacheFragment";
return request_context_->minimal_private_suffix();
}
bool RewriteDriver::SetOrClearPageSpeedOptionCookies(
const GoogleUrl& gurl, ResponseHeaders* response_headers) {
StringPiece required_token(options_->sticky_query_parameters());
StringPiece provided_token(request_context_->sticky_query_parameters_token());
// These are mutually exclusive but provide a way of specifying "do nothing".
bool set_cookies = false;
bool clear_cookies = false;
if (options_->allow_options_to_be_set_by_cookies() &&
!required_token.empty() &&
required_token == provided_token) {
// Make the current options sticky if we allow options to be set by
// cookies (otherwise why bother?), there is a token specified in the
// configuration, and the token specified in the request matches the
// one in the configuration.
set_cookies = true;
} else if (!pagespeed_option_cookies_.empty() &&
!required_token.empty() && !provided_token.empty() &&
required_token != provided_token) {
// Clear the current option cookies if there are any, there is a token
// specified in the configuration, there is a token in the request, and
// the token specified in the request does NOT match the one in the
// configuration - treat that as a specific request to clear the cookies.
clear_cookies = true;
} else if (!pagespeed_option_cookies_.empty() &&
!options_->allow_options_to_be_set_by_cookies()) {
// Clear the current option cookies if there any but we no longer allow
// options to be set by cookies.
clear_cookies = true;
}
if (!set_cookies && !clear_cookies) {
return false;
}
// We need to not set cookies for the option that triggered this.
const GoogleString old_option_name(
StrCat(RewriteQuery::kPageSpeed,
RewriteOptions::kStickyQueryParameters));
const GoogleString new_option_name(
StrCat(RewriteQuery::kModPagespeed,
RewriteOptions::kStickyQueryParameters));
StringPieceVector exclusions;
exclusions.push_back(old_option_name);
exclusions.push_back(new_option_name);
bool result = false;
if (set_cookies) {
int64 expiration_time_ms = (server_context()->timer()->NowMs() +
options_->option_cookies_duration_ms());
result = response_headers->SetQueryParamsAsCookies(gurl,
pagespeed_query_params_,
exclusions,
expiration_time_ms);
} else /* ASSERT: clear_cookies == true */ {
result = response_headers->ClearOptionCookies(gurl,
pagespeed_option_cookies_,
exclusions);
}
if (result) {
response_headers->ComputeCaching();
}
return result;
}
bool RewriteDriver::LookupMetadataForOutputResource(
StringPiece url, GoogleString* error_out,
RewriteContext::CacheLookupResultCallback* callback) {
RewriteFilter* filter = NULL;
GoogleUrl gurl(url);
if (!gurl.IsWebValid()) {
*error_out = "Unable to parse URL.";
return false;
}
// The setup is different depending on if url is .pagespeed. resource or an
// in-place rewritten one.
bool is_pagespeed_resource = server_context_->IsPagespeedResource(gurl);
SetBaseUrlForFetch(gurl.Spec());
OutputResourcePtr output_resource;
if (is_pagespeed_resource) {
output_resource.reset(DecodeOutputResource(gurl, &filter));
} else {
StringPiece base = gurl.AllExceptLeaf();
ResourceNamer namer;
output_resource.reset(
new OutputResource(this, base, base, base, namer, kRewrittenResource));
}
if (output_resource.get() == NULL ||
(filter == NULL && is_pagespeed_resource)) {
*error_out = "Unable to decode resource.";
return false;
}
scoped_ptr<RewriteContext> context;
if (is_pagespeed_resource) {
context.reset(filter->MakeRewriteContext());
} else {
context.reset(new InPlaceRewriteContext(this, gurl.Spec()));
}
return RewriteContext::LookupMetadataForOutputResourceImpl(
output_resource, gurl, context.release(),
this, error_out, callback);
}
} // namespace net_instaweb