| /* |
| * Copyright 2011 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // Author: sligocki@google.com (Shawn Ligocki) |
| |
| #include "pagespeed/automatic/proxy_interface.h" |
| |
| #include "base/callback.h" |
| #include "base/logging.h" |
| #include "net/instaweb/config/rewrite_options_manager.h" |
| #include "net/instaweb/http/public/async_fetch.h" |
| #include "net/instaweb/http/public/log_record.h" |
| #include "net/instaweb/http/public/logging_proto_impl.h" |
| #include "net/instaweb/http/public/request_context.h" |
| #include "net/instaweb/http/public/request_timing_info.h" |
| #include "net/instaweb/rewriter/public/blink_util.h" |
| #include "net/instaweb/rewriter/public/experiment_matcher.h" |
| #include "net/instaweb/rewriter/public/resource_fetch.h" |
| #include "net/instaweb/rewriter/public/rewrite_driver.h" |
| #include "net/instaweb/rewriter/public/rewrite_options.h" |
| #include "net/instaweb/rewriter/public/rewrite_query.h" |
| #include "net/instaweb/rewriter/public/server_context.h" |
| #include "pagespeed/automatic/cache_html_flow.h" |
| #include "pagespeed/automatic/flush_early_flow.h" |
| #include "pagespeed/automatic/proxy_fetch.h" |
| #include "pagespeed/kernel/base/abstract_mutex.h" |
| #include "pagespeed/kernel/base/hasher.h" |
| #include "pagespeed/kernel/base/hostname_util.h" |
| #include "pagespeed/kernel/base/ref_counted_ptr.h" |
| #include "pagespeed/kernel/base/scoped_ptr.h" |
| #include "pagespeed/kernel/base/statistics.h" |
| #include "pagespeed/kernel/base/string.h" |
| #include "pagespeed/kernel/base/string_util.h" |
| #include "pagespeed/kernel/http/content_type.h" |
| #include "pagespeed/kernel/http/google_url.h" |
| #include "pagespeed/kernel/http/http_names.h" |
| #include "pagespeed/kernel/http/query_params.h" |
| #include "pagespeed/kernel/http/request_headers.h" |
| #include "pagespeed/kernel/http/response_headers.h" |
| |
| namespace net_instaweb { |
| |
| class MessageHandler; |
| |
| const char ProxyInterface::kCacheHtmlRequestCount[] = |
| "cache-html-requests"; |
| namespace { |
| |
| // Names for Statistics variables. |
| const char kTotalRequestCount[] = "all-requests"; |
| const char kPagespeedRequestCount[] = "pagespeed-requests"; |
| const char kRejectedRequestCount[] = "publisher-rejected-requests"; |
| const char kRejectedRequestHtmlResponse[] = "Unable to serve " |
| "content as the content is blocked by the administrator of the domain."; |
| const char kNoDomainConfigRequestCount[] = "without-domain-config-requests"; |
| const char kNoDomainConfigResourceRequestCount[] = |
| "without-domain-config-resource-requests"; |
| |
| } // namespace |
| |
| struct ProxyInterface::RequestData { |
| bool is_resource_fetch; |
| scoped_ptr<GoogleUrl> request_url; |
| AsyncFetch* async_fetch; |
| MessageHandler* handler; |
| }; |
| |
| ProxyInterface::ProxyInterface(const StringPiece& hostname, int port, |
| ServerContext* server_context, |
| Statistics* stats) |
| : server_context_(server_context), |
| hostname_(hostname.as_string()), |
| port_(port), |
| all_requests_(stats->GetTimedVariable(kTotalRequestCount)), |
| pagespeed_requests_(stats->GetTimedVariable(kPagespeedRequestCount)), |
| cache_html_flow_requests_( |
| stats->GetTimedVariable(kCacheHtmlRequestCount)), |
| rejected_requests_(stats->GetTimedVariable(kRejectedRequestCount)), |
| requests_without_domain_config_( |
| stats->GetTimedVariable(kNoDomainConfigRequestCount)), |
| resource_requests_without_domain_config_( |
| stats->GetTimedVariable(kNoDomainConfigResourceRequestCount)) { |
| proxy_fetch_factory_.reset(new ProxyFetchFactory(server_context)); |
| } |
| |
| ProxyInterface::~ProxyInterface() { |
| } |
| |
| void ProxyInterface::InitStats(Statistics* statistics) { |
| statistics->AddTimedVariable(kTotalRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kPagespeedRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kCacheHtmlRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kRejectedRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kRejectedRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kNoDomainConfigRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kNoDomainConfigResourceRequestCount, |
| ServerContext::kStatisticsGroup); |
| CacheHtmlFlow::InitStats(statistics); |
| FlushEarlyFlow::InitStats(statistics); |
| } |
| |
| bool ProxyInterface::IsWellFormedUrl(const GoogleUrl& url) { |
| bool ret = false; |
| if (url.IsWebValid()) { |
| if (url.has_path()) { |
| StringPiece path = url.PathAndLeaf(); |
| GoogleString filename = url.ExtractFileName(); |
| int path_len = path.size() - filename.size(); |
| if (path_len >= 0) { |
| ret = true; |
| } |
| } else { |
| LOG(ERROR) << "URL has no path: " << url.Spec(); |
| } |
| } |
| return ret; |
| } |
| |
| bool ProxyInterface::UrlAndPortMatchThisServer(const GoogleUrl& url) { |
| bool ret = false; |
| if (url.IsWebValid() && (url.EffectiveIntPort() == port_)) { |
| // TODO(atulvasu): This should support matching the actual host this |
| // machine can receive requests from. Ideally some flag control would |
| // help. For example this server could be running multiple virtual |
| // servers, and we would like to know what server we are catering to for |
| // pagespeed only queries. |
| // |
| // Allow for exact hostname matches, as well as a URL typed into the |
| // browser window like "exeda.cam", which should match |
| // "exeda.cam.corp.google.com". |
| StringPiece host = url.Host(); |
| if (IsLocalhost(host, hostname_) || |
| StringPiece(hostname_).starts_with(StrCat(host, "."))) { |
| ret = true; |
| } |
| } |
| return ret; |
| } |
| |
| void ProxyInterface::Fetch(const GoogleString& requested_url_string, |
| MessageHandler* handler, |
| AsyncFetch* async_fetch) { |
| GoogleUrl requested_url(requested_url_string); |
| const bool is_get_or_head = |
| (async_fetch->request_headers()->method() == RequestHeaders::kGet) || |
| (async_fetch->request_headers()->method() == RequestHeaders::kHead); |
| |
| all_requests_->IncBy(1); |
| if (!(requested_url.IsWebValid() && IsWellFormedUrl(requested_url))) { |
| LOG(WARNING) << "Bad URL, failing request: " << requested_url_string; |
| async_fetch->response_headers()->SetStatusAndReason(HttpStatus::kNotFound); |
| async_fetch->Done(false); |
| } else { |
| // Try to handle this as a .pagespeed. resource. |
| if (is_get_or_head && server_context_->IsPagespeedResource(requested_url)) { |
| pagespeed_requests_->IncBy(1); |
| LOG(INFO) << "Serving URL as pagespeed resource: " |
| << requested_url.Spec(); |
| ProxyRequest(true, requested_url, async_fetch, handler); |
| } else if (UrlAndPortMatchThisServer(requested_url)) { |
| // Just respond with a 404 for now. |
| async_fetch->response_headers()->SetStatusAndReason( |
| HttpStatus::kNotFound); |
| LOG(INFO) << "Returning 404 for URL: " << requested_url.Spec(); |
| async_fetch->Done(false); |
| } else { |
| // Otherwise we proxy it (rewriting if it is HTML). |
| LOG(INFO) << "Proxying URL normally: " << requested_url.Spec(); |
| ProxyRequest(false, requested_url, async_fetch, handler); |
| } |
| } |
| } |
| |
| void ProxyInterface::ProxyRequest(bool is_resource_fetch, |
| const GoogleUrl& request_url, |
| AsyncFetch* async_fetch, |
| MessageHandler* handler) { |
| RequestData* request_data = new RequestData; |
| request_data->is_resource_fetch = is_resource_fetch; |
| request_data->request_url.reset(new GoogleUrl); |
| request_data->request_url->Reset(request_url); |
| request_data->async_fetch = async_fetch; |
| request_data->handler = handler; |
| |
| server_context_->rewrite_options_manager()->GetRewriteOptions( |
| request_url, |
| *async_fetch->request_headers(), |
| NewCallback(this, &ProxyInterface::GetRewriteOptionsDone, request_data)); |
| } |
| |
| ProxyFetchPropertyCallbackCollector* |
| ProxyInterface::InitiatePropertyCacheLookup( |
| bool is_resource_fetch, |
| const GoogleUrl& request_url, |
| RewriteOptions* options, |
| AsyncFetch* async_fetch, |
| const bool requires_blink_cohort) { |
| return ProxyFetchFactory::InitiatePropertyCacheLookup( |
| is_resource_fetch, request_url, server_context_, options, async_fetch, |
| requires_blink_cohort); |
| } |
| |
| void ProxyInterface::GetRewriteOptionsDone(RequestData* request_data, |
| RewriteOptions* domain_options) { |
| scoped_ptr<RequestData> request_data_deleter(request_data); |
| scoped_ptr<RewriteOptions> scoped_domain_options(domain_options); |
| bool is_resource_fetch = request_data->is_resource_fetch; |
| GoogleUrl* request_url = request_data->request_url.get(); |
| AsyncFetch* async_fetch = request_data->async_fetch; |
| MessageHandler* handler = request_data->handler; |
| |
| if (domain_options == NULL) { |
| requests_without_domain_config_->IncBy(1); |
| if (is_resource_fetch) { |
| resource_requests_without_domain_config_->IncBy(1); |
| } |
| } |
| |
| // Parse the query options, headers, and cookies. |
| RewriteQuery query; |
| if (!server_context_->GetQueryOptions(async_fetch->request_context(), |
| domain_options, request_url, |
| async_fetch->request_headers(), |
| NULL /* response_headers */, &query)) { |
| async_fetch->response_headers()->SetStatusAndReason( |
| HttpStatus::kMethodNotAllowed); |
| async_fetch->Write("Invalid PageSpeed query-params/request headers", |
| handler); |
| async_fetch->Done(false); |
| return; |
| } |
| |
| RewriteOptions* options = server_context_->GetCustomOptions( |
| async_fetch->request_headers(), scoped_domain_options.release(), |
| query.ReleaseOptions()); |
| GoogleString url_string; |
| request_url->Spec().CopyToString(&url_string); |
| RequestHeaders* request_headers = async_fetch->request_headers(); |
| if (options != NULL && |
| options->IsRequestDeclined(url_string, request_headers)) { |
| rejected_requests_->IncBy(1); |
| ResponseHeaders* response_headers = async_fetch->response_headers(); |
| response_headers->SetStatusAndReason(HttpStatus::kProxyDeclinedRequest); |
| response_headers->Replace(HttpAttributes::kContentType, |
| kContentTypeText.mime_type()); |
| response_headers->Replace(HttpAttributes::kCacheControl, |
| "private, max-age=0"); |
| async_fetch->Write(kRejectedRequestHtmlResponse, handler); |
| async_fetch->Done(false); |
| delete options; |
| return; |
| } |
| if (ServerContext::ScanSplitHtmlRequest( |
| async_fetch->request_context(), options, &url_string)) { |
| request_url->Reset(url_string); |
| } |
| |
| // Update request_headers. |
| // We deal with encodings. So strip the users Accept-Encoding headers. |
| async_fetch->request_headers()->RemoveAll(HttpAttributes::kAcceptEncoding); |
| // Note: We preserve the User-Agent and Cookies so that the origin servers |
| // send us the correct HTML. We will need to consider this for caching HTML. |
| |
| async_fetch->request_context()->mutable_timing_info()->ProcessingStarted(); |
| |
| AbstractLogRecord* log_record = async_fetch->request_context()->log_record(); |
| { |
| ScopedMutex lock(log_record->mutex()); |
| log_record->logging_info()->set_is_pagespeed_resource(is_resource_fetch); |
| } |
| |
| // Start fetch and rewrite. If GetCustomOptions found options for us, |
| // the RewriteDriver created by StartNewProxyFetch will take ownership. |
| if (is_resource_fetch) { |
| // TODO(sligocki): Set using_spdy appropriately. |
| bool using_spdy = false; |
| // TODO(pulkitg): Set is_original_resource_cacheable to false if pagespeed |
| // resource is not cacheable. |
| const RewriteOptions* these_options = |
| (options == NULL ? server_context_->global_options() : options); |
| // TODO(sligocki): Should we be setting default options and then overriding |
| // here? It seems like it would be better to only set once, but that |
| // involves a lot of complicated code changes. |
| async_fetch->request_context()->ResetOptions( |
| these_options->ComputeHttpOptions()); |
| ResourceFetch::Start(*request_url, options, using_spdy, |
| server_context_, async_fetch); |
| } else { |
| // TODO(nforman): If we are not running an experiment, remove the |
| // experiment cookie. |
| // If we don't already have custom options, and the global options say we're |
| // running an experiment, then clone them into custom_options so we can |
| // manipulate custom options without affecting the global options. |
| if (options == NULL) { |
| RewriteOptions* global_options = server_context_->global_options(); |
| if (global_options->running_experiment()) { |
| options = global_options->Clone(); |
| } |
| } |
| // TODO(anupama): Adapt the experiment logic below for the FlushEarlyFlow as |
| // well. |
| bool need_to_store_experiment_data = false; |
| if (options != NULL && options->running_experiment()) { |
| need_to_store_experiment_data = |
| server_context_->experiment_matcher()->ClassifyIntoExperiment( |
| *async_fetch->request_headers(), |
| *server_context_->user_agent_matcher(), options); |
| options->set_need_to_store_experiment_data(need_to_store_experiment_data); |
| } |
| const char* user_agent = async_fetch->request_headers()->Lookup1( |
| HttpAttributes::kUserAgent); |
| |
| // Whether it's a cache html request should not change despite the fact |
| // a new driver is created later on. |
| const bool is_cache_html_request = BlinkUtil::IsBlinkRequest( |
| *request_url, async_fetch, options, |
| user_agent, server_context_, |
| RewriteOptions::kCachePartialHtml); |
| |
| ProxyFetchPropertyCallbackCollector* property_callback = NULL; |
| |
| if (options == NULL || |
| (options->enabled() && options->IsAllowed(request_url->Spec()))) { |
| // Ownership of "property_callback" is eventually assumed by either |
| // CacheHtmlFlow or ProxyFetch. |
| property_callback = InitiatePropertyCacheLookup(is_resource_fetch, |
| *request_url, |
| options, |
| async_fetch, |
| is_cache_html_request); |
| } |
| |
| if (options != NULL) { |
| server_context_->ComputeSignature(options); |
| { |
| ScopedMutex lock(log_record->mutex()); |
| log_record->logging_info()->set_options_signature_hash( |
| server_context_->contents_hasher()->HashToUint64( |
| options->signature())); |
| } |
| } |
| |
| RewriteDriver* driver = NULL; |
| RequestContextPtr request_ctx = async_fetch->request_context(); |
| DCHECK(request_ctx.get() != NULL) << "Async fetch must have a request" |
| << "context but does not."; |
| if (options == NULL) { |
| driver = server_context_->NewRewriteDriver(request_ctx); |
| } else { |
| // NewCustomRewriteDriver takes ownership of custom_options_. |
| driver = server_context_->NewCustomRewriteDriver(options, request_ctx); |
| } |
| // TODO(sligocki): Should we be setting default options and then overriding |
| // here? It seems like it would be better to only set once, but that |
| // involves a lot of complicated code changes. |
| request_ctx->ResetOptions(driver->options()->ComputeHttpOptions()); |
| driver->SetRequestHeaders(*async_fetch->request_headers()); |
| // TODO(mmohabey): Factor out the below checks so that they are not |
| // repeated in BlinkUtil::IsBlinkRequest(). |
| |
| // Copy over any PageSpeed query parameters so we can re-add them if we |
| // receive a redirection response to our fetch request. |
| driver->set_pagespeed_query_params( |
| query.pagespeed_query_params().ToEscapedString()); |
| // Copy over any PageSpeed cookies so we know which ones to clear in |
| // ProxyFetch::HandleHeadersComplete(). |
| driver->set_pagespeed_option_cookies( |
| query.pagespeed_option_cookies().ToEscapedString()); |
| |
| if (driver->options() != NULL && driver->options()->enabled() && |
| property_callback != NULL && |
| driver->options()->IsAllowed(url_string)) { |
| if (is_cache_html_request) { |
| cache_html_flow_requests_->IncBy(1); |
| CacheHtmlFlow::Start(url_string, |
| async_fetch, |
| driver, |
| proxy_fetch_factory_.get(), |
| // Takes ownership of property_callback. |
| property_callback); |
| |
| return; |
| } |
| // NOTE: The FlushEarly flow will run in parallel with the ProxyFetch, |
| // but will not begin (FlushEarlyFlwow::FlushEarly) until the |
| // PropertyCache lookup has completed. |
| // Also it does NOT take ownership of property_callback. |
| // FlushEarlyFlow might not start if the request is not GET or if the |
| // useragent is unsupported etc. |
| FlushEarlyFlow::TryStart(url_string, &async_fetch, driver, |
| proxy_fetch_factory_.get(), |
| property_callback); |
| } |
| // Takes ownership of property_callback. |
| proxy_fetch_factory_->StartNewProxyFetch( |
| url_string, async_fetch, driver, property_callback, NULL); |
| } |
| } |
| |
| } // namespace net_instaweb |
