| /* |
| * Copyright 2011 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // Author: sligocki@google.com (Shawn Ligocki) |
| |
| #include "net/instaweb/automatic/public/proxy_interface.h" |
| |
| #include "base/logging.h" |
| #include "net/instaweb/automatic/public/blink_flow_critical_line.h" |
| #include "net/instaweb/automatic/public/cache_html_flow.h" |
| #include "net/instaweb/automatic/public/flush_early_flow.h" |
| #include "net/instaweb/automatic/public/proxy_fetch.h" |
| #include "net/instaweb/http/public/async_fetch.h" |
| #include "net/instaweb/http/public/content_type.h" |
| #include "net/instaweb/http/public/log_record.h" |
| #include "net/instaweb/http/public/logging_proto_impl.h" |
| #include "net/instaweb/http/public/meta_data.h" |
| #include "net/instaweb/http/public/request_context.h" |
| #include "net/instaweb/http/public/request_headers.h" |
| #include "net/instaweb/http/public/response_headers.h" |
| #include "net/instaweb/http/public/user_agent_matcher.h" |
| #include "net/instaweb/rewriter/public/blink_util.h" |
| #include "net/instaweb/rewriter/public/furious_matcher.h" |
| #include "net/instaweb/rewriter/public/server_context.h" |
| #include "net/instaweb/rewriter/public/resource_fetch.h" |
| #include "net/instaweb/rewriter/public/rewrite_driver.h" |
| #include "net/instaweb/rewriter/public/rewrite_options.h" |
| #include "net/instaweb/rewriter/public/url_namer.h" |
| #include "net/instaweb/util/public/basictypes.h" |
| #include "net/instaweb/util/public/google_url.h" |
| #include "net/instaweb/util/public/hostname_util.h" |
| #include "net/instaweb/util/public/property_cache.h" |
| #include "net/instaweb/util/public/ref_counted_ptr.h" |
| #include "net/instaweb/util/public/request_trace.h" |
| #include "net/instaweb/util/public/scoped_ptr.h" |
| #include "net/instaweb/util/public/statistics.h" |
| #include "net/instaweb/util/public/string.h" |
| #include "net/instaweb/util/public/string_util.h" |
| #include "net/instaweb/util/public/thread_system.h" |
| |
| namespace net_instaweb { |
| |
| class AbstractMutex; |
| class MessageHandler; |
| |
| const char ProxyInterface::kBlinkRequestCount[] = "blink-requests"; |
| const char ProxyInterface::kBlinkCriticalLineRequestCount[] = |
| "blink-critical-line-requests"; |
| |
| namespace { |
| |
| // Names for Statistics variables. |
| const char kTotalRequestCount[] = "all-requests"; |
| const char kPagespeedRequestCount[] = "pagespeed-requests"; |
| const char kBlinkRequestCount[] = "blink-requests"; |
| const char kRejectedRequestCount[] = "publisher-rejected-requests"; |
| const char kRejectedRequestHtmlResponse[] = "Unable to serve " |
| "content as the content is blocked by the administrator of the domain."; |
| |
| bool UrlMightHavePropertyCacheEntry(const GoogleUrl& url) { |
| const ContentType* type = NameExtensionToContentType(url.LeafSansQuery()); |
| if (type == NULL) { |
| return true; // http://www.example.com/ -- no extension; could be HTML. |
| } |
| |
| // Use a complete switch-statement rather than type()->IsHtmlLike() |
| // so that every time we add a new content-type we make an explicit |
| // decision about whether it should induce a pcache read. |
| // |
| // TODO(jmarantz): currently this returns false for ".txt". Thus we will |
| // do no optimizations relying on property-cache on HTML files ending with |
| // ".txt". We should determine whether this is the right thing or not. |
| switch (type->type()) { |
| case ContentType::kHtml: |
| case ContentType::kXhtml: |
| case ContentType::kCeHtml: |
| return true; |
| case ContentType::kJavascript: |
| case ContentType::kCss: |
| case ContentType::kText: |
| case ContentType::kXml: |
| case ContentType::kPng: |
| case ContentType::kGif: |
| case ContentType::kJpeg: |
| case ContentType::kSwf: |
| case ContentType::kWebp: |
| case ContentType::kIco: |
| case ContentType::kPdf: |
| case ContentType::kOther: |
| case ContentType::kJson: |
| case ContentType::kVideo: |
| case ContentType::kOctetStream: |
| return false; |
| } |
| LOG(DFATAL) << "URL " << url.Spec() << ": unexpected type:" << type->type() |
| << "; " << type->mime_type() << "; " << type->file_extension(); |
| return false; |
| } |
| |
| bool HasRejectedHeader(const StringPiece& header_name, |
| const RequestHeaders* request_headers, |
| const RewriteOptions* options) { |
| ConstStringStarVector header_values; |
| if (request_headers->Lookup(header_name, &header_values)) { |
| for (int i = 0, n = header_values.size(); i < n; ++i) { |
| if (options->IsRejectedRequest(header_name, *header_values[i])) { |
| return true; |
| } |
| } |
| } |
| return false; |
| } |
| |
| // Provides a callback whose Done() function is executed once we have |
| // rewrite options. |
| class ProxyInterfaceUrlNamerCallback : public UrlNamer::Callback { |
| public: |
| ProxyInterfaceUrlNamerCallback( |
| bool is_resource_fetch, |
| GoogleUrl* request_url, |
| AsyncFetch* async_fetch, |
| ProxyInterface* proxy_interface, |
| RewriteOptions* query_options, |
| MessageHandler* handler) |
| : is_resource_fetch_(is_resource_fetch), |
| request_url_(request_url), |
| async_fetch_(async_fetch), |
| property_callback_(NULL), |
| handler_(handler), |
| proxy_interface_(proxy_interface), |
| query_options_(query_options) { |
| } |
| virtual ~ProxyInterfaceUrlNamerCallback() {} |
| virtual void Done(RewriteOptions* rewrite_options) { |
| proxy_interface_->ProxyRequestCallback( |
| is_resource_fetch_, request_url_, async_fetch_, rewrite_options, |
| query_options_, handler_); |
| delete this; |
| } |
| |
| private: |
| bool is_resource_fetch_; |
| GoogleUrl* request_url_; |
| AsyncFetch* async_fetch_; |
| ProxyFetchPropertyCallbackCollector* property_callback_; |
| MessageHandler* handler_; |
| ProxyInterface* proxy_interface_; |
| RewriteOptions* query_options_; |
| |
| DISALLOW_COPY_AND_ASSIGN(ProxyInterfaceUrlNamerCallback); |
| }; |
| |
| } // namespace |
| |
| ProxyInterface::ProxyInterface(const StringPiece& hostname, int port, |
| ServerContext* manager, |
| Statistics* stats) |
| : server_context_(manager), |
| fetcher_(NULL), |
| timer_(NULL), |
| handler_(manager->message_handler()), |
| hostname_(hostname.as_string()), |
| port_(port), |
| all_requests_(stats->GetTimedVariable(kTotalRequestCount)), |
| pagespeed_requests_(stats->GetTimedVariable(kPagespeedRequestCount)), |
| blink_requests_(stats->GetTimedVariable(kBlinkRequestCount)), |
| blink_critical_line_requests_( |
| stats->GetTimedVariable(kBlinkCriticalLineRequestCount)), |
| rejected_requests_(stats->GetTimedVariable(kRejectedRequestCount)) { |
| proxy_fetch_factory_.reset(new ProxyFetchFactory(manager)); |
| } |
| |
| ProxyInterface::~ProxyInterface() { |
| } |
| |
| void ProxyInterface::InitStats(Statistics* statistics) { |
| statistics->AddTimedVariable(kTotalRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kPagespeedRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kBlinkRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kBlinkCriticalLineRequestCount, |
| ServerContext::kStatisticsGroup); |
| statistics->AddTimedVariable(kRejectedRequestCount, |
| ServerContext::kStatisticsGroup); |
| BlinkFlowCriticalLine::InitStats(statistics); |
| CacheHtmlFlow::InitStats(statistics); |
| FlushEarlyFlow::InitStats(statistics); |
| } |
| |
| bool ProxyInterface::IsWellFormedUrl(const GoogleUrl& url) { |
| bool ret = false; |
| if (url.is_valid()) { |
| if (url.has_path()) { |
| StringPiece path = url.PathAndLeaf(); |
| GoogleString filename = url.ExtractFileName(); |
| int path_len = path.size() - filename.size(); |
| if (path_len >= 0) { |
| ret = true; |
| } |
| } else if (!url.has_scheme()) { |
| LOG(ERROR) << "URL has no scheme: " << url.Spec(); |
| } else { |
| LOG(ERROR) << "URL has no path: " << url.Spec(); |
| } |
| } |
| return ret; |
| } |
| |
| bool ProxyInterface::UrlAndPortMatchThisServer(const GoogleUrl& url) { |
| bool ret = false; |
| if (url.is_valid() && (url.EffectiveIntPort() == port_)) { |
| // TODO(atulvasu): This should support matching the actual host this |
| // machine can receive requests from. Ideally some flag control would |
| // help. For example this server could be running multiple virtual |
| // servers, and we would like to know what server we are catering to for |
| // pagespeed only queries. |
| // |
| // Allow for exact hostname matches, as well as a URL typed into the |
| // browser window like "exeda.cam", which should match |
| // "exeda.cam.corp.google.com". |
| StringPiece host = url.Host(); |
| if (IsLocalhost(host, hostname_) || |
| StringPiece(hostname_).starts_with(StrCat(host, "."))) { |
| ret = true; |
| } |
| } |
| return ret; |
| } |
| |
| void ProxyInterface::Fetch(const GoogleString& requested_url_string, |
| MessageHandler* handler, |
| AsyncFetch* async_fetch) { |
| const GoogleUrl requested_url(requested_url_string); |
| const bool is_get_or_head = |
| (async_fetch->request_headers()->method() == RequestHeaders::kGet) || |
| (async_fetch->request_headers()->method() == RequestHeaders::kHead); |
| |
| all_requests_->IncBy(1); |
| if (!(requested_url.is_valid() && IsWellFormedUrl(requested_url))) { |
| LOG(WARNING) << "Bad URL, failing request: " << requested_url_string; |
| async_fetch->response_headers()->SetStatusAndReason(HttpStatus::kNotFound); |
| async_fetch->Done(false); |
| } else { |
| // Try to handle this as a .pagespeed. resource. |
| if (server_context_->IsPagespeedResource(requested_url) && |
| is_get_or_head) { |
| pagespeed_requests_->IncBy(1); |
| LOG(INFO) << "Serving URL as pagespeed resource: " |
| << requested_url.Spec(); |
| ProxyRequest(true, requested_url, async_fetch, handler); |
| } else if (UrlAndPortMatchThisServer(requested_url)) { |
| // Just respond with a 404 for now. |
| async_fetch->response_headers()->SetStatusAndReason( |
| HttpStatus::kNotFound); |
| LOG(INFO) << "Returning 404 for URL: " << requested_url.Spec(); |
| async_fetch->Done(false); |
| } else { |
| // Otherwise we proxy it (rewriting if it is HTML). |
| LOG(INFO) << "Proxying URL normally: " << requested_url.Spec(); |
| ProxyRequest(false, requested_url, async_fetch, handler); |
| } |
| } |
| } |
| |
| void ProxyInterface::ProxyRequest(bool is_resource_fetch, |
| const GoogleUrl& request_url, |
| AsyncFetch* async_fetch, |
| MessageHandler* handler) { |
| scoped_ptr<GoogleUrl> gurl(new GoogleUrl); |
| gurl->Reset(request_url); |
| |
| // Stripping ModPagespeed query params before the property cache lookup to |
| // make cache key consistent for both lookup and storing in cache. |
| ServerContext::OptionsBoolPair query_options_success = |
| server_context_->GetQueryOptions(gurl.get(), |
| async_fetch->request_headers(), |
| NULL); |
| |
| if (!query_options_success.second) { |
| async_fetch->response_headers()->SetStatusAndReason( |
| HttpStatus::kMethodNotAllowed); |
| async_fetch->Write("Invalid PageSpeed query-params/request headers", |
| handler); |
| async_fetch->Done(false); |
| return; |
| } |
| |
| // Owned by ProxyInterfaceUrlNamerCallback. |
| GoogleUrl* released_gurl(gurl.release()); |
| |
| ProxyInterfaceUrlNamerCallback* proxy_interface_url_namer_callback = |
| new ProxyInterfaceUrlNamerCallback(is_resource_fetch, released_gurl, |
| async_fetch, this, |
| query_options_success.first, handler); |
| |
| server_context_->url_namer()->DecodeOptions( |
| *released_gurl, *async_fetch->request_headers(), |
| proxy_interface_url_namer_callback, handler); |
| } |
| |
| ProxyFetchPropertyCallbackCollector* |
| ProxyInterface::InitiatePropertyCacheLookup( |
| bool is_resource_fetch, |
| const GoogleUrl& request_url, |
| RewriteOptions* options, |
| AsyncFetch* async_fetch, |
| bool* added_page_property_callback) { |
| RequestContextPtr request_ctx = async_fetch->request_context(); |
| DCHECK(request_ctx.get() != NULL); |
| if (request_ctx->root_trace_context() != NULL) { |
| request_ctx->root_trace_context()->TracePrintf( |
| "PropertyCache lookup start"); |
| } |
| StringPiece user_agent = |
| async_fetch->request_headers()->Lookup1(HttpAttributes::kUserAgent); |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> callback_collector( |
| new ProxyFetchPropertyCallbackCollector( |
| server_context_, request_url.Spec(), request_ctx, options, |
| user_agent)); |
| bool added_callback = false; |
| PropertyPageStarVector property_callbacks; |
| |
| ProxyFetchPropertyCallback* client_callback = NULL; |
| PropertyCache* page_property_cache = server_context_->page_property_cache(); |
| PropertyCache* client_property_cache = |
| server_context_->client_property_cache(); |
| if (!is_resource_fetch && |
| server_context_->page_property_cache()->enabled() && |
| UrlMightHavePropertyCacheEntry(request_url) && |
| async_fetch->request_headers()->method() == RequestHeaders::kGet) { |
| if (options != NULL) { |
| server_context_->ComputeSignature(options); |
| } |
| for (int i = 0; |
| i < static_cast<int>(UserAgentMatcher::kEndOfDeviceType); |
| ++i) { |
| UserAgentMatcher::DeviceType device_type = |
| static_cast<UserAgentMatcher::DeviceType>(i); |
| AbstractMutex* mutex = server_context_->thread_system()->NewMutex(); |
| const StringPiece& device_type_suffix = |
| UserAgentMatcher::DeviceTypeSuffix(device_type); |
| GoogleString page_key = server_context_->GetPagePropertyCacheKey( |
| request_url.Spec(), options, device_type_suffix); |
| ProxyFetchPropertyCallback* property_callback = |
| new ProxyFetchPropertyCallback( |
| ProxyFetchPropertyCallback::kPagePropertyCache, |
| *page_property_cache, |
| page_key, device_type, callback_collector.get(), mutex); |
| property_callbacks.push_back(property_callback); |
| callback_collector->AddCallback(property_callback); |
| } |
| added_callback = true; |
| if (added_page_property_callback != NULL) { |
| *added_page_property_callback = true; |
| } |
| } |
| |
| // Initiate client property cache lookup. |
| if (async_fetch != NULL) { |
| const char* client_id = async_fetch->request_headers()->Lookup1( |
| HttpAttributes::kXGooglePagespeedClientId); |
| if (client_id != NULL) { |
| if (client_property_cache->enabled()) { |
| AbstractMutex* mutex = server_context_->thread_system()->NewMutex(); |
| client_callback = new ProxyFetchPropertyCallback( |
| ProxyFetchPropertyCallback::kClientPropertyCache, |
| *client_property_cache, client_id, |
| UserAgentMatcher::kEndOfDeviceType, |
| callback_collector.get(), mutex); |
| callback_collector->AddCallback(client_callback); |
| added_callback = true; |
| } |
| } |
| } |
| |
| // All callbacks need to be registered before Reads to avoid race. |
| if (!property_callbacks.empty()) { |
| page_property_cache->MultiRead(&property_callbacks); |
| } |
| if (client_callback != NULL) { |
| client_property_cache->Read(client_callback); |
| } |
| |
| if (!added_callback) { |
| callback_collector.reset(NULL); |
| } |
| return callback_collector.release(); |
| } |
| |
| void ProxyInterface::ProxyRequestCallback( |
| bool is_resource_fetch, |
| GoogleUrl* url, |
| AsyncFetch* async_fetch, |
| RewriteOptions* domain_options, |
| RewriteOptions* query_options, |
| MessageHandler* handler) { |
| scoped_ptr<GoogleUrl> request_url(url); |
| RewriteOptions* options = server_context_->GetCustomOptions( |
| async_fetch->request_headers(), domain_options, query_options); |
| GoogleString url_string; |
| RequestHeaders* request_headers = async_fetch->request_headers(); |
| request_url->Spec().CopyToString(&url_string); |
| if ((options != NULL) && |
| (options->IsRejectedUrl(url_string) || |
| HasRejectedHeader( |
| HttpAttributes::kUserAgent, request_headers, options) || |
| HasRejectedHeader( |
| HttpAttributes::kXForwardedFor, request_headers, options))) { |
| rejected_requests_->IncBy(1); |
| ResponseHeaders* response_headers = async_fetch->response_headers(); |
| response_headers->SetStatusAndReason(HttpStatus::kProxyDeclinedRequest); |
| response_headers->Replace(HttpAttributes::kContentType, |
| kContentTypeText.mime_type()); |
| response_headers->Replace(HttpAttributes::kCacheControl, |
| "private, max-age=0"); |
| async_fetch->Write(kRejectedRequestHtmlResponse, handler); |
| async_fetch->Done(false); |
| delete options; |
| return; |
| } |
| |
| scoped_ptr<ProxyFetchPropertyCallbackCollector> property_callback; |
| |
| // Update request_headers. |
| // We deal with encodings. So strip the users Accept-Encoding headers. |
| async_fetch->request_headers()->RemoveAll(HttpAttributes::kAcceptEncoding); |
| // Note: We preserve the User-Agent and Cookies so that the origin servers |
| // send us the correct HTML. We will need to consider this for caching HTML. |
| |
| // Start fetch and rewrite. If GetCustomOptions found options for us, |
| // the RewriteDriver created by StartNewProxyFetch will take ownership. |
| if (is_resource_fetch) { |
| // TODO(sligocki): Set using_spdy appropriately. |
| bool using_spdy = false; |
| // TODO(pulkitg): Set is_original_resource_cacheable to false if pagespeed |
| // resource is not cacheable. |
| ResourceFetch::Start(*request_url, options, using_spdy, |
| server_context_, async_fetch); |
| } else { |
| // TODO(nforman): If we are not running an experiment, remove the |
| // furious cookie. |
| // If we don't already have custom options, and the global options |
| // say we're running furious, then clone them into custom_options so we |
| // can manipulate custom options without affecting the global options. |
| if (options == NULL) { |
| RewriteOptions* global_options = server_context_->global_options(); |
| if (global_options->running_furious()) { |
| options = global_options->Clone(); |
| } |
| } |
| // TODO(anupama): Adapt the below furious experiment logic for |
| // FlushEarlyFlow as well. |
| bool need_to_store_experiment_data = false; |
| if (options != NULL && options->running_furious()) { |
| need_to_store_experiment_data = server_context_->furious_matcher()-> |
| ClassifyIntoExperiment(*async_fetch->request_headers(), options); |
| options->set_need_to_store_experiment_data(need_to_store_experiment_data); |
| } |
| const char* user_agent = async_fetch->request_headers()->Lookup1( |
| HttpAttributes::kUserAgent); |
| bool is_blink_request = BlinkUtil::IsBlinkRequest( |
| *request_url, async_fetch, options, user_agent, |
| server_context_->user_agent_matcher(), |
| RewriteOptions::kPrioritizeVisibleContent); |
| bool apply_blink_critical_line = |
| BlinkUtil::ShouldApplyBlinkFlowCriticalLine(server_context_, |
| options); |
| bool page_callback_added = false; |
| property_callback.reset(InitiatePropertyCacheLookup( |
| is_resource_fetch, *request_url, options, async_fetch, |
| &page_callback_added)); |
| if (options != NULL) { |
| server_context_->ComputeSignature(options); |
| LogRecord* log_record = async_fetch->request_context()->log_record(); |
| { |
| ScopedMutex lock(log_record->mutex()); |
| log_record->logging_info()->set_options_signature_hash( |
| server_context_->contents_hasher()->HashToUint64( |
| options->signature())); |
| } |
| } |
| |
| if (is_blink_request && apply_blink_critical_line && page_callback_added) { |
| // In blink flow, we have to modify RewriteOptions after the |
| // property cache read is completed. Hence, we clear the signature to |
| // unfreeze RewriteOptions, which was frozen during signature computation |
| // for generating key for property cache read. |
| // Warning: Please note that using this method is extremely risky and |
| // should be avoided as much as possible. If you are planning to use |
| // this, please discuss this with your team-mates and ensure that you |
| // clearly understand its implications. Also, please do repeat this |
| // warning at every place you use this method. |
| options->ClearSignatureWithCaution(); |
| |
| // TODO(rahulbansal): Remove this LOG once we expect to have |
| // a lot of such requests. |
| LOG(INFO) << "Triggering Blink flow critical line for url " |
| << url_string; |
| blink_critical_line_requests_->IncBy(1); |
| BlinkFlowCriticalLine::Start(url_string, async_fetch, options, |
| proxy_fetch_factory_.get(), |
| server_context_, |
| property_callback.release()); |
| } else { |
| RewriteDriver* driver = NULL; |
| RequestContextPtr request_ctx = async_fetch->request_context(); |
| DCHECK(request_ctx.get() != NULL) << "Async fetch must have a request" |
| << "context but does not."; |
| if (options == NULL) { |
| driver = server_context_->NewRewriteDriver(request_ctx); |
| } else { |
| // NewCustomRewriteDriver takes ownership of custom_options_. |
| driver = server_context_->NewCustomRewriteDriver(options, request_ctx); |
| } |
| |
| // TODO(mmohabey): Remove duplicate setting of user agent and |
| // request headers for different flows. |
| if (user_agent != NULL) { |
| VLOG(1) << "Setting user-agent to " << user_agent; |
| driver->SetUserAgent(user_agent); |
| } else { |
| VLOG(1) << "User-agent empty"; |
| } |
| driver->set_request_headers(async_fetch->request_headers()); |
| // TODO(mmohabey): Factor out the below checks so that they are not |
| // repeated in BlinkUtil::IsBlinkRequest(). |
| if (driver->options() != NULL && driver->options()->enabled() && |
| property_callback != NULL && |
| driver->options()->IsAllowed(url_string)) { |
| bool is_cache_html_request = BlinkUtil::IsBlinkRequest( |
| *request_url, async_fetch, driver->options(), |
| driver->user_agent().c_str(), |
| server_context_->user_agent_matcher(), RewriteOptions::kCacheHtml); |
| if (is_cache_html_request) { |
| CacheHtmlFlow::Start(url_string, async_fetch, driver, |
| proxy_fetch_factory_.get(), |
| property_callback.release()); |
| |
| return; |
| } |
| if (driver->SupportsFlushEarly()) { |
| FlushEarlyFlow::Start(url_string, &async_fetch, driver, |
| proxy_fetch_factory_.get(), |
| property_callback.get()); |
| } |
| } |
| proxy_fetch_factory_->StartNewProxyFetch( |
| url_string, async_fetch, driver, property_callback.release(), NULL); |
| } |
| } |
| |
| if (property_callback.get() != NULL) { |
| // If management of the callback was not transferred to proxy fetch, |
| // then we must detach it so it deletes itself when complete. |
| property_callback.release()->Detach(HttpStatus::kUnknownStatusCode); |
| } |
| } |
| |
| } // namespace net_instaweb |
