trunk-tracking: update from r3852 to r3872
diff --git a/src/ngx_pagespeed.cc b/src/ngx_pagespeed.cc
index 4128b55..c45dfd8 100644
--- a/src/ngx_pagespeed.cc
+++ b/src/ngx_pagespeed.cc
@@ -37,6 +37,7 @@
#include "ngx_server_context.h"
#include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/request_context.h"
@@ -48,6 +49,7 @@
#include "net/instaweb/rewriter/public/resource_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_query.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/in_place_resource_recorder.h"
@@ -908,18 +910,9 @@
return port;
}
+} // namespace
-GoogleString ps_determine_url(ngx_http_request_t* r) {
- int port = ps_determine_port(r);
- GoogleString port_string;
- if ((ps_is_https(r) && (port == 443 || port == -1)) ||
- (!ps_is_https(r) && (port == 80 || port == -1))) {
- // No port specifier needed for requests on default ports.
- port_string = "";
- } else {
- port_string = StrCat(":", IntegerToString(port));
- }
-
+StringPiece ps_determine_host(ngx_http_request_t* r) {
StringPiece host = str_to_string_piece(r->headers_in.server);
if (host.size() == 0) {
// If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
@@ -934,6 +927,23 @@
}
host = str_to_string_piece(s);
}
+ return host;
+}
+
+namespace {
+
+GoogleString ps_determine_url(ngx_http_request_t* r) {
+ int port = ps_determine_port(r);
+ GoogleString port_string;
+ if ((ps_is_https(r) && (port == 443 || port == -1)) ||
+ (!ps_is_https(r) && (port == 80 || port == -1))) {
+ // No port specifier needed for requests on default ports.
+ port_string = "";
+ } else {
+ port_string = StrCat(":", IntegerToString(port));
+ }
+
+ StringPiece host = ps_determine_host(r);
return StrCat(ps_is_https(r) ? "https://" : "http://",
host, port_string, str_to_string_piece(r->unparsed_uri));
@@ -1214,11 +1224,9 @@
// make cache key consistent for both lookup and storing in cache.
//
// Sets option from request headers and url.
- ServerContext::OptionsBoolPair query_options_success =
- cfg_s->server_context->GetQueryOptions(url, request_headers,
- response_headers);
- bool get_query_options_success = query_options_success.second;
- if (!get_query_options_success) {
+ RewriteQuery rewrite_query;
+ if (!cfg_s->server_context->GetQueryOptions(
+ url, request_headers, response_headers, &rewrite_query)) {
// Failed to parse query params or request headers. Treat this as if there
// were no query params given.
ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
@@ -1228,7 +1236,7 @@
// Will be NULL if there aren't any options set with query params or in
// headers.
- return query_options_success.first;
+ return rewrite_query.ReleaseOptions();
}
// Check whether this visitor is already in an experiment. If they're not,
@@ -1307,11 +1315,12 @@
// rebased on the directory options or the global options.
RewriteOptions* request_options = ps_determine_request_options(
r, request_headers, response_headers, cfg_s, url);
+ bool have_request_options = request_options != NULL;
// Because the caller takes ownership of any options we return, the only
// situation in which we can avoid allocating a new RewriteOptions is if the
// global options are ok as are.
- if (directory_options == NULL && request_options == NULL &&
+ if (!have_request_options && directory_options == NULL &&
!global_options->running_experiment()) {
return true;
}
@@ -1324,7 +1333,6 @@
}
// Modify our options in response to request options if specified.
- bool have_request_options = request_options != NULL;
if (have_request_options) {
(*options)->Merge(*request_options);
delete request_options;
@@ -2179,6 +2187,7 @@
ctx->recorder = new InPlaceResourceRecorder(
RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
url,
+ ctx->driver->CacheFragment(),
request_headers.GetProperties(),
options->respect_vary(),
options->ipro_max_response_bytes(),
@@ -2282,6 +2291,23 @@
return ngx_http_output_filter(r, out);
}
+namespace {
+
+// TODO(jefftk): This class is a temporary shim to just support console fetch,
+// but we eventually want to convert everything in ps_simple_handler to use
+// something akin to NgxBaseFetch (which sends results direct to nginx). This
+// is work in progress (but well along) in the mod_pagespeed world.
+class NgxPagespeedConsoleAsyncFetch : public AsyncFetchUsingWriter {
+ public:
+ NgxPagespeedConsoleAsyncFetch(const RequestContextPtr& request_context,
+ Writer* writer)
+ : AsyncFetchUsingWriter(request_context, writer) { }
+ virtual void HandleDone(bool status) { }
+ virtual void HandleHeadersComplete() { }
+};
+
+} // namespace
+
ngx_int_t ps_simple_handler(ngx_http_request_t* r,
NgxServerContext* server_context,
RequestRouting::Response response_category) {
@@ -2325,9 +2351,16 @@
&writer);
break;
}
- case RequestRouting::kConsole:
- server_context->ConsoleHandler(*server_context->config(), &writer);
+ case RequestRouting::kConsole: {
+ ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+ RequestContextPtr request_context(
+ cfg_s->server_context->NewRequestContext(r));
+ NgxPagespeedConsoleAsyncFetch fetch(request_context, &writer);
+ server_context->ConsoleHandler(*server_context->config(), &fetch);
+ status = static_cast<HttpStatus::Code>(
+ fetch.response_headers()->status_code());
break;
+ }
case RequestRouting::kMessages: {
GoogleString log;
StringWriter log_writer(&log);
diff --git a/src/ngx_pagespeed.h b/src/ngx_pagespeed.h
index 51b25f6..c043794 100644
--- a/src/ngx_pagespeed.h
+++ b/src/ngx_pagespeed.h
@@ -117,6 +117,8 @@
const ResponseHeaders& pagespeed_headers,
PreserveCachingHeaders preserve_caching_headers);
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
} // namespace net_instaweb
#endif // NGX_PAGESPEED_H_
diff --git a/src/ngx_server_context.cc b/src/ngx_server_context.cc
index 3e0de11..0ab1da9 100644
--- a/src/ngx_server_context.cc
+++ b/src/ngx_server_context.cc
@@ -72,6 +72,7 @@
return new SystemRequestContext(thread_system()->NewMutex(),
timer(),
+ ps_determine_host(r),
local_port,
str_to_string_piece(local_ip));
}
diff --git a/test/nginx_system_test.sh b/test/nginx_system_test.sh
index 6a3fc77..014b909 100755
--- a/test/nginx_system_test.sh
+++ b/test/nginx_system_test.sh
@@ -1091,8 +1091,8 @@
# spelling it out to avoid test regolds when we add image filter IDs.
WGET_ARGS="--save-headers"
http_proxy=$SECONDARY_HOSTNAME fetch_until -save -recursive \
- http://embed-config-html.example.com/embed-config.html \
- 'fgrep -c .pagespeed.' 3
+ http://embed-config-html.example.org/embed-config.html \
+ 'grep -c \.pagespeed\.' 3
# with the default rewriters in vhost embed-config-resources.example.com
# the image will be >200k. But by enabling resizing & compression 73
@@ -2158,7 +2158,7 @@
fgrep -q "Vary: $OUT_VARY"
fi
check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
- fgrep -q "Cache-Control: ${OUT_CC:-max-age=}"
+ grep -q "Cache-Control: ${OUT_CC:-max-age=[0-9]*}$"
# TODO: check file type of webp. Irrelevant for now.
}
@@ -2184,12 +2184,15 @@
WGETRC=$TEMPDIR/wgetrc-ua
export WGETRC
-# IE 11 does not cache Vary: Accept. For now we send it nonetheless. See
-# TODO above.
-IE11_UA="Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
+# IE 9 and later must re-validate Vary: Accept. We should send CC: private.
+IE9_UA="Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"
+IE11_UA="Mozilla/5.0 (Windows NT 6.1; WOW64; ***********; rv:11.0) like Gecko"
+echo "user_agent = $IE9_UA" > $WGETRC
+# (no accept) Type Out Vary CC
+test_ipro_for_browser_webp "IE 9" "" "" photo jpeg "" "max-age=[0-9]*,private"
+test_ipro_for_browser_webp "IE 9" "" "" synth png
echo "user_agent = $IE11_UA" > $WGETRC
-# (no accept) Type Out Vary
-test_ipro_for_browser_webp "IE 11" "" "" photo jpeg "Accept"
+test_ipro_for_browser_webp "IE 11" "" "" photo jpeg "" "max-age=[0-9]*,private"
test_ipro_for_browser_webp "IE 11" "" "" synth png
# Older Opera did not support webp.
@@ -2387,6 +2390,12 @@
check test $RESOURCE_MAX_AGE -lt 333
check test $RESOURCE_MAX_AGE -gt 300
+# TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
+# MapProxyDomain construct at a static server. Perhaps localhost:8050 will
+# serve, but the tests need to use different urls then. For mod_pagespeed these
+# tests immediately precede "IPRO-optimized resources should have fixed size,
+# not chunked." in system_test.sh.
+
start_test IPRO-optimized resources should have fixed size, not chunked.
URL="$EXAMPLE_ROOT/images/Puzzle.jpg"
URL+="?PageSpeedJpegRecompressionQuality=75"
@@ -2398,6 +2407,16 @@
check_not_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
fgrep -q 'Transfer-Encoding: chunked'
+start_test IPRO 304 with etags
+# Reuses $URL and $FETCH_UNTIL_OUTFILE from previous test.
+check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" fgrep -q 'ETag:'
+ETAG=$(extract_headers $FETCH_UNTIL_OUTFILE | awk '/ETag:/ {print $2}')
+echo $WGET_DUMP --header "If-None-Match: $ETAG" $URL
+OUTFILE=$OUTDIR/etags
+# Note: -o gets debug info which is the only place that 304 message is sent.
+$WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
+check fgrep -q "awaiting response... 304" $OUTFILE
+
start_test PageSpeed resources should have a content length.
URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
OUT=$($WGET_DUMP --save-headers $URL)
diff --git a/test/pagespeed_test.conf.template b/test/pagespeed_test.conf.template
index dd2e655..6dfaf14 100644
--- a/test/pagespeed_test.conf.template
+++ b/test/pagespeed_test.conf.template
@@ -494,7 +494,7 @@
# Note that we test with two distinct caches.
server {
listen @@SECONDARY_PORT@@;
- server_name embed-config-html.example.com;
+ server_name embed-config-html.example.org;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -504,6 +504,9 @@
pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
pagespeed Domain embed-config-resources.example.com;
+ # Share a cache keyspace with embed-config-resources.example.com.
+ pagespeed CacheFragment "embed-config";
+
pagespeed LoadFromFile "http://embed-config-resources.example.com/"
"@@SERVER_ROOT@@/mod_pagespeed_example/";
}
@@ -519,7 +522,10 @@
# Note that we do not set the jpeg quality here, but take
# it from image URL query parameters that we synthesize in
- # from embed-config-html.example.com.
+ # from embed-config-html.example.org.
+
+ # Share a cache keyspace with embed-config-html.example.org.
+ pagespeed CacheFragment "embed-config";
pagespeed LoadFromFile "http://embed-config-resources.example.com/"
"@@SERVER_ROOT@@/mod_pagespeed_example/";
@@ -587,6 +593,7 @@
pagespeed MapRewriteDomain cdn.example.com origin.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css,rewrite_images;
+ pagespeed CacheFragment "example";
}
server {
# Sets up a logical origin for CDNs to fetch content from, on
@@ -599,6 +606,7 @@
pagespeed MapRewriteDomain cdn.example.com origin.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css,rewrite_images;
+ pagespeed CacheFragment "example";
}
server {
# Sets up a logical cdn, which is where we tell browsers to fetch resources
