State dump on tracing url fetcher
diff --git a/net/instaweb/http/cache_url_async_fetcher.cc b/net/instaweb/http/cache_url_async_fetcher.cc
index effc0f5..d7230d3 100644
--- a/net/instaweb/http/cache_url_async_fetcher.cc
+++ b/net/instaweb/http/cache_url_async_fetcher.cc
@@ -614,7 +614,7 @@
}
}
-void CacheUrlAsyncFetcher::Fetch(
+void CacheUrlAsyncFetcher::FetchImpl(
const GoogleString& url, MessageHandler* handler, AsyncFetch* base_fetch) {
switch (base_fetch->request_headers()->method()) {
case RequestHeaders::kHead:
diff --git a/net/instaweb/http/cache_url_async_fetcher_test.cc b/net/instaweb/http/cache_url_async_fetcher_test.cc
index ca5885f..ebf4c42 100644
--- a/net/instaweb/http/cache_url_async_fetcher_test.cc
+++ b/net/instaweb/http/cache_url_async_fetcher_test.cc
@@ -160,12 +160,12 @@
explicit DelayedMockUrlFetcher(ThreadSynchronizer* sync)
: MockUrlFetcher(),
sync_(sync) {}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
sync_->Signal(kFetchTriggeredPrefix);
sync_->Wait(kStartFetchPrefix);
- MockUrlFetcher::Fetch(url, message_handler, fetch);
+ MockUrlFetcher::FetchImpl(url, message_handler, fetch);
}
private:
diff --git a/net/instaweb/http/counting_url_async_fetcher.cc b/net/instaweb/http/counting_url_async_fetcher.cc
index 037e5ca..2e2f0f7 100644
--- a/net/instaweb/http/counting_url_async_fetcher.cc
+++ b/net/instaweb/http/counting_url_async_fetcher.cc
@@ -65,7 +65,7 @@
CountingUrlAsyncFetcher::~CountingUrlAsyncFetcher() {
}
-void CountingUrlAsyncFetcher::Fetch(const GoogleString& url,
+void CountingUrlAsyncFetcher::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* base_fetch) {
{
diff --git a/net/instaweb/http/external_url_fetcher.cc b/net/instaweb/http/external_url_fetcher.cc
index cd11b51..7f74897 100644
--- a/net/instaweb/http/external_url_fetcher.cc
+++ b/net/instaweb/http/external_url_fetcher.cc
@@ -74,7 +74,7 @@
}
}
-void ExternalUrlFetcher::Fetch(
+void ExternalUrlFetcher::FetchImpl(
const GoogleString& url, MessageHandler* handler, AsyncFetch* fetch) {
const RequestHeaders& request_headers = *fetch->request_headers();
ResponseHeaders* response_headers = fetch->response_headers();
diff --git a/net/instaweb/http/http_dump_url_async_writer.cc b/net/instaweb/http/http_dump_url_async_writer.cc
index 3746509..c37b8ec 100644
--- a/net/instaweb/http/http_dump_url_async_writer.cc
+++ b/net/instaweb/http/http_dump_url_async_writer.cc
@@ -153,7 +153,7 @@
HttpDumpUrlAsyncWriter::~HttpDumpUrlAsyncWriter() {
}
-void HttpDumpUrlAsyncWriter::Fetch(const GoogleString& url,
+void HttpDumpUrlAsyncWriter::FetchImpl(const GoogleString& url,
MessageHandler* handler,
AsyncFetch* base_fetch) {
GoogleString filename;
diff --git a/net/instaweb/http/http_dump_url_fetcher.cc b/net/instaweb/http/http_dump_url_fetcher.cc
index 97b72b6..e770183 100644
--- a/net/instaweb/http/http_dump_url_fetcher.cc
+++ b/net/instaweb/http/http_dump_url_fetcher.cc
@@ -185,7 +185,7 @@
DISALLOW_COPY_AND_ASSIGN(HttpResponseWriter);
};
-void HttpDumpUrlFetcher::Fetch(
+void HttpDumpUrlFetcher::FetchImpl(
const GoogleString& url, MessageHandler* handler, AsyncFetch* fetch) {
bool ret = false;
GoogleString filename;
diff --git a/net/instaweb/http/mock_url_fetcher.cc b/net/instaweb/http/mock_url_fetcher.cc
index 97f5116..f1ae680 100644
--- a/net/instaweb/http/mock_url_fetcher.cc
+++ b/net/instaweb/http/mock_url_fetcher.cc
@@ -125,7 +125,7 @@
}
}
-void MockUrlFetcher::Fetch(
+void MockUrlFetcher::FetchImpl(
const GoogleString& url_in, MessageHandler* message_handler,
AsyncFetch* fetch) {
const RequestHeaders& request_headers = *fetch->request_headers();
diff --git a/net/instaweb/http/origin_mapping_url_async_fetcher.cc b/net/instaweb/http/origin_mapping_url_async_fetcher.cc
new file mode 100644
index 0000000..02a1560
--- /dev/null
+++ b/net/instaweb/http/origin_mapping_url_async_fetcher.cc
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#include "net/instaweb/http/public/origin_mapping_url_async_fetcher.h"
+#include <utility>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "pagespeed/kernel/base/abstract_mutex.h"
+#include "pagespeed/kernel/base/file_system.h"
+#include "pagespeed/kernel/base/function.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/http/google_url.h"
+#include "pagespeed/kernel/http/http_names.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/thread/scheduler.h"
+
+namespace net_instaweb {
+
+OriginMappingUrlAsyncFetcher::OriginMappingUrlAsyncFetcher(UrlAsyncFetcher* fetcher) :
+ base_fetcher_(fetcher) {
+}
+
+OriginMappingUrlAsyncFetcher::~OriginMappingUrlAsyncFetcher() {
+}
+
+void OriginMappingUrlAsyncFetcher::FetchImpl(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch) {
+ base_fetcher_->Fetch(url, message_handler, fetch);
+}
+
+bool OriginMappingUrlAsyncFetcher::SupportsHttps() const {
+ return base_fetcher_->SupportsHttps();
+}
+
+int64 OriginMappingUrlAsyncFetcher::timeout_ms() {
+ return base_fetcher_->timeout_ms();
+}
+
+void OriginMappingUrlAsyncFetcher::ShutDown() {
+ base_fetcher_->ShutDown();
+}
+
+} // namespace net_instaweb
diff --git a/net/instaweb/http/public/cache_url_async_fetcher.h b/net/instaweb/http/public/cache_url_async_fetcher.h
index df67e77..82a39a5 100644
--- a/net/instaweb/http/public/cache_url_async_fetcher.h
+++ b/net/instaweb/http/public/cache_url_async_fetcher.h
@@ -84,7 +84,7 @@
virtual bool SupportsHttps() const { return fetcher_->SupportsHttps(); }
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* base_fetch);
diff --git a/net/instaweb/http/public/counting_url_async_fetcher.h b/net/instaweb/http/public/counting_url_async_fetcher.h
index 943a401..445783c 100644
--- a/net/instaweb/http/public/counting_url_async_fetcher.h
+++ b/net/instaweb/http/public/counting_url_async_fetcher.h
@@ -49,7 +49,7 @@
virtual bool SupportsHttps() const { return fetcher_->SupportsHttps(); }
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/external_url_fetcher.h b/net/instaweb/http/public/external_url_fetcher.h
index 30fe4ab..1cc8a85 100644
--- a/net/instaweb/http/public/external_url_fetcher.h
+++ b/net/instaweb/http/public/external_url_fetcher.h
@@ -43,7 +43,7 @@
// TODO(sligocki): Allow protocol version number (e.g. HTTP/1.1)
// and request type (e.g. GET, POST, etc.) to be specified.
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/http_dump_url_async_writer.h b/net/instaweb/http/public/http_dump_url_async_writer.h
index f86b338..71daa19 100644
--- a/net/instaweb/http/public/http_dump_url_async_writer.h
+++ b/net/instaweb/http/public/http_dump_url_async_writer.h
@@ -53,7 +53,7 @@
virtual bool SupportsHttps() const { return base_fetcher_->SupportsHttps(); }
// This is a synchronous/blocking implementation.
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* handler,
AsyncFetch* base_fetch);
diff --git a/net/instaweb/http/public/http_dump_url_fetcher.h b/net/instaweb/http/public/http_dump_url_fetcher.h
index 5f61f60..3b5192d 100644
--- a/net/instaweb/http/public/http_dump_url_fetcher.h
+++ b/net/instaweb/http/public/http_dump_url_fetcher.h
@@ -70,7 +70,7 @@
}
// This is a synchronous/blocking implementation.
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/mock_url_fetcher.h b/net/instaweb/http/public/mock_url_fetcher.h
index cfdf280..ae480d4 100644
--- a/net/instaweb/http/public/mock_url_fetcher.h
+++ b/net/instaweb/http/public/mock_url_fetcher.h
@@ -63,7 +63,7 @@
const StringPiece& response_body);
// Fetching unset URLs will cause EXPECT failures as well as Done(false).
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/origin_mapping_url_async_fetcher.h b/net/instaweb/http/public/origin_mapping_url_async_fetcher.h
new file mode 100644
index 0000000..e51ec67
--- /dev/null
+++ b/net/instaweb/http/public/origin_mapping_url_async_fetcher.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#ifndef NET_INSTAWEB_HTTP_PUBLIC_ORIGIN_MAPPING_URL_ASYNC_FETCHER_H_
+#define NET_INSTAWEB_HTTP_PUBLIC_ORIGIN_MAPPING_URL_ASYNC_FETCHER_H_
+
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/file_system.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_annotations.h"
+
+namespace net_instaweb {
+
+class AbstractMutex;
+class AsyncFetch;
+class MessageHandler;
+class Scheduler;
+class ThreadSystem;
+class Timer;
+
+class OriginMappingUrlAsyncFetcher : public UrlAsyncFetcher {
+ public:
+ OriginMappingUrlAsyncFetcher(UrlAsyncFetcher* fetcher);
+
+ virtual ~OriginMappingUrlAsyncFetcher();
+
+ virtual bool SupportsHttps() const;
+
+ virtual int64 timeout_ms();
+
+ virtual void ShutDown();
+
+ protected:
+ virtual void FetchImpl(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch);
+
+ private:
+ UrlAsyncFetcher* base_fetcher_;
+ DISALLOW_COPY_AND_ASSIGN(OriginMappingUrlAsyncFetcher);
+};
+
+} // namespace net_instaweb
+
+#endif // NET_INSTAWEB_HTTP_PUBLIC_ORIGIN_MAPPING_URL_ASYNC_FETCHER_H_
diff --git a/net/instaweb/http/public/rate_controlling_url_async_fetcher.h b/net/instaweb/http/public/rate_controlling_url_async_fetcher.h
index e6ab267..c004bfc 100644
--- a/net/instaweb/http/public/rate_controlling_url_async_fetcher.h
+++ b/net/instaweb/http/public/rate_controlling_url_async_fetcher.h
@@ -53,7 +53,7 @@
return base_fetcher_->SupportsHttps();
}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/reflecting_test_fetcher.h b/net/instaweb/http/public/reflecting_test_fetcher.h
index 2d48826..114fad1 100644
--- a/net/instaweb/http/public/reflecting_test_fetcher.h
+++ b/net/instaweb/http/public/reflecting_test_fetcher.h
@@ -42,7 +42,7 @@
ReflectingTestFetcher() {}
virtual ~ReflectingTestFetcher() {}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
RequestHeaders* in = fetch->request_headers();
diff --git a/net/instaweb/http/public/simulated_delay_fetcher.h b/net/instaweb/http/public/simulated_delay_fetcher.h
index 812f59e..22faabd 100644
--- a/net/instaweb/http/public/simulated_delay_fetcher.h
+++ b/net/instaweb/http/public/simulated_delay_fetcher.h
@@ -63,7 +63,7 @@
virtual ~SimulatedDelayFetcher();
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/tracing_url_async_fetcher.h b/net/instaweb/http/public/tracing_url_async_fetcher.h
new file mode 100644
index 0000000..e6b67d4
--- /dev/null
+++ b/net/instaweb/http/public/tracing_url_async_fetcher.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#ifndef NET_INSTAWEB_HTTP_PUBLIC_TRACING_URL_ASYNC_FETCHER_H_
+#define NET_INSTAWEB_HTTP_PUBLIC_TRACING_URL_ASYNC_FETCHER_H_
+
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/file_system.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_annotations.h"
+
+namespace net_instaweb {
+
+class AbstractMutex;
+class AsyncFetch;
+class MessageHandler;
+class Scheduler;
+class ThreadSystem;
+class Timer;
+
+class TracingUrlAsyncFetcher : public UrlAsyncFetcher {
+ public:
+ TracingUrlAsyncFetcher(UrlAsyncFetcher* fetcher);
+
+ virtual ~TracingUrlAsyncFetcher();
+
+ virtual bool SupportsHttps() const;
+
+ virtual int64 timeout_ms();
+
+ virtual void ShutDown();
+ protected:
+ virtual void FetchImpl(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch);
+ private:
+ UrlAsyncFetcher* base_fetcher_;
+ DISALLOW_COPY_AND_ASSIGN(TracingUrlAsyncFetcher);
+};
+
+} // namespace net_instaweb
+
+#endif // NET_INSTAWEB_HTTP_PUBLIC_ORIGIN_MAPPING_URL_ASYNC_FETCHER_H_
diff --git a/net/instaweb/http/public/ua_sensitive_test_fetcher.h b/net/instaweb/http/public/ua_sensitive_test_fetcher.h
index 70e7c41..6012433 100644
--- a/net/instaweb/http/public/ua_sensitive_test_fetcher.h
+++ b/net/instaweb/http/public/ua_sensitive_test_fetcher.h
@@ -38,7 +38,7 @@
class UserAgentSensitiveTestFetcher : public UrlAsyncFetcher {
public:
explicit UserAgentSensitiveTestFetcher(UrlAsyncFetcher* base_fetcher);
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/public/url_async_fetcher.h b/net/instaweb/http/public/url_async_fetcher.h
index dc7f6f3..00ffd9e 100644
--- a/net/instaweb/http/public/url_async_fetcher.h
+++ b/net/instaweb/http/public/url_async_fetcher.h
@@ -34,6 +34,8 @@
public:
static const int64 kUnspecifiedTimeout;
+ static int64 fetchId;
+
virtual ~UrlAsyncFetcher();
// Asynchronously fetch a URL, set the response headers and stream the
@@ -45,9 +47,9 @@
//
// TODO(sligocki): GoogleString -> GoogleUrl or at least StringPiece.
// TODO(sligocki): Include the URL in the fetch, like the request headers.
- virtual void Fetch(const GoogleString& url,
- MessageHandler* message_handler,
- AsyncFetch* fetch) = 0;
+ void Fetch(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch);
// Determine if the fetcher supports fetching using HTTPS. By default we
// assume a fetcher can.
@@ -75,10 +77,14 @@
protected:
// Put this in protected to make sure nobody constructs this class except
// for subclasses.
- UrlAsyncFetcher() : fetch_with_gzip_(false) {}
+ UrlAsyncFetcher() : fetch_with_gzip_(false), trace_(true) {}
+ virtual void FetchImpl(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch) = 0;
private:
bool fetch_with_gzip_;
+ bool trace_;
DISALLOW_COPY_AND_ASSIGN(UrlAsyncFetcher);
};
diff --git a/net/instaweb/http/public/url_async_fetcher_stats.h b/net/instaweb/http/public/url_async_fetcher_stats.h
index 536b83c..e0b311c 100644
--- a/net/instaweb/http/public/url_async_fetcher_stats.h
+++ b/net/instaweb/http/public/url_async_fetcher_stats.h
@@ -59,7 +59,7 @@
// Reimplementation of UrlAsyncFetcher methods. See base class
// for API specifications.
virtual bool SupportsHttps() const;
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
virtual int64 timeout_ms();
diff --git a/net/instaweb/http/public/wait_url_async_fetcher.h b/net/instaweb/http/public/wait_url_async_fetcher.h
index 3752178..850cbba 100644
--- a/net/instaweb/http/public/wait_url_async_fetcher.h
+++ b/net/instaweb/http/public/wait_url_async_fetcher.h
@@ -45,7 +45,7 @@
virtual ~WaitUrlAsyncFetcher();
// Initiate fetches that will finish when CallCallbacks is called.
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* handler,
AsyncFetch* fetch);
diff --git a/net/instaweb/http/rate_controlling_url_async_fetcher.cc b/net/instaweb/http/rate_controlling_url_async_fetcher.cc
index b75038c..f88462c 100644
--- a/net/instaweb/http/rate_controlling_url_async_fetcher.cc
+++ b/net/instaweb/http/rate_controlling_url_async_fetcher.cc
@@ -43,7 +43,7 @@
RateControllingUrlAsyncFetcher::~RateControllingUrlAsyncFetcher() {
}
-void RateControllingUrlAsyncFetcher::Fetch(const GoogleString& url,
+void RateControllingUrlAsyncFetcher::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
rate_controller_->Fetch(base_fetcher_, url, message_handler, fetch);
diff --git a/net/instaweb/http/simulated_delay_fetcher.cc b/net/instaweb/http/simulated_delay_fetcher.cc
index 56e8bfd..55fea1a 100644
--- a/net/instaweb/http/simulated_delay_fetcher.cc
+++ b/net/instaweb/http/simulated_delay_fetcher.cc
@@ -65,7 +65,7 @@
file_system_->Close(request_log_, message_handler_);
}
-void SimulatedDelayFetcher::Fetch(const GoogleString& url,
+void SimulatedDelayFetcher::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
fetch = EnableInflation(fetch);
diff --git a/net/instaweb/http/sync_fetcher_adapter_test.cc b/net/instaweb/http/sync_fetcher_adapter_test.cc
index af7b808..758b3e9 100644
--- a/net/instaweb/http/sync_fetcher_adapter_test.cc
+++ b/net/instaweb/http/sync_fetcher_adapter_test.cc
@@ -83,7 +83,7 @@
sync_(NULL) {
}
- virtual void Fetch(const GoogleString& url, MessageHandler* handler,
+ virtual void FetchImpl(const GoogleString& url, MessageHandler* handler,
AsyncFetch* fetch) {
CHECK(!fetch_pending_);
fetch_ = fetch;
diff --git a/net/instaweb/http/tracing_url_async_fetcher.cc b/net/instaweb/http/tracing_url_async_fetcher.cc
new file mode 100644
index 0000000..af7f80f
--- /dev/null
+++ b/net/instaweb/http/tracing_url_async_fetcher.cc
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+#include "net/instaweb/http/public/tracing_url_async_fetcher.h"
+#include <iostream>
+#include <utility>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "pagespeed/kernel/base/abstract_mutex.h"
+#include "pagespeed/kernel/base/file_system.h"
+#include "pagespeed/kernel/base/function.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/http/google_url.h"
+#include "pagespeed/kernel/http/http_names.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/thread/scheduler.h"
+
+namespace net_instaweb {
+
+TracingUrlAsyncFetcher::TracingUrlAsyncFetcher(UrlAsyncFetcher* fetcher) :
+ base_fetcher_(fetcher) {
+}
+
+TracingUrlAsyncFetcher::~TracingUrlAsyncFetcher() {
+}
+
+void TracingUrlAsyncFetcher::FetchImpl(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch) {
+ //std::cerr << "pre trace fetch: " << url << " " << fetch->request_headers()->ToString() << std::endl;
+ base_fetcher_->Fetch(url, message_handler, fetch);
+}
+
+bool TracingUrlAsyncFetcher::SupportsHttps() const {
+ return base_fetcher_->SupportsHttps();
+}
+
+int64 TracingUrlAsyncFetcher::timeout_ms() {
+ return base_fetcher_->timeout_ms();
+}
+
+void TracingUrlAsyncFetcher::ShutDown() {
+ base_fetcher_->ShutDown();
+}
+
+} // namespace net_instaweb
diff --git a/net/instaweb/http/ua_sensitive_test_fetcher.cc b/net/instaweb/http/ua_sensitive_test_fetcher.cc
index b819b75..7dff541 100644
--- a/net/instaweb/http/ua_sensitive_test_fetcher.cc
+++ b/net/instaweb/http/ua_sensitive_test_fetcher.cc
@@ -39,7 +39,7 @@
UrlAsyncFetcher* base_fetcher)
: base_fetcher_(base_fetcher) {}
-void UserAgentSensitiveTestFetcher::Fetch(const GoogleString& url,
+void UserAgentSensitiveTestFetcher::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
GoogleUrl parsed_url(url);
diff --git a/net/instaweb/http/url_async_fetcher.cc b/net/instaweb/http/url_async_fetcher.cc
index 14a5f98..d2104a2 100644
--- a/net/instaweb/http/url_async_fetcher.cc
+++ b/net/instaweb/http/url_async_fetcher.cc
@@ -18,12 +18,18 @@
#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/inflating_fetch.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/string_util.h"
namespace net_instaweb {
const int64 UrlAsyncFetcher::kUnspecifiedTimeout = 0;
+// TODO(oschaaf): we have counters for this, needs a mutex
+int64 UrlAsyncFetcher::fetchId = 0;
+
UrlAsyncFetcher::~UrlAsyncFetcher() {
}
@@ -38,4 +44,91 @@
return inflating_fetch;
}
+class TracingFetch : public SharedAsyncFetch {
+ public:
+ TracingFetch(UrlAsyncFetcher* sender, MessageHandler* message_handler, const GoogleString& url, AsyncFetch* base_fetch)
+ : SharedAsyncFetch(base_fetch), url_(url), base_fetch_(base_fetch), original_fetcher_name_(typeid(*sender).name()), message_handler_(message_handler) {
+ RequestHeaders* request_headers = base_fetch->request_headers();
+ const char* id = request_headers->Lookup1("X-PageSpeed-UrlAsyncFetcher-Id");
+ const char* nested = request_headers->Lookup1("X-PageSpeed-UrlAsyncFetcher-Nested");
+ if (id == nullptr) {
+ request_headers->Add("X-PageSpeed-UrlAsyncFetcher-Id", Integer64ToString(++UrlAsyncFetcher::fetchId));
+ }
+ int64 iNested = 0;
+ if (nested == nullptr) {
+ request_headers->Add("X-PageSpeed-UrlAsyncFetcher-Nested", "0");
+ } else {
+ StringToInt64(nested, &iNested);
+ request_headers->Replace("X-PageSpeed-UrlAsyncFetcher-Nested", Integer64ToString(++iNested));
+ }
+ id = request_headers->Lookup1("X-PageSpeed-UrlAsyncFetcher-Id");
+ id_ = GoogleString(id);
+ nested_ = iNested;
+
+ AsyncFetch* f = base_fetch_;
+ while (dynamic_cast<TracingFetch*>(f) != nullptr) {
+ f = dynamic_cast<TracingFetch*>(f)->base_fetch_;
+ }
+
+ base_fetch_name_ = GoogleString(typeid(*f).name());
+ message_handler_->Message(kInfo, "(f:%s-%ld) %s by %s for %s",
+ id,
+ nested_,
+ url_.c_str(),
+ original_fetcher_name_.c_str(),
+ base_fetch_name_.c_str());
+ }
+
+ virtual bool HandleWrite(const StringPiece& content,
+ MessageHandler* handler) {
+ return SharedAsyncFetch::HandleWrite(content, handler);
+ }
+
+ virtual void HandleHeadersComplete() {
+ traced_request_headers_.CopyFrom(*base_fetch_->request_headers());
+ traced_response_headers_.CopyFrom(*base_fetch_->response_headers());
+ SharedAsyncFetch::HandleHeadersComplete();
+ }
+
+ virtual void HandleDone(bool success) {
+ const char* id = id_.c_str();
+
+ message_handler_->Message(kInfo, "(d:%s-%ld) %s from %s (%d) on behalf of %s",
+ id,
+ nested_,
+ url_.c_str(),
+ base_fetch_name_.c_str(),
+ success ? 1 : 0,
+ original_fetcher_name_.c_str());
+
+ if (!nested_) {
+ message_handler_->Message(kInfo, " (RQ) %s",
+ traced_request_headers_.ToString().c_str());
+ message_handler_->Message(kInfo, " (RS) %s",
+ traced_response_headers_.ToString().c_str());
+ }
+ SharedAsyncFetch::HandleDone(success);
+ delete this;
+ }
+
+ private:
+ GoogleString url_;
+ AsyncFetch* base_fetch_;
+ GoogleString original_fetcher_name_;
+ GoogleString base_fetch_name_;
+ int64 nested_;
+ RequestHeaders traced_request_headers_;
+ ResponseHeaders traced_response_headers_;
+ MessageHandler* message_handler_;
+ GoogleString id_;
+
+ DISALLOW_COPY_AND_ASSIGN(TracingFetch);
+};
+
+void UrlAsyncFetcher::Fetch(const GoogleString& url,
+ MessageHandler* message_handler,
+ AsyncFetch* fetch) {
+ FetchImpl(url, message_handler, new TracingFetch(this, message_handler, url, fetch));
+}
+
} // namespace net_instaweb
diff --git a/net/instaweb/http/url_async_fetcher_stats.cc b/net/instaweb/http/url_async_fetcher_stats.cc
index 0cf60f0..8870b0c 100644
--- a/net/instaweb/http/url_async_fetcher_stats.cc
+++ b/net/instaweb/http/url_async_fetcher_stats.cc
@@ -121,7 +121,7 @@
}
-void UrlAsyncFetcherStats::Fetch(const GoogleString& url,
+void UrlAsyncFetcherStats::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
fetch = EnableInflation(fetch);
diff --git a/net/instaweb/http/wait_url_async_fetcher.cc b/net/instaweb/http/wait_url_async_fetcher.cc
index f493f3a..4d53452 100644
--- a/net/instaweb/http/wait_url_async_fetcher.cc
+++ b/net/instaweb/http/wait_url_async_fetcher.cc
@@ -53,7 +53,7 @@
WaitUrlAsyncFetcher::~WaitUrlAsyncFetcher() {}
-void WaitUrlAsyncFetcher::Fetch(const GoogleString& url,
+void WaitUrlAsyncFetcher::FetchImpl(const GoogleString& url,
MessageHandler* handler,
AsyncFetch* base_fetch) {
DelayedFetch* delayed_fetch =
diff --git a/net/instaweb/instaweb.gyp b/net/instaweb/instaweb.gyp
index 7b3b2df..d87af69 100644
--- a/net/instaweb/instaweb.gyp
+++ b/net/instaweb/instaweb.gyp
@@ -1232,9 +1232,11 @@
'http/http_value.cc',
'http/http_value_writer.cc',
'http/inflating_fetch.cc',
+ 'http/origin_mapping_url_async_fetcher.cc',
'http/rate_controller.cc',
'http/rate_controlling_url_async_fetcher.cc',
'http/sync_fetcher_adapter_callback.cc',
+ 'http/tracing_url_async_fetcher.cc',
'http/url_async_fetcher.cc',
'http/url_async_fetcher_stats.cc',
'http/wait_url_async_fetcher.cc',
diff --git a/net/instaweb/rewriter/in_place_rewrite_context_test.cc b/net/instaweb/rewriter/in_place_rewrite_context_test.cc
index 53d41da..a821130 100644
--- a/net/instaweb/rewriter/in_place_rewrite_context_test.cc
+++ b/net/instaweb/rewriter/in_place_rewrite_context_test.cc
@@ -362,6 +362,7 @@
EXPECT_EQ(expected_body, notifying_fetch.content()) << url;
EXPECT_EQ(expected_ttl, response_headers_.cache_ttl_ms()) << url;
EXPECT_STREQ(etag, response_headers_.Lookup1(HttpAttributes::kEtag)) << url;
+ std::cerr << response_headers_.ToString() << std::endl;
EXPECT_EQ(date_ms, response_headers_.date_ms()) << url;
}
diff --git a/net/instaweb/rewriter/server_context.cc b/net/instaweb/rewriter/server_context.cc
index 575dca3..36f66bc 100644
--- a/net/instaweb/rewriter/server_context.cc
+++ b/net/instaweb/rewriter/server_context.cc
@@ -26,7 +26,9 @@
#include "net/instaweb/config/rewrite_options_manager.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/http_cache.h"
+#include "net/instaweb/http/public/origin_mapping_url_async_fetcher.h"
#include "net/instaweb/http/public/sync_fetcher_adapter_callback.h"
+#include "net/instaweb/http/public/tracing_url_async_fetcher.h"
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/input_info.pb.h"
@@ -1022,6 +1024,10 @@
void ServerContext::ApplySessionFetchers(const RequestContextPtr& req,
RewriteDriver* driver) {
+ driver->SetSessionFetcher(new OriginMappingUrlAsyncFetcher(
+ driver->async_fetcher()));
+ driver->SetSessionFetcher(new TracingUrlAsyncFetcher(
+ driver->async_fetcher()));
}
RequestProperties* ServerContext::NewRequestProperties() {
diff --git a/pagespeed/automatic/proxy_interface.cc b/pagespeed/automatic/proxy_interface.cc
index 0f95a47..e1cdde1 100644
--- a/pagespeed/automatic/proxy_interface.cc
+++ b/pagespeed/automatic/proxy_interface.cc
@@ -152,7 +152,7 @@
return ret;
}
-void ProxyInterface::Fetch(const GoogleString& requested_url_string,
+void ProxyInterface::FetchImpl(const GoogleString& requested_url_string,
MessageHandler* handler,
AsyncFetch* async_fetch) {
GoogleUrl requested_url(requested_url_string);
diff --git a/pagespeed/automatic/proxy_interface.h b/pagespeed/automatic/proxy_interface.h
index db5f29d..90c0066 100644
--- a/pagespeed/automatic/proxy_interface.h
+++ b/pagespeed/automatic/proxy_interface.h
@@ -58,7 +58,7 @@
// All requests use this interface. We decide internally whether the
// request is a pagespeed resource, HTML page to be rewritten or another
// resource to be proxied directly.
- virtual void Fetch(const GoogleString& requested_url,
+ virtual void FetchImpl(const GoogleString& requested_url,
MessageHandler* handler,
AsyncFetch* async_fetch);
diff --git a/pagespeed/automatic/proxy_interface_test_base.h b/pagespeed/automatic/proxy_interface_test_base.h
index 746b2e1..1075dc1 100644
--- a/pagespeed/automatic/proxy_interface_test_base.h
+++ b/pagespeed/automatic/proxy_interface_test_base.h
@@ -155,7 +155,7 @@
num_background_fetches_(0) {}
virtual ~BackgroundFetchCheckingUrlAsyncFetcher() {}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
if (fetch->IsBackgroundFetch()) {
diff --git a/pagespeed/system/add_headers_fetcher.cc b/pagespeed/system/add_headers_fetcher.cc
index 397e7d8..44264ca 100644
--- a/pagespeed/system/add_headers_fetcher.cc
+++ b/pagespeed/system/add_headers_fetcher.cc
@@ -29,7 +29,7 @@
AddHeadersFetcher::~AddHeadersFetcher() {}
-void AddHeadersFetcher::Fetch(const GoogleString& original_url,
+void AddHeadersFetcher::FetchImpl(const GoogleString& original_url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
RequestHeaders* request_headers = fetch->request_headers();
diff --git a/pagespeed/system/add_headers_fetcher.h b/pagespeed/system/add_headers_fetcher.h
index cc140db..e4630f3 100644
--- a/pagespeed/system/add_headers_fetcher.h
+++ b/pagespeed/system/add_headers_fetcher.h
@@ -41,7 +41,7 @@
return backend_fetcher_->SupportsHttps();
}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* callback);
diff --git a/pagespeed/system/loopback_route_fetcher.cc b/pagespeed/system/loopback_route_fetcher.cc
index c04909f..1144265 100644
--- a/pagespeed/system/loopback_route_fetcher.cc
+++ b/pagespeed/system/loopback_route_fetcher.cc
@@ -51,7 +51,7 @@
LoopbackRouteFetcher::~LoopbackRouteFetcher() {
}
-void LoopbackRouteFetcher::Fetch(const GoogleString& original_url,
+void LoopbackRouteFetcher::FetchImpl(const GoogleString& original_url,
MessageHandler* message_handler,
AsyncFetch* fetch) {
GoogleString url = original_url;
diff --git a/pagespeed/system/loopback_route_fetcher.h b/pagespeed/system/loopback_route_fetcher.h
index ff58479..75dc177 100644
--- a/pagespeed/system/loopback_route_fetcher.h
+++ b/pagespeed/system/loopback_route_fetcher.h
@@ -53,7 +53,7 @@
return backend_fetcher_->SupportsHttps();
}
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* fetch);
diff --git a/pagespeed/system/serf_url_async_fetcher.cc b/pagespeed/system/serf_url_async_fetcher.cc
index 6bbb5c2..b1152c1 100644
--- a/pagespeed/system/serf_url_async_fetcher.cc
+++ b/pagespeed/system/serf_url_async_fetcher.cc
@@ -1248,7 +1248,7 @@
return started;
}
-void SerfUrlAsyncFetcher::Fetch(const GoogleString& url,
+void SerfUrlAsyncFetcher::FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* async_fetch) {
async_fetch = EnableInflation(async_fetch);
diff --git a/pagespeed/system/serf_url_async_fetcher.h b/pagespeed/system/serf_url_async_fetcher.h
index 0790f6f..71c66a8 100644
--- a/pagespeed/system/serf_url_async_fetcher.h
+++ b/pagespeed/system/serf_url_async_fetcher.h
@@ -131,7 +131,7 @@
virtual bool SupportsHttps() const;
- virtual void Fetch(const GoogleString& url,
+ virtual void FetchImpl(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* callback);
// TODO(morlovich): Make private once non-thread mode concept removed.
