Log errors if serf error rate seems too high. (#1507)
diff --git a/html/doc/faq.html b/html/doc/faq.html
index 945563b..f842f45 100644
--- a/html/doc/faq.html
+++ b/html/doc/faq.html
@@ -698,7 +698,36 @@
are <code>display: inline</code>.
</p>
+<h2 id="warning-fetch-rate">I've got a warning saying
+"Serf fetch failure rate extremely high". What does this mean?</h2>
+<p>The warning means that, when PageSpeed tried to fetch resources inside your
+web page for optimization, over 50% of attempts inside a 30-minute period
+failed. This may just mean you have some broken resource includes in your
+pages (in which case, it may be a good idea to fix them for better performance),
+but might indicate that PageSpeed's fetching is not working properly. If you
+have in-place resource optimization on, that can result in user requests for
+<code>.pagespeed.</code> URLs returning error 404 intermittently.</p>
+
+<p>First of all, check to see if the log mentions anything else about fetch
+trouble. If what's there is not helpful, the root cause may be more obvious
+if you follow these steps:</p>
+<ol>
+<li>Disable <a href="system#ipro">in-place resource optimization temporarily.
+ </li>
+<li>Clear PageSpeed cache.</li>
+<li>Open a test page with a <code>?PagespeedFilters=+debug</code> query
+ parameter, reload it a few times, and see if resources are getting optimized,
+ and if not if there is an error message.
+<li>Revert the config changes.</li>
+</ol>
+
+<p>Most likely you may need to configure an <a href="domains#mapping_origin">
+origin domain, to specify the host or IP to talk to fetch resources.</p>
+
+<p>You may also want to consider using <a href="domains#ModPagespeedLoadFromFile">
+<code>LoadFromFile</code></a> functionality, as that performs much better if
+your resources are static.</p>
</div>
<!--#include virtual="_footer.html" -->
</body>
diff --git a/pagespeed/system/serf_url_async_fetcher.cc b/pagespeed/system/serf_url_async_fetcher.cc
index 3fd71a3..6bbb5c2 100644
--- a/pagespeed/system/serf_url_async_fetcher.cc
+++ b/pagespeed/system/serf_url_async_fetcher.cc
@@ -65,6 +65,9 @@
kAllowCertificateNotYetValid = 1 << 3,
};
+const int kReliabilityCheckPeriodMs = 30 * net_instaweb::Timer::kMinuteMs;
+const int kReliabilityCheckMinFetches = 5;
+
} // namespace
extern "C" {
@@ -102,6 +105,12 @@
const char SerfStats::kSerfFetchFailureCount[] = "serf_fetch_failure_count";
const char SerfStats::kSerfFetchCertErrors[] = "serf_fetch_cert_errors";
const char SerfStats::kSerfFetchReadCalls[] = "serf_fetch_num_calls_to_read";
+const char SerfStats::kSerfFetchUltimateSuccess[] =
+ "serf_fetch_ultimate_success";
+const char SerfStats::kSerfFetchUltimateFailure[] =
+ "serf_fetch_ultimate_failure";
+const char SerfStats::kSerfFetchLastCheckTimestampMs[] =
+ "serf_fetch_last_check_timestamp_ms";
GoogleString GetAprErrorString(apr_status_t status) {
char error_str[1024];
@@ -166,7 +175,7 @@
return str_url_;
}
-void SerfFetch::Cancel() {
+void SerfFetch::Cancel(CancelCause cause) {
if (connection_ != NULL) {
// We can get here either because we're canceling the connection ourselves
// or because Serf detected an error.
@@ -182,18 +191,20 @@
connection_ = NULL;
}
- CallCallback(false);
+ CallCallback(cause == CancelCause::kClientDecision ?
+ SerfCompletionResult::kClientCancel :
+ SerfCompletionResult::kFailure);
}
-void SerfFetch::CallCallback(bool success) {
+void SerfFetch::CallCallback(SerfCompletionResult result) {
if (ssl_error_message_ != NULL) {
- success = false;
+ result = SerfCompletionResult::kFailure;
}
if (async_fetch_ != NULL) {
fetch_end_ms_ = timer_->NowMs();
fetcher_->ReportCompletedFetchStats(this);
- CallbackDone(success);
+ CallbackDone(result);
fetcher_->FetchComplete(this);
} else if (ssl_error_message_ == NULL) {
LOG(FATAL) << "BUG: Serf callback called more than once on same fetch "
@@ -202,10 +213,10 @@
}
}
-void SerfFetch::CallbackDone(bool success) {
+void SerfFetch::CallbackDone(SerfCompletionResult result) {
// fetcher_==NULL if Start is called during shutdown.
if (fetcher_ != NULL) {
- if (!success) {
+ if (result == SerfCompletionResult::kFailure) {
fetcher_->failure_count_->Add(1);
}
if (fetcher_->track_original_content_length() &&
@@ -214,8 +225,13 @@
async_fetch_->extra_response_headers()->SetOriginalContentLength(
bytes_received_);
}
+
+ if (async_fetch_ != nullptr) {
+ fetcher_->ReportFetchSuccessStats(
+ result, async_fetch_->response_headers(), this);
+ }
}
- async_fetch_->Done(success);
+ async_fetch_->Done(result == SerfCompletionResult::kSuccess);
// We should always NULL the async_fetch_ out after calling otherwise we
// could get weird double calling errors.
async_fetch_ = NULL;
@@ -226,7 +242,7 @@
serf_connection_is_in_error_state(connection_)) {
message_handler_->Message(
kInfo, "Serf cleanup for error'd fetch of: %s", DebugInfo().c_str());
- Cancel();
+ Cancel(CancelCause::kSerfError);
}
}
@@ -434,7 +450,7 @@
// check async_fetch before CallCallback.
if ((ssl_error_message_ != NULL) && (async_fetch_ != NULL)) {
fetcher_->cert_errors_->Add(1);
- CallCallback(false); // sets async_fetch_ to null.
+ CallCallback(SerfCompletionResult::kFailure); // sets async_fetch_ to null.
}
// TODO(jmarantz): I think the design of this system indicates
@@ -458,7 +474,7 @@
message_handler_->Message(
kInfo, "serf HandleResponse called with NULL response for %s",
DebugInfo().c_str());
- CallCallback(false);
+ CallCallback(SerfCompletionResult::kFailure);
return APR_EGENERAL;
}
@@ -506,7 +522,10 @@
}
bool successful_completion =
APR_STATUS_IS_EOF(status) && parser_.headers_complete();
- CallCallback(successful_completion); // Zeros async_fetch_.
+ // Zeros async_fetch_.
+ CallCallback(successful_completion ?
+ SerfCompletionResult::kSuccess :
+ SerfCompletionResult::kFailure);
}
return status;
}
@@ -1077,6 +1096,9 @@
failure_count_(NULL),
cert_errors_(NULL),
read_calls_count_(NULL),
+ ultimate_success_(NULL),
+ ultimate_failure_(NULL),
+ last_check_timestamp_ms_(NULL),
timeout_ms_(timeout_ms),
shutdown_(false),
list_outstanding_urls_on_error_(false),
@@ -1097,6 +1119,12 @@
cert_errors_ = statistics->GetVariable(SerfStats::kSerfFetchCertErrors);
// Using FindVariable for this one since it's only set in debug builds.
read_calls_count_ = statistics->FindVariable(SerfStats::kSerfFetchReadCalls);
+ ultimate_success_ =
+ statistics->GetVariable(SerfStats::kSerfFetchUltimateSuccess);
+ ultimate_failure_ =
+ statistics->GetVariable(SerfStats::kSerfFetchUltimateFailure);
+ last_check_timestamp_ms_ =
+ statistics->GetUpDownCounter(SerfStats::kSerfFetchLastCheckTimestampMs);
Init(pool, proxy);
threaded_fetcher_ = new SerfThreadedFetcher(this, proxy);
}
@@ -1118,6 +1146,9 @@
failure_count_(parent->failure_count_),
cert_errors_(parent->cert_errors_),
read_calls_count_(parent->read_calls_count_),
+ ultimate_success_(parent->ultimate_success_),
+ ultimate_failure_(parent->ultimate_failure_),
+ last_check_timestamp_ms_(parent->last_check_timestamp_ms_),
timeout_ms_(parent->timeout_ms()),
shutdown_(false),
list_outstanding_urls_on_error_(parent->list_outstanding_urls_on_error_),
@@ -1189,7 +1220,7 @@
// trouble, we simply ask for the oldest element, knowing it will go away.
SerfFetch* fetch = active_fetches_.oldest();
LOG(WARNING) << "Aborting fetch of " << fetch->DebugInfo();
- fetch->Cancel();
+ fetch->Cancel(SerfFetch::CancelCause::kClientDecision);
++num_canceled;
}
@@ -1209,7 +1240,9 @@
fetch->DebugInfo().c_str());
active_fetches_.Remove(fetch);
active_count_->Add(-1);
- fetch->CallbackDone(false);
+ fetch->CallbackDone(shutdown_ ?
+ SerfCompletionResult::kClientCancel :
+ SerfCompletionResult::kFailure);
delete fetch;
}
return started;
@@ -1270,7 +1303,7 @@
if (timeout_count_ != NULL) {
timeout_count_->Add(1);
}
- fetch->Cancel();
+ fetch->Cancel(SerfFetch::CancelCause::kFetchTimeout);
}
}
bool success = ((status == APR_SUCCESS) || APR_STATUS_IS_TIMEUP(status));
@@ -1330,7 +1363,7 @@
completed_fetches_.Add(fetch);
}
-void SerfUrlAsyncFetcher::ReportCompletedFetchStats(SerfFetch* fetch) {
+void SerfUrlAsyncFetcher::ReportCompletedFetchStats(const SerfFetch* fetch) {
if (time_duration_ms_) {
time_duration_ms_->Add(fetch->TimeDuration());
}
@@ -1342,6 +1375,45 @@
}
}
+void SerfUrlAsyncFetcher::ReportFetchSuccessStats(
+ SerfCompletionResult result,
+ const ResponseHeaders* headers,
+ const SerfFetch* fetch) {
+ if (result != SerfCompletionResult::kClientCancel) {
+ if (result == SerfCompletionResult::kSuccess &&
+ !headers->IsErrorStatus()) {
+ ultimate_success_->Add(1);
+ } else {
+ ultimate_failure_->Add(1);
+ }
+
+ // We clear "failures" first, read it last, so if we get an
+ // interleaving, failures will be 0, which of course won't
+ // issue a warning.
+ int64 last_check_ms = last_check_timestamp_ms_->Get();
+ int64 success = ultimate_success_->Get();
+ int64 failure = ultimate_failure_->Get();
+
+ int64 now_ms = timer_->NowMs();
+ if (now_ms > (last_check_ms + kReliabilityCheckPeriodMs)) {
+ ultimate_failure_->Clear();
+ ultimate_success_->Clear();
+ last_check_timestamp_ms_->Set(now_ms);
+
+ int64 total = success + failure;
+ if (total >= kReliabilityCheckMinFetches &&
+ (double(success) / total) < 0.5) {
+ message_handler_->Message(
+ kError, "PageSpeed Serf fetch failure rate extremely high; "
+ "only %s of %s recent fetches fully successful; is fetching "
+ "working?",
+ Integer64ToString(success).c_str(),
+ Integer64ToString(total).c_str());
+ }
+ }
+ }
+}
+
bool SerfUrlAsyncFetcher::AnyPendingFetches() {
ScopedMutex lock(mutex_);
return !active_fetches_.empty();
@@ -1421,6 +1493,9 @@
#ifndef NDEBUG
statistics->AddVariable(SerfStats::kSerfFetchReadCalls);
#endif
+ statistics->AddVariable(SerfStats::kSerfFetchUltimateSuccess);
+ statistics->AddVariable(SerfStats::kSerfFetchUltimateFailure);
+ statistics->AddUpDownCounter(SerfStats::kSerfFetchLastCheckTimestampMs);
}
void SerfUrlAsyncFetcher::set_list_outstanding_urls_on_error(bool x) {
diff --git a/pagespeed/system/serf_url_async_fetcher.h b/pagespeed/system/serf_url_async_fetcher.h
index 1da1903..0790f6f 100644
--- a/pagespeed/system/serf_url_async_fetcher.h
+++ b/pagespeed/system/serf_url_async_fetcher.h
@@ -74,6 +74,24 @@
static const char kSerfFetchFailureCount[];
static const char kSerfFetchCertErrors[];
static const char kSerfFetchReadCalls[];
+
+ // A fetch that finished with a 2xx or a 3xx code --- and not just a
+ // mechanically successful one that's a 4xx or such.
+ static const char kSerfFetchUltimateSuccess[];
+
+ // A failure or an error status. Doesn't include fetches dropped due to
+ // process exit and the like.
+ static const char kSerfFetchUltimateFailure[];
+
+ // When we last checked the ultimate failure/success numbers for a
+ // possible concern.
+ static const char kSerfFetchLastCheckTimestampMs[];
+};
+
+enum class SerfCompletionResult {
+ kClientCancel,
+ kSuccess,
+ kFailure
};
// Identifies the set of HTML keywords. This is used in error messages emitted
@@ -128,7 +146,13 @@
void FetchComplete(SerfFetch* fetch);
// Update the statistics object with results of the (completed) fetch.
- void ReportCompletedFetchStats(SerfFetch* fetch);
+ void ReportCompletedFetchStats(const SerfFetch* fetch);
+
+ // Updates states used for success/failure monitoring.
+ void ReportFetchSuccessStats(SerfCompletionResult result,
+ const ResponseHeaders* headers,
+ const SerfFetch* fetch);
+
apr_pool_t* pool() const { return pool_; }
@@ -261,6 +285,9 @@
Variable* failure_count_;
Variable* cert_errors_;
Variable* read_calls_count_; // Non-NULL only on debug builds.
+ Variable* ultimate_success_;
+ Variable* ultimate_failure_;
+ UpDownCounter* last_check_timestamp_ms_;
const int64 timeout_ms_;
bool shutdown_ GUARDED_BY(mutex_);
bool list_outstanding_urls_on_error_;
@@ -276,6 +303,12 @@
// TODO(lsong): Move this to a separate file. Necessary?
class SerfFetch : public PoolElement<SerfFetch> {
public:
+ enum class CancelCause {
+ kClientDecision,
+ kSerfError,
+ kFetchTimeout,
+ };
+
// TODO(lsong): make use of request_headers.
SerfFetch(const GoogleString& url,
AsyncFetch* async_fetch,
@@ -290,7 +323,7 @@
GoogleString DebugInfo();
// This must be called while holding SerfUrlAsyncFetcher's mutex_.
- void Cancel();
+ void Cancel(CancelCause cause);
// Calls the callback supplied by the user. This needs to happen
// exactly once. In some error cases it appears that Serf calls
@@ -300,8 +333,8 @@
//
// Note that when there are SSL error messages, we immediately call
// CallCallback, which is robust against duplicate calls in that case.
- void CallCallback(bool success);
- void CallbackDone(bool success);
+ void CallCallback(SerfCompletionResult result);
+ void CallbackDone(SerfCompletionResult result);
// If last poll of this fetch's connection resulted in an error, clean it up.
// Must be called after serf_context_run, with fetcher's mutex_ held.
diff --git a/pagespeed/system/serf_url_async_fetcher_test.cc b/pagespeed/system/serf_url_async_fetcher_test.cc
index f8658f0..71f5ceb 100644
--- a/pagespeed/system/serf_url_async_fetcher_test.cc
+++ b/pagespeed/system/serf_url_async_fetcher_test.cc
@@ -213,6 +213,10 @@
serf_url_async_fetcher_->SetSslCertificatesFile(ssl_cert_file);
}
#endif
+ // Set initial timestamp so we don't roll-over monitoring stats right after
+ // start.
+ statistics_->GetUpDownCounter(SerfStats::kSerfFetchLastCheckTimestampMs)
+ ->Set(timer_->NowMs());
}
virtual void TearDown() {
@@ -220,7 +224,10 @@
serf_url_async_fetcher_.reset(NULL);
timer_.reset(NULL);
STLDeleteElements(&fetches_);
- apr_pool_destroy(pool_);
+ if (pool_ != nullptr) {
+ apr_pool_destroy(pool_);
+ pool_ = nullptr;
+ }
}
// Adds a new URL & expected response to the url/response structure, returning
@@ -301,6 +308,13 @@
}
}
+ void ValidateMonitoringStats(int64 expect_success, int64 expect_failure) {
+ EXPECT_EQ(expect_success, statistics_->GetVariable(
+ SerfStats::kSerfFetchUltimateSuccess)->Get());
+ EXPECT_EQ(expect_failure, statistics_->GetVariable(
+ SerfStats::kSerfFetchUltimateFailure)->Get());
+ }
+
// Valgrind will not allow the async-fetcher thread to run without a sleep.
void YieldToThread() {
usleep(1);
@@ -340,6 +354,7 @@
ASSERT_TRUE(fetches_[kConnectionRefused]->IsDone());
EXPECT_EQ(HttpStatus::kNotFound,
response_headers(kConnectionRefused)->status_code());
+ ValidateMonitoringStats(0, 1);
}
// Tests that a range of URLs (established with AddTestUrl) all fail
@@ -438,6 +453,7 @@
// We don't care about the exact size, which can change, just that response
// is non-trivial.
EXPECT_LT(7500, bytes_count);
+ ValidateMonitoringStats(1, 0);
}
// Tests that when the fetcher requests using a different request method,
@@ -454,6 +470,7 @@
contents(kModpagespeedSite).find(
"PURGE to /mod_pagespeed_example/index.html not supported.") !=
GoogleString::npos);
+ ValidateMonitoringStats(0, 1);
}
// Tests that when the fetcher requests gzipped data it gets it. Note
@@ -467,6 +484,7 @@
EXPECT_LT(static_cast<size_t>(0), contents(kModpagespeedSite).size());
EXPECT_EQ(200, response_headers(kModpagespeedSite)->status_code());
ASSERT_TRUE(response_headers(kModpagespeedSite)->IsGzipped());
+ ValidateMonitoringStats(1, 0);
GzipInflater inflater(GzipInflater::kGzip);
ASSERT_TRUE(inflater.Init());
@@ -503,6 +521,7 @@
// wget -q -O - http://www.modpagespeed.com/|wc -c --> 2232
EXPECT_LT(2000, bytes_count);
EXPECT_GT(5000, bytes_count);
+ ValidateMonitoringStats(1, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, FetchTwoURLs) {
@@ -519,10 +538,22 @@
// modpagespeed.com so we are not sensitive to favicon changes.
EXPECT_EQ(13988, bytes_count);
EXPECT_EQ(0, ActiveFetches());
+ ValidateMonitoringStats(2, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestCancelThreeThreaded) {
StartFetches(kModpagespeedSite, kGoogleLogo);
+ // Explicit shutdown, it's OK to call this twice.
+ TearDown();
+
+ // Some of the fetches may succeed in the tiny window above, but none should
+ // be considered failed due to the quick shutdown cancelling them.
+ EXPECT_LE(statistics_->GetVariable(
+ SerfStats::kSerfFetchUltimateSuccess)->Get(),
+ 3);
+ EXPECT_EQ(0,
+ statistics_->GetVariable(
+ SerfStats::kSerfFetchUltimateFailure)->Get());
}
TEST_F(SerfUrlAsyncFetcherTest, TestWaitThreeThreaded) {
@@ -534,6 +565,7 @@
fetcher_timeout_ms_, &message_handler_,
SerfUrlAsyncFetcher::kThreadedOnly);
EXPECT_EQ(0, ActiveFetches());
+ ValidateMonitoringStats(3, 0);
}
#if SERF_FLAKY_SLOW_THREADING_TESTS
@@ -586,6 +618,7 @@
ASSERT_EQ(3, completed) << "Async fetches times out before completing";
ValidateFetches(kModpagespeedSite, kGoogleLogo);
EXPECT_EQ(0, ActiveFetches());
+ ValidateMonitoringStats(3, 0);
}
#endif // SERF_FLAKY_SLOW_THREADING_TESTS
@@ -594,6 +627,7 @@
int done = WaitTillDone(kModpagespeedSite, kGoogleLogo);
EXPECT_EQ(3, done);
ValidateFetches(kModpagespeedSite, kGoogleLogo);
+ ValidateMonitoringStats(3, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestTimeout) {
@@ -626,10 +660,12 @@
TestFetch(kNoContent, kNoContent);
EXPECT_EQ(HttpStatus::kNoContent,
response_headers(kNoContent)->status_code());
+ ValidateMonitoringStats(1, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsFailsByDefault) {
TestHttpsFails(https_favicon_url_);
+ ValidateMonitoringStats(0, 1);
}
#if SERF_HTTPS_FETCHING
@@ -638,12 +674,14 @@
serf_url_async_fetcher_->SetHttpsOptions("enable");
EXPECT_TRUE(serf_url_async_fetcher_->SupportsHttps());
TestHttpsFails(https_favicon_url_);
+ ValidateMonitoringStats(0, 1);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsSucceedsForGoogleCom) {
serf_url_async_fetcher_->SetHttpsOptions("enable");
EXPECT_TRUE(serf_url_async_fetcher_->SupportsHttps());
TestHttpsSucceeds("https://www.google.com/intl/en/about/", "<!DOCTYPE html>");
+ ValidateMonitoringStats(1, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsWithExplicitHost) {
@@ -660,6 +698,7 @@
request_headers(index)->Add(HttpAttributes::kHost, original_url.Host());
StartFetches(index, index);
ExpectHttpsSucceeds(index);
+ ValidateMonitoringStats(1, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsFailsWithIncorrectHost) {
@@ -667,6 +706,7 @@
int index = AddTestUrl("https://www.google.com", "");
request_headers(index)->Add(HttpAttributes::kHost, "www.example.com");
TestHttpsFails(index, index);
+ ValidateMonitoringStats(0, 1);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsWithExplicitHostPort) {
@@ -682,6 +722,7 @@
StrCat(original_url.Host(), ":443"));
StartFetches(index, index);
ExpectHttpsSucceeds(index);
+ ValidateMonitoringStats(1, 0);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsFailsForGoogleComWithBogusCertDir) {
@@ -689,12 +730,14 @@
serf_url_async_fetcher_->SetSslCertificatesDir(GTestTempDir());
serf_url_async_fetcher_->SetSslCertificatesFile("");
TestHttpsFails("https://www.google.com/intl/en/about/");
+ ValidateMonitoringStats(0, 1);
}
TEST_F(SerfUrlAsyncFetcherTest, TestHttpsSucceedsWhenEnabled) {
serf_url_async_fetcher_->SetHttpsOptions("enable,allow_self_signed");
EXPECT_TRUE(serf_url_async_fetcher_->SupportsHttps());
TestHttpsSucceeds(https_favicon_url_, favicon_head_);
+ ValidateMonitoringStats(1, 0);
}
#else
@@ -702,6 +745,7 @@
serf_url_async_fetcher_->SetHttpsOptions("enable"); // ignored
EXPECT_FALSE(serf_url_async_fetcher_->SupportsHttps());
TestHttpsFails(https_favicon_url_);
+ ValidateMonitoringStats(0, 1);
}
#endif
@@ -766,6 +810,7 @@
GoogleString msg = StrCat(urls_[kConnectionRefused],
" (connecting to:127.0.0.1:1023)");
EXPECT_TRUE(text.find(msg) != GoogleString::npos) << text;
+ ValidateMonitoringStats(0, 1);
}
// Test that the X-Original-Content-Length header is properly set
@@ -847,7 +892,7 @@
virtual void TearDown() {
async_fetch_->response_headers()->set_status_code(200);
- serf_fetch_->CallbackDone(true /* success */);
+ serf_fetch_->CallbackDone(SerfCompletionResult::kSuccess);
// Fetch must be deleted before fetcher because it has a child pool.
serf_fetch_.reset(NULL);
SerfUrlAsyncFetcherTest::TearDown();
@@ -968,6 +1013,7 @@
ASSERT_EQ(WaitTillDone(index, index), 1);
ASSERT_TRUE(fetches_[index]->IsDone());
EXPECT_EQ(HttpStatus::kNotFound, response_headers(index)->status_code());
+ ValidateMonitoringStats(0, 1);
}
class SerfUrlAsyncFetcherTestFakeWebServer : public SerfUrlAsyncFetcherTest {
