HTRACE-303. Add client-side htraceDropped log file to track dropped spans (cmccabe)
diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
index 6375688..8028cc6 100644
--- a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
+++ b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
@@ -41,7 +41,6 @@
Addr string `json:",omitempty"` // This gets filled in by the RPC layer.
DefaultTrid string `json:",omitempty"`
Spans []*Span
- ClientDropped uint64 `json:",omitempty"`
}
// Info returned by /server/version
@@ -97,20 +96,6 @@
// The total number of spans dropped by the server.
ServerDropped uint64
-
- // The total number of spans dropped by the client.
- //
- // This number is just an estimate and may be incorrect for many reasons.
- // If the client can't contact the server at all, then obviously the server
- // will never increment ClientDropped... even though spans are being
- // dropped. The client may also tell the server about some new spans it
- // has dropped, but then for some reason fail to get the acknowledgement
- // from the server. In that case, the client would re-send its client
- // dropped estimate and it would be double-counted by the server
- //
- // The intention here is to provide a rough estimate of how overloaded
- // htraced clients are, not to provide strongly consistent numbers.
- ClientDroppedEstimate uint64
}
// A map from network address strings to SpanMetrics structures.
@@ -145,10 +130,6 @@
// The total number of spans dropped by the server since the server started.
ServerDroppedSpans uint64
- // An estimate of the total number of spans dropped by the server since the server started.
- // See SpanMetrics#ClientDroppedEstimate
- ClientDroppedEstimate uint64
-
// The maximum latency of a writeSpans request, in milliseconds.
MaxWriteSpansLatencyMs uint32
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
index 9310e6e..1dab5c8 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
@@ -792,7 +792,7 @@
}
}
-func (ing *SpanIngestor) Close(clientDropped int, startTime time.Time) {
+func (ing *SpanIngestor) Close(startTime time.Time) {
for shardIdx := range(ing.batches) {
batch := ing.batches[shardIdx]
if len(batch.incoming) > 0 {
@@ -809,7 +809,7 @@
endTime := time.Now()
ing.store.msink.UpdateIngested(ing.addr, ing.totalIngested,
- ing.serverDropped, clientDropped, endTime.Sub(startTime))
+ ing.serverDropped, endTime.Sub(startTime))
}
func (store *dataStore) WriteSpans(shardIdx int, ispans []*IncomingSpan) {
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
index e6d1df7..d38c1b0 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
@@ -77,7 +77,7 @@
for idx := range spans {
ing.IngestSpan(&spans[idx])
}
- ing.Close(0, time.Now())
+ ing.Close(time.Now())
store.WrittenSpans.Waits(int64(len(spans)))
}
@@ -364,7 +364,7 @@
for n := 0; n < b.N; n++ {
ing.IngestSpan(allSpans[n])
}
- ing.Close(0, time.Now())
+ ing.Close(time.Now())
// Wait for all the spans to be written.
ht.Store.WrittenSpans.Waits(int64(b.N))
assertNumWrittenEquals(b, ht.Store.msink, b.N)
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
index a649420..0d569a0 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
@@ -270,7 +270,7 @@
for spanIdx := range req.Spans {
ing.IngestSpan(req.Spans[spanIdx])
}
- ing.Close(int(req.ClientDropped), startTime)
+ ing.Close(startTime)
return nil
}
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
index 5ce3339..7bf42fd 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
@@ -55,9 +55,6 @@
// The total number of spans dropped by the server.
ServerDropped uint64
- // The total number of spans dropped by the client (self-reported).
- ClientDroppedEstimate uint64
-
// Per-host Span Metrics
HostSpanMetrics common.SpanMetricsMap
@@ -80,13 +77,12 @@
// Update the total number of spans which were ingested, as well as other
// metrics that get updated during span ingest.
func (msink *MetricsSink) UpdateIngested(addr string, totalIngested int,
- serverDropped int, clientDroppedEstimate int, wsLatency time.Duration) {
+ serverDropped int, wsLatency time.Duration) {
msink.lock.Lock()
defer msink.lock.Unlock()
msink.IngestedSpans += uint64(totalIngested)
msink.ServerDropped += uint64(serverDropped)
- msink.ClientDroppedEstimate += uint64(clientDroppedEstimate)
- msink.updateSpanMetrics(addr, 0, serverDropped, clientDroppedEstimate)
+ msink.updateSpanMetrics(addr, 0, serverDropped)
wsLatencyMs := wsLatency.Nanoseconds() / 1000000
var wsLatency32 uint32
if wsLatencyMs > math.MaxUint32 {
@@ -99,7 +95,7 @@
// Update the per-host span metrics. Must be called with the lock held.
func (msink *MetricsSink) updateSpanMetrics(addr string, numWritten int,
- serverDropped int, clientDroppedEstimate int) {
+ serverDropped int) {
mtx, found := msink.HostSpanMetrics[addr]
if !found {
// Ensure that the per-host span metrics map doesn't grow too large.
@@ -117,7 +113,6 @@
}
mtx.Written += uint64(numWritten)
mtx.ServerDropped += uint64(serverDropped)
- mtx.ClientDroppedEstimate += uint64(clientDroppedEstimate)
}
// Update the total number of spans which were persisted to disk.
@@ -127,7 +122,7 @@
defer msink.lock.Unlock()
msink.WrittenSpans += uint64(totalWritten)
msink.ServerDropped += uint64(serverDropped)
- msink.updateSpanMetrics(addr, totalWritten, serverDropped, 0)
+ msink.updateSpanMetrics(addr, totalWritten, serverDropped)
}
// Read the server stats.
@@ -137,7 +132,6 @@
stats.IngestedSpans = msink.IngestedSpans
stats.WrittenSpans = msink.WrittenSpans
stats.ServerDroppedSpans = msink.ServerDropped
- stats.ClientDroppedEstimate = msink.ClientDroppedEstimate
stats.MaxWriteSpansLatencyMs = msink.wsLatencyCircBuf.Max()
stats.AverageWriteSpansLatencyMs = msink.wsLatencyCircBuf.Average()
stats.HostSpanMetrics = make(common.SpanMetricsMap)
@@ -145,7 +139,6 @@
stats.HostSpanMetrics[k] = &common.SpanMetrics {
Written: v.Written,
ServerDropped: v.ServerDropped,
- ClientDroppedEstimate: v.ClientDroppedEstimate,
}
}
}
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
index 0140dbb..b354a2c 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
@@ -56,7 +56,7 @@
for spanIdx := range testSpans {
ing.IngestSpan(testSpans[spanIdx])
}
- ing.Close(0, time.Now())
+ ing.Close(time.Now())
// Wait the spans to be created
ht.Store.WrittenSpans.Waits(NUM_TEST_SPANS)
// Set a reaper date that will remove all the spans except final one.
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
index 1b90bd4..c327cdd 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
@@ -257,7 +257,7 @@
for spanIdx := range msg.Spans {
ing.IngestSpan(msg.Spans[spanIdx])
}
- ing.Close(int(msg.ClientDropped), startTime)
+ ing.Close(startTime)
}
type queryHandler struct {
diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
index c81bbb7..9837e94 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
@@ -218,8 +218,6 @@
fmt.Fprintf(w, "Spans ingested\t%d\n", stats.IngestedSpans)
fmt.Fprintf(w, "Spans written\t%d\n", stats.WrittenSpans)
fmt.Fprintf(w, "Spans dropped by server\t%d\n", stats.ServerDroppedSpans)
- fmt.Fprintf(w, "Estimated spans dropped by clients\t%d\n",
- stats.ClientDroppedEstimate)
dur := time.Millisecond * time.Duration(stats.AverageWriteSpansLatencyMs)
fmt.Fprintf(w, "Average WriteSpan Latency\t%s\n", dur.String())
dur = time.Millisecond * time.Duration(stats.MaxWriteSpansLatencyMs)
@@ -247,8 +245,8 @@
sort.Sort(keys)
for k := range keys {
mtx := mtxMap[keys[k]]
- fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\tclient dropped estimate: %d\n",
- keys[k], mtx.Written, mtx.ServerDropped, mtx.ClientDroppedEstimate)
+ fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\n",
+ keys[k], mtx.Written, mtx.ServerDropped)
}
w.Flush()
return EXIT_SUCCESS
diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
index cdd176f..3206dd6 100644
--- a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
+++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
@@ -18,6 +18,7 @@
package org.apache.htrace.impl;
import java.io.IOException;
+import java.io.File;
import java.net.InetSocketAddress;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -144,6 +145,18 @@
"htraced.error.log.period.ms";
final static long ERROR_LOG_PERIOD_MS_DEFAULT = 30000L;
+ final static String DROPPED_SPANS_LOG_PATH_KEY =
+ "htraced.dropped.spans.log.path";
+
+ final static String DROPPED_SPANS_LOG_PATH_DEFAULT =
+ new File(System.getProperty("java.io.tmpdir", "/tmp"), "htraceDropped").
+ getAbsolutePath();
+
+ final static String DROPPED_SPANS_LOG_MAX_SIZE_KEY =
+ "htraced.dropped.spans.log.max.size";
+
+ final static long DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT = 1024L * 1024L;
+
@JsonProperty("ioTimeoutMs")
final int ioTimeoutMs;
@@ -180,6 +193,12 @@
@JsonProperty("endpoint")
final InetSocketAddress endpoint;
+ @JsonProperty("droppedSpansLogPath")
+ final String droppedSpansLogPath;
+
+ @JsonProperty("droppedSpansLogMaxSize")
+ final long droppedSpansLogMaxSize;
+
private static int getBoundedInt(final HTraceConfiguration conf,
String key, int defaultValue, int minValue, int maxValue) {
int val = conf.getInt(key, defaultValue);
@@ -341,6 +360,11 @@
throw new IOException("Error reading " + ADDRESS_KEY + ": " +
e.getMessage());
}
+ this.droppedSpansLogPath = conf.get(
+ DROPPED_SPANS_LOG_PATH_KEY, DROPPED_SPANS_LOG_PATH_DEFAULT);
+ this.droppedSpansLogMaxSize = getBoundedLong(conf,
+ DROPPED_SPANS_LOG_MAX_SIZE_KEY, DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT,
+ 0, Long.MAX_VALUE);
}
@Override
diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
index f5f493c..22b64f6 100644
--- a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
+++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
@@ -17,7 +17,20 @@
*/
package org.apache.htrace.impl;
+import static java.nio.file.StandardOpenOption.APPEND;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
+
import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+//import java.nio.file.attribute.FileAttribute;
+//import java.util.EnumSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -75,6 +88,8 @@
private long lastBufferClearedTimeMs = 0;
+ private long unbufferableSpans = 0;
+
static class FaultInjector {
static FaultInjector NO_OP = new FaultInjector();
public void handleContentLengthTrigger(int len) { }
@@ -140,10 +155,12 @@
}
long deltaMs = TimeUtil.deltaMs(startTimeMs, TimeUtil.nowMs());
if (deltaMs > conf.spanDropTimeoutMs) {
+ StringBuilder bld = new StringBuilder();
spanDropLog.error("Dropping a span after unsuccessfully " +
"attempting to add it for " + deltaMs + " ms. There is not " +
"enough buffer space. Please increase " + Conf.BUFFER_SIZE_KEY +
" or decrease the rate of spans being generated.");
+ unbufferableSpans++;
return;
} else if (LOG.isDebugEnabled()) {
LOG.debug("Unable to write span to buffer #" + activeBuf +
@@ -296,6 +313,16 @@
return;
}
int flushTries = 0;
+ if (unbufferableSpans > 0) {
+ try {
+ appendToDroppedSpansLog("Dropped " + unbufferableSpans +
+ " spans because of lack of local buffer space.\n");
+ } catch (IOException e) {
+ // Ignore. We already logged a message about the dropped spans
+ // earlier.
+ }
+ unbufferableSpans = 0;
+ }
while (true) {
Throwable exc;
try {
@@ -311,17 +338,22 @@
return;
}
int numSpans = flushBufManager.getNumberOfSpans();
- String excMessage = "Failed to flush " + numSpans + " htrace " +
- "spans to " + conf.endpointStr + " on try " + (flushTries + 1);
+ flushErrorLog.error("Failed to flush " + numSpans + " htrace " +
+ "spans to " + conf.endpointStr + " on try " + (flushTries + 1),
+ exc);
if (flushTries >= conf.flushRetryDelays.length) {
- excMessage += ". Discarding all spans.";
- }
- if (LOG.isDebugEnabled()) {
- LOG.error(excMessage, exc);
- } else {
- flushErrorLog.error(excMessage, exc);
- }
- if (flushTries >= conf.flushRetryDelays.length) {
+ StringBuilder bld = new StringBuilder();
+ bld.append("Failed to flush ").append(numSpans).
+ append(" spans to htraced at").append(conf.endpointStr).
+ append(" after ").append(flushTries).append(" tries: ").
+ append(exc.getMessage());
+ try {
+ appendToDroppedSpansLog(bld.toString());
+ } catch (IOException e) {
+ bld.append(". Failed to write to dropped spans log: ").
+ append(e.getMessage());
+ }
+ spanDropLog.error(bld.toString());
return;
}
int delayMs = conf.flushRetryDelays[flushTries];
@@ -330,4 +362,43 @@
}
}
}
+
+ void appendToDroppedSpansLog(String text) throws IOException {
+ // Is the dropped spans log is disabled?
+ if (conf.droppedSpansLogPath.isEmpty() ||
+ (conf.droppedSpansLogMaxSize == 0)) {
+ return;
+ }
+ FileLock lock = null;
+ ByteBuffer bb = ByteBuffer.wrap(
+ text.getBytes(StandardCharsets.UTF_8));
+ // FileChannel locking corresponds to advisory locking on UNIX. It will
+ // protect multiple processes from attempting to write to the same dropped
+ // spans log at once. However, within a single process, we need this
+ // synchronized block to ensure that multiple HTracedSpanReceiver objects
+ // don't try to write to the same log at once. (It is unusal to configure
+ // multiple HTracedSpanReceiver objects, but possible.)
+ synchronized(HTracedSpanReceiver.class) {
+ FileChannel channel = FileChannel.open(
+ Paths.get(conf.droppedSpansLogPath), APPEND, CREATE, WRITE);
+ try {
+ lock = channel.lock();
+ long size = channel.size();
+ if (size > conf.droppedSpansLogMaxSize) {
+ throw new IOException("Dropped spans log " +
+ conf.droppedSpansLogPath + " is already " + size +
+ " bytes; will not add to it.");
+ }
+ channel.write(bb);
+ } finally {
+ try {
+ if (lock != null) {
+ lock.release();
+ }
+ } finally {
+ channel.close();
+ }
+ }
+ }
+ }
}
diff --git a/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java
new file mode 100644
index 0000000..8947365
--- /dev/null
+++ b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.htrace.impl;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.htrace.core.HTraceConfiguration;
+import org.apache.htrace.core.MilliSpan;
+import org.apache.htrace.core.TracerId;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestDroppedSpans {
+ private static final Log LOG = LogFactory.getLog(TestDroppedSpans.class);
+
+ private static Path tempDir;
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ // Allow setting really small buffer sizes for testing purposes.
+ // We do not allow setting such small sizes in production.
+ Conf.BUFFER_SIZE_MIN = 0;
+
+ // Create a temporary directory to hold the dropped spans logs.
+ String tmp = System.getProperty("java.io.tmpdir", "/tmp");
+ File dir = new File(tmp,
+ "TestDroppedSpans." + UUID.randomUUID().toString());
+ Files.createDirectory(dir.toPath());
+ tempDir = dir.toPath();
+ }
+
+ @BeforeClass
+ public static void afterClass() throws IOException {
+ if (tempDir != null) {
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(tempDir)) {
+ for (final Iterator<Path> it = stream.iterator(); it.hasNext();) {
+ Files.delete(it.next());
+ }
+ }
+ Files.delete(tempDir);
+ tempDir = null;
+ }
+ }
+
+ /**
+ * Test that we can disable the dropped spans log.
+ */
+ @Test(timeout = 60000)
+ public void testDisableDroppedSpansLog() throws Exception {
+ HTraceConfiguration conf = HTraceConfiguration.fromMap(
+ new HashMap<String, String>() {{
+ put(Conf.ADDRESS_KEY, "127.0.0.1:8080");
+ put(TracerId.TRACER_ID_KEY, "testAppendToDroppedSpansLog");
+ put(Conf.DROPPED_SPANS_LOG_PATH_KEY, "/");
+ put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "0");
+ }});
+ HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf);
+ try {
+ rcvr.appendToDroppedSpansLog("this won't get written");
+ } finally {
+ rcvr.close();
+ }
+ }
+
+ /**
+ * Test that we can write to the dropped spans log.
+ */
+ @Test(timeout = 60000)
+ public void testWriteToDroppedSpansLog() throws Exception {
+ final String logPath = new File(
+ tempDir.toFile(), "testWriteToDroppedSpansLog").getAbsolutePath();
+ HTraceConfiguration conf = HTraceConfiguration.fromMap(
+ new HashMap<String, String>() {{
+ put(Conf.ADDRESS_KEY, "127.0.0.1:8080");
+ put(TracerId.TRACER_ID_KEY, "testWriteToDroppedSpansLog");
+ put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath);
+ put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78");
+ }});
+ HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf);
+ try {
+ final String LINE1 = "This is a test of the dropped spans log.";
+ rcvr.appendToDroppedSpansLog(LINE1 + "\n");
+ final String LINE2 = "These lines should appear in the log.";
+ rcvr.appendToDroppedSpansLog(LINE2 + "\n");
+ try {
+ rcvr.appendToDroppedSpansLog("This line won't be written because we're " +
+ "out of space.");
+ Assert.fail("expected append to fail because of lack of space");
+ } catch (IOException e) {
+ // ignore
+ }
+ List<String> lines =
+ Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8);
+ Assert.assertEquals(2, lines.size());
+ Assert.assertEquals(LINE1, lines.get(0));
+ Assert.assertEquals(LINE2, lines.get(1));
+ } finally {
+ rcvr.close();
+ }
+ }
+
+ /**
+ * Test that we write to the dropped spans log when htraced is unreachable.
+ */
+ @Test(timeout = 60000)
+ public void testSpansDroppedBecauseOfUnreachableHTraced() throws Exception {
+ final String logPath = new File(tempDir.toFile(),
+ "testSpansDroppedBecauseOfUnreachableHTraced").getAbsolutePath();
+ // Open a local socket. We know that nobody is listening on this socket, so
+ // all attempts to send to it will fail.
+ final ServerSocket serverSocket = new ServerSocket(0);
+ HTracedSpanReceiver rcvr = null;
+ try {
+ HTraceConfiguration conf = HTraceConfiguration.fromMap(
+ new HashMap<String, String>() {{
+ put(Conf.ADDRESS_KEY, "127.0.0.1:" + serverSocket.getLocalPort());
+ put(TracerId.TRACER_ID_KEY,
+ "testSpansDroppedBecauseOfUnreachableHTraced");
+ put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath);
+ put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78");
+ put(Conf.CONNECT_TIMEOUT_MS_KEY, "1");
+ put(Conf.IO_TIMEOUT_MS_KEY, "1");
+ put(Conf.FLUSH_RETRY_DELAYS_KEY, "1,1");
+ }});
+ rcvr = new HTracedSpanReceiver(conf);
+ rcvr.receiveSpan(new MilliSpan.Builder().
+ begin(123).end(456).description("FooBar").build());
+ HTracedSpanReceiver tmpRcvr = rcvr;
+ rcvr = null;
+ tmpRcvr.close();
+ List<String> lines =
+ Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8);
+ Assert.assertTrue(lines.size() >= 1);
+ Assert.assertTrue(lines.get(0).contains("Failed to flush "));
+ } finally {
+ serverSocket.close();
+ if (rcvr != null) {
+ rcvr.close();
+ }
+ }
+ }
+}
diff --git a/htrace-webapp/src/main/webapp/app/server_info_view.js b/htrace-webapp/src/main/webapp/app/server_info_view.js
index efb7545..43533d4 100644
--- a/htrace-webapp/src/main/webapp/app/server_info_view.js
+++ b/htrace-webapp/src/main/webapp/app/server_info_view.js
@@ -50,7 +50,6 @@
'<th>Remote</th>' +
'<th>Written</th>' +
'<th>ServerDropped</th>' +
- '<th>ClientDroppedEstimate</th>' +
'</tr>' +
'</thead>';
var remotes = [];
@@ -69,7 +68,6 @@
"<td>" + remote + "</td>" +
"<td>" + smtx.Written + "</td>" +
"<td>" + smtx.ServerDropped + "</td>" +
- "<td>" + smtx.ClientDroppedEstimate + "</td>" +
"</tr>";
}
out = out + '</table>';
diff --git a/htrace-webapp/src/main/webapp/app/server_stats.js b/htrace-webapp/src/main/webapp/app/server_stats.js
index 4cfea92..65d8c60 100644
--- a/htrace-webapp/src/main/webapp/app/server_stats.js
+++ b/htrace-webapp/src/main/webapp/app/server_stats.js
@@ -26,7 +26,6 @@
"IngestedSpans": "(unknown)",
"WrittenSpans": "(unknown)",
"ServerDroppedSpans": "(unknown)",
- "ClientDroppedSpans": "(unknown)",
"MaxWriteSpansLatencyMs": "(unknown)",
"AverageWriteSpansLatencyMs": "(unknown)"
},
diff --git a/htrace-webapp/src/main/webapp/index.html b/htrace-webapp/src/main/webapp/index.html
index 1e20ec0..edbfd26 100644
--- a/htrace-webapp/src/main/webapp/index.html
+++ b/htrace-webapp/src/main/webapp/index.html
@@ -92,10 +92,6 @@
<td><%= model.stats.get("ServerDroppedSpans") %></td>
</tr>
<tr>
- <td>Estimated Client Dropped Spans</td>
- <td><%= model.stats.get("ClientDroppedEstimate") %></td>
- </tr>
- <tr>
<td>Maximum WriteSpans Latency (ms)</td>
<td><%= model.stats.get("MaxWriteSpansLatencyMs") %></td>
</tr>
