Add stats to new Compactable classes. Fixes #1930 (#1937)
* Also fix split stats in the monitor
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
index a6fa3b7..0cb87b6 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
@@ -222,7 +222,7 @@
majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
splitStdDev =
- stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
+ stddev(historical.splits.elapsed, historical.splits.num, historical.splits.sumDev);
TabletServerDetailInformation details = doDetails(tsStats.size());
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
index 76b5f6b..3469a8c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
@@ -83,7 +83,7 @@
Optional<Files> getFiles(CompactionServiceId service, CompactionKind kind);
void compact(CompactionServiceId service, CompactionJob job, RateLimiter readLimiter,
- RateLimiter writeLimiter);
+ RateLimiter writeLimiter, long queuedTime);
CompactionServiceId getConfiguredService(CompactionKind kind);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
index 21cf147..2346cd8 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
@@ -69,6 +69,7 @@
private Compactable compactable;
private CompactionServiceId csid;
private Consumer<Compactable> completionCallback;
+ private final long queuedTime;
public CompactionTask(CompactionJob job, Compactable compactable, CompactionServiceId csid,
Consumer<Compactable> completionCallback) {
@@ -77,6 +78,7 @@
this.csid = csid;
this.completionCallback = completionCallback;
queuedTask.add(this);
+ queuedTime = System.currentTimeMillis();
}
@Override
@@ -85,7 +87,7 @@
try {
if (status.compareAndSet(Status.QUEUED, Status.RUNNING)) {
queuedTask.remove(this);
- compactable.compact(csid, getJob(), readLimiter, writeLimiter);
+ compactable.compact(csid, getJob(), readLimiter, writeLimiter, queuedTime);
completionCallback.accept(compactable);
}
} catch (Exception e) {
@@ -134,7 +136,6 @@
return canceled;
}
-
}
private static CompactionJob getJob(Runnable r) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index a6ca747..d0d8669 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.tserver.tablet;
+import static org.apache.accumulo.tserver.TabletStatsKeeper.Operation.MAJOR;
+
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Collection;
@@ -539,7 +541,7 @@
@Override
public void compact(CompactionServiceId service, CompactionJob job, RateLimiter readLimiter,
- RateLimiter writeLimiter) {
+ RateLimiter writeLimiter, long queuedTime) {
Set<StoredTabletFile> jobFiles = job.getFiles().stream()
.map(cf -> ((CompactableFileImpl) cf).getStortedTabletFile()).collect(Collectors.toSet());
@@ -641,13 +643,17 @@
}
StoredTabletFile metaFile = null;
+ long startTime = System.currentTimeMillis();
+ // create an empty stats object to be populated by CompactableUtils.compact()
+ CompactionStats stats = new CompactionStats();
try {
TabletLogger.compacting(getExtent(), job, localCompactionCfg);
+ tablet.incrementStatusMajor();
metaFile = CompactableUtils.compact(tablet, job, jobFiles, compactionId, propogateDeletes,
localHelper, iters, new CompactionCheck(service, job.getKind(), checkCompactionId),
- readLimiter, writeLimiter);
+ readLimiter, writeLimiter, stats);
TabletLogger.compacted(getExtent(), job, metaFile);
@@ -679,6 +685,8 @@
selectedCompactionCompleted(job, jobFiles, metaFile);
else
selectFiles();
+
+ tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(), metaFile == null);
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
index 8589ad5..c4de249 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
@@ -532,7 +532,8 @@
static StoredTabletFile compact(Tablet tablet, CompactionJob job, Set<StoredTabletFile> jobFiles,
Long compactionId, boolean propogateDeletes, CompactableImpl.CompactionHelper helper,
List<IteratorSetting> iters, CompactionCheck compactionCheck, RateLimiter readLimiter,
- RateLimiter writeLimiter) throws IOException, CompactionCanceledException {
+ RateLimiter writeLimiter, CompactionStats stats)
+ throws IOException, CompactionCanceledException {
StoredTabletFile metaFile;
CompactionEnv cenv = new CompactionEnv() {
@Override
@@ -580,6 +581,8 @@
}
});
}
+ // mutate the empty stats to allow returning their values
+ stats.add(mcs);
metaFile = tablet.getDatafileManager().bringMajorCompactionOnline(compactFiles.keySet(),
compactTmpName, newFile, compactionId,
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 7605500..1a90b1c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2114,6 +2114,15 @@
queryBytes += numBytes;
}
+ public void updateTimer(Operation operation, long queued, long start, long count,
+ boolean failed) {
+ timer.updateTime(operation, queued, start, count, failed);
+ }
+
+ public void incrementStatusMajor() {
+ timer.incrementStatusMajor();
+ }
+
TabletServer getTabletServer() {
return tabletServer;
}
