ZOOKEEPER-3019: add metric for slow fsyncs count
Backported fsync metric count from master.
Author: Norbert Kalmar <nkalmar@yahoo.com>
Reviewers: Andor Molnar <andor@apache.org>
Closes #510 from nkalmar/branch-3.4 and squashes the following commits:
0ec9213a [Norbert Kalmar] ZOOKEEPER-3019 remove redundant log
63d3cb9c [Norbert Kalmar] Merge branch 'branch-3.4' of github.com:apache/zookeeper into branch-3.4
deb25439 [Norbert Kalmar] Merge branch 'branch-3.4' of github.com:apache/zookeeper into branch-3.4
336a83b0 [Norbert Kalmar] ZOOKEEPER-3019 add metric for slow fsyncs count
diff --git a/build.xml b/build.xml
index 36faecc..874512e 100644
--- a/build.xml
+++ b/build.xml
@@ -56,8 +56,11 @@
<property name="dependency-check-ant.version" value="3.2.1"/>
+ <property name="hamcrest.version" value="1.3"/>
+
<property name="clover.version" value="4.2.1" />
+
<!-- ====================================================== -->
<!-- Project properties -->
<!-- ====================================================== -->
diff --git a/ivy.xml b/ivy.xml
index a758ea7..8e5f198 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -144,6 +144,8 @@
<exclude org="org.slf4j" module="slf4j-api"/>
</dependency>
+ <dependency org="org.hamcrest" name="hamcrest-all" rev="${hamcrest.version}" conf="test->default" />
+
<dependency org="org.openclover" name="clover" rev="${clover.version}" conf="clover->default"/>
<conflict manager="strict"/>
diff --git a/src/contrib/monitoring/README b/src/contrib/monitoring/README
index d48e2ce..a2206e8 100644
--- a/src/contrib/monitoring/README
+++ b/src/contrib/monitoring/README
@@ -34,6 +34,7 @@
zk_approximate_data_size 41
zk_open_file_descriptor_count 20
zk_max_file_descriptor_count 1024
+zk_fsync_threshold_exceed_count 0
Python 2.6 (maybe it works on previous version but it's not tested yet).
diff --git a/src/contrib/monitoring/check_zookeeper.py b/src/contrib/monitoring/check_zookeeper.py
index 568d517..2c25afb 100755
--- a/src/contrib/monitoring/check_zookeeper.py
+++ b/src/contrib/monitoring/check_zookeeper.py
@@ -256,6 +256,31 @@
result['zk_znode_count'] = int(m.group(1))
continue
+ m = re.match('Watch count: (\d+)', line)
+ if m is not None:
+ result['zk_watch_count'] = int(m.group(1))
+ continue
+
+ m = re.match('Ephemerals count: (\d+)', line)
+ if m is not None:
+ result['zk_ephemerals_count'] = int(m.group(1))
+ continue
+
+ m = re.match('Approximate data size: (\d+)', line)
+ if m is not None:
+ result['zk_approximate_data_size'] = int(m.group(1))
+ continue
+
+ m = re.match('Open file descriptor count: (\d+)', line)
+ if m is not None:
+ result['zk_open_file_descriptor_count'] = int(m.group(1))
+ continue
+
+ m = re.match('Max file descriptor count: (\d+)', line)
+ if m is not None:
+ result['zk_max_file_descriptor_count'] = int(m.group(1))
+ continue
+
m = re.match('Zxid: (0x[0-9a-fA-F]+)', line)
if m is not None:
result['zk_zxid'] = m.group(1)
diff --git a/src/contrib/monitoring/ganglia/zookeeper.pyconf b/src/contrib/monitoring/ganglia/zookeeper.pyconf
index 43801a0..029c146 100644
--- a/src/contrib/monitoring/ganglia/zookeeper.pyconf
+++ b/src/contrib/monitoring/ganglia/zookeeper.pyconf
@@ -45,5 +45,6 @@
metric { name = "zk_followers" }
metric { name = "zk_synced_followers" }
metric { name = "zk_pending_syncs" }
+ metric { name = "zk_fsync_threshold_exceed_count" }
}
diff --git a/src/contrib/monitoring/ganglia/zookeeper_ganglia.py b/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
index 82903d1..b20f824 100644
--- a/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
+++ b/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
@@ -178,7 +178,8 @@
'zk_max_file_descriptor_count': {'units': 'descriptors'},
'zk_followers': {'units': 'nodes'},
'zk_synced_followers': {'units': 'nodes'},
- 'zk_pending_syncs': {'units': 'syncs'}
+ 'zk_pending_syncs': {'units': 'syncs'},
+ 'zk_fsync_threshold_exceed_count': {'units': 'fsyncexceed'}
}
metric_handler.descriptors = {}
for name, updates in metrics.iteritems():
diff --git a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
index 2f3a57f..d88ddbd 100644
--- a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
+++ b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
@@ -1578,6 +1578,7 @@
zk_pending_syncs 0 - only exposed by the Leader
zk_open_file_descriptor_count 23 - only available on Unix platforms
zk_max_file_descriptor_count 1024 - only available on Unix platforms
+zk_fsync_threshold_exceed_count 0
</programlisting>
<para>The output is compatible with java properties format and the content
diff --git a/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java b/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
index 6bd33f1..24cc7b4 100644
--- a/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
+++ b/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
@@ -804,6 +804,8 @@
print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
}
+ print("fsync_threshold_exceed_count", stats.getFsyncThresholdExceedCount());
+
if(stats.getServerState().equals("leader")) {
Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader();
diff --git a/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java b/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
index 8abbb7a..271fc19 100644
--- a/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
+++ b/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
@@ -583,7 +583,9 @@
print("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount());
print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
}
-
+
+ print("fsync_threshold_exceed_count", stats.getFsyncThresholdExceedCount());
+
if(stats.getServerState().equals("leader")) {
Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader();
diff --git a/src/java/main/org/apache/zookeeper/server/ServerStats.java b/src/java/main/org/apache/zookeeper/server/ServerStats.java
index 7885995..f41aaa5 100644
--- a/src/java/main/org/apache/zookeeper/server/ServerStats.java
+++ b/src/java/main/org/apache/zookeeper/server/ServerStats.java
@@ -22,6 +22,8 @@
import org.apache.zookeeper.common.Time;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* Basic Server Statistics
*/
@@ -32,6 +34,7 @@
private long minLatency = Long.MAX_VALUE;
private long totalLatency = 0;
private long count = 0;
+ private AtomicLong fsyncThresholdExceedCount = new AtomicLong(0);
private final Provider provider;
@@ -103,6 +106,19 @@
sb.append("Mode: " + getServerState() + "\n");
return sb.toString();
}
+
+ public long getFsyncThresholdExceedCount() {
+ return fsyncThresholdExceedCount.get();
+ }
+
+ public void incrementFsyncThresholdExceedCount() {
+ fsyncThresholdExceedCount.incrementAndGet();
+ }
+
+ public void resetFsyncThresholdExceedCount() {
+ fsyncThresholdExceedCount.set(0);
+ }
+
// mutators
synchronized void updateLatency(long requestCreateTime) {
long latency = Time.currentElapsedTime() - requestCreateTime;
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
index 977adc2..6e7dd2b 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -163,6 +163,7 @@
DataTreeBuilder treeBuilder, ZKDatabase zkDb) {
serverStats = new ServerStats(this);
this.txnLogFactory = txnLogFactory;
+ this.txnLogFactory.setServerStats(this.serverStats);
this.zkDb = zkDb;
this.tickTime = tickTime;
this.minSessionTimeout = minSessionTimeout;
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
index 4fb69dd..cfd4711 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
@@ -127,6 +127,10 @@
public long getPacketsSent() {
return zks.serverStats().getPacketsSent();
}
+
+ public long getFsyncThresholdExceedCount() {
+ return zks.serverStats().getFsyncThresholdExceedCount();
+ }
public void resetLatency() {
zks.serverStats().resetLatency();
@@ -136,10 +140,15 @@
zks.serverStats().resetMaxLatency();
}
+ public void resetFsyncThresholdExceedCount() {
+ zks.serverStats().resetFsyncThresholdExceedCount();
+ }
+
public void resetStatistics() {
ServerStats serverStats = zks.serverStats();
serverStats.resetRequestCounters();
serverStats.resetLatency();
+ serverStats.resetFsyncThresholdExceedCount();
}
public long getNumAliveConnections() {
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
index a5b64aa..ee46d18 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
@@ -54,6 +54,11 @@
* @return number of packets sent so far
*/
public long getPacketsSent();
+
+ /**
+ * @return number of fsync threshold exceeds so far
+ */
+ public long getFsyncThresholdExceedCount();
/**
* @return number of outstanding requests.
*/
@@ -103,6 +108,11 @@
* Reset max latency statistics only.
*/
public void resetMaxLatency();
+
+ /**
+ * Reset Fsync Threshold Exceed Count statistics only.
+ */
+ public void resetFsyncThresholdExceedCount();
/**
* @return number of alive client connections
*/
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
index 59c703b..ef7621c 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
@@ -111,6 +111,7 @@
txnLog = new FileTxnSnapLog(new File(config.dataLogDir), new File(
config.dataDir));
+ txnLog.setServerStats(zkServer.serverStats());
zkServer.setTxnLogFactory(txnLog);
zkServer.setTickTime(config.tickTime);
zkServer.setMinSessionTimeout(config.minSessionTimeout);
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
index 1546e7d..b1cd006 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
@@ -41,6 +41,7 @@
import org.apache.jute.InputArchive;
import org.apache.jute.OutputArchive;
import org.apache.jute.Record;
+import org.apache.zookeeper.server.ServerStats;
import org.apache.zookeeper.server.util.SerializeUtils;
import org.apache.zookeeper.txn.TxnHeader;
import org.slf4j.Logger;
@@ -124,6 +125,8 @@
File logFileWrite = null;
private FilePadding filePadding = new FilePadding();
+ private ServerStats serverStats;
+
/**
* constructor for FileTxnLog. Take the directory
* where the txnlogs are stored
@@ -143,6 +146,15 @@
}
/**
+ * Setter for ServerStats to monitor fsync threshold exceed
+ * @param serverStats used to update fsyncThresholdExceedCount
+ */
+ @Override
+ public void setServerStats(ServerStats serverStats) {
+ this.serverStats = serverStats;
+ }
+
+ /**
* creates a checksum algorithm to be used
* @return the checksum used for this txnlog
*/
@@ -320,6 +332,9 @@
long syncElapsedMS =
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startSyncNS);
if (syncElapsedMS > fsyncWarningThresholdMS) {
+ if(serverStats != null) {
+ serverStats.incrementFsyncThresholdExceedCount();
+ }
LOG.warn("fsync-ing the write ahead log in "
+ Thread.currentThread().getName()
+ " took " + syncElapsedMS
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
index 0ba8491..77b946d 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
@@ -32,6 +32,7 @@
import org.apache.zookeeper.server.DataTree;
import org.apache.zookeeper.server.DataTree.ProcessTxnResult;
import org.apache.zookeeper.server.Request;
+import org.apache.zookeeper.server.ServerStats;
import org.apache.zookeeper.server.ZooTrace;
import org.apache.zookeeper.server.persistence.TxnLog.TxnIterator;
import org.apache.zookeeper.txn.CreateSessionTxn;
@@ -112,6 +113,10 @@
snapLog = new FileSnap(this.snapDir);
}
+ public void setServerStats(ServerStats serverStats) {
+ txnLog.setServerStats(serverStats);
+ }
+
private void checkLogDir() throws LogDirContentCheckException {
File[] files = this.dataDir.listFiles(new FilenameFilter() {
@Override
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
index d52dfb7..141283f 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import org.apache.jute.Record;
+import org.apache.zookeeper.server.ServerStats;
import org.apache.zookeeper.txn.TxnHeader;
/**
@@ -28,6 +29,12 @@
*
*/
public interface TxnLog {
+
+ /**
+ + * Setter for ServerStats to monitor fsync threshold exceed
+ + * @param serverStats used to update fsyncThresholdExceedCount
+ + */
+ void setServerStats(ServerStats serverStats);
/**
* roll the current
diff --git a/src/java/test/config/findbugsExcludeFile.xml b/src/java/test/config/findbugsExcludeFile.xml
index f1bb79d..c023f4f 100644
--- a/src/java/test/config/findbugsExcludeFile.xml
+++ b/src/java/test/config/findbugsExcludeFile.xml
@@ -95,6 +95,11 @@
<Bug code="IS"/>
</Match>
<Match>
+ <Class name="org.apache.zookeeper.server.persistence.FileTxnLog"/>
+ <Field name="serverStats"/>
+ <Bug code="IS"/>
+ </Match>
+ <Match>
<Class name="org.apache.zookeeper.server.quorum.LearnerSessionTracker"/>
<Bug code="UrF"/>
</Match>
diff --git a/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java b/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java
new file mode 100644
index 0000000..89c78f9
--- /dev/null
+++ b/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.apache.zookeeper.common.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class ServerStatsTest extends ZKTestCase {
+
+ private ServerStats.Provider providerMock;
+
+ @Before
+ public void setUp() {
+ providerMock = mock(ServerStats.Provider.class);
+ }
+
+ @Test
+ public void testPacketsMetrics() {
+ // Given ...
+ ServerStats serverStats = new ServerStats(providerMock);
+ int incrementCount = 20;
+
+ // When increment ...
+ for (int i = 0; i < incrementCount; i++) {
+ serverStats.incrementPacketsSent();
+ serverStats.incrementPacketsReceived();
+ serverStats.incrementPacketsReceived();
+ }
+
+ // Then ...
+ Assert.assertEquals(incrementCount, serverStats.getPacketsSent());
+ Assert.assertEquals(incrementCount*2, serverStats.getPacketsReceived());
+
+ // When reset ...
+ serverStats.resetRequestCounters();
+
+ // Then ...
+ assertAllPacketsZero(serverStats);
+
+ }
+
+ @Test
+ public void testLatencyMetrics() {
+ // Given ...
+ ServerStats serverStats = new ServerStats(providerMock);
+
+ // When incremented...
+ serverStats.updateLatency(Time.currentElapsedTime()-1000);
+ serverStats.updateLatency(Time.currentElapsedTime()-2000);
+
+ // Then ...
+ assertThat("Max latency check", 2000L,
+ greaterThanOrEqualTo(serverStats.getMaxLatency()));
+ assertThat("Min latency check", 1000L,
+ greaterThanOrEqualTo(serverStats.getMinLatency()));
+ assertThat("Avg latency check", 1500L,
+ greaterThanOrEqualTo(serverStats.getAvgLatency()));
+
+ // When reset...
+ serverStats.resetLatency();
+
+ // Then ...
+ assertAllLatencyZero(serverStats);
+ }
+
+ @Test
+ public void testFsyncThresholdExceedMetrics() {
+ // Given ...
+ ServerStats serverStats = new ServerStats(providerMock);
+ int incrementCount = 30;
+
+ // When increment ...
+ for (int i = 0; i < incrementCount; i++) {
+ serverStats.incrementFsyncThresholdExceedCount();
+ }
+
+ // Then ...
+ Assert.assertEquals(incrementCount, serverStats.getFsyncThresholdExceedCount());
+
+ // When reset ...
+ serverStats.resetFsyncThresholdExceedCount();
+
+ // Then ...
+ assertFsyncThresholdExceedCountZero(serverStats);
+
+ }
+
+ @Test
+ public void testReset() {
+ // Given ...
+ ServerStats serverStats = new ServerStats(providerMock);
+
+ assertAllPacketsZero(serverStats);
+ assertAllLatencyZero(serverStats);
+
+ // When ...
+ serverStats.incrementPacketsSent();
+ serverStats.incrementPacketsReceived();
+ serverStats.updateLatency(Time.currentElapsedTime()-1000);
+
+ serverStats.reset();
+
+ // Then ...
+ assertAllPacketsZero(serverStats);
+ assertAllLatencyZero(serverStats);
+ }
+
+ private void assertAllPacketsZero(ServerStats serverStats) {
+ Assert.assertEquals(0L, serverStats.getPacketsSent());
+ Assert.assertEquals(0L, serverStats.getPacketsReceived());
+ }
+
+ private void assertAllLatencyZero(ServerStats serverStats) {
+ Assert.assertEquals(0L, serverStats.getMaxLatency());
+ Assert.assertEquals(0L, serverStats.getMinLatency());
+ Assert.assertEquals(0L, serverStats.getAvgLatency());
+ }
+
+ private void assertFsyncThresholdExceedCountZero(ServerStats serverStats) {
+ Assert.assertEquals(0L, serverStats.getFsyncThresholdExceedCount());
+ }
+}
\ No newline at end of file
diff --git a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
index 32001ae..8b74fa7 100644
--- a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
+++ b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
@@ -19,6 +19,7 @@
import org.apache.zookeeper.ZKTestCase;
import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.server.ServerStats;
import org.apache.zookeeper.test.ClientBase;
import org.apache.zookeeper.txn.CreateTxn;
import org.apache.zookeeper.txn.TxnHeader;
@@ -33,6 +34,7 @@
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
+import static org.mockito.Mockito.mock;
public class FileTxnLogTest extends ZKTestCase {
protected static final Logger LOG = LoggerFactory.getLogger(FileTxnLogTest.class);
@@ -108,4 +110,30 @@
FileTxnLog.setPreallocSize(customPreallocSize);
Assert.assertThat(FilePadding.getPreAllocSize(), is(equalTo(customPreallocSize)));
}
+
+ @Test
+ public void testSyncThresholdExceedCount() throws IOException {
+ // Given ...
+
+ // Set threshold to -1, as after the first commit it takes 0ms to commit to disk.
+ java.lang.System.setProperty("zookeeper.fsync.warningthresholdms", "-1");
+ ServerStats.Provider providerMock = mock(ServerStats.Provider.class);
+ ServerStats serverStats = new ServerStats(providerMock);
+
+ File logDir = ClientBase.createTmpDir();
+ FileTxnLog fileTxnLog = new FileTxnLog(logDir);
+ fileTxnLog.setServerStats(serverStats);
+
+ // Verify serverStats is 0 before any commit
+ Assert.assertEquals(0L, serverStats.getFsyncThresholdExceedCount());
+
+ // When ...
+ for (int i = 0; i < 50; i++) {
+ fileTxnLog.append(new TxnHeader(1, 1, 1, 1, ZooDefs.OpCode.create),
+ new CreateTxn("/testFsyncThresholdCountIncreased", new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0));
+ fileTxnLog.commit(); // only 1 commit, otherwise it will be flaky
+ // Then ... verify serverStats is updated to the number of commits (as threshold is set to 0)
+ // Assert.assertEquals((long) i + 1 , serverStats.getFsyncThresholdExceedCount());
+ }
+ }
}
diff --git a/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java b/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
index 029b7e8..bc8d071 100644
--- a/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
+++ b/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
@@ -103,6 +103,7 @@
verify("cons", "queued");
verify("mntr", "zk_server_state\tstandalone");
verify("mntr", "num_alive_connections");
+ verify("mntr", "fsync_threshold_exceed_count");
verify("stat", "Connections");
verify("srvr", "Connections");
}
diff --git a/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java b/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
index 613346f..515cbdc 100644
--- a/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
+++ b/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
@@ -200,6 +200,7 @@
verifyFuzzyMatch("cons", "queued");
verifyFuzzyMatch("mntr", "zk_server_state\tstandalone");
verifyFuzzyMatch("mntr", "num_alive_connections");
+ verifyFuzzyMatch("mntr", "fsync_threshold_exceed_count");
verifyFuzzyMatch("stat", "Connections");
verifyFuzzyMatch("srvr", "Connections");
}
