fixed peformance test (#39)
diff --git a/bin/performance-test b/bin/performance-test
index b606ad3..7fa1e68 100755
--- a/bin/performance-test
+++ b/bin/performance-test
@@ -74,9 +74,9 @@
echo "Running test $test_class"
pt_tmp=$(mktemp -d -t accumulo_pt_XXXXXXX)
setup_accumulo
- get_config_file accumulo-site.xml "$pt_tmp"
+ get_config_file accumulo.properties "$pt_tmp"
CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.MergeSiteConfig "$test_class" "$pt_tmp"
- put_config_file "$pt_tmp/accumulo-site.xml"
+ put_config_file "$pt_tmp/accumulo.properties"
put_server_code "$at_home/core/target/accumulo-testing-core-$at_version.jar"
start_accumulo
get_config_file accumulo-client.properties "$pt_tmp"
diff --git a/conf/cluster-control.sh.uno b/conf/cluster-control.sh.uno
index 2b9abdb..678ee35 100644
--- a/conf/cluster-control.sh.uno
+++ b/conf/cluster-control.sh.uno
@@ -30,21 +30,33 @@
ACCUMULO)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $ACCUMULO_VERSION
)
;;
HADOOP)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $HADOOP_VERSION
)
;;
ZOOKEEPER)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $ZOOKEEPER_VERSION
)
;;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
index 499be09..375c619 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
@@ -28,7 +28,6 @@
import java.util.zip.CRC32;
import java.util.zip.Checksum;
-import com.google.common.base.Preconditions;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
@@ -36,14 +35,16 @@
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.trace.CountSampler;
import org.apache.accumulo.core.trace.Trace;
+import org.apache.accumulo.core.trace.TraceSamplers;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
public class ContinuousIngest {
private static final Logger log = LoggerFactory.getLogger(ContinuousIngest.class);
@@ -129,7 +130,7 @@
}
BatchWriter bw = client.createBatchWriter(tableName);
- bw = Trace.wrapAll(bw, new CountSampler(1024));
+ bw = Trace.wrapAll(bw, TraceSamplers.countSampler(1024));
Random r = new Random();
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
index 8c4c14d..56b7aa0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
@@ -36,12 +36,12 @@
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.server.cli.ClientOnDefaultTable;
import org.apache.hadoop.io.Text;
import com.beust.jcommander.Parameter;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
index e5617c4..bc1b305 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
@@ -18,10 +18,10 @@
import java.io.IOException;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.server.cli.ClientOnRequiredTable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
index 04ef98f..d957286 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
@@ -19,7 +19,7 @@
public interface PerformanceTest {
- SystemConfiguration getConfiguration();
+ SystemConfiguration getSystemConfig();
Report runTest(Environment env) throws Exception;
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
index 60f02e5..c626a01 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
@@ -31,7 +31,7 @@
return this;
}
- public Map<String,String> getAccumuloSite() {
+ public Map<String,String> getAccumuloConfig() {
return accumuloSite;
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
index 2f79a88..aeacf20 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
@@ -17,36 +17,32 @@
package org.apache.accumulo.testing.core.performance.impl;
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.Reader;
+import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Properties;
import org.apache.accumulo.testing.core.performance.PerformanceTest;
-import org.apache.hadoop.conf.Configuration;
public class MergeSiteConfig {
public static void main(String[] args) throws Exception {
String className = args[0];
- Path confFile = Paths.get(args[1], "accumulo-site.xml");
+ Path confFile = Paths.get(args[1], "accumulo.properties");
PerformanceTest perfTest = Class.forName(className).asSubclass(PerformanceTest.class).newInstance();
- Configuration conf = new Configuration(false);
- byte[] newConf;
+ Properties props = new Properties();
-
- try(BufferedInputStream in = new BufferedInputStream(Files.newInputStream(confFile))){
- conf.addResource(in);
- perfTest.getConfiguration().getAccumuloSite().forEach((k,v) -> conf.set(k, v));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- conf.writeXml(baos);
- baos.close();
- newConf = baos.toByteArray();
+ try(Reader in = Files.newBufferedReader(confFile)){
+ props.load(in);
}
+ perfTest.getSystemConfig().getAccumuloConfig().forEach((k,v) -> props.setProperty(k, v));
- Files.write(confFile, newConf);
+ try(Writer out = Files.newBufferedWriter(confFile)){
+ props.store(out, "Modified by performance test");
+ }
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
index 28d34bb..5655c3f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
@@ -59,7 +59,7 @@
private static final int NUM_ROWS = 100000;
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "1000");
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
index 994af06..8f1e2d0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
@@ -64,7 +64,7 @@
+ "a dedicated executor. If the scan prioritizer or dispatcher is not working properly, " + "then the short scans will be orders of magnitude slower.";
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "200");
@@ -85,6 +85,7 @@
Map<String,String> props = new HashMap<>();
props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "executor", "se1");
props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "heed_hints", "true");
+ props.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
env.getClient().tableOperations().create(tableName,
new NewTableConfiguration().setProperties(props));
@@ -100,7 +101,7 @@
TestExecutor<Long> longScans = startLongScans(env, tableName, stop);
LongSummaryStatistics shortStats1 = runShortScans(env, tableName, 50000);
- LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 500000);
+ LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 100000);
stop.set(true);
long t4 = System.currentTimeMillis();
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
index b098ba0..4f675f5 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
@@ -45,7 +45,7 @@
private static final int NUM_QUALS = 1;
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
return new SystemConfiguration();
}