Fix warnings and clean up pom.xml
POM updates:
* Update plugins
* Add m2e profile for ensuring compiler compliance and ignoring plugins
in Eclipse that Eclipse doesn't understand
* Require minimum java version of 1.8 and Maven 3.5
* Remove unneeded plugins (felix bundle and exec-maven-plugin)
Fix warnings in code:
* Use try-with-resources for TestEnv
* Use Double.parseDouble instead of new Double
* Compute iterable sizes from scanners without unused variables
* Suppress deprecation warnings where needed
* Update BulkPlusOne RW test to use new bulk import API
* Add missing case statement for GET_SUMMARIES in TableOp
* Remove unused variables
diff --git a/pom.xml b/pom.xml
index 79463d0..9f53552 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,15 +126,19 @@
<pluginManagement>
<plugins>
<plugin>
- <!-- Allows us to get the apache-ds bundle artifacts -->
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <version>3.0.1</version>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.1</version>
</plugin>
<plugin>
<groupId>com.github.ekryd.sortpom</groupId>
<artifactId>sortpom-maven-plugin</artifactId>
- <version>2.8.0</version>
+ <version>2.10.0</version>
<configuration>
<predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
<createBackupFile>false</createBackupFile>
@@ -149,7 +153,7 @@
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
- <version>2.8.1</version>
+ <version>2.10.0</version>
<configuration>
<configFile>${eclipseFormatterStyle}</configFile>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
@@ -171,7 +175,7 @@
<plugin>
<groupId>net.revelc.code</groupId>
<artifactId>impsort-maven-plugin</artifactId>
- <version>1.2.0</version>
+ <version>1.3.0</version>
<configuration>
<removeUnused>true</removeUnused>
<groups>java.,javax.,org.,com.</groups>
@@ -185,6 +189,30 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <!-- must be same id as in the apache parent pom, to override the version -->
+ <id>enforce-maven-version</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <phase>validate</phase>
+ <configuration>
+ <rules>
+ <requireMavenVersion>
+ <version>[3.5.0,)</version>
+ </requireMavenVersion>
+ <requireJavaVersion>
+ <version>[${maven.compiler.target},)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<optimize>true</optimize>
@@ -201,13 +229,6 @@
</configuration>
</plugin>
<plugin>
- <!-- Allows us to get the apache-ds bundle artifacts -->
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <extensions>true</extensions>
- <inherited>true</inherited>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
@@ -221,18 +242,10 @@
</executions>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
- <configuration>
- <cleanupDaemonThreads>false</cleanupDaemonThreads>
- </configuration>
- </plugin>
- <plugin>
<!-- This was added to ensure project only uses public API -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>3.0.0</version>
+ <version>3.1.0</version>
<configuration>
<configLocation>contrib/checkstyle.xml</configLocation>
</configuration>
@@ -240,7 +253,7 @@
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
- <version>8.18</version>
+ <version>8.23</version>
</dependency>
</dependencies>
<executions>
@@ -351,6 +364,48 @@
</build>
</profile>
<profile>
+ <id>m2e</id>
+ <activation>
+ <property>
+ <name>m2e.version</name>
+ </property>
+ </activation>
+ <properties>
+ <maven.compiler.release>8</maven.compiler.release>
+ </properties>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ <versionRange>[0,)</versionRange>
+ <goals>
+ <goal>process</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
+ <profile>
<id>jdk-release-flag</id>
<activation>
<jdk>[9,)</jdk>
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
index c7d102c..99e2fdf 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
@@ -112,44 +112,45 @@
@Override
public int run(String[] args) throws Exception {
+ try (ContinuousEnv env = new ContinuousEnv(args)) {
- ContinuousEnv env = new ContinuousEnv(args);
+ Job job = Job.getInstance(getConf(),
+ this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ job.setInputFormatClass(AccumuloInputFormat.class);
- Job job = Job.getInstance(getConf(),
- this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
- job.setJarByClass(this.getClass());
- job.setInputFormatClass(AccumuloInputFormat.class);
+ int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
+ Set<Range> ranges = env.getAccumuloClient().tableOperations()
+ .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
- int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
- Set<Range> ranges = env.getAccumuloClient().tableOperations()
- .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
+ AccumuloInputFormat.configure().clientProperties(env.getClientProps())
+ .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
- AccumuloInputFormat.configure().clientProperties(env.getClientProps())
- .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
+ job.setMapperClass(CMapper.class);
+ job.setNumReduceTasks(0);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
- job.setMapperClass(CMapper.class);
- job.setNumReduceTasks(0);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+ .defaultTable(env.getAccumuloTableName()).store(job);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
- .defaultTable(env.getAccumuloTableName()).store(job);
+ Configuration conf = job.getConfiguration();
+ conf.setLong(MIN, env.getRowMin());
+ conf.setLong(MAX, env.getRowMax());
+ conf.setInt(MAX_CF, env.getMaxColF());
+ conf.setInt(MAX_CQ, env.getMaxColQ());
+ conf.set(CI_ID, UUID.randomUUID().toString());
+ conf.set("mapreduce.job.classloader", "true");
- Configuration conf = job.getConfiguration();
- conf.setLong(MIN, env.getRowMin());
- conf.setLong(MAX, env.getRowMax());
- conf.setInt(MAX_CF, env.getMaxColF());
- conf.setInt(MAX_CQ, env.getMaxColQ());
- conf.set(CI_ID, UUID.randomUUID().toString());
- conf.set("mapreduce.job.classloader", "true");
-
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
}
public static void main(String[] args) throws Exception {
- ContinuousEnv env = new ContinuousEnv(args);
- int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args);
- if (res != 0)
- System.exit(res);
+ try (ContinuousEnv env = new ContinuousEnv(args)) {
+ int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args);
+ if (res != 0)
+ System.exit(res);
+ }
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
index 14015e3..5db4350 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
@@ -140,69 +140,71 @@
@Override
public int run(String[] args) throws Exception {
- ContinuousEnv env = new ContinuousEnv(args);
+ try (ContinuousEnv env = new ContinuousEnv(args)) {
- String tableName = env.getAccumuloTableName();
+ String tableName = env.getAccumuloTableName();
- Job job = Job.getInstance(getConf(),
- this.getClass().getSimpleName() + "_" + tableName + "_" + System.currentTimeMillis());
- job.setJarByClass(this.getClass());
+ Job job = Job.getInstance(getConf(),
+ this.getClass().getSimpleName() + "_" + tableName + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
- job.setInputFormatClass(AccumuloInputFormat.class);
+ job.setInputFormatClass(AccumuloInputFormat.class);
- boolean scanOffline = Boolean
- .parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
- int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
- int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
- String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
+ boolean scanOffline = Boolean
+ .parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
+ int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
+ int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
+ String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
- Set<Range> ranges;
- String clone = "";
- AccumuloClient client = env.getAccumuloClient();
- String table;
+ Set<Range> ranges;
+ String clone = "";
+ AccumuloClient client = env.getAccumuloClient();
+ String table;
- if (scanOffline) {
- Random random = new Random();
- clone = tableName + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffL));
- client.tableOperations().clone(tableName, clone, true, new HashMap<>(), new HashSet<>());
- ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
- client.tableOperations().offline(clone);
- table = clone;
- } else {
- ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
- table = tableName;
+ if (scanOffline) {
+ Random random = new Random();
+ clone = tableName + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffL));
+ client.tableOperations().clone(tableName, clone, true, new HashMap<>(), new HashSet<>());
+ ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
+ client.tableOperations().offline(clone);
+ table = clone;
+ } else {
+ ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
+ table = tableName;
+ }
+
+ AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table)
+ .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job);
+
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
+
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(reducers);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
+ job.getConfiguration().set("mapreduce.job.classloader", "true");
+
+ TextOutputFormat.setOutputPath(job, new Path(outputDir));
+
+ job.waitForCompletion(true);
+
+ if (scanOffline) {
+ client.tableOperations().delete(clone);
+ }
+ return job.isSuccessful() ? 0 : 1;
}
-
- AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table)
- .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job);
-
- job.setMapperClass(CMapper.class);
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(VLongWritable.class);
-
- job.setReducerClass(CReducer.class);
- job.setNumReduceTasks(reducers);
-
- job.setOutputFormatClass(TextOutputFormat.class);
-
- job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
- job.getConfiguration().set("mapreduce.job.classloader", "true");
-
- TextOutputFormat.setOutputPath(job, new Path(outputDir));
-
- job.waitForCompletion(true);
-
- if (scanOffline) {
- client.tableOperations().delete(clone);
- }
- return job.isSuccessful() ? 0 : 1;
}
public static void main(String[] args) throws Exception {
- ContinuousEnv env = new ContinuousEnv(args);
+ try (ContinuousEnv env = new ContinuousEnv(args)) {
- int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args);
- if (res != 0)
- System.exit(res);
+ int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java b/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
index 2b3a2c3..2da2c62 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
@@ -41,6 +41,7 @@
String failures = null;
}
+ @SuppressWarnings("deprecation")
public static void main(String[] args)
throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
final FileSystem fs = FileSystem.get(new Configuration());
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
index 992a982..b783e8c 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
@@ -62,10 +62,9 @@
opts.parseArgs(VerifyIngest.class.getName(), args);
try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) {
if (opts.trace) {
- String name = VerifyIngest.class.getSimpleName();
/*
- * DistributedTrace.enable(); Trace.on(name); Trace.data("cmdLine",
- * Arrays.asList(args).toString());
+ * String name = VerifyIngest.class.getSimpleName(); DistributedTrace.enable();
+ * Trace.on(name); Trace.data("cmdLine", Arrays.asList(args).toString());
*/
}
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
index 4722799..45e24f0 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
@@ -58,44 +58,46 @@
@Override
public int run(String[] args) throws Exception {
- TestEnv env = new TestEnv(args);
- Job job = Job.getInstance(getConf());
- job.setJobName(this.getClass().getName());
- job.setJarByClass(this.getClass());
- job.setInputFormatClass(AccumuloInputFormat.class);
+ try (TestEnv env = new TestEnv(args)) {
+ Job job = Job.getInstance(getConf());
+ job.setJobName(this.getClass().getName());
+ job.setJarByClass(this.getClass());
+ job.setInputFormatClass(AccumuloInputFormat.class);
- Properties props = env.getTestProperties();
- String col = props.getProperty(TestProps.ROWHASH_COLUMN);
- int idx = col.indexOf(":");
- Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
- Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
- Collection<IteratorSetting.Column> cols = Collections.emptyList();
- if (cf.getLength() > 0)
- cols = Collections.singleton(new IteratorSetting.Column(cf, cq));
+ Properties props = env.getTestProperties();
+ String col = props.getProperty(TestProps.ROWHASH_COLUMN);
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ Collection<IteratorSetting.Column> cols = Collections.emptyList();
+ if (cf.getLength() > 0)
+ cols = Collections.singleton(new IteratorSetting.Column(cf, cq));
- String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
- String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
+ String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
+ String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
- AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
- .fetchColumns(cols).store(job);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
- .defaultTable(outputTable).createTables(true).store(job);
+ AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
+ .fetchColumns(cols).store(job);
+ AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
+ .defaultTable(outputTable).createTables(true).store(job);
- job.getConfiguration().set("mapreduce.job.classloader", "true");
- job.setMapperClass(HashDataMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Mutation.class);
+ job.getConfiguration().set("mapreduce.job.classloader", "true");
+ job.setMapperClass(HashDataMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
- job.setNumReduceTasks(0);
+ job.setNumReduceTasks(0);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
}
public static void main(String[] args) throws Exception {
- TestEnv env = new TestEnv(args);
- ToolRunner.run(env.getHadoopConfiguration(), new RowHash(), args);
+ try (TestEnv env = new TestEnv(args)) {
+ ToolRunner.run(env.getHadoopConfiguration(), new RowHash(), args);
+ }
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
index ee2b5d0..1961b93 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
@@ -352,42 +352,44 @@
}
public static void main(String[] args) throws Exception {
- TestEnv env = new TestEnv(args);
- ToolRunner.run(env.getHadoopConfiguration(), new TeraSortIngest(), args);
+ try (TestEnv env = new TestEnv(args)) {
+ ToolRunner.run(env.getHadoopConfiguration(), new TeraSortIngest(), args);
+ }
}
@Override
public int run(String[] args) throws Exception {
- TestEnv env = new TestEnv(args);
+ try (TestEnv env = new TestEnv(args)) {
- Job job = Job.getInstance(getConf());
- job.setJobName("TeraSortIngest");
- job.setJarByClass(this.getClass());
- job.setInputFormatClass(RangeInputFormat.class);
- job.setMapperClass(SortGenMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Mutation.class);
- job.setNumReduceTasks(0);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
+ Job job = Job.getInstance(getConf());
+ job.setJobName("TeraSortIngest");
+ job.setJarByClass(this.getClass());
+ job.setInputFormatClass(RangeInputFormat.class);
+ job.setMapperClass(SortGenMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
+ job.setNumReduceTasks(0);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
- Properties testProps = env.getTestProperties();
- String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
- Objects.requireNonNull(tableName);
+ Properties testProps = env.getTestProperties();
+ String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
+ Objects.requireNonNull(tableName);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
- .defaultTable(tableName).store(job);
+ AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+ .defaultTable(tableName).store(job);
- Configuration conf = job.getConfiguration();
- conf.set("mapreduce.job.classloader", "true");
- for (Object keyObj : testProps.keySet()) {
- String key = (String) keyObj;
- if (key.startsWith(TestProps.TERASORT)) {
- conf.set(key, testProps.getProperty(key));
+ Configuration conf = job.getConfiguration();
+ conf.set("mapreduce.job.classloader", "true");
+ for (Object keyObj : testProps.keySet()) {
+ String key = (String) keyObj;
+ if (key.startsWith(TestProps.TERASORT)) {
+ conf.set(key, testProps.getProperty(key));
+ }
}
- }
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
index 2d4f8b1..bf86a6a 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
@@ -22,10 +22,8 @@
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Condition;
import org.apache.accumulo.core.data.ConditionalMutation;
-import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.testing.performance.Environment;
import org.apache.accumulo.testing.performance.PerformanceTest;
@@ -35,6 +33,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.common.hash.Hashing;
public class ConditionalMutationsPT implements PerformanceTest {
@@ -85,7 +84,7 @@
}
reportBuilder.result("avgRate: 1-19",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"ConditionalMutationsTest: average rate (conditions/sec) to run sequence 1-19");
env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -96,7 +95,7 @@
}
reportBuilder.result("avgRate: 20-39",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"ConditionalMutationsTest: average rate (conditions/sec) to run sequence 20-39");
}
@@ -157,7 +156,7 @@
}
reportBuilder.result("avgRate: 1-19",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeConditionalMutationsTest: average rate (conditions/sec) to run sequence 1-19");
env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -168,7 +167,7 @@
}
reportBuilder.result("avgRate: 20-39",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeConditionalMutationsTest: average rate (conditions/sec) to run sequence 20-39");
}
@@ -241,7 +240,7 @@
}
reportBuilder.result("avgRate: 1-19",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeBatchScanAndWriteTest: average rate (conditions/sec) to write and scan sequence 1-19");
env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -252,7 +251,7 @@
}
reportBuilder.result("avgRate: 20-39",
- new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeBatchScanAndWriteTest: average rate (conditions/sec) to write and scan sequence 20-39 post flush");
}
@@ -282,10 +281,7 @@
bs.setRanges(ranges);
- int count = 0;
- for (Map.Entry<Key,Value> entry : bs) {
- count++;
- }
+ int count = Iterables.size(bs);
if (0 != count) {
throw new RuntimeException("count = " + count);
}
@@ -324,7 +320,7 @@
}
reportBuilder.result("avgRate1",
- new Double(new DecimalFormat("#0.00").format(rateSum / numTest)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
"SetBlockSizeTest: average rate in conditions/sec");
env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -335,7 +331,7 @@
}
reportBuilder.result("avgRate2",
- new Double(new DecimalFormat("#0.00").format(rateSum / numTest)),
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
"SetBlockSizeTest: average rate in conditions/sec post flush");
env.getClient().tableOperations().compact(tableName, null, null, true, true);
@@ -345,7 +341,8 @@
rateSum += setBlockSizeTime(cw, numRows, numCols);
}
- reportBuilder.result("avgRate3", new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+ reportBuilder.result("avgRate3",
+ Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"SetBlockSizeTest: average rate in conditions/sec post compaction");
reportBuilder.parameter("numRows", numRows, "SetBlockSizeTest: The number of rows");
reportBuilder.parameter("numCols", numCols, "SetBlockSizeTest: The number of columns");
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
index 20911cc..ae9b8c0 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
@@ -18,6 +18,7 @@
package org.apache.accumulo.testing.performance.tests;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LongSummaryStatistics;
import java.util.Map;
import java.util.Map.Entry;
@@ -161,7 +162,7 @@
while (!stop.get()) {
try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
scanner.setExecutionHints(hints);
- for (Entry<Key,Value> entry : scanner) {
+ for (Iterator<Entry<Key,Value>> iter = scanner.iterator(); iter.hasNext(); iter.next()) {
count++;
if (stop.get()) {
return count;
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
index 59571fb..8fe424f 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
@@ -182,7 +182,7 @@
scanner.addScanIterator(is);
// scanner.setExecutionHints(hints);
- for (Entry<Key,Value> entry : scanner) {
+ for (Iterator<Entry<Key,Value>> iter = scanner.iterator(); iter.hasNext(); iter.next()) {
count++;
if (stop.get()) {
return count;
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index 60821e2..55d81c1 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -30,7 +30,6 @@
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.randomwalk.State;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -58,9 +57,7 @@
final FileSystem fs = (FileSystem) state.get("fs");
final Path dir = new Path(fs.getUri() + "/tmp", "bulk_" + UUID.randomUUID().toString());
log.debug("Bulk loading from {}", dir);
- final Path fail = new Path(dir.toString() + "_fail");
final Random rand = state.getRandom();
- fs.mkdirs(fail);
final int parts = rand.nextInt(10) + 1;
TreeSet<Integer> startRows = new TreeSet<>();
@@ -96,15 +93,9 @@
}
writer.close();
}
- env.getAccumuloClient().tableOperations().importDirectory(Setup.getTableName(), dir.toString(),
- fail.toString(), true);
+ env.getAccumuloClient().tableOperations().importDirectory(dir.toString())
+ .to(Setup.getTableName()).tableTime(true);
fs.delete(dir, true);
- FileStatus[] failures = fs.listStatus(fail);
- if (failures != null && failures.length > 0) {
- state.set("bulkImportSuccess", "false");
- throw new Exception(failures.length + " failure files found importing files from " + dir);
- }
- fs.delete(fail, true);
log.debug("Finished bulk import, start rows " + printRows + " last row "
+ String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
}
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
index 1d1d0c1..4dd9736 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
@@ -126,8 +126,8 @@
bw.close();
}
- client.tableOperations().importDirectory(tableName, bulkDir, bulkDir + "_f",
- rand.nextBoolean());
+ client.tableOperations().importDirectory(bulkDir).to(tableName).tableTime(rand.nextBoolean())
+ .load();
log.debug("BulkImported to " + tableName);
} catch (TableNotFoundException e) {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
index b209d33..992fad6 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
@@ -52,6 +52,8 @@
return new Setting(property, min, max);
}
+ @SuppressWarnings("deprecation")
+ Property TSERV_READ_AHEAD_MAXCONCURRENT_deprecated = Property.TSERV_READ_AHEAD_MAXCONCURRENT;
// @formatter:off
Setting[] settings = {
s(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, 1, 10),
@@ -69,7 +71,7 @@
s(Property.TSERV_DEFAULT_BLOCKSIZE, 100000, 10000000L),
s(Property.TSERV_MAX_IDLE, 10000, 500 * 1000),
s(Property.TSERV_MAXMEM, 1000000, 3 * 1024 * 1024 * 1024L),
- s(Property.TSERV_READ_AHEAD_MAXCONCURRENT, 1, 25),
+ s(TSERV_READ_AHEAD_MAXCONCURRENT_deprecated, 1, 25),
s(Property.TSERV_MIGRATE_MAXCONCURRENT, 1, 10),
s(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, 10000, 1024 * 1024),
s(Property.TSERV_RECOVERY_MAX_CONCURRENT, 1, 100),
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java b/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
index b632249..0c8c474 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
@@ -45,7 +45,6 @@
private static final String authsMap = "authorizationsCountMap";
private static final String lastKey = "lastMutationKey";
- private static final String filesystem = "securityFileSystem";
public static String getTableName(State state) {
return state.getString(tableName);
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java b/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
index 7e217c3..bf58fdb 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
@@ -240,7 +240,7 @@
rFileWriter.append(k, new Value("Value".getBytes(UTF_8)));
rFileWriter.close();
try {
- tableOps.importDirectory(tableName, dir.toString(), fail.toString(), true);
+ tableOps.importDirectory(dir.toString()).to(tableName).tableTime(true).load();
} catch (TableNotFoundException tnfe) {
if (tableExists)
throw new AccumuloException("Table didn't exist when it should have: " + tableName);
@@ -293,6 +293,9 @@
props.setProperty("source", "table");
DropTable.dropTable(state, env, props);
break;
+
+ case GET_SUMMARIES:
+ throw new UnsupportedOperationException("GET_SUMMARIES not implemented");
}
}
}
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
index e1c377a..1916e57 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
@@ -142,6 +142,7 @@
fs.delete(new Path(rootDir), true);
}
+ @SuppressWarnings("deprecation")
private void bulkImport(FileSystem fs, State state, RandWalkEnv env, String tableName,
String rootDir, String prefix) throws Exception {
while (true) {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
index 5206417..91ff5eb 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
@@ -18,8 +18,6 @@
import java.util.Collection;
-import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
-import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.conf.Configured;
@@ -45,6 +43,7 @@
this.splits = splits;
}
+ @SuppressWarnings("deprecation")
@Override
public int run(String[] args) throws Exception {
Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
@@ -58,16 +57,20 @@
job.setInputFormatClass(SequenceFileInputFormat.class);
SequenceFileInputFormat.setInputPaths(job, seqFile);
- job.setPartitionerClass(KeyRangePartitioner.class);
- KeyRangePartitioner.setSplitFile(job, splitFile);
+ job.setPartitionerClass(
+ org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.class);
+ org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.setSplitFile(job,
+ splitFile);
job.setMapOutputKeyClass(Key.class);
job.setMapOutputValueClass(Value.class);
job.setNumReduceTasks(splits.size() + 1);
- job.setOutputFormatClass(AccumuloFileOutputFormat.class);
- AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
+ job.setOutputFormatClass(
+ org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.class);
+ org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.setOutputPath(job,
+ new Path(outputDir));
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;
