Updates due to Accumulo 2.0 changes (#35)
- using accumulo-client.properties to create clients
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index 34fadb0..465e11b 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -128,7 +128,7 @@
datanode_log="${log_base}_datanode-agitator"
master_cmd="nohup ${libexec}/master-agitator.pl $AGTR_MASTER_KILL_SLEEP_TIME $AGTR_MASTER_RESTART_SLEEP_TIME"
tserver_cmd="nohup ${libexec}/tserver-agitator.pl $AGTR_TSERVER_KILL_SLEEP_TIME $AGTR_TSERVER_RESTART_SLEEP_TIME $AGTR_TSERVER_MIN_KILL $AGTR_TSERVER_MAX_KILL"
- datanode_cmd="nohup ${libexec}/datanode-agitator.pl $AGTR_DATANODE_KILL_SLEEP_TIME $AGTR_DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $AGTR_DATANODE_MIN_KILL $AGTR_DATANODE_MAX_KILL"
+ datanode_cmd="nohup ${libexec}/datanode-agitator.pl $AGTR_DATANODE_KILL_SLEEP_TIME $AGTR_DATANODE_RESTART_SLEEP_TIME $HADOOP_HOME $AGTR_DATANODE_MIN_KILL $AGTR_DATANODE_MAX_KILL"
[[ -n $AGITATOR_USER ]] || AGITATOR_USER=$(whoami)
if [[ $AGITATOR_USER == root ]]; then
@@ -187,7 +187,7 @@
case "$1" in
ci-createtable)
build_shade_jar
- java -Dlog4j.configuration="file:$log4j_config" org.apache.accumulo.testing.core.continuous.CreateTable "$at_props"
+ java -Dlog4j.configuration="file:$log4j_config" org.apache.accumulo.testing.core.continuous.CreateTable "$at_props" "$ACCUMULO_CLIENT_PROPS"
;;
ci-local)
if [ -z "$2" ]; then
@@ -197,7 +197,7 @@
fi
determine_app_main "$2"
build_shade_jar
- java -Dlog4j.configuration="file:$log4j_config" "$ci_main" "$at_props"
+ java -Dlog4j.configuration="file:$log4j_config" "$ci_main" "$at_props" "$ACCUMULO_CLIENT_PROPS"
;;
ci-yarn)
if [ -z "$2" ]; then
@@ -212,7 +212,7 @@
fi
determine_app_main "$3"
build_shade_jar
- mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloCITest-$3 -j $at_shaded_jar -m $ci_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties"
+ mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloCITest-$3 -j $at_shaded_jar -m $ci_main -n $2 -p $at_props -c $ACCUMULO_CLIENT_PROPS -l $log4j_config -a ./accumulo-testing.properties ./accumulo-client.properties"
;;
ci-mapred)
if [ -z "$2" ]; then
@@ -222,7 +222,7 @@
fi
determine_mapred_main "$2"
build_shade_jar
- "$HADOOP_PREFIX"/bin/yarn jar "$at_shaded_jar" "$ci_main" "$at_props"
+ "$HADOOP_HOME"/bin/yarn jar "$at_shaded_jar" "$ci_main" "$at_props" "$ACCUMULO_CLIENT_PROPS"
;;
rw-local)
if [ -z "$2" ]; then
@@ -231,7 +231,7 @@
exit 1
fi
build_shade_jar
- java -Dlog4j.configuration="file:$log4j_config" "$randomwalk_main" "$at_props" "$2"
+ java -Dlog4j.configuration="file:$log4j_config" "$randomwalk_main" "$at_props" "$ACCUMULO_CLIENT_PROPS" "$2"
;;
rw-yarn)
if [ -z "$2" ]; then
@@ -245,7 +245,7 @@
exit 1
fi
build_shade_jar
- mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRWTest-$3 -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3"
+ mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRWTest-$3 -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -c $ACCUMULO_CLIENT_PROPS -l $log4j_config -a ./accumulo-testing.properties ./accumulo-client.properties $3"
;;
agitator)
case "$2" in
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 677c23d..3876718 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -15,18 +15,22 @@
# General
# =======
-test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+
+## Hadoop installation
+export HADOOP_HOME="${HADOOP_HOME:-/path/to/hadoop}"
+## Accumulo installation
+export ACCUMULO_HOME="${ACCUMULO_HOME:-/path/to/accumulo}"
+## Path to Accumulo client properties
+export ACCUMULO_CLIENT_PROPS="$ACCUMULO_HOME/conf/accumulo-client.properties"
# Versions set below will be what is included in the shaded jar
-export ACCUMULO_VERSION=`accumulo version`
-export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
+export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
+export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
export ZOOKEEPER_VERSION=3.4.9
# Make sure Hadoop configuration directory is on the classpath
-export CLASSPATH=$HADOOP_PREFIX/etc/hadoop
+export CLASSPATH=$HADOOP_HOME/etc/hadoop
# Agitator
# ========
-# Agitator needs know where Accumulo is installed
-test -z "$ACCUMULO_HOME" && export ACCUMULO_HOME=/path/to/accumulo
# Accumulo user
AGTR_ACCUMULO_USER=$(whoami)
# Time (in minutes) between killing Accumulo masters
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
index 4fff104..b084c4c 100644
--- a/conf/accumulo-testing.properties.example
+++ b/conf/accumulo-testing.properties.example
@@ -17,24 +17,6 @@
# Common properties
###################
-# Accumulo instance name
-test.common.accumulo.instance=instance
-# Accumulo username
-test.common.accumulo.username=root
-# Accumulo password
-test.common.accumulo.password=secret
-# Max memory (in bytes) each batch writer will use to buffer writes
-test.common.accumulo.bw.max.memory.bytes=100000000
-# Max latency (in milliseconds) that each batch writer will buffer data
-test.common.accumulo.bw.max.latency.ms=600000
-# Number of write thread for each batch writer
-test.common.accumulo.bw.num.threads=4
-# Number of threads used by batch scanner
-test.common.accumulo.bs.num.threads=8
-# Number of key/value entries to pull during scan
-test.common.accumulo.scanner.batch.size=1000
-# Accumulo keytab
-#test.common.accumulo.keytab=
# HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
test.common.hdfs.root=hdfs://localhost:8020
# YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
@@ -43,8 +25,6 @@
test.common.yarn.container.memory.mb=1024
# Number of cores given to each container (if running in YARN)
test.common.yarn.container.cores=1
-# Zookeeper connection string
-test.common.zookeepers=localhost:2181
###################################
# Continuous ingest test properties
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
index 48266f5..5ec94c7 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
@@ -2,62 +2,55 @@
import static java.util.Objects.requireNonNull;
-import java.io.File;
-import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Properties;
-import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientInfo;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.client.security.tokens.KerberosToken;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
public class TestEnv {
- protected final Properties p;
- private Instance instance = null;
- private Connector connector = null;
+ protected final Properties testProps;
+ private String clientPropsPath;
+ private ClientInfo info;
+ private AccumuloClient client = null;
private Configuration hadoopConfig = null;
- /**
- * Creates new test environment using provided properties
- *
- * @param p
- * Properties
- */
- public TestEnv(Properties p) {
- requireNonNull(p);
- this.p = p;
+ public TestEnv(String testPropsPath, String clientPropsPath) {
+ requireNonNull(testPropsPath);
+ requireNonNull(clientPropsPath);
+ this.testProps = TestProps.loadFromFile(testPropsPath);
+ this.clientPropsPath = clientPropsPath;
+ this.info = ClientInfo.from(TestProps.loadFromFile(clientPropsPath));
}
/**
- * Gets a copy of the configuration properties.
- *
- * @return a copy of the configuration properties
+ * @return a copy of the test properties
*/
- public Properties copyConfigProperties() {
- return new Properties(p);
+ public Properties getTestProperties() {
+ return new Properties(testProps);
}
/**
- * Gets a configuration property.
- *
- * @param key
- * key
- * @return property value
+ * @return a test property value given a key
*/
- public String getConfigProperty(String key) {
- return p.getProperty(key);
+ public String getTestProperty(String key) {
+ return testProps.getProperty(key);
+ }
+
+ public String getClientPropsPath() {
+ return clientPropsPath;
+ }
+
+ public ClientInfo getInfo() {
+ return info;
}
/**
@@ -66,7 +59,7 @@
* @return username
*/
public String getAccumuloUserName() {
- return p.getProperty(TestProps.ACCUMULO_USERNAME);
+ return info.getPrincipal();
}
/**
@@ -75,16 +68,11 @@
* @return password
*/
public String getAccumuloPassword() {
- return p.getProperty(TestProps.ACCUMULO_PASSWORD);
- }
-
- /**
- * Gets the configured keytab.
- *
- * @return path to keytab
- */
- public String getAccumuloKeytab() {
- return p.getProperty(TestProps.ACCUMULO_KEYTAB);
+ String authType = info.getProperties().getProperty(ClientProperty.AUTH_TYPE.getKey());
+ if (authType.equals("password")) {
+ return info.getProperties().getProperty(ClientProperty.AUTH_TOKEN.getKey());
+ }
+ return null;
}
/**
@@ -114,79 +102,28 @@
* Gets an authentication token based on the configured password.
*/
public AuthenticationToken getToken() {
- String password = getAccumuloPassword();
- if (null != password) {
- return new PasswordToken(getAccumuloPassword());
- }
- String keytab = getAccumuloKeytab();
- if (null != keytab) {
- File keytabFile = new File(keytab);
- if (!keytabFile.exists() || !keytabFile.isFile()) {
- throw new IllegalArgumentException("Provided keytab is not a normal file: " + keytab);
- }
- try {
- UserGroupInformation.loginUserFromKeytab(getAccumuloUserName(), keytabFile.getAbsolutePath());
- return new KerberosToken();
- } catch (IOException e) {
- throw new RuntimeException("Failed to login", e);
- }
- }
- throw new IllegalArgumentException("Must provide password or keytab in configuration");
- }
-
- public String getAccumuloInstanceName() {
- return p.getProperty(TestProps.ACCUMULO_INSTANCE);
+ return info.getAuthenticationToken();
}
public String getHdfsRoot() {
- return p.getProperty(TestProps.HDFS_ROOT);
+ return testProps.getProperty(TestProps.HDFS_ROOT);
}
public String getYarnResourceManager() {
- return p.getProperty(TestProps.YARN_RESOURCE_MANAGER);
- }
-
- public String getZookeepers() {
- return p.getProperty(TestProps.ZOOKEEPERS);
- }
-
- public ClientConfiguration getClientConfiguration() {
- return ClientConfiguration.loadDefault().withInstance(getAccumuloInstanceName()).withZkHosts(getZookeepers());
- }
-
- /**
- * Gets an Accumulo instance object. The same instance is reused after the first call.
- */
- public Instance getAccumuloInstance() {
- if (instance == null) {
- this.instance = new ZooKeeperInstance(getClientConfiguration());
- }
- return instance;
+ return testProps.getProperty(TestProps.YARN_RESOURCE_MANAGER);
}
/**
* Gets an Accumulo connector. The same connector is reused after the first call.
*/
- public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException {
- if (connector == null) {
- connector = getAccumuloInstance().getConnector(getAccumuloUserName(), getToken());
+ public AccumuloClient getAccumuloClient() throws AccumuloException, AccumuloSecurityException {
+ if (client == null) {
+ client = Accumulo.newClient().usingClientInfo(info).build();
}
- return connector;
+ return client;
}
- public BatchWriterConfig getBatchWriterConfig() {
- int numThreads = Integer.parseInt(p.getProperty(TestProps.ACCUMULO_BW_NUM_THREADS));
- long maxLatency = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_LATENCY_MS));
- long maxMemory = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_MEM_BYTES));
-
- BatchWriterConfig config = new BatchWriterConfig();
- config.setMaxWriteThreads(numThreads);
- config.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS);
- config.setMaxMemory(maxMemory);
- return config;
- }
-
- public int getScannerBatchSize() {
- return Integer.parseInt(p.getProperty(TestProps.ACCUMULO_SCANNER_BATCH_SIZE));
+ public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException {
+ return Connector.from(getAccumuloClient());
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
index 638d2db..3b2c98f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
@@ -34,26 +34,6 @@
private static final String CI_VERIFY = CI + "verify.";
/** Common properties **/
- // Zookeeper connection string
- public static final String ZOOKEEPERS = COMMON + "zookeepers";
- // Accumulo instance name
- public static final String ACCUMULO_INSTANCE = COMMON + "accumulo.instance";
- // Accumulo username
- public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username";
- // Accumulo password
- public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password";
- // Max memory (in bytes) each batch writer will use to buffer writes
- public static final String ACCUMULO_BW_MAX_MEM_BYTES = COMMON + "accumulo.bw.max.memory.bytes";
- // Max the maximum time (in ms) each batch writer will buffer data
- public static final String ACCUMULO_BW_MAX_LATENCY_MS = COMMON + "accumulo.bw.max.latency.ms";
- // Number of threads each batch writer will use to write data
- public static final String ACCUMULO_BW_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
- // Number of thread for each batch scanner
- public static final String ACCUMULO_BS_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
- // Number of key/value entries to pull during scan
- public static final String ACCUMULO_SCANNER_BATCH_SIZE = COMMON + "accumulo.scanner.batch.size";
- // Accumulo keytab
- public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab";
// HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
public static final String HDFS_ROOT = COMMON + "hdfs.root";
// YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in
@@ -132,8 +112,12 @@
// Location in HDFS to store output
public static final String CI_VERIFY_OUTPUT_DIR = CI_VERIFY + "output.dir";
- public static Properties loadFromFile(String propsFilePath) throws IOException {
- return loadFromStream(new FileInputStream(propsFilePath));
+ public static Properties loadFromFile(String propsFilePath) {
+ try {
+ return loadFromStream(new FileInputStream(propsFilePath));
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
}
public static Properties loadFromStream(FileInputStream fis) throws IOException {
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
index 88134a0..2917800 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
@@ -21,7 +21,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -42,22 +41,22 @@
public static void main(String[] args) throws Exception {
- Properties props = TestProps.loadFromFile(args[0]);
-
- ContinuousEnv env = new ContinuousEnv(props);
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousBatchWalker <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Authorizations auths = env.getRandomAuthorizations();
Connector conn = env.getAccumuloConnector();
Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), auths);
- int scanBatchSize = Integer.parseInt(props.getProperty(TestProps.CI_BW_BATCH_SIZE));
+ int scanBatchSize = Integer.parseInt(env.getTestProperty(TestProps.CI_BW_BATCH_SIZE));
scanner.setBatchSize(scanBatchSize);
Random r = new Random();
- int scanThreads = Integer.parseInt(props.getProperty(TestProps.ACCUMULO_BS_NUM_THREADS));
-
while (true) {
- BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths, scanThreads);
+ BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths);
Set<Text> batch = getBatch(scanner, env.getRowMin(), env.getRowMax(), scanBatchSize, r);
List<Range> ranges = new ArrayList<>(batch.size());
@@ -68,7 +67,7 @@
runBatchScan(scanBatchSize, bs, batch, ranges);
- int bwSleepMs = Integer.parseInt(props.getProperty(TestProps.CI_BW_SLEEP_MS));
+ int bwSleepMs = Integer.parseInt(env.getTestProperty(TestProps.CI_BW_SLEEP_MS));
sleepUninterruptibly(bwSleepMs, TimeUnit.MILLISECONDS);
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java
index 7907ffd..76d6705 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java
@@ -3,7 +3,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Properties;
import java.util.Random;
import org.apache.accumulo.core.security.Authorizations;
@@ -14,8 +13,8 @@
private List<Authorizations> authList;
- ContinuousEnv(Properties props) {
- super(props);
+ ContinuousEnv(String testPropsPath, String clientPropsPath) {
+ super(testPropsPath, clientPropsPath);
}
/**
@@ -23,7 +22,7 @@
*/
private List<Authorizations> getAuthList() {
if (authList == null) {
- String authValue = p.getProperty(TestProps.CI_COMMON_AUTHS);
+ String authValue = testProps.getProperty(TestProps.CI_COMMON_AUTHS);
if (authValue == null || authValue.trim().isEmpty()) {
authList = Collections.singletonList(Authorizations.EMPTY);
} else {
@@ -45,22 +44,22 @@
}
long getRowMin() {
- return Long.parseLong(p.getProperty(TestProps.CI_INGEST_ROW_MIN));
+ return Long.parseLong(testProps.getProperty(TestProps.CI_INGEST_ROW_MIN));
}
long getRowMax() {
- return Long.parseLong(p.getProperty(TestProps.CI_INGEST_ROW_MAX));
+ return Long.parseLong(testProps.getProperty(TestProps.CI_INGEST_ROW_MAX));
}
int getMaxColF() {
- return Integer.parseInt(p.getProperty(TestProps.CI_INGEST_MAX_CF));
+ return Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_MAX_CF));
}
int getMaxColQ() {
- return Integer.parseInt(p.getProperty(TestProps.CI_INGEST_MAX_CQ));
+ return Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_MAX_CQ));
}
String getAccumuloTableName() {
- return p.getProperty(TestProps.CI_COMMON_ACCUMULO_TABLE);
+ return testProps.getProperty(TestProps.CI_COMMON_ACCUMULO_TABLE);
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
index 4afd00c..729fc53 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
@@ -99,14 +99,14 @@
public static void main(String[] args) throws Exception {
- if (args.length != 1) {
- System.err.println("Usage: ContinuousIngest <propsPath>");
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousIngest <testPropsPath> <clientPropsPath>");
System.exit(-1);
}
- Properties props = TestProps.loadFromFile(args[0]);
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
- String vis = props.getProperty(TestProps.CI_INGEST_VISIBILITIES);
+ String vis = env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES);
if (vis == null) {
visibilities = Collections.singletonList(new ColumnVisibility());
} else {
@@ -116,8 +116,6 @@
}
}
- ContinuousEnv env = new ContinuousEnv(props);
-
long rowMin = env.getRowMin();
long rowMax = env.getRowMax();
if (rowMin < 0 || rowMax < 0 || rowMax <= rowMin) {
@@ -130,7 +128,7 @@
throw new TableNotFoundException(null, tableName, "Consult the README and create the table before starting ingest.");
}
- BatchWriter bw = conn.createBatchWriter(tableName, env.getBatchWriterConfig());
+ BatchWriter bw = conn.createBatchWriter(tableName);
bw = Trace.wrapAll(bw, new CountSampler(1024));
Random r = new Random();
@@ -159,12 +157,13 @@
int maxColF = env.getMaxColF();
int maxColQ = env.getMaxColQ();
- boolean checksum = Boolean.parseBoolean(props.getProperty(TestProps.CI_INGEST_CHECKSUM));
- long numEntries = Long.parseLong(props.getProperty(TestProps.CI_INGEST_CLIENT_ENTRIES));
+ boolean checksum = Boolean.parseBoolean(env.getTestProperty(TestProps.CI_INGEST_CHECKSUM));
+ long numEntries = Long.parseLong(env.getTestProperty(TestProps.CI_INGEST_CLIENT_ENTRIES));
- if (pauseEnabled(props)) {
+ Properties testProps = env.getTestProperties();
+ if (pauseEnabled(testProps)) {
lastPauseNs = System.nanoTime();
- pauseWaitSec = getPauseWaitSec(props, r);
+ pauseWaitSec = getPauseWaitSec(testProps, r);
log.info("PAUSING enabled");
log.info("INGESTING for " + pauseWaitSec + "s");
}
@@ -208,7 +207,7 @@
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
if (count >= numEntries)
break out;
- pauseCheck(props, r);
+ pauseCheck(testProps, r);
}
// create one big linked list, this makes all of the first inserts
@@ -222,7 +221,7 @@
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
if (count >= numEntries)
break out;
- pauseCheck(props, r);
+ pauseCheck(testProps, r);
}
bw.close();
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java
index 81f9bd3..abaccc9 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java
@@ -19,7 +19,6 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
-import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@@ -113,19 +112,21 @@
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
- Properties props = TestProps.loadFromFile(args[0]);
- ContinuousEnv env = new ContinuousEnv(props);
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousMoru <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
job.setInputFormatClass(AccumuloInputFormat.class);
- AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken());
+ AccumuloInputFormat.setClientInfo(job, env.getInfo());
AccumuloInputFormat.setInputTableName(job, env.getAccumuloTableName());
- AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration());
- int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS));
+ int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
// set up ranges
try {
@@ -139,11 +140,9 @@
job.setMapperClass(CMapper.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setBatchWriterOptions(job, env.getBatchWriterConfig());
- AccumuloOutputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken());
+ AccumuloOutputFormat.setClientInfo(job, env.getInfo());
AccumuloOutputFormat.setCreateTables(job, true);
AccumuloOutputFormat.setDefaultTableName(job, env.getAccumuloTableName());
- AccumuloOutputFormat.setZooKeeperInstance(job, env.getClientConfiguration());
Configuration conf = job.getConfiguration();
conf.setLong(MIN, env.getRowMin());
@@ -157,7 +156,11 @@
}
public static void main(String[] args) throws Exception {
- ContinuousEnv env = new ContinuousEnv(TestProps.loadFromFile(args[0]));
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousMoru <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args);
if (res != 0)
System.exit(res);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java
index 1531269..c051691 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java
@@ -20,7 +20,6 @@
import java.util.Iterator;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@@ -38,9 +37,11 @@
public class ContinuousScanner {
public static void main(String[] args) throws Exception {
-
- Properties props = TestProps.loadFromFile(args[0]);
- ContinuousEnv env = new ContinuousEnv(props);
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousScanner <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Random r = new Random();
@@ -49,10 +50,9 @@
Connector conn = env.getAccumuloConnector();
Authorizations auths = env.getRandomAuthorizations();
Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), auths);
- scanner.setBatchSize(env.getScannerBatchSize());
- int numToScan = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_ENTRIES));
- int scannerSleepMs = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_SLEEP_MS));
+ int numToScan = Integer.parseInt(env.getTestProperty(TestProps.CI_SCANNER_ENTRIES));
+ int scannerSleepMs = Integer.parseInt(env.getTestProperty(TestProps.CI_SCANNER_SLEEP_MS));
double delta = Math.min(.05, .05 / (numToScan / 1000.0));
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java
index d343c9b..6eef2fe 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java
@@ -22,7 +22,6 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -140,20 +139,22 @@
@Override
public int run(String[] args) throws Exception {
-
- Properties props = TestProps.loadFromFile(args[0]);
- ContinuousEnv env = new ContinuousEnv(props);
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousVerify <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
job.setInputFormatClass(AccumuloInputFormat.class);
- boolean scanOffline = Boolean.parseBoolean(props.getProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
+ boolean scanOffline = Boolean.parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
String tableName = env.getAccumuloTableName();
- int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS));
- int reducers = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_REDUCERS));
- String outputDir = props.getProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
+ int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
+ int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
+ String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
Set<Range> ranges;
String clone = "";
@@ -174,8 +175,7 @@
AccumuloInputFormat.setRanges(job, ranges);
AccumuloInputFormat.setAutoAdjustRanges(job, false);
- AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken());
- AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration());
+ AccumuloInputFormat.setClientInfo(job, env.getInfo());
job.setMapperClass(CMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
@@ -199,8 +199,11 @@
}
public static void main(String[] args) throws Exception {
-
- ContinuousEnv env = new ContinuousEnv(TestProps.loadFromFile(args[0]));
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousVerify <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args);
if (res != 0)
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java
index 49c10c9..aa5b899 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java
@@ -20,7 +20,6 @@
import java.util.ArrayList;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.Random;
import java.util.zip.CRC32;
@@ -46,9 +45,11 @@
}
public static void main(String[] args) throws Exception {
-
- Properties props = TestProps.loadFromFile(args[0]);
- ContinuousEnv env = new ContinuousEnv(props);
+ if (args.length != 2) {
+ System.err.println("Usage: ContinuousWalk <testPropsPath> <clientPropsPath>");
+ System.exit(-1);
+ }
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Connector conn = env.getAccumuloConnector();
@@ -56,7 +57,7 @@
ArrayList<Value> values = new ArrayList<>();
- int sleepTime = Integer.parseInt(props.getProperty(TestProps.CI_WALKER_SLEEP_MS));
+ int sleepTime = Integer.parseInt(env.getTestProperty(TestProps.CI_WALKER_SLEEP_MS));
while (true) {
Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), env.getRandomAuthorizations());
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java
index bac8368..c2386cb 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.testing.core.continuous;
-import java.util.Properties;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -28,13 +27,11 @@
public static void main(String[] args) throws Exception {
- if (args.length != 1) {
- System.err.println("Usage: CreateTable <propsPath>");
+ if (args.length != 2) {
+ System.err.println("Usage: CreateTable <testPropsPath> <clientPropsPath>");
System.exit(-1);
}
-
- Properties props = TestProps.loadFromFile(args[0]);
- ContinuousEnv env = new ContinuousEnv(props);
+ ContinuousEnv env = new ContinuousEnv(args[0], args[1]);
Connector conn = env.getAccumuloConnector();
String tableName = env.getAccumuloTableName();
@@ -43,7 +40,7 @@
System.exit(-1);
}
- int numTablets = Integer.parseInt(props.getProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS));
+ int numTablets = Integer.parseInt(env.getTestProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS));
if (numTablets < 1) {
System.err.println("ERROR: numTablets < 1");
System.exit(-1);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
index 70df0c0..ccd5a78 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
@@ -38,7 +38,6 @@
import org.apache.accumulo.core.cli.BatchScannerOpts;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java
index 2d43bee..57a3cf1 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java
@@ -30,13 +30,11 @@
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.TabletServerBatchWriter;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.ConstraintViolationSummary;
import org.apache.accumulo.core.data.Key;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java
index be3de45..2fd62b1 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java
@@ -25,7 +25,6 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java
index a2286bb..b3d664e 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java
@@ -26,7 +26,6 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java
index 4c9ac70..126ea5f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java
@@ -39,7 +39,6 @@
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
index 5531d81..2ff6046 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
@@ -21,7 +21,6 @@
import org.apache.accumulo.core.cli.ClientOpts;
import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
index 2a0f26d..1c63575 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
@@ -23,7 +23,6 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.io.Text;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
index 43c66b4..96083fa 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
@@ -19,7 +19,6 @@
import java.util.HashMap;
import java.util.Properties;
-import org.apache.accumulo.testing.core.TestProps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,18 +80,16 @@
public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.out.println("Usage: Framework <propsPath> <module>");
+ if (args.length != 3) {
+ System.out.println("Usage: Framework <testPropsPath> <clientPropsPath> <module>");
System.exit(-1);
}
- Properties props = TestProps.loadFromFile(args[0]);
-
- log.info("Running random walk test with module: " + args[1]);
+ log.info("Running random walk test with module: " + args[2]);
State state = new State();
- RandWalkEnv env = new RandWalkEnv(props);
- getInstance().run(args[1], state, env);
+ RandWalkEnv env = new RandWalkEnv(args[0], args[1]);
+ getInstance().run(args[2], state, env);
log.info("Test finished");
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
index 68ffe6a..ef26591 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
@@ -342,7 +342,7 @@
log.debug(" " + entry.getKey() + ": " + entry.getValue());
}
log.debug("Overall Configuration Properties");
- for (Entry<Object,Object> entry : env.copyConfigProperties().entrySet()) {
+ for (Entry<Object,Object> entry : env.getTestProperties().entrySet()) {
log.debug(" " + entry.getKey() + ": " + entry.getValue());
}
log.debug("State information");
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java
index f85f560..207b364 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java
@@ -16,8 +16,6 @@
*/
package org.apache.accumulo.testing.core.randomwalk;
-import java.util.Properties;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
@@ -35,14 +33,8 @@
private MultiTableBatchWriter mtbw = null;
- /**
- * Creates a new test environment.
- *
- * @param p
- * configuration properties
- */
- public RandWalkEnv(Properties p) {
- super(p);
+ public RandWalkEnv(String testPropsPath, String clientPropsPath) {
+ super(testPropsPath, clientPropsPath);
}
/**
@@ -56,7 +48,7 @@
*/
public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException {
if (mtbw == null) {
- mtbw = getAccumuloConnector().createMultiTableBatchWriter(getBatchWriterConfig());
+ mtbw = getAccumuloClient().createMultiTableBatchWriter();
}
return mtbw;
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
index 9324861..4d987d4 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
@@ -40,8 +40,6 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.admin.TableOperations;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
index cd9fd1e..c70efaa 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
@@ -26,7 +26,6 @@
import java.util.TreeSet;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
@@ -44,7 +43,6 @@
public void setUp(State state, RandWalkEnv env) throws Exception {
Connector conn = env.getAccumuloConnector();
- Instance instance = env.getAccumuloInstance();
SortedSet<Text> splits = new TreeSet<>();
for (int i = 1; i < 256; i++) {
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
index 6417913..cbaf0b8 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -21,7 +21,6 @@
import java.util.Random;
import java.util.TreeSet;
-import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -53,16 +52,10 @@
int nextId = ((Integer) state.get("nextId")).intValue();
String dstTableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
- String[] args = new String[6];
- args[0] = env.getAccumuloUserName();
- args[1] = env.getAccumuloPassword();
- if (null == args[1]) {
- args[1] = env.getAccumuloKeytab();
- }
- args[2] = srcTableName;
- args[3] = env.getAccumuloInstance().getInstanceName();
- args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[5] = dstTableName;
+ String[] args = new String[3];
+ args[0] = env.getClientPropsPath();
+ args[1] = srcTableName;
+ args[2] = dstTableName;
log.debug("copying " + srcTableName + " to " + dstTableName);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
index 5c979d3..df265eb 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
@@ -18,26 +18,18 @@
import java.io.IOException;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.ClientInfo;
import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.client.security.tokens.KerberosToken;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.SystemPermission;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,70 +56,21 @@
return 1;
}
- ClientConfiguration clientConf = ClientConfiguration.create().withInstance(args[3]).withZkHosts(args[4]);
-
+ ClientInfo info = Accumulo.newClient().usingProperties(args[0]).info();
job.setInputFormatClass(AccumuloInputFormat.class);
- AccumuloInputFormat.setInputTableName(job, args[2]);
+ AccumuloInputFormat.setClientInfo(job, info);
+ AccumuloInputFormat.setInputTableName(job, args[1]);
AccumuloInputFormat.setScanAuthorizations(job, Authorizations.EMPTY);
- AccumuloInputFormat.setZooKeeperInstance(job, clientConf);
-
- final String principal;
- final AuthenticationToken token;
- if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) {
- // Use the Kerberos creds to request a DelegationToken for MapReduce
- // to use
- // We could use the specified keytab (args[1]), but we're already
- // logged in and don't need to, so we can just use the current user
- KerberosToken kt = new KerberosToken();
- try {
- UserGroupInformation user = UserGroupInformation.getCurrentUser();
- if (!user.hasKerberosCredentials()) {
- throw new IllegalStateException("Expected current user to have Kerberos credentials");
- }
-
- // Get the principal via UGI
- principal = user.getUserName();
-
- // Connector w/ the Kerberos creds
- ZooKeeperInstance inst = new ZooKeeperInstance(clientConf);
- Connector conn = inst.getConnector(principal, kt);
-
- // Do the explicit check to see if the user has the permission
- // to get a delegation token
- if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
- log.error(principal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name()
- + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's"
- + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials.");
- throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token");
- }
-
- // Fetch a delegation token from Accumulo
- token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
-
- } catch (Exception e) {
- final String msg = "Failed to acquire DelegationToken for use with MapReduce";
- log.error(msg, e);
- throw new RuntimeException(msg, e);
- }
- } else {
- // Simple principal + password
- principal = args[0];
- token = new PasswordToken(args[1]);
- }
-
- AccumuloInputFormat.setConnectorInfo(job, principal, token);
- AccumuloOutputFormat.setConnectorInfo(job, principal, token);
job.setMapperClass(SeqMapClass.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Mutation.class);
-
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setClientInfo(job, info);
AccumuloOutputFormat.setCreateTables(job, true);
- AccumuloOutputFormat.setDefaultTableName(job, args[5]);
- AccumuloOutputFormat.setZooKeeperInstance(job, clientConf);
+ AccumuloOutputFormat.setDefaultTableName(job, args[2]);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
index 1283fd7..c973e11 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
@@ -19,9 +19,9 @@
import java.net.InetAddress;
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
@@ -36,15 +36,15 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
String systemUser = WalkingSecurity.get(state, env).getSysUserName();
- Connector conn = env.getAccumuloInstance().getConnector(systemUser, WalkingSecurity.get(state, env).getSysToken());
+ AccumuloClient client = env.getAccumuloClient().changeUser(systemUser, WalkingSecurity.get(state, env).getSysToken());
String tableName = WalkingSecurity.get(state, env).getTableName();
boolean exists = WalkingSecurity.get(state, env).getTableExists();
boolean hasPermission;
try {
- hasPermission = conn.securityOperations().hasTablePermission(systemUser, tableName, TablePermission.ALTER_TABLE)
- || conn.securityOperations().hasSystemPermission(systemUser, SystemPermission.ALTER_TABLE);
+ hasPermission = client.securityOperations().hasTablePermission(systemUser, tableName, TablePermission.ALTER_TABLE)
+ || client.securityOperations().hasSystemPermission(systemUser, SystemPermission.ALTER_TABLE);
} catch (AccumuloSecurityException ae) {
if (ae.getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST)) {
if (exists)
@@ -58,13 +58,13 @@
String newTableName = String.format("security_%s_%s_%d", InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_"), env.getPid(),
System.currentTimeMillis());
- renameTable(conn, state, env, tableName, newTableName, hasPermission, exists);
+ renameTable(client, state, env, tableName, newTableName, hasPermission, exists);
}
- public static void renameTable(Connector conn, State state, RandWalkEnv env, String oldName, String newName, boolean hasPermission, boolean tableExists)
+ public static void renameTable(AccumuloClient client, State state, RandWalkEnv env, String oldName, String newName, boolean hasPermission, boolean tableExists)
throws AccumuloSecurityException, AccumuloException, TableExistsException {
try {
- conn.tableOperations().rename(oldName, newName);
+ client.tableOperations().rename(oldName, newName);
} catch (AccumuloSecurityException ae) {
if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (hasPermission)
@@ -72,7 +72,7 @@
else
return;
} else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
- if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ if (WalkingSecurity.get(state, env).userPassTransient(client.whoami()))
return;
}
throw new AccumuloException("Got unexpected ae error code", ae);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
index 94a972d..553c766 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
@@ -19,9 +19,9 @@
import java.util.Properties;
import java.util.Random;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.admin.SecurityOperations;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
@@ -76,8 +76,8 @@
sourceUser = env.getAccumuloUserName();
sourceToken = env.getToken();
}
- Connector conn = env.getAccumuloInstance().getConnector(sourceUser, sourceToken);
- SecurityOperations secOps = conn.securityOperations();
+ AccumuloClient client = env.getAccumuloClient().changeUser(sourceUser, sourceToken);
+ SecurityOperations secOps = client.securityOperations();
try {
canGive = secOps.hasSystemPermission(sourceUser, SystemPermission.ALTER_TABLE) || secOps.hasTablePermission(sourceUser, tableName, TablePermission.GRANT);
@@ -97,7 +97,7 @@
try {
boolean res;
if (hasPerm != (res = env.getAccumuloConnector().securityOperations().hasTablePermission(target, tableName, tabPerm)))
- throw new AccumuloException("Test framework and accumulo are out of sync for user " + conn.whoami() + " for perm " + tabPerm.name()
+ throw new AccumuloException("Test framework and accumulo are out of sync for user " + client.whoami() + " for perm " + tabPerm.name()
+ " with local vs. accumulo being " + hasPerm + " " + res);
if (hasPerm)
@@ -122,17 +122,17 @@
}
}
- boolean trans = WalkingSecurity.get(state, env).userPassTransient(conn.whoami());
+ boolean trans = WalkingSecurity.get(state, env).userPassTransient(client.whoami());
if ("take".equals(action)) {
try {
- conn.securityOperations().revokeTablePermission(target, tableName, tabPerm);
+ client.securityOperations().revokeTablePermission(target, tableName, tabPerm);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case GRANT_INVALID:
throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae);
case PERMISSION_DENIED:
if (canGive)
- throw new AccumuloException(conn.whoami() + " failed to revoke permission to " + target + " when it should have worked", ae);
+ throw new AccumuloException(client.whoami() + " failed to revoke permission to " + target + " when it should have worked", ae);
return;
case USER_DOESNT_EXIST:
if (userExists)
@@ -144,7 +144,7 @@
return;
case BAD_CREDENTIALS:
if (!trans)
- throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ throw new AccumuloException("Bad credentials for user " + client.whoami());
return;
default:
throw new AccumuloException("Got unexpected exception", ae);
@@ -153,14 +153,14 @@
WalkingSecurity.get(state, env).revokeTablePermission(target, tableName, tabPerm);
} else if ("give".equals(action)) {
try {
- conn.securityOperations().grantTablePermission(target, tableName, tabPerm);
+ client.securityOperations().grantTablePermission(target, tableName, tabPerm);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case GRANT_INVALID:
throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae);
case PERMISSION_DENIED:
if (canGive)
- throw new AccumuloException(conn.whoami() + " failed to give permission to " + target + " when it should have worked", ae);
+ throw new AccumuloException(client.whoami() + " failed to give permission to " + target + " when it should have worked", ae);
return;
case USER_DOESNT_EXIST:
if (userExists)
@@ -172,7 +172,7 @@
return;
case BAD_CREDENTIALS:
if (!trans)
- throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ throw new AccumuloException("Bad credentials for user " + client.whoami());
return;
default:
throw new AccumuloException("Got unexpected exception", ae);
@@ -186,7 +186,7 @@
if (!tableExists)
throw new AccumuloException("Table shouldn't have existed, but apparently does");
if (!canGive)
- throw new AccumuloException(conn.whoami() + " shouldn't have been able to grant privilege");
+ throw new AccumuloException(client.whoami() + " shouldn't have been able to grant privilege");
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
index 63105f4..037abbc 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
@@ -19,9 +19,9 @@
import java.util.Arrays;
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.security.SystemPermission;
@@ -40,7 +40,7 @@
String targetProp = props.getProperty("target");
boolean success = Boolean.parseBoolean(props.getProperty("valid"));
- Connector conn = env.getAccumuloInstance().getConnector(principal, token);
+ AccumuloClient client = env.getAccumuloClient().changeUser(principal, token);
String target;
@@ -52,7 +52,7 @@
boolean exists = WalkingSecurity.get(state, env).userExists(target);
// Copy so if failed it doesn't mess with the password stored in state
byte[] password = Arrays.copyOf(WalkingSecurity.get(state, env).getUserPassword(target), WalkingSecurity.get(state, env).getUserPassword(target).length);
- boolean hasPermission = conn.securityOperations().hasSystemPermission(principal, SystemPermission.SYSTEM) || principal.equals(target);
+ boolean hasPermission = client.securityOperations().hasSystemPermission(principal, SystemPermission.SYSTEM) || principal.equals(target);
if (!success)
for (int i = 0; i < password.length; i++)
@@ -61,7 +61,7 @@
boolean result;
try {
- result = conn.securityOperations().authenticateUser(target, new PasswordToken(password));
+ result = client.securityOperations().authenticateUser(target, new PasswordToken(password));
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case PERMISSION_DENIED:
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
index 585a2c1..b409892 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
@@ -19,9 +19,9 @@
import java.util.Properties;
import java.util.Random;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.security.SystemPermission;
@@ -45,7 +45,7 @@
principal = WalkingSecurity.get(state, env).getTabUserName();
token = WalkingSecurity.get(state, env).getTabToken();
}
- Connector conn = env.getAccumuloInstance().getConnector(principal, token);
+ AccumuloClient client = env.getAccumuloClient().changeUser(principal, token);
boolean hasPerm;
boolean targetExists;
@@ -56,7 +56,7 @@
targetExists = WalkingSecurity.get(state, env).userExists(target);
- hasPerm = conn.securityOperations().hasSystemPermission(principal, SystemPermission.ALTER_USER) || principal.equals(target);
+ hasPerm = client.securityOperations().hasSystemPermission(principal, SystemPermission.ALTER_USER) || principal.equals(target);
Random r = new Random();
@@ -66,7 +66,7 @@
PasswordToken newPass = new PasswordToken(newPassw);
try {
- conn.securityOperations().changeLocalUserPassword(target, newPass);
+ client.securityOperations().changeLocalUserPassword(target, newPass);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case PERMISSION_DENIED:
@@ -78,8 +78,8 @@
throw new AccumuloException("User " + target + " doesn't exist and they SHOULD.", ae);
return;
case BAD_CREDENTIALS:
- if (!WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
- throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ if (!WalkingSecurity.get(state, env).userPassTransient(client.whoami()))
+ throw new AccumuloException("Bad credentials for user " + client.whoami());
return;
default:
throw new AccumuloException("Got unexpected exception", ae);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
index de11d62..e4e8190 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
@@ -18,9 +18,9 @@
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
import org.apache.accumulo.core.security.SystemPermission;
@@ -33,15 +33,15 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
- Connector conn = env.getAccumuloInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
+ AccumuloClient client = env.getAccumuloClient().changeUser(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
String tableName = WalkingSecurity.get(state, env).getTableName();
boolean exists = WalkingSecurity.get(state, env).getTableExists();
- boolean hasPermission = conn.securityOperations().hasSystemPermission(WalkingSecurity.get(state, env).getSysUserName(), SystemPermission.CREATE_TABLE);
+ boolean hasPermission = client.securityOperations().hasSystemPermission(WalkingSecurity.get(state, env).getSysUserName(), SystemPermission.CREATE_TABLE);
try {
- conn.tableOperations().create(tableName);
+ client.tableOperations().create(tableName);
} catch (AccumuloSecurityException ae) {
if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (hasPermission)
@@ -69,7 +69,7 @@
}
WalkingSecurity.get(state, env).initTable(tableName);
for (TablePermission tp : TablePermission.values())
- WalkingSecurity.get(state, env).grantTablePermission(conn.whoami(), tableName, tp);
+ WalkingSecurity.get(state, env).grantTablePermission(client.whoami(), tableName, tp);
if (!hasPermission)
throw new AccumuloException("Didn't get Security Exception when we should have");
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
index ca9afbe..f23f87c 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
@@ -18,9 +18,9 @@
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
@@ -32,15 +32,15 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
String sysPrincipal = WalkingSecurity.get(state, env).getSysUserName();
- Connector conn = env.getAccumuloInstance().getConnector(sysPrincipal, WalkingSecurity.get(state, env).getSysToken());
+ AccumuloClient client = env.getAccumuloClient().changeUser(sysPrincipal, WalkingSecurity.get(state, env).getSysToken());
String tableUserName = WalkingSecurity.get(state, env).getTabUserName();
boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName);
- boolean hasPermission = conn.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.CREATE_USER);
+ boolean hasPermission = client.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.CREATE_USER);
PasswordToken tabUserPass = new PasswordToken("Super Sekret Table User Password");
try {
- conn.securityOperations().createLocalUser(tableUserName, tabUserPass);
+ client.securityOperations().createLocalUser(tableUserName, tabUserPass);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case PERMISSION_DENIED:
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
index 66fc0e2..ddfee12 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
@@ -18,9 +18,9 @@
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
@@ -50,16 +50,16 @@
principal = WalkingSecurity.get(state, env).getSysUserName();
token = WalkingSecurity.get(state, env).getSysToken();
}
- Connector conn = env.getAccumuloInstance().getConnector(principal, token);
+ AccumuloClient client = env.getAccumuloClient().changeUser(principal, token);
String tableName = WalkingSecurity.get(state, env).getTableName();
boolean exists = WalkingSecurity.get(state, env).getTableExists();
try {
- hasPermission = conn.securityOperations().hasTablePermission(principal, tableName, TablePermission.DROP_TABLE)
- || conn.securityOperations().hasSystemPermission(principal, SystemPermission.DROP_TABLE);
- conn.tableOperations().delete(tableName);
+ hasPermission = client.securityOperations().hasTablePermission(principal, tableName, TablePermission.DROP_TABLE)
+ || client.securityOperations().hasSystemPermission(principal, SystemPermission.DROP_TABLE);
+ client.tableOperations().delete(tableName);
} catch (AccumuloSecurityException ae) {
if (ae.getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST)) {
if (exists)
@@ -76,7 +76,7 @@
return;
}
} else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
- if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ if (WalkingSecurity.get(state, env).userPassTransient(client.whoami()))
return;
}
throw new AccumuloException("Got unexpected ae error code", ae);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
index 933c26d..d5cf910 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
@@ -18,9 +18,9 @@
import java.util.Properties;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -31,15 +31,15 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
String sysPrincipal = WalkingSecurity.get(state, env).getSysUserName();
- Connector conn = env.getAccumuloInstance().getConnector(sysPrincipal, WalkingSecurity.get(state, env).getSysToken());
+ AccumuloClient client = env.getAccumuloClient().changeUser(sysPrincipal, WalkingSecurity.get(state, env).getSysToken());
String tableUserName = WalkingSecurity.get(state, env).getTabUserName();
boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName);
- boolean hasPermission = conn.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.DROP_USER);
+ boolean hasPermission = client.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.DROP_USER);
try {
- conn.securityOperations().dropLocalUser(tableUserName);
+ client.securityOperations().dropLocalUser(tableUserName);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case PERMISSION_DENIED:
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
index 1bffba0..8782fdb 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
@@ -19,10 +19,9 @@
import java.net.InetAddress;
import java.util.Set;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
@@ -36,8 +35,7 @@
public void setUp(State state, RandWalkEnv env) throws Exception {
String secTableName, systemUserName, tableUserName, secNamespaceName;
// A best-effort sanity check to guard against not password-based auth
- ClientConfiguration clientConf = ClientConfiguration.loadDefault();
- if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) {
+ if (env.getInfo().getProperties().getProperty(ClientProperty.AUTH_TYPE.getKey()).equals("kerberos")) {
throw new IllegalStateException("Security module currently cannot support Kerberos/SASL instances");
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
index feee2d8..c309f35 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
@@ -19,9 +19,9 @@
import java.util.Properties;
import java.util.Random;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
@@ -48,10 +48,10 @@
authPrincipal = env.getAccumuloUserName();
authToken = env.getToken();
}
- Connector conn = env.getAccumuloInstance().getConnector(authPrincipal, authToken);
+ AccumuloClient client = env.getAccumuloClient().changeUser(authPrincipal, authToken);
boolean exists = WalkingSecurity.get(state, env).userExists(target);
- boolean hasPermission = conn.securityOperations().hasSystemPermission(authPrincipal, SystemPermission.ALTER_USER);
+ boolean hasPermission = client.securityOperations().hasSystemPermission(authPrincipal, SystemPermission.ALTER_USER);
Authorizations auths;
if (authsString.equals("_random")) {
@@ -74,7 +74,7 @@
}
try {
- conn.securityOperations().changeUserAuthorizations(target, auths);
+ client.securityOperations().changeUserAuthorizations(target, auths);
} catch (AccumuloSecurityException ae) {
switch (ae.getSecurityErrorCode()) {
case PERMISSION_DENIED:
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
index 2fee84f..3029542 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
@@ -19,7 +19,6 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import java.util.Iterator;
-import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Random;
@@ -27,11 +26,11 @@
import java.util.TreeSet;
import java.util.UUID;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -58,9 +57,9 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
String tablePrincipal = WalkingSecurity.get(state, env).getTabUserName();
- Connector conn = env.getAccumuloInstance().getConnector(tablePrincipal, WalkingSecurity.get(state, env).getTabToken());
- TableOperations tableOps = conn.tableOperations();
- SecurityOperations secOps = conn.securityOperations();
+ AccumuloClient client = env.getAccumuloClient().changeUser(tablePrincipal, WalkingSecurity.get(state, env).getTabToken());
+ TableOperations tableOps = client.tableOperations();
+ SecurityOperations secOps = client.securityOperations();
String action = props.getProperty("action", "_random");
TablePermission tp;
@@ -85,12 +84,12 @@
return;
}
Authorizations auths = secOps.getUserAuthorizations(tablePrincipal);
- boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp);
- boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(conn.whoami());
+ boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(client.whoami(), tp);
+ boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(client.whoami());
Scanner scan = null;
try {
- scan = conn.createScanner(tableName, secOps.getUserAuthorizations(conn.whoami()));
+ scan = client.createScanner(tableName, secOps.getUserAuthorizations(client.whoami()));
int seen = 0;
Iterator<Entry<Key,Value>> iter = scan.iterator();
while (iter.hasNext()) {
@@ -101,7 +100,7 @@
throw new AccumuloException("Got data I should not be capable of seeing: " + k + " table " + tableName);
}
if (!canRead && !ambiguousZone)
- throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName);
+ throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + client.whoami() + " table " + tableName);
for (Entry<String,Integer> entry : WalkingSecurity.get(state, env).getAuthsMap().entrySet()) {
if (auths.contains(entry.getKey().getBytes(UTF_8)))
seen = seen - entry.getValue();
@@ -171,7 +170,7 @@
BatchWriter writer = null;
try {
try {
- writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1));
+ writer = client.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1));
} catch (TableNotFoundException tnfe) {
if (tableExists)
throw new AccumuloException("Table didn't exist when it should have: " + tableName);
@@ -230,7 +229,7 @@
throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName);
return;
} else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
- if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ if (WalkingSecurity.get(state, env).userPassTransient(client.whoami()))
return;
}
throw new AccumuloException("Unexpected exception!", ae);
@@ -252,7 +251,7 @@
throw new AccumuloException("Table didn't exist when it should have: " + tableName, ase);
return;
}
- AlterTable.renameTable(conn, state, env, tableName, tableName + "plus", tablePerm, tableExists);
+ AlterTable.renameTable(client, state, env, tableName, tableName + "plus", tablePerm, tableExists);
break;
case GRANT:
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
index 9e36c86..2cc0f4f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
@@ -22,7 +22,6 @@
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
index 924b8d9..9b080ea 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -25,7 +25,6 @@
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -36,16 +35,10 @@
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws Exception {
- String[] args = new String[6];
- args[0] = env.getAccumuloUserName();
- args[1] = env.getAccumuloPassword();
- if (null == args[1]) {
- args[1] = env.getAccumuloKeytab();
- }
- args[2] = state.getString("seqTableName");
- args[3] = env.getAccumuloInstance().getInstanceName();
- args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[5] = args[2] + "_MR";
+ String[] args = new String[3];
+ args[0] = env.getClientPropsPath();
+ args[1] = state.getString("seqTableName");
+ args[2] = args[2] + "_MR";
if (ToolRunner.run(env.getHadoopConfiguration(), new MapRedVerifyTool(), args) != 0) {
log.error("Failed to run map/red verify");
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
index e8dcca1..de952ee 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
@@ -19,20 +19,13 @@
import java.io.IOException;
import java.util.Iterator;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.ClientInfo;
import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.client.security.tokens.KerberosToken;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.SystemPermission;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
@@ -40,7 +33,6 @@
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -95,53 +87,14 @@
return 1;
}
- ClientConfiguration clientConf = ClientConfiguration.loadDefault().withInstance(args[3]).withZkHosts(args[4]);
+ ClientInfo info = Accumulo.newClient().usingProperties(args[0]).info();
+ AccumuloInputFormat.setClientInfo(job, info);
+ AccumuloInputFormat.setInputTableName(job, args[1]);
- AccumuloInputFormat.setInputTableName(job, args[2]);
- AccumuloInputFormat.setZooKeeperInstance(job, clientConf);
- AccumuloOutputFormat.setDefaultTableName(job, args[5]);
- AccumuloOutputFormat.setZooKeeperInstance(job, clientConf);
+ AccumuloOutputFormat.setClientInfo(job, info);
+ AccumuloOutputFormat.setDefaultTableName(job, args[2]);
job.setInputFormatClass(AccumuloInputFormat.class);
- if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) {
- // Better be logged in
- KerberosToken token = new KerberosToken();
- try {
- UserGroupInformation user = UserGroupInformation.getCurrentUser();
- if (!user.hasKerberosCredentials()) {
- throw new IllegalStateException("Expected current user to have Kerberos credentials");
- }
-
- String newPrincipal = user.getUserName();
-
- ZooKeeperInstance inst = new ZooKeeperInstance(clientConf);
- Connector conn = inst.getConnector(newPrincipal, token);
-
- // Do the explicit check to see if the user has the permission
- // to get a delegation token
- if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
- log.error(newPrincipal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name()
- + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's"
- + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials.");
- throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token");
- }
-
- // Fetch a delegation token from Accumulo
- AuthenticationToken dt = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
-
- // Set the delegation token instead of the kerberos token
- AccumuloInputFormat.setConnectorInfo(job, newPrincipal, dt);
- AccumuloOutputFormat.setConnectorInfo(job, newPrincipal, dt);
- } catch (Exception e) {
- final String msg = "Failed to acquire DelegationToken for use with MapReduce";
- log.error(msg, e);
- throw new RuntimeException(msg, e);
- }
- } else {
- AccumuloInputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1]));
- AccumuloOutputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1]));
- }
-
job.setMapperClass(SeqMapClass.class);
job.setMapOutputKeyClass(NullWritable.class);
job.setMapOutputValueClass(IntWritable.class);
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
index 0514f05..3fe34a0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
@@ -19,7 +19,6 @@
import java.net.InetAddress;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
index b520437..108dda1 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
@@ -36,7 +36,6 @@
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
index 65b7a2b..708658e 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
@@ -23,7 +23,6 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
index d27afa5..1cd4739 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
@@ -18,7 +18,6 @@
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
diff --git a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
index 02e7409..a8da367 100644
--- a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
+++ b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
@@ -19,15 +19,12 @@
import static org.apache.accumulo.core.conf.Property.TSERV_NATIVEMAP_ENABLED;
import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
-import java.util.Properties;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
import org.apache.accumulo.testing.core.randomwalk.concurrent.Replication;
import org.apache.hadoop.conf.Configuration;
+import org.easymock.EasyMock;
import org.junit.Test;
public class ReplicationRandomWalkIT extends ConfigurableMacBase {
@@ -43,23 +40,13 @@
public void runReplicationRandomWalkStep() throws Exception {
Replication r = new Replication();
- RandWalkEnv env = new RandWalkEnv(new Properties()) {
- @Override
- public String getAccumuloUserName() {
- return "root";
- }
+ RandWalkEnv env = EasyMock.createMock(RandWalkEnv.class);
+ EasyMock.expect(env.getAccumuloUserName()).andReturn("root").anyTimes();
+ EasyMock.expect(env.getAccumuloPassword()).andReturn(ROOT_PASSWORD).anyTimes();
+ EasyMock.expect(env.getAccumuloClient()).andReturn(this.getClient()).anyTimes();
+ EasyMock.expect(env.getAccumuloConnector()).andReturn(Connector.from(this.getClient())).anyTimes();
+ EasyMock.replay(env);
- @Override
- public String getAccumuloPassword() {
- return ROOT_PASSWORD;
- }
-
- @Override
- public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException {
- return Connector.from(ReplicationRandomWalkIT.this.getClient());
- }
-
- };
r.visit(null, env, null);
}
diff --git a/pom.xml b/pom.xml
index 69f7097..0de1eed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,7 +39,7 @@
<properties>
<accumulo.version>2.0.0-SNAPSHOT</accumulo.version>
- <hadoop.version>2.6.4</hadoop.version>
+ <hadoop.version>3.0.3</hadoop.version>
<zookeeper.version>3.4.9</zookeeper.version>
<slf4j.version>1.7.21</slf4j.version>
<twill.version>0.12.1</twill.version>
@@ -53,12 +53,12 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>26.0-jre</version>
</dependency>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
- <version>1.48</version>
+ <version>1.72</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -73,7 +73,7 @@
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
- <version>1.6</version>
+ <version>1.10</version>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index 30b2191..56fe6f3 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -23,11 +23,8 @@
import org.apache.accumulo.testing.core.TestEnv;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
-import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.TwillSpecification;
import org.apache.twill.ext.BundledJarRunnable;
@@ -39,7 +36,6 @@
import java.io.File;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.Properties;
@@ -69,12 +65,13 @@
.setMemory(memory, ResourceSpecification.SizeUnit.MEGA).setInstances(opts.numContainers).build();
File jarFile = new File(opts.jarPath);
+ File clientProps = new File(opts.clientProps);
File testProps = new File(opts.testProps);
File log4jProps = new File(opts.logProps);
return TwillSpecification.Builder.with().setName(opts.testName).withRunnable().add(RUNNABLE_ID, new BundledJarRunnable(), resourceSpec).withLocalFiles()
- .add(jarFile.getName(), jarFile.toURI(), false).add(testProps.getName(), testProps.toURI()).add(log4jProps.getName(), log4jProps.toURI()).apply()
- .anyOrder().build();
+ .add(jarFile.getName(), jarFile.toURI(), false).add(testProps.getName(), testProps.toURI()).add(clientProps.getName(), clientProps.toURI())
+ .add(log4jProps.getName(), log4jProps.toURI()).apply().anyOrder().build();
}
}
@@ -92,6 +89,9 @@
@Parameter(names = {"--main", "-m"}, required = true, description = "Main class")
String mainClass;
+ @Parameter(names = {"--clientProps", "-c"}, required = true, description = "Accumulo client properties path")
+ String clientProps;
+
@Parameter(names = {"--testProps", "-p"}, required = true, description = "Test properties path")
String testProps;
@@ -108,15 +108,6 @@
Preconditions.checkState(f.canRead());
}
- private static int getNumRunning(TwillController controller) {
- ResourceReport report = controller.getResourceReport();
- if (report == null) {
- return 0;
- }
- Collection<TwillRunResources> resources = report.getRunnableResources(RUNNABLE_ID);
- return resources == null ? 0 : resources.size();
- }
-
public static void main(String[] args) throws Exception {
TestRunnerOpts opts = new TestRunnerOpts();
@@ -124,6 +115,7 @@
verifyPath(opts.jarPath);
verifyPath(opts.testProps);
+ verifyPath(opts.clientProps);
verifyPath(opts.logProps);
String jarFileName = Paths.get(opts.jarPath).getFileName().toString();
@@ -131,26 +123,21 @@
String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(jarFileName, "/lib",
opts.mainClass, mainArgs);
-
- Properties props = TestProps.loadFromFile(opts.testProps);
- TestEnv env = new TestEnv(props);
+ TestEnv env = new TestEnv(opts.testProps, opts.clientProps);
YarnConfiguration yarnConfig = new YarnConfiguration(env.getHadoopConfiguration());
-
- TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers());
+ TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getInfo().getZooKeepers());
twillRunner.start();
- TwillController controller = twillRunner.prepare(new YarnTestApp(opts, props))
- .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName()).withArguments("BundledJarRunnable", arguments.toArray()).start();
+ twillRunner.prepare(new YarnTestApp(opts, env.getTestProperties()))
+ .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
+ .withArguments("BundledJarRunnable", arguments.toArray()).start();
- int numRunning = getNumRunning(controller);
- while (numRunning != opts.numContainers) {
- LOG.info("{} of {} containers have started in YARN.", numRunning, opts.numContainers);
- Thread.sleep(5000);
- numRunning = getNumRunning(controller);
+ LOG.info("{} containers will start in YARN.", opts.numContainers);
+ LOG.info("Press Ctrl-C when these containers have started.");
+
+ while (true) {
+ Thread.sleep(1000);
}
-
- LOG.info("{} of {} containers have started in YARN", numRunning, opts.numContainers);
- LOG.info("{} application was successfully started in YARN", opts.testName);
}
}
