MD-5043 Added minio server to test S3 API in the drill-test-framework (#554)
diff --git a/.gitignore b/.gitignore
index d326281..2481071 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,9 @@
*.class
**/target/*
logs/*
+framework/resources/Datasources/s3/minio/minio_data/.minio.sys
+framework/resources/Datasources/s3/minio/minio_data/tstbucket/tmp/ppruning/
+framework/resources/Datasources/s3/minio/minio_data/tstbucket/tmp/gitignore
# Hive stuff
metastore_db
@@ -40,6 +43,7 @@
framework/resources/Datasources/ctas/decimal/fragments/decimal_big_zero_prec.tsv
framework/resources/Datasources/ctas/decimal/fragments/decimal_big_zero_scale.tsv
framework/resources/Datasources/ctas/decimal/fragments/negative/decimal_big_zero_scale.tsv
+framework/resources/Datasources/s3/minio/minio_data/tstbucket/tmp/big_files/
# Logs
drill.log*
diff --git a/conf/plugin-templates/common/s3-storage-plugin.template b/conf/plugin-templates/common/s3-storage-plugin.template
new file mode 100644
index 0000000..dc1155e
--- /dev/null
+++ b/conf/plugin-templates/common/s3-storage-plugin.template
@@ -0,0 +1,74 @@
+{
+ "name" : "s3",
+ "config" : {
+ "type": "file",
+ "enabled": true,
+ "connection": "s3a://tstbucket",
+ "config": {
+ "fs.s3a.endpoint": "http://localhost:9000",
+ "fs.s3a.connection.ssl.enabled": "false",
+ "fs.s3a.path.style.access": "true",
+ "fs.s3a.access.key": "ZYVR33VM12IAHETIJYF9",
+ "fs.s3a.secret.key": "BjGb9WpNd74t5T7FcBlXRgUcSyq505G598T8Yxg6",
+ "fs.s3a.impl.disable.cache": "false",
+ "fs.s3a.connection.maximum": "30"
+ },
+ "workspaces": {
+ "tmp": {
+ "location": "/tmp",
+ "writable": true,
+ "defaultInputFormat": null,
+ "allowAccessOutsideWorkspace": false
+ },
+ "root": {
+ "location": "/",
+ "writable": false,
+ "defaultInputFormat": null,
+ "allowAccessOutsideWorkspace": false
+ }
+ },
+ "formats": {
+ "csv": {
+ "type": "text",
+ "extensions": [
+ "csv"
+ ],
+ "delimiter": ","
+ },
+ "tsv": {
+ "type": "text",
+ "extensions": [
+ "tsv"
+ ],
+ "delimiter": "\t"
+ },
+ "parquet": {
+ "type": "parquet"
+ },
+ "json": {
+ "type": "json",
+ "extensions": [
+ "json"
+ ]
+ },
+ "avro": {
+ "type": "avro"
+ },
+ "csvh": {
+ "type": "text",
+ "extensions": [
+ "csvh"
+ ],
+ "extractHeader": true,
+ "delimiter": ","
+ },
+ "image": {
+ "type": "image",
+ "extensions": [
+ "jpg",
+ "png"
+ ]
+ }
+ }
+ }
+}
diff --git a/framework/resources/Datasources/s3/minio/minio b/framework/resources/Datasources/s3/minio/minio
new file mode 100755
index 0000000..d5f81f7
--- /dev/null
+++ b/framework/resources/Datasources/s3/minio/minio
Binary files differ
diff --git a/framework/resources/Datasources/s3/minio/run_mn.sh b/framework/resources/Datasources/s3/minio/run_mn.sh
new file mode 100755
index 0000000..d925814
--- /dev/null
+++ b/framework/resources/Datasources/s3/minio/run_mn.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+source conf/drillTestConfig.properties
+
+export MINIO_ACCESS_KEY=ZYVR33VM12IAHETIJYF9
+export MINIO_SECRET_KEY=BjGb9WpNd74t5T7FcBlXRgUcSyq505G598T8Yxg6
+
+if [ -z "$(top -bn1 |grep minio |grep -oE '^[ ]*[0-9]+' |grep -oE '[0-9]+')" ]
+then
+ nohup ${DRILL_TEST_DATA_DIR}/Datasources/s3/minio/minio server ${DRILL_TEST_DATA_DIR}/Datasources/s3/minio/minio_data >> ~/nohup.out &
+ sleep 1
+fi
diff --git a/framework/resources/Datasources/s3/minio/stop_mn.sh b/framework/resources/Datasources/s3/minio/stop_mn.sh
new file mode 100755
index 0000000..624bfca
--- /dev/null
+++ b/framework/resources/Datasources/s3/minio/stop_mn.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ ! -z "$(top -bn1 |grep minio |grep -oE '^[ ]*[0-9]+' |grep -oE '[0-9]+')" ]
+then
+ kill -9 $(top -bn1 |grep minio |grep -oE '^[ ]*[0-9]+' |grep -oE '[0-9]+')
+fi
diff --git a/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java b/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
index 9aa0495..39fcf28 100644
--- a/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
+++ b/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
@@ -693,6 +693,11 @@
}
}
connectionPool.releaseConnection(connection);
+
+ // Stop Apache Minio server if it was started for the S3 storage
+ if(cmdParam.excludeDependencies == null || !cmdParam.excludeDependencies.contains("s3")) {
+ Utils.stopMinio();
+ }
}
private void prepareData(List<DrillTestCase> tests) throws Exception {
@@ -750,6 +755,12 @@
}
final Stopwatch stopwatch = Stopwatch.createStarted();
+
+ // Run Apache Minio server if needed for the S3 storage
+ if(cmdParam.excludeDependencies == null || !cmdParam.excludeDependencies.contains("s3")) {
+ Utils.startMinio();
+ }
+
LOG.info("> Copying Data");
copyExecutor.executeAll(copyTasks);
copyExecutor.close();
diff --git a/framework/src/main/java/org/apache/drill/test/framework/Utils.java b/framework/src/main/java/org/apache/drill/test/framework/Utils.java
index c82adea..d08f24d 100755
--- a/framework/src/main/java/org/apache/drill/test/framework/Utils.java
+++ b/framework/src/main/java/org/apache/drill/test/framework/Utils.java
@@ -673,6 +673,33 @@
}
}
+ public static void startMinio() {
+ LOG.info("> Starting Apache Minio server\n");
+ String cmd = DrillTestDefaults.CWD + "/" + DrillTestDefaults.DRILL_TESTDATA_DIR + "/Datasources/s3/minio/run_mn.sh";
+ try {
+ Runtime.getRuntime().exec(cmd);
+ } catch (Throwable e) {
+ LOG.warn("Fail to run command " + cmd, e);
+ }
+ }
+
+ public static void stopMinio() {
+ LOG.info("> Stopping Apache Minio server\n");
+ String cmd = DrillTestDefaults.CWD + "/" + DrillTestDefaults.DRILL_TESTDATA_DIR + "/Datasources/s3/minio/stop_mn.sh";
+ try {
+ Runtime.getRuntime().exec(cmd);
+ } catch (Throwable e) {
+ LOG.warn("Fail to run command " + cmd, e);
+ }
+ LOG.info("> Disabling S3 storage plugin\n");
+ String templatePath = DrillTestDefaults.CWD + "/conf/plugin-templates/common/s3-storage-plugin.template";
+
+ boolean isSuccess = Utils.disableStoragePlugin(templatePath, "s3");
+ if(!isSuccess){
+ LOG.info(">> Fail to disable S3 storage plugin");
+ }
+ }
+
/**
* Updates storage plugin for drill
*
@@ -699,6 +726,26 @@
}
/**
+ * Turns off a storage plugin for drill
+ *
+ * @param filename
+ * name of file containing drill storage plugin
+ * @param pluginType
+ * type of plugin; e.g.: "dfs", "cp"
+ * @return true if operation is successful
+ */
+ private static boolean disableStoragePlugin(String filename, String pluginType) {
+ try {
+ String content = getFileContent(filename)
+ .replace("\"enabled\": true", "\"enabled\": false");
+ return postDrillStoragePlugin(content, pluginType);
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ return false;
+ }
+
+ /**
* Update drill storage plugin content
*
* @param content string containing drill storage plugin
