Refactored remaining test suites to use data preparation phases instead of scripts
Also updated README.md and CONTRIBUTING.md to reflect the changes
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 09634b4..7e0e4ea 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -85,6 +85,18 @@
}
</code></pre>
+The data described in "datasources" section is prepared in several stages. The stages are defined by "mode" field and executed in the following order:
+- **rm** - remove data specified in "dest" field
+- **cp** - copy data from "src" in local file system to "dest" in the target fs
+- **mkdir** - make directory specified in "dest"
+- **gen** - execute command specified in "src". In case of a file with .ddl extension it will be send to Drill through JDBC
+- **post_rm** - the same as "rm" but executed later
+- **post_cp** - the same as "cp" but executed later
+- **dfs_cp** - copy data between dfs locations
+- **ddl** - executes specified ddl file through JDBC
+
+Note that local paths are based on DRILL_TEST_DATA_DIR property (by default it is framework/resources), and all dfs paths are based on DRILL_TESTDATA property (by default it is /drill/testdata)
+
### Special handling for Drill version in regex based baseline verification
The Drill version in a regex based baseline file can be substituted with "{DRILL_VERSION}".
diff --git a/README.md b/README.md
index 9a7f2d9..01bcb38 100644
--- a/README.md
+++ b/README.md
@@ -36,11 +36,25 @@
If you've already downloaded the datasets previously, you can simply skip the download.
+If you're going to run tests in docker, you'll need to build the docker image first:
+
+```
+mvn clean install docker:build -DskipTests
+```
+
## Execute Tests
In the root directory of your repository, execute the following command to run tests:
-`bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
+```
+bin/run_tests -s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d`
+```
+
+Alternatively, you can run tests in docker:
+
+```
+mvn docker:start -Dtest.args="-s <suites> -g <groups> -t <Timeout> -x <Exclude> -n <Concurrency> -d"
+```
Example:
<pre><code> bin/run_tests -s Functional/aggregates,Functional/joins -g functional -x hbase -t 180 -n 20 -d
@@ -57,7 +71,8 @@
-n concurrency (optional)
Here, '20' queries can execute concurrently
-x exclude dependencies (optional)
- Here, any 'hbase' test suites within the specified directory are excluded
+ Here, any test suites within the specified directory that have specified dependencies are excluded.
+ In case of "-x all" all tests suites that have at least one dependency are excluded
-h help (optional)
Use this option to provide the usage of the command, which includes additional options
</code></pre>
diff --git a/framework/resources/Datasources/schema_change_empty_batch/json/setup.sh b/framework/resources/Datasources/schema_change_empty_batch/json/setup.sh
index 5fb583e..e1ea2ec 100755
--- a/framework/resources/Datasources/schema_change_empty_batch/json/setup.sh
+++ b/framework/resources/Datasources/schema_change_empty_batch/json/setup.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-source conf/drillTestConfig.properties
+DRILL_TEST_DATA_DIR=$1
set -x
set -e
@@ -39,21 +39,4 @@
fi
-if ! $(hadoop fs -test -d ${DRILL_TESTDATA}/schema_change_empty_batch/json)
- then
-
- echo "Copying to hadoop"
-
- hadoop fs -mkdir -p ${DRILL_TESTDATA}/schema_change_empty_batch/json
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/json/part ${DRILL_TESTDATA}/schema_change_empty_batch/json/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/json/partsupp ${DRILL_TESTDATA}/schema_change_empty_batch/json/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/json/empty ${DRILL_TESTDATA}/schema_change_empty_batch/json/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/json/part_small ${DRILL_TESTDATA}/schema_change_empty_batch/json/
-
-fi
-
set +x
diff --git a/framework/resources/Datasources/schema_change_empty_batch/text/dfs/setup.sh b/framework/resources/Datasources/schema_change_empty_batch/text/dfs/setup.sh
index 0836dfe..5fca382 100755
--- a/framework/resources/Datasources/schema_change_empty_batch/text/dfs/setup.sh
+++ b/framework/resources/Datasources/schema_change_empty_batch/text/dfs/setup.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-source conf/drillTestConfig.properties
+DRILL_TEST_DATA_DIR=$1
set -x
set -e
@@ -35,24 +35,7 @@
touch ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/partsupp/partsuppa{f..h}.tbl
- touch ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/empty/empty{a..d}.tbl
-
-fi
-
-if ! $(hadoop fs -test -d ${DRILL_TESTDATA}/schema_change_empty_batch/psv)
- then
-
- echo "Copying to hadoop"
-
- hadoop fs -mkdir -p ${DRILL_TESTDATA}/schema_change_empty_batch/psv
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/part ${DRILL_TESTDATA}/schema_change_empty_batch/psv/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/partsupp ${DRILL_TESTDATA}/schema_change_empty_batch/psv/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/empty ${DRILL_TESTDATA}/schema_change_empty_batch/psv/
-
- hadoop fs -put ${DRILL_TEST_DATA_DIR}/Datasources/schema_change_empty_batch/data/psv/json_field ${DRILL_TESTDATA}/schema_change_empty_batch/psv/
+ touch ${DRILL_TEST_DATA_DIR}/Datasources/spsv/json_field/empty_json_field.tblchema_change_empty_batch/data/psv/empty/empty{a..d}.tbl
fi
diff --git a/framework/resources/Datasources/table_function/DRILL-5166_generate_data.sh b/framework/resources/Datasources/table_function/DRILL-5166_generate_data.sh
index 901bd21..9f4d4de 100755
--- a/framework/resources/Datasources/table_function/DRILL-5166_generate_data.sh
+++ b/framework/resources/Datasources/table_function/DRILL-5166_generate_data.sh
@@ -1,8 +1,7 @@
#!/bin/bash
-source conf/drillTestConfig.properties
+DRILL_TEST_DATA_DIR=$1
test_dir=${DRILL_TEST_DATA_DIR}/Datasources/table_function
-dfs_location=${DRILL_TESTDATA}/table_function
untar_data () {
local location=$1
@@ -12,37 +11,21 @@
tar xzvf $location/$tar_name -C $location ./$file_name
}
-copy_to_dfs () {
- local location=$1
- local file_name=$2
- local file=$location/$file_name
- local destination=$3
-
- hadoop fs -test -f $destination/$file_name
- if [ $? -eq 0 ]
- then
- hadoop fs -rm -r $destination/$file_name
- fi
- hadoop fs -put $file $destination/
-}
-
prepare_dataset () {
local location=$1
local file_name=$2
local file=$test_dir/$file_name
local tar_name=$3
- local destination=$4
# Reusing of existing file if exists
if [ ! -f $file ]
then
untar_data $location $tar_name $file_name
- copy_to_dfs $location $file_name $destination
fi
}
base_file_name="DRILL-5166_NPE_with_table_function"
tar_name="DRILL-5166_data.tar.gz"
-prepare_dataset $test_dir ${base_file_name}_group_select_1.csv $tar_name $dfs_location
-prepare_dataset $test_dir ${base_file_name}_group_select_2.csv $tar_name $dfs_location
-prepare_dataset $test_dir ${base_file_name}_large_file.csv $tar_name $dfs_location
+prepare_dataset $test_dir ${base_file_name}_group_select_1.csv $tar_name
+prepare_dataset $test_dir ${base_file_name}_group_select_2.csv $tar_name
+prepare_dataset $test_dir ${base_file_name}_large_file.csv $tar_name
diff --git a/framework/resources/Datasources/table_stats/analyze_tables.sql b/framework/resources/Datasources/table_stats/analyze_tables.sql
index bf1c723..3079f3a 100755
--- a/framework/resources/Datasources/table_stats/analyze_tables.sql
+++ b/framework/resources/Datasources/table_stats/analyze_tables.sql
@@ -1,4 +1,5 @@
set `store.parquet.reader.int96_as_timestamp`=true;
+use dfs.drillTestDir;
analyze table `table_stats/alltypes_with_nulls` compute statistics;
analyze table `table_stats/date_intervals` compute statistics;
analyze table `table_stats/impala_parquet` compute statistics;
diff --git a/framework/resources/Functional/complex/parquet/complex.json b/framework/resources/Functional/complex/parquet/complex.json
index 1cd5b1d..b734509 100644
--- a/framework/resources/Functional/complex/parquet/complex.json
+++ b/framework/resources/Functional/complex/parquet/complex.json
@@ -19,7 +19,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "complex/parquet/complex.json"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/csv/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/csv/data/ctas_auto_partition.json
index 03c22d5..0f7f7be 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/csv/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/csv/data/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/data/ctas_auto_partition.json
index 4646d36..a59220a 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/data/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/plan/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/plan/ctas_auto_partition.json
index e447f4b..e473772 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/plan/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/plan/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/json/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/json/data/ctas_auto_partition.json
index 8a405ea..e2bca86 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/json/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/json/data/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/data/ctas_auto_partition.json
index e5f3808..876c97b 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/data/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/plan/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/plan/ctas_auto_partition.json
index 745f684..c6db2d5 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/plan/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/plan/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/general/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/general/data/ctas_auto_partition.json
index 5824cd7..094cdd2 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/general/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/general/data/ctas_auto_partition.json
@@ -33,12 +33,12 @@
"dest": "/drill/testdata/drill-3947"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_single_partition1"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/general/plan/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/general/plan/ctas_auto_partition.json
index 27ae033..e592c61 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/general/plan/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/general/plan/ctas_auto_partition.json
@@ -33,12 +33,12 @@
"dest": "/drill/testdata/drill-3947"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_single_partition1"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/data/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/data/ctas_auto_partition.json
index 99232a5..2594f95 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/data/ctas_auto_partition.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_multiple_partitions"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/plan/ctas_auto_partition.json b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/plan/ctas_auto_partition.json
index 3a625e4..4bdcbac 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/plan/ctas_auto_partition.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/plan/ctas_auto_partition.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_multiple_partitions"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition/tpch.json b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition/tpch.json
index d9bd966..b7dfff6 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition/tpch.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition/tpch.json
@@ -58,7 +58,7 @@
"dest": "Tpch0.01/parquet/orders/orders.parquet"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_single_partition"
},
diff --git a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition1/tpch.json b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition1/tpch.json
index bb7febf..d38d95e 100644
--- a/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition1/tpch.json
+++ b/framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition1/tpch.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/tpch_single_partition1"
},
diff --git a/framework/resources/Functional/ctas/ctas_flatten/100000rows/ctas_flatten.json b/framework/resources/Functional/ctas/ctas_flatten/100000rows/ctas_flatten.json
index 36b59d0..723da34 100644
--- a/framework/resources/Functional/ctas/ctas_flatten/100000rows/ctas_flatten.json
+++ b/framework/resources/Functional/ctas/ctas_flatten/100000rows/ctas_flatten.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_flatten"
},
diff --git a/framework/resources/Functional/ctas/ctas_flatten/2rows/ctas_flatten.json b/framework/resources/Functional/ctas/ctas_flatten/2rows/ctas_flatten.json
index 30e2ab5..63d5cb8 100644
--- a/framework/resources/Functional/ctas/ctas_flatten/2rows/ctas_flatten.json
+++ b/framework/resources/Functional/ctas/ctas_flatten/2rows/ctas_flatten.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_flatten"
},
diff --git a/framework/resources/Functional/ctas/ctas_joins_aggregates/ctas.json b/framework/resources/Functional/ctas/ctas_joins_aggregates/ctas.json
index 8775bfa..fe77ca9 100644
--- a/framework/resources/Functional/ctas/ctas_joins_aggregates/ctas.json
+++ b/framework/resources/Functional/ctas/ctas_joins_aggregates/ctas.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas/parquet"
},
diff --git a/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal.json b/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal.json
index 86af4f9..184307b 100644
--- a/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal.json
+++ b/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "filter/pushdown/varchar_decimal"
},
diff --git a/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal_plan.json b/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal_plan.json
index 520e511..7db8988 100644
--- a/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal_plan.json
+++ b/framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal_plan.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "filter/pushdown/varchar_decimal"
},
diff --git a/framework/resources/Functional/int96/int96_data.json b/framework/resources/Functional/int96/int96_data.json
index 48e2ef4..69df6c6 100644
--- a/framework/resources/Functional/int96/int96_data.json
+++ b/framework/resources/Functional/int96/int96_data.json
@@ -21,7 +21,7 @@
"dest": "subqueries"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/hive1_parquet_part"
},
diff --git a/framework/resources/Functional/int96/int96_plan.json b/framework/resources/Functional/int96/int96_plan.json
index 35112c2..6b8813a 100644
--- a/framework/resources/Functional/int96/int96_plan.json
+++ b/framework/resources/Functional/int96/int96_plan.json
@@ -21,7 +21,7 @@
"dest": "subqueries"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/hive1_parquet_part"
},
diff --git a/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/data/ctas_auto_partition.json b/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/data/ctas_auto_partition.json
index 6c8cd3e..8742ad9 100644
--- a/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/data/ctas_auto_partition.json
+++ b/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/data/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/plan/ctas_auto_partition.json b/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/plan/ctas_auto_partition.json
index fcc7fc1..b53b5fd 100644
--- a/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/plan/ctas_auto_partition.json
+++ b/framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/plan/ctas_auto_partition.json
@@ -28,7 +28,7 @@
"dest": "partition_pruning/hive"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "ctas_auto_partition/existing_partition_pruning"
},
diff --git a/framework/resources/Functional/limit0/functions/data/limit0_functions.json b/framework/resources/Functional/limit0/functions/data/limit0_functions.json
index 7c55072..290ecee 100644
--- a/framework/resources/Functional/limit0/functions/data/limit0_functions.json
+++ b/framework/resources/Functional/limit0/functions/data/limit0_functions.json
@@ -21,12 +21,12 @@
"dest": "subqueries"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/optional_type_v.view.drill"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/required_type_v.view.drill"
},
diff --git a/framework/resources/Functional/limit0/functions/plan/limit0_functions.json b/framework/resources/Functional/limit0/functions/plan/limit0_functions.json
index 03e2634..a7b0367 100644
--- a/framework/resources/Functional/limit0/functions/plan/limit0_functions.json
+++ b/framework/resources/Functional/limit0/functions/plan/limit0_functions.json
@@ -21,12 +21,12 @@
"dest": "subqueries"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/optional_type_v.view.drill"
},
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "subqueries/required_type_v.view.drill"
},
diff --git a/framework/resources/Functional/metadata_caching/data/metadata_caching_small.json b/framework/resources/Functional/metadata_caching/data/metadata_caching_small.json
index 6f833cf..475f5f6 100644
--- a/framework/resources/Functional/metadata_caching/data/metadata_caching_small.json
+++ b/framework/resources/Functional/metadata_caching/data/metadata_caching_small.json
@@ -1,51 +1,157 @@
{
- "testId": "Metadata_Caching_Small_Data",
- "type": "group",
- "description": "Metadata Caching Tests with smaller data sets",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs.metadata_caching",
- "output-format": "tsv",
- "expected-file": ".*.e_tsv",
- "verification-type": [
- "in-memory"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/metadata_caching/data/",
- "dest": "metadata_caching/"
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/nation",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/addremove_files.sh",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_toplevel_cache.sh",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_toplevel_cache.sh",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/refresh_metadata_multilevel.sh",
- "dest": ""
- }
- ]
+ "testId": "Metadata_Caching_Small_Data",
+ "type": "group",
+ "description": "Metadata Caching Tests with smaller data sets",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs.metadata_caching",
+ "output-format": "tsv",
+ "expected-file": ".*.e_tsv",
+ "verification-type": [
+ "in-memory"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/nation"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addfiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_adddir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/orders"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/metadata_caching/data/",
+ "dest": "metadata_caching/"
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_addremovefiles.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_deletecache.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_multilevel.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir/feb"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files/lineitem2.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/generated_caches"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_deletecache/.drill.parquet_metadata"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem1.parquet",
+ "dest": "metadata_caching/lineitem_addfiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/feb",
+ "dest": "metadata_caching/lineitem_adddir/"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem2.parquet",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files/"
+ },
+
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_summary_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ }
+ ]
}
diff --git a/framework/resources/Functional/metadata_caching/generated_caches/metadata_caching_small.json b/framework/resources/Functional/metadata_caching/generated_caches/metadata_caching_small.json
index 13c9e60..7d17667 100644
--- a/framework/resources/Functional/metadata_caching/generated_caches/metadata_caching_small.json
+++ b/framework/resources/Functional/metadata_caching/generated_caches/metadata_caching_small.json
@@ -1,41 +1,152 @@
{
- "testId": "Metadata_Caching_GeneratedCaches",
- "type": "group",
- "description": "Metadata Caching Tests with smaller data sets",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs.metadata_caching",
- "output-format": "tsv",
- "expected-file": ".*.e_tsv",
- "verification-type": [
- "regex"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/metadata_caching/data/",
- "dest": "metadata_caching/"
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/nation",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/addremove_files.sh",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_toplevel_cache.sh",
- "dest": ""
- }
- ]
+ "testId": "Metadata_Caching_GeneratedCaches",
+ "type": "group",
+ "description": "Metadata Caching Tests with smaller data sets",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs.metadata_caching",
+ "output-format": "tsv",
+ "expected-file": ".*.e_tsv",
+ "verification-type": [
+ "regex"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/nation"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addfiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_adddir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/orders"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/metadata_caching/data/",
+ "dest": "metadata_caching/"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/metadata_caching/refresh_metadata_addremovefiles.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/metadata_caching/refresh_metadata_deletecache.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir/feb"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files/lineitem2.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/generated_caches"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_deletecache/.drill.parquet_metadata"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem1.parquet",
+ "dest": "metadata_caching/lineitem_addfiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/feb",
+ "dest": "metadata_caching/lineitem_adddir/"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem2.parquet",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files/"
+ },
+
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_summary_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ }
+ ]
}
diff --git a/framework/resources/Functional/metadata_caching/partition_pruning/data/metadata_caching_pp.json b/framework/resources/Functional/metadata_caching/partition_pruning/data/metadata_caching_pp.json
index 2002e27..3c8a044 100644
--- a/framework/resources/Functional/metadata_caching/partition_pruning/data/metadata_caching_pp.json
+++ b/framework/resources/Functional/metadata_caching/partition_pruning/data/metadata_caching_pp.json
@@ -1,31 +1,31 @@
{
- "testId": "Metadata_Caching_Small_PP_data",
- "type": "group",
- "description": "Metadata Caching Tests with smaller data sets",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs.metadata_caching_pp",
- "output-format": "tsv",
- "expected-file": ".*.e",
- "verification-type": [
- "in-memory"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/metadata_caching/data_pp/",
- "dest": "metadata_caching_pp/"
- } ,
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/refresh_metadata_multilevel.sh",
- "dest": ""
- }
- ]
+ "testId": "Metadata_Caching_Small_PP_data",
+ "type": "group",
+ "description": "Metadata Caching Tests with smaller data sets",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs.metadata_caching_pp",
+ "output-format": "tsv",
+ "expected-file": ".*.e",
+ "verification-type": [
+ "in-memory"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "cp",
+ "src": "Datasources/metadata_caching/data_pp/",
+ "dest": "metadata_caching_pp/"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/metadata_caching/refresh_metadata_multilevel.ddl",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/metadata_caching/partition_pruning/plan/metadata_caching_pp.json b/framework/resources/Functional/metadata_caching/partition_pruning/plan/metadata_caching_pp.json
index b238504..ce00416 100644
--- a/framework/resources/Functional/metadata_caching/partition_pruning/plan/metadata_caching_pp.json
+++ b/framework/resources/Functional/metadata_caching/partition_pruning/plan/metadata_caching_pp.json
@@ -1,31 +1,31 @@
{
- "testId": "Metadata_Caching_Small_PP",
- "type": "group",
- "description": "Metadata Caching Tests with smaller data sets",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs.metadata_caching_pp",
- "output-format": "tsv",
- "expected-file": ".*.e",
- "verification-type": [
- "regex"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/metadata_caching/data_pp/",
- "dest": "metadata_caching_pp/"
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/refresh_metadata_multilevel.sh",
- "dest": ""
- }
- ]
+ "testId": "Metadata_Caching_Small_PP",
+ "type": "group",
+ "description": "Metadata Caching Tests with smaller data sets",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs.metadata_caching_pp",
+ "output-format": "tsv",
+ "expected-file": ".*.e",
+ "verification-type": [
+ "regex"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "cp",
+ "src": "Datasources/metadata_caching/data_pp/",
+ "dest": "metadata_caching_pp/"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/metadata_caching/refresh_metadata_multilevel.ddl",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/metadata_caching/plan/metadata_caching_small.json b/framework/resources/Functional/metadata_caching/plan/metadata_caching_small.json
index e274eae..c036e85 100644
--- a/framework/resources/Functional/metadata_caching/plan/metadata_caching_small.json
+++ b/framework/resources/Functional/metadata_caching/plan/metadata_caching_small.json
@@ -3,7 +3,7 @@
"type": "group",
"description": "Metadata Caching Tests with smaller data sets",
"categories": [
- "excluded"
+ "functional"
],
"matrices": [
{
@@ -17,25 +17,141 @@
}
],
"datasources": [
- {
- "mode": "cp",
- "src": "Datasources/metadata_caching/data/",
- "dest": "metadata_caching/"
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/delete_cache.sh /drill/testdata/metadata_caching/nation",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/addremove_files.sh",
- "dest": ""
- },
- {
- "mode": "gen",
- "src": "Datasources/metadata_caching/refresh_metadata_multilevel.sh",
- "dest": ""
- }
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/nation"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addfiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_adddir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "metadata_caching/orders"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/metadata_caching/data/",
+ "dest": "metadata_caching/"
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_addremovefiles.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_deletecache.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "gen",
+ "src": "Datasources/metadata_caching/refresh_metadata_multilevel.ddl",
+ "dest": ""
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removefiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removedir/feb"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_removeautopartitioned_files/lineitem2.parquet"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/generated_caches"
+ },
+ {
+ "mode": "post_rm",
+ "src": "",
+ "dest": "metadata_caching/lineitem_deletecache/.drill.parquet_metadata"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem1.parquet",
+ "dest": "metadata_caching/lineitem_addfiles/lineitem1.parquet"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/feb",
+ "dest": "metadata_caching/lineitem_adddir/"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/metadata_caching/data/lineitem2.parquet",
+ "dest": "metadata_caching/lineitem_addautopartitioned_files/"
+ },
+
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_summary_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ },
+ {
+ "mode": "dfs_cp",
+ "src": "metadata_caching/fewtypes/.drill.parquet_file_metadata.v4",
+ "dest": "metadata_caching/generated_caches/fewtypes_cache/parquet_file_metadata.json"
+ }
]
}
diff --git a/framework/resources/Functional/min_max_dir/min_max_dir.json b/framework/resources/Functional/min_max_dir/min_max_dir.json
index e9dd098..cd23d55 100644
--- a/framework/resources/Functional/min_max_dir/min_max_dir.json
+++ b/framework/resources/Functional/min_max_dir/min_max_dir.json
@@ -18,7 +18,7 @@
],
"datasources": [
{
- "mode": "del",
+ "mode": "rm",
"src": "",
"dest": "min_max_dir_metadatacache"
},
diff --git a/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/data/parquet_date.json b/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/data/parquet_date.json
index 2b9f87f..7b66d45 100644
--- a/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/data/parquet_date.json
+++ b/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/data/parquet_date.json
@@ -1,36 +1,66 @@
{
- "testId": "mcAutoPartitionParquetDate",
- "type": "group",
- "description": "Test drill's parquet date compatibility",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs",
- "output-format": "tsv",
- "expected-file": ".*.e",
- "verification-type": [
- "in-memory"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "mc_parquet_date"
- },
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "parquet_date"
- },
- {
- "mode": "gen",
- "src": "Datasources/parquet_date/gen.sh",
- "dest": ""
- }
- ]
+ "testId": "mcAutoPartitionParquetDate",
+ "type": "group",
+ "description": "Test drill's parquet date compatibility",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs",
+ "output-format": "tsv",
+ "expected-file": ".*.e",
+ "verification-type": [
+ "in-memory"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/metadata_cache"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/auto_partition"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "mc_parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.2",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.2_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.6",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.6_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_multipart",
+ "dest": "parquet_date/auto_partition/item_multipart_autorefresh"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_single/1.2",
+ "dest": "parquet_date/auto_partition/item_single_1.2"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/parquet_date/metadata_cache.ddl",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/plan/parquet_date.json b/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/plan/parquet_date.json
index 57b3332..59a2118 100644
--- a/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/plan/parquet_date.json
+++ b/framework/resources/Functional/parquet_storage/parquet_date/auto_partition/plan/parquet_date.json
@@ -1,36 +1,66 @@
{
- "testId": "mcAutoPartitionParquetDate_Plan",
- "type": "group",
- "description": "Test drill's parquet date compatibility",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs",
- "output-format": "tsv",
- "expected-file": ".*.e",
- "verification-type": [
- "regex"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "mc_parquet_date"
- },
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "parquet_date"
- },
- {
- "mode": "gen",
- "src": "Datasources/parquet_date/gen.sh",
- "dest": ""
- }
- ]
+ "testId": "mcAutoPartitionParquetDate_Plan",
+ "type": "group",
+ "description": "Test drill's parquet date compatibility",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs",
+ "output-format": "tsv",
+ "expected-file": ".*.e",
+ "verification-type": [
+ "regex"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/metadata_cache"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/auto_partition"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "mc_parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.2",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.2_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.6",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.6_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_multipart",
+ "dest": "parquet_date/auto_partition/item_multipart_autorefresh"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_single/1.2",
+ "dest": "parquet_date/auto_partition/item_single_1.2"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/parquet_date/metadata_cache.ddl",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/parquet_storage/parquet_date/mc_parquet_date/parquet_date.json b/framework/resources/Functional/parquet_storage/parquet_date/mc_parquet_date/parquet_date.json
index 8fd85f2..f2fa89a 100644
--- a/framework/resources/Functional/parquet_storage/parquet_date/mc_parquet_date/parquet_date.json
+++ b/framework/resources/Functional/parquet_storage/parquet_date/mc_parquet_date/parquet_date.json
@@ -1,36 +1,66 @@
{
- "testId": "mcParquetDate",
- "type": "group",
- "description": "Test drill's parquet date compatibility",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.q",
- "schema": "dfs",
- "output-format": "tsv",
- "expected-file": ".*.e",
- "verification-type": [
- "in-memory"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "mc_parquet_date"
- },
- {
- "mode": "cp",
- "src": "Datasources/parquet_date",
- "dest": "parquet_date"
- },
- {
- "mode": "gen",
- "src": "Datasources/parquet_date/gen.sh",
- "dest": ""
- }
- ]
+ "testId": "mcParquetDate",
+ "type": "group",
+ "description": "Test drill's parquet date compatibility",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.q",
+ "schema": "dfs",
+ "output-format": "tsv",
+ "expected-file": ".*.e",
+ "verification-type": [
+ "in-memory"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/metadata_cache"
+ },
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "parquet_date/auto_partition"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "mc_parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date",
+ "dest": "parquet_date"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.2",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.2_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/metadata_cache/metadata_cache1.6",
+ "dest": "parquet_date/metadata_cache/metadata_cache1.6_autogen"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_multipart",
+ "dest": "parquet_date/auto_partition/item_multipart_autorefresh"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/parquet_date/auto_partition/item_single/1.2",
+ "dest": "parquet_date/auto_partition/item_single_1.2"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/parquet_date/metadata_cache.ddl",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/schema_change_empty_batch/json/empty_batch_json.json b/framework/resources/Functional/schema_change_empty_batch/json/empty_batch_json.json
index fb5e3ad..a3e8d16 100644
--- a/framework/resources/Functional/schema_change_empty_batch/json/empty_batch_json.json
+++ b/framework/resources/Functional/schema_change_empty_batch/json/empty_batch_json.json
@@ -4,7 +4,7 @@
"description": "Test drill queries involving empty batches for json files",
"submit-type": "jdbc",
"categories": [
- "excluded"
+ "functional"
],
"matrices": [
{
@@ -20,8 +20,28 @@
"datasources" : [
{
"mode": "gen",
- "src": "Datasources/schema_change_empty_batch/json/setup.sh",
+ "src": "Datasources/schema_change_empty_batch/json/setup.sh $DRILL_TESTDATA_DIR",
"dest": ""
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/json/part",
+ "dest": "schema_change_empty_batch/json/part"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/json/partsupp",
+ "dest": "schema_change_empty_batch/json/partsupp"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/json/empty",
+ "dest": "schema_change_empty_batch/json/empty"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/json/part_small",
+ "dest": "schema_change_empty_batch/json/part_small"
}
]
}
diff --git a/framework/resources/Functional/schema_change_empty_batch/text/dfs/empty_batch_text_dfs.json b/framework/resources/Functional/schema_change_empty_batch/text/dfs/empty_batch_text_dfs.json
index 1852c8a..d77bdfd 100644
--- a/framework/resources/Functional/schema_change_empty_batch/text/dfs/empty_batch_text_dfs.json
+++ b/framework/resources/Functional/schema_change_empty_batch/text/dfs/empty_batch_text_dfs.json
@@ -20,8 +20,28 @@
"datasources" : [
{
"mode": "gen",
- "src": "Datasources/schema_change_empty_batch/text/dfs/setup.sh",
+ "src": "Datasources/schema_change_empty_batch/text/dfs/setup.sh $DRILL_TESTDATA_DIR",
"dest": ""
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/psv/part",
+ "dest": "schema_change_empty_batch/psv/part"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/psv/partsupp",
+ "dest": "schema_change_empty_batch/psv/partsupp"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/psv/empty",
+ "dest": "schema_change_empty_batch/psv/empty"
+ },
+ {
+ "mode": "post_cp",
+ "src": "Datasources/schema_change_empty_batch/data/psv/json_field",
+ "dest": "schema_change_empty_batch/psv/json_field"
}
]
}
diff --git a/framework/resources/Functional/table_function/positive/data/table_function.json b/framework/resources/Functional/table_function/positive/data/table_function.json
index 12c35a0..802b0d4 100644
--- a/framework/resources/Functional/table_function/positive/data/table_function.json
+++ b/framework/resources/Functional/table_function/positive/data/table_function.json
@@ -3,7 +3,7 @@
"type": "group",
"description": "Test table function queries via jdbc",
"categories": [
- "excluded"
+ "functional"
],
"matrices": [
{
@@ -18,13 +18,13 @@
],
"datasources": [
{
- "mode": "cp",
+ "mode": "post_cp",
"src": "Datasources/table_function",
"dest": "table_function/"
},
{
"mode": "gen",
- "src": "Datasources/table_function/DRILL-5166_generate_data.sh",
+ "src": "Datasources/table_function/DRILL-5166_generate_data.sh $DRILL_TESTDATA_DIR",
"dest": ""
}
]
diff --git a/framework/resources/Functional/table_function/positive/plan/table_function_plan_check.json b/framework/resources/Functional/table_function/positive/plan/table_function_plan_check.json
index 46a628b..cfd242b 100644
--- a/framework/resources/Functional/table_function/positive/plan/table_function_plan_check.json
+++ b/framework/resources/Functional/table_function/positive/plan/table_function_plan_check.json
@@ -18,13 +18,13 @@
],
"datasources": [
{
- "mode": "cp",
+ "mode": "post_cp",
"src": "Datasources/table_function",
"dest": "table_function/"
},
{
"mode": "gen",
- "src": "Datasources/table_function/DRILL-5166_generate_data.sh",
+ "src": "Datasources/table_function/DRILL-5166_generate_data.sh $DRILL_TESTDATA_DIR",
"dest": ""
}
]
diff --git a/framework/resources/Functional/table_stats/stats/positive/stats.json b/framework/resources/Functional/table_stats/stats/positive/stats.json
index 4c18510..72064df 100644
--- a/framework/resources/Functional/table_stats/stats/positive/stats.json
+++ b/framework/resources/Functional/table_stats/stats/positive/stats.json
@@ -1,31 +1,36 @@
{
- "testId": "Stats_positive_tests",
- "type": "group",
- "description": "Test queries using stats",
- "categories": [
- "excluded"
- ],
- "matrices": [
- {
- "query-file": ".*.sql",
- "schema": "dfs.drillTestDir",
- "output-format": "tsv",
- "expected-file": ".*.plan",
- "verification-type": [
- "regex"
- ]
- }
- ],
- "datasources": [
- {
- "mode": "cp",
- "src": "Datasources/table_stats/tables",
- "dest": "table_stats"
- },
- {
- "mode": "gen",
- "src": "Datasources/table_stats/table_stats.sh",
- "dest": ""
- }
- ]
+ "testId": "Stats_positive_tests",
+ "type": "group",
+ "description": "Test queries using stats",
+ "categories": [
+ "functional"
+ ],
+ "matrices": [
+ {
+ "query-file": ".*.sql",
+ "schema": "dfs.drillTestDir",
+ "output-format": "tsv",
+ "expected-file": ".*.plan",
+ "verification-type": [
+ "regex"
+ ]
+ }
+ ],
+ "datasources": [
+ {
+ "mode": "rm",
+ "src": "",
+ "dest": "table_stats"
+ },
+ {
+ "mode": "cp",
+ "src": "Datasources/table_stats/tables",
+ "dest": "table_stats"
+ },
+ {
+ "mode": "ddl",
+ "src": "Datasources/table_stats/analyze_tables.sql",
+ "dest": ""
+ }
+ ]
}
diff --git a/framework/resources/Functional/table_stats/stats/query/query.json b/framework/resources/Functional/table_stats/stats/query/query.json
index 977ef17..58312a9 100644
--- a/framework/resources/Functional/table_stats/stats/query/query.json
+++ b/framework/resources/Functional/table_stats/stats/query/query.json
@@ -3,7 +3,7 @@
"type": "group",
"description": "Test queries using stats",
"categories": [
- "excluded"
+ "functional"
],
"matrices": [
{
@@ -18,13 +18,18 @@
],
"datasources": [
{
+ "mode": "rm",
+ "src": "",
+ "dest": "table_stats"
+ },
+ {
"mode": "cp",
"src": "Datasources/table_stats/tables",
"dest": "table_stats"
},
{
- "mode": "gen",
- "src": "Datasources/table_stats/table_stats.sh",
+ "mode": "ddl",
+ "src": "Datasources/table_stats/analyze_tables.sql",
"dest": ""
}
]
diff --git a/framework/resources/Functional/text_storage/testcases/textReadGroup.json b/framework/resources/Functional/text_storage/testcases/textReadGroup.json
index 73e7b58..0523c61 100644
--- a/framework/resources/Functional/text_storage/testcases/textReadGroup.json
+++ b/framework/resources/Functional/text_storage/testcases/textReadGroup.json
@@ -3,7 +3,7 @@
"type": "group",
"description": "Test text read queries",
"categories": [
- "exclude"
+ "functional"
],
"matrices": [
{
@@ -18,14 +18,39 @@
],
"datasources": [
{
+ "mode": "rm",
+ "src": "",
+ "dest": "text_storage/drill-4484"
+ },
+ {
+ "mode": "mkdir",
+ "src": "",
+ "dest": "text_storage/drill-4484/20160401/3"
+ },
+ {
+ "mode": "mkdir",
+ "src": "",
+ "dest": "text_storage/drill-4484/20160404"
+ },
+ {
+ "mode": "mkdir",
+ "src": "",
+ "dest": "text_storage/drill-4484/20160501/1"
+ },
+ {
+ "mode": "mkdir",
+ "src": "",
+ "dest": "text_storage/drill-4484/20160501/2"
+ },
+ {
+ "mode": "mkdir",
+ "src": "",
+ "dest": "text_storage/drill-4484/20160502"
+ },
+ {
"mode": "cp",
"src": "Datasources/text_storage",
"dest": "text_storage"
- },
- {
- "mode": "gen",
- "src": "Datasources/text_storage/drill-4484.sh",
- "dest": ""
}
]
}
diff --git a/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java b/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
index cb22d55..c7998ba 100644
--- a/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
+++ b/framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
@@ -741,16 +741,18 @@
boolean restartDrillbits = false;
- List<Cancelable> delTasks = new ArrayList<>();
+ List<Cancelable> rmTasks = new ArrayList<>();
List<Cancelable> copyTasks = new ArrayList<>();
List<Cancelable> mkdirTasks = new ArrayList<>();
List<Cancelable> genTasks = new ArrayList<>();
+ List<Cancelable> postRmTasks = new ArrayList<>();
List<Cancelable> postCopyTasks = new ArrayList<>();
+ List<Cancelable> dfsCopyTasks = new ArrayList<>();
List<Cancelable> ddlTasks = new ArrayList<>();
for (final TestCaseModeler.DataSource datasource : dataSources) {
String mode = datasource.mode;
switch (mode) {
- case "del": {
+ case "rm": {
Cancelable task = new Cancelable() {
@Override
public void cancel() {
@@ -767,7 +769,7 @@
}
}
};
- delTasks.add(task);
+ rmTasks.add(task);
break;
}
case "cp": {
@@ -826,6 +828,26 @@
genTasks.add(task);
break;
}
+ case "post_rm": {
+ Cancelable task = new Cancelable() {
+ @Override
+ public void cancel() {
+ // no op, as this will not time out
+ }
+
+ @Override
+ public void run() {
+ try {
+ Path dest = new Path(DrillTestDefaults.DRILL_TESTDATA, datasource.dest);
+ dfsDelete(dest, DrillTestDefaults.FS_MODE);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ postRmTasks.add(task);
+ break;
+ }
case "post_cp": {
Cancelable task = new Cancelable() {
@Override
@@ -847,6 +869,27 @@
postCopyTasks.add(task);
break;
}
+ case "dfs_cp": {
+ Cancelable task = new Cancelable() {
+ @Override
+ public void cancel() {
+ // no op, as this will not time out
+ }
+
+ @Override
+ public void run() {
+ try {
+ Path src = new Path(DrillTestDefaults.DRILL_TESTDATA, datasource.src);
+ Path dest = new Path(DrillTestDefaults.DRILL_TESTDATA, datasource.dest);
+ dfsToDfsCopy(src, dest, DrillTestDefaults.FS_MODE);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ dfsCopyTasks.add(task);
+ break;
+ }
case "ddl": {
Cancelable task = new Cancelable() {
@Override
@@ -872,21 +915,41 @@
final Stopwatch stopwatch = Stopwatch.createStarted();
try (CancelingExecutor executor = new CancelingExecutor(cmdParam.threads, Integer.MAX_VALUE)) {
- LOG.info("> Clearing data");
- executor.executeAll(delTasks);
- LOG.info("> Copying Data");
- executor.executeAll(copyTasks);
- LOG.info("> Making directories");
- executor.executeAll(mkdirTasks);
- LOG.info(">> Copy duration: " + stopwatch + "\n");
- stopwatch.reset().start();
- LOG.info("> Generating Data");
- executor.executeAll(genTasks);
- LOG.info("> Copying generated Data");
- executor.executeAll(postCopyTasks);
- LOG.info("> Executing DDL scripts");
- executor.executeAll(ddlTasks);
- LOG.info("\n>> Generation duration: " + stopwatch + "\n");
+ if (!rmTasks.isEmpty()) {
+ LOG.info("> Clearing Data");
+ executor.executeAll(rmTasks);
+ }
+ if (!copyTasks.isEmpty()) {
+ LOG.info("> Copying Data");
+ executor.executeAll(copyTasks);
+ LOG.info(">> Copy duration: " + stopwatch + "\n");
+ stopwatch.reset().start();
+ }
+ if (!mkdirTasks.isEmpty()) {
+ LOG.info("> Making directories");
+ executor.executeAll(mkdirTasks);
+ }
+ if (!genTasks.isEmpty()) {
+ LOG.info("> Generating Data");
+ executor.executeAll(genTasks);
+ LOG.info("\n>> Generation duration: " + stopwatch + "\n");
+ }
+ if (!rmTasks.isEmpty()) {
+ LOG.info("> Clearing Data after generating");
+ executor.executeAll(postRmTasks);
+ }
+ if (!postCopyTasks.isEmpty()) {
+ LOG.info("> Copying generated Data");
+ executor.executeAll(postCopyTasks);
+ }
+ if (!dfsCopyTasks.isEmpty()) {
+ LOG.info("> Rearranging Data on DFS");
+ executor.executeAll(dfsCopyTasks);
+ }
+ if (!ddlTasks.isEmpty()) {
+ LOG.info("> Executing DDL scripts");
+ executor.executeAll(ddlTasks);
+ }
}
if (restartDrillbits) {
@@ -938,8 +1001,40 @@
LOG.debug("File " + src + " already exists as " + dest);
}
}
- } catch (FileAlreadyExistsException e) {
+ } catch (IOException e) {
LOG.debug("File " + src + " already exists as " + dest);
+ }
+ }
+
+ private void dfsToDfsCopy(Path src, Path dest, String fsMode)
+ throws IOException {
+
+ FileSystem fs;
+
+ if (fsMode.equals("distributedFS")) {
+ fs = FileSystem.get(conf);
+ } else {
+ fs = FileSystem.getLocal(conf);
+ }
+
+ try {
+ if (fs.getFileStatus(src).isDirectory()) {
+ for (FileStatus file : fs.listStatus(src)) {
+ Path srcChild = file.getPath();
+ Path newDest = new Path(dest + "/" + srcChild.getName());
+ dfsCopy(srcChild, newDest, fsMode);
+ }
+ } else {
+ if (!fs.exists(dest.getParent())) {
+ fs.mkdirs(dest.getParent());
+ }
+ if (!fs.exists(dest)) {
+ FileUtil.copy(fs, src, fs, dest, false, fs.getConf());
+ LOG.debug("Copying file " + src + " to " + dest);
+ } else {
+ LOG.debug("File " + src + " already exists as " + dest);
+ }
+ }
} catch (IOException e) {
LOG.debug("File " + src + " already exists as " + dest);
}
@@ -961,22 +1056,26 @@
private void runGenerateScript(DataSource datasource) {
String command = DrillTestDefaults.TEST_ROOT_DIR + "/" + DrillTestDefaults.DRILL_TESTDATA_DIR + "/" + datasource.src;
- command = Utils.substituteArguments(command);
- LOG.info("Running command " + command);
- CmdConsOut cmdConsOut;
- try {
- cmdConsOut = Utils.execCmd(command);
- LOG.debug(cmdConsOut);
- } catch (Exception e) {
- cmdConsOut = new CmdConsOut();
- cmdConsOut.cmd = command;
- cmdConsOut.consoleErr = e.getMessage();
- LOG.error("Error: Failed to execute the command " + cmdConsOut);
- throw new RuntimeException(e);
- }
- if (cmdConsOut.exitCode != 0) {
- throw new RuntimeException("Error executing the command " + command
- + " has return code " + cmdConsOut.exitCode);
+ if (command.endsWith(".ddl") && command.split(" ").length == 1) {
+ runDDL(new Path(command));
+ } else {
+ command = Utils.substituteArguments(command);
+ LOG.info("Running command " + command);
+ CmdConsOut cmdConsOut;
+ try {
+ cmdConsOut = Utils.execCmd(command);
+ LOG.debug(cmdConsOut);
+ } catch (Exception e) {
+ cmdConsOut = new CmdConsOut();
+ cmdConsOut.cmd = command;
+ cmdConsOut.consoleErr = e.getMessage();
+ LOG.error("Error: Failed to execute the command " + cmdConsOut);
+ throw new RuntimeException(e);
+ }
+ if (cmdConsOut.exitCode != 0) {
+ throw new RuntimeException("Error executing the command " + command
+ + " has return code " + cmdConsOut.exitCode);
+ }
}
}
