IMPALA-9117, IMPALA-7726: Fixed a few unit tests for ABFS
This test makes the following changes / fixes when running Impala tests
on ABFS:
* Skips some tests in test_lineage.py that don't work on ABFS / ADLS
(they were already skipped for S3)
* Skips some tests in test_mt_dop.py; the test creates a directory that
ends with a period (and ABFS does not support writing files or
directories that end with a period)
* Removes the ABFS skip flag SkipIfABFS.trash (IMPALA-7726: Drop with
purge tests fail against ABFS due to trash misbehavior"); I removed
these flags and looped the tests overnight with no failures, so it is
likely whatever bug was causing this has now been fixed
* Now that HADOOP-15860 has been resolved, and the agreed upon behavior
for ABFS is that it will fail if a client tries to write a file /
directory that ends with a period, I added a new entry to the SkipIfABFS
class called file_or_folder_name_ends_with_period and applied it where
necessary
Testing:
* Ran core tests on ABFS
Change-Id: I18ae5b0f7de6aa7628a1efd780ff30a0cc3c5285
Reviewed-on: http://gerrit.cloudera.org:8080/14636
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 1426b86..5203de5 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -75,6 +75,8 @@
reason="HDFS encryption is not supported with ABFS")
trash = pytest.mark.skipif(IS_ABFS,
reason="Drop/purge not working as expected on ABFS, IMPALA-7726")
+ file_or_folder_name_ends_with_period = pytest.mark.skipif(IS_ABFS,
+ reason="ABFS does not support file / directories that end with a period")
# These need test infra work to re-enable.
udfs = pytest.mark.skipif(IS_ABFS, reason="udas/udfs not copied to ABFS")
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 850ae65..73f785d 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -26,7 +26,7 @@
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3
+from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3
LOG = logging.getLogger(__name__)
@@ -132,6 +132,8 @@
assert lineage_json["queryText"] == query
assert lineage_json["tableLocation"] is not None
+ @SkipIfABFS.hbase
+ @SkipIfADLS.hbase
@SkipIfS3.hbase
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
diff --git a/tests/custom_cluster/test_mt_dop.py b/tests/custom_cluster/test_mt_dop.py
index 0bf8a1b..d4f664e 100644
--- a/tests/custom_cluster/test_mt_dop.py
+++ b/tests/custom_cluster/test_mt_dop.py
@@ -20,7 +20,7 @@
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import build_flavor_timeout
-from tests.common.skip import SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfABFS, SkipIfNotHdfsMinicluster
WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=100000)
@@ -36,6 +36,7 @@
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(impalad_args="--unlock_mt_dop=true")
+ @SkipIfABFS.file_or_folder_name_ends_with_period
def test_mt_dop_all(self, vector, unique_database):
"""Test joins, inserts and runtime filters with mt_dop > 0"""
vector = deepcopy(vector)
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 3a3c982..c51e0f4 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -40,7 +40,6 @@
# Validates DDL statements (create, drop)
class TestDdlStatements(TestDdlBase):
@SkipIfLocal.hdfs_client
- @SkipIfABFS.trash
def test_drop_table_with_purge(self, unique_database):
"""This test checks if the table data is permanently deleted in
DROP TABLE <tbl> PURGE queries"""
@@ -451,7 +450,6 @@
use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
@SkipIfLocal.hdfs_client
- @SkipIfABFS.trash
def test_drop_partition_with_purge(self, vector, unique_database):
"""Verfies whether alter <tbl> drop partition purge actually skips trash"""
self.client.execute(
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 173a272..7807030 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -129,7 +129,7 @@
# Erasure coding doesn't respect memory limit
@SkipIfEC.fix_later
# ABFS partition names cannot end in periods
- @SkipIfABFS.jira(reason="HADOOP-15860")
+ @SkipIfABFS.file_or_folder_name_ends_with_period
def test_insert(self, vector):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
