blob: 2ab4250f6505b8a4bf7b01d16d2c4922cd11344d [file] [log] [blame]
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Impala py.test skipif markers. When a test can't be run against S3,
# choose the appropriate reason (or add a new one if needed) and
# annotate the class or test routine with the marker.
import os
import pytest
from functools import partial
from tests.common.environ import (ImpalaTestClusterProperties,
from tests.common.kudu_test_suite import get_kudu_master_flag
from tests.util.filesystem_utils import (
IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance()
class SkipIfS3:
# These are skipped due to product limitations.
caching = pytest.mark.skipif(IS_S3, reason="SET CACHED not implemented for S3")
hive = pytest.mark.skipif(IS_S3, reason="Hive doesn't work with S3")
hdfs_block_size = pytest.mark.skipif(IS_S3, reason="S3 uses it's own block size")
hdfs_acls = pytest.mark.skipif(IS_S3, reason="HDFS acls are not supported on S3")
jira = partial(pytest.mark.skipif, IS_S3)
hdfs_encryption = pytest.mark.skipif(IS_S3,
reason="HDFS encryption is not supported with S3")
empty_directory = pytest.mark.skipif(IS_S3,
reason="Empty directories are not supported on S3")
# These need test infra work to re-enable.
udfs = pytest.mark.skipif(IS_S3, reason="udas/udfs not copied to S3")
datasrc = pytest.mark.skipif(IS_S3, reason="data sources not copied to S3")
hbase = pytest.mark.skipif(IS_S3, reason="HBase not started with S3")
qualified_path = pytest.mark.skipif(IS_S3,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
eventually_consistent = pytest.mark.skipif(IS_S3,
reason="Flakiness on account of S3 eventual consistency.")
class SkipIfABFS:
# These are skipped due to product limitations.
caching = pytest.mark.skipif(IS_ABFS, reason="SET CACHED not implemented for ABFS")
hive = pytest.mark.skipif(IS_ABFS, reason="Hive doesn't work with ABFS")
hdfs_block_size = pytest.mark.skipif(IS_ABFS, reason="ABFS uses it's own block size")
hdfs_acls = pytest.mark.skipif(IS_ABFS, reason="HDFS acls are not supported on ABFS")
jira = partial(pytest.mark.skipif, IS_ABFS)
hdfs_encryption = pytest.mark.skipif(IS_ABFS,
reason="HDFS encryption is not supported with ABFS")
trash = pytest.mark.skipif(IS_ABFS,
reason="Drop/purge not working as expected on ABFS, IMPALA-7726")
file_or_folder_name_ends_with_period = pytest.mark.skipif(IS_ABFS,
reason="ABFS does not support file / directories that end with a period")
# These need test infra work to re-enable.
udfs = pytest.mark.skipif(IS_ABFS, reason="udas/udfs not copied to ABFS")
datasrc = pytest.mark.skipif(IS_ABFS, reason="data sources not copied to ABFS")
hbase = pytest.mark.skipif(IS_ABFS, reason="HBase not started with ABFS")
qualified_path = pytest.mark.skipif(IS_ABFS,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
class SkipIfADLS:
# These are skipped due to product limitations.
caching = pytest.mark.skipif(IS_ADLS, reason="SET CACHED not implemented for ADLS")
hive = pytest.mark.skipif(IS_ADLS, reason="Hive doesn't work with ADLS")
hdfs_block_size = pytest.mark.skipif(IS_ADLS, reason="ADLS uses it's own block size")
hdfs_acls = pytest.mark.skipif(IS_ADLS, reason="HDFS acls are not supported on ADLS")
jira = partial(pytest.mark.skipif, IS_ADLS)
hdfs_encryption = pytest.mark.skipif(IS_ADLS,
reason="HDFS encryption is not supported with ADLS")
# These need test infra work to re-enable.
udfs = pytest.mark.skipif(IS_ADLS, reason="udas/udfs not copied to ADLS")
datasrc = pytest.mark.skipif(IS_ADLS, reason="data sources not copied to ADLS")
hbase = pytest.mark.skipif(IS_ADLS, reason="HBase not started with ADLS")
qualified_path = pytest.mark.skipif(IS_ADLS,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
eventually_consistent = pytest.mark.skipif(IS_ADLS,
reason="The client is slow to realize changes to file metadata")
class SkipIfKudu:
unsupported_env = pytest.mark.skipif(os.environ["KUDU_IS_SUPPORTED"] == "false",
reason="Kudu is not supported in this environment")
no_hybrid_clock = pytest.mark.skipif(
get_kudu_master_flag("--use_hybrid_clock") == "false",
reason="Test relies on --use_hybrid_clock=true in Kudu.")
hms_integration_enabled = pytest.mark.skipif(
get_kudu_master_flag("--hive_metastore_uris") != "",
reason="Test assumes Kudu/HMS integration is not enabled.")
class SkipIf:
skip_hbase = pytest.mark.skipif(pytest.config.option.skip_hbase,
reason="--skip_hbase argument specified")
kudu_not_supported = pytest.mark.skipif(os.environ["KUDU_IS_SUPPORTED"] == "false",
reason="Kudu is not supported")
not_s3 = pytest.mark.skipif(not IS_S3, reason="S3 Filesystem needed")
not_hdfs = pytest.mark.skipif(not IS_HDFS, reason="HDFS Filesystem needed")
not_ec = pytest.mark.skipif(not IS_EC, reason="Erasure Coding needed")
no_secondary_fs = pytest.mark.skipif(not SECONDARY_FILESYSTEM,
reason="Secondary filesystem needed")
is_buggy_el6_kernel = pytest.mark.skipif(
IS_BUGGY_EL6_KERNEL, reason="Kernel is affected by KUDU-1508")
class SkipIfIsilon:
caching = pytest.mark.skipif(IS_ISILON, reason="SET CACHED not implemented for Isilon")
hbase = pytest.mark.skipif(IS_ISILON, reason="HBase not tested with Isilon")
hive = pytest.mark.skipif(IS_ISILON, reason="Hive not tested with Isilon")
hdfs_acls = pytest.mark.skipif(IS_ISILON, reason="HDFS acls are not supported on Isilon")
hdfs_block_size = pytest.mark.skipif(IS_ISILON,
reason="Isilon uses its own block size")
hdfs_encryption = pytest.mark.skipif(IS_ISILON,
reason="HDFS encryption is not supported with Isilon")
untriaged = pytest.mark.skipif(IS_ISILON,
reason="This Isilon issue has yet to be triaged.")
jira = partial(pytest.mark.skipif, IS_ISILON)
class SkipIfLocal:
# These are skipped due to product limitations.
caching = pytest.mark.skipif(IS_LOCAL,
reason="HDFS caching not supported on local file system")
hdfs_blocks = pytest.mark.skipif(IS_LOCAL,
reason="Files on local filesystem are not split into blocks")
hdfs_encryption = pytest.mark.skipif(IS_LOCAL,
reason="HDFS encryption is not supported on local filesystem")
hive = pytest.mark.skipif(IS_LOCAL,
reason="Hive not started when using local file system")
multiple_impalad = pytest.mark.skipif(IS_LOCAL,
reason="Multiple impalads are not supported when using local file system")
parquet_file_size = pytest.mark.skipif(IS_LOCAL,
reason="Parquet block size incorrectly determined")
hdfs_fd_caching = pytest.mark.skipif(IS_LOCAL,
reason="HDFS file handle caching not supported for local non-HDFS files")
# These need test infra work to re-enable.
hbase = pytest.mark.skipif(IS_LOCAL,
reason="HBase not started when using local file system")
hdfs_client = pytest.mark.skipif(IS_LOCAL,
reason="HDFS not started when using local file system")
qualified_path = pytest.mark.skipif(IS_LOCAL,
reason="Tests rely on HDFS qualified paths")
root_path = pytest.mark.skipif(IS_LOCAL,
reason="Tests rely on the root directory")
class SkipIfNotHdfsMinicluster:
# These are skipped when not running against a local HDFS mini-cluster.
plans = pytest.mark.skipif(
not IS_HDFS or IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
reason="Test assumes plans from local HDFS mini-cluster")
tuned_for_minicluster = pytest.mark.skipif(
not IS_HDFS or IS_EC or IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
reason="Test is tuned for 3-node HDFS minicluster with no EC")
scheduling = pytest.mark.skipif(
not IS_HDFS or IS_EC or pytest.config.option.testing_remote_cluster,
reason="Test is tuned for scheduling decisions made on a 3-node HDFS minicluster "
"with no EC")
class SkipIfBuildType:
not_dev_build = pytest.mark.skipif(not IMPALA_TEST_CLUSTER_PROPERTIES.is_dev(),
reason="Test depends on debug build startup option.")
remote = pytest.mark.skipif(IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
reason="Test depends on running against a local Impala cluster")
class SkipIfEC:
remote_read = pytest.mark.skipif(IS_EC, reason="EC files are read remotely and "
"features relying on local read do not work.")
oom = pytest.mark.skipif(IS_EC, reason="Probably broken by HDFS-13540.")
fix_later = pytest.mark.skipif(IS_EC, reason="It should work but doesn't.")
class SkipIfDockerizedCluster:
catalog_service_not_exposed = pytest.mark.skipif(
IS_DOCKERIZED_TEST_CLUSTER, reason="Catalog service not exposed.")
statestore_not_exposed = pytest.mark.skipif(
IS_DOCKERIZED_TEST_CLUSTER, reason="Statestore service not exposed.")
internal_hostname = pytest.mark.skipif(
IS_DOCKERIZED_TEST_CLUSTER, reason="Internal hostname is used, not local hostname.")
daemon_logs_not_exposed = pytest.mark.skipif(
IS_DOCKERIZED_TEST_CLUSTER, reason="Daemon logs not exposed in host.")
accesses_host_filesystem = pytest.mark.skipif(
IS_DOCKERIZED_TEST_CLUSTER, reason="Daemon would need to access host filesystem.")
insert_acls = pytest.mark.skipif(IS_DOCKERIZED_TEST_CLUSTER,
reason="IMPALA-8384: insert ACL tests are broken on dockerised minicluster.")
class SkipIfHive3:
sentry_not_supported = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
reason="Sentry HMS follower does not work with HMS-3. See SENTRY-2518 for details")
kudu_hms_notifications_not_supported = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
reason="Kudu is not tested with Hive 3 notifications yet, see IMPALA-8751.")
col_stat_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
reason="Hive 3 separates column statistics by engine")
class SkipIfHive2:
acid = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
reason="Acid tables are only supported with Hive 3.")
col_stat_not_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
reason="Hive 2 doesnt support separating column statistics by engine")
create_external_kudu_table = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
reason="Hive 2 does not support creating external.table.purge Kudu tables."
" See IMPALA-9092 for details.")
class SkipIfCatalogV2:
"""Expose decorators as methods so that is_catalog_v2_cluster() can be evaluated lazily
when needed, instead of whenever this module is imported."""
def stats_pulling_disabled(self):
return pytest.mark.skipif(
reason="Local catalog does not use incremental stats pulling.")
def catalog_v1_test(self):
return pytest.mark.skipif(
reason="Test is specific to old implementation of catalog.")
# TODO: IMPALA-7131: add support or update tests to reflect expected behaviour.
def data_sources_unsupported(self):
return pytest.mark.skipif(
reason="IMPALA-7131: data sources not supported.")
# TODO: IMPALA-7538: add support or update tests to reflect expected behaviour.
def hdfs_caching_ddl_unsupported(self):
return pytest.mark.skipif(
reason="IMPALA-7538: HDFS caching DDL not supported.")
# TODO: IMPALA-8489: fix this bug.
def impala_8489(self):
return pytest.mark.skipif(
reason="IMPALA-8489: TestRecoverPartitions.test_post_invalidate "
# TODO: IMPALA-7539: fix this bug.
def impala_7539(self):
return pytest.mark.skipif(
reason="IMPALA-7539: support HDFS permission checks for LocalCatalog")
def hms_event_polling_enabled(self):
return pytest.mark.skipif(
reason="Table isn't invalidated with Local catalog and enabled hms_event_polling.")
class SkipIfOS:
redhat6 = pytest.mark.skipif(IS_REDHAT_6_DERIVATIVE,
reason="Flaky on redhat or centos 6")