tests/metadata/test_hidden_files.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from subprocess import check_call

from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
    create_single_exec_option_dimension,
    create_uncompressed_text_dimension)
from tests.util.filesystem_utils import WAREHOUSE, IS_S3

class TestHiddenFiles(ImpalaTestSuite):
  """
  Tests that files with special prefixes/suffixes are considered 'hidden' when
  loading table metadata and running queries.
  """

  # The .test file run in these tests relies this table name.
  TBL_NAME = "test_hidden_files"

  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestHiddenFiles, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
    cls.ImpalaTestMatrix.add_dimension(
        create_uncompressed_text_dimension(cls.get_workload()))
    # Only run in exhaustive mode on hdfs since this test takes a long time.
    if cls.exploration_strategy() != 'exhaustive' and not IS_S3:
      cls.ImpalaTestMatrix.clear()

  def __prepare_test_table(self, db_name, tbl_name):
    """Creates a test table with two partitions, and copies files into the HDFS
    directories of the two partitions. The goal is to have both an empty and non-empty
    partition with hidden files."""

    self.client.execute(
      "create table %s.%s like functional.alltypes" % (db_name, tbl_name))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=1)" % (db_name, tbl_name))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=2)" % (db_name, tbl_name))

    ALLTYPES_LOC = "%s/alltypes" % WAREHOUSE
    TEST_TBL_LOC = "%s/%s.db/%s" % (WAREHOUSE, db_name, tbl_name)
    # Copy a visible file into one of the partitions.
    self.filesystem_client.copy(
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt" % TEST_TBL_LOC, overwrite=True)
    # Add hidden files to the non-empty partition. Use upper case hidden suffixes.
    self.filesystem_client.copy(
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/.100101.txt" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/_100101.txt" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.COPYING" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.TMP" % TEST_TBL_LOC, overwrite=True)
    # Add hidden files to the empty partition. Use lower case hidden suffixes.
    self.filesystem_client.copy(
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/.100201.txt" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/_100201.txt" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.copying" % TEST_TBL_LOC, overwrite=True)
    self.filesystem_client.copy(
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.tmp" % TEST_TBL_LOC, overwrite=True)

  def test_hidden_files_load(self, vector, unique_database):
    """Tests that an incremental refresh ignores hidden files."""
    self.__prepare_test_table(unique_database, self.TBL_NAME)
    self.client.execute("invalidate metadata %s.%s" % (unique_database, self.TBL_NAME))
    self.run_test_case('QueryTest/hidden-files', vector, unique_database)

  # This test runs on one dimension. Therefore, running in it parallel is safe, given no
  # other method in this test class is run.
  def test_hidden_files_refresh(self, vector, unique_database):
    """Tests that an incremental refresh ignores hidden files."""
    self.__prepare_test_table(unique_database, self.TBL_NAME)
    self.client.execute("refresh %s.%s" % (unique_database, self.TBL_NAME))
    self.run_test_case('QueryTest/hidden-files', vector, unique_database)