blob: 2850e493484b6e601c82f8cc6866aea8e96d450f [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import unittest
from pathlib import Path
from unittest import mock
from sparktestsupport import SPARK_HOME, modules
from sparktestsupport.shellutils import is_exe
from sparktestsupport.utils import determine_modules_for_files, determine_modules_to_test
class IsIgnoredFileTests(unittest.TestCase):
def test_dots_in_patterns_are_literal(self):
self.assertTrue(modules.is_ignored_file(".asf.yaml"))
self.assertFalse(modules.is_ignored_file("xasfZyaml"))
def test_sparktestsupport_tests_are_ignored(self):
self.assertTrue(modules.is_ignored_file("dev/sparktestsupport/tests/test_modules.py"))
def test_sparktestsupport_source_is_not_ignored(self):
self.assertFalse(modules.is_ignored_file("dev/sparktestsupport/modules.py"))
def test_agent_docs_are_ignored(self):
for path in [
".agents/pr-workflow.md",
"sql/core/.agents/x.md",
".ignore",
"python/.ignore",
"dev/agent_docs_lint.py",
]:
self.assertTrue(modules.is_ignored_file(path), path)
self.assertFalse(modules.is_ignored_file("x.agents/y.md"))
class ShellScriptTestGoalTests(unittest.TestCase):
def test_default_stage(self):
goal = modules.ShellScriptTestGoal("dev/example")
self.assertIs(goal.stage, modules.TestStage.PRE_BUILD)
def test_normalizes_and_hashes_equivalent_paths(self):
goal = modules.ShellScriptTestGoal("dev/example")
equivalents = (
modules.ShellScriptTestGoal(Path("dev/example")),
modules.ShellScriptTestGoal("./dev//example"),
)
self.assertEqual(goal.path, Path("dev/example"))
for equivalent in equivalents:
with self.subTest(goal=equivalent):
self.assertEqual(equivalent.path, goal.path)
self.assertEqual(equivalent, goal)
self.assertEqual(len({goal, *equivalents}), 1)
def test_rejects_invalid_stages(self):
for stage in ("post_build", "pre_build", None):
with self.subTest(stage=stage):
with self.assertRaises(TypeError) as error:
modules.ShellScriptTestGoal("dev/example", stage)
self.assertEqual(str(error.exception), f"Unsupported test stage: {stage!r}")
def test_rejects_invalid_paths(self):
absolute_path = Path.cwd() / "example"
for path in (
str(absolute_path),
absolute_path,
"",
" ",
".",
"../example",
"dev/../example",
):
with self.subTest(path=path):
with self.assertRaises(ValueError) as error:
modules.ShellScriptTestGoal(path)
self.assertEqual(
str(error.exception),
f"Shell script path must be relative to repo root: {str(path)!r}",
)
class ModuleTests(unittest.TestCase):
def test_duplicate_name_does_not_change_module_graph(self):
with mock.patch.object(modules, "all_modules", []):
dependency = modules.Module("dependency", [], [])
original = modules.Module("core", [], [])
with self.assertRaisesRegex(ValueError, "module name 'core' is already in use"):
modules.Module("core", [dependency], [])
self.assertEqual(modules.all_modules, [dependency, original])
self.assertEqual(dependency.dependent_modules, set())
def test_shell_script_test_goals_accept_sequences(self):
goal = modules.ShellScriptTestGoal("dev/lint")
with mock.patch.object(modules, "all_modules", []):
for index, goals in enumerate(([goal], (goal,))):
with self.subTest(goals=goals):
module = modules.Module(f"valid-{index}", [], [], shell_script_test_goals=goals)
self.assertEqual(module.shell_script_test_goals, goals)
def test_shell_script_test_goals_require_a_sequence(self):
goal = modules.ShellScriptTestGoal("dev/lint")
invalid_goals = (
"dev/lint-java",
b"dev/lint-java",
goal,
(item for item in [goal]),
)
with mock.patch.object(modules, "all_modules", []):
dependency = modules.Module("dependency", [], [])
for goals in invalid_goals:
with self.subTest(goals=goals):
with self.assertRaisesRegex(
TypeError, "^invalid: shell_script_test_goals must be a sequence$"
):
modules.Module("invalid", [dependency], [], shell_script_test_goals=goals)
self.assertEqual(modules.all_modules, [dependency])
self.assertEqual(dependency.dependent_modules, set())
def test_shell_script_test_goals_reject_invalid_items(self):
goal = modules.ShellScriptTestGoal("dev/lint")
with mock.patch.object(modules, "all_modules", []):
dependency = modules.Module("dependency", [], [])
for goals in (["dev/lint-java"], [goal, "dev/lint-java"]):
with self.subTest(goals=goals):
with self.assertRaisesRegex(
TypeError, "^invalid: expected ShellScriptTestGoal, got 'dev/lint-java'$"
):
modules.Module("invalid", [dependency], [], shell_script_test_goals=goals)
self.assertEqual(modules.all_modules, [dependency])
self.assertEqual(dependency.dependent_modules, set())
class ModuleDefinitionsTests(unittest.TestCase):
def test_all_pyspark_modules_depend_on_base(self):
for module in modules.all_modules:
if module.name.startswith("pyspark-") and module is not modules.pyspark_base:
with self.subTest(module=module.name):
self.assertIn(modules.pyspark_base, modules.transitive_dependencies(module))
def test_no_module_depends_on_root(self):
for module in modules.all_modules:
with self.subTest(module=module.name):
self.assertNotIn(modules.root, module.dependencies)
def test_root_includes_all_build_profiles(self):
expected = {
flag
for module in modules.all_modules
if module is not modules.root
for flag in module.build_profile_flags
}
self.assertEqual(set(modules.root.build_profile_flags), expected)
def test_shell_script_test_goals_are_executable(self):
for module in modules.all_modules:
for goal in module.shell_script_test_goals:
with self.subTest(module=module.name, goal=str(goal.path)):
self.assertTrue(is_exe(os.path.join(SPARK_HOME, goal.path)))
def test_streaming_kafka_does_not_match_sql_kafka(self):
self.assertTrue(
modules.streaming_kafka_0_10.contains_file("connector/kafka-0-10/test.scala")
)
self.assertFalse(
modules.streaming_kafka_0_10.contains_file("connector/kafka-0-10-sql/test.scala")
)
def test_any_python_change_selects_periodic_tests(self):
for filename in (
"python/pyspark/test.py",
"python/run-tests.py",
"python/packaging/setup.py",
):
with self.subTest(filename=filename):
self.assertIn(modules.pyspark_periodic, determine_modules_for_files([filename]))
def test_kinesis_changes_select_kinesis_module(self):
kinesis_files = ["connector/kinesis-asl/src/test/scala/KinesisTest.scala"]
self.assertIn(modules.streaming_kinesis_asl, determine_modules_for_files(kinesis_files))
def test_streaming_changes_do_not_select_kinesis_tests(self):
for deduplicated in (True, False):
with self.subTest(deduplicated=deduplicated):
modules_to_test = determine_modules_to_test(
[modules.streaming], deduplicated=deduplicated
)
self.assertNotIn(modules.root, modules_to_test)
self.assertNotIn(modules.streaming_kinesis_asl, modules_to_test)
def test_rust_sdk_changes_select_rust_sdk_module(self):
for filename in (
"sql/native/rust/src/lib.rs",
"examples/src/main/rust/native-data-source/src/lib.rs",
"sql/core/src/main/resources/org/apache/spark/sql/datasource/spark_datasource.h",
):
with self.subTest(filename=filename):
changed_modules = determine_modules_for_files([filename])
self.assertIn(modules.rust_sdk, changed_modules)
self.assertNotIn(modules.root, changed_modules)
def test_cargo_only_changes_select_only_rust_sdk_module(self):
for filename in (
"sql/native/rust/src/lib.rs",
"examples/src/main/rust/native-data-source/src/lib.rs",
):
with self.subTest(filename=filename):
changed_modules = determine_modules_for_files([filename])
self.assertEqual({modules.rust_sdk}, changed_modules)
def test_non_rust_example_changes_select_examples_module(self):
changed_modules = determine_modules_for_files(["examples/src/main/scala/Example.scala"])
self.assertIn(modules.examples, changed_modules)
def test_sql_changes_do_not_select_rust_sdk_tests(self):
self.assertNotIn(modules.rust_sdk, determine_modules_to_test([modules.sql]))