| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| """End-to-end tests for the session's Python worker environment. | |
| These run a real Python worker, so they cover what the unit tests cannot: that a value set through | |
| `spark.conf` actually reaches `os.environ` inside a UDF. The mixin is shared by the classic suite | |
| here and the Spark Connect parity suite, so both front ends are held to the same behaviour. | |
| """ | |
| import os | |
| from pyspark.sql.functions import lit, udf | |
| from pyspark.testing.sqlutils import ReusedSQLTestCase | |
| PREFIX = "spark.pythonWorkerEnv." | |
| # A process environment cannot carry NUL. Built from its code point rather than written as an | |
| # escape, so nothing along the way can turn it into a literal control character. | |
| NUL = chr(0) | |
| class PythonWorkerEnvMixin: | |
| """Behaviour that must hold identically on classic Spark and on Spark Connect.""" | |
| def _read_in_worker(self, name, use_arrow=None): | |
| """The value of `name` as seen by os.environ inside a Python worker. | |
| The UDF is defined here rather than at module scope because cloudpickle serializes a | |
| module-level function by reference: the worker would try to import this test module and | |
| fail. A local function is serialized by value. | |
| `use_arrow` selects the execution path. Left as None it follows | |
| `spark.sql.execution.pythonUDF.arrow.enabled`, which is enabled by default, so the default | |
| case exercises the Arrow path. | |
| """ | |
| def read_env(variable): | |
| import os | |
| return os.environ.get(variable, "<unset>") | |
| read_env_udf = udf(read_env, "string", useArrow=use_arrow) | |
| df = self.spark.range(1).select(read_env_udf(lit(name)).alias("value")) | |
| return df.collect()[0]["value"] | |
| def _set_env(self, name, value): | |
| self.spark.conf.set(PREFIX + name, value) | |
| def _unset_env(self, name): | |
| self.spark.conf.unset(PREFIX + name) | |
| # -- the two execution paths of a regular scalar Python UDF ---------------- | |
| def test_env_var_reaches_arrow_udf(self): | |
| # Arrow is the default, so this is the ordinary case. | |
| self._set_env("MY_SETTING", "abc") | |
| try: | |
| self.assertEqual(self._read_in_worker("MY_SETTING", use_arrow=True), "abc") | |
| finally: | |
| self._unset_env("MY_SETTING") | |
| def test_env_var_reaches_non_arrow_udf(self): | |
| self._set_env("MY_SETTING", "abc") | |
| try: | |
| self.assertEqual(self._read_in_worker("MY_SETTING", use_arrow=False), "abc") | |
| finally: | |
| self._unset_env("MY_SETTING") | |
| def test_env_var_reaches_udf_with_the_default_path(self): | |
| self._set_env("MY_SETTING", "abc") | |
| try: | |
| self.assertEqual(self._read_in_worker("MY_SETTING"), "abc") | |
| finally: | |
| self._unset_env("MY_SETTING") | |
| # -- lifecycle ------------------------------------------------------------ | |
| def test_env_var_unset_is_not_visible(self): | |
| self.assertEqual(self._read_in_worker("NEVER_SET_BY_THIS_TEST"), "<unset>") | |
| def test_env_var_change_takes_effect_on_the_next_action(self): | |
| # The environment is installed when a worker is launched, so a change is picked up without | |
| # rebuilding the UDF, and a cached or reused plan cannot pin an old value. | |
| self._set_env("ROTATING", "first") | |
| try: | |
| self.assertEqual(self._read_in_worker("ROTATING"), "first") | |
| self._set_env("ROTATING", "second") | |
| self.assertEqual(self._read_in_worker("ROTATING"), "second") | |
| finally: | |
| self._unset_env("ROTATING") | |
| def test_env_var_change_takes_effect_on_a_reused_dataframe(self): | |
| # The same DataFrame, and so the same plan and the same built function, observed twice | |
| # across a change. This is what a plan cache would otherwise get wrong. | |
| def read_env(variable): | |
| import os | |
| return os.environ.get(variable, "<unset>") | |
| read_env_udf = udf(read_env, "string") | |
| df = self.spark.range(1).select(read_env_udf(lit("REUSED")).alias("value")) | |
| self._set_env("REUSED", "first") | |
| try: | |
| self.assertEqual(df.collect()[0]["value"], "first") | |
| self._set_env("REUSED", "second") | |
| self.assertEqual(df.collect()[0]["value"], "second") | |
| finally: | |
| self._unset_env("REUSED") | |
| def test_empty_value_is_visible_as_empty(self): | |
| # `FOO=` in a shell: the variable exists and its value is the empty string. | |
| self._set_env("EMPTY", "") | |
| try: | |
| self.assertEqual(self._read_in_worker("EMPTY"), "") | |
| finally: | |
| self._unset_env("EMPTY") | |
| def test_platform_owned_variable_is_not_overridden(self): | |
| # The worker factory sets PYTHONUNBUFFERED after applying the session's map, so the platform | |
| # value wins by write order and the name does not need reserving. | |
| self._set_env("PYTHONUNBUFFERED", "NO") | |
| try: | |
| self.assertEqual(self._read_in_worker("PYTHONUNBUFFERED"), "YES") | |
| finally: | |
| self._unset_env("PYTHONUNBUFFERED") | |
| def test_pythonpath_reaches_the_worker_import_path(self): | |
| # PYTHONPATH is neither reserved nor overridden: PythonWorkerFactory folds the session's | |
| # value into the path it computes, so the entry reaches the worker rather than being | |
| # dropped. That is the whole guarantee. The relative order is deliberately not asserted: | |
| # `sparkPythonPath` is empty when SPARK_HOME is unset, and `mergePythonPaths` drops empty | |
| # entries, so the session's path can legitimately come first. Leaving the name settable | |
| # rests on the session already choosing the code its own worker runs, not on Spark's | |
| # entries winning. | |
| marker = "/tmp/spark-58752-extra-modules" | |
| self._set_env("PYTHONPATH", marker) | |
| try: | |
| entries = self._read_in_worker("PYTHONPATH").split(os.pathsep) | |
| self.assertIn(marker, entries) | |
| finally: | |
| self._unset_env("PYTHONPATH") | |
| def test_reserved_name_is_refused(self): | |
| # Spark sets SPARK_PIPELINED_UDF only when pipelined execution is on, and the worker reads | |
| # it to choose its wire protocol, so a session must not be able to set it at all. One name | |
| # is enough end to end: `PythonWorkerEnvironmentSuite` iterates every reserved prefix and | |
| # name, including the conditional PYTHON_* ones and PYTHON_WORKER_FACTORY_SOCK_DIR. | |
| with self.assertRaises(Exception) as context: | |
| self.spark.conf.set(PREFIX + "SPARK_PIPELINED_UDF", "1") | |
| self.assertIn("RESERVED_PYTHON_WORKER_ENV_VAR_NAME", str(context.exception)) | |
| self.assertIsNone(self.spark.conf.get(PREFIX + "SPARK_PIPELINED_UDF", None)) | |
| def test_sql_set_is_refused_too(self): | |
| # SQL `SET` reaches `RuntimeConfig.set` through `SetCommand`, so it goes through the same | |
| # check as `spark.conf.set` rather than storing the value for a later query to trip over. | |
| with self.assertRaises(Exception) as context: | |
| self.spark.sql("SET {}1INVALID=x".format(PREFIX)).collect() | |
| message = str(context.exception) | |
| self.assertIn("1INVALID", message) | |
| # A rejection names the variable but must never quote its value. | |
| self.assertNotIn("is not valid: x", message) | |
| self.assertIsNone(self.spark.conf.get(PREFIX + "1INVALID", None)) | |
| def test_oversized_environment_is_refused(self): | |
| # Exceeds the default 128 KiB spark.sql.pythonWorkerEnv.maxTotalSizeBytes. The limits are | |
| # static configs, so a shared session cannot lower one for a test; going over the default is | |
| # what lets this run end to end. The Scala suite covers the other limits by overriding them | |
| # on the SparkConf directly. | |
| big = "x" * (200 * 1024) | |
| with self.assertRaises(Exception) as context: | |
| self.spark.conf.set(PREFIX + "BIG", big) | |
| message = str(context.exception) | |
| self.assertIn("PYTHON_WORKER_ENV_TOO_LARGE", message) | |
| # The rejection reports sizes, never the offending value. | |
| self.assertNotIn("xxxxxxxxxx", message) | |
| self.assertIsNone(self.spark.conf.get(PREFIX + "BIG", None)) | |
| # -- families that are not covered yet ------------------------------------ | |
| # -- the vectorized and specialized families ------------------------------- | |
| def test_env_var_reaches_map_in_pandas(self): | |
| import pandas as pd | |
| self._set_env("MAP_SETTING", "mapped") | |
| try: | |
| def func(iterator): | |
| import os | |
| for _ in iterator: | |
| yield pd.DataFrame({"value": [os.environ.get("MAP_SETTING", "<unset>")]}) | |
| result = self.spark.range(1).mapInPandas(func, "value string").collect() | |
| self.assertEqual(result[0]["value"], "mapped") | |
| finally: | |
| self._unset_env("MAP_SETTING") | |
| def test_env_var_reaches_map_in_arrow(self): | |
| import pyarrow as pa | |
| self._set_env("MAP_SETTING", "mapped") | |
| try: | |
| def func(iterator): | |
| import os | |
| for _ in iterator: | |
| yield pa.RecordBatch.from_pydict( | |
| {"value": [os.environ.get("MAP_SETTING", "<unset>")]} | |
| ) | |
| result = self.spark.range(1).mapInArrow(func, "value string").collect() | |
| self.assertEqual(result[0]["value"], "mapped") | |
| finally: | |
| self._unset_env("MAP_SETTING") | |
| def test_env_var_reaches_scalar_pandas_udf(self): | |
| import pandas as pd | |
| from pyspark.sql.functions import pandas_udf | |
| self._set_env("PANDAS_SETTING", "vectorized") | |
| try: | |
| @pandas_udf("string") | |
| def read_env(values: pd.Series) -> pd.Series: | |
| import os | |
| return values.map(lambda _: os.environ.get("PANDAS_SETTING", "<unset>")) | |
| result = self.spark.range(1).select(read_env("id").alias("value")).collect() | |
| self.assertEqual(result[0]["value"], "vectorized") | |
| finally: | |
| self._unset_env("PANDAS_SETTING") | |
| def test_env_var_reaches_scalar_pandas_iter_udf(self): | |
| from typing import Iterator | |
| import pandas as pd | |
| from pyspark.sql.functions import pandas_udf | |
| self._set_env("PANDAS_SETTING", "iterated") | |
| try: | |
| @pandas_udf("string") | |
| def read_env(batches: Iterator[pd.Series]) -> Iterator[pd.Series]: | |
| import os | |
| value = os.environ.get("PANDAS_SETTING", "<unset>") | |
| for batch in batches: | |
| yield batch.map(lambda _: value) | |
| result = self.spark.range(1).select(read_env("id").alias("value")).collect() | |
| self.assertEqual(result[0]["value"], "iterated") | |
| finally: | |
| self._unset_env("PANDAS_SETTING") | |
| def test_env_var_reaches_apply_in_pandas(self): | |
| import pandas as pd | |
| self._set_env("GROUP_SETTING", "grouped") | |
| try: | |
| def func(pdf): | |
| import os | |
| return pd.DataFrame({"value": [os.environ.get("GROUP_SETTING", "<unset>")]}) | |
| result = self.spark.range(1).groupBy("id").applyInPandas(func, "value string").collect() | |
| self.assertEqual(result[0]["value"], "grouped") | |
| finally: | |
| self._unset_env("GROUP_SETTING") | |
| def test_env_var_reaches_cogrouped_apply_in_pandas(self): | |
| # The cogrouped path has a runner of its own, so it is not covered by the grouped test. | |
| import pandas as pd | |
| self._set_env("COGROUP_SETTING", "cogrouped") | |
| try: | |
| def func(left, right): | |
| import os | |
| return pd.DataFrame({"value": [os.environ.get("COGROUP_SETTING", "<unset>")]}) | |
| left = self.spark.range(1) | |
| right = self.spark.range(1) | |
| result = ( | |
| left.groupBy("id") | |
| .cogroup(right.groupBy("id")) | |
| .applyInPandas(func, "value string") | |
| .collect() | |
| ) | |
| self.assertEqual(result[0]["value"], "cogrouped") | |
| finally: | |
| self._unset_env("COGROUP_SETTING") | |
| def test_env_var_reaches_grouped_agg_pandas_udf(self): | |
| import pandas as pd | |
| from pyspark.sql.functions import pandas_udf | |
| self._set_env("AGG_SETTING", "aggregated") | |
| try: | |
| @pandas_udf("string") | |
| def read_env(values: pd.Series) -> str: | |
| import os | |
| return os.environ.get("AGG_SETTING", "<unset>") | |
| result = self.spark.range(1).groupBy("id").agg(read_env("id").alias("value")).collect() | |
| self.assertEqual(result[0]["value"], "aggregated") | |
| finally: | |
| self._unset_env("AGG_SETTING") | |
| def test_env_var_reaches_window_agg_pandas_udf(self): | |
| import pandas as pd | |
| from pyspark.sql.functions import pandas_udf | |
| from pyspark.sql.window import Window | |
| self._set_env("WINDOW_SETTING", "windowed") | |
| try: | |
| @pandas_udf("string") | |
| def read_env(values: pd.Series) -> str: | |
| import os | |
| return os.environ.get("WINDOW_SETTING", "<unset>") | |
| window = Window.partitionBy("id").rowsBetween( | |
| Window.unboundedPreceding, Window.unboundedFollowing | |
| ) | |
| result = ( | |
| self.spark.range(1).select(read_env("id").over(window).alias("value")).collect() | |
| ) | |
| self.assertEqual(result[0]["value"], "windowed") | |
| finally: | |
| self._unset_env("WINDOW_SETTING") | |
| def test_env_var_reaches_python_udtf(self): | |
| from pyspark.sql.functions import udtf | |
| self._set_env("UDTF_SETTING", "tabular") | |
| try: | |
| @udtf(returnType="value string", useArrow=False) | |
| class ReadEnv: | |
| def eval(self): | |
| import os | |
| yield (os.environ.get("UDTF_SETTING", "<unset>"),) | |
| self.assertEqual(ReadEnv().collect()[0]["value"], "tabular") | |
| finally: | |
| self._unset_env("UDTF_SETTING") | |
| def test_env_var_reaches_arrow_python_udtf(self): | |
| # The Arrow UDTF has a runner of its own, separate from the row UDTF's. | |
| from pyspark.sql.functions import udtf | |
| self._set_env("UDTF_SETTING", "tabular") | |
| try: | |
| @udtf(returnType="value string", useArrow=True) | |
| class ReadEnv: | |
| def eval(self): | |
| import os | |
| yield (os.environ.get("UDTF_SETTING", "<unset>"),) | |
| self.assertEqual(ReadEnv().collect()[0]["value"], "tabular") | |
| finally: | |
| self._unset_env("UDTF_SETTING") | |
| def test_env_var_reaches_a_streaming_foreach_writer(self): | |
| """`writeStream.foreach` runs the user's `process` in a worker, on classic and on Connect. | |
| The worker cannot return a value, so it records what it saw in a file. A file source with | |
| `availableNow` keeps the query bounded rather than relying on wall-clock timing. | |
| """ | |
| import os | |
| import shutil | |
| import tempfile | |
| source_dir = tempfile.mkdtemp() | |
| observed_dir = tempfile.mkdtemp() | |
| self._set_env("FOREACH_SETTING", "written") | |
| try: | |
| with open(os.path.join(source_dir, "input.txt"), "w") as handle: | |
| handle.write("a row\n") | |
| def process_row(row): | |
| import os | |
| import uuid | |
| path = os.path.join(observed_dir, "observed-{}.txt".format(uuid.uuid4().hex)) | |
| with open(path, "w") as out: | |
| out.write(os.environ.get("FOREACH_SETTING", "<unset>")) | |
| query = ( | |
| self.spark.readStream.format("text") | |
| .load(source_dir) | |
| .writeStream.foreach(process_row) | |
| .trigger(availableNow=True) | |
| .start() | |
| ) | |
| try: | |
| query.awaitTermination(timeout=120) | |
| finally: | |
| query.stop() | |
| observed = sorted(os.listdir(observed_dir)) | |
| self.assertNotEqual(observed, [], "foreach never processed a row") | |
| with open(os.path.join(observed_dir, observed[0])) as handle: | |
| self.assertEqual(handle.read(), "written") | |
| finally: | |
| self._unset_env("FOREACH_SETTING") | |
| shutil.rmtree(source_dir, ignore_errors=True) | |
| shutil.rmtree(observed_dir, ignore_errors=True) | |
| def test_env_var_reaches_a_native_arrow_udtf(self): | |
| # `arrow_udtf` is a separate API from `udtf(useArrow=True)` and produces a different | |
| # evaluation type, so it does not follow from the other UDTF tests. | |
| import pyarrow as pa | |
| from pyspark.sql.functions import arrow_udtf | |
| self._set_env("UDTF_SETTING", "arrow_native") | |
| try: | |
| @arrow_udtf(returnType="value string") | |
| class ReadEnv: | |
| def eval(self): | |
| import os | |
| yield pa.table({"value": [os.environ.get("UDTF_SETTING", "<unset>")]}) | |
| self.assertEqual(ReadEnv().collect()[0]["value"], "arrow_native") | |
| finally: | |
| self._unset_env("UDTF_SETTING") | |
| def test_a_registered_udf_uses_the_environment_of_each_execution(self): | |
| # Registration keeps the function as supplied; the environment is merged when a worker | |
| # launches. So a value set, changed or unset after registration takes effect on the next | |
| # execution rather than being captured at registration. | |
| def read_env(): | |
| import os | |
| return os.environ.get("REGISTERED_SETTING", "<unset>") | |
| self.spark.udf.register("read_registered_env", read_env, "string") | |
| query = "SELECT read_registered_env() AS value" | |
| try: | |
| self._set_env("REGISTERED_SETTING", "first") | |
| self.assertEqual(self.spark.sql(query).collect()[0]["value"], "first") | |
| self._set_env("REGISTERED_SETTING", "second") | |
| self.assertEqual(self.spark.sql(query).collect()[0]["value"], "second") | |
| finally: | |
| self._unset_env("REGISTERED_SETTING") | |
| self.assertEqual(self.spark.sql(query).collect()[0]["value"], "<unset>") | |
| def test_env_var_reaches_a_data_source_planning_worker(self): | |
| """A planning worker is a separate process from the ones that run a query's partitions. | |
| The variable is read in `schema()`, which runs in that planning worker, and decides the | |
| column name. Asserting on the resulting schema therefore fails if only the execution-side | |
| workers received the environment. | |
| """ | |
| from pyspark.sql.datasource import DataSource, DataSourceReader | |
| self._set_env("PLANNING_SETTING", "planned") | |
| try: | |
| class EnvNamedReader(DataSourceReader): | |
| def read(self, partition): | |
| yield ("row",) | |
| class EnvNamedDataSource(DataSource): | |
| @classmethod | |
| def name(cls): | |
| return "env_named_source" | |
| def schema(self): | |
| import os | |
| return "{} string".format(os.environ.get("PLANNING_SETTING", "unset")) | |
| def reader(self, schema): | |
| return EnvNamedReader() | |
| self.spark.dataSource.register(EnvNamedDataSource) | |
| df = self.spark.read.format("env_named_source").load() | |
| self.assertEqual(df.schema.fieldNames(), ["planned"]) | |
| finally: | |
| self._unset_env("PLANNING_SETTING") | |
| # -- rejection ------------------------------------------------------------ | |
| def test_invalid_name_fails_the_set(self): | |
| # `spark.conf.set` goes through `RuntimeConfig.set` on both front ends -- the Spark Connect | |
| # server writes through a classic `RuntimeConfig` too -- so an invalid variable is refused | |
| # at the call rather than at a later query. | |
| with self.assertRaises(Exception) as context: | |
| self.spark.conf.set(PREFIX + "1INVALID", "x") | |
| message = str(context.exception) | |
| # Assert on the message text rather than the condition name, which a client is not required | |
| # to surface in the string form of the exception. | |
| self.assertIn("is not valid", message) | |
| self.assertIn("1INVALID", message) | |
| # The write was refused, so nothing was stored to break later queries. | |
| self.assertIsNone(self.spark.conf.get(PREFIX + "1INVALID", None)) | |
| def test_value_containing_nul_fails_the_set(self): | |
| with self.assertRaises(Exception) as context: | |
| self.spark.conf.set(PREFIX + "WITH_NUL", "abc" + NUL + "def") | |
| message = str(context.exception) | |
| self.assertIn("NUL character", message) | |
| self.assertIn("WITH_NUL", message) | |
| # A value can be a secret, so it must not appear in the failure. | |
| self.assertNotIn("abc", message) | |
| self.assertIsNone(self.spark.conf.get(PREFIX + "WITH_NUL", None)) | |
| def test_env_var_reaches_a_udf_inside_a_higher_order_function(self): | |
| # A UDF written inside a lambda is lifted to the element-wise eval type, which is enabled by | |
| # default. It is the same UDF to the user, so it has to receive the environment too. | |
| from pyspark.sql.functions import array, transform | |
| self._set_env("HOF_SETTING", "lifted") | |
| try: | |
| def read_env(_): | |
| import os | |
| return os.environ.get("HOF_SETTING", "<unset>") | |
| read_env_udf = udf(read_env, "string") | |
| df = self.spark.range(1).select( | |
| transform(array(lit(1), lit(2)), lambda x: read_env_udf(x)).alias("value") | |
| ) | |
| self.assertEqual(df.collect()[0]["value"], ["lifted", "lifted"]) | |
| finally: | |
| self._unset_env("HOF_SETTING") | |
| class PythonWorkerEnvTests(PythonWorkerEnvMixin, ReusedSQLTestCase): | |
| @classmethod | |
| def conf(cls): | |
| # An application-scoped variable, so the tests can pin the precedence between it and a | |
| # session variable of the same name. | |
| return super().conf().set("spark.executorEnv.SHARED", "from_application") | |
| # `spark.executorEnv.*` reaches a Python worker only on classic: `SparkContext` copies those | |
| # entries into `sc.environment` (core/context.py), which becomes the function's own envVars. | |
| # A Spark Connect client has no SparkContext and the server builds the function with an empty | |
| # map, so there is no inherited value there to fall back to or to override. | |
| def test_unset_reveals_the_inherited_value(self): | |
| self._set_env("SHARED", "from_session") | |
| try: | |
| self.assertEqual(self._read_in_worker("SHARED"), "from_session") | |
| finally: | |
| self._unset_env("SHARED") | |
| self.assertEqual(self._read_in_worker("SHARED"), "from_application") | |
| def test_an_environment_stored_by_a_bypass_path_fails_at_worker_launch(self): | |
| # `RuntimeConfig.set` now refuses an invalid variable, and SQL `SET` reaches it too, so the | |
| # only way left into a bad session state is a write straight to `SQLConf` -- which is what | |
| # the configurations merged into a new session by `SparkSession.builder` do. Reached here | |
| # through the JVM, so this is classic-only, and it is the end-to-end proof that the check | |
| # performed when a worker is launched is still what makes an invalid environment unusable. | |
| jconf = self.spark._jsparkSession.sessionState().conf() | |
| jconf.setConfString(PREFIX + "1INVALID", "x") | |
| try: | |
| self.assertEqual(self.spark.conf.get(PREFIX + "1INVALID"), "x") | |
| with self.assertRaises(Exception) as context: | |
| self._read_in_worker("ANY") | |
| self.assertIn("1INVALID", str(context.exception)) | |
| finally: | |
| self._unset_env("1INVALID") | |
| if __name__ == "__main__": | |
| from pyspark.testing import main | |
| main() |