)]}' { "commit": "264ca4dc320e516bd88679ed2404f1c15d2f24eb", "tree": "89bccb1440c2004b3b8c47b4ceb18ec3dafa2ee0", "parents": [ "3ac4a48fb8c5e0bdd9e7f0a4b98094531a7807f6" ], "author": { "name": "vicennial", "email": "venkata.gudesa@databricks.com", "time": "Mon Oct 13 11:01:58 2025 -0400" }, "committer": { "name": "Herman van Hovell", "email": "herman@databricks.com", "time": "Mon Oct 13 11:01:58 2025 -0400" }, "message": "[SPARK-53455][CONNECT] Add `CloneSession` RPC\n\n### What changes were proposed in this pull request?\n\nAdds a new experimental/developer RPC `CloneSession` to the `SparkConnectService`.\n\n ✅ CLONED (from SparkSession)\n\n - SessionState - SQL configs, temp views, UDFs, catalog metadata\n - ArtifactManager - JARs, files, classes added to session\n - ManagedJobTags - Job group tags for tracking\n - SharedState (reference) - Metastore, global temp views\n - SparkContext (reference) - Core Spark engine\n\n ❌ NOT CLONED (SessionHolder - Spark Connect layer)\n\n - planCache - (Partially analyzed) Logical plans for query optimization\n - operationIds - Currently executing operations\n - errorIdToError - Recent errors for debugging\n - eventManager - Session lifecycle events\n - dataFrameCache - DataFrames for foreachBatch callbacks\n - mlCache - ML models and pipelines\n - listenerCache - Streaming query listeners\n - pipelineExecutions - Active pipeline contexts\n - dataflowGraphRegistry - Registered dataflow graphs\n - streamingForeachBatchRunnerCleanerCache - Python streaming workers\n - pythonAccumulator - Python metrics collection\n - Session timings - Start time, last access, custom timeout\n\nThe clone preserves all SQL/catalog state but creates a fresh runtime environment. An analogy is cloning a database schema/config but not the active connections, caches, or running jobs.\n### Why are the changes needed?\n\nSpark Connect introduced the concept of resource isolation (via `ArtifactManager`, which has been ported to classic Spark) and thus, jars/pyfiles/artifacts added to each session are isolated from other sessions.\n\nA slight rough edge is that if a user wishes to fork the state of a session but maintain independence, the only possible way is to create a new session and reupload/reinit all base jars/artifacts/pyfiles, etc.\n\nSupport for cloning through the API helps address the rough edge while maintaining all the benefits of session resource isolation.\n### Does this PR introduce _any_ user-facing change?\n\nYes\n\n```python\n spark \u003d SparkSession.builder.remote(\"sc://localhost\").getOrCreate()\n spark.conf.set(\"my.custom.config\", \"value\")\n spark.addArtifact(\"/path/to/my.jar\")\n spark.sql(\"CREATE TEMP VIEW my_view AS SELECT 1 AS id\")\n\n # Clone the session\n cloned_spark \u003d spark.cloneSession()\n\n # The cloned session has all the same state\n assert cloned_spark.conf.get(\"my.custom.config\") \u003d\u003d \"value\"\n assert cloned_spark.sql(\"SELECT * FROM my_view\").collect() \u003d\u003d [Row(id\u003d1)]\n\n # But operations are isolated between sessions\n cloned_spark.sql(\"DROP VIEW my_view\") # Only affects cloned session\n spark.sql(\"SELECT * FROM my_view\").collect() # Original still works\n```\n\n### How was this patch tested?\n\nNew individual unit tests along with new test suites.\n\n### Was this patch authored or co-authored using generative AI tooling?\n\nCo-authored with assistance from Claude Code.\n\nCloses #52200 from vicennial/cloneAPI.\n\nAuthored-by: vicennial \u003cvenkata.gudesa@databricks.com\u003e\nSigned-off-by: Herman van Hovell \u003cherman@databricks.com\u003e\n", "tree_diff": [ { "type": "modify", "old_id": "8f6687587f781de6b191ed714f567c87011dfc9a", "old_mode": 33188, "old_path": "common/utils/src/main/resources/error/error-conditions.json", "new_id": "afe56f6db2fd2c8f641980ada3bb227e3c0e1e0a", "new_mode": 33188, "new_path": "common/utils/src/main/resources/error/error-conditions.json" }, { "type": "modify", "old_id": "8aab600071dc10209a5a63ffe23bddaf9f9b9f2d", "old_mode": 33188, "old_path": "dev/sparktestsupport/modules.py", "new_id": "945a2ac9189b09a099d036208ad52c87e69d76b8", "new_mode": 33188, "new_path": "dev/sparktestsupport/modules.py" }, { "type": "modify", "old_id": "741d612f53f47b38a4c2a66e70886dd318cd33bf", "old_mode": 33188, "old_path": "python/pyspark/sql/connect/client/core.py", "new_id": "d0d191dbd7fde3eeb29ca5a332effd819b6cf7e2", "new_mode": 33188, "new_path": "python/pyspark/sql/connect/client/core.py" }, { "type": "modify", "old_id": "269f5eefbd267ebc6f531b2415d8aa93387e5a29", "old_mode": 33188, "old_path": "python/pyspark/sql/connect/proto/base_pb2.py", "new_id": "069e0830fcea768aa25d449a2d575d5a06d19e29", "new_mode": 33188, "new_path": "python/pyspark/sql/connect/proto/base_pb2.py" }, { "type": "modify", "old_id": "3361649d9323cde7ee4f8e2b46fa761dba41402c", "old_mode": 33188, "old_path": "python/pyspark/sql/connect/proto/base_pb2.pyi", "new_id": "dc3099ecdffcaca4de602d0a17949e6b2380518e", "new_mode": 33188, "new_path": "python/pyspark/sql/connect/proto/base_pb2.pyi" }, { "type": "modify", "old_id": "7501aaf0a3a235795f7d00bcb1b14d0e14c289e7", "old_mode": 33188, "old_path": "python/pyspark/sql/connect/proto/base_pb2_grpc.py", "new_id": "16dd20b563f3b125ec8a596d751f75b099b7335d", "new_mode": 33188, "new_path": "python/pyspark/sql/connect/proto/base_pb2_grpc.py" }, { "type": "modify", "old_id": "6ccffc718d0645e883dcd1c70f00672089e4f8c3", "old_mode": 33188, "old_path": "python/pyspark/sql/connect/session.py", "new_id": "f759137fac1d20ebdaf3a2b535373dc58d4e0bc4", "new_mode": 33188, "new_path": "python/pyspark/sql/connect/session.py" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "1cef628a158cf6073344a6272263892dd7b7fbf7", "new_mode": 33188, "new_path": "python/pyspark/sql/tests/connect/test_connect_clone_session.py" }, { "type": "modify", "old_id": "b2e0cc6229c4ba68e3897af58bcda6797249ec82", "old_mode": 33188, "old_path": "python/pyspark/sql/tests/test_connect_compatibility.py", "new_id": "4ca1c04bf2d07f93e5732f856d224fc8deadc991", "new_mode": 33188, "new_path": "python/pyspark/sql/tests/test_connect_compatibility.py" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "97ad21b2675fdaf43dbc183f82d35cd3381cdbb9", "new_mode": 33188, "new_path": "sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/SparkSessionCloneSuite.scala" }, { "type": "modify", "old_id": "f34b169f821d1873f44e219de1a931ff45606b3f", "old_mode": 33188, "old_path": "sql/connect/common/src/main/protobuf/spark/connect/base.proto", "new_id": "6e1029bf0a6a268f7c9c97c8da9993d3adcebfcb", "new_mode": 33188, "new_path": "sql/connect/common/src/main/protobuf/spark/connect/base.proto" }, { "type": "modify", "old_id": "646db83981fe21474a7df5bfe792dd21a8adc88a", "old_mode": 33188, "old_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/SparkSession.scala", "new_id": "f7869a8b4dd88eda9259afd6cf5fcaa3c1e5c35d", "new_mode": 33188, "new_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/SparkSession.scala" }, { "type": "modify", "old_id": "d7867229248b81e531f8a29a4ff411c9722fad7d", "old_mode": 33188, "old_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/CustomSparkConnectBlockingStub.scala", "new_id": "913f068fcf34593abb8e549d80af09d8a6fb3254", "new_mode": 33188, "new_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/CustomSparkConnectBlockingStub.scala" }, { "type": "modify", "old_id": "48f01a8042a645f1d985dca21ffb018999a25451", "old_mode": 33188, "old_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala", "new_id": "3c328681dd9a8366dffb8814d6287c27efbdd2c7", "new_mode": 33188, "new_path": "sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "c0d6da9389324a5e0cef922e7710cdf9ef006c90", "new_mode": 33188, "new_path": "sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectCloneSessionHandler.scala" }, { "type": "modify", "old_id": "cc1cb95b66c468dfbfddecc1783d2d1a383dcb41", "old_mode": 33188, "old_path": "sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala", "new_id": "13ce2d64256b46b2b225454e8e13600a9210e9f4", "new_mode": 33188, "new_path": "sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala" }, { "type": "modify", "old_id": "1c3cfd67f132ca407ec2eea5ca43e3184fbef2a5", "old_mode": 33188, "old_path": "sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala", "new_id": "f28af0379a04c542c72f56ada47773d3e2027183", "new_mode": 33188, "new_path": "sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "922c239526f31d3fc0cdebe7812b084662ea9800", "new_mode": 33188, "new_path": "sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectCloneSessionSuite.scala" } ] }