)]}'
{
  "commit": "aa10c87dd3194b0866c8e34884875bd63aff7de7",
  "tree": "9b747fb52171da7eb388a3be42b43ad27d3d3fbf",
  "parents": [
    "8e1e126c68aaa94f0fa6861b93042e1385615418"
  ],
  "author": {
    "name": "Eddie Bkheet",
    "email": "eddie.bkheet@databricks.com",
    "time": "Fri Jan 23 13:34:27 2026 -0800"
  },
  "committer": {
    "name": "Takuya Ueshin",
    "email": "ueshin@databricks.com",
    "time": "Fri Jan 23 13:34:27 2026 -0800"
  },
  "message": "[SPARK-55046][PYTHON] PySpark add udf processing time metric\n\n### What changes were proposed in this pull request?\n\nAdds a new **pythonProcessingTime** metric to capture the actual Python code execution time in Python UDFs, UDTFs, and Python-based operations, separate from worker boot and initialization overhead.\n\n### Why are the changes needed?\n\nMotivation: the existing pythonTotalTime metric includes:\n\nPython worker boot time (pythonBootTime)\nPython worker initialization time (pythonInitTime)\nActual Python code execution time\nThis makes it difficult to identify whether performance issues are due to worker startup overhead or the UDF logic itself. The new processingTimeNs metric isolates just the Python code execution time for better observability.\n\n### Does this PR introduce _any_ user-facing change?\n\nan additional metric will be displayed in the spark UI\n\n### How was this patch tested?\n\nnew UTs\n\n### Was this patch authored or co-authored using generative AI tooling?\n\nYes\n\nCloses #53831 from eddiebkheet/user/eddiebkheet/metrics/udf-processing-time.\n\nAuthored-by: Eddie Bkheet \u003ceddie.bkheet@databricks.com\u003e\nSigned-off-by: Takuya Ueshin \u003cueshin@databricks.com\u003e\n",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "5015dafbbb2d369bb43d57bd79688ebf594bc6f0",
      "old_mode": 33188,
      "old_path": "core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala",
      "new_id": "f830dd2d8b6e71b372a7656a7c35c70307bb073d",
      "new_mode": 33188,
      "new_path": "core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala"
    },
    {
      "type": "modify",
      "old_id": "0df7eaf709c83ca9b53ebfaedb9920d22e3ed1fe",
      "old_mode": 33188,
      "old_path": "python/pyspark/worker.py",
      "new_id": "2e79c981d818718f5b9da43c91f28411f76ee00e",
      "new_mode": 33188,
      "new_path": "python/pyspark/worker.py"
    },
    {
      "type": "modify",
      "old_id": "43a47d1f7d90761563aac8759900ea3ef7de92d6",
      "old_mode": 33188,
      "old_path": "sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala",
      "new_id": "cbce07977f16df6d3f5d9bf6fa1f90079df5f809",
      "new_mode": 33188,
      "new_path": "sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala"
    },
    {
      "type": "modify",
      "old_id": "9b40226c2049b51afebaf57723c485e038415cc6",
      "old_mode": 33188,
      "old_path": "sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala",
      "new_id": "6d5c459514e58e95b8f3f08a2f769edc2783f2ad",
      "new_mode": 33188,
      "new_path": "sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala"
    }
  ]
}