)]}' { "log": [ { "commit": "09094f5da4c0ac4e0605563768114c536a817a1b", "tree": "704dd7092b9e8d7ca5427146d49e47e315e8aa90", "parents": [ "1738c33e5669b5778a3be3cdfc6cfa385f085062" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Feb 18 07:02:32 2026 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Feb 18 07:02:32 2026 -0800" }, "message": "[SPARK-55548] Enable `GitHub Issues` feature (#106)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to enable `GitHub Issues` feature.\n\n### Why are the changes needed?\n\nThis will enable `GitHub Issues` feature after the vote passes.\n\n[[VOTE] Open github issues as an experimental option](https://lists.apache.org/thread/kv11qlr8j05cwqjoyddybclwcn0nv2n7).\n\n### Does this PR introduce _any_ user-facing change?\n\nNo. This is an infra change.\n\n### How was this patch tested?\n\nManual review.\n\n### Was this patch authored or co-authored using generative AI tooling?\n\nNo." }, { "commit": "1738c33e5669b5778a3be3cdfc6cfa385f085062", "tree": "3f2eb221cbe0dfe07eadc7f84ef62328bbf0e77c", "parents": [ "1e6fe4d2c19bd4ac1bfd3b1ec7c1f90ccd25cf12" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Sun Feb 15 23:21:17 2026 -0800" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Sun Feb 15 23:21:17 2026 -0800" }, "message": "[SPARK-55549] Update PR template according to the ASF Generative Tooling Guidance recommendations\n\n### What changes were proposed in this pull request?\n\nThis PR aims to update PR template according to the ASF Generative Tooling Guidance recommendations.\n\n### Why are the changes needed?\n\nLike the Apache Spark main repositories, we had better follow the ASF guideline.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review.\n\nCloses #107 from dongjoon-hyun/SPARK-55549.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "1e6fe4d2c19bd4ac1bfd3b1ec7c1f90ccd25cf12", "tree": "ee1420cb6c8558a2b0c4bb117310f9dbba43a517", "parents": [ "b07cb7ac2815143315ea9ebaddcb0f5c3aa887c2" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 05 15:37:59 2026 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Feb 05 15:37:59 2026 -0800" }, "message": "[SPARK-55378] Publish Apache Spark 4.0.2 to docker registry (#104)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark `4.0.2` to docker registry.\n\n### Why are the changes needed?\n\nApache Spark 4.0.2 is released officially.\n- https://github.com/apache/spark/releases/tag/v4.0.2\n- https://spark.apache.org/docs/4.0.2/\n- https://dist.apache.org/repos/dist/release/spark/spark-4.0.2/\n\nWe need to provide a docker image of Apache Spark `4.0.2`.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "b07cb7ac2815143315ea9ebaddcb0f5c3aa887c2", "tree": "c9047bf35ace9d9f2fa0062f0de35a5f06fba5ee", "parents": [ "dd49d350dc0ee517299bd79f4be369f1e39fb9c2" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 05 14:52:34 2026 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Feb 05 14:52:34 2026 -0800" }, "message": "[SPARK-55375] Remove all 4.0 and 4.1 preview releases (#103)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to remove all 4.0 and 4.1 preview releases.\n\n### Why are the changes needed?\n\nSince we successfully delivered 4.0 and 4.1, we don\u0027t need these preview contents. Technically, we have too many preview contents because we made monthly releases. \n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "dd49d350dc0ee517299bd79f4be369f1e39fb9c2", "tree": "0a92daef1d18b5154c5a72eef91db38ed69e9ac6", "parents": [ "b0876281ebd5ef739bf6103a5980b8a38cd957eb" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Jan 16 11:20:12 2026 +0900" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Jan 16 11:20:12 2026 +0900" }, "message": "[SPARK-55060] Publish Apache Spark 3.5.8 to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark `3.5.8` to docker registry.\n\n### Why are the changes needed?\n\nApache Spark 3.5.8 is released officially.\n- https://github.com/apache/spark/releases/tag/v3.5.8\n- https://dist.apache.org/repos/dist/release/spark/spark-3.5.8/\n\nWe need to provide a docker image of Apache Spark `3.5.8`.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #102 from dongjoon-hyun/SPARK-55060.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "b0876281ebd5ef739bf6103a5980b8a38cd957eb", "tree": "0320a73e2b68180d696933f2aecefc9a01099e30", "parents": [ "205cc96059fdeb61eef7a2f17cdb37a3744f8d80" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Sat Jan 10 10:22:00 2026 +0900" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Jan 10 10:22:00 2026 +0900" }, "message": "[SPARK-54994] Publish Apache Spark `4.1.1` to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark `4.1.1` to docker registry.\n\n### Why are the changes needed?\n\nApache Spark 4.1.1 is released officially.\n- https://github.com/apache/spark/releases/tag/v4.1.1\n- https://dist.apache.org/repos/dist/release/spark/spark-4.1.1/\n\nWe need to provide a docker image of Apache Spark `4.1.1`.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "205cc96059fdeb61eef7a2f17cdb37a3744f8d80", "tree": "cfe96f536a2570b2ada1c10c661fd411efecd56f", "parents": [ "763fbfda79c3bd5e43c68932c6f8bdd8c21d00ab" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Dec 17 23:20:50 2025 -0800" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Dec 17 23:20:50 2025 -0800" }, "message": "[SPARK-54748][FOLLOWUP] Add 4.1.0 to `publish-java*.yaml` and `test.yml` files\n\n### What changes were proposed in this pull request?\n\nThis PR is a follow-up of the following to add `4.1.0` to `publish-java*.yaml` and `test.yml` GitHub Action CI files.\n- #99\n\n### Why are the changes needed?\n\nPreviously, these was missed.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual reviews because these CIs are triggered manually only.\n\nCloses #100 from dongjoon-hyun/SPARK-54748-2.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "763fbfda79c3bd5e43c68932c6f8bdd8c21d00ab", "tree": "b5950521919eaae30da37ff39784f78256221a2e", "parents": [ "331850d47f43fbe96fd77c7635b611a65d520844" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Dec 17 23:01:59 2025 -0800" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Dec 17 23:01:59 2025 -0800" }, "message": "[SPARK-54748] Publish Apache Spark `4.1.0` to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark `4.1.0` to docker registry.\n\n### Why are the changes needed?\n\nApache Spark 4.1.0 is released officially.\n- https://github.com/apache/spark/releases/tag/v4.1.0\n- https://spark.apache.org/docs/4.1.0/\n- https://dist.apache.org/repos/dist/release/spark/spark-4.1.0/\n\nWe need to provide a docker image of Apache Spark `4.1.0`.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #99 from dongjoon-hyun/SPARK-54748.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "331850d47f43fbe96fd77c7635b611a65d520844", "tree": "31f1b6fe5f5b07b5ef7d25ef9fafd52a56182a2a", "parents": [ "57da4875301b3b5983f0bb80e80562cb072ab844" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Nov 19 17:41:52 2025 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Nov 19 17:41:52 2025 -0800" }, "message": "[SPARK-54429] Publish Apache Spark `4.1.0-preview4` to docker registry (#98)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark `4.1.0-preview4` to docker registry.\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark `4.1.0-preview4`.\n\nNote that this PR doesn\u0027t change `latest`, `scala`, `python3`, `python3-java17`, `r` tags intentionally because this is a preview version.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "57da4875301b3b5983f0bb80e80562cb072ab844", "tree": "9db51466535beb2fd1c86ce3302373bd14cc7c4b", "parents": [ "2ebf694ad45fee6f4beeeb4204bcdb01d73c988f" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Oct 30 20:56:20 2025 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Oct 30 20:56:20 2025 -0700" }, "message": "[SPARK-54084] Publish Apache Spark 4.1.0-preview3 to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark 4.1.0-preview3 to docker registry.\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 4.1.0-preview3.\n\nNote that this PR doesn\u0027t change `latest`, `scala`, `python3`, `python3-java17`, `r` tags intentionally because this is a preview version.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #97 from dongjoon-hyun/SPARK-54084.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "2ebf694ad45fee6f4beeeb4204bcdb01d73c988f", "tree": "20ea2d1669b08c6aa003879767aee40c6a12883d", "parents": [ "29bc18a98e1145132d7a28250a2f31e8d066ba2e" ], "author": { "name": "Peter Toth", "email": "peter.toth@gmail.com", "time": "Tue Oct 07 18:34:28 2025 +0200" }, "committer": { "name": "Peter Toth", "email": "peter.toth@gmail.com", "time": "Tue Oct 07 18:34:28 2025 +0200" }, "message": "[SPARK-53730] Switch Spark 3.5 Java 11 docker registry images to 11-jammy\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to switch future Apache Spark 3.5 Java 11 docket images to `11-jammy`.\nWe have applied a similar change to Apache Spark repo Java 17 dockerfile in [SPARK-49005](https://issues.apache.org/jira/browse/SPARK-49005) / https://github.com/apache/spark/pull/47488.\n\n### Why are the changes needed?\n\nThe current `11-jre-focal` is no longer supported by Docker Library: https://github.com/docker-library/official-images/pull/19948#issuecomment-3335268352\n\n### Does this PR introduce _any_ user-facing change?\n\nNo, we will not republish the 3.5.7 Apache Docker Hub image.\n\n### How was this patch tested?\n\nManual review.\n\nCloses #95 from peter-toth/SPARK-53730-switch-spark-3.5-to-11-jammy.\n\nAuthored-by: Peter Toth \u003cpeter.toth@gmail.com\u003e\nSigned-off-by: Peter Toth \u003cpeter.toth@gmail.com\u003e\n" }, { "commit": "29bc18a98e1145132d7a28250a2f31e8d066ba2e", "tree": "2b6e51cee6855e164940ec4022663e9d17c0a1ed", "parents": [ "8e8bdcd6a559fbf1eab6cee73c60a8e72a9e4a80" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Mon Sep 29 14:44:58 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Sep 29 14:44:58 2025 -0700" }, "message": "[SPARK-53752] Publish Apache Spark 4.1.0-preview2 to docker registry (#96)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark 4.1.0-preview2 to docker registry.\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 4.1.0-preview2.\n\nNote that this PR doesn\u0027t change `latest`, `scala`, `python3`, `python3-java17`, `r` tags intentionally because this is a preview version.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "8e8bdcd6a559fbf1eab6cee73c60a8e72a9e4a80", "tree": "bc3f1d05c9af919ca599be53f12943d255dc9817", "parents": [ "a5edefc5f796902d5cc15f7d840bb6335f283797" ], "author": { "name": "Peter Toth", "email": "peter.toth@gmail.com", "time": "Wed Sep 24 20:41:35 2025 +0200" }, "committer": { "name": "Peter Toth", "email": "p_toth@apple.com", "time": "Wed Sep 24 20:41:35 2025 +0200" }, "message": "[SPARK-53693] Publish Apache Spark 3.5.7 to docker registry\n\n### What changes were proposed in this pull request?\nThis PR proposes to publish Apache Spark 3.5.7 to docker registry\n\n### Why are the changes needed?\nTo provide a docker image of Apache Spark 3.5.7.\n\n### Does this PR introduce _any_ user-facing change?\nNo.\n\n### How was this patch tested?\nManual review.\n\nCloses #94 from peter-toth/spark-3.5.7.\n\nLead-authored-by: Peter Toth \u003cpeter.toth@gmail.com\u003e\nCo-authored-by: Peter Toth \u003cp_toth@apple.com\u003e\nSigned-off-by: Peter Toth \u003cp_toth@apple.com\u003e\n" }, { "commit": "a5edefc5f796902d5cc15f7d840bb6335f283797", "tree": "22b1a74bda274800e40127b5437f25ca7a88f3d1", "parents": [ "a88154a8b0d8a7af4ced1272721f415de54868da" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Sat Sep 06 14:15:57 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Sep 06 14:15:57 2025 -0700" }, "message": "[SPARK-53509] Publish Apache Spark 4.0.1 to docker registry (#93)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to Publish Apache Spark 4.0.1 to docker registry.\n\n### Why are the changes needed?\n\nTo provide the latest docker images.\n\n### Does this PR introduce _any_ user-facing change?\n\nYes, the latest tags are changed to 4.0.1 from 4.0.0. These are expected changes.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "a88154a8b0d8a7af4ced1272721f415de54868da", "tree": "472265cc82835ffe072c09b04656aa46d6b15ad6", "parents": [ "67603f320982c8176be2d96f9d4ee1fc10d90376" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 03 11:20:55 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Sep 03 11:20:55 2025 -0700" }, "message": "[SPARK-53475] Use `setup-minikube` in `spark-docker` repository (#92)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to use `setup-minikube` in `spark-docker` repository.\n\n### Why are the changes needed?\n\nTo simplify the GitHub Action:\n- `setup-minikube` is ASF-approved GitHub Action.\n- All Apache Spark repositories (except `spark-docker`) have been using `setup-minikube` as of now.\n\nCurrently, `v0.0.20` is the latest one.\n- https://github.com/medyagh/setup-minikube/releases/tag/v0.0.20 (2025-07-08)\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs." }, { "commit": "67603f320982c8176be2d96f9d4ee1fc10d90376", "tree": "96e7e9b5236228fde5633b11d13d7dde48eaf467", "parents": [ "0f76cd1f98e924a07fb6a5551807015b634a92a2" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 03 11:14:57 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Sep 03 11:14:57 2025 -0700" }, "message": "[SPARK-53473] Publish Apache Spark 4.1.0-preview1 to docker registry (#91)\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to publish Apache Spark 4.1.0-preview1 to docker registry\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 4.1.0-preview1.\n\nNote that this PR doesn\u0027t change `latest`, `scala`, `python3`, `python3-java17`, `r` tags intentionally because this is a preview version.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review." }, { "commit": "0f76cd1f98e924a07fb6a5551807015b634a92a2", "tree": "9f613a61f32302831056764fc6e38a1bd2ab38fe", "parents": [ "bbfad0134f7a3534f4cd735ccd7b1ffe84ce6ae8" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Mon Jun 23 09:16:26 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jun 23 09:16:26 2025 -0700" }, "message": "[SPARK-52542] Use `/nonexistent` instead of nonexistent `/opt/spark` (#87)\n\n### What changes were proposed in this pull request?\n\nThis PR aims to use `/nonexistent` explicitly instead of nonexistent `/home/spark` because the current status is misleading.\n\nPlease note that SPARK-40528 introduced `useradd --system` which created `spark` user with a non-existent `/home/spark` directory from the beginning of this repository, `spark-docker`.\n\n- #12 \n\n https://github.com/apache/spark-docker/blob/c264d48dc510018095700ed33e700ccc34268bf2/Dockerfile.template#L21-L22\n\n**Rejected Alternatives**\n\n- We can set `HOME` to `/opt/spark` like Apache Spark behavior. However, it\u0027s also different from `WORKDIR` (`/opt/spark/work-dir`).\n- We can create `/home/spark`, but it could be more vulnerable than AS-IS status. For `system` account, `/nonexistent` is frequently used as the security practice to prevent any side effects of `HOME` directory.\n\n```\n$ docker run -it --rm apache/spark:4.0.0 cat /etc/passwd | grep /nonexistent\nnobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin\n_apt:x:100:65534::/nonexistent:/usr/sbin/nologin\n```\n\n### Why are the changes needed?\n\n**Apache Spark 3.3.3**\n\n```\n$ docker run -it --rm apache/spark:3.3.3 /opt/spark/bin/spark-sql\n...\n25/06/20 20:15:41 WARN SparkSQLCLIDriver: WARNING: Directory for Hive history file: /home/spark does not exist. History will not be available during this session.\n```\n\n```\n$ docker run -it --rm -uroot apache/spark:3.3.3 tail -1 /etc/passwd\nspark:x:185:185::/home/spark:/bin/sh\n\n$ docker run -it --rm -uroot apache/spark:3.3.3 ls -al /home/spark\nls: cannot access \u0027/home/spark\u0027: No such file or directory\n```\n\n**Apache Spark 3.4.4**\n\n```\n$ docker run -it --rm -uroot apache/spark:3.4.4 tail -1 /etc/passwd\nspark:x:185:185::/home/spark:/bin/sh\n\n$ docker run -it --rm -uroot apache/spark:3.4.4 ls -al /home/spark\nls: cannot access \u0027/home/spark\u0027: No such file or directory\n```\n\n**Apache Spark 3.5.6**\n\n```\n$ docker run -it --rm -uroot apache/spark:3.5.6 tail -1 /etc/passwd\nspark:x:185:185::/home/spark:/bin/sh\n\n$ docker run -it --rm -uroot apache/spark:3.5.6 ls /home/spark\nls: cannot access \u0027/home/spark\u0027: No such file or directory\n```\n\n**Apache Spark 4.0.0**\n```\n$ docker run -it --rm -uroot apache/spark:4.0.0 tail -1 /etc/passwd\nspark:x:185:185::/home/spark:/bin/sh\n\n$ docker run -it --rm -uroot apache/spark:4.0.0 ls /home/spark\nls: cannot access \u0027/home/spark\u0027: No such file or directory\n```\n\n### Does this PR introduce _any_ user-facing change?\n\nNo behavior change because it doesn\u0027t exist already.\n\n### How was this patch tested?\n\nManual review." }, { "commit": "bbfad0134f7a3534f4cd735ccd7b1ffe84ce6ae8", "tree": "d68a89c325dd6b8c9dd59165b72eb1e25ef9f426", "parents": [ "c264d48dc510018095700ed33e700ccc34268bf2" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Jun 20 13:10:26 2025 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Jun 20 13:10:26 2025 -0700" }, "message": "[SPARK-52543] Download Apache Spark distributions via ASF Mirrors site\n\n### What changes were proposed in this pull request?\n\nThis PR aims to download `Apache Spark` distributions via ASF Mirrors site.\n\n### Why are the changes needed?\n\nASF Mirros are the recommended place instead of the flaky `archive.apache.org` .\n\n- https://github.com/apache/spark-docker/actions/runs/15784378078/job/44497425327?pr\u003d87\n\n```\n0.064 + wget -nv -O spark.tgz https://archive.apache.org/dist/spark/spark-4.0.0/spark-4.0.0-bin-hadoop3.tgz\n133.8 failed: Connection timed out.\n133.8 failed: Network is unreachable.\n```\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #88 from dongjoon-hyun/SPARK-52543.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "c264d48dc510018095700ed33e700ccc34268bf2", "tree": "8d4448f0488eaee9fd1fb64fbbc70b0d87ae5b5c", "parents": [ "4bd1dbce94797b5b387b784db6b378069a8b6328" ], "author": { "name": "Hyukjin Kwon", "email": "gurwls223@gmail.com", "time": "Thu May 29 05:13:50 2025 +0900" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed May 28 13:13:50 2025 -0700" }, "message": "[SPARK-52325] Publish Apache Spark 3.5.6 to docker registry (#85)\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to publish Apache Spark 3.5.6 to docker registry\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 3.5.6.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review." }, { "commit": "4bd1dbce94797b5b387b784db6b378069a8b6328", "tree": "71411cee42fdaa82bf9beb24b00bbd061bda6e63", "parents": [ "f81e80c3f0786adbf8979a940251c9401f2837e3" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Tue May 27 08:48:12 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue May 27 08:48:12 2025 -0700" }, "message": "[SPARK-52286][FOLLOWUP] Move `latest`, `scala`, `python3`, `python3-java17`, `r` (#86)\n\n### What changes were proposed in this pull request?\n\nThis is a follow-up to move `latest`, `scala`, `python3`, `python3-java17`, `r` tags.\n\n### Why are the changes needed?\n\nThese tags should be aliases for the latest version, 4.0.0.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review." }, { "commit": "f81e80c3f0786adbf8979a940251c9401f2837e3", "tree": "4c1af7abe6115c5038db93f9b4b127a022afadec", "parents": [ "86f243f6df77cf247f8eb9931558d719e0c56144" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri May 23 19:18:19 2025 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri May 23 19:18:19 2025 -0700" }, "message": "[SPARK-52286] Publish Apache Spark 4.0.0 to docker registry (#84)\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to publish Apache Spark 4.0.0 to docker registry\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 4.0.0.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review." }, { "commit": "86f243f6df77cf247f8eb9931558d719e0c56144", "tree": "78d8a0d586e095603fd7176752c093d7822d7b16", "parents": [ "6a209875c42a9e878baa41baa436bde402e8cf57" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 20:45:43 2025 -0800" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 20:45:43 2025 -0800" }, "message": "[SPARK-51335] Publish Apache Spark 3.5.5 to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to publish Apache Spark 3.5.5 to docker registry\n\n### Why are the changes needed?\n\nTo provide a docker image of Apache Spark 3.5.5.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review.\n\nCloses #80 from dongjoon-hyun/SPARK-51335.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "6a209875c42a9e878baa41baa436bde402e8cf57", "tree": "fe72dce35ade966b57f4d7fab29db79c10adda99", "parents": [ "0fcf56b614503a03cde7eb3427ec883086e713e5" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 18:25:18 2025 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Feb 28 10:25:18 2025 +0800" }, "message": "[SPARK-51345] Remove all EOL versions from `spark-docker` repository (#83)\n\n" }, { "commit": "0fcf56b614503a03cde7eb3427ec883086e713e5", "tree": "d6dd82e639746131d031f19d81031ecabf5c8c14", "parents": [ "e05e0d91fa35ff0f93381227237155e66f2e07d3" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 13:48:08 2025 -0800" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 13:48:08 2025 -0800" }, "message": "[SPARK-51344] Fix `ENV` key value format in `*.template`\n\n### What changes were proposed in this pull request?\n\nThis PR aims to fix `ENV` key value format in `*.template` in order to generate the future Dockerfiles correctly.\n\n### Why are the changes needed?\n\nTo follow the Docker guideline to fix the following legacy format.\n- https://docs.docker.com/reference/build-checks/legacy-key-value-format/\n\n```\n - LegacyKeyValueFormat: \"ENV key\u003dvalue\" should be used instead of legacy \"ENV key value\" format\n```\n\nWe already fixed Apache Spark Dockerfiles in the main repository.\n- https://github.com/apache/spark/pull/47357\n\n### Does this PR introduce _any_ user-facing change?\n\nNo. This removes a warning for now.\n\n### How was this patch tested?\n\nManual review.\n\nCloses #82 from dongjoon-hyun/SPARK-51344.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "e05e0d91fa35ff0f93381227237155e66f2e07d3", "tree": "96a5485fed9129d25e1d913a6ef45a718844f418", "parents": [ "6b917ced4279dd7b3a33a81a08db37b3f27e037b" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Thu Feb 27 08:04:30 2025 -0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Feb 27 08:04:30 2025 -0800" }, "message": "[SPARK-51336] Upgrade `upload-artifact` to v4 in `spark-docker` repository (#81)\n\n" }, { "commit": "6b917ced4279dd7b3a33a81a08db37b3f27e037b", "tree": "4d2e654f3639a8392ba617e6f3a8f00d6cf0c41e", "parents": [ "5b6c9a8583f2ec8dc523694e19dfa926e836dc47" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Sun Dec 22 13:28:31 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Dec 22 13:28:31 2024 +0800" }, "message": "Publish 3.5.4 to docker registry (#77)\n\n### What changes were proposed in this pull request?\r\nPublish 3.5.4 to docker registry\r\n\r\n### Why are the changes needed?\r\n3.5.4 release step\r\n\r\n### Does this PR introduce _any_ user-facing change?\r\nno\r\n\r\n### How was this patch tested?\r\nCI" }, { "commit": "5b6c9a8583f2ec8dc523694e19dfa926e836dc47", "tree": "b261764b93f9921f5e1aaddb9e1ebe73f5f91e92", "parents": [ "51acf3d1d38effffa7843e84497b189e1e5c9ff4" ], "author": { "name": "yangjie01", "email": "yangjie01@baidu.com", "time": "Sun Dec 22 12:31:05 2024 +0800" }, "committer": { "name": "yangjie01", "email": "yangjie01@baidu.com", "time": "Sun Dec 22 12:31:05 2024 +0800" }, "message": "Install r for ubuntu-latest\n\n### What changes were proposed in this pull request?\nubuntu-latest no longer includes the runtime environment for R, which has led to the following error in testing:\n\n```\n+++ dirname /home/runner/work/spark-docker/spark-docker/spark/core/../R/install-dev.sh\n++ cd /home/runner/work/spark-docker/spark-docker/spark/core/../R\n++ pwd\n+ FWDIR\u003d/home/runner/work/spark-docker/spark-docker/spark/R\n+ LIB_DIR\u003d/home/runner/work/spark-docker/spark-docker/spark/R/lib\n+ mkdir -p /home/runner/work/spark-docker/spark-docker/spark/R/lib\n+ pushd /home/runner/work/spark-docker/spark-docker/spark/R\n+ . /home/runner/work/spark-docker/spark-docker/spark/R/find-r.sh\n++ \u0027[\u0027 -z \u0027\u0027 \u0027]\u0027\n++ \u0027[\u0027 \u0027!\u0027 -z \u0027\u0027 \u0027]\u0027\n+++ command -v R\n++ \u0027[\u0027 \u0027!\u0027 \u0027]\u0027\n++ echo \u0027Cannot find \u0027\\\u0027\u0027R_HOME\u0027\\\u0027\u0027. Please specify \u0027\\\u0027\u0027R_HOME\u0027\\\u0027\u0027 or make sure R is properly installed.\u0027\n++ exit 1\n[error] java.lang.RuntimeException: Nonzero exit value: 1\n[error] \tat scala.sys.package$.error(package.scala:30)\n[error] \tat scala.sys.process.ProcessBuilderImpl$AbstractBuilder.slurp(ProcessBuilderImpl.scala:138)\n[error] \tat scala.sys.process.ProcessBuilderImpl$AbstractBuilder.$bang$bang(ProcessBuilderImpl.scala:108)\n[error] \tat SparkR$.$anonfun$settings$124(SparkBuild.scala:1355)\n[error] \tat SparkR$.$anonfun$settings$124$adapted(SparkBuild.scala:1352)\n[error] \tat scala.Function1.$anonfun$compose$1(Function1.scala:49)\n[error] \tat sbt.internal.util.$tilde$greater.$anonfun$$u2219$1(TypeFunctions.scala:63)\n[error] \tat sbt.std.Transform$$anon$4.work(Transform.scala:69)\n[error] \tat sbt.Execute.$anonfun$submit$2(Execute.scala:283)\n[error] \tat sbt.internal.util.ErrorHandling$.wideConvert(ErrorHandling.scala:24)\n[error] \tat sbt.Execute.work(Execute.scala:292)\n[error] \tat sbt.Execute.$anonfun$submit$1(Execute.scala:283)\n[error] \tat sbt.ConcurrentRestrictions$$anon$4.$anonfun$submitValid$1(ConcurrentRestrictions.scala:265)\n[error] \tat sbt.CompletionService$$anon$2.call(CompletionService.scala:65)\n[error] \tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n[error] \tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n[error] \tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n[error] \tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n[error] \tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n[error] \tat java.base/java.lang.Thread.run(Thread.java:829)\n[error] (core / buildRPackage) Nonzero exit value: 1\n[error] Total time: 12 s, completed Dec 20, 2024, 5:24:08 PM\n\nError: Process completed with exit code 1.\n```\n\nThis pr manually installs `r-base` to fix the GitHub Actions testing environment. This fix was refer to SPARK-49920\n\n### Why are the changes needed?\nInstall r for ubuntu-latest\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI\n\nCloses #78 from LuciferYang/install-r.\n\nAuthored-by: yangjie01 \u003cyangjie01@baidu.com\u003e\nSigned-off-by: yangjie01 \u003cyangjie01@baidu.com\u003e\n" }, { "commit": "51acf3d1d38effffa7843e84497b189e1e5c9ff4", "tree": "cd8443134b8426c4ceeae0fd89a7f0783f2a61ab", "parents": [ "18c599ec44230c48fc982eb52d6cdf069883a57d" ], "author": { "name": "Kent Yao", "email": "yao@apache.org", "time": "Thu Oct 31 23:19:28 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 31 08:19:28 2024 -0700" }, "message": "Update example symbolic link creation in Dockerfile.template (#76)\n\n" }, { "commit": "18c599ec44230c48fc982eb52d6cdf069883a57d", "tree": "9550f4180c4e53dc9e6d73a978f9f040a04a7cd3", "parents": [ "cf333e1f7403fa68c7b359cff77b7949ec0990b3" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Sun Oct 27 00:49:23 2024 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Oct 27 00:49:23 2024 -0700" }, "message": "Publish 3.4.4 to docker registry (#75)\n\n### What changes were proposed in this pull request?\r\n\r\nPublish 3.4.4 to docker registry\r\n\r\n### Why are the changes needed?\r\n\r\nTo provide the docker image.\r\n\r\n### Does this PR introduce _any_ user-facing change?\r\n\r\nNo.\r\n\r\n### How was this patch tested?\r\n\r\nPass the CIs." }, { "commit": "cf333e1f7403fa68c7b359cff77b7949ec0990b3", "tree": "3036dfb39700113366380f4838bd7e9d6b918d09", "parents": [ "059a2817e53ac7c0c408196f9eb91397a99ec84e" ], "author": { "name": "Kent Yao", "email": "yao@apache.org", "time": "Thu Oct 10 14:58:10 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 10 14:58:10 2024 +0800" }, "message": "[SPARK-45497][FOLLOWUP][3.5] Simplify symbolic link creation of spark-examples.jar (#74)\n\n" }, { "commit": "059a2817e53ac7c0c408196f9eb91397a99ec84e", "tree": "c87fc10c0599318b81dc778c6f424115842ca245", "parents": [ "e498bfc1509009c74151b0df559867c73be7deb9" ], "author": { "name": "Kent Yao", "email": "yao@apache.org", "time": "Tue Oct 08 17:12:28 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 08 17:12:28 2024 +0800" }, "message": "[SPARK-45497][FOLLOWUP] Simplify symbolic link creation of spark-examples.jar (#73)\n\n### What changes were proposed in this pull request?\r\n\r\nAddress comments https://github.com/docker-library/official-images/pull/17622#issuecomment-2384397951 from docker official owners\r\n\r\n\r\n### Why are the changes needed?\r\n\r\nUse less fiddly subshell layers in between where things could unintentionally go wrong\r\n\r\n\r\n### Does this PR introduce _any_ user-facing change?\r\n\r\nno\r\n### How was this patch tested?\r\n\r\n```\r\ndocker run -it --rm scala2.13-java17-ubuntu ls -al /opt/spark/examples/jars | tail -n6\r\ndrwxr-xr-x 4 spark spark 4096 Sep 16 04:02 ..\r\n-rw-r--r-- 1 spark spark 232248 Sep 16 04:02 jackson-core-asl-1.9.13.jar\r\n-rw-r--r-- 1 spark spark 780664 Sep 16 04:02 jackson-mapper-asl-1.9.13.jar\r\n-rw-r--r-- 1 spark spark 80424 Sep 16 04:02 scopt_2.13-3.7.1.jar\r\n-rw-r--r-- 1 spark spark 1591043 Sep 16 04:02 spark-examples_2.13-4.0.0-preview2.jar\r\nlrwxrwxrwx 1 root root 38 Oct 8 05:41 spark-examples.jar -\u003e spark-examples_2.13-4.0.0-preview2.jar\r\n```\r\n" }, { "commit": "e498bfc1509009c74151b0df559867c73be7deb9", "tree": "c962a5ceea06e24bbc7a9a02b9eaed8cb061832b", "parents": [ "63c0ce3c0abfc94b591da7bc8cd5f90e02f16464" ], "author": { "name": "Haejoon Lee", "email": "haejoon.lee@databricks.com", "time": "Fri Sep 27 19:57:12 2024 +0900" }, "committer": { "name": "Haejoon Lee", "email": "haejoon.lee@databricks.com", "time": "Fri Sep 27 19:57:12 2024 +0900" }, "message": "[SPARK-49813] Publish Apache Spark 3.5.3 to docker registry\n\n### What changes were proposed in this pull request?\n\nThis PR proposes to publish Apache Spark 3.5.3 to docker registry\n\n### Why are the changes needed?\n\nThis is one of required release step for Apache Spark\n\n### Does this PR introduce _any_ user-facing change?\n\nNo\n\n### How was this patch tested?\n\nCI\n\nCloses #72 from itholic/docker_for_3.5.3.\n\nAuthored-by: Haejoon Lee \u003chaejoon.lee@databricks.com\u003e\nSigned-off-by: Haejoon Lee \u003chaejoon.lee@databricks.com\u003e\n" }, { "commit": "63c0ce3c0abfc94b591da7bc8cd5f90e02f16464", "tree": "905d76e9a25c33ac31e6c6309e0b8bf32ab58123", "parents": [ "217a9422bedb7fc3aab47c0bebed32acf0e1a737" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Sep 20 11:15:32 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Sep 20 11:15:32 2024 -0700" }, "message": "[SPARK-49740] Update `publish-java17.yaml` and `publish-java21.yaml` to use `preview2` by default\n\n### What changes were proposed in this pull request?\n\nThis PR aims to update `publish-java17.yaml` and `publish-java21.yaml` to use `preview2` by default.\n\n### Why are the changes needed?\n\nTo publish the latest images.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nManual review.\n\nCloses #71 from dongjoon-hyun/SPARK-49740.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "217a9422bedb7fc3aab47c0bebed32acf0e1a737", "tree": "95120ee3af7d48f75efb94bfd396dcc6d7db4b08", "parents": [ "0402e13bb797363f6b99d6aa56c4185317deeaf4" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Sep 20 10:54:45 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Fri Sep 20 10:54:45 2024 -0700" }, "message": "[SPARK-49736] Add Apache Spark `4.0.0-preview2` Dockerfiles\n\n### What changes were proposed in this pull request?\n\nThis PR aims to add `4.0.0-preview2` Dockerfiles.\n\n### Why are the changes needed?\n\nNew release.\n\n### Does this PR introduce _any_ user-facing change?\n\nNew release.\n\n### How was this patch tested?\n\nNew release.\n\nCloses #70 from dongjoon-hyun/SPARK-49736.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "0402e13bb797363f6b99d6aa56c4185317deeaf4", "tree": "669978f579e4cf8fd16172fb678be173a8f5c8d3", "parents": [ "a5f016858ca0ce6809a06d3c159bfdb221df68e0" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 20:37:56 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 20:37:56 2024 -0700" }, "message": "[SPARK-49703] Publish Java 21 Docker image for preview1\n\n### What changes were proposed in this pull request?\n\nThis PR aims to publish Java 21 Docker image for `preview1` and will be extended for `preview2`.\n\n### Why are the changes needed?\n\nApache Spark supports Java 21 via SPARK-43831.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo, this is a new image.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #69 from dongjoon-hyun/SPARK-49703.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "a5f016858ca0ce6809a06d3c159bfdb221df68e0", "tree": "dfbc73df19a2e8fcf4f3d5ceb6892b7314ae7592", "parents": [ "b69d21da5da1f5d35ea0125188a700c4ca9897bb" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:12:41 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:12:41 2024 -0700" }, "message": "[SPARK-44935] Fix `RELEASE` file to have the correct information in Docker images if exists\n\n### What changes were proposed in this pull request?\n\nThis PR aims to fix `RELEASE` file to have the correct information in Docker images if exists.\n\nApache Spark repository already fixed this.\n- https://github.com/apache/spark/pull/42636\n\n### Why are the changes needed?\n\nTo provide a correct information for Spark 3.4+\n\n### Does this PR introduce _any_ user-facing change?\n\nNo behavior change. Only `RELEASE` file.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #68 from dongjoon-hyun/SPARK-44935.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "b69d21da5da1f5d35ea0125188a700c4ca9897bb", "tree": "95dd3ec5899f203a73adba89935da7189d4817c0", "parents": [ "daa6f940a133d63a4e813bff811b98b1f05f1c4a" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:09:30 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:09:30 2024 -0700" }, "message": "[SPARK-45497] Add a symbolic link file `spark-examples.jar` in K8s Docker images\n\n### What changes were proposed in this pull request?\n\nThis PR aims to add a symbolic link file, `spark-examples.jar`, in the example jar directory.\n\nApache Spark repository is updated already via\n- https://github.com/apache/spark/pull/43324\n\n```\n$ docker run -it --rm spark:latest ls -al /opt/spark/examples/jars | tail -n6\ntotal 1620\ndrwxr-xr-x 1 root root 4096 Oct 11 04:37 .\ndrwxr-xr-x 1 root root 4096 Sep 9 02:08 ..\n-rw-r--r-- 1 root root 78803 Sep 9 02:08 scopt_2.12-3.7.1.jar\n-rw-r--r-- 1 root root 1564255 Sep 9 02:08 spark-examples_2.12-3.5.0.jar\nlrwxrwxrwx 1 root root 29 Oct 11 04:37 spark-examples.jar -\u003e spark-examples_2.12-3.5.0.jar\n```\n\n### Why are the changes needed?\n\nLike PySpark example (`pi.py`), we can submit the examples without considering the version numbers which was painful before.\n```\nbin/spark-submit \\\n--master k8s://$K8S_MASTER \\\n--deploy-mode cluster \\\n...\n--class org.apache.spark.examples.SparkPi \\\nlocal:///opt/spark/examples/jars/spark-examples.jar 10000\n```\n\nThe following is the driver pod log.\n```\n+ exec /usr/bin/tini -s -- /opt/spark/bin/spark-submit ...\n--deploy-mode client\n--properties-file /opt/spark/conf/spark.properties\n--class org.apache.spark.examples.SparkPi\nlocal:///opt/spark/examples/jars/spark-examples.jar 10000\nFiles local:///opt/spark/examples/jars/spark-examples.jar from /opt/spark/examples/jars/spark-examples.jar to /opt/spark/work-dir/./spark-examples.jar\n```\n\n### Does this PR introduce _any_ user-facing change?\n\nNo, this is an additional file.\n\n### How was this patch tested?\n\nManually build the docker image and do `ls`.\n\n### Was this patch authored or co-authored using generative AI tooling?\n\nNo.\n\nCloses #67 from dongjoon-hyun/SPARK-45497.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "daa6f940a133d63a4e813bff811b98b1f05f1c4a", "tree": "b65baebd8caea464db77eb821cc7fdd6045a499a", "parents": [ "6f10acb452a360966ec9a971641280b7b4a3619a" ], "author": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:08:32 2024 -0700" }, "committer": { "name": "Dongjoon Hyun", "email": "dongjoon@apache.org", "time": "Wed Sep 18 16:08:32 2024 -0700" }, "message": "[SPARK-49701] Use JDK for Spark 3.5+ Docker image\n\n### What changes were proposed in this pull request?\n\nThis PR aims to use JDK for Spark 3.5+ Docker image. Apache Spark Dockerfile are updated already.\n- https://github.com/apache/spark/pull/45762\n- https://github.com/apache/spark/pull/45761\n\n### Why are the changes needed?\n\nSince Apache Spark 3.5.0, SPARK-44153 starts to use `jmap` like the following.\n\n- https://github.com/apache/spark/pull/41709\n\nhttps://github.com/apache/spark/blob/c832e2ac1d04668c77493577662c639785808657/core/src/main/scala/org/apache/spark/util/Utils.scala#L2030\n\n### Does this PR introduce _any_ user-facing change?\n\nYes, the user can use `Heap Histogram` feature.\n\n### How was this patch tested?\n\nPass the CIs.\n\nCloses #66 from dongjoon-hyun/SPARK-49701.\n\nAuthored-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\nSigned-off-by: Dongjoon Hyun \u003cdongjoon@apache.org\u003e\n" }, { "commit": "6f10acb452a360966ec9a971641280b7b4a3619a", "tree": "edc8ecae3444d91837aea2f277304dc18408613a", "parents": [ "b9f1f8e8ebed1959c2be3864a114b52f67519092" ], "author": { "name": "Kent Yao", "email": "yao@apache.org", "time": "Wed Aug 14 02:44:34 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Aug 14 02:44:34 2024 +0800" }, "message": "Publish 3.4.3 to docker registry (#65)\n\n" }, { "commit": "b9f1f8e8ebed1959c2be3864a114b52f67519092", "tree": "a5461fd97d34e12b903c10afb98abcafa1aecac4", "parents": [ "b1b1019fb8f3034d2b31c407eee1a95301cd4eab" ], "author": { "name": "Kent Yao", "email": "yao@apache.org", "time": "Mon Aug 12 17:09:28 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Aug 12 17:09:28 2024 +0800" }, "message": "Publish 3.5.2 to docker registry (#64)\n\n* Publish 3.5.2 to docker registry\r\n\r\n* Publish 3.5.2 to docker registry\r\n\r\n* Publish 3.5.2 to docker registry" }, { "commit": "b1b1019fb8f3034d2b31c407eee1a95301cd4eab", "tree": "85029f55ff2375034e2e9111b633509dc5718cf8", "parents": [ "04e85239a8fcc9b3dcfe146bc144ee2b981f8f42" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Jun 24 09:05:02 2024 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Jun 24 09:05:02 2024 +0800" }, "message": "[SPARK-48664][FOLLOWUP] Update 4.0.0-preview1\n\n### What changes were proposed in this pull request?\n- Remove `wget -nv -O KEYS https://downloads.apache.org/spark/KEYS` and `gpg --import KEYS;`, it works but not meet the [security concern](https://github.com/docker-library/official-images?tab\u003dreadme-ov-file#security) from DOI.\n- add `add-dockerfiles.sh 4.0.0-preview1` support to address https://github.com/apache/spark-docker/pull/61#issuecomment-2178827705\n- Fix `versions.json`: consider java17 as default version, so we can remove java17 tag.\n\n### Why are the changes needed?\nupdate 4.0.0-preview1\n\n### Does this PR introduce _any_ user-facing change?\nnew release\n\n### How was this patch tested?\n./add-dockerfiles.sh 4.0.0-preview1 , no diff\n\nCloses #63 from Yikun/4.0.0-preview1.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "04e85239a8fcc9b3dcfe146bc144ee2b981f8f42", "tree": "036ba6f02f734279ae9e5d1023155b63567c39c8", "parents": [ "300ae7980492792c9717a2415159c4f9ac1d4b36" ], "author": { "name": "Wenchen Fan", "email": "cloud0fan@gmail.com", "time": "Thu Jun 20 00:18:27 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jun 20 00:18:27 2024 +0800" }, "message": "minor fix (#62)\n\n" }, { "commit": "300ae7980492792c9717a2415159c4f9ac1d4b36", "tree": "8d4624f9d6cf88d4e2ec48f92393dbfdd96769e9", "parents": [ "4f2d96a415c89cfe0fde89a55e9034d095224c94" ], "author": { "name": "Wenchen Fan", "email": "cloud0fan@gmail.com", "time": "Wed Jun 19 22:13:10 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jun 19 22:13:10 2024 +0800" }, "message": "[SPARK-48664] Add Apache Spark 4.0.0-preview1 Dockerfiles (#61)\n\n* Add Apache Spark 4.0.0-preview1 Dockerfiles\r\n\r\n* Update 4.0.0-preview1/scala2.13-java17-ubuntu/Dockerfile\r\n\r\n* fix permission\r\n\r\n* upgrade minikube\r\n\r\n---------\r\n\r\nCo-authored-by: Kent Yao \u003cyao@apache.org\u003e" }, { "commit": "4f2d96a415c89cfe0fde89a55e9034d095224c94", "tree": "5b677112a6212e6ae71c88fa569285574368eda3", "parents": [ "7216374855ba57ce14c8ddbf56890538f678ec3d" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Feb 29 09:49:01 2024 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Feb 29 09:49:01 2024 +0800" }, "message": "[SPARK-47206][FOLLOWUP] Fix wrong path version\n\n### What changes were proposed in this pull request?\nFix wrong path version.\n\n### Why are the changes needed?\nThis will be used by https://github.com/docker-library/official-images .\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\n\n```\n$ tools/manifest.py manifest\n\nMaintainers: Apache Spark Developers \u003cdevspark.apache.org\u003e (ApacheSpark)\nGitRepo: https://github.com/apache/spark-docker.git\n\nTags: 3.5.1-scala2.12-java17-python3-ubuntu, 3.5.1-java17-python3, 3.5.1-java17, python3-java17\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java17-python3-ubuntu\n\nTags: 3.5.1-scala2.12-java17-r-ubuntu, 3.5.1-java17-r\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java17-r-ubuntu\n\nTags: 3.5.1-scala2.12-java17-ubuntu, 3.5.1-java17-scala\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java17-ubuntu\n\nTags: 3.5.1-scala2.12-java17-python3-r-ubuntu\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java17-python3-r-ubuntu\n\nTags: 3.5.1-scala2.12-java11-python3-ubuntu, 3.5.1-python3, 3.5.1, python3, latest\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java11-python3-ubuntu\n\nTags: 3.5.1-scala2.12-java11-r-ubuntu, 3.5.1-r, r\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java11-r-ubuntu\n\nTags: 3.5.1-scala2.12-java11-ubuntu, 3.5.1-scala, scala\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java11-ubuntu\n\nTags: 3.5.1-scala2.12-java11-python3-r-ubuntu\nArchitectures: amd64, arm64v8\nGitCommit: 8b4329162bbbd1ce5c9d885a1edcd6d61ebcc676\nDirectory: ./3.5.1/scala2.12-java11-python3-r-ubuntu\n```\n\nCloses #60 from Yikun/3.5.1-follow.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "7216374855ba57ce14c8ddbf56890538f678ec3d", "tree": "cec1e612a6f1c1c96320738bc96fea983865165c", "parents": [ "431aa516ba58985c902bf2d2a07bf0eaa1df6740" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Feb 29 08:55:47 2024 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Feb 29 08:55:47 2024 +0800" }, "message": "[SPARK-47206] Add official image Dockerfile for Apache Spark 3.5.1\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.5.1 Dockerfiles.\n\n- Add 3.5.1 GPG key\n- Add .github/workflows/build_3.5.1.yaml\n- `./add-dockerfiles.sh 3.5.1` to generate dockerfiles\n- Add version and tag info\n\n### Why are the changes needed?\nApache Spark 3.5.1 released\n\n### Does this PR introduce _any_ user-facing change?\nDocker image will be published.\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #59 from Yikun/3.5.1.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "431aa516ba58985c902bf2d2a07bf0eaa1df6740", "tree": "480416d7ea6cdfac3aa49d18264cee1ba7afe91a", "parents": [ "ec69b9c77bc733ed5937f5068d23f7407eb51ea9" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat Dec 02 20:36:29 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat Dec 02 20:36:29 2023 +0800" }, "message": "[SPARK-46209] Add java 11 only yml for version before 3.5\n\n### What changes were proposed in this pull request?\nAdd Java11 only workflow for version before 3.5.0.\n\n### Why are the changes needed?\notherwise, the publish will failed due to no java 17 file founded in version before v 3.5.0.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nTest on my repo: https://github.com/Yikun/spark-docker/actions/workflows/publish-java11.yml\n\nCloses #58 from Yikun/java11-publish.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "ec69b9c77bc733ed5937f5068d23f7407eb51ea9", "tree": "7f9423b2449bba520f7690a2397f76f3d029d667", "parents": [ "6f68fe0f7051c10f2bf43a50a7decfce2e97baf0" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat Dec 02 10:00:48 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat Dec 02 10:00:48 2023 +0800" }, "message": "[SPARK-46185] Add official image Dockerfile for Apache Spark 3.4.2\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.4.2 Dockerfiles.\n\n- Add 3.4.2 GPG key\n- Add .github/workflows/build_3.4.2.yaml\n- `./add-dockerfiles.sh 3.4.2` to generate dockerfiles (and remove master changes: https://github.com/apache/spark-docker/pull/55/commits/24cbf40abdc252fdcf48303efa33ba7f84adefaf)\n- Add version and tag info\n\n### Why are the changes needed?\nApache Spark 3.4.2 released\n\n### Does this PR introduce _any_ user-facing change?\nDocker image will be published.\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #57 from Yikun/3.4.2.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "6f68fe0f7051c10f2bf43a50a7decfce2e97baf0", "tree": "caf2d3932bf2272b5b4b7f2235988b62070f5ca2", "parents": [ "028efd4637fb2cf791d5bd9ea70b2fca472de4b7" ], "author": { "name": "vakarisbk", "email": "vakaris.bashkirov@gmail.com", "time": "Fri Nov 10 11:33:39 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Nov 10 11:33:39 2023 +0800" }, "message": "Add support for java 17 from spark 3.5.0\n\n### What changes were proposed in this pull request?\n1. Create Java17 base images alongside Java11 images starting from spark 3.5.0\n2. Change ubuntu version to 22.04 for `scala2.12-java17-*`\n\n### Why are the changes needed?\n\nSpark supports multiple Java versions, but the images are currently built only with Java 11.\n\n### Does this PR introduce _any_ user-facing change?\n\nNew images would be available in the repositories.\n\n### How was this patch tested?\n\nCloses #56 from vakarisbk/master.\n\nAuthored-by: vakarisbk \u003cvakaris.bashkirov@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "028efd4637fb2cf791d5bd9ea70b2fca472de4b7", "tree": "48e2ae63d6d64df0f60e2113e7a9f01c6dfbf01b", "parents": [ "a690e663273a5eb889aa90a9d1ef9e38bffae8d1" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Sep 14 21:22:32 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Sep 14 21:22:32 2023 +0800" }, "message": "[SPARK-45169] Add official image Dockerfile for Apache Spark 3.5.0\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.5.0 Dockerfiles.\n\n- Add 3.5.0 GPG key\n- Add .github/workflows/build_3.5.0.yaml\n- `./add-dockerfiles.sh 3.5.0` to generate dockerfiles\n- Add version and tag info\n- Backport https://github.com/apache/spark/commit/1d2c338c867c69987d8ed1f3666358af54a040e3 and https://github.com/apache/spark/commit/0c7b4306c7c5fbdd6c577774f8172f82e1d23e3b entrypoint changes\n\n### Why are the changes needed?\nApache Spark 3.5.0 released\n\n### Does this PR introduce _any_ user-facing change?\nDocker image will be published.\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #55 from Yikun/3.5.0.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "a690e663273a5eb889aa90a9d1ef9e38bffae8d1", "tree": "e22d04ab365afa912842570cb45440ab0a762178", "parents": [ "6fd201e7c6e6a36c7a18e3b5877c3616081a05cf" ], "author": { "name": "Yuming Wang", "email": "yumwang@ebay.com", "time": "Tue Aug 22 15:13:33 2023 +0800" }, "committer": { "name": "Yuming Wang", "email": "yumwang@ebay.com", "time": "Tue Aug 22 15:13:33 2023 +0800" }, "message": "[SPARK-44892] Add official image Dockerfile for Spark 3.3.3\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.3.3 Dockerfiles.\n- Add 3.3.3 GPG key\n- Add .github/workflows/build_3.3.3.yaml\n- ./add-dockerfiles.sh 3.3.3\n- Add version and tag info\n\n### Why are the changes needed?\nApache Spark 3.3.3 released:\nhttps://spark.apache.org/releases/spark-release-3-3-3.html\n\n### Does this PR introduce _any_ user-facing change?\nDocker image will be published.\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #54 from wangyum/spark-3.3.3.\n\nAuthored-by: Yuming Wang \u003cyumwang@ebay.com\u003e\nSigned-off-by: Yuming Wang \u003cyumwang@ebay.com\u003e\n" }, { "commit": "6fd201e7c6e6a36c7a18e3b5877c3616081a05cf", "tree": "116936fac2a43f3bb506fad280c561471583d809", "parents": [ "58d288546e8419d229f14b62b6a653999e0390f1" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Aug 17 15:30:59 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Aug 17 15:30:59 2023 +0800" }, "message": "[SPARK-44494] Pin minikube to v1.30.1 to fix spark-docker K8s CI\n\n### What changes were proposed in this pull request?\nPin minikube to v1.30.1 to fix spark-docker K8s CI.\n\n### Why are the changes needed?\nPin minikube to v1.30.1 to fix spark-docker K8s CI\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #53 from Yikun/minikube.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "58d288546e8419d229f14b62b6a653999e0390f1", "tree": "2d59023233f962433f4bde02d08e65075ebfdbf5", "parents": [ "39264c502cf21b71a1ab5da71760e5864abce099" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 29 16:05:47 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 29 16:05:47 2023 +0800" }, "message": "[SPARK-40513] Add --batch to gpg command\n\n### What changes were proposed in this pull request?\nAdd --batch to gpg command which essentially puts GnuPG into \"API mode\" instead of \"UI mode\".\nApply changes to 3.4.x dockerfile.\n\n### Why are the changes needed?\nAddress DOI comments: https://github.com/docker-library/official-images/pull/13089#issuecomment-1611814491\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #51 from Yikun/batch.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "39264c502cf21b71a1ab5da71760e5864abce099", "tree": "491e1a0c0e9a98015138a2cc79b4cdc0715aae78", "parents": [ "d02ff6091835311a32c7ccc73d8ebae1d5817ecc" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 29 16:04:50 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 29 16:04:50 2023 +0800" }, "message": "[SPARK-44168][FOLLOWUP] Change v3.4 GPG_KEY to full key fingerprint\n\n### What changes were proposed in this pull request?\n\nChange GPG key from `34F0FC5C` to `F28C9C925C188C35E345614DEDA00CE834F0FC5C` to avoid pontential collision.\n\nThe full finger print can get from below cmd:\n```\n$ wget https://dist.apache.org/repos/dist/dev/spark/KEYS\n$ gpg --import KEYS\n$ gpg --fingerprint 34F0FC5C\n\npub rsa4096 2015-05-05 [SC]\n F28C 9C92 5C18 8C35 E345 614D EDA0 0CE8 34F0 FC5C\nuid [ unknown] Dongjoon Hyun (CODE SIGNING KEY) \u003cdongjoonapache.org\u003e\nsub rsa4096 2015-05-05 [E]\n\n```\n\n### Why are the changes needed?\n\n- A short gpg key had been added as v3.4.0 gpg key in https://github.com/apache/spark-docker/pull/46 .\n- The short key `34F0FC5C` is from https://dist.apache.org/repos/dist/dev/spark/KEYS\n- According DOI review comments, https://github.com/docker-library/official-images/pull/13089#issuecomment-1609990551 , `this should be the full key fingerprint: F28C9C925C188C35E345614DEDA00CE834F0FC5C (generating a collision for such a short key ID is trivial.`\n- We\u0027d better to switch the short key to full fingerprint\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #50 from Yikun/gpg_key.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "d02ff6091835311a32c7ccc73d8ebae1d5817ecc", "tree": "a100777df8a53f741ac19dd04f7d61779a99d4ba", "parents": [ "5405b49b52aa1661d31ac80cdb8c9aad530d6847" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 14:28:21 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 14:28:21 2023 +0800" }, "message": "[SPARK-40513][DOCS] Add apache/spark docker image overview\n\n### What changes were proposed in this pull request?\nThis PR add the `OVERVIEW.md`.\n\n### Why are the changes needed?\n\nThis will be used in the page of https://hub.docker.com/r/apache/spark to introduce the spark docker image and tag info.\n\n### Does this PR introduce _any_ user-facing change?\nYes, doc only\n\n### How was this patch tested?\nDoc only, review.\n\nCloses #34 from Yikun/overview.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "5405b49b52aa1661d31ac80cdb8c9aad530d6847", "tree": "b842b4562c6528737522ecba57ea16b7f1ca4bff", "parents": [ "60222892836549f05c56edd49ac81c688c8e7356" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 14:09:34 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 14:09:34 2023 +0800" }, "message": "[SPARK-44175] Remove useless lib64 path link in dockerfile\n\n### What changes were proposed in this pull request?\nRemove useless lib64 path\n\n### Why are the changes needed?\nAddress comments: https://github.com/docker-library/official-images/pull/13089#issuecomment-1601813499\n\nIt was introduced by https://github.com/apache/spark/commit/f13ea15d79fb4752a0a75a05a4a89bd8625ea3d5 to address the issue about snappy on alpine OS, but we already switch the OS to ubuntu, so `/lib64` hack can be cleanup.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #48 from Yikun/rm-lib64-hack.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "60222892836549f05c56edd49ac81c688c8e7356", "tree": "bd4dd5b7e1794c3491d6ec26f26237164622d80e", "parents": [ "6f1a0a5fbb8034ebc4ea04e4f0b2fda728a4dd1e" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 08:59:03 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 08:59:03 2023 +0800" }, "message": "[SPARK-44177] Add \u0027set -eo pipefail\u0027 to entrypoint and quote variables\n\n### What changes were proposed in this pull request?\nAdd \u0027set -eo pipefail\u0027 to entrypoint and quote variables\n\n### Why are the changes needed?\nAddress DOI comments:\n1. Have you considered a set -eo pipefail on the entrypoint script to help prevent any errors from being silently ignored?\n2. You probably want to quote this (and many of the other variables in this execution); ala --driver-url \"$SPARK_DRIVER_URL\"\n\n[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601334895\n[2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601813499\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #49 from Yikun/quote.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "6f1a0a5fbb8034ebc4ea04e4f0b2fda728a4dd1e", "tree": "b503423414eb997514be19c110d4da0599c1b378", "parents": [ "6f3641534a97a80491cba926cc7a5e67972494ea" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 08:56:54 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Jun 27 08:56:54 2023 +0800" }, "message": "[SPARK-44176] Change apt to apt-get and remove useless cleanup\n\n### What changes were proposed in this pull request?\nThis patch change `apt` to `apt-get` and also remove useless `rm -rf /var/cache/apt/*; \\`.\nAnd also apply the change to 3.4.0 and 3.4.1\n\n### Why are the changes needed?\nAddress comments from DOI:\n- `apt install ...`, This should be apt-get (apt is not intended for unattended use, as the warning during build makes clear).\n- `rm -rf /var/cache/apt/*; \\` This is harmless, but should be unnecessary (the base image configuration already makes sure this directory stays empty).\n\nSee more in:\n[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601813499\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #47 from Yikun/apt-get.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "6f3641534a97a80491cba926cc7a5e67972494ea", "tree": "cd45e94303ecde9f0dd2cb2cfcaab568cc040b78", "parents": [ "c07ae18355678370fd270bedb8b39ab2aceb5ac2" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sun Jun 25 10:51:46 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sun Jun 25 10:51:46 2023 +0800" }, "message": "[SPARK-44168] Add Apache Spark 3.4.1 Dockerfiles\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.4.1 Dockerfiles.\n- Add 3.4.1 GPG key\n- Add .github/workflows/build_3.4.1.yaml\n- ./add-dockerfiles.sh 3.4.1\n- Add version and tag info\n\n### Why are the changes needed?\nApache Spark 3.4.1 released:\nhttps://spark.apache.org/releases/spark-release-3-4-1.html\n\n### Does this PR introduce _any_ user-facing change?\nDocker image will be published.\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #46 from Yikun/3.4.1.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "c07ae18355678370fd270bedb8b39ab2aceb5ac2", "tree": "5806e30b2752bdf87d9f4268ef05e5a8eea33654", "parents": [ "2dc12d96910710aa6ee2d717c4c723ddd75127a1" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Jun 02 10:27:01 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Jun 02 10:27:01 2023 +0800" }, "message": "[SPARK-43368] Use `libnss_wrapper` to fake passwd entry\n\n### What changes were proposed in this pull request?\nUse `libnss_wrapper` to fake passwd entry instead of changing passwd to resolve random UID problem. And also we only attempt to setup fake passwd entry for driver/executor, but for cmd like `bash`, the fake passwd will not be set.\n\n### Why are the changes needed?\nIn the past, we add the entry to `/etc/passwd` directly for current UID, it\u0027s mainly for [OpenShift anonymous random `uid` case](https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523) (See also in https://github.com/apache-spark-on-k8s/spark/pull/404), but this way bring the pontential security issue about widely permision of `/etc/passwd`.\n\nAccording to DOI reviewer [suggestion](https://github.com/docker-library/official-images/pull/13089#issuecomment-1561793792), we\u0027d better to resolve this problem by using [libnss_wrapper](https://cwrap.org/nss_wrapper.html). It\u0027s a library to help set a fake passwd entry by setting `LD_PRELOAD`, `NSS_WRAPPER_PASSWD`, `NSS_WRAPPER_GROUP`. Such as random UID is `1000`, the env will be:\n\n```\nspark6f41b8e5be9b:/opt/spark/work-dir$ id -u\n1000\nspark6f41b8e5be9b:/opt/spark/work-dir$ id -g\n1000\nspark6f41b8e5be9b:/opt/spark/work-dir$ whoami\nspark\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $LD_PRELOAD\n/usr/lib/libnss_wrapper.so\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $NSS_WRAPPER_PASSWD\n/tmp/tmp.r5x4SMX35B\nspark6f41b8e5be9b:/opt/spark/work-dir$ cat /tmp/tmp.r5x4SMX35B\nspark:x:1000:1000:${SPARK_USER_NAME:-anonymous uid}:/opt/spark:/bin/false\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $NSS_WRAPPER_GROUP\n/tmp/tmp.XcnnYuD68r\nspark6f41b8e5be9b:/opt/spark/work-dir$ cat /tmp/tmp.XcnnYuD68r\nspark:x:1000:\n```\n\n### Does this PR introduce _any_ user-facing change?\nYes, setup fake ENV rather than changing `/etc/passwd`.\n\n### How was this patch tested?\n#### 1. Without `attempt_setup_fake_passwd_entry`, the user is `I have no name!`\n```\n# docker run -it --rm --user 1000:1000 spark-test bash\ngroups: cannot find name for group ID 1000\nI have no name!998110cd5a26:/opt/spark/work-dir$\nI have no name!0fea1d27d67d:/opt/spark/work-dir$ id -u\n1000\nI have no name!0fea1d27d67d:/opt/spark/work-dir$ id -g\n1000\nI have no name!0fea1d27d67d:/opt/spark/work-dir$ whoami\nwhoami: cannot find name for user ID 1000\n```\n\n#### 2. Mannual stub the `attempt_setup_fake_passwd_entry`, the user is `spark`.\n2.1 Apply a tmp change to cmd\n\n```patch\ndiff --git a/entrypoint.sh.template b/entrypoint.sh.template\nindex 08fc925..77d5b04 100644\n--- a/entrypoint.sh.template\n+++ b/entrypoint.sh.template\n -118,6 +118,7 case \"$1\" in\n\n *)\n # Non-spark-on-k8s command provided, proceeding in pass-through mode...\n+ attempt_setup_fake_passwd_entry\n exec \"$\"\n ;;\n esac\n```\n\n2.2 Build and run the image, specify a random UID/GID 1000\n\n```bash\n$ docker build . -t spark-test\n$ docker run -it --rm --user 1000:1000 spark-test bash\n# the user is set to spark rather than unknow user\nspark6f41b8e5be9b:/opt/spark/work-dir$\nspark6f41b8e5be9b:/opt/spark/work-dir$ id -u\n1000\nspark6f41b8e5be9b:/opt/spark/work-dir$ id -g\n1000\nspark6f41b8e5be9b:/opt/spark/work-dir$ whoami\nspark\n\n```\n\n```\n# NSS env is set right\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $LD_PRELOAD\n/usr/lib/libnss_wrapper.so\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $NSS_WRAPPER_PASSWD\n/tmp/tmp.r5x4SMX35B\nspark6f41b8e5be9b:/opt/spark/work-dir$ cat /tmp/tmp.r5x4SMX35B\nspark:x:1000:1000:${SPARK_USER_NAME:-anonymous uid}:/opt/spark:/bin/false\nspark6f41b8e5be9b:/opt/spark/work-dir$ echo $NSS_WRAPPER_GROUP\n/tmp/tmp.XcnnYuD68r\nspark6f41b8e5be9b:/opt/spark/work-dir$ cat /tmp/tmp.XcnnYuD68r\nspark:x:1000:\n```\n\n#### 3. If specify current exsiting user (such as `spark`, `root`), no fake setup\n```bash\n# docker run -it --rm --user 0 spark-test bash\nroote5bf55d4df22:/opt/spark/work-dir# echo $LD_PRELOAD\n\n```\n\n```bash\n# docker run -it --rm spark-test bash\nsparkdef8d8ca4e7d:/opt/spark/work-dir$ echo $LD_PRELOAD\n\n```\n\nCloses #45 from Yikun/SPARK-43368.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "2dc12d96910710aa6ee2d717c4c723ddd75127a1", "tree": "25a88f904b2cd7e606d5ff6996c8e8a198f9e12b", "parents": [ "9d4c98c62c4ce517e69e65d1f6f7bf412d775b75" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 01 14:36:17 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Jun 01 14:36:17 2023 +0800" }, "message": "[SPARK-43370] Switch spark user only when run driver and executor\n\n### What changes were proposed in this pull request?\nSwitch spark user only when run driver and executor\n\n### Why are the changes needed?\nAddress doi comments: question 7 [1]\n\n[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388\n[2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1561793792\n\n### Does this PR introduce _any_ user-facing change?\nYes\n\n### How was this patch tested?\n1. test mannuly\n```\ncd ~/spark-docker/3.4.0/scala2.12-java11-ubuntu\n$ docker build . -t spark-test\n\n$ docker run -ti spark-test bash\nsparkafa78af05cf8:/opt/spark/work-dir$\n\n$ docker run --user root -ti spark-test bash\nroot095e0d7651fd:/opt/spark/work-dir#\n```\n2. ci passed\n\nCloses: https://github.com/apache/spark-docker/pull/44\n\nCloses #43 from Yikun/SPARK-43370.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "9d4c98c62c4ce517e69e65d1f6f7bf412d775b75", "tree": "10c6349bf1330f5d53a0368bc9fa418747c055bd", "parents": [ "ce3e12266ef82264b814f6f7823165f7c7ae215a" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri May 26 09:53:20 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri May 26 09:53:20 2023 +0800" }, "message": "[SPARK-43806] Add awesome-spark-docker.md\n\n### What changes were proposed in this pull request?\nAdd links to more related images and dockerfile reference.\n\n### Why are the changes needed?\nSomething we talked about in \"Spark on Kube Coffe Chats“[1] to add links to more related images and dockerfile reference. Init with [2].\n[1] https://lists.apache.org/thread/26gpmlhqhk5cp2fhtzrpl5f61p8jc551\n[2] https://github.com/awesome-spark/awesome-spark/blob/main/README.md#docker-images\n\n### Does this PR introduce _any_ user-facing change?\nDoc only\n\n### How was this patch tested?\nNo\n\nCloses #28 from Yikun/awesome-spark-docker.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "ce3e12266ef82264b814f6f7823165f7c7ae215a", "tree": "52d3bb9b54a2c17c39b45f9e319d88d4903c9c28", "parents": [ "006e8fade69f148a05fc73f591f52c7678e48f04" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 19:07:55 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 19:07:55 2023 +0800" }, "message": "[SPARK-43367] Recover sh in dockerfile\n\n### What changes were proposed in this pull request?\nRecover `sh`, we remove `sh` due to https://github.com/apache-spark-on-k8s/spark/pull/444/files#r134075892 , now `SPARK_DRIVER_JAVA_OPTS` related code already move to `entrypoint.sh` with `#!/bin/bash`, so we don\u0027t need this hack way.\n\nSee also:\n[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388\n[2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1561793792\n\n### Why are the changes needed?\nRecover sh\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #41 from Yikun/SPARK-43367.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "006e8fade69f148a05fc73f591f52c7678e48f04", "tree": "c65bc199502c3903b4e46eb347f70c0fbcd3ea72", "parents": [ "f2d2b2d1ffbb951aed29221a040861327c09441e" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 19:05:26 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 19:05:26 2023 +0800" }, "message": "[SPARK-43793] Fix SPARK_EXECUTOR_JAVA_OPTS assignment bug\n\n### What changes were proposed in this pull request?\nIn previous code, this is susceptible to a few bugs particularly around newlines in values.\n```\nenv | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed \u0027s/[^\u003d]*\u003d\\(.*\\)/\\1/g\u0027 \u003e /tmp/java_opts.txt\nreadarray -t SPARK_EXECUTOR_JAVA_OPTS \u003c /tmp/java_opts.txt\n```\n\n### Why are the changes needed?\nTo address DOI comments: https://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388 , question 6.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\n1. Test mannully\n```\nexport SPARK_JAVA_OPT_0\u003d\"foo\u003dbar\"\nexport SPARK_JAVA_OPT_1\u003d\"foo1\u003dbar1\"\n\nfor v in \"${!SPARK_JAVA_OPT_}\"; do\n SPARK_EXECUTOR_JAVA_OPTS+\u003d( \"${!v}\" )\ndone\n\nfor v in ${SPARK_EXECUTOR_JAVA_OPTS[]}; do\n echo $v\ndone\n\n# foo\u003dbar\n# foo1\u003dbar1\n```\n2. CI passed\n\nCloses #42 from Yikun/SPARK-43793.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "f2d2b2d1ffbb951aed29221a040861327c09441e", "tree": "62fb5fb5afff075fe64bf4c7161df8214032c484", "parents": [ "7f9b414de48639d69c64acfd81e6792517b86f61" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 16:13:44 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu May 25 16:13:44 2023 +0800" }, "message": "[SPARK-43365][FOLLWUP] Refactor publish workflow based on base image\n\n### What changes were proposed in this pull request?\n- This patch changes the `build-args` to `patch in test` in build and publish workflow, because the docker official image do not support **parameterized FROM** values. https://github.com/docker-library/official-images/pull/13089#issuecomment-1555352902\n- And also Refactor publish workflow:\n![image](https://user-images.githubusercontent.com/1736354/236613626-96f8fbf6-7df7-4d10-b4fb-be4d57c56dce.png)\n### Why are the changes needed?\nSame change with build workflow refactor, to avoid the publish issue like:\n```\n#5 [linux/amd64 internal] load metadata for docker.io/library/spark:3.4.0-scala2.12-java11-ubuntu\n#5 ERROR: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed\n------\n \u003e [linux/amd64 internal] load metadata for docker.io/library/spark:3.4.0-scala2.12-java11-ubuntu:\n------\nDockerfile:18\n--------------------\n 16 | #\n 17 | ARG BASE_IMAGE\u003dspark:3.4.0-scala2.12-java11-ubuntu\n 18 | \u003e\u003e\u003e FROM $BASE_IMAGE\n 19 |\n 20 | RUN set -ex \u0026\u0026 \\\n--------------------\nERROR: failed to solve: spark:3.4.0-scala2.12-java11-ubuntu: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed\nError: buildx failed with: ERROR: failed to solve: spark:3.4.0-scala2.12-java11-ubuntu: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed\n```\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nPublish test in my local fork:\n- https://github.com/Yikun/spark-docker/actions/runs/5076986823/jobs/9120029759: Skip the local base build use the [published base](https://github.com/Yikun/spark-docker/actions/runs/5076986823/jobs/9120029759#step:11:135) image:\n\n![image](https://user-images.githubusercontent.com/1736354/236612540-2b454c14-e194-4d73-b859-0df001570d27.png)\n\n```\n#3 [linux/amd64 internal] load metadata for ghcr.io/yikun/spark-docker/spark:3.4.0-scala2.12-java11-ubuntu\n#3 DONE 0.9s\n\n#4 [linux/arm64 internal] load metadata for ghcr.io/yikun/spark-docker/spark:3.4.0-scala2.12-java11-ubuntu\n#4 DONE 0.9s\n```\n\n- CI passed: do local base build first and build base on the local build\n\nCloses #39 from Yikun/publish-build.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "7f9b414de48639d69c64acfd81e6792517b86f61", "tree": "69732c41be6a4e745386b095db4a3bfa3b1e0d3c", "parents": [ "406eb86c2cc722458e0a4787e759802dda5c73eb" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon May 08 11:19:36 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon May 08 11:19:36 2023 +0800" }, "message": "[SPARK-43372] Use ; instead of \u0026\u0026 when enable set -ex\n\n### What changes were proposed in this pull request?\n- Use ; instead of \u0026\u0026 when enable set -ex\n- ./add-dockerfiles.sh 3.4.0 to apply changes\n\n### Why are the changes needed?\nAddress DOI comments: `9. using set -ex means you can use ; instead of \u0026\u0026 (really only matters for complex expressions, like the || in the later RUN that does use ;)`\n\nhttps://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #38 from Yikun/SPARK-43372.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "406eb86c2cc722458e0a4787e759802dda5c73eb", "tree": "509c8201904dda50e546379143f51b43fd8ee10b", "parents": [ "7f836378d8bfe453b7e1dba304b54cb1cfacda49" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat May 06 17:24:12 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat May 06 17:24:12 2023 +0800" }, "message": "[SPARK-43371] Minimize duplication across layers for chmod\n\n### What changes were proposed in this pull request?\nThis patch minimizes duplication across layers for chmod:\n- Move `chmod g+w /opt/spark/work-dir` to layer of `/opt/spark/work-dir` creation\n- Move `chmod a+x /opt/decom.sh` to layer of spark extration layer.\n- `chmod a+x $VERSION/$TAG/entrypoint.sh` when generate the entrypoint.sh\n- ./add-dockerfiles.sh 3.4.0 to apply changes\n\n### Why are the changes needed?\nAddress DOI review comments to minimize duplication across layers for chmod\n\u003e To minimize duplication across layers, chmod\u0027s should be done in the layer that creates the file/folder (or in the case of a file from the context via COPY, it should have the +x committed to git)\n\nhttps://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #37 from Yikun/SPARK-43371.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "7f836378d8bfe453b7e1dba304b54cb1cfacda49", "tree": "5b3330de57914394265166f787b6f58ed21f6408", "parents": [ "fe05e38f0ffad271edccd6ae40a77d5f14f3eef7" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat May 06 09:15:41 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Sat May 06 09:15:41 2023 +0800" }, "message": "[SPARK-43365] Refactor Dockerfile and workflow based on base image\n\n### What changes were proposed in this pull request?\nThis PR changes Dockerfile and workflow based on base image to save space by sharing layers by having one image from another.\n\nAfter this PR:\n- The spark / PySpark / SparkR related files extract into base image\n- Install PySpark / SparkR deps in PySpark / SparkR images.\n- Add the base image build step\n- Apply changes to template: `./add-dockerfiles.sh 3.4.0` to make it work.\n- This PR didn\u0027t contain changes on 3.3.X Dockerfiles to make PR more clear, the 3.3.x changes will be a separate PR when we address all comments for 3.4.0.\n\n[1] https://github.com/docker-library/official-images/pull/13089?notification_referrer_id\u003dNT_kwDOABp-orI0MzIwMzMwNzY5OjE3MzYzNTQ#issuecomment-1533540388\n\n### Why are the changes needed?\nAddress DOI comments, and also to save space by sharing layers by having one image from another.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed.\n\nCloses #36 from Yikun/official.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "fe05e38f0ffad271edccd6ae40a77d5f14f3eef7", "tree": "3e1273f6734cb10d71e45c6305371f06e4d73a2b", "parents": [ "02bc9054d757f8defbc2baf6af1d2a9aa84b2b35" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Apr 18 10:58:59 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Apr 18 10:58:59 2023 +0800" }, "message": "[SPARK-43148] Add Apache Spark 3.4.0 Dockerfiles\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.4.0 Dockerfiles.\n- Add 3.4.0 GPG key\n- Add .github/workflows/build_3.4.0.yaml\n- ./add-dockerfiles.sh 3.4.0\n\n### Why are the changes needed?\nApache Spark 3.4.0 released:\nhttps://spark.apache.org/releases/spark-release-3-4-0.html\n\n### Does this PR introduce _any_ user-facing change?\nYes in future, new image will publised in future (after DOI reviewed)\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #33 from Yikun/3.4.0.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "02bc9054d757f8defbc2baf6af1d2a9aa84b2b35", "tree": "e1a58d9097076ca0538d5e708e59825fe3a6d6ff", "parents": [ "e8f5b0a1151c349d9c7fdb09cf76300b42a6946b" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Feb 21 17:02:29 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Feb 21 17:02:29 2023 +0800" }, "message": "[SPARK-42505] Apply entrypoint template change to 3.3.0/3.3.1\n\n### What changes were proposed in this pull request?\nApply entrypoint template change to 3.3.0/3.3.1\n\n### Why are the changes needed?\nWe remove the redundant PySpark related vars in https://github.com/apache/spark-docker/commit/e8f5b0a1151c349d9c7fdb09cf76300b42a6946b . This change also should be apply to 3.3.0/3.3.1.\n\n### Does this PR introduce _any_ user-facing change?\nNo, because the image hasn\u0027t plublished yet.\n\n### How was this patch tested?\nCI for 3.3.0/3.3.1 passed\n\nCloses #31 from Yikun/SPARK-42505.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "e8f5b0a1151c349d9c7fdb09cf76300b42a6946b", "tree": "20ad83f9a33dab4459ff94a3c64a7d55da3f2958", "parents": [ "7bb8661f7d57356f94fd5874696df1b1c058cb0b" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Feb 21 14:22:19 2023 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Feb 21 14:22:19 2023 +0800" }, "message": "[SPARK-42494] Add official image Dockerfile for Spark v3.3.2\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.3.2 Dockerfiles.\n- Add 3.3.2 GPG key\n- Add .github/workflows/build_3.3.2.yaml\n- ./add-dockerfiles.sh 3.3.2\n\n### Why are the changes needed?\nApache Spark 3.3.2 released.\n\nhttps://lists.apache.org/thread/k8skf16wyn6rg9n0vd0t6l3bhw7c9svq\n\n### Does this PR introduce _any_ user-facing change?\nYes in future, new image will publised in future (after DOI reviewed)\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #30 from Yikun/SPARK-42494.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "7bb8661f7d57356f94fd5874696df1b1c058cb0b", "tree": "5a463b2f892fe4a8e448eecc6e346f633d3a5f19", "parents": [ "cfcbeac5d2b922a5ee7dfd2b4a5cf08072c827b7" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Dec 21 10:15:44 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Dec 21 10:15:44 2022 +0800" }, "message": "[SPARK-40520] Add support to generate DOI mainifest\n\n### What changes were proposed in this pull request?\nThis patch add support to generate DOI mainifest from versions.json.\n\n### Why are the changes needed?\nTo help generate DOI mainifest\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\n```shell\n$ flake8 ./tools/manifest.py --max-line-length\u003d100\n$ black ./tools/manifest.py\nAll done! ✨ 🍰 ✨\n1 file left unchanged.\n```\n\n```shell\n$ tools/manifest.py manifest\nMaintainers: Apache Spark Developers \u003cdevspark.apache.org\u003e (ApacheSpark)\nGitRepo: https://github.com/apache/spark-docker.git\n\nTags: 3.3.1-scala2.12-java11-python3-ubuntu, 3.3.1-python3, 3.3.1, python3, latest\nArchitectures: amd64, arm64v8\nGitCommit: 496edb6dee0ade08bc5d180d7a6da0ff8b5d91ff\nDirectory: ./3.3.1/scala2.12-java11-python3-ubuntu\n\nTags: 3.3.1-scala2.12-java11-r-ubuntu, 3.3.1-r, r\nArchitectures: amd64, arm64v8\nGitCommit: 496edb6dee0ade08bc5d180d7a6da0ff8b5d91ff\nDirectory: ./3.3.1/scala2.12-java11-r-ubuntu\n\n// ... ...\n```\n\nCloses #27 from Yikun/SPARK-40520.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "cfcbeac5d2b922a5ee7dfd2b4a5cf08072c827b7", "tree": "d27feac784d80e39122db4bc8378f3c613851dea", "parents": [ "d58e17890f07b4c8c8d212775a53c48dc3a6ce42" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Nov 28 17:55:18 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Nov 28 17:55:18 2022 +0800" }, "message": "[SPARK-41287][INFRA] Add test workflow to help self-build image test in fork repo\n\n### What changes were proposed in this pull request?\nThis patch adds a test workflow to help fork repo to test image in their fork repos.\n\n![image](https://user-images.githubusercontent.com/1736354/204183109-e2341397-251e-42a0-b5f7-c1c1f9334ff9.png)\n\nsuch like:\n- https://github.com/Yikun/spark-docker/actions/runs/3552072792/jobs/5966742869\n- https://github.com/Yikun/spark-docker/actions/runs/3561513498/jobs/5982485960\n\n### Why are the changes needed?\nHelp devs/users test their own image in their fork repo\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nTest in my fork repo:\nhttps://github.com/Yikun/spark-docker/actions/workflows/test.yml\n\nCloses #26 from Yikun/test-workflow.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "d58e17890f07b4c8c8d212775a53c48dc3a6ce42", "tree": "1789611df438c221117cf0a38f06739536747eb7", "parents": [ "33abc1894f3de135e827ce393842ca355229c117" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Nov 28 09:36:54 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Nov 28 09:36:54 2022 +0800" }, "message": "[SPARK-41269][INFRA] Move image matrix into version\u0027s workflow\n\n### What changes were proposed in this pull request?\nThis patch refactors main workflow:\n- Move image matrix into version\u0027s workflow to make the main workflow more clear. And also will help downstream repo to only validate specified image type.\n- Move build steps into a same section\n\n### Why are the changes needed?\nThis will help downstream repo to only validate specified image type.\n\nAfter this patch, we will add a test to reuse spark docker workflow like: https://github.com/yikun/spark-docker/commit/45044cee2e8919de7e7353e74f8ca612ad16629a to help developers/users test their self build image.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #25 from Yikun/matrix-refactor.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "33abc1894f3de135e827ce393842ca355229c117", "tree": "db25ea9962dd5a652786f36fde6de7619a22e2c6", "parents": [ "f488d732d254caa78c1e1a2ef74958e6c867dad6" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Nov 25 14:57:27 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Nov 25 14:57:27 2022 +0800" }, "message": "[SPARK-41258][INFRA] Upgrade docker and actions to cleanup warnning\n\n### What changes were proposed in this pull request?\n- Upgrade `actions/checkout` from v2 to v3\n- Upgrade `docker/build-push-action` from v2 to v3\n\n### Why are the changes needed?\nCleanup set output and lower version node warnning\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nTest passed\n\nCloses #24 from Yikun/upgrade-actions.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "f488d732d254caa78c1e1a2ef74958e6c867dad6", "tree": "025cceb32f47d8fa4218eeacef601dfdb755718a", "parents": [ "52152c1b6d70acc2e7c5e32bffe0265b55df7b6f" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Nov 15 21:32:30 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Nov 15 21:32:30 2022 +0800" }, "message": "[SPARK-40519] Add \"Publish\" workflow to help release apache/spark image\n\n### What changes were proposed in this pull request?\nThe publish step will include 3 steps:\n1. First build the local image.\n2. Pass related test (K8s test / Standalone test) using image of first step.\n3. After pass all test, will publish to `ghcr` (This might help RC test) or `dockerhub`\n\nIt\u0027s about 30-40 mins to publish all images.\n\nAdd \"Publish\" workflow to help release apache/spark image.\n![image](https://user-images.githubusercontent.com/1736354/201015477-30428444-0ed5-4436-8b59-7420c678c4a6.png)\n\n### Why are the changes needed?\nOne click to create the `apche/spark` image.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\n1. Set default branch in my fork repo\n2. Run workflow manually, https://github.com/Yikun/spark-docker/actions/workflows/publish.yml?query\u003dis%3Asuccess\n\nCloses #23 from Yikun/workflow.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "52152c1b6d70acc2e7c5e32bffe0265b55df7b6f", "tree": "631a348d333b134bf9e9ea920d6230ce9f6edc25", "parents": [ "243ce201296c20ae48b32a87d254800e8ad197ef" ], "author": { "name": "Qian.Sun", "email": "qian.sun2020@gmail.com", "time": "Wed Nov 09 09:34:47 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Nov 09 09:34:47 2022 +0800" }, "message": "[SPARK-40569][TESTS] Add smoke test in standalone cluster for spark-docker\n\n### What changes were proposed in this pull request?\n\nThis PR aims to add smoke test in standalone cluster for spark-docker repo.\n\n### Why are the changes needed?\n\nVerify spark docker works normally in standalone cluster.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo\n\n### How was this patch tested?\n\nNew test in GA.\n\nCloses #21 from dcoliversun/SPARK-40569.\n\nAuthored-by: Qian.Sun \u003cqian.sun2020@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "243ce201296c20ae48b32a87d254800e8ad197ef", "tree": "5c421154a4da4a7d33a682ec61a458a2fd269220", "parents": [ "5ba4b67d88f101a8491f39030b38ca8f2c6d6351" ], "author": { "name": "Qian.Sun", "email": "qian.sun2020@gmail.com", "time": "Tue Nov 01 11:14:04 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Nov 01 11:14:04 2022 +0800" }, "message": "[SPARK-40969] Replace spark TGZ url with apache archive url\n\n### What changes were proposed in this pull request?\n\nThis PR aims to replace spark TGZ url with apache archive url.\n\n### Why are the changes needed?\n\n```\n#13 [linux/amd64 4/9] RUN set -ex; export SPARK_TMP\u003d\"$(mktemp -d)\"; cd $SPARK_TMP; wget -nv -O spark.tgz \"https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz\"; wget -nv -O spark.tgz.asc \"https://downloads.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc\"; export GNUPGHOME\u003d\"$(mktemp -d)\"; gpg --keyserver hkps://keys.openpgp.org --recv-key \"80FB8EBE8EBA68504989703491B5DC815DBF10D3\" || gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys \"80FB8EBE8EBA68504989703491B5DC815DBF10D3\"; gpg --batch --verify spark.tgz.asc spark.tgz; gpgconf --kill all; rm -rf \"$GNUPGHOME\" spark.tgz.asc; tar -xf spark.tgz --strip-components\u003d1; chown -R spark:spark .; mv jars /opt/spark/; mv bin /opt/spark/; mv sbin /opt/spark/; mv kubernetes/dockerfiles/spark/decom.sh /opt/; mv examples /opt/spark/; mv kubernetes/tests /opt/spark/; mv data /opt/spark/; mv python/pyspark /opt/spark/python/pyspark/; mv python/lib /opt/spark/python/lib/; cd ..; rm -rf \"$SPARK_TMP\";\n#0 0.132 ++ mktemp -d\n#0 0.133 + export SPARK_TMP\u003d/tmp/tmp.oEdW8CyP9h\n#0 0.133 + SPARK_TMP\u003d/tmp/tmp.oEdW8CyP9h\n#0 0.133 + cd /tmp/tmp.oEdW8CyP9h\n#0 0.133 + wget -nv -O spark.tgz https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz\n#0 0.152 https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz:\n#0 0.152 2022-10-31 04:06:44 ERROR 404: Not Found.\n#13 ERROR: process \"/bin/sh -c set -ex; export SPARK_TMP\u003d\\\"$(mktemp -d)\\\"; cd $SPARK_TMP; wget -nv -O spark.tgz \\\"$SPARK_TGZ_URL\\\"; wget -nv -O spark.tgz.asc \\\"$SPARK_TGZ_ASC_URL\\\"; export GNUPGHOME\u003d\\\"$(mktemp -d)\\\"; gpg --keyserver hkps://keys.openpgp.org --recv-key \\\"$GPG_KEY\\\" || gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys \\\"$GPG_KEY\\\"; gpg --batch --verify spark.tgz.asc spark.tgz; gpgconf --kill all; rm -rf \\\"$GNUPGHOME\\\" spark.tgz.asc; tar -xf spark.tgz --strip-components\u003d1; chown -R spark:spark .; mv jars /opt/spark/; mv bin /opt/spark/; mv sbin /opt/spark/; mv kubernetes/dockerfiles/spark/decom.sh /opt/; mv examples /opt/spark/; mv kubernetes/tests /opt/spark/; mv data /opt/spark/; mv python/pyspark /opt/spark/python/pyspark/; mv python/lib /opt/spark/python/lib/; cd ..; rm -rf \\\"$SPARK_TMP\\\";\" did not complete successfully: exit code: 8\n```\nOld url `https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz` is not found. Better to use unity apache archive url.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\nNo need to add new tests.\n\nCloses #22 from dcoliversun/SPARK-40969.\n\nAuthored-by: Qian.Sun \u003cqian.sun2020@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "5ba4b67d88f101a8491f39030b38ca8f2c6d6351", "tree": "85199af074830ae5d3e2b20b2aff87ddd581245e", "parents": [ "f6bab6be5ddcd41d2b6c1b0c139316bc311e13aa" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Oct 27 13:56:21 2022 +0800" }, "committer": { "name": "Yuming Wang", "email": "yumwang@ebay.com", "time": "Thu Oct 27 13:56:21 2022 +0800" }, "message": "[SPARK-40929] Add Apache Spark 3.3.1 Dockerfiles\n\n### What changes were proposed in this pull request?\nAdd Apache Spark 3.3.1 Dockerfiles.\n- Add 3.3.1 GPG key\n- Add .github/workflows/build_3.3.1.yaml\n- ./add-dockerfiles.sh 3.3.1\n\n### Why are the changes needed?\nApache Spark 3.3.1 released.\n\nhttps://lists.apache.org/thread/7x5s8fpg16jvlm4cng5kvfk3njhwd0zh\n\n### Does this PR introduce _any_ user-facing change?\nYes in future, new image will publised in future\n\n### How was this patch tested?\nAdd workflow and CI passed\n\nCloses #20 from Yikun/SPARK-40929.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yuming Wang \u003cyumwang@ebay.com\u003e\n" }, { "commit": "f6bab6be5ddcd41d2b6c1b0c139316bc311e13aa", "tree": "46c392629dc84da4e98c2af056acc4274563a0a3", "parents": [ "52e5856d81e70a9d9e87292c6caf42587ce433df" ], "author": { "name": "Qian.Sun", "email": "qian.sun2020@gmail.com", "time": "Tue Oct 25 10:15:10 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Oct 25 10:15:10 2022 +0800" }, "message": "[SPARK-40855] Add CONTRIBUTING.md for apache/spark-docker\n\n### What changes were proposed in this pull request?\n\nThis PR aims to add `CONTRIBUTING.md` for apache/spark-docker.\n\n### Why are the changes needed?\n\nBetter to briefly explain how to contribute DOI.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo.\n\n### How was this patch tested?\n\n![image](https://user-images.githubusercontent.com/44011673/197155544-bfae0c70-ee01-44b0-851d-ed5c288129d9.png)\n\nCloses #19 from dcoliversun/SPARK-40855.\n\nAuthored-by: Qian.Sun \u003cqian.sun2020@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "52e5856d81e70a9d9e87292c6caf42587ce433df", "tree": "c9c631bbc5df84be92559a2f949b74760fc86907", "parents": [ "6f56ef1c8c8bccd05069d4590f7ae084d4c72b4d" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Oct 21 17:02:54 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Oct 21 17:02:54 2022 +0800" }, "message": "[SPARK-40864] Remove pip/setuptools dynamic upgrade\n\n### What changes were proposed in this pull request?\nRemove pip/setuptools dynamic upgrade in dockerfile\n\n### Why are the changes needed?\nAccording to [official image suggestion](https://github.com/docker-library/official-images#repeatability), `Rebuilding the same Dockerfile should result in the same version of the image being packaged`.\n\nBut we used to upgrade pip/setuptools to latest, actually we don\u0027t need a latest pip/setuptools for any reason I can think out. I also take a look on [initial commits](https://github.com/apache-spark-on-k8s/spark/commit/befcf0a30651d0335bb57c242a824e43748db33f) for this line, according merge history no more reason for it.\n\n### Does this PR introduce _any_ user-facing change?\nThe OS recommand pip/setuptools version is used.\n\n### How was this patch tested?\n\nCI passed.\n\nCloses #17 from Yikun/remove-pip.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "6f56ef1c8c8bccd05069d4590f7ae084d4c72b4d", "tree": "b7031609b9af7d7000450ed56a394153b037c5af", "parents": [ "896a36e36c094bf1480f4819005e2982ea8af417" ], "author": { "name": "Qian.Sun", "email": "qian.sun2020@gmail.com", "time": "Fri Oct 21 16:02:50 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Oct 21 16:02:50 2022 +0800" }, "message": "[SPARK-40866][INFRA] Rename Spark repository as Spark Docker repository in GA\n\n### What changes were proposed in this pull request?\n\nThis PR aim to rename `Spark repository` as `Spark Docker repository` in GA, discussion as https://github.com/apache/spark-docker/pull/15#discussion_r1001440707\n\n### Why are the changes needed?\n\nActually repository is apache/spark-docker.\n\n### Does this PR introduce _any_ user-facing change?\n\nNo\n\n### How was this patch tested?\n\nPass the GA\n\nCloses #18 from dcoliversun/SPARK-40866.\n\nAuthored-by: Qian.Sun \u003cqian.sun2020@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "896a36e36c094bf1480f4819005e2982ea8af417", "tree": "b61bf8f759221851d940182f259a1041bd88732a", "parents": [ "95f5a1f3e846ad3b6550e151fa76b70f6fe0b946" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Oct 20 15:38:03 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Oct 20 15:38:03 2022 +0800" }, "message": "[SPARK-40845] Add template support for SPARK_GPG_KEY and fix GPG verify\n\n### What changes were proposed in this pull request?\nThis patch:\n- Add template support for `SPARK_GPG_KEY`.\n- Fix a bug on GPG verified. (Change `||` to `;`)\n- Use opengpg.org instead of gpg.com becasue it would be uploaded in [spark release process](https://spark.apache.org/release-process.html).\n\n### Why are the changes needed?\nEach version have specific GPG key to verified, so we need to set GPG version separately.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\nRun `./add-dockerfiles.sh 3.3.0` and see GPG set correctly\n\nCloses #16 from Yikun/GPG.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "95f5a1f3e846ad3b6550e151fa76b70f6fe0b946", "tree": "c5d2fe2a387e14ab111ef920e6089b810c763e18", "parents": [ "c1353a377176d9f2a84641323840130bd160e436" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 19 10:17:58 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 19 10:17:58 2022 +0800" }, "message": "[SPARK-40833] Cleanup apt lists cache\n\n### What changes were proposed in this pull request?\nRemove unused apt lists cache and apply `./add-dockerfiles.sh 3.3.0`\n\n### Why are the changes needed?\nClean cache to reduce docker image size.\n\nThis is also [recommanded](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run) by docker community:\n\n```\n$ docker run --user 0:0 -ti apache/spark bash\nroot5d1ca347279e:/opt/spark/work-dir# ls /var/lib/apt/lists/\nauxfiles\t\t\t\t\t\t\t\t lock\ndeb.debian.org_debian_dists_bullseye-updates_InRelease\t\t\t partial\ndeb.debian.org_debian_dists_bullseye-updates_main_binary-arm64_Packages.lz4 security.debian.org_debian-security_dists_bullseye-security_InRelease\ndeb.debian.org_debian_dists_bullseye_InRelease\t\t\t\t security.debian.org_debian-security_dists_bullseye-security_main_binary-arm64_Packages.lz4\ndeb.debian.org_debian_dists_bullseye_main_binary-arm64_Packages.lz4\nroot5d1ca347279e:/opt/spark/work-dir# du --max-depth\u003d1 -h /var/lib/apt/lists/\n4.0K\t/var/lib/apt/lists/partial\n4.0K\t/var/lib/apt/lists/auxfiles\n17M\t/var/lib/apt/lists/\n```\n\n### Does this PR introduce _any_ user-facing change?\nYes in some level, image size is reduced.\n\n### How was this patch tested?\nK8s CI passed\n\nCloses #14 from Yikun/clean-apt-list.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "c1353a377176d9f2a84641323840130bd160e436", "tree": "f70d8cda5a7113ca0c8b90f61e18a4b10379aaab", "parents": [ "6459e3d09a2e009573be355e63c404bb35139d28" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 19 10:16:41 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 19 10:16:41 2022 +0800" }, "message": "[SPARK-40832][DOCS] Add README for spark-docker\n\n### What changes were proposed in this pull request?\nAdd README for spark-docker\n\n### Why are the changes needed?\nAlthough the PR of DOI has not been merged yet, but we\u0027d better to briefly explain what this repository does.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nPreview manually:\n\n\u003cimg width\u003d\"975\" alt\u003d\"image\" src\u003d\"https://user-images.githubusercontent.com/1736354/196381318-cb3d72e1-1ba7-479c-82cb-4412dde91179.png\"\u003e\n\nCloses #13 from Yikun/readme.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "6459e3d09a2e009573be355e63c404bb35139d28", "tree": "a6ae87f1f087a7f02ef562dca0b4947a42b8a759", "parents": [ "a75ecb13dee5580a149f2ef0bd9f8a4371d3d956" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 17 16:23:23 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 17 16:23:23 2022 +0800" }, "message": "[SPARK-40528] Support dockerfile template\n\n### What changes were proposed in this pull request?\nThis patch:\n- Add dockerfile template: `Dockerfile.template` contains 3 vars: `BASE_IMAGE` for base image name, `HAVE_PY` for adding python support, `HAVE_R` for adding sparkr support.\n- Add a script: `add-dockerfiles.sh`, you can `./add-dockerfiles.sh 3.3.0`\n- Add a tool: `tempalte.py` to help generate dockerfile from jinja template.\n\n### Why are the changes needed?\nGenerate the dockerfiles to make life easier.\n\n### Does this PR introduce _any_ user-facing change?\nNo, dev only.\n\n### How was this patch tested?\n```shell\n# Prepare new env\npython3 -m venv ~/xxx\npip install -r ./tools/requirements.txt\nsource ~/xxx/bin/activate\n\n# Generate 3.3.0\n./add-dockerfiles.sh 3.3.0\n\n# no diff\ngit diff\n```\n\nlint:\n```\n$ flake8 ./tools/template.py\n$ black ./tools/template.py\nAll done! ✨ 🍰 ✨\n1 file left unchanged.\n```\n\nCloses #12 from Yikun/SPARK-40528.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "a75ecb13dee5580a149f2ef0bd9f8a4371d3d956", "tree": "e3e3b56e08eda4f568f19141df6cd122956a40fd", "parents": [ "3037f75a88ca7ea57746c7d1bf49c125a828f56e" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 17 16:09:30 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 17 16:09:30 2022 +0800" }, "message": "[SPARK-40805] Use `spark` username in official image\n\n### What changes were proposed in this pull request?\nThis patch:\n- Add spark uid/gid in dockerfile (useradd and groupadd). (used in entrypoint) This way is also used by [others DOI](https://github.com/search?p\u003d2\u0026q\u003dorg%3Adocker-library+useradd\u0026type\u003dCode) and apache DOI (such as [zookeeper](https://github.com/31z4/zookeeper-docker/blob/master/3.8.0/Dockerfile#L17-L21), [solr](https://github.com/apache/solr-docker/blob/a20477ed123cd1a72132aebcc0742cee46b5f976/9.0/Dockerfile#L108-L110), [flink](https://github.com/apache/flink-docker/blob/master/1.15/scala_2.12-java11-ubuntu/Dockerfile#L55-L56)).\n- Use `spark` user in `entrypoint.sh` rather than Dockerfile. (make sure the spark process is executed as non-root users)\n- Remove `USER` setting in Dockerfile. (make sure base image has permission to extend dockerifle, such as execute `apt update`)\n- Chown script to `spark:spark` instead of `root:root`. (avoid permission issue such like standalone mode)\n- Add `gosu` deps, a `sudo` replacement recommanded by [docker](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user) and [docker official image](https://github.com/docker-library/official-images/blob/9a4d54f1a42ea82970baa4e6f3d0bc75e98fc961/README.md#consistency), and also are used by other DOI images.\n\nThis change also follow the rules of docker official images, see also [consistency](https://github.com/docker-library/official-images/blob/9a4d54f1a42ea82970baa4e6f3d0bc75e98fc961/README.md#consistency) and [dockerfile best practices about user](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user).\n\n### Why are the changes needed?\n\nThe below issues are what I have found so far\n\n1. **Irregular login username**\n Docker images username is not very standard, docker run with `185` username is a little bit weird.\n\n ```\n $ docker run -ti apache/spark bash\n 185d88a24357413:/opt/spark/work-dir$\n ```\n\n2. **Permission issue of spark sbin**\nAnd also there are some permission issue when running some spark script, such as standalone mode:\n\n ```\n $ docker run -ti apache/spark /opt/spark/sbin/start-master.sh\n\n mkdir: cannot create directory ‘/opt/spark/logs’: Permission denied\n chown: cannot access \u0027/opt/spark/logs\u0027: No such file or directory\n starting org.apache.spark.deploy.master.Master, logging to /opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-1c345a00e312.out\n /opt/spark/sbin/spark-daemon.sh: line 135: /opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-1c345a00e312.out: No such file or directory\n failed to launch: nice -n 0 /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master --host 1c345a00e312 --port 7077 --webui-port 8080\n tail: cannot open \u0027/opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-1c345a00e312.out\u0027 for reading: No such file or directory\n full log in /opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-1c345a00e312.out\n ```\n\n \u003c/details\u003e\n\n3. **spark as base image case is not supported well**\n Due to static USER set in Dockerfile.\n ```\n $ cat Dockerfile\n FROM apache/spark\n RUN apt update\n\n $ docker build -t spark-test:1015 .\n // ...\n ------\n \u003e [2/2] RUN apt update:\n #5 0.405 E: Could not open lock file /var/lib/apt/lists/lock - open (13: Permission denied)\n #5 0.405 E: Unable to lock directory /var/lib/apt/lists/\n ------\n executor failed running [/bin/sh -c apt update]: exit code: 100\n\n ```\n\n### Does this PR introduce _any_ user-facing change?\nYes.\n\n### How was this patch tested?\n- CI passed: all k8s test\n\n- Regression test:\n```\n# Username is set to spark rather than 185\ndocker run -ti spark:scala2.12-java11-python3-r-ubuntu bash\nspark27bbfca0a581:/opt/spark/work-dir$\n```\n```\n# start-master.sh no permission issue\n$ docker run -ti spark:scala2.12-java11-python3-r-ubuntu bash\n\nspark8d1118e26766:~/work-dir$ /opt/spark/sbin/start-master.sh\nstarting org.apache.spark.deploy.master.Master, logging to /opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-8d1118e26766.out\n```\n```\n# Image as parent case\n$ cat Dockerfile\nFROM spark:scala2.12-java11-python3-r-ubuntu\nRUN apt update\n$ docker build -t spark-test:1015 .\n[+] Building 7.8s (6/6) FINISHED\n \u003d\u003e [1/2] FROM docker.io/library/spark:scala2.12-java11-python3-r-ubuntu 0.0s\n \u003d\u003e [2/2] RUN apt update 7.7s\n```\n\n- Other test:\n```\n# Test on pyspark\n$ cd spark-docker/3.3.0/scala2.12-java11-python3-r-ubuntu\n$ docker build -t spark:scala2.12-java11-python3-r-ubuntu .\n$ docker run -p 4040:4040 -ti spark:scala2.12-java11-python3-r-ubuntu /opt/spark/bin/pyspark\n```\n\n```\n# A simple test for `start-master.sh` (standalone mode)\n$ docker run -ti spark:scala2.12-java11-python3-r-ubuntu bash\nspark8d1118e26766:~/work-dir$ /opt/spark/sbin/start-master.sh\nstarting org.apache.spark.deploy.master.Master, logging to /opt/spark/logs/spark--org.apache.spark.deploy.master.Master-1-8d1118e26766.out\n```\n\nCloses #11 from Yikun/spark-user.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "3037f75a88ca7ea57746c7d1bf49c125a828f56e", "tree": "90bd697933de474ddbb18afffc46c65d43a00f45", "parents": [ "fc07aeda1f48eb2aae9a441dfe94ae95f697e222" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Oct 14 11:57:01 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Fri Oct 14 11:57:01 2022 +0800" }, "message": "[SPARK-40783][INFRA] Enable Spark on K8s integration test\n\n### What changes were proposed in this pull request?\nThis patch enable the Spark on K8s integration test:\n\n- **scala2.12-java11-python3-ubuntu**: Run scala / PySpark basic test\n- **scala2.12-java11-ubuntu**: Run scala basic test\n- **scala2.12-java11-r-ubuntu**: Run scala / SparkR basic test\n- **scala2.12-java11-python3-r-ubuntu**: Run all K8s integration test\n\nCurrently, we use the local registry as a bridge between build and test:\n\u003cimg width\u003d\"646\" alt\u003d\"image\" src\u003d\"https://user-images.githubusercontent.com/1736354/195758243-abfbea7f-05e9-4678-a3a5-cfd38cc1b8f5.png\"\u003e\n\n- Build: generate the image and push to local registry\n- Test: load to minikube docker, run K8s test using specific image\n\nDue to the multi-platform images cannot be exported with the `docker` export type, the local registry (push) is used here rather than local build (load). Compare to `ghcr` it reduces the network transmition and permission required.\n\nAlso:\n- Upgrade `setup-qemu-action` to v2\n- Upgrade `setup-buildx-action` to v2\n- Remove ununsed `Image digest` step\n\n### Why are the changes needed?\nTo ensure the quality of official dockerfiles.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses #9 from Yikun/enable-k8s-it.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "fc07aeda1f48eb2aae9a441dfe94ae95f697e222", "tree": "936d8b9fa463ff7ae56f72ae800bb09996e10762", "parents": [ "c11669850c0c03212df6d5c84c01050e6c933076" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Oct 13 21:47:15 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Thu Oct 13 21:47:15 2022 +0800" }, "message": "[SPARK-40754][DOCS] Add LICENSE and NOTICE\n\n### What changes were proposed in this pull request?\nThis pach adds LICENSE and NOTICE:\n- LICENSE: https://www.apache.org/licenses/LICENSE-2.0.txt\n- NOTICE: https://github.com/apache/spark/blob/master/NOTICE\n\n### Why are the changes needed?\nhttps://www.apache.org/licenses/LICENSE-2.0#apply\n\nSee also: https://github.com/apache/spark-docker/pull/2#issuecomment-1274807917\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nNo need\n\nCloses #6 from Yikun/SPARK-40754.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "c11669850c0c03212df6d5c84c01050e6c933076", "tree": "4040a2fb300c6ede19c2fdac7c9cae3af39aa967", "parents": [ "30fd82f313c4ecd44f4181e6a4cf2e1d9463c628" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 12 10:48:51 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 12 10:48:51 2022 +0800" }, "message": "[SPARK-40746][INFRA] Fix Dockerfile build workflow\n\n### What changes were proposed in this pull request?\nThis patch is to make the workflow work in apache repo:\n- Add `.github/workflows/build_3.3.0.yaml` and `3.3.0/**` to trigger paths\n- Change `apache/spark-docker:TAG` to `ghcr.io/apache/spark-docker/spark:TAG`\n- Remove the push, we only need to build locally to validate dockerfile, even in future K8s IT test we can also refactor to use minikube docker, it still can be local build.\n\n### Why are the changes needed?\nTo make the workflow works well in apache repo.\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\nCI passed\n\nCloses: https://github.com/apache/spark-docker/pull/5\n\nCloses #7 from Yikun/SPARK-40746.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "30fd82f313c4ecd44f4181e6a4cf2e1d9463c628", "tree": "dc990ab74d336a6c614a24daaa1089b51995f76d", "parents": [ "e61aba1ed4ca8e747f38cae5f6bd72a3a50f57cd" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 12 10:47:31 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Wed Oct 12 10:47:31 2022 +0800" }, "message": "[SPARK-40757][INFRA] Add PULL_REQUEST_TEMPLATE for spark-docker\n\n### What changes were proposed in this pull request?\nInitialize with https://github.com/apache/spark/blob/master/.github/PULL_REQUEST_TEMPLATE and remove some unsued note\n\n### Why are the changes needed?\nAdd PULL_REQUEST_TEMPLATE for `spark-docker`\n\n### Does this PR introduce _any_ user-facing change?\nNo, dev only\n\n### How was this patch tested?\nNew PR after this merged\n\nCloses #8 from Yikun/SPARK-40757.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "e61aba1ed4ca8e747f38cae5f6bd72a3a50f57cd", "tree": "d0d660ba61528b2274ce613007e1cc3b6e493ad5", "parents": [ "fa2d1a59b6e47b1e4072154de0b1f215780af595" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Oct 11 10:45:57 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Tue Oct 11 10:45:57 2022 +0800" }, "message": "[SPARK-40516] Add Apache Spark 3.3.0 Dockerfile\n\n### What changes were proposed in this pull request?\nThis patch adds Apache Spark 3.3.0 Dockerfile:\n- 3.3.0-scala2.12-java11-python3-ubuntu: pyspark + scala\n- 3.3.0-scala2.12-java11-ubuntu: scala\n- 3.3.0-scala2.12-java11-r-ubuntu: sparkr + scala\n- 3.3.0-scala2.12-java11-python3-r-ubuntu: All in one image\n\n### Why are the changes needed?\nThis is needed by Docker Official Image\n\nSee also in: https://docs.google.com/document/d/1nN-pKuvt-amUcrkTvYAQ-bJBgtsWb9nAkNoVNRM2S2o\n\n### Does this PR introduce _any_ user-facing change?\nNo\n\n### How was this patch tested?\n**The action won\u0027t be triggered until the workflow is merged to the default branch**, so I can only test it in my local repo:\n\n- local test: https://github.com/Yikun/spark-docker/pull/1\n![image](https://user-images.githubusercontent.com/1736354/194975185-d5843c84-bbba-48d0-bbf0-363532c6712d.png)\n- Dockerfile E2E K8S Local test: https://github.com/Yikun/spark-docker-bak/pull/7\n![image](https://user-images.githubusercontent.com/1736354/194975267-6dca0de5-c715-4e0f-b735-22752b7912de.png)\n\nCloses #2 from Yikun/SPARK-40516.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "fa2d1a59b6e47b1e4072154de0b1f215780af595", "tree": "2937db9c2ab166864ac9b9a7cfaf6b4d424b7e59", "parents": [ "c5b015ac2014bfeb47daafd454b610ae9633f676" ], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 10 20:34:43 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 10 20:34:43 2022 +0800" }, "message": "[SPARK-40727][INFRA] Add merge_spark_docker_pr.py\n\n### What changes were proposed in this pull request?\nThis patch add the merge_spark_docker_pr.py to help to merge `spark-docker` commits and resolve spark JIRA issue.\n\nThe script is from https://github.com/apache/spark/blob/ef837ca71020950b841f9891c70dc4b29d968bf1/dev/merge_spark_pr.py\n\nAnd change `spark` to `spark-docker`: https://github.com/apache/spark-docker/commit/e4107a74d348656041612ff68a647c6051894240\n\n### Why are the changes needed?\nHelp to merge spark-docker commits.\n\n### Does this PR introduce _any_ user-facing change?\nNo, dev only\n\n### How was this patch tested?\nwill merge it by using itself\n\nCloses #1 from Yikun/merge_script.\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" }, { "commit": "c5b015ac2014bfeb47daafd454b610ae9633f676", "tree": "5b4ddf378a12a09607d582d428bd97abc75c35b9", "parents": [], "author": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 10 16:40:34 2022 +0800" }, "committer": { "name": "Yikun Jiang", "email": "yikunkero@gmail.com", "time": "Mon Oct 10 16:40:34 2022 +0800" }, "message": "[SPARK-40723][INFRA] Add .asf.yaml to spark-docker\n\n### What changes were proposed in this pull request?\nThis change add the .asf.yaml as first commit.\n\n### Why are the changes needed?\nInitialize the repo setting.\n\n### Does this PR introduce _any_ user-facing change?\nNo, dev only\n\n### How was this patch tested?\nSee result after merged\n\nAuthored-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\nSigned-off-by: Yikun Jiang \u003cyikunkero@gmail.com\u003e\n" } ] }