| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| # Spark master (5.0.0-SNAPSHOT) release image | |
| # Extends the base image with: | |
| # - Java 21 | |
| # - Python 3.11 with required packages | |
| FROM spark-rm-base:latest | |
| # Install Java 21 for Spark 5.x | |
| RUN apt-get update && apt-get install -y \ | |
| openjdk-21-jdk-headless \ | |
| && rm -rf /var/lib/apt/lists/* | |
| # Install Python 3.11 | |
| RUN apt-get update && apt-get install -y \ | |
| python3.11 \ | |
| python3.11-dev \ | |
| python3-psutil \ | |
| libpython3-dev \ | |
| && rm -rf /var/lib/apt/lists/* | |
| # Install pip for Python 3.11 | |
| RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 | |
| # Basic Python packages for master | |
| ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.3 scipy plotly<6.0.0 \ | |
| mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2 twine==3.4.1" | |
| # Python deps for Spark Connect | |
| ARG CONNECT_PIP_PKGS="grpcio==1.84.1 grpcio-status==1.84.1 protobuf==6.33.5 \ | |
| googleapis-common-protos==1.71.0 graphviz==0.20.3" | |
| # Install Python 3.11 packages | |
| RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' && \ | |
| python3.11 -m pip install --ignore-installed 'six==1.16.0' && \ | |
| python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ | |
| python3.11 -m pip install 'torch<2.6.0' torchvision --index-url https://download.pytorch.org/whl/cpu && \ | |
| python3.11 -m pip install deepspeed torcheval && \ | |
| python3.11 -m pip cache purge | |
| # Sphinx and documentation packages | |
| # See 'docutils' in SPARK-39421 | |
| RUN python3.11 -m pip install 'sphinx==8.2.3' mkdocs 'pydata_sphinx_theme>=0.13' \ | |
| sphinx-copybutton nbsphinx numpydoc markupsafe 'pyzmq<24.0.0' \ | |
| ipython sphinx_plotly_directive 'numpy>=1.23.2' pyarrow pandas \ | |
| 'plotly>=4.8' 'docutils' 'mypy==1.19.1' 'pytest==7.1.3' \ | |
| 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' 'pandas-stubs==1.2.0.53' \ | |
| 'grpcio==1.84.1' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' | |
| # Set Python 3.11 as the default | |
| RUN ln -sf "$(which python3.11)" "/usr/local/bin/python" && \ | |
| ln -sf "$(which python3.11)" "/usr/local/bin/python3" | |
| # Create user for release manager | |
| ARG UID | |
| RUN useradd -m -s /bin/bash -p spark-rm -u $UID spark-rm | |
| USER spark-rm:spark-rm | |
| ENTRYPOINT [ "/opt/spark-rm/do-release.sh" ] |