| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| name: python-sdk |
| |
| # Trigger the workflow on push or pull request |
| on: [push, pull_request] |
| |
| env: |
| KUBERNETES_VERSION: "v1.21.14" |
| |
| jobs: |
| check-style: |
| runs-on: ubuntu-latest |
| timeout-minutes: 30 |
| steps: |
| - uses: actions/checkout@v3 |
| - name: Install dependencies |
| run: | |
| pip install --upgrade pip setuptools==65.7.0 wheel |
| pip install -r ./dev-support/style-check/python/lint-requirements.txt |
| pip install -r ./dev-support/style-check/python/mypy-requirements.txt |
| - name: List installed packages |
| run: pip list |
| - name: Check python sdk code style |
| run: ./dev-support/style-check/python/lint.sh |
| |
| unit: |
| runs-on: ubuntu-latest |
| timeout-minutes: 30 |
| strategy: |
| matrix: |
| python-version: ['3.7', '3.8', '3.9', '3.10'] |
| tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] |
| exclude: |
| - python-version: '3.8' |
| tf-version: '1.15.0' |
| - python-version: '3.9' |
| tf-version: '1.15.0' |
| - python-version: '3.10' |
| tf-version: '1.15.0' |
| - python-version: '3.10' |
| tf-version: '2.6.0' |
| - python-version: '3.10' |
| tf-version: '2.7.0' |
| fail-fast: false |
| steps: |
| - uses: actions/checkout@v3 |
| - name: Setup python environment |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ matrix.python-version }} |
| - name: Install requirements |
| run: | |
| pip install --upgrade pip setuptools==65.7.0 wheel |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt |
| - name: Install pysubmarine with tf1 and pytorch |
| if: ${{ matrix.tf-version == '1.15.0' }} |
| run: | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] |
| - name: Install pysubmarine with tf2 and pytorch |
| if: startsWith(matrix.tf-version, '2.') |
| run: | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] |
| pip install --no-cache-dir 'tensorflow~=${{ matrix.tf-version }}' |
| - name: List installed packages |
| run: pip list && pipdeptree |
| - name: Run unit test |
| working-directory: ./submarine-sdk/pysubmarine |
| run: pytest -m "not e2e" --cov-report xml |
| - uses: codecov/codecov-action@v3 |
| with: |
| flags: python-unit |
| files: ./submarine-sdk/pysubmarine/coverage.xml |
| name: codecov-unit |
| |
| integration: |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| strategy: |
| matrix: |
| python-version: ['3.7', '3.8', '3.9', '3.10'] |
| tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] |
| exclude: |
| - python-version: '3.8' |
| tf-version: '1.15.0' |
| - python-version: '3.9' |
| tf-version: '1.15.0' |
| - python-version: '3.10' |
| tf-version: '1.15.0' |
| - python-version: '3.10' |
| tf-version: '2.6.0' |
| - python-version: '3.10' |
| tf-version: '2.7.0' |
| fail-fast: false |
| steps: |
| - uses: actions/checkout@v3 |
| with: |
| fetch-depth: 50 |
| - name: Set up JDK 1.8 |
| uses: actions/setup-java@v1 |
| with: |
| java-version: "1.8" |
| - name: Set up Maven 3.6.3 |
| uses: stCarolas/setup-maven@v4 |
| with: |
| maven-version: 3.6.3 |
| - name: Check version |
| run: | |
| mvn --version |
| java -version |
| - name: Create kind cluster |
| run: kind create cluster --config ./.github/config/kind-config-kind.yaml --wait 3m --image kindest/node:${KUBERNETES_VERSION} |
| - name: Download Istio 1.13.0 |
| run: | |
| curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.13.0 sh - |
| cd istio-1.13.0 |
| echo "$PWD/bin" >> $GITHUB_PATH |
| - name: Install Istio |
| run: istioctl install -y --set values.global.proxy.resources.requests.cpu=10m |
| - name: Show K8s cluster information |
| run: | |
| kubectl cluster-info |
| kubectl version |
| kubectl get pods -n kube-system |
| echo "current-context:" $(kubectl config current-context) |
| echo "environment-kubeconfig:" ${KUBECONFIG} |
| # Cache maven (for submarine server) |
| - uses: actions/cache@v3 |
| with: |
| path: ~/.m2/repository |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-maven- |
| - name: Build Image locally |
| run: .github/scripts/build-image-locally-v3.sh |
| - name: Start submarine |
| run: bash ./.github/scripts/start-submarine.sh |
| - name: Setup python environment |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ matrix.python-version }} |
| - name: Install tf1 dependencies |
| if: ${{ matrix.tf-version == '1.15.0' }} |
| run: | |
| pip install --upgrade pip setuptools==65.7.0 wheel |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt |
| - name: Install tf2 dependencies |
| if: startsWith(matrix.tf-version, '2.') |
| run: | |
| pip install --upgrade pip setuptools==65.7.0 wheel |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] |
| pip install --no-cache-dir 'tensorflow~=${{ matrix.tf-version }}' |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt |
| - name: List installed packages |
| run: pip list |
| - name: Set CLI config for e2e pytest |
| run: | |
| submarine config set connection.hostname localhost |
| submarine config set connection.port 8080 |
| submarine config list |
| - name: Run integration test |
| working-directory: ./submarine-sdk/pysubmarine |
| run: pytest -m "e2e" --cov-report xml |
| - uses: codecov/codecov-action@v3 |
| with: |
| flags: python-integration |
| files: ./submarine-sdk/pysubmarine/coverage.xml |
| name: codecov-integration |
| - name: Failure status |
| run: | |
| kubectl describe nodes |
| kubectl get pods |
| kubectl -n default get events --sort-by='{.lastTimestamp}' |
| if: ${{ failure() }} |
| |
| sonarcloud: |
| if: github.repository == 'apache/submarine' && github.event_name == 'push' && github.ref == 'refs/heads/master' |
| runs-on: ubuntu-latest |
| steps: |
| - uses: actions/checkout@v3 |
| - name: SonarCloud Scan |
| uses: SonarSource/sonarcloud-github-action@master |
| env: |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} |
| with: |
| projectBaseDir: submarine-sdk/pysubmarine |
| args: > |
| -Dsonar.organization=apache |
| -Dsonar.projectKey=apache_submarine_python_sdk |
| -Dsonar.python.version=3.7,3.8,3.9,3.10 |