| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| name: Collect And Deploy Playground Examples |
| |
| on: |
| workflow_dispatch: |
| env: |
| BEAM_ROOT_DIR: "../../" |
| SDK_CONFIG: "../../playground/sdks.yaml" |
| BEAM_EXAMPLE_CATEGORIES: "../categories.yaml" |
| BEAM_VERSION: 2.40.0 |
| K8S_NAMESPACE: playground-backend |
| HELM_APP_NAME: playground-backend |
| SUBDIRS: "./learning/katas ./examples ./sdks" |
| ORIGIN: PG_EXAMPLES |
| jobs: |
| check_examples: |
| name: Check examples |
| runs-on: ubuntu-latest |
| outputs: |
| example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }} |
| steps: |
| - name: Check out the repo |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 0 |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: '3.8' |
| - name: install deps |
| run: pip install -r requirements.txt |
| working-directory: playground/infrastructure |
| - name: get Difference |
| id: check_file_changed |
| run: | |
| set -xeu |
| # define the base ref |
| BASE_REF=$GITHUB_BASE_REF |
| if [ -z "$BASE_REF" ] || [ "$BASE_REF" == "master" ]; then |
| BASE_REF=origin/master |
| fi |
| DIFF=$(git diff --name-only $BASE_REF $GITHUB_SHA -- $SUBDIRS) |
| echo "example_diff=$DIFF" >> $GITHUB_OUTPUT |
| - name: has Examples |
| run: | |
| output=$(python3 checker.py ${{ steps.check_file_changed.outputs.example_diff }}) |
| echo "example_has_changed=$output" >> $GITHUB_OUTPUT |
| id: check_has_example |
| working-directory: playground/infrastructure |
| - name: Print has_example |
| run: echo "${{ steps.check_has_example.outputs.example_has_changed }}" |
| deploy_examples: |
| name: Deploy examples |
| runs-on: ubuntu-latest |
| needs: [ check_examples ] |
| if: needs.check_examples.outputs.example_has_changed == 'True' |
| env: |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json |
| GOOGLE_CLOUD_PROJECT: ${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} |
| SA_KEY_CONTENT: ${{ secrets.GCP_PLAYGROUND_SA_KEY }} |
| REGION: us-central1 |
| steps: |
| - name: Check out the repo |
| uses: actions/checkout@v3 |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: '3.8' |
| - uses: actions/setup-java@v3.8.0 |
| with: |
| distribution: 'zulu' |
| java-version: '8' |
| - name: Install kubectl |
| run: | |
| curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\ |
| chmod +x kubectl &&\ |
| mv kubectl /usr/local/bin/ |
| - name: Install helm |
| run: | |
| curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\ |
| chmod 700 get_helm.sh &&\ |
| ./get_helm.sh |
| - name: Set up Cloud SDK |
| uses: google-github-actions/setup-gcloud@v0 |
| with: |
| install_components: 'gke-gcloud-auth-plugin' |
| - name: install deps |
| run: pip install -r requirements.txt |
| working-directory: playground/infrastructure |
| - name: Remove default github maven configuration |
| # This step is a workaround to avoid a decryption issue |
| run: rm ~/.m2/settings.xml |
| - name: Setup GCP account |
| run: | |
| echo "$SA_KEY_CONTENT" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS |
| which gcloud |
| gcloud auth activate-service-account --project=$GOOGLE_CLOUD_PROJECT \ |
| --key-file=$GOOGLE_APPLICATION_CREDENTIALS |
| - name: Set Docker Tag |
| run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV |
| - name: Set Docker Tag If Github Tag was trigger |
| run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV |
| if: startsWith(github.ref, 'refs/tags/') |
| - name: Set Docker Registry env |
| run: echo "DOCKER_REGISTRY=$DOCKER_REGISTRY" >> $GITHUB_ENV |
| env: |
| DOCKER_REGISTRY: ${{ env.REGION }}-docker.pkg.dev/${{ env.GOOGLE_CLOUD_PROJECT }}/playground-repository |
| |
| - name: Get K8s Config |
| run: gcloud container clusters get-credentials --region us-central1-a playground-examples |
| - name: Login to Docker Registry |
| run: cat $GOOGLE_APPLICATION_CREDENTIALS | docker login -u _json_key --password-stdin "$PLAYGROUND_REGISTRY_URL" |
| env: |
| PLAYGROUND_REGISTRY_URL: https://${{ env.REGION }}-docker.pkg.dev |
| - name: Build And Push Java Backend |
| run: | |
| ./gradlew playground:backend:containers:java:dockerTagPush \ |
| -Pdocker-repository-root="$DOCKER_REGISTRY" \ |
| -Pbase-image="apache/beam_java8_sdk:$BEAM_VERSION" \ |
| -Pdocker-tag="$DOCKERTAG" |
| - name: Build And Push Go Backend |
| run: | |
| ./gradlew playground:backend:containers:go:dockerTagPush \ |
| -Pdocker-repository-root="$DOCKER_REGISTRY" \ |
| -Pdocker-tag="$DOCKERTAG" |
| - name: Build And Push Python Backend |
| run: | |
| ./gradlew playground:backend:containers:python:dockerTagPush \ |
| -Pdocker-repository-root="$DOCKER_REGISTRY" \ |
| -Pdocker-tag="$DOCKERTAG" |
| - name: Clean All Build directories |
| run: ./gradlew clean |
| - name: Install helm chart |
| run: | |
| kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\ |
| helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . \ |
| --set global.registry="$DOCKER_REGISTRY" \ |
| --set global.tag="$DOCKERTAG" &&\ |
| sleep 120 |
| working-directory: playground/infrastructure/helm |
| |
| - name: Run Python Examples CI |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| echo $K8S_SERVER_ADDRESS |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| |
| - name: Run Python Examples CD |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| |
| - name: Run Go Examples CI |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| |
| - name: Run Go Examples CD |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| |
| - name: Run Java Examples CI |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| - name: Run Java Examples CD |
| run: | |
| K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') |
| export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" |
| python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS |
| working-directory: playground/infrastructure |
| - name: Delete Helm Chart |
| if: always() |
| run: | |
| helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME |