.github/workflows/ephemeral-env.yml - superset - Git at Google

 name: Ephemeral env workflow

 on:
   issue_comment:
     types: [created]

 jobs:
   config:
     runs-on: "ubuntu-22.04"
     if: github.event.issue.pull_request
     outputs:
       has-secrets: ${{ steps.check.outputs.has-secrets }}
     steps:
       - name: "Check for secrets"
         id: check
         shell: bash
         run: |
           if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
             echo "has-secrets=1" >> "$GITHUB_OUTPUT"
           fi

   ephemeral-env-comment:
     concurrency:
       group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
       cancel-in-progress: true
     needs: config
     if: needs.config.outputs.has-secrets
     name: Evaluate ephemeral env comment trigger (/testenv)
     runs-on: ubuntu-22.04
     permissions:
       pull-requests: write
     outputs:
       slash-command: ${{ steps.eval-body.outputs.result }}
       feature-flags: ${{ steps.eval-feature-flags.outputs.result }}

     steps:
     - name: Debug
       run: |
         echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}"

     - name: Eval comment body for /testenv slash command
       uses: actions/github-script@v7
       id: eval-body
       with:
         result-encoding: string
         script: |
           const pattern = /^\/testenv (up|down)/
           const result = pattern.exec(context.payload.comment.body)
           return result === null ? 'noop' : result[1]

     - name: Eval comment body for feature flags
       uses: actions/github-script@v7
       id: eval-feature-flags
       with:
         script: |
           const pattern = /FEATURE_(\w+)=(\w+)/g;
           let results = [];
           [...context.payload.comment.body.matchAll(pattern)].forEach(match => {
             const config = {
               name: `SUPERSET_FEATURE_${match[1]}`,
               value: match[2],
             };
             results.push(config);
           });
           return results;

     - name: Limit to committers
       if: >
         steps.eval-body.outputs.result != 'noop' &&
         github.event.comment.author_association != 'MEMBER' &&
         github.event.comment.author_association != 'OWNER'
       uses: actions/github-script@v7
       with:
         github-token: ${{github.token}}
         script: |
           const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
           github.rest.issues.createComment({
             issue_number: ${{ github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: errMsg
           })
           core.setFailed(errMsg)

   ephemeral-docker-build:
     concurrency:
       group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
       cancel-in-progress: true
     needs: ephemeral-env-comment
     name: ephemeral-docker-build
     runs-on: ubuntu-22.04
     steps:
       - name: Get Info from comment
         uses: actions/github-script@v7
         id: get-pr-info
         with:
           script: |
             const request = {
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 pull_number: ${{ github.event.issue.number }},
             }
             core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
             const pr = await github.rest.pulls.get(request);
             return pr.data;

       - name: Debug
         id: get-sha
         run: |
           echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT

       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )"
         uses: actions/checkout@v4
         with:
           ref: ${{ steps.get-sha.outputs.sha }}
           persist-credentials: false

       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

       - name: Build ephemeral env image
         run: |
           ./scripts/build_docker.py \
             "ci" \
             "pull_request" \
             --build_context_ref ${{ github.event.issue.number }}

       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2

       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2

       - name: Load, tag and push image to ECR
         id: push-image
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: superset-ci
           IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
         run: |
           docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
           docker push -a $ECR_REGISTRY/$ECR_REPOSITORY

   ephemeral-env-up:
     needs: [ephemeral-env-comment, ephemeral-docker-build]
     if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
     name: Spin up an ephemeral environment
     runs-on: ubuntu-22.04
     permissions:
       contents: read
       pull-requests: write

     steps:
     - uses: actions/checkout@v4
       with:
         persist-credentials: false

     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@v4
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         aws-region: us-west-2

     - name: Login to Amazon ECR
       id: login-ecr
       uses: aws-actions/amazon-ecr-login@v2

     - name: Check target image exists in ECR
       id: check-image
       continue-on-error: true
       run: |
         aws ecr describe-images \
         --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
         --repository-name superset-ci \
         --image-ids imageTag=pr-${{ github.event.issue.number }}-ci

     - name: Fail on missing container image
       if: steps.check-image.outcome == 'failure'
       uses: actions/github-script@v7
       with:
         github-token: ${{github.token}}
         script: |
           const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
           github.rest.issues.createComment({
             issue_number: ${{ github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: errMsg
           })
           core.setFailed(errMsg)

     - name: Fill in the new image ID in the Amazon ECS task definition
       id: task-def
       uses: aws-actions/amazon-ecs-render-task-definition@v1
       with:
         task-definition: .github/workflows/ecs-task-definition.json
         container-name: superset-ci
         image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci

     - name: Update env vars in the Amazon ECS task definition
       run: |
         cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}

     - name: Describe ECS service
       id: describe-services
       run: |
         echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
     - name: Create ECS service
       if: steps.describe-services.outputs.active != 'true'
       id: create-service
       env:
         ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
         ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
       run: |
         aws ecs create-service \
         --cluster superset-ci \
         --service-name pr-${{ github.event.issue.number }}-service \
         --task-definition superset-ci \
         --launch-type FARGATE \
         --desired-count 1 \
         --platform-version LATEST \
         --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
         --tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}

     - name: Deploy Amazon ECS task definition
       id: deploy-task
       uses: aws-actions/amazon-ecs-deploy-task-definition@v2
       with:
         task-definition: ${{ steps.task-def.outputs.task-definition }}
         service: pr-${{ github.event.issue.number }}-service
         cluster: superset-ci
         wait-for-service-stability: true
         wait-for-minutes: 10

     - name: List tasks
       id: list-tasks
       run: |
         echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT

     - name: Get network interface
       id: get-eni
       run: |
         echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT

     - name: Get public IP
       id: get-ip
       run: |
         echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT

     - name: Comment (success)
       if: ${{ success() }}
       uses: actions/github-script@v7
       with:
         github-token: ${{github.token}}
         script: |
           github.rest.issues.createComment({
             issue_number: ${{ github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
           })

     - name: Comment (failure)
       if: ${{ failure() }}
       uses: actions/github-script@v7
       with:
         github-token: ${{github.token}}
         script: |
           github.rest.issues.createComment({
             issue_number: ${{ github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
           })
	name: Ephemeral env workflow

	on:
	issue_comment:
	types: [created]

	jobs:
	config:
	runs-on: "ubuntu-22.04"
	if: github.event.issue.pull_request
	outputs:
	has-secrets: ${{ steps.check.outputs.has-secrets }}
	steps:
	- name: "Check for secrets"
	id: check
	shell: bash
	run: \|
	if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') \|\| '' }}" ]; then
	echo "has-secrets=1" >> "$GITHUB_OUTPUT"
	fi

	ephemeral-env-comment:
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.issue.number \|\| github.run_id }}-comment
	cancel-in-progress: true
	needs: config
	if: needs.config.outputs.has-secrets
	name: Evaluate ephemeral env comment trigger (/testenv)
	runs-on: ubuntu-22.04
	permissions:
	pull-requests: write
	outputs:
	slash-command: ${{ steps.eval-body.outputs.result }}
	feature-flags: ${{ steps.eval-feature-flags.outputs.result }}

	steps:
	- name: Debug
	run: \|
	echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}"

	- name: Eval comment body for /testenv slash command
	uses: actions/github-script@v7
	id: eval-body
	with:
	result-encoding: string
	script: \|
	const pattern = /^\/testenv (up\|down)/
	const result = pattern.exec(context.payload.comment.body)
	return result === null ? 'noop' : result[1]

	- name: Eval comment body for feature flags
	uses: actions/github-script@v7
	id: eval-feature-flags
	with:
	script: \|
	const pattern = /FEATURE_(\w+)=(\w+)/g;
	let results = [];
	[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
	const config = {
	name: `SUPERSET_FEATURE_${match[1]}`,
	value: match[2],
	};
	results.push(config);
	});
	return results;

	- name: Limit to committers
	if: >
	steps.eval-body.outputs.result != 'noop' &&
	github.event.comment.author_association != 'MEMBER' &&
	github.event.comment.author_association != 'OWNER'
	uses: actions/github-script@v7
	with:
	github-token: ${{github.token}}
	script: \|
	const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
	github.rest.issues.createComment({
	issue_number: ${{ github.event.issue.number }},
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: errMsg
	})
	core.setFailed(errMsg)

	ephemeral-docker-build:
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.issue.number \|\| github.run_id }}-build
	cancel-in-progress: true
	needs: ephemeral-env-comment
	name: ephemeral-docker-build
	runs-on: ubuntu-22.04
	steps:
	- name: Get Info from comment
	uses: actions/github-script@v7
	id: get-pr-info
	with:
	script: \|
	const request = {
	owner: context.repo.owner,
	repo: context.repo.repo,
	pull_number: ${{ github.event.issue.number }},
	}
	core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
	const pr = await github.rest.pulls.get(request);
	return pr.data;

	- name: Debug
	id: get-sha
	run: \|
	echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT

	- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )"
	uses: actions/checkout@v4
	with:
	ref: ${{ steps.get-sha.outputs.sha }}
	persist-credentials: false

	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build ephemeral env image
	run: \|
	./scripts/build_docker.py \
	"ci" \
	"pull_request" \
	--build_context_ref ${{ github.event.issue.number }}

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: us-west-2

	- name: Login to Amazon ECR
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@v2

	- name: Load, tag and push image to ECR
	id: push-image
	env:
	ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
	ECR_REPOSITORY: superset-ci
	IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
	run: \|
	docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
	docker push -a $ECR_REGISTRY/$ECR_REPOSITORY

	ephemeral-env-up:
	needs: [ephemeral-env-comment, ephemeral-docker-build]
	if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
	name: Spin up an ephemeral environment
	runs-on: ubuntu-22.04
	permissions:
	contents: read
	pull-requests: write

	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: us-west-2

	- name: Login to Amazon ECR
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@v2

	- name: Check target image exists in ECR
	id: check-image
	continue-on-error: true
	run: \|
	aws ecr describe-images \
	--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" \| grep -Eo "^[0-9]+") \
	--repository-name superset-ci \
	--image-ids imageTag=pr-${{ github.event.issue.number }}-ci

	- name: Fail on missing container image
	if: steps.check-image.outcome == 'failure'
	uses: actions/github-script@v7
	with:
	github-token: ${{github.token}}
	script: \|
	const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
	github.rest.issues.createComment({
	issue_number: ${{ github.event.issue.number }},
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: errMsg
	})
	core.setFailed(errMsg)

	- name: Fill in the new image ID in the Amazon ECS task definition
	id: task-def
	uses: aws-actions/amazon-ecs-render-task-definition@v1
	with:
	task-definition: .github/workflows/ecs-task-definition.json
	container-name: superset-ci
	image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci

	- name: Update env vars in the Amazon ECS task definition
	run: \|
	cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}

	- name: Describe ECS service
	id: describe-services
	run: \|
	echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service \| jq '.services[] \| select(.status == "ACTIVE") \| any')" >> $GITHUB_OUTPUT
	- name: Create ECS service
	if: steps.describe-services.outputs.active != 'true'
	id: create-service
	env:
	ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
	ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
	run: \|
	aws ecs create-service \
	--cluster superset-ci \
	--service-name pr-${{ github.event.issue.number }}-service \
	--task-definition superset-ci \
	--launch-type FARGATE \
	--desired-count 1 \
	--platform-version LATEST \
	--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
	--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}

	- name: Deploy Amazon ECS task definition
	id: deploy-task
	uses: aws-actions/amazon-ecs-deploy-task-definition@v2
	with:
	task-definition: ${{ steps.task-def.outputs.task-definition }}
	service: pr-${{ github.event.issue.number }}-service
	cluster: superset-ci
	wait-for-service-stability: true
	wait-for-minutes: 10

	- name: List tasks
	id: list-tasks
	run: \|
	echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service \| jq '.taskArns \| first')" >> $GITHUB_OUTPUT

	- name: Get network interface
	id: get-eni
	run: \|
	echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} \| jq '.tasks \| .[0] \| .attachments \| .[0] \| .details \| map(select(.name=="networkInterfaceId")) \| .[0] \| .value')" >> $GITHUB_OUTPUT

	- name: Get public IP
	id: get-ip
	run: \|
	echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} \| jq -r '.NetworkInterfaces \| first \| .Association.PublicIp')" >> $GITHUB_OUTPUT

	- name: Comment (success)
	if: ${{ success() }}
	uses: actions/github-script@v7
	with:
	github-token: ${{github.token}}
	script: \|
	github.rest.issues.createComment({
	issue_number: ${{ github.event.issue.number }},
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
	})

	- name: Comment (failure)
	if: ${{ failure() }}
	uses: actions/github-script@v7
	with:
	github-token: ${{github.token}}
	script: \|
	github.rest.issues.createComment({
	issue_number: ${{ github.event.issue.number }},
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
	})