.test-infra/dataproc/cleanup.sh

#!/usr/bin/env bash
#    Licensed to the Apache Software Foundation (ASF) under one or more
#    contributor license agreements.  See the NOTICE file distributed with
#    this work for additional information regarding copyright ownership.
#    The ASF licenses this file to You under the Apache License, Version 2.0
#    (the "License"); you may not use this file except in compliance with
#    the License.  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

set -Eeuo pipefail
GCLOUD_REGION="us-central1"
clustersList=( )
toDeleteList=( )


generatedResources=("beam-loadtests-go-*-flink" "beam-loadtests-python-*-flink" \
"beam-loadtests-py-*-flink" "beam-postcommit-python-chicago" \
"beam-precommit-flink-container" )

function deleteFilteredClusters(){
    for cluster in ${toDeleteList[@]};do
		echo "Deleting cluster: $cluster"
        gcloud dataproc clusters delete $cluster --region=$GCLOUD_REGION --quiet
    done

    if  [ ${#toDeleteList[@]} -eq 0 ]; then
		echo "No leaked resources were found"
    fi
}

function filterClusters(){
    echo "Searching for leaked dataproc resources ... "
    local currentDate=$(date +%s)
    #To avoid the deletion of active clusters, a date comparison is required
    for cluster in ${clustersList[@]}; do
        clusterStartTime=$(gcloud dataproc clusters describe $cluster --region=$GCLOUD_REGION --format="get(status.stateStartTime)" )
		clusterStartTime=$(date -d$clusterStartTime +%s)

		elapsedHours=$((($currentDate - $clusterStartTime)/3600))

		# teardown clusters >= 6 hr old
		if [[ $elapsedHours -ge 6 ]]; then
			for name in ${generatedResources[@]}; do
				# Only resources generated by the groovy jobs set are queued for deletion
				if [[ "$cluster" == *${name}* && ! ("$cluster" =~ nokill) ]]; then
					toDeleteList+=( "$cluster" )
					break
				fi
			done
		fi
    done
}




clustersList=( $( gcloud dataproc clusters list --region=$GCLOUD_REGION --format="get(NAME)" ) )
filterClusters
deleteFilteredClusters