[SYSTEMDS-3528] Refine allocation of GitHub Artifacts
This commit change the allocation of the GitHub artifacts to remove the
individual sub tests artifacts once the entire code coverage is
constructed. The PR sets a policy that delete the code coverage on
the main branch after 30 days, PRs 7 days, and forks 3 days.
Closes #1820
diff --git a/.github/workflows/cleanup-transient-artifacts.yml b/.github/workflows/cleanup-transient-artifacts.yml
new file mode 100644
index 0000000..8360f00
--- /dev/null
+++ b/.github/workflows/cleanup-transient-artifacts.yml
@@ -0,0 +1,54 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+name: Cleanup Transient Artifacts
+
+on:
+ workflow_run:
+ workflows: [Java Test]
+ types:
+ - completed
+
+jobs:
+ on-success:
+ name: Delete transient artifacts of successful workflow
+ runs-on: ubuntu-latest
+ permissions:
+ # write permissions required for deleting artifacts
+ actions: write
+ # only run after successful jobs since some tests require retries
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v3
+
+ - name: Delete Artifacts
+ run: |
+ python .github/workflows/delete_artifacts.py \
+ -t ${{ secrets.GITHUB_TOKEN }} \
+ -a "transient_.*" \
+ -r ${{ github.event.workflow_run.id }}
+
+ env:
+ GITHUB_REPOSITORY: ${{ vars.GITHUB_REPOSITORY }}
+
+
+
diff --git a/.github/workflows/delete_artifacts.py b/.github/workflows/delete_artifacts.py
new file mode 100644
index 0000000..2aa42b1
--- /dev/null
+++ b/.github/workflows/delete_artifacts.py
@@ -0,0 +1,160 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+import argparse
+import logging
+import math
+import os
+import requests
+import re
+
+logging.basicConfig(level=logging.INFO)
+
+def list_workflow_artifacts(
+ owner_repo: str,
+ run_id: str,
+ token: str,
+ per_page: int = 30,
+ page: int = 1,
+) -> requests.Response:
+ headers = {
+ "Accept": "application/vnd.github+json",
+ "Authorization": f"Bearer {token}",
+ "X-GitHub-Api-Version": "2022-11-28",
+ }
+
+ url_base = "https://api.github.com"
+ query_params = f"per_page={per_page}&page={page}"
+ url = (
+ f"{url_base}/repos/{owner_repo}/actions/runs/{run_id}/artifacts?{query_params}"
+ )
+
+ return requests.get(url=url, headers=headers)
+
+
+def delete_artifact(
+ owner_repo: str,
+ artifact_id: str,
+ token: str,
+) -> requests.Response:
+ headers = {
+ "Accept": "application/vnd.github+json",
+ "Authorization": f"Bearer {token}",
+ "X-GitHub-Api-Version": "2022-11-28",
+ }
+
+ url_base = "https://api.github.com"
+ url = f"{url_base}/repos/{owner_repo}/actions/artifacts/{artifact_id}"
+
+ logging.info(f"Deleting artifact at url: {url}")
+ return requests.delete(url=url, headers=headers)
+
+
+if __name__ == "__main__":
+ logging.info("Running delete-artifacts.py")
+ parser = argparse.ArgumentParser(description="Deletes Artifacts")
+ parser.add_argument(
+ "-t",
+ "--token",
+ type=str,
+ required=True,
+ help="From github action pass ${{ secrets.GITHUB_TOKEN }}",
+ )
+ parser.add_argument(
+ "-o",
+ "--owner-repo",
+ type=str,
+ default=os.getenv("GITHUB_REPOSITORY", None),
+ help="Defaults to envvar 'GITHUB_REPOSITORY'",
+ )
+ parser.add_argument(
+ "-r",
+ "--run-id",
+ type=str,
+ default=os.getenv("GITHUB_RUN_ID"),
+ help="Defaults to envvar 'GITHUB_RUN_ID'",
+ )
+ parser.add_argument(
+ "-a",
+ "--artifact-name",
+ type=str,
+ default=".*",
+ help="Provide a regex to filter artifacts of a given name expression"
+ )
+
+ args = parser.parse_args()
+ if args.token is None:
+ logging.info(f"--token: is not set! Aborting")
+ exit(1)
+ else:
+ logging.info(f"--token: is set, continue")
+ logging.info(f"--owner-repository: {args.owner_repo}")
+ logging.info(f"--run-id: {args.run_id}")
+ logging.info(f"--artifact-name: {args.artifact_name}")
+
+ page = 1
+ items_per_page = 85
+ resp = list_workflow_artifacts(
+ owner_repo=args.owner_repo,
+ run_id=args.run_id,
+ token=args.token,
+ page=page,
+ per_page=items_per_page,
+ )
+
+ resp_dict = resp.json()
+ artifacts_count = resp_dict.get("total_count")
+ artifact_ids = []
+ for artifact in resp_dict.get("artifacts"):
+ name = artifact.get("name")
+ if re.fullmatch(args.artifact_name, name):
+ logging.info(f"fullmatch: {name} ")
+ artifact_ids.append(artifact.get("id"))
+ else:
+ logging.info(f"no match: {name}")
+
+ logging.info(f"Artifacts count: {len(artifact_ids)} of {artifacts_count}")
+
+ if items_per_page < artifacts_count:
+ pages = math.ceil(artifacts_count / items_per_page)
+ logging.info(f"Pagecount to retrieve: {pages}")
+ for page in range(2, pages + 1):
+ resp = list_workflow_artifacts(
+ owner_repo=args.owner_repo,
+ run_id=args.run_id,
+ token=args.token,
+ page=page, per_page=items_per_page,
+ )
+ for artifact in resp.json().get("artifacts"):
+ name = artifact.get("name")
+ if re.fullmatch(args.artifact_name, name):
+ logging.info(f"fullmatch: {name} ")
+ artifact_ids.append(artifact.get("id"))
+ else:
+ logging.info(f"no match: {name}")
+
+ for artifact_id in artifact_ids:
+ resp = delete_artifact(
+ owner_repo=args.owner_repo,
+ artifact_id=artifact_id,
+ token=args.token,
+ )
+ resp.raise_for_status()
diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml
index 4b21e3c..da6d9ed 100644
--- a/.github/workflows/javaTests.yml
+++ b/.github/workflows/javaTests.yml
@@ -93,7 +93,7 @@
- name: Clean Github Artifact Name of Asterisks
run: |
- ARTIFACT_NAME="jacoco-component-tests"
+ ARTIFACT_NAME="transient_jacoco"
ARTIFACT_NAME+="-${{ matrix.os }}"
ARTIFACT_NAME+="-java-${{ matrix.java }}"
ARTIFACT_NAME+="-${{ matrix.javadist }}"
@@ -140,8 +140,26 @@
- name: Generate Code Coverage Report
run: mvn jacoco:report
- - name: Upload Jacoco Report Artifact
+ - name: Upload Jacoco Report Artifact PR
+ if: (github.repository_owner == 'apache') && (github.ref_name != 'main')
uses: actions/upload-artifact@v3
with:
name: Java Code Coverage (Jacoco)
path: target/site/jacoco
+ retention-days: 7
+
+ - name: Upload Jacoco Report Artifact Main
+ if: (github.repository_owner == 'apache') && (github.ref_name == 'main')
+ uses: actions/upload-artifact@v3
+ with:
+ name: Java Code Coverage (Jacoco)
+ path: target/site/jacoco
+ retention-days: 30
+
+ - name: Upload Jacoco Report Artifact Fork
+ if: (github.repository_owner != 'apache')
+ uses: actions/upload-artifact@v3
+ with:
+ name: Java Code Coverage (Jacoco)
+ path: target/site/jacoco
+ retention-days: 3
