blob: 6746073abeb334bcee7b055823f9b9bb28ffd312 [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Dependency baseline submission
on:
workflow_call:
inputs:
artifact-id:
required: true
type: string
inventory-artifact-id:
required: true
type: string
permissions:
contents: read
jobs:
submit:
if: >-
((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) ||
github.repository == 'hugegraph/hugegraph') &&
github.ref == 'refs/heads/master' &&
(github.event_name == 'push' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: write
actions: read
steps:
- name: Validate and submit snapshot data
env:
SNAPSHOT_TOKEN: ${{ github.token }}
EXPECTED_REPOSITORY: ${{ github.repository }}
EXPECTED_SHA: ${{ github.sha }}
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
ARTIFACT_ID: ${{ inputs.artifact-id }}
INVENTORY_ARTIFACT_ID: ${{ inputs.inventory-artifact-id }}
run: |
python3 -I - <<'PY'
import datetime
import hashlib
import io
import json
import os
import re
import stat
import urllib.error
import urllib.parse
import urllib.request
import zipfile
LIMIT = 25 * 1024 * 1024
def require(condition, message):
if not condition:
raise ValueError(message)
def unique_object(pairs):
result = {}
for key, value in pairs:
require(key not in result, 'Duplicate JSON key')
result[key] = value
return result
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
def github_request(repository, path, data=None):
return urllib.request.Request('https://api.github.com/repos/' + repository + '/' + path,
data=None if data is None else json.dumps(data).encode(),
headers={'Authorization': 'Bearer ' + os.environ['SNAPSHOT_TOKEN'],
'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json',
'X-GitHub-Api-Version': '2022-11-28'})
def bounded_body(response):
raw = response.read(LIMIT + 1)
require(len(raw) <= LIMIT, 'Response too large')
return raw
def api(repository, path, data=None):
request = github_request(repository, path, data)
with urllib.request.build_opener(NoRedirect()).open(request, timeout=60) as response:
require(response.status == (200 if data is None else 201), 'Unexpected API status')
raw = bounded_body(response)
return json.loads(raw, object_pairs_hook=unique_object)
def read_snapshot(raw, member_name='snapshot.json'):
require(0 < len(raw) <= LIMIT, 'Invalid ZIP size')
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
members = archive.infolist()
require(len(members) == 1, 'ZIP must contain one data file')
member = members[0]
require(member.orig_filename == member_name and not member.is_dir() and
not (member.external_attr & 0x10) and
stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and
not (member.flag_bits & 1) and
member.compress_type in [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED] and
0 < member.file_size <= LIMIT, 'Invalid ZIP member')
with archive.open(member) as stream:
payload = bounded_body(stream)
require(len(payload) == member.file_size, 'Incomplete ZIP member')
return json.loads(payload, object_pairs_hook=unique_object)
def download_snapshot(repository, artifact_id, artifact, member_name='snapshot.json'):
opener = urllib.request.build_opener(NoRedirect())
try:
response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60)
except urllib.error.HTTPError as error:
require(error.code == 302, 'Expected artifact redirect')
location = error.headers.get('Location', '')
error.close()
else:
response.close()
raise ValueError('Expected artifact redirect')
target = urllib.parse.urlsplit(location)
require(target.scheme == 'https' and target.hostname and
target.hostname.endswith(('.blob.core.windows.net', '.githubusercontent.com')) and
target.username is None and target.password is None and
target.port in [None, 443] and not target.fragment, 'Invalid artifact redirect')
# A separate request carries no GitHub Authorization to blob storage.
with opener.open(urllib.request.Request(location), timeout=60) as response:
require(response.status == 200, 'Artifact download failed')
raw = bounded_body(response)
require(len(raw) == artifact['size_in_bytes'] and
'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch')
return read_snapshot(raw, member_name)
def timestamp(value):
parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00'))
require(parsed.tzinfo is not None, 'Timestamp lacks timezone')
return parsed
def main():
repository, sha = os.environ['EXPECTED_REPOSITORY'], os.environ['EXPECTED_SHA']
run_id, attempt, artifact_id = (os.environ[key] for key in ['RUN_ID', 'RUN_ATTEMPT', 'ARTIFACT_ID'])
require(repository in ['apache/hugegraph', 'hugegraph/hugegraph'], 'Repository refused')
require(re.fullmatch('[0-9a-f]{40}', sha), 'Invalid commit')
require(all(re.fullmatch('[1-9][0-9]*', x) for x in [run_id, attempt, artifact_id]), 'Invalid ID')
run = api(repository, f'actions/runs/{run_id}/attempts/{attempt}')
require(run['id'] == int(run_id) and run['run_attempt'] == int(attempt) and
run['head_sha'] == sha and run['head_branch'] == 'master' and
run['event'] in ['push', 'workflow_dispatch'] and
run['repository']['full_name'] == repository and
run['path'] in ['.github/workflows/server-ci.yml',
'.github/workflows/server-ci.yml@master'], 'Run identity')
jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100')
require(jobs['total_count'] <= 100, 'Too many jobs')
producers = [job for job in jobs['jobs'] if job['name'] == 'dependency_license / dependency-check']
require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and
producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and
producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit')
inventory_id = os.environ['INVENTORY_ARTIFACT_ID']
require(re.fullmatch('[1-9][0-9]*', inventory_id) and inventory_id != artifact_id, 'Inventory ID')
def checked_artifact(identifier, kind):
item = api(repository, f'actions/artifacts/{identifier}')
require(item['id'] == int(identifier) and item['expired'] is False and
item['name'] == f'dependency-{kind}-{run_id}-{attempt}' and
item['workflow_run']['id'] == int(run_id) and item['workflow_run']['head_sha'] == sha and
item['workflow_run']['head_branch'] == 'master' and
type(item['size_in_bytes']) is int and 0 < item['size_in_bytes'] <= LIMIT and
isinstance(item['digest'], str) and re.fullmatch('sha256:[0-9a-f]{64}', item['digest']),
'Artifact identity')
return item
artifact = checked_artifact(artifact_id, 'snapshot')
inventory_artifact = checked_artifact(inventory_id, 'inventory')
snapshot = download_snapshot(repository, artifact_id, artifact)
require(isinstance(snapshot, dict) and set(snapshot) ==
{'version', 'sha', 'ref', 'job', 'detector', 'metadata', 'scanned', 'manifests'}, 'Snapshot schema')
require(type(snapshot['version']) is int and snapshot['version'] == 0 and
snapshot['sha'] == sha and snapshot['ref'] == 'refs/heads/master', 'Snapshot source')
require(snapshot['job'] == {'id': run_id,
'correlator': '.github/workflows/submit-dependencies.yml:submit',
'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'}, 'Snapshot job')
require(snapshot['detector'] == {'name': 'maven-dependency-submission-action', 'version': '6.0.1',
'url': 'https://github.com/advanced-security/maven-dependency-submission-action'}, 'Detector')
require(snapshot['metadata'] == {'serializer':
'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1',
'producer': 'generate', 'run_attempt': attempt}, 'Producer identity')
now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=1)
require(timestamp(run['run_started_at']) <= timestamp(snapshot['scanned']) <=
timestamp(artifact['created_at']) + datetime.timedelta(minutes=1) and
timestamp(artifact['created_at']) <= now and timestamp(snapshot['scanned']) <= now,
'Snapshot freshness')
inventory = download_snapshot(repository, inventory_id, inventory_artifact, 'inventory.json')
require(isinstance(inventory, dict) and set(inventory) ==
{'version', 'sha', 'repository', 'run_id', 'run_attempt', 'producer', 'resolver',
'scanned', 'manifests'}, 'Inventory schema')
require(type(inventory['version']) is int and inventory['version'] == 1 and
inventory['sha'] == sha and inventory['repository'] == repository and
inventory['run_id'] == run_id and inventory['run_attempt'] == attempt and
inventory['producer'] == 'inventory' and inventory['resolver'] ==
'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree', 'Inventory source')
require(timestamp(run['run_started_at']) <= timestamp(inventory['scanned']) <=
timestamp(inventory_artifact['created_at']) + datetime.timedelta(minutes=1) and
timestamp(inventory_artifact['created_at']) <= now and timestamp(inventory['scanned']) <= now,
'Inventory freshness')
tree = api(repository, f'git/trees/{sha}?recursive=1')
require(tree['truncated'] is False, 'Incomplete source tree')
poms = {node['path'] for node in tree['tree'] if node['type'] == 'blob' and
node['path'].split('/')[-1] == 'pom.xml' and node['mode'] != '120000'}
require(isinstance(inventory['manifests'], dict) and set(inventory['manifests']) == poms,
'Incomplete independent inventory')
manifests, paths = snapshot['manifests'], set()
require(isinstance(manifests, dict) and manifests and poms, 'Empty inventory')
component = r"(?:[A-Za-z0-9._!~*'():+-]|%[0-9A-F]{2})+"
qualifier = r'(?:[A-Za-z0-9._*\-]|%[0-9A-F]{2})+'
purl_pattern = (r'pkg:maven/' + component + '(?:/' + component + ')+@' + component +
r'(?:\?classifier=' + qualifier + '(?:&type=' + qualifier + r')?|\?type=' + qualifier + ')?')
for name, manifest in manifests.items():
require(isinstance(name, str) and name.strip() and isinstance(manifest, dict) and
set(manifest) == {'name', 'file', 'resolved'} and manifest['name'] == name and
isinstance(manifest['file'], dict) and set(manifest['file']) == {'source_location'},
'Manifest schema')
location = manifest['file']['source_location']
require(isinstance(location, str) and location in poms and location not in paths, 'Manifest source')
paths.add(location)
resolved = manifest['resolved']
require(isinstance(resolved, dict), 'Resolved schema')
for purl, package in resolved.items():
require(isinstance(purl, str) and len(purl) <= 4096 and
re.fullmatch(purl_pattern, purl), 'Package URL')
require(isinstance(package, dict) and set(package) ==
{'package_url', 'relationship', 'scope', 'dependencies'} and
package['package_url'] == purl and package['relationship'] in ['direct', 'indirect'] and
package['scope'] in ['runtime', 'development'] and
isinstance(package['dependencies'], list) and
all(isinstance(child, str) and child in resolved for child in package['dependencies']),
'Package schema')
require(not resolved or any(p['relationship'] == 'direct' for p in resolved.values()),
'Missing direct dependencies')
observed = {purl: {'scope': package['scope'], 'relationship': package['relationship']}
for purl, package in resolved.items()}
require(observed == inventory['manifests'][location], 'Dependency inventory mismatch')
require(paths == poms and any(m['resolved'] for m in manifests.values()), 'Incomplete inventory')
require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Stale master before submission')
result = api(repository, 'dependency-graph/snapshots', snapshot)
require(result['result'] == 'SUCCESS', 'Snapshot not accepted')
require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Master advanced during submission')
print('Dependency baseline submitted for ' + sha)
if __name__ == '__main__':
main()
PY