blob: 8642815c5396d840ae1c18520ed7c00a609efda4 [file]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
module.exports = async ({github, context, core, env = process.env, log = console}) => {
// Open PRs are read with one GraphQL query per page of PRs. For each PR it returns
// the Build checks on the head commit (statusCheckRollup.commit) and the fork's
// workflow runs for that commit (headRef.target), both as GitHub Actions check
// suites: fork runs are Actions runs, and Build checks written with the GITHUB_TOKEN
// land in Actions suites too. On the PR side, only suites containing Build checks
// are relevant; unrelated upstream workflow runs must not fill the page.
//
// A PR rarely has more than a few such suites. A fork's default branch can collect
// thousands from scheduled workflows, but suites are listed oldest first and a push
// creates its runs' suites right away, so they are still among the first. Should a
// page ever not hold everything a PR needs, the PR fails loudly rather than being
// synced from partial data.
//
// GitHub aborts a GraphQL request after 10 seconds and allows 60 seconds of GraphQL
// CPU time per minute. A page of 25 PRs takes about 3 seconds, so each page is
// followed by a pause as long as it took.
// https://docs.github.com/en/graphql/overview/rate-limits-and-query-limits-for-the-graphql-api
const GITHUB_ACTIONS_APP_ID = 15368
const PR_PAGE_SIZE = 25
// Name of the mirrored PR check, not the workflow.
const BUILD_CHECK_NAME = 'Build'
// Quick syncs cover only open PRs updated in the last QUICK_SYNC_LOOKBACK_DAYS. A
// push, comment, review, or label change updates a PR, but a check change does not.
// Fork runs usually finish within hours of a push, and most reruns happen within
// days. A later rerun, or a gap in quick syncs longer than the lookback, is picked
// up by the next full sync, which covers all open PRs. Both schedules are defined
// in update_build_status.yml.
const QUICK_SYNC_LOOKBACK_DAYS = 3
const fullSync = env.FULL_SYNC == 'true'
const updatedSince = fullSync ? 0
: Date.now() - QUICK_SYNC_LOOKBACK_DAYS * 24 * 60 * 60 * 1000
// A run-linked Build check points at the fork's build_main.yml run for the PR branch.
// Right after a push that run may not be registered yet, so a PR updated in the last
// NEW_RUN_GRACE_MINUTES without a usable run is left for a later pass instead of
// receiving action_required guidance.
const BUILD_WORKFLOW = '.github/workflows/build_main.yml'
const NEW_RUN_GRACE_MINUTES = 5
const runGraceSince = Date.now() - NEW_RUN_GRACE_MINUTES * 60 * 1000
const prPageQuery = `
query($owner: String!, $repo: String!, $size: Int!, $cursor: String, $appId: Int!) {
repository(owner: $owner, name: $repo) {
pullRequests(states: OPEN, first: $size, after: $cursor,
orderBy: {field: UPDATED_AT, direction: DESC}) {
pageInfo { hasNextPage endCursor }
nodes {
number
updatedAt
headRefOid
statusCheckRollup {
commit {
oid
checkSuites(first: 10, filterBy: {
appId: $appId, checkName: "${BUILD_CHECK_NAME}"
}) {
pageInfo { hasNextPage }
nodes {
checkRuns(first: 10,
filterBy: {
checkName: "${BUILD_CHECK_NAME}", checkType: LATEST
}) {
pageInfo { hasNextPage }
nodes { databaseId status conclusion text }
}
}
}
}
}
headRef {
name
repository { name owner { login } }
target {
oid
... on Commit {
checkSuites(first: 10, filterBy: {appId: $appId}) {
pageInfo { hasNextPage }
nodes {
status
conclusion
branch { name }
workflowRun { databaseId file { path } }
}
}
}
}
}
}
}
}
}`
// Yield the open PRs updated since updatedSince one page at a time, most recently
// updated first, pausing after each page as long as it took (see above). A PR
// updated during the pass moves to the front and may be missed, but it is then
// recent enough for the next pass.
async function* listOpenPRs() {
let cursor = null
for (;;) {
const started = Date.now()
const data = await github.graphql(prPageQuery, {
...context.repo,
size: PR_PAGE_SIZE,
cursor,
appId: GITHUB_ACTIONS_APP_ID
})
const took = Date.now() - started
const page = data.repository.pullRequests
const recent = page.nodes.filter(node =>
Date.parse(node.updatedAt) >= updatedSince)
yield recent
if (recent.length < page.nodes.length || !page.pageInfo.hasNextPage) {
return
}
cursor = page.pageInfo.endCursor
await new Promise(resolve => setTimeout(resolve, took))
}
}
// Flatten a PR from a page into its number, updatedAt, head sha, fork owner, repo and
// branch, and:
// - buildChecks: the head commit's Build checks.
// - forkRuns: the fork's workflow runs for the head commit, by run ID, and
// forkRunsTruncated: whether the page held only some of them.
// - buildRun: the fork's newest build_main.yml run for the PR branch, if any. The
// branch matters because the same commit can have runs from pushes to different
// branches, including a skipped master run and a build on the PR branch.
// Statuses and conclusions are lowercased to match the check-runs REST API.
// Return null if the fork branch is gone or not at the PR head commit (the branch
// was just pushed to and the PR has yet to follow). Throw if the page does not hold
// all of the PR's Build checks.
const toPR = (node) => {
const target = node.headRef?.target
const commit = node.statusCheckRollup?.commit
if (target?.oid != node.headRefOid || (commit && commit.oid != node.headRefOid)) {
return null
}
// A commit without any checks or statuses has no rollup.
const suites = commit?.checkSuites.nodes ?? []
if (commit?.checkSuites.pageInfo.hasNextPage
|| suites.some(s => s.checkRuns.pageInfo.hasNextPage)) {
throw new Error('The PR head commit has more check suites or Build checks '
+ 'than a page holds')
}
// checkName filtering can repeat suites; process each Build check only once.
const checks = new Map(suites.flatMap(s => s.checkRuns.nodes).map(cr => [cr.databaseId, cr]))
const lower = (value) => value?.toLowerCase()
const runs = target.checkSuites.nodes.filter(s => s.workflowRun).map(s => ({
id: s.workflowRun.databaseId,
status: lower(s.status),
conclusion: lower(s.conclusion),
workflow: s.workflowRun.file?.path,
branch: s.branch?.name
}))
return {
number: node.number,
updatedAt: node.updatedAt,
sha: node.headRefOid,
forkOwner: node.headRef.repository.owner.login,
forkRepo: node.headRef.repository.name,
branch: node.headRef.name,
buildChecks: [...checks.values()].map(cr => ({
id: cr.databaseId,
status: lower(cr.status),
conclusion: lower(cr.conclusion),
text: cr.text
})),
forkRuns: new Map(runs.map(run => [String(run.id), run])),
forkRunsTruncated: target.checkSuites.pageInfo.hasNextPage,
buildRun: runs.findLast(run => run.workflow == BUILD_WORKFLOW
&& run.branch == node.headRef.name)
}
}
// The fork run ID a Build check points at, stored as JSON in its output text, or
// undefined if the text is not that JSON (e.g. a check from an older version or a
// manual run).
const runIdOf = (cr) => {
try {
return JSON.parse(cr.text).run_id
} catch {
return undefined
}
}
// The Build check state that shows a fork run's state. The check-runs API accepts
// fewer values than a run reports: the other pending statuses (requested, waiting,
// pending) become queued, and the other conclusions (startup_failure, stale, or
// none), which all mean the run produced no test result, become failure.
const checkState = (run) => {
if (run.status != 'completed') {
return {status: run.status == 'in_progress' ? 'in_progress' : 'queued'}
}
const conclusions = ['success', 'failure', 'neutral', 'cancelled', 'timed_out',
'action_required', 'skipped']
return {
status: 'completed',
conclusion: conclusions.includes(run.conclusion) ? run.conclusion : 'failure'
}
}
const label = (state) => state.status == 'completed' ? state.conclusion : state.status
const sameState = (a, b) => a.status == b.status
&& (a.status != 'completed' || a.conclusion == b.conclusion)
// Record PR-specific failures and fail the step after syncing the rest.
// A failed page query instead aborts the pass.
const failedPRs = new Set()
// The log lists only the Build checks a pass creates or updates, and errors, each
// with its PR and head commit. Everything else is counted for a summary at the end.
const count = {prs: 0, skipped: 0, updated: 0, created: 0, unchanged: 0, notApplied: 0}
const prUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/`
log.log(fullSync ? 'Full sync: all open PRs'
: 'Quick sync: open PRs updated in the last ' + QUICK_SYNC_LOOKBACK_DAYS + ' days')
for await (const prs of listOpenPRs()) {
for (const node of prs) {
count.prs++
const ref = prUrl + node.number + ' ' + node.headRefOid.slice(0, 7)
let pr
try {
pr = toPR(node)
} catch (error) {
log.error(ref + ': ' + error.message + '; raise the limits in prPageQuery')
failedPRs.add(node.number)
continue
}
if (!pr) {
count.skipped++
continue
}
// Sync every Build check that points at a fork run, not just the first: a commit
// can carry more than one (e.g. left by the notify workflow this one replaced).
// An action_required guidance check is not linked to a run.
let syncableBuildCheck = false
for (const cr of pr.buildChecks) {
if (cr.conclusion == 'action_required') {
continue
}
const runId = runIdOf(cr)
const run = pr.forkRuns.get(String(runId))
if (!run) {
if (runId && pr.forkRunsTruncated) {
// The run may be among the fork's check suites the page left out. Keep
// the backfill from replacing the check, and fail the PR to get noticed.
log.error(ref + ': fork run ' + runId + ' not in the first fork check '
+ 'suites; raise the limits in prPageQuery')
failedPRs.add(pr.number)
syncableBuildCheck = true
}
// Otherwise the check's output is unusable or its fork run was deleted, so
// it can never be synced; the backfill below replaces it.
continue
}
syncableBuildCheck = true
// Leave a check alone if it already shows the run's state, as nearly all do.
const state = checkState(run)
if (sameState(cr, state)) {
count.unchanged++
continue
}
const change = label(cr) + ' -> ' + label(state)
try {
const {data: updated} = await github.rest.checks.update({
...context.repo,
check_run_id: cr.id,
...state
})
// A successful request can leave the state unchanged, notably when trying
// to reopen a completed check after a rerun.
if (sameState(cr, updated)) {
count.unchanged++
} else {
log.log(ref + ': ' + label(cr) + ' -> ' + label(updated))
count.updated++
}
if (!sameState(updated, state)) {
count.notApplied++
}
} catch (error) {
log.error(ref + ': ' + change + ' failed: ' + error.message)
failedPRs.add(pr.number)
}
}
// No syncable Build check: link one to the PR branch's build_main.yml run, or
// create action_required guidance if no run exists, it was skipped, or the PR
// is from fork master. Fork master is reserved for syncing upstream; new PR
// builds must use other branches, even if an older workflow ran on master.
// Existing run-linked checks are still synchronized above.
if (!syncableBuildCheck) {
const run = pr.branch == 'master' || pr.buildRun?.conclusion == 'skipped'
? undefined : pr.buildRun
if (!run) {
// Already provided guidance, or still within the run-registration grace period.
if (pr.buildChecks.some(cr => cr.conclusion == 'action_required')
|| Date.parse(pr.updatedAt) > runGraceSince) {
count.skipped++
continue
}
}
const state = run ? checkState(run)
: {status: 'completed', conclusion: 'action_required'}
const check = {
...context.repo, name: BUILD_CHECK_NAME, head_sha: pr.sha, ...state
}
const change = 'none -> ' + label(state)
try {
if (run) {
const actions_url = 'https://github.com/' + pr.forkOwner + '/' + pr.forkRepo
+ '/actions/runs/' + run.id
check.output = {
title: 'Test results',
summary: '[See test results](' + actions_url + ')\n\n'
+ 'If the tests fail for reasons unrelated to this pull request, '
+ 'please rerun the workflow in your forked repository.\n'
+ 'If the failures are related to this pull request, '
+ 'please investigate them and push follow-up changes.',
text: JSON.stringify({
owner: pr.forkOwner,
repo: pr.forkRepo,
run_id: run.id
})
}
check.details_url = actions_url
} else {
// No usable run, or a fork-master PR: explain the required setup.
check.output = {
title: 'PR build setup requires attention',
summary: `
A supported Build check is not available for this PR head commit.
1. If you did not enable GitHub Actions in your forked repository, please enable it by clicking the button as shown in the image below. See also [Managing GitHub Actions settings for a repository](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository) for more details.
2. For PR build status, use a branch other than \`master\` in your fork.`
+ (pr.branch == 'master' ? '' : `
After fixing the cause, push a new commit to your PR branch to start the build.`),
images: [
{
alt: 'enabling workflows button',
image_url: 'https://raw.githubusercontent.com/apache/spark/master/.github/workflows/images/workflow-enable-button.png'
}
]
}
}
await github.rest.checks.create(check)
log.log(ref + ': ' + change)
count.created++
} catch (error) {
log.error(ref + ': ' + change + ' failed: ' + error.message)
failedPRs.add(pr.number)
}
}
}
}
log.log(count.prs + ' PRs checked, ' + count.skipped + ' skipped. '
+ 'Build checks: ' + count.updated + ' updated, ' + count.created + ' created, '
+ count.unchanged + ' unchanged.')
if (count.notApplied > 0) {
core.warning(count.notApplied + ' check update(s) did not reach the requested '
+ 'state. GitHub cannot reopen completed checks.')
}
if (failedPRs.size > 0) {
core.setFailed('Failed to sync ' + failedPRs.size + ' PR(s), will retry next pass: #'
+ [...failedPRs].join(', #'))
}
};