agents: retry GitHub Actions jobs and handle setup-bazel network flakes (#4320) GitHub Actions jobs occasionally fail during `setup-bazel` with `Unable to find Bazelisk version` due to transient network errors. Previously, `buildkite-retry-job` only retried Buildkite jobs, and early matrix failures could not be rerun via `gh run rerun` until sibling jobs in the workflow run finished. Extend the CI monitoring, failure analysis, and retry skills to classify `setup-bazel` Bazelisk lookup failures as infrastructure flakes, fetch job logs via the GitHub API when runs are still in progress, and wait for parent GitHub Actions runs to complete before rerunning failed jobs.
diff --git a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py index 31d8dd7..88df0f5 100755 --- a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py +++ b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py
@@ -39,19 +39,45 @@ log_url = f"https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/{build_id}/jobs/{job_id}/download.txt" # Check if this is a GitHub Actions job - gh_match = re.search(r"github\.com/.*/job/(\d+)", log_url) + gh_repo_match = re.search( + r"github\.com/([^/]+/[^/]+)/actions/runs/\d+/job/(\d+)", log_url + ) + gh_match = gh_repo_match or re.search(r"github\.com/.*/job/(\d+)", log_url) if not gh_match and "github" in job_name.lower() and re.match(r"^\d+$", job_id): gh_match = re.match(r"^(\d+)$", job_id) if gh_match: - gh_job_id = gh_match.group(1) + if gh_repo_match: + gh_repo = gh_repo_match.group(1) + gh_job_id = gh_repo_match.group(2) + else: + gh_repo = "bazel-contrib/rules_python" + gh_job_id = gh_match.group(1) print(f"đĨ Fetching GitHub Action log for job {gh_job_id} using gh CLI...") - cmd = ["gh", "run", "view", "--job", gh_job_id, "--log"] + api_cmd = ["gh", "api", f"repos/{gh_repo}/actions/jobs/{gh_job_id}/logs"] + try: + res = subprocess.run(api_cmd, capture_output=True, text=True, check=True) + if res.stdout.strip(): + with open(output_path, "w") as f: + f.write(res.stdout) + return True + except Exception as e: + print( + f"â ī¸ Failed to fetch GitHub log via gh api for job {gh_job_id}: {e}", + file=sys.stderr, + ) + + cmd = ["gh", "run", "view", "--job", gh_job_id, "--repo", gh_repo, "--log"] try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) - with open(output_path, "w") as f: - f.write(res.stdout) - return True + if ( + res.stdout.strip() + and "is still in progress; logs will be available when it is complete" + not in res.stdout + ): + with open(output_path, "w") as f: + f.write(res.stdout) + return True except Exception as e: print( f"â ī¸ Failed to fetch GitHub log via gh CLI for job {gh_job_id}: {e}", @@ -118,6 +144,7 @@ "connection reset by peer", "input/output error", "errno 5", + "unable to find bazelisk version", ] ): if clean_line: @@ -168,6 +195,13 @@ ): is_flake = True flake_reason = "Transient runner host disk / Darwin sandbox I/O error (OSError: [Errno 5] Input/output error). This is an infrastructure flake, not a codebase defect." + elif any("unable to find bazelisk version" in e.lower() for e in errors): + is_flake = True + flake_reason = ( + "Transient network error fetching Bazelisk release metadata during " + "GitHub Actions `setup-bazel` step (`Unable to find Bazelisk version`). " + "This is an infrastructure network flake, not a codebase bug." + ) classification = ( "⥠**Classification**: **Infrastructure / Flake Issue** (Not a codebase bug)" @@ -215,7 +249,8 @@ os.makedirs(scratch_dir, exist_ok=True) safe_jname = re.sub(r"[^a-zA-Z0-9]", "_", args.job_name) - log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{args.job_id}.log") + safe_jid = re.sub(r"[^a-zA-Z0-9_-]", "_", args.job_id) + log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{safe_jid}.log") fetch_log(args.job_name, args.build_id, args.job_id, log_path)
diff --git a/.agents/skills/buildkite-retry-job/SKILL.md b/.agents/skills/buildkite-retry-job/SKILL.md index d01707c..7acbc62 100644 --- a/.agents/skills/buildkite-retry-job/SKILL.md +++ b/.agents/skills/buildkite-retry-job/SKILL.md
@@ -1,21 +1,32 @@ --- name: buildkite-retry-job -description: Retry failed Buildkite jobs, unblock manual approval steps, or - rebuild full builds +description: Retry failed Buildkite or GitHub Actions jobs, unblock manual + approval steps, or rebuild full builds --- -Use `scripts/retry_buildkite_jobs.py` to retry individual jobs, unblock manual -approval steps (such as `.bazelci/presubmit.yml` changes), or rebuild full -Buildkite builds. +Use `scripts/retry_buildkite_jobs.py` to retry individual Buildkite or GitHub +Actions jobs, unblock manual approval steps (such as `.bazelci/presubmit.yml` +changes), or rebuild full builds. + +When a GitHub Actions job fails early (for example, `Setup Bazel` failing with +`Error: Unable to find Bazelisk version ...` due to a transient network error) +while sibling matrix jobs in the same workflow run are still running, GitHub +rejects rerun requests until the parent workflow run completes. The script +automatically waits for an in-progress workflow run to finish before triggering +`gh run rerun`. ### Retrying & Unblocking Modes -1. **Default Retry & Unblock**: - Unblocks any blocked manual approval steps (`bk job unblock <job_id> - --no-input`) and retries specific failing jobs (`bk job retry <job_id>`): +1. **Default Retry & Unblock (by PR, Buildkite URL, or GitHub Actions URL)**: + Unblocks any blocked Buildkite manual approval steps (`bk job unblock + <job_id> --no-input`), retries specific failing Buildkite jobs (`bk job + retry <job_id>`), and reruns failing GitHub Actions jobs/runs (`gh run + rerun`): ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ <pr_number_or_url> + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + "https://github.com/<owner>/<repo>/actions/runs/<run_id>/job/<job_id>" ``` 2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**: @@ -27,16 +38,21 @@ ``` 3. **Retry Specific Jobs by Name or ID**: + Retries matching Buildkite or GitHub Actions jobs by name/regex pattern, + Buildkite job UUID, or numeric GitHub Actions job ID: ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ <pr_number> --jobs "compile_pip_requirements" ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --job-id <job_uuid> + <pr_number> --jobs "Build linux" + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --job-id <job_uuid_or_github_job_id> ``` 4. **Rebuild Full Build**: - Re-runs the entire pipeline build from scratch via `bk build rebuild`: + Re-runs the entire pipeline/workflow build from scratch via `bk build + rebuild` or `gh run rerun`: ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - <pr_number> --rebuild + <pr_number_or_url> --rebuild ```
diff --git a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py index 97e5fb5..168b1da 100755 --- a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py +++ b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py
@@ -5,6 +5,12 @@ import re import subprocess import sys +import time + +DEFAULT_GH_REPO = "bazel-contrib/rules_python" +GH_ACTIONS_URL_RE = re.compile( + r"github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/job/(\d+))?" +) def check_cli(cmd_name, install_url): @@ -24,21 +30,194 @@ sys.exit(1) -def get_build_url_from_pr(pr_number): +def get_pr_checks(pr_number): check_cli("gh", "https://cli.github.com/") - cmd = ["gh", "pr", "checks", str(pr_number), "--json", "name,link"] + cmd = ["gh", "pr", "checks", str(pr_number), "--json", "name,link,state,bucket"] + res = subprocess.run(cmd, capture_output=True, text=True) + out = res.stdout + if "[" not in out: + print( + f"â Error fetching PR checks for PR #{pr_number}: {res.stderr}", + file=sys.stderr, + ) + sys.exit(1) + json_str = out[out.find("[") : out.rfind("]") + 1] + return json.loads(json_str) + + +def parse_github_actions_url(target): + m = GH_ACTIONS_URL_RE.search(target) + if m: + return m.group(1), m.group(2), m.group(3) + return None + + +def get_github_run_data(repo, run_id): + cmd = [ + "gh", + "run", + "view", + str(run_id), + "--repo", + repo, + "--json", + "status,conclusion,jobs", + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + out = res.stdout + json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" + return json.loads(json_str) + + +def wait_for_github_run_completion( + repo, run_id, job_id=None, poll_interval=15, max_attempts=60 +): + for attempt in range(max_attempts): + try: + data = get_github_run_data(repo, str(run_id)) + except Exception as e: + print( + f"â ī¸ Warning: Could not inspect GitHub Actions run {run_id}: {e}", + file=sys.stderr, + ) + return None + + status = (data.get("status") or "").lower() + jobs = data.get("jobs", []) + + if job_id is not None: + active_job_ids = {str(j.get("databaseId")) for j in jobs} + if str(job_id) not in active_job_ids: + # The job ID from an earlier attempt is no longer in the active jobs + # list, meaning a newer run_attempt is already queued/running/done. + if status != "completed" or data.get("conclusion") == "success": + print( + f"âšī¸ GitHub Actions job {job_id} has already been superseded " + f"by a newer attempt in run {run_id} (status: {status})." + ) + return data + + if status == "completed": + return data + + print( + f"âŗ GitHub Actions run {run_id} is currently '{status}' " + f"(attempt {attempt + 1}/{max_attempts}); waiting {poll_interval}s " + f"for run to complete before rerunning..." + ) + time.sleep(poll_interval) + + print( + f"â ī¸ Timed out waiting for GitHub Actions run {run_id} to complete.", + file=sys.stderr, + ) + return None + + +def retry_github_actions(repo, run_id, job_id=None, job_pattern=None, rebuild=False): + check_cli("gh", "https://cli.github.com/") + data = wait_for_github_run_completion(repo, run_id, job_id=job_id) + jobs = data.get("jobs", []) if isinstance(data, dict) else [] + + if rebuild: + print(f"đ Re-running entire GitHub Actions run {run_id} ({repo})...") + res = subprocess.run(["gh", "run", "rerun", str(run_id), "--repo", repo]) + return res.returncode + + if job_pattern: + matched = [] + for j in jobs: + name = j.get("name", "") + jid = j.get("databaseId") + conclusion = j.get("conclusion") or j.get("status") or "" + if jid and ( + re.search(job_pattern, name, re.IGNORECASE) + or job_pattern.lower() in name.lower() + ): + matched.append((str(jid), name, conclusion)) + + if not matched: + print( + f"â ī¸ No GitHub Actions jobs found matching '{job_pattern}' in run {run_id}.", + file=sys.stderr, + ) + return 1 + + rc = 0 + for jid, name, conclusion in matched: + print( + f"đ Retrying GitHub Actions job '{name}' " + f"(ID: {jid}, conclusion: {conclusion})..." + ) + res = subprocess.run(["gh", "run", "rerun", "--job", jid, "--repo", repo]) + if res.returncode != 0: + print( + f"â ī¸ Single-job rerun failed for {jid}; falling back to " + f"'gh run rerun {run_id} --failed'..." + ) + res = subprocess.run( + ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] + ) + return res.returncode + return rc + + if job_id: + active_jobs_by_id = {str(j.get("databaseId")): j for j in jobs} + if data is not None and str(job_id) not in active_jobs_by_id: + status = (data.get("status") or "").lower() + conclusion = (data.get("conclusion") or "").lower() + if status != "completed" or conclusion == "success": + return 0 + + target_job = active_jobs_by_id.get(str(job_id)) + if target_job and (target_job.get("conclusion") or "").lower() == "success": + print( + f"âšī¸ GitHub Actions job '{target_job.get('name')}' " + f"(ID: {job_id}) already succeeded." + ) + return 0 + + print(f"đ Retrying GitHub Actions job {job_id} in run {run_id} ({repo})...") + res = subprocess.run( + ["gh", "run", "rerun", "--job", str(job_id), "--repo", repo] + ) + if res.returncode == 0: + return 0 + print( + f"â ī¸ Single-job rerun for {job_id} exited with {res.returncode}; " + f"falling back to rerunning failed jobs in run {run_id}..." + ) + + if data is not None and (data.get("conclusion") or "").lower() == "success": + print(f"âšī¸ GitHub Actions run {run_id} already succeeded.") + return 0 + + print(f"đ Retrying failed jobs in GitHub Actions run {run_id} ({repo})...") + res = subprocess.run( + ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] + ) + return res.returncode + + +def retry_github_job_by_id(job_id, repo=DEFAULT_GH_REPO): + check_cli("gh", "https://cli.github.com/") + cmd = ["gh", "api", f"repos/{repo}/actions/jobs/{job_id}"] try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) - checks = json.loads(res.stdout) - for c in checks: - link = c.get("link", "") - if "buildkite.com" in link: - return link.split("#")[0] - print(f"â No Buildkite checks found for PR #{pr_number}.", file=sys.stderr) - sys.exit(1) - except subprocess.CalledProcessError as e: - print(f"â Error fetching PR checks: {e.stderr}", file=sys.stderr) - sys.exit(1) + out = res.stdout + json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" + job_data = json.loads(json_str) + run_id = job_data.get("run_id") + if run_id: + return retry_github_actions(repo, str(run_id), job_id=str(job_id)) + except Exception as e: + print( + f"â ī¸ Could not resolve run_id for GitHub job {job_id} via API: {e}", + file=sys.stderr, + ) + print(f"đ Retrying GitHub Actions job ID: {job_id}...") + res = subprocess.run(["gh", "run", "rerun", "--job", str(job_id), "--repo", repo]) + return res.returncode def normalize_build_target(target): @@ -57,7 +236,7 @@ def retry_job_by_uuid(job_id): check_cli("bk", "https://github.com/buildkite/cli") - print(f"đ Retrying individual job UUID: {job_id}") + print(f"đ Retrying individual Buildkite job UUID: {job_id}") return subprocess.run(["bk", "job", "retry", job_id]) @@ -75,7 +254,7 @@ return unblocked -def retry_jobs_by_name(pipeline, build_number, job_pattern): +def retry_jobs_by_name(pipeline, build_number, job_pattern, exit_on_missing=True): check_cli("bk", "https://github.com/buildkite/cli") cmd = ["bk", "build", "view", str(build_number), "-p", pipeline, "--json"] try: @@ -94,78 +273,37 @@ matched.append((jid, name, state)) if not matched: - print( - f"â ī¸ No jobs found matching pattern '{job_pattern}' in build #{build_number}", - file=sys.stderr, - ) - sys.exit(1) + if exit_on_missing: + print( + f"â ī¸ No jobs found matching pattern '{job_pattern}' in build #{build_number}", + file=sys.stderr, + ) + sys.exit(1) + return False for jid, name, state in matched: print(f"đ Retrying job '{name}' (ID: {jid}, status: {state})...") subprocess.run(["bk", "job", "retry", jid]) + return True except Exception as e: print(f"â Error fetching build jobs: {e}", file=sys.stderr) - sys.exit(1) + if exit_on_missing: + sys.exit(1) + return False -def main(): - parser = argparse.ArgumentParser( - description="Retry Buildkite jobs individually using 'bk job retry' or rebuild the whole build using 'bk build rebuild'." - ) - parser.add_argument( - "args", - nargs="+", - help="Target build (PR#, Buildkite URL, or org/pipeline/build)", - ) - parser.add_argument( - "--jobs", - "--job-name", - dest="job_name", - help="Specific job name or regex pattern to retry individually via 'bk job retry'", - ) - parser.add_argument( - "--job-id", - dest="job_id", - help="Specific job UUID to retry individually via 'bk job retry'", - ) - parser.add_argument( - "--rebuild", - action="store_true", - help="Rebuild the entire build afresh via 'bk build rebuild'", - ) - parser.add_argument( - "--unblock", - action="store_true", - help="Unblock manual approval steps (e.g. .bazelci/presubmit.yml changes) via 'bk job unblock'", - ) - args = parser.parse_args() - +def retry_buildkite_build( + pipeline, + build_number, + job_name=None, + unblock=False, + rebuild=False, + rebuild_if_no_failed_jobs=True, +): check_cli("bk", "https://github.com/buildkite/cli") - - if args.job_id: - res = retry_job_by_uuid(args.job_id) - sys.exit(res.returncode) - - if len(args.args) == 3: - target = f"{args.args[0]}/{args.args[1]}/{args.args[2]}" - elif len(args.args) == 1: - target = args.args[0] - else: - print( - "â Error: Invalid arguments. Provide a single target (PR#, URL, or org/pipeline/build).", - file=sys.stderr, - ) - sys.exit(1) - - if target.isdigit() and len(target) < 10: - print(f"đ Inspecting PR #{target} via gh to find Buildkite URL...") - target = get_build_url_from_pr(target) - - pipeline, build_number = normalize_build_target(target) - - if args.job_name: - retry_jobs_by_name(pipeline, build_number, args.job_name) - elif args.unblock: + if job_name: + return retry_jobs_by_name(pipeline, build_number, job_name) + elif unblock: cmd = [ "bk", "build", @@ -183,14 +321,14 @@ f"âšī¸ No blocked manual approval jobs found in build #{build_number}.", file=sys.stderr, ) - elif args.rebuild: + return True + elif rebuild: print(f"đ Rebuilding entire build #{build_number} for pipeline: {pipeline}") res = subprocess.run( ["bk", "build", "rebuild", build_number, "-p", pipeline, "--yes"] ) - sys.exit(res.returncode) + return res.returncode == 0 else: - # Default behavior: unblock manual approval steps and retry failed jobs if any, otherwise rebuild cmd = [ "bk", "build", @@ -208,7 +346,10 @@ failed_jobs = [ j for j in jobs - if j.get("state") in ("failed", "broken", "timed_out") and j.get("id") + if j.get("state") in ("failed", "broken", "timed_out") + and j.get("id") + and not j.get("retried_in_job_uuid") + and not j.get("retried_at") ] if failed_jobs: for j in failed_jobs: @@ -216,7 +357,10 @@ jname = j.get("name") or j.get("label") or jid print(f"đ Retrying failed job '{jname}' (ID: {jid})...") subprocess.run(["bk", "job", "retry", jid]) - elif not unblocked_jobs: + return True + elif unblocked_jobs: + return True + elif rebuild_if_no_failed_jobs: print( f"đ Rebuilding entire build #{build_number} for pipeline: {pipeline}" ) @@ -231,23 +375,240 @@ "--yes", ] ) - sys.exit(res.returncode) + return res.returncode == 0 + return False except Exception: + if rebuild_if_no_failed_jobs: + print( + f"đ Rebuilding entire build #{build_number} for pipeline: {pipeline}" + ) + res = subprocess.run( + [ + "bk", + "build", + "rebuild", + build_number, + "-p", + pipeline, + "--yes", + ] + ) + return res.returncode == 0 + return False + + +def handle_pr_target(pr_number, job_name=None, unblock=False, rebuild=False): + print(f"đ Inspecting PR #{pr_number} checks via gh...") + checks = get_pr_checks(pr_number) + + bk_checks = [] + gh_checks = [] + for c in checks: + link = c.get("link", "") + if "buildkite.com" in link: + bk_checks.append(c) + elif parse_github_actions_url(link): + gh_checks.append(c) + + if unblock: + if not bk_checks: + print(f"âšī¸ No Buildkite checks found for PR #{pr_number}.", file=sys.stderr) + return 0 + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + retry_buildkite_build(pipeline, build_number, unblock=True) + return 0 + + if job_name: + matched_any = False + matched_gh_runs = {} + for c in gh_checks: + cname = c.get("name", "") + parsed = parse_github_actions_url(c.get("link", "")) + if parsed and ( + re.search(job_name, cname, re.IGNORECASE) + or job_name.lower() in cname.lower() + ): + repo, run_id, job_id = parsed + matched_gh_runs.setdefault((repo, run_id), []).append(job_id) + + for (repo, run_id), job_ids in matched_gh_runs.items(): + matched_any = True + if len(job_ids) == 1 and job_ids[0]: + retry_github_actions(repo, run_id, job_id=job_ids[0]) + else: + retry_github_actions(repo, run_id, job_pattern=job_name) + + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + if retry_jobs_by_name( + pipeline, build_number, job_name, exit_on_missing=False + ): + matched_any = True + + if not matched_any: print( - f"đ Rebuilding entire build #{build_number} for pipeline: {pipeline}" + f"â ī¸ No GitHub Actions or Buildkite jobs found matching " + f"'{job_name}' for PR #{pr_number}.", + file=sys.stderr, ) - res = subprocess.run( - [ - "bk", - "build", - "rebuild", - build_number, - "-p", - pipeline, - "--yes", - ] + return 1 + return 0 + + # Retry failed GitHub Actions checks (or rebuild all failed runs if --rebuild) + failed_gh_runs = {} + for c in gh_checks: + state = (c.get("state") or "").upper() + bucket = (c.get("bucket") or "").lower() + is_failed = ( + state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE") + or bucket == "fail" + ) + if is_failed or rebuild: + parsed = parse_github_actions_url(c.get("link", "")) + if parsed: + repo, run_id, job_id = parsed + if is_failed: + failed_gh_runs.setdefault((repo, run_id), []).append(job_id) + + gh_retried = False + for (repo, run_id), job_ids in failed_gh_runs.items(): + gh_retried = True + if rebuild: + retry_github_actions(repo, run_id, rebuild=True) + elif len(job_ids) == 1 and job_ids[0]: + retry_github_actions(repo, run_id, job_id=job_ids[0]) + else: + retry_github_actions(repo, run_id) + + # Check Buildkite builds + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + state = (c.get("state") or "").upper() + bucket = (c.get("bucket") or "").lower() + bk_failed = ( + state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT") or bucket == "fail" + ) + retry_buildkite_build( + pipeline, + build_number, + rebuild=rebuild, + rebuild_if_no_failed_jobs=(bk_failed and not gh_retried), + ) + + if not bk_checks and not gh_retried: + print( + f"âšī¸ No failing GitHub Actions or Buildkite checks found for PR #{pr_number}." + ) + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Retry failed Buildkite or GitHub Actions jobs individually, " + "unblock Buildkite approval steps, or rebuild entire builds." + ) + ) + parser.add_argument( + "args", + nargs="*", + help=( + "Target build (PR#, Buildkite URL, GitHub Actions run/job URL, " + "or org/pipeline/build)" + ), + ) + parser.add_argument( + "--jobs", + "--job-name", + dest="job_name", + help="Specific job name or regex pattern to retry individually", + ) + parser.add_argument( + "--job-id", + dest="job_id", + help=( + "Specific Buildkite job UUID or GitHub Actions job ID to retry individually" + ), + ) + parser.add_argument( + "--rebuild", + action="store_true", + help="Rebuild the entire build afresh", + ) + parser.add_argument( + "--unblock", + action="store_true", + help=( + "Unblock manual approval steps (e.g. .bazelci/presubmit.yml " + "changes) via 'bk job unblock'" + ), + ) + args = parser.parse_args() + + if args.job_id: + gh_parsed = parse_github_actions_url(args.job_id) + if gh_parsed: + repo, run_id, job_id = gh_parsed + sys.exit( + retry_github_actions( + repo, + run_id, + job_id=job_id, + job_pattern=args.job_name, + rebuild=args.rebuild, + ) ) - sys.exit(res.returncode) + if args.job_id.isdigit(): + sys.exit(retry_github_job_by_id(args.job_id)) + res = retry_job_by_uuid(args.job_id) + sys.exit(res.returncode) + + if len(args.args) == 3: + target = f"{args.args[0]}/{args.args[1]}/{args.args[2]}" + elif len(args.args) == 1: + target = args.args[0] + else: + print( + "â Error: Invalid arguments. Provide a single target " + "(PR#, Buildkite URL, GitHub Actions URL, or org/pipeline/build).", + file=sys.stderr, + ) + sys.exit(1) + + gh_parsed = parse_github_actions_url(target) + if gh_parsed: + repo, run_id, job_id = gh_parsed + sys.exit( + retry_github_actions( + repo, + run_id, + job_id=job_id, + job_pattern=args.job_name, + rebuild=args.rebuild, + ) + ) + + if target.isdigit() and len(target) < 10: + sys.exit( + handle_pr_target( + target, + job_name=args.job_name, + unblock=args.unblock, + rebuild=args.rebuild, + ) + ) + + pipeline, build_number = normalize_build_target(target) + ok = retry_buildkite_build( + pipeline, + build_number, + job_name=args.job_name, + unblock=args.unblock, + rebuild=args.rebuild, + rebuild_if_no_failed_jobs=True, + ) + sys.exit(0 if ok else 1) if __name__ == "__main__":
diff --git a/.agents/skills/merge-pr/SKILL.md b/.agents/skills/merge-pr/SKILL.md index f410692..99f8176 100644 --- a/.agents/skills/merge-pr/SKILL.md +++ b/.agents/skills/merge-pr/SKILL.md
@@ -25,7 +25,8 @@ - Have the subagent use the **`buildkite-retry-job`** skill (`retry_buildkite_jobs.py <pr_number>`) to automatically retry any transient network flakes (e.g., HTTP 504 gateway timeouts, downloader - errors) and unblock manual approval steps. + errors, `setup-bazel` `Unable to find Bazelisk version` errors) and + unblock manual approval steps. - **Blocked Builds (`.bazelci/presubmit.yml` Approval)**: When `.bazelci/presubmit.yml` is modified, Buildkite blocks the build until manually approved. Approve and unblock it immediately via
diff --git a/.agents/skills/monitor-ci-results/SKILL.md b/.agents/skills/monitor-ci-results/SKILL.md index b5a0af7..29cba5d 100644 --- a/.agents/skills/monitor-ci-results/SKILL.md +++ b/.agents/skills/monitor-ci-results/SKILL.md
@@ -50,7 +50,8 @@ `analyze-ci-failure` skill on the reported failure. 5. **Autonomous Flake Retry**: If the CI Failure Analyzer subagent confirms that the failure is a transient infrastructure or network flake (e.g. disk - I/O error, 504 gateway, sandbox initialization failure), **immediately and + I/O error, 504 gateway, sandbox initialization failure, or `setup-bazel` + failing with `Unable to find Bazelisk version`), **immediately and autonomously** launch a separate subagent using `invoke_subagent` with the role "CI Job Retrier" and the prompt template in `.agents/skills/monitor-ci-results/retry-job-prompt.md` to run the
diff --git a/.agents/skills/monitor-ci-results/retry-job-prompt.md b/.agents/skills/monitor-ci-results/retry-job-prompt.md index 4d2edad..9d948b2 100644 --- a/.agents/skills/monitor-ci-results/retry-job-prompt.md +++ b/.agents/skills/monitor-ci-results/retry-job-prompt.md
@@ -1,3 +1,5 @@ -Use the `buildkite-retry-job` skill to retry the failed Buildkite job(s) for PR {pr_number}. -Follow the instructions in `.agents/skills/buildkite-retry-job/SKILL.md` to retry the failing jobs using the skill. +Use the `buildkite-retry-job` skill to retry the failed Buildkite or GitHub +Actions job(s) for PR {pr_number}. +Follow the instructions in `.agents/skills/buildkite-retry-job/SKILL.md` to +retry the failing jobs using the skill. Once the retry request completes, report back to the parent conversation.
diff --git a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py index 19a7f11..669d117 100755 --- a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py +++ b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py
@@ -324,26 +324,30 @@ elif ( state.upper() in ["FAILURE", "FAIL", "CANCELLED", "TIMED_OUT"] or bucket == "fail" - ) and name not in monitored: - print(f"đ¨ Notifying failure for GitHub check '{name}'...") - job_id = link.split("/")[-1] if "/" in link else link - msg = ( - f"â ī¸ Remote CI GitHub Check '{name}' completed with errors!\n\n" - f"Link: {link}\n\n" - f"Action required: Start a subagent to run the analyze-ci-failure skill for '{name}' with link '{link}' and job ID '{job_id}'." - ) - subprocess.run( - [ - "agentapi", - "send-message", - "--title=CI Check Failed", - args.conv_id, - msg, - ] - ) - monitored[name] = time.time() - with open(state_file, "w") as f: - json.dump(monitored, f) + ): + job_id = link.split("/")[-1].split("?")[0] if "/" in link else link + gh_key = f"gh_{job_id}" if job_id else name + if gh_key not in monitored: + print(f"đ¨ Notifying failure for GitHub check '{name}'...") + msg = ( + f"â ī¸ Remote CI GitHub Check '{name}' completed with errors!\n\n" + f"Link: {link}\n\n" + f"Action required: Start a subagent to run the " + f"analyze-ci-failure skill for '{name}' with link " + f"'{link}' and job ID '{job_id}'." + ) + subprocess.run( + [ + "agentapi", + "send-message", + "--title=CI Check Failed", + args.conv_id, + msg, + ] + ) + monitored[gh_key] = time.time() + with open(state_file, "w") as f: + json.dump(monitored, f) time.sleep(args.interval)