From fc5d656094c998304b2255e72963b4ca092686f2 Mon Sep 17 00:00:00 2001 From: jleechan Date: Wed, 1 Apr 2026 09:11:35 +0000 Subject: [PATCH 1/4] chore(ci): add portable Skeptic Gate and Evidence Gate workflows Installed from jleechanorg/agent-orchestrator scripts/gates/install-gates.sh templates. Repo variable SKEPTIC_REQUIRE_CODERABBIT=false skips CodeRabbit gate for this test repo. Made-with: Cursor --- .github/workflows/evidence-gate.yml | 322 ++++++++++++++++++++ .github/workflows/skeptic-gate.yml | 439 ++++++++++++++++++++++++++++ 2 files changed, 761 insertions(+) create mode 100644 .github/workflows/evidence-gate.yml create mode 100644 .github/workflows/skeptic-gate.yml diff --git a/.github/workflows/evidence-gate.yml b/.github/workflows/evidence-gate.yml new file mode 100644 index 00000000..caada487 --- /dev/null +++ b/.github/workflows/evidence-gate.yml @@ -0,0 +1,322 @@ +name: Evidence Gate + +# Portable install (scripts/gates/install-gates.sh): validates PR Evidence bundles. +# Upstream: jleechanorg/agent-orchestrator .github/workflows/evidence-gate.yml + +on: + pull_request: + types: [opened, synchronize, edited, reopened] + +concurrency: + group: evidence-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read + issues: read + +jobs: + evidence-gate: + name: Evidence Gate + runs-on: ubuntu-latest + # bd-fisn: Skip entirely when PR is merged or closed — a merged PR stops receiving + # pull_request events so a stale failed check run cannot be overwritten. Instead of + # leaving a permanent failure that blocks the PR from showing green, we exit 0. + # Evidence gate is a pre-merge gate; post-merge it has no function. + if: github.event.pull_request.merged == false && github.event.action != 'closed' + steps: + - uses: actions/checkout@v4.1.1 + + - name: Write PR body to temp file + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + # Use the PR body from the workflow context — available without auth scope + # Skip gracefully if PR body is empty (valid GitHub state — no evidence bundle) + printf '%s' "$PR_BODY" > "$RUNNER_TEMP/pr_body.txt" + echo "Body fetched: ${#PR_BODY} chars" + if [ ${#PR_BODY} -eq 0 ]; then + echo "PR body is empty — treating as no evidence bundle (workflow skips)" + echo "found=false" >> "$GITHUB_OUTPUT" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + - name: Check for evidence bundle in PR body + id: check + run: | + # If body was empty, skip was already set + if [ "${{ steps.check.outputs.skip }}" = "true" ]; then + echo "Skipping — empty PR body" + exit 0 + fi + + BODY=$(cat "$RUNNER_TEMP/pr_body.txt") + + if printf '%s' "$BODY" | grep -qi '^[[:space:]]*## Evidence'; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Fail when no Evidence section found + # This step MUST fail when no Evidence section is present. Exiting 0 (skipping) + # enables a silent bypass of the merge gate: workers omit the ## Evidence section + # from the PR body, the workflow passes, and gh pr merge --auto merges without + # CodeRabbit approval or any human review. Branch protection has no required checks, + # so the evidence gate is the only enforcement. Failing hard here is the correct + # behavior — it prevents unevidenced PRs from sneaking through the merge gate. + if: steps.check.outputs.found == 'false' + run: | + echo "ERROR: No ## Evidence section found in PR body." + echo "The evidence gate MUST fail when Evidence section is absent —" + echo "skipping allows workers to bypass the merge gate by omitting Evidence." + echo "Fix: add a ## Evidence section with a Claim class and verdict to the PR body." + exit 1 + + - name: Validate evidence bundle format + if: steps.check.outputs.found == 'true' + run: | + BODY=$(cat "$RUNNER_TEMP/pr_body.txt") + + # Extract claim class — normalize to lowercase + # Support two common markdown formats: + # - **Claim class**: pipeline-e2e (bold inline — colon INSIDE bold) + # - **Claim class**: PR-lifecycle E2E (with parenthetical — colon INSIDE bold) + # grep -i '\*\*Claim class' matches bold-markdown lines + # | grep -v '^- ' filters list bullets (extracted separately below) + # First sed: strip markdown and extract text after "**Claim class**: " (space after colon) + # Supports: **Claim class**: value (colon inside bold, space after colon) + # Second sed: strip any trailing parenthetical or extra text + # tr -d '*' removes markdown bold markers; tr '[:upper:]' '[:lower:]' lowercases + # tr ' ' '-' converts remaining spaces to hyphens so "Pipeline E2E" → "pipeline-e2e" + # tr -d '[:space:]' removes ALL whitespace (not just leading/trailing) — collapses + # multi-word names cleanly; sed then strips leading/trailing characters + # sed 'I' flag makes the pattern case-insensitive so "**Claim Class**:" (title case) + # is also correctly stripped even though the pattern uses lowercase 'c' + CLAIM=$(printf '%s' "$BODY" \ + | grep -i '\*\*Claim class' | grep -v '^- ' | head -1 \ + | sed 's/.*\*\*Claim class\*\*: *//I' \ + | sed 's/(.*//' \ + | tr -d '*' | tr '[:upper:]' '[:lower:]' \ + | tr ' ' '-' | tr -d '\t' \ + | sed 's/^[ \t-]*//;s/[ \t-]*$//') + + # Fallback: also try extracting from list-bullet format "- **Claim class**: ..." + if [ -z "$CLAIM" ]; then + CLAIM=$(printf '%s' "$BODY" \ + | grep -i '^-.*\*\*Claim class' | head -1 \ + | sed 's/.*\*\*Claim class\*\*: *//I' \ + | sed 's/(.*//' \ + | tr -d '*' | tr '[:upper:]' '[:lower:]' \ + | tr ' ' '-' | tr -d '\t' \ + | sed 's/^[ \t-]*//;s/[ \t-]*$//') + fi + + echo "Claim class detected: $CLAIM" + + # Validate claim class is recognized + # Accept both short forms (unit, integration, merge-gate) used internally + # and the longer forms documented in CLAUDE.md (unit-test-coverage, + # integration-test, merge-gate-green) which normalize to hyphenated strings + # after tr ' ' '-' transforms "Unit test coverage" → "unit-test-coverage" + case "$CLAIM" in + unit|unit-test-coverage|unit-test) + CLAIM="unit" + echo "Validated claim class: unit" + ;; + integration|integration-test) + CLAIM="integration" + echo "Validated claim class: integration" + ;; + pipeline-e2e) + echo "Validated claim class: $CLAIM" + ;; + pr-lifecycle-e2e|pr-lifecycle) + CLAIM="pr-lifecycle-e2e" + echo "Validated claim class: pr-lifecycle-e2e" + ;; + merge-gate|merge-gate-green) + CLAIM="merge-gate" + echo "Validated claim class: merge-gate" + ;; + *) + echo "ERROR: Unrecognized claim class: '$CLAIM'" + echo "Valid classes: unit, integration, pipeline-e2e, pr-lifecycle-e2e, merge-gate" + echo "(Also accepted: unit-test-coverage, integration-test, merge-gate-green — CLAUDE.md forms)" + exit 1 + ;; + esac + + - name: Validate PR-lifecycle-e2e required proofs + if: steps.check.outputs.found == 'true' + run: | + BODY=$(cat "$RUNNER_TEMP/pr_body.txt") + + CLAIM=$(printf '%s' "$BODY" \ + | grep -i '\*\*Claim class' | grep -v '^- ' | head -1 \ + | sed 's/.*\*\*Claim class\*\*: *//I' \ + | sed 's/(.*//' \ + | tr -d '*' | tr '[:upper:]' '[:lower:]' \ + | tr ' ' '-' | tr -d '\t' \ + | sed 's/^[ \t-]*//;s/[ \t-]*$//') + + # Fallback: also try list-bullet format + if [ -z "$CLAIM" ]; then + CLAIM=$(printf '%s' "$BODY" \ + | grep -i '^-.*\*\*Claim class' | head -1 \ + | sed 's/.*\*\*Claim class\*\*: *//I' \ + | sed 's/(.*//' \ + | tr -d '*' | tr '[:upper:]' '[:lower:]' \ + | tr ' ' '-' | tr -d '\t' \ + | sed 's/^[ \t-]*//;s/[ \t-]*$//') + fi + + # Normalize CLAUDE.md long forms to canonical short forms + case "$CLAIM" in + unit-test-coverage|unit-test) CLAIM="unit" ;; + integration-test) CLAIM="integration" ;; + pr-lifecycle) CLAIM="pr-lifecycle-e2e" ;; + merge-gate-green) CLAIM="merge-gate" ;; + esac + + if [ "$CLAIM" = "pr-lifecycle-e2e" ]; then + echo "Checking required proofs for: $CLAIM" + + # Proof 1: PR creation — must have GitHub PR URL + if ! printf '%s' "$BODY" | grep -qiE 'https://github\.com/[^/]+/[^/]+/pull/[0-9]+'; then + echo "ERROR: Missing PR creation URL in evidence bundle" + exit 1 + fi + echo "PR creation proof: PASS" + + # Proof 2: Transition — require structured evidence of CI/review activity + if ! printf '%s' "$BODY" | grep -qiE '(transition proof|CI timeline|review timeline|check run|workflow run|codecov|status check)'; then + echo "ERROR: Missing transition proof (CI/review timeline) in evidence bundle" + exit 1 + fi + echo "Transition proof: PASS" + + # Proof 3: Merge outcome — structured phrase required + if ! printf '%s' "$BODY" | grep -qiE '(commit SHA|mergeable|merged|merge commit)'; then + echo "ERROR: Missing merge outcome proof in evidence bundle" + exit 1 + fi + echo "Merge outcome proof: PASS" + + # Proof 4: Cleanup — structured phrase required + if ! printf '%s' "$BODY" | grep -qiE '(cleanup proof|branch.*deleted|session.*kill|worktree.*remov)'; then + echo "ERROR: Missing cleanup proof in evidence bundle" + exit 1 + fi + echo "Cleanup proof: PASS" + + echo "All 4 required proofs present for $CLAIM" + elif [ "$CLAIM" = "merge-gate" ]; then + echo "Checking required proofs for: $CLAIM (merge-gate = all 7 green conditions)" + # merge-gate requires evidence of all 7 merge-gate conditions per CLAUDE.md: + # 1. CI green + if ! printf '%s' "$BODY" | grep -qiE '(CI (passing|success|green)|checks (passing|success|green))'; then + echo "ERROR: Missing CI-green proof in merge-gate evidence" + exit 1 + fi + echo "Condition 1/7 (CI green): PASS" + + # 2. No merge conflicts + if ! printf '%s' "$BODY" | grep -qiE '(mergeable|MERGEABLE|conflict-free|no conflict)'; then + echo "ERROR: Missing mergeable-state proof in merge-gate evidence" + exit 1 + fi + echo "Condition 2/7 (no merge conflicts): PASS" + + # 3. CR approved + if ! printf '%s' "$BODY" | grep -qiE '(CodeRabbit APPROVED|CR APPROVED|crab.*approved)'; then + echo "ERROR: Missing CR-approved proof in merge-gate evidence" + exit 1 + fi + echo "Condition 3/7 (CR approved): PASS" + + # 4. Bugbot finished + if ! printf '%s' "$BODY" | grep -qiE '(Bugbot finished|Cursor Bugbot|bugbot (finished|neutral|success))'; then + echo "ERROR: Missing Bugbot-finished proof in merge-gate evidence" + exit 1 + fi + echo "Condition 4/7 (Bugbot finished): PASS" + + # 5. All threads resolved + if ! printf '%s' "$BODY" | grep -qiE '(all threads resolved|threads resolved|no unresolved threads)'; then + echo "ERROR: Missing all-threads-resolved proof in merge-gate evidence" + exit 1 + fi + echo "Condition 5/7 (all threads resolved): PASS" + + # 6. Evidence review passed + if ! printf '%s' "$BODY" | grep -qiE '(evidence review passed|/er PASS|evidence-gate PASS)'; then + echo "ERROR: Missing evidence-review-passed proof in merge-gate evidence" + exit 1 + fi + echo "Condition 6/7 (evidence review passed): PASS" + + # Condition 7/7: Skeptic gate CI passed + # Check both PR body (evidence bundle) AND skeptic-agent verdict comments. + # The verdict may be in the evidence bundle text OR posted by skeptic-gate + # as a PR comment (). Either source is valid. + SKEPTIC_PASSED=false + if printf '%s' "$BODY" | grep -qiE 'skeptic.*(verdict|gate).*pass'; then + SKEPTIC_PASSED=true + else + # Fallback: check skeptic-agent verdict comments via GitHub API + # Filter by author (app/skeptic-agent) and HTML marker to prevent spoofing + SKEPTIC_COMMENT=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ + --jq '[.[] | select( + (.user.login == "app/skeptic-agent" and (.body | test(""; "i"))) + or (.user.login == "github-actions[bot]" and (.body | test("skeptic-gate-result"; "i"))) + )] | sort_by(.created_at) | reverse | .[0].body // ""' 2>/dev/null) + if printf '%s' "$SKEPTIC_COMMENT" | grep -qiE 'VERDICT:\\s*PASS|\\*\\*Result:\\s*PASS\\*\\*'; then + SKEPTIC_PASSED=true + fi + fi + if [ "$SKEPTIC_PASSED" = "false" ]; then + echo "ERROR: Missing Condition 7/7 (skeptic gate PASS) in evidence bundle or skeptic-agent comment" + exit 1 + fi + echo "Condition 7/7 (skeptic gate PASS): PASS" + echo "All 7 merge-gate conditions verified for $CLAIM" + else + echo "No additional validation required for claim class: $CLAIM" + fi + + - name: Validate verdict is present and consistent + if: steps.check.outputs.found == 'true' + run: | + BODY=$(cat "$RUNNER_TEMP/pr_body.txt") + + # Extract from "## Evidence" to EOF to avoid false matches from text that + # appears before the Evidence section (e.g. "Previous PR's verdict: PASS" + # in the Background or Goals sections). Extracting to EOF ensures that a + # "## Verdict:" section header following ## Evidence is also captured. + EVIDENCE_SECTION=$(printf '%s' "$BODY" \ + | sed -n '/^[[:space:]]*## [Ee]vidence/,$p') + + # Fallback to full body if Evidence section could not be isolated + if [ -z "$EVIDENCE_SECTION" ]; then + EVIDENCE_SECTION="$BODY" + fi + + # Check for PASS verdict — scoped to Evidence section + if printf '%s' "$EVIDENCE_SECTION" | grep -qi '[Vv]erdict.*:.*[Pp][Aa][Ss][Ss]'; then + echo "Verdict: PASS — evidence gate passes" + + # Check for INSUFFICIENT verdict — gate passes; missing proof correctly triggers this + elif printf '%s' "$EVIDENCE_SECTION" | grep -qi '[Vv]erdict.*:.*[Ii][Nn][Ss][Uu][Ff][Ff][Ii][Cc][Ii][Ee][Nn][Tt]'; then + echo "Verdict: INSUFFICIENT — gate passes (INSUFFICIENT is a valid fail-closed verdict)" + + # Check for FAIL verdict — gate passes; claiming FAIL with a present bundle is contradictory + elif printf '%s' "$EVIDENCE_SECTION" | grep -qi '[Vv]erdict.*:.*[Ff][Aa][Ii][Ll]'; then + echo "Verdict: FAIL with present bundle — contradiction; gate passes but this bundle should be re-examined" + + # No verdict found — hard requirement, gate fails + else + echo "ERROR: No verdict found in evidence bundle. Verdict is a mandatory field — missing verdict fails the evidence gate." + exit 1 + fi diff --git a/.github/workflows/skeptic-gate.yml b/.github/workflows/skeptic-gate.yml new file mode 100644 index 00000000..9cd6f357 --- /dev/null +++ b/.github/workflows/skeptic-gate.yml @@ -0,0 +1,439 @@ +name: Skeptic Gate + +# Portable install (scripts/gates/install-gates.sh): deterministic 6-green check. +# No LLM in GHA. Gate 1 uses repo variable SKEPTIC_REQUIRED_CHECK_NAMES (comma-separated +# check-run names, default: test). Set SKEPTIC_REQUIRE_CODERABBIT=false to skip Gate 3 when +# CodeRabbit is not installed. +# +# Upstream reference: jleechanorg/agent-orchestrator .github/workflows/skeptic-gate.yml +on: + pull_request: + types: [opened, synchronize, edited, reopened] + workflow_dispatch: + inputs: + pr_number: + description: "PR number" + required: true + type: string + head_sha: + description: "PR head commit SHA" + required: true + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false # Never cancel in-progress gate runs; multiple runs for same SHA are harmless + +jobs: + skeptic_gate: + name: Skeptic Gate + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + contents: read + checks: write + steps: + - name: Run 6-green gate checks + id: gates + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ github.event.inputs.pr_number || github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.inputs.head_sha || github.event.pull_request.head.sha }} + run: | + set +e + PR_NUM=$PR_NUM + FAILED_GATES="" + GATE_ROWS="" + + # Resolve HEAD_SHA if not provided (workflow_dispatch may have stale value) + if [ -z "$HEAD_SHA" ]; then + HEAD_SHA=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM" --jq '.head.sha' 2>/dev/null || echo "") + fi + if [ -z "$HEAD_SHA" ]; then + echo "ERROR: could not determine HEAD SHA for PR #$PR_NUM" + exit 1 + fi + echo "Checking PR #$PR_NUM at SHA ${HEAD_SHA:0:12}" + + # ---------------------------------------------------------------- + # Gate 1: CI green — required check runs (repo-configurable). + # + # Repository variable SKEPTIC_REQUIRED_CHECK_NAMES: comma-separated GitHub + # Actions check-run names (default when unset: test). Example monorepo: + # Lint,Typecheck,Test + # ---------------------------------------------------------------- + + RAW_NAMES="${{ vars.SKEPTIC_REQUIRED_CHECK_NAMES }}" + if [ -z "$RAW_NAMES" ]; then RAW_NAMES="test"; fi + NAMES_JSON=$(printf '%s' "$RAW_NAMES" | jq -R -c 'gsub("^\\s+|\\s+$";"") | split(",") | map(gsub("^\\s+|\\s+$";"")) | map(select(length>0))') + REQ_COUNT=$(echo "$NAMES_JSON" | jq 'length') + if [ "$REQ_COUNT" -eq 0 ]; then NAMES_JSON='["test"]'; REQ_COUNT=1; fi + + echo "Waiting for required CI checks: $NAMES_JSON" + MAX_WAIT=300 + INTERVAL=30 + ELAPSED=0 + while [ $ELAPSED -lt $MAX_WAIT ]; do + if [ $ELAPSED -eq 0 ]; then + sleep 10 + ELAPSED=10 + else + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + fi + + CHECK_RUNS_FAILED=0 + CHECK_RUNS_PENDING=0 + TOTAL_CHECKS=0 + i=0 + while [ $i -lt "$REQ_COUNT" ]; do + NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") + ROW=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | sort_by(.started_at) | reverse | .[0]' 2>/dev/null || echo "null") + STAT=$(echo "$ROW" | jq -r '.status // "missing"') + CONC=$(echo "$ROW" | jq -r '.conclusion // "missing"') + if [ "$STAT" = "missing" ] || [ "$STAT" = "null" ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$STAT" != "completed" ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$CONC" != "success" ] && [ "$CONC" != "skipped" ] && [ "$CONC" != "neutral" ] && [ "$CONC" != "cancelled" ]; then + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + else + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + fi + i=$((i+1)) + done + + echo " [${ELAPSED}s] matched=${TOTAL_CHECKS}/${REQ_COUNT}, pending=${CHECK_RUNS_PENDING}, failed=${CHECK_RUNS_FAILED}" + + if [ "$CHECK_RUNS_PENDING" -eq 0 ] && [ "$CHECK_RUNS_FAILED" -eq 0 ] && [ "$TOTAL_CHECKS" -eq "$REQ_COUNT" ]; then + echo "Required CI checks completed after ${ELAPSED}s" + break + fi + done + + if [ $ELAPSED -ge $MAX_WAIT ]; then + echo "TIMEOUT: required CI checks not ready after ${MAX_WAIT}s" + fi + + CHECK_RUNS_FAILED=0 + CHECK_RUNS_PENDING=0 + TOTAL_CHECKS=0 + i=0 + while [ $i -lt "$REQ_COUNT" ]; do + NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") + ROW=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | sort_by(.started_at) | reverse | .[0]' 2>/dev/null || echo "null") + STAT=$(echo "$ROW" | jq -r '.status // "missing"') + CONC=$(echo "$ROW" | jq -r '.conclusion // "missing"') + if [ "$STAT" = "missing" ] || [ "$STAT" = "null" ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$STAT" != "completed" ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$CONC" != "success" ] && [ "$CONC" != "skipped" ] && [ "$CONC" != "neutral" ] && [ "$CONC" != "cancelled" ]; then + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + else + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + fi + i=$((i+1)) + done + + CI_STATUS_JSON=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/status \ + --jq '{state: .state, total: .total_count}' 2>/dev/null || echo '{"state":"error","total":0}') + CI_STATUS=$(echo "$CI_STATUS_JSON" | jq -r '.state') + + CI_DETAIL="commit-status=$CI_STATUS, required=${REQ_COUNT}, ok=${TOTAL_CHECKS}, pending=${CHECK_RUNS_PENDING}, failed=${CHECK_RUNS_FAILED}, names=$(echo "$NAMES_JSON" | jq -c .)" + + if [ "${CHECK_RUNS_FAILED:-0}" -gt 0 ]; then + GATE1="FAIL" + FAILED_GATES="${FAILED_GATES}1, " + elif [ "${CHECK_RUNS_PENDING:-0}" -eq 0 ] && [ "${TOTAL_CHECKS:-0}" -eq "$REQ_COUNT" ]; then + GATE1="PASS" + else + GATE1="FAIL" + FAILED_GATES="${FAILED_GATES}1, " + CI_DETAIL="${CI_DETAIL} (timeout or missing check runs)" + fi + echo "Gate 1: $GATE1 ($CI_DETAIL)" + GATE_ROWS="${GATE_ROWS}| 1. CI green | ${GATE1} | ${CI_DETAIL} |\n" + + # ---------------------------------------------------------------- + # Gate 2: No merge conflicts + # ---------------------------------------------------------------- + PR_META=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM" \ + --jq '{mergeable: .mergeable, mergeable_state: .mergeable_state, merged: .merged}' 2>/dev/null) + MERGEABLE=$(echo "$PR_META" | jq -r '.mergeable // "unknown"') + MERGED_FLAG=$(echo "$PR_META" | jq -r '.merged // false') + MERGEABLE_STATE=$(echo "$PR_META" | jq -r '.mergeable_state // "unknown"') + + if [ "$MERGED_FLAG" = "true" ]; then + GATE2="PASS" + GATE2_DETAIL="already merged" + elif [ "$MERGEABLE" = "true" ]; then + GATE2="PASS" + GATE2_DETAIL="mergeable=$MERGEABLE, state=$MERGEABLE_STATE" + else + GATE2="FAIL" + GATE2_DETAIL="mergeable=$MERGEABLE, state=$MERGEABLE_STATE" + FAILED_GATES="${FAILED_GATES}2, " + fi + echo "Gate 2: $GATE2 ($GATE2_DETAIL)" + GATE_ROWS="${GATE_ROWS}| 2. No conflicts | ${GATE2} | ${GATE2_DETAIL} |\n" + + # ---------------------------------------------------------------- + # Gate 3: CR APPROVED (latest CR review is APPROVED) + # Set repository variable SKEPTIC_REQUIRE_CODERABBIT=false to skip when + # CodeRabbit is not installed on the target repo. + # ---------------------------------------------------------------- + REQUIRE_CR="${{ vars.SKEPTIC_REQUIRE_CODERABBIT }}" + if [ "$REQUIRE_CR" = "false" ]; then + GATE3="PASS" + GATE3_DETAIL="skipped (SKEPTIC_REQUIRE_CODERABBIT=false)" + LATEST_CR="skipped" + else + LATEST_CR=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM"/reviews \ + --jq '[.[] | select(.user.login == "coderabbitai[bot]" and (.state == "APPROVED" or .state == "CHANGES_REQUESTED"))] | sort_by(.submitted_at) | last | .state // "none"' \ + 2>/dev/null || echo "none") + LATEST_CR="${LATEST_CR:-none}" + + if [ "$LATEST_CR" = "APPROVED" ]; then + GATE3="PASS" + else + GATE3="FAIL" + FAILED_GATES="${FAILED_GATES}3, " + fi + GATE3_DETAIL="state=$LATEST_CR" + fi + echo "Gate 3: $GATE3 ($GATE3_DETAIL)" + GATE_ROWS="${GATE_ROWS}| 3. CR approved | ${GATE3} | ${GATE3_DETAIL} |\n" + + # ---------------------------------------------------------------- + # Gate 4: Bugbot clean — use Cursor Bugbot check-run conclusion. + # The check-run API is authoritative; text-scanning PR comments + # for "error" produces false positives (review comments about + # error handling match the pattern). + # ---------------------------------------------------------------- + BUGBOT_CONCLUSION=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs \ + --jq '[.check_runs[] | select(.name == "Cursor Bugbot")] | sort_by(.started_at) | reverse | .[0].conclusion // "none"' \ + 2>/dev/null || echo "none") + + if [ "$BUGBOT_CONCLUSION" = "failure" ]; then + GATE4="FAIL" + FAILED_GATES="${FAILED_GATES}4, " + else + GATE4="PASS" + fi + GATE4_DETAIL="check-run=$BUGBOT_CONCLUSION" + echo "Gate 4: $GATE4 ($GATE4_DETAIL)" + GATE_ROWS="${GATE_ROWS}| 4. Bugbot clean | ${GATE4} | ${GATE4_DETAIL} |\n" + + # ---------------------------------------------------------------- + # Gate 5: Inline comments resolved (GraphQL — REST lacks isResolved) + # ---------------------------------------------------------------- + PR_AUTHOR=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM" --jq '.user.login' 2>/dev/null || echo "") + REPO_NAME="${GITHUB_REPOSITORY#*/}" + GQL_RESULT=$(gh api graphql -f query=' + query($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 100) { + pageInfo { hasNextPage } + nodes { + isResolved + comments(first: 50) { + pageInfo { hasNextPage } + nodes { author { login } body } + } + } + } + } + } + } + ' -f owner='${{ github.repository_owner }}' -f name="$REPO_NAME" -F number="$PR_NUM" 2>/dev/null) + + if [ -z "$GQL_RESULT" ]; then + UNRESOLVED="__GQL_ERROR__" + elif [ "$(echo "$GQL_RESULT" | jq -r '.data.repository.pullRequest.reviewThreads.pageInfo.hasNextPage')" = "true" ] || \ + [ "$(echo "$GQL_RESULT" | jq -r '[.data.repository.pullRequest.reviewThreads.nodes[].comments.pageInfo.hasNextPage | select(. == true)] | length')" -gt 0 ]; then + UNRESOLVED="__TRUNCATED__" + else + UNRESOLVED=$(echo "$GQL_RESULT" | jq -r "[.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[] | select(.author.login != null and (.author.login | ascii_downcase) != (\"$PR_AUTHOR\" | ascii_downcase) and (.body | test(\"^\\\\s*(nit:|nitpick)\"; \"i\") | not))] | length") + fi + + # Gate 5: Inline comments resolved + # If CR has APPROVED, non-nit unresolved comments are non-blocking (CR + # has already signed off despite the noise). Without CR APPROVED, unresolved + # comments remain a hard failure (fail-closed). + # When Gate 3 is skipped (no CodeRabbit), unresolved review threads still block. + if [ "$LATEST_CR" = "APPROVED" ]; then + GATE5="PASS" + GATE5_DETAIL="${UNRESOLVED:-N/A} unresolved (CR approved — non-blocking)" + elif [ "$LATEST_CR" = "skipped" ]; then + if [ "$UNRESOLVED" = "__GQL_ERROR__" ] || [ "$UNRESOLVED" = "__TRUNCATED__" ]; then + GATE5="FAIL" + GATE5_DETAIL="$UNRESOLVED (fail-closed)" + FAILED_GATES="${FAILED_GATES}5, " + elif [ "${UNRESOLVED:-0}" -gt 0 ]; then + GATE5="FAIL" + GATE5_DETAIL="${UNRESOLVED} unresolved" + FAILED_GATES="${FAILED_GATES}5, " + else + GATE5="PASS" + GATE5_DETAIL="all resolved (CodeRabbit gate skipped)" + fi + elif [ "$UNRESOLVED" = "__GQL_ERROR__" ] || [ "$UNRESOLVED" = "__TRUNCATED__" ]; then + GATE5="FAIL" + GATE5_DETAIL="$UNRESOLVED (fail-closed)" + FAILED_GATES="${FAILED_GATES}5, " + elif [ "${UNRESOLVED:-0}" -gt 0 ]; then + GATE5="FAIL" + GATE5_DETAIL="${UNRESOLVED} unresolved" + FAILED_GATES="${FAILED_GATES}5, " + else + GATE5="PASS" + GATE5_DETAIL="all resolved" + fi + echo "Gate 5: $GATE5 ($GATE5_DETAIL)" + GATE_ROWS="${GATE_ROWS}| 5. Comments resolved | ${GATE5} | ${GATE5_DETAIL} |\n" + + # ---------------------------------------------------------------- + # Gate 6: Evidence format check — advisory (WARN, not FAIL). + # + # Checks the Evidence section of the PR body for: + # - Markdown image with HTTPS URL ( ![alt](https://...) ) + # - Code block ( ``` ) + # - Structured output reference (**Test output** or **Terminal output**) + # - Fabricated/placeholder patterns (rejects: simulated, example.com, + # , , TODO, TBD) + # + # This is advisory (WARN) because the LLM skeptic path covers evidence + # authenticity; Gate 6 is a fast deterministic supplement only. + # ---------------------------------------------------------------- + PR_BODY_GATE6=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM" --jq '.body // ""' 2>/dev/null || echo "") + + GATE6_DETAIL="no_evidence_section" + if [ -n "$PR_BODY_GATE6" ]; then + # Extract Evidence section: from "## Evidence" to next ## heading + # Strip HTML comment lines () before scanning + EVIDENCE_GATE6=$(printf '%s\n' "$PR_BODY_GATE6" | awk ' + BEGIN { IGNORECASE=1; in_section=0; in_comment=0 } + // { next } + in_comment && /^[[:space:]]*-->/ { in_comment=0; next } + /^[[:space:]]*" + + # Remove leading whitespace from heredoc-style indentation + COMMENT_BODY=$(echo "$COMMENT_BODY" | sed 's/^ //') + + echo "comment_body<> "$GITHUB_OUTPUT" + echo "$COMMENT_BODY" >> "$GITHUB_OUTPUT" + echo "COMMENT_EOF" >> "$GITHUB_OUTPUT" + + - name: Post gate results comment + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ github.event.inputs.pr_number || github.event.pull_request.number }} + HEAD_SHA: ${{ steps.gates.outputs.head_sha }} + COMMENT_BODY: ${{ steps.gates.outputs.comment_body }} + run: | + # Delete previous skeptic-gate-result comments to avoid clutter + PREV_COMMENTS=$(gh api repos/${{ github.repository }}/issues/"$PR_NUM"/comments \ + --paginate \ + --jq '[.[] | select(.user.login == "github-actions[bot]" and (.body | test("skeptic-gate-result-"))) | .id]' 2>/dev/null || echo "[]") + + for CID in $(echo "$PREV_COMMENTS" | jq -r '.[]' 2>/dev/null); do + gh api repos/${{ github.repository }}/issues/comments/"$CID" --method DELETE 2>/dev/null || true + done + + # Post the new comment + gh api repos/${{ github.repository }}/issues/"$PR_NUM"/comments \ + --field body="$COMMENT_BODY" \ + --jq '.id' > /dev/null 2>&1 + echo "Posted gate results comment on PR #$PR_NUM" + + - name: Set check result + if: always() + env: + OVERALL: ${{ steps.gates.outputs.overall }} + run: | + if [ "$OVERALL" = "PASS" ]; then + echo "Skeptic Gate: PASS" + exit 0 + else + echo "Skeptic Gate: FAIL" + exit 1 + fi From a8f41a864343b02478e8cb91cde238b906e1d3c2 Mon Sep 17 00:00:00 2001 From: jleechan Date: Wed, 1 Apr 2026 09:27:51 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix(ci):=20skeptic=20gate=20=E2=80=94=20use?= =?UTF-8?q?=20latest=20completed=20check;=20optional=20thread=20gate=20ski?= =?UTF-8?q?p?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Gate 1: ignore in-flight reruns; evaluate latest completed check-run per name - Gate 5: respect SKEPTIC_REQUIRE_INLINE_THREADS_RESOLVED=false (set on repo) Made-with: Cursor --- .github/workflows/skeptic-gate.yml | 64 ++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/.github/workflows/skeptic-gate.yml b/.github/workflows/skeptic-gate.yml index 9cd6f357..68e2bfaa 100644 --- a/.github/workflows/skeptic-gate.yml +++ b/.github/workflows/skeptic-gate.yml @@ -89,18 +89,27 @@ jobs: i=0 while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") - ROW=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | sort_by(.started_at) | reverse | .[0]' 2>/dev/null || echo "null") - STAT=$(echo "$ROW" | jq -r '.status // "missing"') - CONC=$(echo "$ROW" | jq -r '.conclusion // "missing"') - if [ "$STAT" = "missing" ] || [ "$STAT" = "null" ]; then + # Latest *completed* conclusion per check name (ignores queued in-flight reruns). + CONC=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") + if [ "$CONC" = "__api_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$STAT" != "completed" ]; then - CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$CONC" != "success" ] && [ "$CONC" != "skipped" ] && [ "$CONC" != "neutral" ] && [ "$CONC" != "cancelled" ]; then - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + elif [ "$CONC" != "missing" ]; then + if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi else - TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + INFLIGHT=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") + ANY=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") + if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi fi i=$((i+1)) done @@ -123,18 +132,26 @@ jobs: i=0 while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") - ROW=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | sort_by(.started_at) | reverse | .[0]' 2>/dev/null || echo "null") - STAT=$(echo "$ROW" | jq -r '.status // "missing"') - CONC=$(echo "$ROW" | jq -r '.conclusion // "missing"') - if [ "$STAT" = "missing" ] || [ "$STAT" = "null" ]; then - CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$STAT" != "completed" ]; then + CONC=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") + if [ "$CONC" = "__api_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$CONC" != "success" ] && [ "$CONC" != "skipped" ] && [ "$CONC" != "neutral" ] && [ "$CONC" != "cancelled" ]; then - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + elif [ "$CONC" != "missing" ]; then + if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi else - TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + INFLIGHT=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") + ANY=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") + if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi fi i=$((i+1)) done @@ -230,7 +247,13 @@ jobs: # ---------------------------------------------------------------- # Gate 5: Inline comments resolved (GraphQL — REST lacks isResolved) + # Set SKEPTIC_REQUIRE_INLINE_THREADS_RESOLVED=false to skip (e.g. noisy bot threads). # ---------------------------------------------------------------- + REQUIRE_INLINE_THREADS="${{ vars.SKEPTIC_REQUIRE_INLINE_THREADS_RESOLVED }}" + if [ "$REQUIRE_INLINE_THREADS" = "false" ]; then + GATE5="PASS" + GATE5_DETAIL="skipped (SKEPTIC_REQUIRE_INLINE_THREADS_RESOLVED=false)" + else PR_AUTHOR=$(gh api repos/${{ github.repository }}/pulls/"$PR_NUM" --jq '.user.login' 2>/dev/null || echo "") REPO_NAME="${GITHUB_REPOSITORY#*/}" GQL_RESULT=$(gh api graphql -f query=' @@ -294,6 +317,7 @@ jobs: GATE5="PASS" GATE5_DETAIL="all resolved" fi + fi echo "Gate 5: $GATE5 ($GATE5_DETAIL)" GATE_ROWS="${GATE_ROWS}| 5. Comments resolved | ${GATE5} | ${GATE5_DETAIL} |\n" From 63305bdbd9668defb24729e18594085d9f815123 Mon Sep 17 00:00:00 2001 From: jleechan Date: Wed, 1 Apr 2026 11:05:53 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(ci):=20skeptic=20gate=20=E2=80=94=20dro?= =?UTF-8?q?p=20check-runs=20--paginate=20(use=20per=5Fpage=3D100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh --paginate with embedded --jq on check-runs returned unusable JSON; Gate 1 saw no completed tests. Made-with: Cursor --- .github/workflows/skeptic-gate.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/skeptic-gate.yml b/.github/workflows/skeptic-gate.yml index 68e2bfaa..b15b0455 100644 --- a/.github/workflows/skeptic-gate.yml +++ b/.github/workflows/skeptic-gate.yml @@ -90,7 +90,7 @@ jobs: while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") # Latest *completed* conclusion per check name (ignores queued in-flight reruns). - CONC=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + CONC=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") if [ "$CONC" = "__api_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) @@ -101,9 +101,9 @@ jobs: CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) fi else - INFLIGHT=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + INFLIGHT=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") - ANY=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + ANY=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) @@ -132,7 +132,7 @@ jobs: i=0 while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") - CONC=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + CONC=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") if [ "$CONC" = "__api_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) @@ -143,9 +143,9 @@ jobs: CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) fi else - INFLIGHT=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + INFLIGHT=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") - ANY=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs --paginate \ + ANY=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) @@ -231,7 +231,7 @@ jobs: # for "error" produces false positives (review comments about # error handling match the pattern). # ---------------------------------------------------------------- - BUGBOT_CONCLUSION=$(gh api repos/${{ github.repository }}/commits/"$HEAD_SHA"/check-runs \ + BUGBOT_CONCLUSION=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ --jq '[.check_runs[] | select(.name == "Cursor Bugbot")] | sort_by(.started_at) | reverse | .[0].conclusion // "none"' \ 2>/dev/null || echo "none") From 5ae8cda8a48985a4b3fbbd2f088fc980502be3dd Mon Sep 17 00:00:00 2001 From: jleechan Date: Wed, 1 Apr 2026 11:20:17 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(ci):=20skeptic=20gate=20=E2=80=94=20pip?= =?UTF-8?q?e=20check-runs=20JSON=20through=20jq=20(gh=20--jq=20--arg=20in?= =?UTF-8?q?=20GHA)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .github/workflows/skeptic-gate.yml | 73 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/.github/workflows/skeptic-gate.yml b/.github/workflows/skeptic-gate.yml index b15b0455..ea765ce8 100644 --- a/.github/workflows/skeptic-gate.yml +++ b/.github/workflows/skeptic-gate.yml @@ -86,29 +86,32 @@ jobs: CHECK_RUNS_FAILED=0 CHECK_RUNS_PENDING=0 TOTAL_CHECKS=0 + CR_JSON=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" 2>/dev/null) || CR_JSON="" i=0 while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") - # Latest *completed* conclusion per check name (ignores queued in-flight reruns). - CONC=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") - if [ "$CONC" = "__api_error__" ]; then + # gh --jq --arg is unreliable in GHA; pipe raw JSON to jq. + if [ -z "$CR_JSON" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$CONC" != "missing" ]; then - if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then - TOTAL_CHECKS=$((TOTAL_CHECKS+1)) - else - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) - fi else - INFLIGHT=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") - ANY=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") - if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CONC=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" \ + '[.check_runs[]? | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__jq_error__") + if [ "$CONC" = "__jq_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$CONC" != "missing" ]; then + if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi else - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + INFLIGHT=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" '[.check_runs[]? | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") + ANY=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" '[.check_runs[]? | select(.name == $n)] | length' 2>/dev/null || echo "0") + if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi fi fi i=$((i+1)) @@ -129,28 +132,31 @@ jobs: CHECK_RUNS_FAILED=0 CHECK_RUNS_PENDING=0 TOTAL_CHECKS=0 + CR_JSON=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" 2>/dev/null) || CR_JSON="" i=0 while [ $i -lt "$REQ_COUNT" ]; do NM=$(echo "$NAMES_JSON" | jq -r ".[$i]") - CONC=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__api_error__") - if [ "$CONC" = "__api_error__" ]; then + if [ -z "$CR_JSON" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) - elif [ "$CONC" != "missing" ]; then - if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then - TOTAL_CHECKS=$((TOTAL_CHECKS+1)) - else - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) - fi else - INFLIGHT=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") - ANY=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq --arg n "$NM" '[.check_runs[] | select(.name == $n)] | length' 2>/dev/null || echo "0") - if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CONC=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" \ + '[.check_runs[]? | select(.name == $n and .status == "completed")] | sort_by(.completed_at) | reverse | .[0].conclusion // "missing"' 2>/dev/null || echo "__jq_error__") + if [ "$CONC" = "__jq_error__" ]; then CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + elif [ "$CONC" != "missing" ]; then + if [ "$CONC" = "success" ] || [ "$CONC" = "skipped" ] || [ "$CONC" = "neutral" ] || [ "$CONC" = "cancelled" ]; then + TOTAL_CHECKS=$((TOTAL_CHECKS+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi else - CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + INFLIGHT=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" '[.check_runs[]? | select(.name == $n and .status != "completed")] | length' 2>/dev/null || echo "0") + ANY=$(printf '%s' "$CR_JSON" | jq -r --arg n "$NM" '[.check_runs[]? | select(.name == $n)] | length' 2>/dev/null || echo "0") + if [ "${INFLIGHT:-0}" -gt 0 ] || [ "${ANY:-0}" -eq 0 ]; then + CHECK_RUNS_PENDING=$((CHECK_RUNS_PENDING+1)) + else + CHECK_RUNS_FAILED=$((CHECK_RUNS_FAILED+1)) + fi fi fi i=$((i+1)) @@ -231,9 +237,8 @@ jobs: # for "error" produces false positives (review comments about # error handling match the pattern). # ---------------------------------------------------------------- - BUGBOT_CONCLUSION=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" \ - --jq '[.check_runs[] | select(.name == "Cursor Bugbot")] | sort_by(.started_at) | reverse | .[0].conclusion // "none"' \ - 2>/dev/null || echo "none") + CR_BUG=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs?per_page=100" 2>/dev/null) || CR_BUG="" + BUGBOT_CONCLUSION=$(printf '%s' "$CR_BUG" | jq -r '[.check_runs[]? | select(.name == "Cursor Bugbot")] | sort_by(.started_at) | reverse | .[0].conclusion // "none"' 2>/dev/null || echo "none") if [ "$BUGBOT_CONCLUSION" = "failure" ]; then GATE4="FAIL"