From 2e1743e7a35b3502d84f590705c2c7ef75937cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Miko=C5=82ajczak?= Date: Mon, 22 Jun 2026 14:17:39 +0200 Subject: [PATCH 1/3] Add reviewer prompt-cache health monitoring + cold-cache regression warning --- .github/scripts/report-cache-health.sh | 55 ++++++++++++++++++++++++++ .github/workflows/claude-review.yml | 9 +++++ 2 files changed, 64 insertions(+) create mode 100755 .github/scripts/report-cache-health.sh diff --git a/.github/scripts/report-cache-health.sh b/.github/scripts/report-cache-health.sh new file mode 100755 index 000000000000..5d7775633fb9 --- /dev/null +++ b/.github/scripts/report-cache-health.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Reports prompt-cache health for a reviewer run. +# +# Source is the action's execution file (always written, independent of +# `show_full_output`). We read ONLY token counts — never message content — so it +# is safe to emit in public CI logs. +# +# Output: +# - a per-run table in the GitHub step summary, +# - a single parseable line (for external monitoring / scraping), +# - a NON-BLOCKING warning annotation when the rule prefix was written instead +# of read, i.e. the cache went cold. +# +# A cold run is expected occasionally (first review after an idle gap > TTL). +# A *sustained* cold signal means caching regressed — typically a rules edit, an +# action/CLI bump (changes the system-prompt bytes), a model change, or the TTL +# reverting to 5m (ENABLE_PROMPT_CACHING_1H lost). +set -euo pipefail + +f="${EXEC_FILE:-}" +if [ -z "$f" ] || [ ! -s "$f" ]; then + echo "report-cache-health: no execution file, skipping" + exit 0 +fi + +# Aggregate cache tokens across assistant turns; cost from the result message. +read -r CREATION READ COST < <(jq -r ' + [.[] | select(.type=="assistant") | .message.usage // empty] as $u + | (($u | map(.cache_creation_input_tokens // 0) | add) // 0) as $c + | (($u | map(.cache_read_input_tokens // 0) | add) // 0) as $r + | (([.[] | select(.type=="result")] | last | .total_cost_usd) // 0) as $cost + | "\($c) \($r) \($cost)"' "$f") + +TOTAL=$(( CREATION + READ )) +RATIO=0 +[ "$TOTAL" -gt 0 ] && RATIO=$(( READ * 100 / TOTAL )) +WARN_TOKENS="${CACHE_CREATION_WARN_TOKENS:-15000}" + +{ + echo "### Reviewer prompt-cache health" + echo "" + echo "| metric | value |" + echo "| --- | --- |" + echo "| cache_read tokens | ${READ} |" + echo "| cache_creation tokens | ${CREATION} |" + echo "| warm ratio (read / read+create) | ${RATIO}% |" + echo "| total_cost_usd | ${COST} |" +} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +# Parseable single line (counts only — safe to log). +echo "cache_health read=${READ} creation=${CREATION} warm_ratio=${RATIO}% cost=${COST}" + +if [ "$CREATION" -gt "$WARN_TOKENS" ]; then + echo "::warning title=Reviewer cache cold::cache_creation=${CREATION} tokens (> ${WARN_TOKENS}) — the rule prefix was written, not read. Expected occasionally (first run after an idle gap); if sustained across runs, caching has regressed (rules edit, action/CLI bump, model change, or 1h TTL lost)." +fi diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 4cfc0f46fcf2..ab09c868e547 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -110,6 +110,15 @@ jobs: STRUCTURED_OUTPUT: ${{ steps.code-review.outputs.structured_output }} run: postCodeReviewResults.sh "$PR_NUMBER" + # Monitoring: report prompt-cache health (token counts only) and warn, + # non-blocking, if the cached rule prefix went cold. Detects silent caching + # regressions (rules edit, action/CLI bump, model change, TTL revert). + - name: Report reviewer cache health + if: always() && steps.code-review.outputs.execution_file != '' + env: + EXEC_FILE: ${{ steps.code-review.outputs.execution_file }} + run: .github/scripts/report-cache-health.sh + - name: Run Claude Code (docs) if: steps.set-authorized.outputs.IS_AUTHORIZED == 'true' && steps.filter.outputs.docs == 'true' uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # v1.0.149 From 7cf5f2db4a00d9aa250d5a71e8f7a14b379ed0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Miko=C5=82ajczak?= Date: Mon, 22 Jun 2026 23:22:13 +0200 Subject: [PATCH 2/3] Split cache monitoring: minimal in-run emit + scheduled collector --- .github/scripts/collect-cache-health.sh | 63 ++++++++++++++++++++ .github/scripts/emit-cache-usage.sh | 23 +++++++ .github/scripts/report-cache-health.sh | 55 ----------------- .github/workflows/claude-review.yml | 10 ++-- .github/workflows/reviewer-cache-monitor.yml | 39 ++++++++++++ 5 files changed, 130 insertions(+), 60 deletions(-) create mode 100755 .github/scripts/collect-cache-health.sh create mode 100755 .github/scripts/emit-cache-usage.sh delete mode 100755 .github/scripts/report-cache-health.sh create mode 100644 .github/workflows/reviewer-cache-monitor.yml diff --git a/.github/scripts/collect-cache-health.sh b/.github/scripts/collect-cache-health.sh new file mode 100755 index 000000000000..d7cd5c78a22d --- /dev/null +++ b/.github/scripts/collect-cache-health.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Scheduled collector: scans recent reviewer runs' LOGS for the CACHE_USAGE line +# emitted by emit-cache-usage.sh, computes the warm-hit trend, and fails (a red +# scheduled run = built-in alert) if caching looks regressed across a meaningful +# sample. +# +# Reads only run logs (token counts) - no message content, no artifacts. Runs +# entirely out of the review hot path. +# +# Tunables (env): +# LOOKBACK how many recent reviewer runs to scan (default 30) +# CACHE_CREATION_WARN_TOKENS a run is "cold" if creation exceeds this (default 15000) +# COLD_ALERT_PCT alert if more than this % of the sample is cold (default 40) +# MIN_SAMPLE need at least this many data-carrying runs to judge (default 10) +set -euo pipefail + +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY not set}" +LOOKBACK="${LOOKBACK:-30}" +WARN_TOKENS="${CACHE_CREATION_WARN_TOKENS:-15000}" +COLD_ALERT_PCT="${COLD_ALERT_PCT:-40}" +MIN_SAMPLE="${MIN_SAMPLE:-10}" + +mapfile -t RUNS < <(gh run list --repo "$REPO" --workflow claude-review.yml \ + --status completed --limit "$LOOKBACK" --json databaseId --jq '.[].databaseId') + +scanned=0 +cold=0 +rows="" +for id in "${RUNS[@]}"; do + line=$(gh run view "$id" --repo "$REPO" --log 2>/dev/null | grep -m1 'CACHE_USAGE' || true) + [ -n "$line" ] || continue + c=$(sed -n 's/.*creation=\([0-9][0-9]*\).*/\1/p' <<<"$line") + r=$(sed -n 's/.*read=\([0-9][0-9]*\).*/\1/p' <<<"$line") + [ -n "$c" ] && [ -n "$r" ] || continue + t=$(( c + r )); ratio=0; [ "$t" -gt 0 ] && ratio=$(( r * 100 / t )) + scanned=$(( scanned + 1 )) + [ "$c" -gt "$WARN_TOKENS" ] && cold=$(( cold + 1 )) + rows="${rows}| ${id} | ${c} | ${r} | ${ratio}% |"$'\n' +done + +{ + echo "### Reviewer cache monitor" + echo "" + echo "Scanned **${scanned}** data-carrying runs of the last ${LOOKBACK}." + echo "" + echo "| run | creation | read | warm% |" + echo "| --- | --- | --- | --- |" + printf '%s' "$rows" +} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +if [ "$scanned" -lt "$MIN_SAMPLE" ]; then + echo "Only ${scanned} runs carried cache data (< ${MIN_SAMPLE}); not enough to judge — skipping." + exit 0 +fi + +coldpct=$(( cold * 100 / scanned )) +echo "cold=${cold}/${scanned} (${coldpct}%) alert threshold=${COLD_ALERT_PCT}%" + +if [ "$coldpct" -gt "$COLD_ALERT_PCT" ]; then + echo "::error title=Reviewer cache regressed::${coldpct}% of the last ${scanned} reviewer runs were cold (> ${COLD_ALERT_PCT}%). Likely cause: a rules edit, an action/CLI bump, a model change, or the 1h TTL being lost." + exit 1 +fi +echo "Reviewer cache healthy: ${coldpct}% cold over ${scanned} runs." diff --git a/.github/scripts/emit-cache-usage.sh b/.github/scripts/emit-cache-usage.sh new file mode 100755 index 000000000000..82a2f1866b1f --- /dev/null +++ b/.github/scripts/emit-cache-usage.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# In-run, minimal: emit the reviewer's prompt-cache token counts to the run log. +# +# Token COUNTS ONLY (no message content) -> safe in public logs. Source is the +# action's execution file (always written, independent of `show_full_output`). +# +# Analysis, thresholds and alerting live OUT of the review hot path: the +# scheduled collector (.github/workflows/reviewer-cache-monitor.yml) scans these +# lines across recent runs to detect caching regressions over time. +set -euo pipefail + +f="${EXEC_FILE:-}" +if [ -z "$f" ] || [ ! -s "$f" ]; then + echo "emit-cache-usage: no execution file, skipping" + exit 0 +fi + +jq -r ' + [.[] | select(.type=="assistant") | .message.usage // empty] as $u + | (($u | map(.cache_creation_input_tokens // 0) | add) // 0) as $c + | (($u | map(.cache_read_input_tokens // 0) | add) // 0) as $r + | (([.[] | select(.type=="result")] | last | .total_cost_usd) // 0) as $cost + | "CACHE_USAGE creation=\($c) read=\($r) cost=\($cost)"' "$f" diff --git a/.github/scripts/report-cache-health.sh b/.github/scripts/report-cache-health.sh deleted file mode 100755 index 5d7775633fb9..000000000000 --- a/.github/scripts/report-cache-health.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Reports prompt-cache health for a reviewer run. -# -# Source is the action's execution file (always written, independent of -# `show_full_output`). We read ONLY token counts — never message content — so it -# is safe to emit in public CI logs. -# -# Output: -# - a per-run table in the GitHub step summary, -# - a single parseable line (for external monitoring / scraping), -# - a NON-BLOCKING warning annotation when the rule prefix was written instead -# of read, i.e. the cache went cold. -# -# A cold run is expected occasionally (first review after an idle gap > TTL). -# A *sustained* cold signal means caching regressed — typically a rules edit, an -# action/CLI bump (changes the system-prompt bytes), a model change, or the TTL -# reverting to 5m (ENABLE_PROMPT_CACHING_1H lost). -set -euo pipefail - -f="${EXEC_FILE:-}" -if [ -z "$f" ] || [ ! -s "$f" ]; then - echo "report-cache-health: no execution file, skipping" - exit 0 -fi - -# Aggregate cache tokens across assistant turns; cost from the result message. -read -r CREATION READ COST < <(jq -r ' - [.[] | select(.type=="assistant") | .message.usage // empty] as $u - | (($u | map(.cache_creation_input_tokens // 0) | add) // 0) as $c - | (($u | map(.cache_read_input_tokens // 0) | add) // 0) as $r - | (([.[] | select(.type=="result")] | last | .total_cost_usd) // 0) as $cost - | "\($c) \($r) \($cost)"' "$f") - -TOTAL=$(( CREATION + READ )) -RATIO=0 -[ "$TOTAL" -gt 0 ] && RATIO=$(( READ * 100 / TOTAL )) -WARN_TOKENS="${CACHE_CREATION_WARN_TOKENS:-15000}" - -{ - echo "### Reviewer prompt-cache health" - echo "" - echo "| metric | value |" - echo "| --- | --- |" - echo "| cache_read tokens | ${READ} |" - echo "| cache_creation tokens | ${CREATION} |" - echo "| warm ratio (read / read+create) | ${RATIO}% |" - echo "| total_cost_usd | ${COST} |" -} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - -# Parseable single line (counts only — safe to log). -echo "cache_health read=${READ} creation=${CREATION} warm_ratio=${RATIO}% cost=${COST}" - -if [ "$CREATION" -gt "$WARN_TOKENS" ]; then - echo "::warning title=Reviewer cache cold::cache_creation=${CREATION} tokens (> ${WARN_TOKENS}) — the rule prefix was written, not read. Expected occasionally (first run after an idle gap); if sustained across runs, caching has regressed (rules edit, action/CLI bump, model change, or 1h TTL lost)." -fi diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index ab09c868e547..efd3fd1b06e3 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -110,14 +110,14 @@ jobs: STRUCTURED_OUTPUT: ${{ steps.code-review.outputs.structured_output }} run: postCodeReviewResults.sh "$PR_NUMBER" - # Monitoring: report prompt-cache health (token counts only) and warn, - # non-blocking, if the cached rule prefix went cold. Detects silent caching - # regressions (rules edit, action/CLI bump, model change, TTL revert). - - name: Report reviewer cache health + # Monitoring (emit only): print prompt-cache token counts to the log so the + # out-of-band collector (.github/workflows/reviewer-cache-monitor.yml) can + # track cache health across runs. Counts only - no message content. + - name: Emit reviewer cache usage if: always() && steps.code-review.outputs.execution_file != '' env: EXEC_FILE: ${{ steps.code-review.outputs.execution_file }} - run: .github/scripts/report-cache-health.sh + run: .github/scripts/emit-cache-usage.sh - name: Run Claude Code (docs) if: steps.set-authorized.outputs.IS_AUTHORIZED == 'true' && steps.filter.outputs.docs == 'true' diff --git a/.github/workflows/reviewer-cache-monitor.yml b/.github/workflows/reviewer-cache-monitor.yml new file mode 100644 index 000000000000..2de9fe5886bc --- /dev/null +++ b/.github/workflows/reviewer-cache-monitor.yml @@ -0,0 +1,39 @@ +name: Reviewer cache monitor + +# Out-of-band collector for reviewer prompt-cache health. Scans recent +# "PR Reviews with Claude Code" runs' logs for the CACHE_USAGE line and fails +# (a red scheduled run = built-in alert) if caching has regressed across a +# meaningful sample. Reads logs only - no message content, no artifacts. + +on: + schedule: + - cron: "23 * * * *" # hourly + workflow_dispatch: + inputs: + lookback: + description: How many recent reviewer runs to scan + required: false + default: "30" + +permissions: + contents: read + actions: read + +concurrency: + group: reviewer-cache-monitor + cancel-in-progress: true + +jobs: + collect: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Scan reviewer runs for cache health + env: + GH_TOKEN: ${{ github.token }} + LOOKBACK: ${{ github.event.inputs.lookback || '30' }} + run: .github/scripts/collect-cache-health.sh From e34569e08a9582f731a062ce170e31b41d018349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Miko=C5=82ajczak?= <62747088+kacper-mikolajczak@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:25:17 +0200 Subject: [PATCH 3/3] collect-cache-health: drop zero/zero runs from the sample A run that emits creation=0 read=0 (reviewer skipped or no usage-bearing turns) passed the parse guard, counted toward scanned/MIN_SAMPLE, and read as not-cold - diluting the cold fraction and masking regressions. Treat no cache activity as no data, not a warm sample. --- .github/scripts/collect-cache-health.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/collect-cache-health.sh b/.github/scripts/collect-cache-health.sh index d7cd5c78a22d..e6c19a70e687 100755 --- a/.github/scripts/collect-cache-health.sh +++ b/.github/scripts/collect-cache-health.sh @@ -32,6 +32,8 @@ for id in "${RUNS[@]}"; do c=$(sed -n 's/.*creation=\([0-9][0-9]*\).*/\1/p' <<<"$line") r=$(sed -n 's/.*read=\([0-9][0-9]*\).*/\1/p' <<<"$line") [ -n "$c" ] && [ -n "$r" ] || continue + # no cache activity at all = no data, not a warm sample + [ "$c" -eq 0 ] && [ "$r" -eq 0 ] && continue t=$(( c + r )); ratio=0; [ "$t" -gt 0 ] && ratio=$(( r * 100 / t )) scanned=$(( scanned + 1 )) [ "$c" -gt "$WARN_TOKENS" ] && cold=$(( cold + 1 ))