From dcec3025a27363f1ef7ca3e191caec7425f1e7f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 2 Jul 2026 23:42:27 +0000 Subject: [PATCH 1/7] Initial plan From 6a71e6238a22296130354f6a48d8340b9d6ea2c2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 2 Jul 2026 23:59:31 +0000 Subject: [PATCH 2/7] Use private temp dir for safe-output evaluations Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- ...jective-impact-safe-output-evaluations.cjs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/scripts/prepare-objective-impact-safe-output-evaluations.cjs b/scripts/prepare-objective-impact-safe-output-evaluations.cjs index 757e81a9924..5e27f63b71b 100644 --- a/scripts/prepare-objective-impact-safe-output-evaluations.cjs +++ b/scripts/prepare-objective-impact-safe-output-evaluations.cjs @@ -1,6 +1,5 @@ #!/usr/bin/env node -const crypto = require("crypto"); const fs = require("fs"); const path = require("path"); const { execFileSync } = require("child_process"); @@ -23,25 +22,22 @@ function readJSON(filePath, fallback) { } } -// Atomically write content to a file using a temp-file-then-rename pattern. -// Using O_EXCL with a cryptographically random suffix prevents TOCTOU and -// symlink attacks (CWE-377, CWE-378). crypto.randomBytes is used instead of -// process.pid to make the temp file name unpredictable. +// Atomically write content to a file using a private temp directory created +// alongside the destination. mkdtempSync creates the directory with restricted +// permissions, which avoids predictable temp-file races and keeps rename on the +// same filesystem as the destination file. function writeFileAtomic(filePath, content) { - const tmp = filePath + "." + crypto.randomBytes(8).toString("hex") + ".tmp"; - let fd; + const baseName = path.basename(filePath); + const parentDir = path.dirname(filePath); + const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); + const tmpFile = path.join(tmpDir, baseName); try { - fd = fs.openSync(tmp, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o666); - fs.writeFileSync(fd, content); - fs.closeSync(fd); - fd = undefined; - fs.renameSync(tmp, filePath); + fs.writeFileSync(tmpFile, content); + fs.renameSync(tmpFile, filePath); } catch (err) { - if (typeof fd === "number") { - try { fs.closeSync(fd); } catch {} - } - try { fs.unlinkSync(tmp); } catch {} throw err; + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); } } From 55b6520f5dcd2028cc94c8195195b9c4b91269d7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:03:10 +0000 Subject: [PATCH 3/7] Harden temp file writes in safe-output evaluations Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- ...re-objective-impact-safe-output-evaluations.cjs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/prepare-objective-impact-safe-output-evaluations.cjs b/scripts/prepare-objective-impact-safe-output-evaluations.cjs index 5e27f63b71b..f0007ef04c7 100644 --- a/scripts/prepare-objective-impact-safe-output-evaluations.cjs +++ b/scripts/prepare-objective-impact-safe-output-evaluations.cjs @@ -32,10 +32,16 @@ function writeFileAtomic(filePath, content) { const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); const tmpFile = path.join(tmpDir, baseName); try { - fs.writeFileSync(tmpFile, content); - fs.renameSync(tmpFile, filePath); - } catch (err) { - throw err; + try { + fs.writeFileSync(tmpFile, content); + } catch (err) { + throw new Error(`failed to write temp file ${tmpFile}`, { cause: err }); + } + try { + fs.renameSync(tmpFile, filePath); + } catch (err) { + throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err }); + } } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } From ac1ade4cf34629d58ee740ab31257e23dedaca54 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:29:54 +0000 Subject: [PATCH 4/7] Inline objective impact safe-output prep Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../objective-impact-report.lock.yml | 4 +- .github/workflows/objective-impact-report.md | 173 ++++++++++++++++- ...prepare-objective-impact-report-dataset.sh | 2 - ...jective-impact-safe-output-evaluations.cjs | 175 ------------------ 4 files changed, 174 insertions(+), 180 deletions(-) delete mode 100644 scripts/prepare-objective-impact-safe-output-evaluations.cjs diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 3a83cc76828..4d622c34ce2 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"fa6fd7fc54661e38506d4bca907a5bfa81b52dbd6f3c8dcd68d8319d73330c17","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"68887beaac914ad7f94b1b247e617dbbd8f8fff9461625b5aad661c2b052d60b","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.32","digest":"sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -514,7 +514,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Prepare deterministic impact datasets - run: bash scripts/prepare-objective-impact-report-dataset.sh + run: "bash scripts/prepare-objective-impact-report-dataset.sh\nnode <<'NODE'\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\nNODE\n" - name: Download container images run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.22 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/squid:0.27.22 ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 diff --git a/.github/workflows/objective-impact-report.md b/.github/workflows/objective-impact-report.md index fd0ca69edbe..11b7830e25b 100644 --- a/.github/workflows/objective-impact-report.md +++ b/.github/workflows/objective-impact-report.md @@ -38,7 +38,178 @@ pre-agent-steps: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} EXPR_GITHUB_REPOSITORY: ${{ github.repository }} - run: bash scripts/prepare-objective-impact-report-dataset.sh + run: | + bash scripts/prepare-objective-impact-report-dataset.sh + node <<'NODE' + const fs = require("fs"); + const path = require("path"); + const { execFileSync } = require("child_process"); + const { + evaluateItem, + normalizeOutcome, + readJSONL, + } = require(path.join(process.cwd(), "actions/setup/js/evaluate_outcomes.cjs")); + + const DATA_DIR = "/tmp/gh-aw/agent/objective-impact-report"; + const RUNS_DIR = path.join(DATA_DIR, "safe-output-runs"); + const OUTPUT_JSONL = path.join(DATA_DIR, "safe-output-issue-evaluations.jsonl"); + const OUTPUT_SUMMARY = path.join(DATA_DIR, "safe-output-issue-summary.json"); + + function readJSON(filePath, fallback) { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return fallback; + } + } + + function writeFileAtomic(filePath, content) { + const baseName = path.basename(filePath); + const parentDir = path.dirname(filePath); + const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); + const tmpFile = path.join(tmpDir, baseName); + try { + try { + fs.writeFileSync(tmpFile, content); + } catch (err) { + throw new Error(`failed to write temp file ${tmpFile}`, { cause: err }); + } + try { + fs.renameSync(tmpFile, filePath); + } catch (err) { + throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err }); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + } + + function writeJSON(filePath, value) { + writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n"); + } + + function gh(args) { + try { + return execFileSync("gh", args, { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); + } catch { + return null; + } + } + + function ensureIssueURL(item, repo) { + if (item.url || typeof item.number !== "number" || !repo) { + return item; + } + return { + ...item, + url: `https://github.com/${repo}/issues/${item.number}`, + }; + } + + function loadRuns() { + const workflowLogs = readJSON(path.join(DATA_DIR, "workflow-logs.json"), {}); + const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : []; + return runs + .map(run => ({ + id: Number(run.id ?? run.databaseId ?? 0), + workflow_name: run.workflow_name || run.workflowName || "", + aic: run.aic ?? null, + created_at: run.created_at || run.createdAt || "", + status: run.status || "", + conclusion: run.conclusion || "", + url: run.html_url || run.url || "", + })) + .filter(run => Number.isInteger(run.id) && run.id > 0); + } + + function loadManifest(runDir) { + const manifestPath = path.join(runDir, "safe-output-items.jsonl"); + if (!fs.existsSync(manifestPath)) return []; + return readJSONL(manifestPath); + } + + function downloadManifest(repo, runId, runDir) { + fs.mkdirSync(runDir, { recursive: true }); + const manifestPath = path.join(runDir, "safe-output-items.jsonl"); + if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) { + return true; + } + const result = gh(["run", "download", String(runId), "--repo", repo, "--name", "safe-outputs-items", "--dir", runDir]); + return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0; + } + + function main() { + const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || ""; + if (!repo) { + console.error("EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required"); + process.exit(1); + } + + fs.mkdirSync(DATA_DIR, { recursive: true }); + fs.mkdirSync(RUNS_DIR, { recursive: true }); + + const runs = loadRuns(); + const rows = []; + + for (const run of runs) { + const runDir = path.join(RUNS_DIR, `run-${run.id}`); + if (!downloadManifest(repo, run.id, runDir)) { + continue; + } + + const items = loadManifest(runDir) + .filter(item => item && (item.type === "create_issue" || item.type === "close_issue")) + .map(item => ensureIssueURL(item, item.repo || repo)); + + for (const item of items) { + const evalResult = evaluateItem(item, repo); + const normalized = normalizeOutcome(evalResult.result, evalResult.detail); + rows.push({ + run_id: run.id, + workflow_name: run.workflow_name, + workflow_aic: run.aic, + workflow_run_created_at: run.created_at, + workflow_run_url: run.url, + type: item.type, + repo: item.repo || repo, + number: typeof item.number === "number" ? item.number : null, + url: item.url || "", + timestamp: item.timestamp || "", + result: evalResult.result, + detail: evalResult.detail, + outcome_status: normalized.outcome_status, + evidence_strength: normalized.evidence_strength, + signal: normalized.signal, + resolution_sec: evalResult.resolution_sec, + pending_age_sec: evalResult.pending_age_sec, + comments: evalResult.comments, + reactions_total: evalResult.reactions_total, + reactions_positive: evalResult.reactions_positive, + reactions_negative: evalResult.reactions_negative, + zero_touch: evalResult.zero_touch, + }); + } + } + + writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "")); + + const summary = { + total_issue_outcomes: rows.length, + create_issue_count: rows.filter(row => row.type === "create_issue").length, + close_issue_count: rows.filter(row => row.type === "close_issue").length, + accepted_count: rows.filter(row => row.outcome_status === "accepted").length, + rejected_count: rows.filter(row => row.outcome_status === "rejected").length, + pending_count: rows.filter(row => row.outcome_status === "pending").length, + ignored_count: rows.filter(row => row.outcome_status === "ignored").length, + unknown_count: rows.filter(row => row.outcome_status === "unknown").length, + distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length, + distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length, + }; + writeJSON(OUTPUT_SUMMARY, summary); + } + + main(); + NODE safe-outputs: close-issue: required-title-prefix: "Impact Efficiency Report - " diff --git a/scripts/prepare-objective-impact-report-dataset.sh b/scripts/prepare-objective-impact-report-dataset.sh index 38bcca9cee9..fa93b8e7d78 100644 --- a/scripts/prepare-objective-impact-report-dataset.sh +++ b/scripts/prepare-objective-impact-report-dataset.sh @@ -226,5 +226,3 @@ jq -n \ ] } ' > "$DATA_DIR/dataset-manifest.json" - -node scripts/prepare-objective-impact-safe-output-evaluations.cjs \ No newline at end of file diff --git a/scripts/prepare-objective-impact-safe-output-evaluations.cjs b/scripts/prepare-objective-impact-safe-output-evaluations.cjs deleted file mode 100644 index f0007ef04c7..00000000000 --- a/scripts/prepare-objective-impact-safe-output-evaluations.cjs +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env node - -const fs = require("fs"); -const path = require("path"); -const { execFileSync } = require("child_process"); -const { - evaluateItem, - normalizeOutcome, - readJSONL, -} = require("../actions/setup/js/evaluate_outcomes.cjs"); - -const DATA_DIR = "/tmp/gh-aw/agent/objective-impact-report"; -const RUNS_DIR = path.join(DATA_DIR, "safe-output-runs"); -const OUTPUT_JSONL = path.join(DATA_DIR, "safe-output-issue-evaluations.jsonl"); -const OUTPUT_SUMMARY = path.join(DATA_DIR, "safe-output-issue-summary.json"); - -function readJSON(filePath, fallback) { - try { - return JSON.parse(fs.readFileSync(filePath, "utf8")); - } catch { - return fallback; - } -} - -// Atomically write content to a file using a private temp directory created -// alongside the destination. mkdtempSync creates the directory with restricted -// permissions, which avoids predictable temp-file races and keeps rename on the -// same filesystem as the destination file. -function writeFileAtomic(filePath, content) { - const baseName = path.basename(filePath); - const parentDir = path.dirname(filePath); - const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); - const tmpFile = path.join(tmpDir, baseName); - try { - try { - fs.writeFileSync(tmpFile, content); - } catch (err) { - throw new Error(`failed to write temp file ${tmpFile}`, { cause: err }); - } - try { - fs.renameSync(tmpFile, filePath); - } catch (err) { - throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err }); - } - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -} - -function writeJSON(filePath, value) { - writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n"); -} - -function gh(args) { - try { - return execFileSync("gh", args, { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); - } catch { - return null; - } -} - -function ensureIssueURL(item, repo) { - if (item.url || typeof item.number !== "number" || !repo) { - return item; - } - return { - ...item, - url: `https://github.com/${repo}/issues/${item.number}`, - }; -} - -function loadRuns() { - const workflowLogs = readJSON(path.join(DATA_DIR, "workflow-logs.json"), {}); - const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : []; - return runs - .map(run => ({ - id: Number(run.id ?? run.databaseId ?? 0), - workflow_name: run.workflow_name || run.workflowName || "", - aic: run.aic ?? null, - created_at: run.created_at || run.createdAt || "", - status: run.status || "", - conclusion: run.conclusion || "", - url: run.html_url || run.url || "", - })) - .filter(run => Number.isInteger(run.id) && run.id > 0); -} - -function loadManifest(runDir) { - const manifestPath = path.join(runDir, "safe-output-items.jsonl"); - if (!fs.existsSync(manifestPath)) return []; - return readJSONL(manifestPath); -} - -function downloadManifest(repo, runId, runDir) { - fs.mkdirSync(runDir, { recursive: true }); - const manifestPath = path.join(runDir, "safe-output-items.jsonl"); - if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) { - return true; - } - const result = gh(["run", "download", String(runId), "--repo", repo, "--name", "safe-outputs-items", "--dir", runDir]); - return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0; -} - -function main() { - const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || ""; - if (!repo) { - console.error("EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required"); - process.exit(1); - } - - fs.mkdirSync(DATA_DIR, { recursive: true }); - fs.mkdirSync(RUNS_DIR, { recursive: true }); - - const runs = loadRuns(); - /** @type {any[]} */ - const rows = []; - - for (const run of runs) { - const runDir = path.join(RUNS_DIR, `run-${run.id}`); - if (!downloadManifest(repo, run.id, runDir)) { - continue; - } - - const items = loadManifest(runDir) - .filter(item => item && (item.type === "create_issue" || item.type === "close_issue")) - .map(item => ensureIssueURL(item, item.repo || repo)); - - for (const item of items) { - const evalResult = evaluateItem(item, repo); - const normalized = normalizeOutcome(evalResult.result, evalResult.detail); - rows.push({ - run_id: run.id, - workflow_name: run.workflow_name, - workflow_aic: run.aic, - workflow_run_created_at: run.created_at, - workflow_run_url: run.url, - type: item.type, - repo: item.repo || repo, - number: typeof item.number === "number" ? item.number : null, - url: item.url || "", - timestamp: item.timestamp || "", - result: evalResult.result, - detail: evalResult.detail, - outcome_status: normalized.outcome_status, - evidence_strength: normalized.evidence_strength, - signal: normalized.signal, - resolution_sec: evalResult.resolution_sec, - pending_age_sec: evalResult.pending_age_sec, - comments: evalResult.comments, - reactions_total: evalResult.reactions_total, - reactions_positive: evalResult.reactions_positive, - reactions_negative: evalResult.reactions_negative, - zero_touch: evalResult.zero_touch, - }); - } - } - - writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "")); - - const summary = { - total_issue_outcomes: rows.length, - create_issue_count: rows.filter(row => row.type === "create_issue").length, - close_issue_count: rows.filter(row => row.type === "close_issue").length, - accepted_count: rows.filter(row => row.outcome_status === "accepted").length, - rejected_count: rows.filter(row => row.outcome_status === "rejected").length, - pending_count: rows.filter(row => row.outcome_status === "pending").length, - ignored_count: rows.filter(row => row.outcome_status === "ignored").length, - unknown_count: rows.filter(row => row.outcome_status === "unknown").length, - distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length, - distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length, - }; - writeJSON(OUTPUT_SUMMARY, summary); -} - -main(); \ No newline at end of file From 4fe156a9f96cdd68375e66a15873de761e6672ab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:36:15 +0000 Subject: [PATCH 5/7] Split objective impact prep steps Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/objective-impact-report.lock.yml | 10 ++++++++-- .github/workflows/objective-impact-report.md | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 4d622c34ce2..f7261472d4b 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"68887beaac914ad7f94b1b247e617dbbd8f8fff9461625b5aad661c2b052d60b","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"32f48927408bffce6f57b3ad55d277466cbd098f38c65b561cac8aee37f5826b","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.32","digest":"sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -514,7 +514,13 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Prepare deterministic impact datasets - run: "bash scripts/prepare-objective-impact-report-dataset.sh\nnode <<'NODE'\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\nNODE\n" + run: bash scripts/prepare-objective-impact-report-dataset.sh + - env: + EXPR_GITHUB_REPOSITORY: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Prepare safe-output issue evaluations + run: "node <<'NODE'\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\nNODE\n" - name: Download container images run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.22 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/squid:0.27.22 ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 diff --git a/.github/workflows/objective-impact-report.md b/.github/workflows/objective-impact-report.md index 11b7830e25b..35ce3ab505e 100644 --- a/.github/workflows/objective-impact-report.md +++ b/.github/workflows/objective-impact-report.md @@ -34,12 +34,17 @@ tools: - "head -n * /tmp/gh-aw/agent/objective-impact-report/*.json /tmp/gh-aw/agent/objective-impact-report/*.jsonl" pre-agent-steps: - name: Prepare deterministic impact datasets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXPR_GITHUB_REPOSITORY: ${{ github.repository }} + run: bash scripts/prepare-objective-impact-report-dataset.sh + - name: Prepare safe-output issue evaluations env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} EXPR_GITHUB_REPOSITORY: ${{ github.repository }} run: | - bash scripts/prepare-objective-impact-report-dataset.sh node <<'NODE' const fs = require("fs"); const path = require("path"); @@ -48,7 +53,7 @@ pre-agent-steps: evaluateItem, normalizeOutcome, readJSONL, - } = require(path.join(process.cwd(), "actions/setup/js/evaluate_outcomes.cjs")); + } = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), "actions/setup/js/evaluate_outcomes.cjs")); const DATA_DIR = "/tmp/gh-aw/agent/objective-impact-report"; const RUNS_DIR = path.join(DATA_DIR, "safe-output-runs"); From 6f072cee314857b763469eadd559c1fe88d9ed4f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:46:39 +0000 Subject: [PATCH 6/7] Use github-script in objective impact workflow Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../objective-impact-report.lock.yml | 7 +- .github/workflows/objective-impact-report.md | 305 +++++++++--------- 2 files changed, 158 insertions(+), 154 deletions(-) diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index f7261472d4b..7de5b013041 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"32f48927408bffce6f57b3ad55d277466cbd098f38c65b561cac8aee37f5826b","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d99c0c1b113ac5c44a15f29adf7a8544dbd46b549e4451f4d2f75d29fa0bb1ea","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.32","digest":"sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -520,7 +520,10 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Prepare safe-output issue evaluations - run: "node <<'NODE'\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\nNODE\n" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: "const fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\n" - name: Download container images run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.22 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22 ghcr.io/github/gh-aw-firewall/squid:0.27.22 ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 diff --git a/.github/workflows/objective-impact-report.md b/.github/workflows/objective-impact-report.md index 35ce3ab505e..e12ec2ea9e5 100644 --- a/.github/workflows/objective-impact-report.md +++ b/.github/workflows/objective-impact-report.md @@ -44,177 +44,178 @@ pre-agent-steps: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} EXPR_GITHUB_REPOSITORY: ${{ github.repository }} - run: | - node <<'NODE' - const fs = require("fs"); - const path = require("path"); - const { execFileSync } = require("child_process"); - const { - evaluateItem, - normalizeOutcome, - readJSONL, - } = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), "actions/setup/js/evaluate_outcomes.cjs")); - - const DATA_DIR = "/tmp/gh-aw/agent/objective-impact-report"; - const RUNS_DIR = path.join(DATA_DIR, "safe-output-runs"); - const OUTPUT_JSONL = path.join(DATA_DIR, "safe-output-issue-evaluations.jsonl"); - const OUTPUT_SUMMARY = path.join(DATA_DIR, "safe-output-issue-summary.json"); - - function readJSON(filePath, fallback) { - try { - return JSON.parse(fs.readFileSync(filePath, "utf8")); - } catch { - return fallback; - } - } - - function writeFileAtomic(filePath, content) { - const baseName = path.basename(filePath); - const parentDir = path.dirname(filePath); - const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); - const tmpFile = path.join(tmpDir, baseName); - try { + uses: actions/github-script@v9.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require("fs"); + const path = require("path"); + const { execFileSync } = require("child_process"); + const { + evaluateItem, + normalizeOutcome, + readJSONL, + } = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), "actions/setup/js/evaluate_outcomes.cjs")); + + const DATA_DIR = "/tmp/gh-aw/agent/objective-impact-report"; + const RUNS_DIR = path.join(DATA_DIR, "safe-output-runs"); + const OUTPUT_JSONL = path.join(DATA_DIR, "safe-output-issue-evaluations.jsonl"); + const OUTPUT_SUMMARY = path.join(DATA_DIR, "safe-output-issue-summary.json"); + + function readJSON(filePath, fallback) { try { - fs.writeFileSync(tmpFile, content); - } catch (err) { - throw new Error(`failed to write temp file ${tmpFile}`, { cause: err }); + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return fallback; } + } + + function writeFileAtomic(filePath, content) { + const baseName = path.basename(filePath); + const parentDir = path.dirname(filePath); + const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`)); + const tmpFile = path.join(tmpDir, baseName); try { - fs.renameSync(tmpFile, filePath); - } catch (err) { - throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err }); + try { + fs.writeFileSync(tmpFile, content); + } catch (err) { + throw new Error(`failed to write temp file ${tmpFile}`, { cause: err }); + } + try { + fs.renameSync(tmpFile, filePath); + } catch (err) { + throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err }); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); } - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); } - } - - function writeJSON(filePath, value) { - writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n"); - } - function gh(args) { - try { - return execFileSync("gh", args, { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); - } catch { - return null; + function writeJSON(filePath, value) { + writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n"); } - } - function ensureIssueURL(item, repo) { - if (item.url || typeof item.number !== "number" || !repo) { - return item; + function gh(args) { + try { + return execFileSync("gh", args, { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); + } catch { + return null; + } } - return { - ...item, - url: `https://github.com/${repo}/issues/${item.number}`, - }; - } - - function loadRuns() { - const workflowLogs = readJSON(path.join(DATA_DIR, "workflow-logs.json"), {}); - const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : []; - return runs - .map(run => ({ - id: Number(run.id ?? run.databaseId ?? 0), - workflow_name: run.workflow_name || run.workflowName || "", - aic: run.aic ?? null, - created_at: run.created_at || run.createdAt || "", - status: run.status || "", - conclusion: run.conclusion || "", - url: run.html_url || run.url || "", - })) - .filter(run => Number.isInteger(run.id) && run.id > 0); - } - - function loadManifest(runDir) { - const manifestPath = path.join(runDir, "safe-output-items.jsonl"); - if (!fs.existsSync(manifestPath)) return []; - return readJSONL(manifestPath); - } - - function downloadManifest(repo, runId, runDir) { - fs.mkdirSync(runDir, { recursive: true }); - const manifestPath = path.join(runDir, "safe-output-items.jsonl"); - if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) { - return true; + + function ensureIssueURL(item, repo) { + if (item.url || typeof item.number !== "number" || !repo) { + return item; + } + return { + ...item, + url: `https://github.com/${repo}/issues/${item.number}`, + }; } - const result = gh(["run", "download", String(runId), "--repo", repo, "--name", "safe-outputs-items", "--dir", runDir]); - return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0; - } - - function main() { - const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || ""; - if (!repo) { - console.error("EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required"); - process.exit(1); + + function loadRuns() { + const workflowLogs = readJSON(path.join(DATA_DIR, "workflow-logs.json"), {}); + const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : []; + return runs + .map(run => ({ + id: Number(run.id ?? run.databaseId ?? 0), + workflow_name: run.workflow_name || run.workflowName || "", + aic: run.aic ?? null, + created_at: run.created_at || run.createdAt || "", + status: run.status || "", + conclusion: run.conclusion || "", + url: run.html_url || run.url || "", + })) + .filter(run => Number.isInteger(run.id) && run.id > 0); } - fs.mkdirSync(DATA_DIR, { recursive: true }); - fs.mkdirSync(RUNS_DIR, { recursive: true }); + function loadManifest(runDir) { + const manifestPath = path.join(runDir, "safe-output-items.jsonl"); + if (!fs.existsSync(manifestPath)) return []; + return readJSONL(manifestPath); + } - const runs = loadRuns(); - const rows = []; + function downloadManifest(repo, runId, runDir) { + fs.mkdirSync(runDir, { recursive: true }); + const manifestPath = path.join(runDir, "safe-output-items.jsonl"); + if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) { + return true; + } + const result = gh(["run", "download", String(runId), "--repo", repo, "--name", "safe-outputs-items", "--dir", runDir]); + return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0; + } - for (const run of runs) { - const runDir = path.join(RUNS_DIR, `run-${run.id}`); - if (!downloadManifest(repo, run.id, runDir)) { - continue; + function main() { + const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || ""; + if (!repo) { + console.error("EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required"); + process.exit(1); } - const items = loadManifest(runDir) - .filter(item => item && (item.type === "create_issue" || item.type === "close_issue")) - .map(item => ensureIssueURL(item, item.repo || repo)); - - for (const item of items) { - const evalResult = evaluateItem(item, repo); - const normalized = normalizeOutcome(evalResult.result, evalResult.detail); - rows.push({ - run_id: run.id, - workflow_name: run.workflow_name, - workflow_aic: run.aic, - workflow_run_created_at: run.created_at, - workflow_run_url: run.url, - type: item.type, - repo: item.repo || repo, - number: typeof item.number === "number" ? item.number : null, - url: item.url || "", - timestamp: item.timestamp || "", - result: evalResult.result, - detail: evalResult.detail, - outcome_status: normalized.outcome_status, - evidence_strength: normalized.evidence_strength, - signal: normalized.signal, - resolution_sec: evalResult.resolution_sec, - pending_age_sec: evalResult.pending_age_sec, - comments: evalResult.comments, - reactions_total: evalResult.reactions_total, - reactions_positive: evalResult.reactions_positive, - reactions_negative: evalResult.reactions_negative, - zero_touch: evalResult.zero_touch, - }); + fs.mkdirSync(DATA_DIR, { recursive: true }); + fs.mkdirSync(RUNS_DIR, { recursive: true }); + + const runs = loadRuns(); + const rows = []; + + for (const run of runs) { + const runDir = path.join(RUNS_DIR, `run-${run.id}`); + if (!downloadManifest(repo, run.id, runDir)) { + continue; + } + + const items = loadManifest(runDir) + .filter(item => item && (item.type === "create_issue" || item.type === "close_issue")) + .map(item => ensureIssueURL(item, item.repo || repo)); + + for (const item of items) { + const evalResult = evaluateItem(item, repo); + const normalized = normalizeOutcome(evalResult.result, evalResult.detail); + rows.push({ + run_id: run.id, + workflow_name: run.workflow_name, + workflow_aic: run.aic, + workflow_run_created_at: run.created_at, + workflow_run_url: run.url, + type: item.type, + repo: item.repo || repo, + number: typeof item.number === "number" ? item.number : null, + url: item.url || "", + timestamp: item.timestamp || "", + result: evalResult.result, + detail: evalResult.detail, + outcome_status: normalized.outcome_status, + evidence_strength: normalized.evidence_strength, + signal: normalized.signal, + resolution_sec: evalResult.resolution_sec, + pending_age_sec: evalResult.pending_age_sec, + comments: evalResult.comments, + reactions_total: evalResult.reactions_total, + reactions_positive: evalResult.reactions_positive, + reactions_negative: evalResult.reactions_negative, + zero_touch: evalResult.zero_touch, + }); + } } + + writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "")); + + const summary = { + total_issue_outcomes: rows.length, + create_issue_count: rows.filter(row => row.type === "create_issue").length, + close_issue_count: rows.filter(row => row.type === "close_issue").length, + accepted_count: rows.filter(row => row.outcome_status === "accepted").length, + rejected_count: rows.filter(row => row.outcome_status === "rejected").length, + pending_count: rows.filter(row => row.outcome_status === "pending").length, + ignored_count: rows.filter(row => row.outcome_status === "ignored").length, + unknown_count: rows.filter(row => row.outcome_status === "unknown").length, + distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length, + distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length, + }; + writeJSON(OUTPUT_SUMMARY, summary); } - writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join("\n") + (rows.length > 0 ? "\n" : "")); - - const summary = { - total_issue_outcomes: rows.length, - create_issue_count: rows.filter(row => row.type === "create_issue").length, - close_issue_count: rows.filter(row => row.type === "close_issue").length, - accepted_count: rows.filter(row => row.outcome_status === "accepted").length, - rejected_count: rows.filter(row => row.outcome_status === "rejected").length, - pending_count: rows.filter(row => row.outcome_status === "pending").length, - ignored_count: rows.filter(row => row.outcome_status === "ignored").length, - unknown_count: rows.filter(row => row.outcome_status === "unknown").length, - distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length, - distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length, - }; - writeJSON(OUTPUT_SUMMARY, summary); - } - - main(); - NODE + main(); safe-outputs: close-issue: required-title-prefix: "Impact Efficiency Report - " From c50ad3afbc9e1ecf7d1ab5ee2e3c23e589556c4b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:51:55 +0000 Subject: [PATCH 7/7] Drop redundant github-script token input Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/objective-impact-report.lock.yml | 3 +-- .github/workflows/objective-impact-report.md | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 7de5b013041..d8fc637fff3 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d99c0c1b113ac5c44a15f29adf7a8544dbd46b549e4451f4d2f75d29fa0bb1ea","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"60aceebc5dc3a5cd50b3fc16f740d3c9f3b7ee070f14f1e207e7f439558f10ec","body_hash":"3a8f742f0503e76530aef2cf03ccce2ef6cc5a5abe2f66b7aa61025c28ed797e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.22"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.32","digest":"sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -522,7 +522,6 @@ jobs: name: Prepare safe-output issue evaluations uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: "const fs = require(\"fs\");\nconst path = require(\"path\");\nconst { execFileSync } = require(\"child_process\");\nconst {\n evaluateItem,\n normalizeOutcome,\n readJSONL,\n} = require(path.join(process.env.GITHUB_WORKSPACE || process.cwd(), \"actions/setup/js/evaluate_outcomes.cjs\"));\n\nconst DATA_DIR = \"/tmp/gh-aw/agent/objective-impact-report\";\nconst RUNS_DIR = path.join(DATA_DIR, \"safe-output-runs\");\nconst OUTPUT_JSONL = path.join(DATA_DIR, \"safe-output-issue-evaluations.jsonl\");\nconst OUTPUT_SUMMARY = path.join(DATA_DIR, \"safe-output-issue-summary.json\");\n\nfunction readJSON(filePath, fallback) {\n try {\n return JSON.parse(fs.readFileSync(filePath, \"utf8\"));\n } catch {\n return fallback;\n }\n}\n\nfunction writeFileAtomic(filePath, content) {\n const baseName = path.basename(filePath);\n const parentDir = path.dirname(filePath);\n const tmpDir = fs.mkdtempSync(path.join(parentDir, `.${baseName}.tmp-`));\n const tmpFile = path.join(tmpDir, baseName);\n try {\n try {\n fs.writeFileSync(tmpFile, content);\n } catch (err) {\n throw new Error(`failed to write temp file ${tmpFile}`, { cause: err });\n }\n try {\n fs.renameSync(tmpFile, filePath);\n } catch (err) {\n throw new Error(`failed to rename temp file ${tmpFile} to ${filePath}`, { cause: err });\n }\n } finally {\n fs.rmSync(tmpDir, { recursive: true, force: true });\n }\n}\n\nfunction writeJSON(filePath, value) {\n writeFileAtomic(filePath, JSON.stringify(value, null, 2) + \"\\n\");\n}\n\nfunction gh(args) {\n try {\n return execFileSync(\"gh\", args, { encoding: \"utf8\", stdio: [\"pipe\", \"pipe\", \"pipe\"] }).trim();\n } catch {\n return null;\n }\n}\n\nfunction ensureIssueURL(item, repo) {\n if (item.url || typeof item.number !== \"number\" || !repo) {\n return item;\n }\n return {\n ...item,\n url: `https://github.com/${repo}/issues/${item.number}`,\n };\n}\n\nfunction loadRuns() {\n const workflowLogs = readJSON(path.join(DATA_DIR, \"workflow-logs.json\"), {});\n const runs = Array.isArray(workflowLogs.runs) ? workflowLogs.runs : [];\n return runs\n .map(run => ({\n id: Number(run.id ?? run.databaseId ?? 0),\n workflow_name: run.workflow_name || run.workflowName || \"\",\n aic: run.aic ?? null,\n created_at: run.created_at || run.createdAt || \"\",\n status: run.status || \"\",\n conclusion: run.conclusion || \"\",\n url: run.html_url || run.url || \"\",\n }))\n .filter(run => Number.isInteger(run.id) && run.id > 0);\n}\n\nfunction loadManifest(runDir) {\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (!fs.existsSync(manifestPath)) return [];\n return readJSONL(manifestPath);\n}\n\nfunction downloadManifest(repo, runId, runDir) {\n fs.mkdirSync(runDir, { recursive: true });\n const manifestPath = path.join(runDir, \"safe-output-items.jsonl\");\n if (fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0) {\n return true;\n }\n const result = gh([\"run\", \"download\", String(runId), \"--repo\", repo, \"--name\", \"safe-outputs-items\", \"--dir\", runDir]);\n return result !== null && fs.existsSync(manifestPath) && fs.statSync(manifestPath).size > 0;\n}\n\nfunction main() {\n const repo = process.env.EXPR_GITHUB_REPOSITORY || process.env.GITHUB_REPOSITORY || \"\";\n if (!repo) {\n console.error(\"EXPR_GITHUB_REPOSITORY or GITHUB_REPOSITORY is required\");\n process.exit(1);\n }\n\n fs.mkdirSync(DATA_DIR, { recursive: true });\n fs.mkdirSync(RUNS_DIR, { recursive: true });\n\n const runs = loadRuns();\n const rows = [];\n\n for (const run of runs) {\n const runDir = path.join(RUNS_DIR, `run-${run.id}`);\n if (!downloadManifest(repo, run.id, runDir)) {\n continue;\n }\n\n const items = loadManifest(runDir)\n .filter(item => item && (item.type === \"create_issue\" || item.type === \"close_issue\"))\n .map(item => ensureIssueURL(item, item.repo || repo));\n\n for (const item of items) {\n const evalResult = evaluateItem(item, repo);\n const normalized = normalizeOutcome(evalResult.result, evalResult.detail);\n rows.push({\n run_id: run.id,\n workflow_name: run.workflow_name,\n workflow_aic: run.aic,\n workflow_run_created_at: run.created_at,\n workflow_run_url: run.url,\n type: item.type,\n repo: item.repo || repo,\n number: typeof item.number === \"number\" ? item.number : null,\n url: item.url || \"\",\n timestamp: item.timestamp || \"\",\n result: evalResult.result,\n detail: evalResult.detail,\n outcome_status: normalized.outcome_status,\n evidence_strength: normalized.evidence_strength,\n signal: normalized.signal,\n resolution_sec: evalResult.resolution_sec,\n pending_age_sec: evalResult.pending_age_sec,\n comments: evalResult.comments,\n reactions_total: evalResult.reactions_total,\n reactions_positive: evalResult.reactions_positive,\n reactions_negative: evalResult.reactions_negative,\n zero_touch: evalResult.zero_touch,\n });\n }\n }\n\n writeFileAtomic(OUTPUT_JSONL, rows.map(row => JSON.stringify(row)).join(\"\\n\") + (rows.length > 0 ? \"\\n\" : \"\"));\n\n const summary = {\n total_issue_outcomes: rows.length,\n create_issue_count: rows.filter(row => row.type === \"create_issue\").length,\n close_issue_count: rows.filter(row => row.type === \"close_issue\").length,\n accepted_count: rows.filter(row => row.outcome_status === \"accepted\").length,\n rejected_count: rows.filter(row => row.outcome_status === \"rejected\").length,\n pending_count: rows.filter(row => row.outcome_status === \"pending\").length,\n ignored_count: rows.filter(row => row.outcome_status === \"ignored\").length,\n unknown_count: rows.filter(row => row.outcome_status === \"unknown\").length,\n distinct_workflows: [...new Set(rows.map(row => row.workflow_name).filter(Boolean))].length,\n distinct_runs_with_issue_outcomes: [...new Set(rows.map(row => row.run_id))].length,\n };\n writeJSON(OUTPUT_SUMMARY, summary);\n}\n\nmain();\n" - name: Download container images diff --git a/.github/workflows/objective-impact-report.md b/.github/workflows/objective-impact-report.md index e12ec2ea9e5..5829b7f0ab3 100644 --- a/.github/workflows/objective-impact-report.md +++ b/.github/workflows/objective-impact-report.md @@ -46,7 +46,6 @@ pre-agent-steps: EXPR_GITHUB_REPOSITORY: ${{ github.repository }} uses: actions/github-script@v9.0.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require("fs"); const path = require("path");