From d9201c36ecfcfd33eefeeddfe30fca0d81470e66 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 11:55:57 +0000
Subject: [PATCH 1/7] feat: add A/B experiments to frontmatter with runtime
 variant selection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add `experiments` section to frontmatter schema (map of name → variant list)
- Add `Experiments map[string][]string` to FrontmatterConfig and WorkflowData
- Transform `${{ experiments.name }}` expressions to `${{ env.GH_AW_EXPERIMENTS_NAME }}`
  via new `transformExperimentsExpression()` in expression_extraction.go
- Generate activation-job steps: restore cache → pick variants (pick_experiment.cjs) →
  save cache → upload "experiment" artifact
- Add pick_experiment.cjs: balanced round-robin variant selection using actions/cache
  state file, GITHUB_ENV export, and Markdown step summary
- Add `ArtifactSetExperiment` (\"experiment\") to CLI artifact sets for logs/audit
- Add ExperimentArtifactName constant
- Full test coverage: Go unit tests and vitest JS tests (16 passing)"

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/6603825b-d8d5-4381-a44c-3d6fd823ab92

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 actions/setup/js/pick_experiment.cjs          | 187 +++++++++++++++++
 actions/setup/js/pick_experiment.test.cjs     | 198 ++++++++++++++++++
 pkg/cli/logs_artifact_set.go                  |   6 +
 pkg/cli/logs_artifact_set_test.go             |   2 +-
 pkg/constants/job_constants.go                |   4 +
 pkg/parser/schemas/main_workflow_schema.json  |  19 ++
 pkg/workflow/compiler_activation_job.go       |   8 +
 pkg/workflow/compiler_experiments.go          | 180 ++++++++++++++++
 pkg/workflow/compiler_experiments_test.go     | 164 +++++++++++++++
 .../compiler_orchestrator_workflow.go         |   3 +
 pkg/workflow/compiler_types.go                |   1 +
 pkg/workflow/expression_extraction.go         |  29 +++
 pkg/workflow/frontmatter_types.go             |   5 +
 13 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 actions/setup/js/pick_experiment.cjs
 create mode 100644 actions/setup/js/pick_experiment.test.cjs
 create mode 100644 pkg/workflow/compiler_experiments.go
 create mode 100644 pkg/workflow/compiler_experiments_test.go

diff --git a/actions/setup/js/pick_experiment.cjs b/actions/setup/js/pick_experiment.cjs
new file mode 100644
index 00000000000..925b5a26da2
--- /dev/null
+++ b/actions/setup/js/pick_experiment.cjs
@@ -0,0 +1,187 @@
+// @ts-check
+/// <reference types="@actions/github-script" />
+
+/**
+ * pick_experiment
+ *
+ * Selects A/B experiment variants for the current workflow run.
+ *
+ * Environment variables (set by the compiled workflow step):
+ *   GH_AW_EXPERIMENT_SPEC       - JSON object mapping experiment name → array of variant strings
+ *                                  e.g. '{"feature1":["A","B"],"style":["concise","detailed"]}'
+ *   GH_AW_EXPERIMENT_STATE_FILE - Absolute path to the JSON state file to read/write
+ *                                  e.g. /tmp/gh-aw/experiments/state.json
+ *   GH_AW_EXPERIMENT_STATE_DIR  - Directory that holds the state file (created if missing)
+ *                                  e.g. /tmp/gh-aw/experiments
+ *
+ * Algorithm:
+ *   For each experiment the function maintains a counter per variant in the state file.
+ *   The variant with the lowest invocation count is selected next (ties are broken by
+ *   variant order, yielding a deterministic round-robin across runs).
+ *   This ensures that across N runs every variant is used approximately N/K times where
+ *   K is the number of variants, satisfying basic A/B statistical balance.
+ *
+ * Outputs:
+ *   - Sets GH_AW_EXPERIMENTS_<NAME>=<variant> in GITHUB_ENV for each experiment.
+ *   - Writes the updated counter state back to GH_AW_EXPERIMENT_STATE_FILE.
+ *   - Appends a Markdown step summary with the assignment table and cumulative counts.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * @typedef {Object} ExperimentState
+ * @property {Record<string, Record<string, number>>} counts
+ *   Maps experiment name → variant → cumulative invocation count.
+ */
+
+/**
+ * Load and parse the state JSON file.  Returns an empty state if the file does not exist
+ * or cannot be parsed (e.g. first run or corrupted cache).
+ *
+ * @param {string} stateFile
+ * @returns {ExperimentState}
+ */
+function loadState(stateFile) {
+  try {
+    const raw = fs.readFileSync(stateFile, "utf8");
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed.counts === "object") {
+      return parsed;
+    }
+  } catch {
+    // File missing, unreadable, or invalid JSON – start fresh.
+  }
+  return { counts: {} };
+}
+
+/**
+ * Persist the state JSON file to disk.
+ *
+ * @param {string} stateFile
+ * @param {ExperimentState} state
+ */
+function saveState(stateFile, state) {
+  const dir = path.dirname(stateFile);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(stateFile, JSON.stringify(state, null, 2) + "\n", "utf8");
+}
+
+/**
+ * Pick the variant for one experiment using a balanced least-used selection.
+ * The variant with the lowest cumulative count is chosen; ties are broken by
+ * the order of the variants array so selection is deterministic.
+ *
+ * @param {string} name       - Experiment name
+ * @param {string[]} variants - Array of variant values (length >= 2)
+ * @param {ExperimentState} state
+ * @returns {string} The selected variant
+ */
+function pickVariant(name, variants, state) {
+  const counts = state.counts[name] || {};
+  let minCount = Infinity;
+  let selected = variants[0];
+  for (const variant of variants) {
+    const c = counts[variant] || 0;
+    if (c < minCount) {
+      minCount = c;
+      selected = variant;
+    }
+  }
+  return selected;
+}
+
+/**
+ * Increment the counter for the chosen variant.
+ *
+ * @param {string} name    - Experiment name
+ * @param {string} variant - Chosen variant
+ * @param {ExperimentState} state
+ */
+function recordVariant(name, variant, state) {
+  if (!state.counts[name]) {
+    state.counts[name] = {};
+  }
+  state.counts[name][variant] = (state.counts[name][variant] || 0) + 1;
+}
+
+/**
+ * Append a Markdown step summary describing the experiment assignments.
+ *
+ * @param {Record<string, string>} assignments  - Maps experiment name → selected variant
+ * @param {Record<string, string[]>} spec       - Maps experiment name → variants array
+ * @param {ExperimentState} state               - Updated state (post-selection)
+ * @param {any} core                            - @actions/core
+ */
+async function writeSummary(assignments, spec, state, core) {
+  const names = Object.keys(assignments).sort();
+  const lines = ["## 🧪 A/B Experiment Assignments", "", "| Experiment | Selected Variant | All Variants | Cumulative Counts |", "| --- | --- | --- | --- |"];
+  for (const name of names) {
+    const selected = assignments[name];
+    const variants = spec[name] || [];
+    const counts = state.counts[name] || {};
+    const countsStr = variants.map(v => `${v}: ${counts[v] || 0}`).join(", ");
+    lines.push(`| \`${name}\` | **${selected}** | ${variants.join(", ")} | ${countsStr} |`);
+  }
+  lines.push("");
+  lines.push("_Variants are selected by balanced round-robin to ensure statistical relevance across runs._");
+  await core.summary.addRaw(lines.join("\n")).write();
+}
+
+/**
+ * Main entry point called by the actions/github-script step.
+ */
+async function main() {
+  const specRaw = process.env.GH_AW_EXPERIMENT_SPEC || "{}";
+  const stateFile = process.env.GH_AW_EXPERIMENT_STATE_FILE || "/tmp/gh-aw/experiments/state.json";
+  const stateDir = process.env.GH_AW_EXPERIMENT_STATE_DIR || "/tmp/gh-aw/experiments";
+
+  /** @type {Record<string, string[]>} */
+  let spec;
+  try {
+    spec = JSON.parse(specRaw);
+  } catch (e) {
+    core.setFailed(`Failed to parse GH_AW_EXPERIMENT_SPEC: ${e.message}`);
+    return;
+  }
+
+  const experimentNames = Object.keys(spec).sort();
+  if (experimentNames.length === 0) {
+    core.info("No experiments defined – nothing to do.");
+    return;
+  }
+
+  // Ensure the state directory exists so that the cache-save step can find it.
+  fs.mkdirSync(stateDir, { recursive: true });
+
+  const state = loadState(stateFile);
+
+  /** @type {Record<string, string>} */
+  const assignments = {};
+
+  for (const name of experimentNames) {
+    const variants = spec[name];
+    if (!Array.isArray(variants) || variants.length < 2) {
+      core.warning(`Experiment "${name}" has fewer than 2 variants – skipping.`);
+      continue;
+    }
+    const selected = pickVariant(name, variants, state);
+    recordVariant(name, selected, state);
+    assignments[name] = selected;
+
+    // Export the selected variant so subsequent steps can read it via $GITHUB_ENV.
+    const envVarName = `GH_AW_EXPERIMENTS_${name.toUpperCase().replace(/[^A-Z0-9]/g, "_")}`;
+    core.exportVariable(envVarName, selected);
+    core.info(`Experiment "${name}": selected variant "${selected}" → ${envVarName}=${selected}`);
+  }
+
+  // Persist updated counts.
+  saveState(stateFile, state);
+  core.info(`Experiment state written to ${stateFile}`);
+
+  // Write step summary.
+  await writeSummary(assignments, spec, state, core);
+}
+
+module.exports = { main, pickVariant, loadState, saveState, recordVariant };
diff --git a/actions/setup/js/pick_experiment.test.cjs b/actions/setup/js/pick_experiment.test.cjs
new file mode 100644
index 00000000000..d69345a6d5f
--- /dev/null
+++ b/actions/setup/js/pick_experiment.test.cjs
@@ -0,0 +1,198 @@
+// @ts-check
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import fs from "fs";
+import path from "path";
+import os from "os";
+
+// Mock @actions/core globals
+const mockCore = {
+  info: vi.fn(),
+  warning: vi.fn(),
+  error: vi.fn(),
+  setFailed: vi.fn(),
+  exportVariable: vi.fn(),
+  summary: {
+    addRaw: vi.fn().mockReturnThis(),
+    write: vi.fn().mockResolvedValue(undefined),
+  },
+};
+
+global.core = mockCore;
+
+const { pickVariant, loadState, saveState, recordVariant, main } = await import("./pick_experiment.cjs");
+
+describe("pick_experiment", () => {
+  /** @type {string} */
+  let tmpDir;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "pick_experiment_test_"));
+  });
+
+  // ── pickVariant ────────────────────────────────────────────────────────────
+
+  describe("pickVariant", () => {
+    it("selects the first variant when counts are equal", () => {
+      const state = { counts: {} };
+      expect(pickVariant("f", ["A", "B"], state)).toBe("A");
+    });
+
+    it("selects the least-used variant", () => {
+      const state = { counts: { f: { A: 3, B: 1 } } };
+      expect(pickVariant("f", ["A", "B"], state)).toBe("B");
+    });
+
+    it("handles three variants in round-robin fashion", () => {
+      const state = { counts: { f: { A: 2, B: 2, C: 1 } } };
+      expect(pickVariant("f", ["A", "B", "C"], state)).toBe("C");
+    });
+
+    it("returns the first variant when all counts are equal (tie-break by order)", () => {
+      const state = { counts: { f: { A: 1, B: 1, C: 1 } } };
+      expect(pickVariant("f", ["A", "B", "C"], state)).toBe("A");
+    });
+
+    it("handles unknown experiment name (no counts yet)", () => {
+      const state = { counts: {} };
+      expect(pickVariant("new", ["X", "Y"], state)).toBe("X");
+    });
+  });
+
+  // ── recordVariant ──────────────────────────────────────────────────────────
+
+  describe("recordVariant", () => {
+    it("increments the variant counter", () => {
+      const state = { counts: {} };
+      recordVariant("f", "A", state);
+      expect(state.counts["f"]["A"]).toBe(1);
+    });
+
+    it("accumulates counts across multiple calls", () => {
+      const state = { counts: {} };
+      recordVariant("f", "A", state);
+      recordVariant("f", "A", state);
+      recordVariant("f", "B", state);
+      expect(state.counts["f"]["A"]).toBe(2);
+      expect(state.counts["f"]["B"]).toBe(1);
+    });
+  });
+
+  // ── loadState / saveState ──────────────────────────────────────────────────
+
+  describe("loadState", () => {
+    it("returns empty state when file does not exist", () => {
+      const state = loadState(path.join(tmpDir, "nonexistent.json"));
+      expect(state).toEqual({ counts: {} });
+    });
+
+    it("returns empty state on invalid JSON", () => {
+      const file = path.join(tmpDir, "bad.json");
+      fs.writeFileSync(file, "not valid json");
+      const state = loadState(file);
+      expect(state).toEqual({ counts: {} });
+    });
+
+    it("round-trips state through save and load", () => {
+      const file = path.join(tmpDir, "state.json");
+      const orig = { counts: { f: { A: 3, B: 1 } } };
+      saveState(file, orig);
+      const loaded = loadState(file);
+      expect(loaded).toEqual(orig);
+    });
+  });
+
+  // ── statistical balance ────────────────────────────────────────────────────
+
+  describe("statistical balance", () => {
+    it("distributes two variants evenly across 10 runs", () => {
+      const state = { counts: {} };
+      const selections = [];
+      for (let i = 0; i < 10; i++) {
+        const v = pickVariant("f", ["A", "B"], state);
+        recordVariant("f", v, state);
+        selections.push(v);
+      }
+      const countA = selections.filter(v => v === "A").length;
+      const countB = selections.filter(v => v === "B").length;
+      expect(countA).toBe(5);
+      expect(countB).toBe(5);
+    });
+
+    it("distributes three variants evenly across 9 runs", () => {
+      const state = { counts: {} };
+      const selections = [];
+      for (let i = 0; i < 9; i++) {
+        const v = pickVariant("f", ["A", "B", "C"], state);
+        recordVariant("f", v, state);
+        selections.push(v);
+      }
+      const countA = selections.filter(v => v === "A").length;
+      const countB = selections.filter(v => v === "B").length;
+      const countC = selections.filter(v => v === "C").length;
+      expect(countA).toBe(3);
+      expect(countB).toBe(3);
+      expect(countC).toBe(3);
+    });
+  });
+
+  // ── main ───────────────────────────────────────────────────────────────────
+
+  describe("main", () => {
+    it("exports GITHUB_ENV variables for each experiment", async () => {
+      const stateFile = path.join(tmpDir, "state.json");
+      process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({
+        feature1: ["A", "B"],
+      });
+      process.env.GH_AW_EXPERIMENT_STATE_FILE = stateFile;
+      process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir;
+
+      await main();
+
+      expect(mockCore.exportVariable).toHaveBeenCalledWith("GH_AW_EXPERIMENTS_FEATURE1", "A");
+      expect(mockCore.setFailed).not.toHaveBeenCalled();
+    });
+
+    it("persists state between calls to simulate multi-run balance", async () => {
+      const stateFile = path.join(tmpDir, "state.json");
+      process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({
+        feat: ["X", "Y"],
+      });
+      process.env.GH_AW_EXPERIMENT_STATE_FILE = stateFile;
+      process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir;
+
+      // First run → X
+      await main();
+      const firstCall = mockCore.exportVariable.mock.calls.find(c => c[0] === "GH_AW_EXPERIMENTS_FEAT");
+      expect(firstCall?.[1]).toBe("X");
+
+      vi.clearAllMocks();
+
+      // Second run → Y (state persisted from first call)
+      await main();
+      const secondCall = mockCore.exportVariable.mock.calls.find(c => c[0] === "GH_AW_EXPERIMENTS_FEAT");
+      expect(secondCall?.[1]).toBe("Y");
+    });
+
+    it("does nothing when spec is empty", async () => {
+      process.env.GH_AW_EXPERIMENT_SPEC = "{}";
+      process.env.GH_AW_EXPERIMENT_STATE_FILE = path.join(tmpDir, "state.json");
+      process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir;
+
+      await main();
+
+      expect(mockCore.exportVariable).not.toHaveBeenCalled();
+      expect(mockCore.setFailed).not.toHaveBeenCalled();
+    });
+
+    it("calls setFailed on invalid JSON spec", async () => {
+      process.env.GH_AW_EXPERIMENT_SPEC = "not-json";
+      process.env.GH_AW_EXPERIMENT_STATE_FILE = path.join(tmpDir, "state.json");
+      process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir;
+
+      await main();
+
+      expect(mockCore.setFailed).toHaveBeenCalled();
+    });
+  });
+});
diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go
index 7196ea50662..0fc11837ca8 100644
--- a/pkg/cli/logs_artifact_set.go
+++ b/pkg/cli/logs_artifact_set.go
@@ -58,6 +58,10 @@ const (
 	// logs (github_rate_limits.jsonl), which are included in both the activation and
 	// agent artifacts.
 	ArtifactSetGitHubAPI ArtifactSet = "github-api"
+
+	// ArtifactSetExperiment downloads the experiment artifact containing A/B experiment
+	// state (state.json) uploaded by the activation job when experiments are declared.
+	ArtifactSetExperiment ArtifactSet = "experiment"
 )
 
 // artifactSetArtifacts maps each named set to the list of artifact base names it includes.
@@ -73,6 +77,8 @@ var artifactSetArtifacts = map[ArtifactSet][]string{
 	ArtifactSetDetection:  {constants.DetectionArtifactName},
 	// github-api: both jobs upload github_rate_limits.jsonl; fetch both for a complete view.
 	ArtifactSetGitHubAPI: {constants.ActivationArtifactName, constants.AgentArtifactName},
+	// experiment: A/B experiment state uploaded by the activation job.
+	ArtifactSetExperiment: {constants.ExperimentArtifactName},
 }
 
 // ValidArtifactSetNames returns a sorted list of valid artifact set names,
diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go
index e3ad282e020..2d67263852d 100644
--- a/pkg/cli/logs_artifact_set_test.go
+++ b/pkg/cli/logs_artifact_set_test.go
@@ -242,7 +242,7 @@ func TestValidArtifactSetNames(t *testing.T) {
 	names := ValidArtifactSetNames()
 	require.NotEmpty(t, names, "ValidArtifactSetNames should return non-empty slice")
 
-	expected := []string{"all", "activation", "agent", "detection", "firewall", "github-api", "mcp"}
+	expected := []string{"all", "activation", "agent", "detection", "experiment", "firewall", "github-api", "mcp"}
 	assert.ElementsMatch(t, expected, names, "ValidArtifactSetNames should contain all known sets")
 }
 
diff --git a/pkg/constants/job_constants.go b/pkg/constants/job_constants.go
index bcf39443322..1a65abf7e7f 100644
--- a/pkg/constants/job_constants.go
+++ b/pkg/constants/job_constants.go
@@ -133,6 +133,10 @@ const ArtifactPrefixOutputName = "artifact_prefix"
 // (aw_info.json and prompt.txt).
 const ActivationArtifactName = "activation"
 
+// ExperimentArtifactName is the artifact name for A/B experiment state
+// uploaded by the activation job when experiments are declared in the frontmatter.
+const ExperimentArtifactName = "experiment"
+
 // SafeOutputItemsArtifactName is the artifact name for the safe output items manifest.
 // This artifact contains the JSONL manifest of all items created by safe output handlers
 // and is uploaded by the safe_outputs job to avoid conflicting with the "agent" artifact
diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json
index 5eab3b20bd1..1a225a47e41 100644
--- a/pkg/parser/schemas/main_workflow_schema.json
+++ b/pkg/parser/schemas/main_workflow_schema.json
@@ -2730,6 +2730,25 @@
         }
       ]
     },
+    "experiments": {
+      "description": "A/B testing experiments. Each key is an experiment name; the value is an array of two or more variant strings. At runtime the activation job picks a variant using actions/cache to maintain consistent assignment across runs. Use ${{ experiments.<name> }} in the workflow prompt to reference the selected variant. When multiple experiments are declared, assignments are statistically balanced using a counter that round-robins across variants.",
+      "type": "object",
+      "additionalProperties": {
+        "type": "array",
+        "items": { "type": "string" },
+        "minItems": 2,
+        "description": "Array of variant values for this experiment. Must contain at least two variants."
+      },
+      "examples": [
+        {
+          "feature1": ["A", "B"]
+        },
+        {
+          "prompt_style": ["concise", "detailed", "structured"],
+          "model_temp": ["low", "high"]
+        }
+      ]
+    },
     "infer": {
       "type": "boolean",
       "description": "DEPRECATED: Use 'disable-model-invocation' instead. Controls whether the custom agent should infer additional context from the conversation. This field is maintained for backward compatibility with existing custom agent files.",
diff --git a/pkg/workflow/compiler_activation_job.go b/pkg/workflow/compiler_activation_job.go
index dec20f84efd..98ca0179781 100644
--- a/pkg/workflow/compiler_activation_job.go
+++ b/pkg/workflow/compiler_activation_job.go
@@ -40,6 +40,14 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate
 		return nil, err
 	}
 
+	// Generate experiment selection steps when experiments are declared in the frontmatter.
+	// These steps run before the prompt is built so that ${{ experiments.name }} placeholders
+	// can be resolved by the substitute_placeholders step.
+	if experimentSteps := c.generateExperimentSteps(data); len(experimentSteps) > 0 {
+		compilerActivationJobLog.Printf("Adding %d experiment step(s) for %d experiment(s)", len(experimentSteps), len(data.Experiments))
+		ctx.steps = append(ctx.steps, experimentSteps...)
+	}
+
 	c.configureActivationNeedsAndCondition(ctx)
 	compilerActivationJobLog.Print("Generating prompt in activation job")
 	c.generatePromptInActivationJob(&ctx.steps, data, preActivationJobCreated, ctx.customJobsBeforeActivation)
diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go
new file mode 100644
index 00000000000..1a2dd0cd45d
--- /dev/null
+++ b/pkg/workflow/compiler_experiments.go
@@ -0,0 +1,180 @@
+package workflow
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/github/gh-aw/pkg/constants"
+	"github.com/github/gh-aw/pkg/logger"
+)
+
+var experimentsLog = logger.New("workflow:compiler_experiments")
+
+// experimentsCacheDir is the runtime directory where the experiment state JSON is stored.
+const experimentsCacheDir = "/tmp/gh-aw/experiments"
+
+// experimentStateFile is the path to the experiment state JSON written by pick_experiment.cjs.
+const experimentStateFile = experimentsCacheDir + "/state.json"
+
+// extractExperimentsFromFrontmatter reads the "experiments" map from a raw frontmatter map.
+// Each key is an experiment name; each value must be a []string (or []any of strings) of
+// variant values.  Invalid entries are silently skipped.
+func extractExperimentsFromFrontmatter(frontmatter map[string]any) map[string][]string {
+	raw, ok := frontmatter["experiments"]
+	if !ok || raw == nil {
+		return nil
+	}
+	rawMap, ok := raw.(map[string]any)
+	if !ok {
+		return nil
+	}
+	result := make(map[string][]string, len(rawMap))
+	for name, val := range rawMap {
+		switch v := val.(type) {
+		case []string:
+			if len(v) >= 2 {
+				result[name] = v
+			}
+		case []any:
+			var variants []string
+			for _, item := range v {
+				if s, ok := item.(string); ok {
+					variants = append(variants, s)
+				}
+			}
+			if len(variants) >= 2 {
+				result[name] = variants
+			}
+		}
+	}
+	if len(result) == 0 {
+		return nil
+	}
+	return result
+}
+
+// generateExperimentSteps creates the steps that pick and upload A/B experiment variants.
+//
+// Steps generated (only when experiments are declared):
+//  1. Restore experiment cache   – actions/cache/restore keyed by workflow ID
+//  2. Pick variants              – pick_experiment.cjs (reads/writes state.json, sets GITHUB_ENV)
+//  3. Experiment step summary    – write a Markdown summary to GITHUB_STEP_SUMMARY
+//  4. Save experiment cache      – actions/cache/save keyed by workflow ID
+//  5. Upload experiment artifact – actions/upload-artifact named "experiment"
+func (c *Compiler) generateExperimentSteps(data *WorkflowData) []string {
+	if len(data.Experiments) == 0 {
+		return nil
+	}
+
+	experimentNames := sortedExperimentNames(data.Experiments)
+	experimentsLog.Printf("Generating experiment steps for %d experiment(s): %v", len(experimentNames), experimentNames)
+
+	cacheKey := "experiments-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }}"
+	restoreKey := "experiments-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-"
+
+	var steps []string
+
+	// ── Step 1: Restore experiment cache ──────────────────────────────────────
+	steps = append(steps,
+		"      - name: Restore experiment state\n",
+		"        id: restore-experiment-cache\n",
+		fmt.Sprintf("        uses: %s\n", getActionPin("actions/cache/restore")),
+		"        with:\n",
+		fmt.Sprintf("          key: %s\n", cacheKey),
+		fmt.Sprintf("          restore-keys: %s\n", restoreKey),
+		fmt.Sprintf("          path: %s\n", experimentsCacheDir),
+	)
+
+	// ── Step 2: Pick experiment variants ──────────────────────────────────────
+	// Build the JSON spec: {"feature1":["A","B"],...}
+	specJSON := buildExperimentSpecJSON(data.Experiments, experimentNames)
+
+	steps = append(steps,
+		"      - name: Pick experiment variants\n",
+		"        id: pick-experiment\n",
+		fmt.Sprintf("        uses: %s\n", getCachedActionPin("actions/github-script", data)),
+		"        env:\n",
+		fmt.Sprintf("          GH_AW_EXPERIMENT_SPEC: '%s'\n", specJSON),
+		fmt.Sprintf("          GH_AW_EXPERIMENT_STATE_FILE: %s\n", experimentStateFile),
+		fmt.Sprintf("          GH_AW_EXPERIMENT_STATE_DIR: %s\n", experimentsCacheDir),
+		"        with:\n",
+		"          script: |\n",
+		"            const { setupGlobals } = require('"+SetupActionDestination+"/setup_globals.cjs');\n",
+		"            setupGlobals(core, github, context, exec, io, getOctokit);\n",
+		"            const { main } = require('"+SetupActionDestination+"/pick_experiment.cjs');\n",
+		"            await main();\n",
+	)
+
+	// ── Step 3: Save experiment cache ─────────────────────────────────────────
+	steps = append(steps,
+		"      - name: Save experiment state\n",
+		"        if: always()\n",
+		fmt.Sprintf("        uses: %s\n", getActionPin("actions/cache/save")),
+		"        with:\n",
+		fmt.Sprintf("          key: %s\n", cacheKey),
+		fmt.Sprintf("          path: %s\n", experimentsCacheDir),
+	)
+
+	// ── Step 4: Upload experiment artifact ────────────────────────────────────
+	experimentArtifactName := artifactPrefixExprForActivationJob(data) + constants.ExperimentArtifactName
+	steps = append(steps,
+		"      - name: Upload experiment artifact\n",
+		"        if: always()\n",
+		fmt.Sprintf("        uses: %s\n", getActionPin("actions/upload-artifact")),
+		"        with:\n",
+		fmt.Sprintf("          name: %s\n", experimentArtifactName),
+		fmt.Sprintf("          path: %s\n", experimentsCacheDir),
+		"          if-no-files-found: ignore\n",
+		"          retention-days: 30\n",
+	)
+
+	return steps
+}
+
+// buildExperimentSpecJSON builds a compact JSON object from the experiments map.
+// The JSON is safe to embed in a YAML single-quoted scalar (no single quotes in output).
+func buildExperimentSpecJSON(experiments map[string][]string, names []string) string {
+	var sb strings.Builder
+	sb.WriteString("{")
+	for i, name := range names {
+		if i > 0 {
+			sb.WriteString(",")
+		}
+		sb.WriteString("\"")
+		sb.WriteString(escapeJSONString(name))
+		sb.WriteString("\":[")
+		variants := experiments[name]
+		for j, v := range variants {
+			if j > 0 {
+				sb.WriteString(",")
+			}
+			sb.WriteString("\"")
+			sb.WriteString(escapeJSONString(v))
+			sb.WriteString("\"")
+		}
+		sb.WriteString("]")
+	}
+	sb.WriteString("}")
+	return sb.String()
+}
+
+// escapeJSONString escapes a string for embedding in a JSON value.
+func escapeJSONString(s string) string {
+	s = strings.ReplaceAll(s, `\`, `\\`)
+	s = strings.ReplaceAll(s, `"`, `\"`)
+	s = strings.ReplaceAll(s, "\n", `\n`)
+	s = strings.ReplaceAll(s, "\r", `\r`)
+	s = strings.ReplaceAll(s, "\t", `\t`)
+	return s
+}
+
+// sortedExperimentNames returns the experiment names in sorted order for deterministic output.
+func sortedExperimentNames(experiments map[string][]string) []string {
+	names := make([]string, 0, len(experiments))
+	for name := range experiments {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return names
+}
diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go
new file mode 100644
index 00000000000..e78cbae95b0
--- /dev/null
+++ b/pkg/workflow/compiler_experiments_test.go
@@ -0,0 +1,164 @@
+//go:build !integration
+
+package workflow
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// ── extractExperimentsFromFrontmatter ─────────────────────────────────────
+
+func TestExtractExperimentsFromFrontmatter(t *testing.T) {
+	tests := []struct {
+		name        string
+		frontmatter map[string]any
+		want        map[string][]string
+	}{
+		{
+			name:        "nil frontmatter returns nil",
+			frontmatter: map[string]any{},
+			want:        nil,
+		},
+		{
+			name: "basic two-variant experiment",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"feature1": []any{"A", "B"},
+				},
+			},
+			want: map[string][]string{"feature1": {"A", "B"}},
+		},
+		{
+			name: "three variants",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"style": []any{"concise", "detailed", "structured"},
+				},
+			},
+			want: map[string][]string{"style": {"concise", "detailed", "structured"}},
+		},
+		{
+			name: "skips experiment with fewer than two variants",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"bad":  []any{"only-one"},
+					"good": []any{"A", "B"},
+				},
+			},
+			want: map[string][]string{"good": {"A", "B"}},
+		},
+		{
+			name: "multiple experiments",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"feat1": []any{"X", "Y"},
+					"feat2": []any{"P", "Q", "R"},
+				},
+			},
+			want: map[string][]string{
+				"feat1": {"X", "Y"},
+				"feat2": {"P", "Q", "R"},
+			},
+		},
+		{
+			name: "returns nil when experiments map is empty",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{},
+			},
+			want: nil,
+		},
+		{
+			name: "handles native []string slice",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"feature1": []string{"A", "B"},
+				},
+			},
+			want: map[string][]string{"feature1": {"A", "B"}},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := extractExperimentsFromFrontmatter(tt.frontmatter)
+			assert.Equal(t, tt.want, got, "extracted experiments should match")
+		})
+	}
+}
+
+// ── sortedExperimentNames ─────────────────────────────────────────────────
+
+func TestSortedExperimentNames(t *testing.T) {
+	experiments := map[string][]string{
+		"z_exp": {"A", "B"},
+		"a_exp": {"X", "Y"},
+		"m_exp": {"P", "Q"},
+	}
+	got := sortedExperimentNames(experiments)
+	require.Equal(t, []string{"a_exp", "m_exp", "z_exp"}, got, "names should be sorted alphabetically")
+}
+
+// ── buildExperimentSpecJSON ───────────────────────────────────────────────
+
+func TestBuildExperimentSpecJSON(t *testing.T) {
+	experiments := map[string][]string{
+		"feature1": {"A", "B"},
+		"style":    {"concise", "detailed"},
+	}
+	names := []string{"feature1", "style"}
+	got := buildExperimentSpecJSON(experiments, names)
+	assert.JSONEq(t, `{"feature1":["A","B"],"style":["concise","detailed"]}`, got, "JSON spec should match expected structure")
+}
+
+func TestBuildExperimentSpecJSONEscaping(t *testing.T) {
+	experiments := map[string][]string{
+		`quote"test`: {`val"1`, `val\2`},
+	}
+	names := []string{`quote"test`}
+	got := buildExperimentSpecJSON(experiments, names)
+	assert.Contains(t, got, `\"`, "double quotes should be escaped in JSON")
+}
+
+// ── generateExperimentSteps ───────────────────────────────────────────────
+
+func TestGenerateExperimentSteps_Empty(t *testing.T) {
+	c := &Compiler{}
+	data := &WorkflowData{}
+	steps := c.generateExperimentSteps(data)
+	assert.Empty(t, steps, "no steps should be generated when experiments is nil")
+}
+
+func TestGenerateExperimentSteps_Generated(t *testing.T) {
+	c := &Compiler{}
+	data := &WorkflowData{
+		Experiments: map[string][]string{
+			"feature1": {"A", "B"},
+		},
+	}
+	steps := c.generateExperimentSteps(data)
+	require.NotEmpty(t, steps, "steps should be generated when experiments are declared")
+
+	joined := strings.Join(steps, "")
+	assert.Contains(t, joined, "Restore experiment state", "should include cache restore step")
+	assert.Contains(t, joined, "Pick experiment variants", "should include pick step")
+	assert.Contains(t, joined, "pick_experiment.cjs", "should reference pick_experiment.cjs")
+	assert.Contains(t, joined, "Save experiment state", "should include cache save step")
+	assert.Contains(t, joined, "Upload experiment artifact", "should include artifact upload step")
+	assert.Contains(t, joined, "experiment", "artifact name should include 'experiment'")
+}
+
+func TestGenerateExperimentSteps_SpecJSON(t *testing.T) {
+	c := &Compiler{}
+	data := &WorkflowData{
+		Experiments: map[string][]string{
+			"style": {"concise", "detailed"},
+		},
+	}
+	steps := c.generateExperimentSteps(data)
+	joined := strings.Join(steps, "")
+	assert.Contains(t, joined, `{"style":["concise","detailed"]}`, "spec JSON should be embedded in the step")
+}
diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go
index d341a4404cb..f2271641fc1 100644
--- a/pkg/workflow/compiler_orchestrator_workflow.go
+++ b/pkg/workflow/compiler_orchestrator_workflow.go
@@ -379,6 +379,9 @@ func (c *Compiler) extractAdditionalConfigurations(
 	// This runs last so that all section-specific configurations have been resolved first.
 	applyTopLevelGitHubAppFallbacks(workflowData)
 
+	// Extract experiments configuration
+	workflowData.Experiments = extractExperimentsFromFrontmatter(frontmatter)
+
 	return nil
 }
 
diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go
index b46934438ef..d619fb1e9a4 100644
--- a/pkg/workflow/compiler_types.go
+++ b/pkg/workflow/compiler_types.go
@@ -512,6 +512,7 @@ type WorkflowData struct {
 	CachedPermissions             *Permissions                    // cached parsed Permissions object (for performance optimization); populated by applyDefaults after all permission mutations
 	ConcurrencyGroupExpr          string                          // cached concurrency group expression extracted from Concurrency YAML (for performance optimization); populated by applyDefaults
 	CachedConcurrencyGroupExprErr error                           // cached result of validateConcurrencyGroupExpression(ConcurrencyGroupExpr); nil = valid; populated by applyDefaults
+	Experiments                   map[string][]string             // A/B testing experiments: maps experiment name to variant list (from frontmatter)
 	CachedConcurrencyGroupExprSet bool                            // true once CachedConcurrencyGroupExprErr has been populated; distinguishes "valid (nil)" from "not yet computed"
 	CachedParsedToolsets          []string                        // cached result of ParseGitHubToolsets for the GitHub tool (for performance optimization); populated by applyDefaults
 	CachedAllowedDomainsStr       string                          // cached allowed-domains string for sanitization (for performance optimization); computed once and reused across multiple compilation steps
diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go
index 74773200976..5d12e203a8d 100644
--- a/pkg/workflow/expression_extraction.go
+++ b/pkg/workflow/expression_extraction.go
@@ -76,6 +76,14 @@ func (e *ExpressionExtractor) ExtractExpressions(markdown string) ([]*Expression
 			content = transformedContent
 		}
 
+		// Detect experiments.NAME expressions and remap them to env.GH_AW_EXPERIMENTS_NAME
+		// so the substitution step reads the value set by the experiment selection step via GITHUB_ENV.
+		transformedContent = transformExperimentsExpression(content)
+		if transformedContent != content {
+			expressionExtractionLog.Printf("Transformed experiment expression: %s -> %s", content, transformedContent)
+			content = transformedContent
+		}
+
 		// Skip if we've already seen this expression (also prevents duplicate deprecation warnings)
 		if _, exists := e.mappings[originalExpr]; exists {
 			continue
@@ -189,6 +197,27 @@ func transformActivationOutputs(expr string) string {
 	return expr
 }
 
+// experimentNameRegex matches experiments.<name> expressions where name is a simple identifier.
+var experimentNameRegex = regexp.MustCompile(`^experiments\.([a-zA-Z_][a-zA-Z0-9_]*)$`)
+
+// ExperimentEnvVarName returns the GITHUB_ENV / GITHUB_OUTPUT variable name used by the
+// pick_experiment step for the given experiment name.
+// Example: "feature1" → "GH_AW_EXPERIMENTS_FEATURE1"
+func ExperimentEnvVarName(experimentName string) string {
+	return "GH_AW_EXPERIMENTS_" + strings.ToUpper(experimentName)
+}
+
+// transformExperimentsExpression detects expressions of the form "experiments.<name>"
+// and rewrites them to "env.GH_AW_EXPERIMENTS_<NAME>" so that the placeholder
+// substitution step reads the value that the pick_experiment step wrote to GITHUB_ENV.
+func transformExperimentsExpression(expr string) string {
+	m := experimentNameRegex.FindStringSubmatch(expr)
+	if m == nil {
+		return expr
+	}
+	return "env." + ExperimentEnvVarName(m[1])
+}
+
 // simpleIdentifierRegex matches simple JavaScript property access chains like
 // "github.event.issue.number" or "needs.activation.outputs.text"
 // Each identifier must start with a letter or underscore, followed by alphanumeric or underscore
diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go
index 46761b4d185..1f90bb1c9d0 100644
--- a/pkg/workflow/frontmatter_types.go
+++ b/pkg/workflow/frontmatter_types.go
@@ -207,6 +207,11 @@ type FrontmatterConfig struct {
 	SecretMasking *SecretMaskingConfig `json:"secret-masking,omitempty"`
 	Observability *ObservabilityConfig `json:"observability,omitempty"`
 
+	// A/B testing experiments: maps experiment name to a list of variant values.
+	// Variants are picked at runtime using actions/cache to maintain state across runs.
+	// Use ${{ experiments.name }} in the workflow prompt to reference the selected variant.
+	Experiments map[string][]string `json:"experiments,omitempty"`
+
 	// Rate limiting configuration
 	RateLimit *RateLimitConfig `json:"rate-limit,omitempty"`
 

From a25dd7f1c1b874f1e8753978b9f7722012158c67 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 12:03:41 +0000
Subject: [PATCH 2/7] refactor: address code review feedback on experiments
 feature

- Fix expression_extraction.go: use originalContent variable and cleaner
  branching to avoid reusing transformedContent across two transformations;
  scope deprecation warning precisely to activation output transformations
- Fix ExperimentEnvVarName: use normalizeJobNameForEnvVar instead of
  strings.ToUpper to align Go and JS normalization (handles hyphens etc.)
- Fix buildExperimentSpecJSON: replace manual escapeJSONString with
  encoding/json.Marshal for correct escaping of all control characters"

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/6603825b-d8d5-4381-a44c-3d6fd823ab92

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 pkg/workflow/compiler_experiments.go  | 35 +++++++++------------------
 pkg/workflow/expression_extraction.go | 26 +++++++++++---------
 2 files changed, 25 insertions(+), 36 deletions(-)

diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go
index 1a2dd0cd45d..ef565b2b218 100644
--- a/pkg/workflow/compiler_experiments.go
+++ b/pkg/workflow/compiler_experiments.go
@@ -1,6 +1,7 @@
 package workflow
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -133,42 +134,28 @@ func (c *Compiler) generateExperimentSteps(data *WorkflowData) []string {
 }
 
 // buildExperimentSpecJSON builds a compact JSON object from the experiments map.
-// The JSON is safe to embed in a YAML single-quoted scalar (no single quotes in output).
+// Uses encoding/json for proper escaping of all special characters.
+// The resulting JSON is safe to embed in a YAML single-quoted scalar (no single quotes in output).
 func buildExperimentSpecJSON(experiments map[string][]string, names []string) string {
+	// Build JSON manually with encoding/json for individual values to ensure
+	// correct escaping of all special characters.  We iterate names (a sorted slice)
+	// rather than the map directly to produce deterministic output.
 	var sb strings.Builder
 	sb.WriteString("{")
 	for i, name := range names {
 		if i > 0 {
 			sb.WriteString(",")
 		}
-		sb.WriteString("\"")
-		sb.WriteString(escapeJSONString(name))
-		sb.WriteString("\":[")
-		variants := experiments[name]
-		for j, v := range variants {
-			if j > 0 {
-				sb.WriteString(",")
-			}
-			sb.WriteString("\"")
-			sb.WriteString(escapeJSONString(v))
-			sb.WriteString("\"")
-		}
-		sb.WriteString("]")
+		keyBytes, _ := json.Marshal(name)
+		varBytes, _ := json.Marshal(experiments[name])
+		sb.Write(keyBytes)
+		sb.WriteString(":")
+		sb.Write(varBytes)
 	}
 	sb.WriteString("}")
 	return sb.String()
 }
 
-// escapeJSONString escapes a string for embedding in a JSON value.
-func escapeJSONString(s string) string {
-	s = strings.ReplaceAll(s, `\`, `\\`)
-	s = strings.ReplaceAll(s, `"`, `\"`)
-	s = strings.ReplaceAll(s, "\n", `\n`)
-	s = strings.ReplaceAll(s, "\r", `\r`)
-	s = strings.ReplaceAll(s, "\t", `\t`)
-	return s
-}
-
 // sortedExperimentNames returns the experiment names in sorted order for deterministic output.
 func sortedExperimentNames(experiments map[string][]string) []string {
 	names := make([]string, 0, len(experiments))
diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go
index 5d12e203a8d..c0020c578e2 100644
--- a/pkg/workflow/expression_extraction.go
+++ b/pkg/workflow/expression_extraction.go
@@ -65,23 +65,22 @@ func (e *ExpressionExtractor) ExtractExpressions(markdown string) ([]*Expression
 
 		// Extract the content (without ${{ }})
 		content := strings.TrimSpace(match[1])
+		originalContent := content
 
 		// Apply activation output transformation for backward compatibility
 		// This transforms needs.activation.outputs.{text|title|body} to steps.sanitized.outputs.{text|title|body}
 		// Users should now use steps.sanitized.outputs.* directly; this transformation exists only for
 		// backward compatibility with existing workflows.
-		transformedContent := transformActivationOutputs(content)
-		if transformedContent != content {
-			expressionExtractionLog.Printf("Transformed expression: %s -> %s", content, transformedContent)
-			content = transformedContent
+		if t := transformActivationOutputs(content); t != content {
+			expressionExtractionLog.Printf("Transformed expression: %s -> %s", content, t)
+			content = t
 		}
 
 		// Detect experiments.NAME expressions and remap them to env.GH_AW_EXPERIMENTS_NAME
 		// so the substitution step reads the value set by the experiment selection step via GITHUB_ENV.
-		transformedContent = transformExperimentsExpression(content)
-		if transformedContent != content {
-			expressionExtractionLog.Printf("Transformed experiment expression: %s -> %s", content, transformedContent)
-			content = transformedContent
+		if t := transformExperimentsExpression(content); t != content {
+			expressionExtractionLog.Printf("Transformed experiment expression: %s -> %s", content, t)
+			content = t
 		}
 
 		// Skip if we've already seen this expression (also prevents duplicate deprecation warnings)
@@ -89,10 +88,10 @@ func (e *ExpressionExtractor) ExtractExpressions(markdown string) ([]*Expression
 			continue
 		}
 
-		// Emit deprecation warning once per unique deprecated expression
-		if transformedContent != strings.TrimSpace(match[1]) {
+		// Emit deprecation warning once per unique deprecated activation-output expression
+		if content != originalContent && strings.HasPrefix(content, "steps.sanitized.outputs.") {
 			fmt.Fprintln(os.Stderr, console.FormatWarningMessage(
-				fmt.Sprintf("Deprecated expression ${{ %s }}: use ${{ %s }} instead.", strings.TrimSpace(match[1]), transformedContent),
+				fmt.Sprintf("Deprecated expression ${{ %s }}: use ${{ %s }} instead.", originalContent, content),
 			))
 		}
 
@@ -202,9 +201,12 @@ var experimentNameRegex = regexp.MustCompile(`^experiments\.([a-zA-Z_][a-zA-Z0-9
 
 // ExperimentEnvVarName returns the GITHUB_ENV / GITHUB_OUTPUT variable name used by the
 // pick_experiment step for the given experiment name.
+// The name is uppercased and any character that is not A-Z, 0-9, or underscore is replaced
+// with underscore, matching the JavaScript normalization in pick_experiment.cjs.
 // Example: "feature1" → "GH_AW_EXPERIMENTS_FEATURE1"
+// Example: "my-flag"  → "GH_AW_EXPERIMENTS_MY_FLAG"
 func ExperimentEnvVarName(experimentName string) string {
-	return "GH_AW_EXPERIMENTS_" + strings.ToUpper(experimentName)
+	return "GH_AW_EXPERIMENTS_" + normalizeJobNameForEnvVar(experimentName)
 }
 
 // transformExperimentsExpression detects expressions of the form "experiments.<name>"

From 2acaa085c3c80bdf3e1db1d31fd3685b7a062872 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 15:05:13 +0000
Subject: [PATCH 3/7] refactor: use step outputs for experiments, add caveman
 experiment to smoke-copilot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Replace core.exportVariable with core.setOutput in pick_experiment.cjs:
  - Per-experiment output: setOutput(name, variant) e.g. 'caveman=yes'
  - Combined JSON output: setOutput('experiments', JSON) accessible as
    needs.activation.outputs.experiments from downstream jobs
- Wire activation job output: ctx.outputs["experiments"] = "${{ steps.pick-experiment.outputs.experiments }}"
- Add ExperimentExpressionMappings() in compiler_experiments.go: generates
  GH_AW_EXPERIMENTS_* → steps.pick-experiment.outputs.* mappings so the
  interpolation and substitution steps have the env vars set from step outputs
- Add Step 2.5 in interpolate_prompt.cjs: substitute __GH_AW_EXPERIMENTS_*__
  placeholders from env vars BEFORE renderMarkdownTemplate() so {{#if experiments.name}}
  conditionals evaluate the actual variant value not the placeholder string
- Extend isTruthy to treat 'no' as falsy (enables [yes, no] variant pattern)
- Update transformExperimentsExpression: map experiments.NAME →
  steps.pick-experiment.outputs.NAME (was env.GH_AW_EXPERIMENTS_NAME)
- Add caveman experiment to smoke-copilot.md with {{#if experiments.caveman}}
  conditional that injects 'Talk like a caveman' instructions when selected
- Update tests: pick_experiment.test.cjs, is_truthy.test.cjs,
  compiler_experiments_test.go, expression_extraction_test.go"

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/3ca87b03-00f9-40d6-b348-bdc026ad85c0

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 .github/workflows/smoke-copilot.lock.yml   | 93 +++++++++++++++-------
 .github/workflows/smoke-copilot.md         |  6 ++
 actions/setup/js/interpolate_prompt.cjs    | 27 +++++++
 actions/setup/js/is_truthy.cjs             |  2 +-
 actions/setup/js/is_truthy.test.cjs        |  6 ++
 actions/setup/js/pick_experiment.cjs       | 14 +++-
 actions/setup/js/pick_experiment.test.cjs  | 16 ++--
 pkg/workflow/compiler_activation_job.go    |  5 +-
 pkg/workflow/compiler_experiments.go       | 35 +++++++-
 pkg/workflow/compiler_experiments_test.go  | 24 ++++++
 pkg/workflow/compiler_yaml.go              | 12 +++
 pkg/workflow/expression_extraction.go      |  9 ++-
 pkg/workflow/expression_extraction_test.go |  6 ++
 13 files changed, 212 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml
index 2a0d47048aa..fa062cb9f64 100644
--- a/.github/workflows/smoke-copilot.lock.yml
+++ b/.github/workflows/smoke-copilot.lock.yml
@@ -1,4 +1,4 @@
-# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"b17dc062451a27c76b3efaf0a662ec506bcfb537952e772d9cd893c44c136031","agent_id":"copilot"}
+# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"8addbcc752f01d9f4e9dd85e45ca260c0d49cfb94496818813ff67ea4883022f","agent_id":"copilot"}
 # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/setup-go","sha":"4a3601121dd01d1626a1e23e37211e3254c1c06c","version":"v6.4.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"bcafcacb16a39f128d818304e6c9c0c18556b85f","version":"v7.1.0"},{"repo":"docker/setup-buildx-action","sha":"4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd","version":"v4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.29","digest":"sha256:e68f37e36962dcb3f3d1de680a49bc2302cefd001b941a7dc377155ec7ce42f4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.29@sha256:e68f37e36962dcb3f3d1de680a49bc2302cefd001b941a7dc377155ec7ce42f4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.29","digest":"sha256:d1219e4110684402aabbeb5a43858f26790c9d0be210581cf3f7a521bd2c87b6","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.29@sha256:d1219e4110684402aabbeb5a43858f26790c9d0be210581cf3f7a521bd2c87b6"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.29","digest":"sha256:29917488eb90a01ff9544ffeeb5cc26434a8ea16d69ae8972f5f6be0e567e276","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.29@sha256:29917488eb90a01ff9544ffeeb5cc26434a8ea16d69ae8972f5f6be0e567e276"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.29","digest":"sha256:8a71ad9e40454051672312917e51567abfb8251d7c294d086c48f63d84e4cb53","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.29@sha256:8a71ad9e40454051672312917e51567abfb8251d7c294d086c48f63d84e4cb53"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.3"},{"image":"ghcr.io/github/github-mcp-server:v1.0.3","digest":"sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.3@sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959"},{"image":"ghcr.io/github/serena-mcp-server:latest","digest":"sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5","pinned_image":"ghcr.io/github/serena-mcp-server:latest@sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5"},{"image":"mcr.microsoft.com/playwright/mcp","digest":"sha256:7b82f29c6ef83480a97f612d53ac3fd5f30a32df3fea1e06923d4204d3532bb2","pinned_image":"mcr.microsoft.com/playwright/mcp@sha256:7b82f29c6ef83480a97f612d53ac3fd5f30a32df3fea1e06923d4204d3532bb2"},{"image":"node:lts-alpine","digest":"sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f","pinned_image":"node:lts-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f"}]}
 #    ___                   _   _      
 #   / _ \                 | | (_)     
@@ -104,6 +104,7 @@ jobs:
       comment_repo: ${{ steps.add-comment.outputs.comment-repo }}
       comment_url: ${{ steps.add-comment.outputs.comment-url }}
       engine_id: ${{ steps.generate_aw_info.outputs.engine_id }}
+      experiments: ${{ steps.pick-experiment.outputs.experiments }}
       label_command: ${{ steps.remove_trigger_label.outputs.label_name }}
       lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }}
       model: ${{ steps.generate_aw_info.outputs.model }}
@@ -229,10 +230,45 @@ jobs:
             setupGlobals(core, github, context, exec, io, getOctokit);
             const { main } = require('${{ runner.temp }}/gh-aw/actions/remove_trigger_label.cjs');
             await main();
+      - name: Restore experiment state
+        id: restore-experiment-cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          key: experiments-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }}
+          restore-keys: experiments-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-
+          path: /tmp/gh-aw/experiments
+      - name: Pick experiment variants
+        id: pick-experiment
+        uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9
+        env:
+          GH_AW_EXPERIMENT_SPEC: '{"caveman":["yes","no"]}'
+          GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json
+          GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments
+        with:
+          script: |
+            const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+            setupGlobals(core, github, context, exec, io, getOctokit);
+            const { main } = require('${{ runner.temp }}/gh-aw/actions/pick_experiment.cjs');
+            await main();
+      - name: Save experiment state
+        if: always()
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          key: experiments-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }}
+          path: /tmp/gh-aw/experiments
+      - name: Upload experiment artifact
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: experiment
+          path: /tmp/gh-aw/experiments
+          if-no-files-found: ignore
+          retention-days: 30
       - name: Create prompt with built-in context
         env:
           GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
           GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl
+          GH_AW_EXPERIMENTS_CAVEMAN: ${{ steps.pick-experiment.outputs.caveman }}
           GH_AW_GITHUB_ACTOR: ${{ github.actor }}
           GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }}
           GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }}
@@ -246,9 +282,9 @@ jobs:
         run: |
           bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh"
           {
-          cat << 'GH_AW_PROMPT_0df92f983a5f9a09_EOF'
+          cat << 'GH_AW_PROMPT_4f448fe4f579f978_EOF'
           <system>
-          GH_AW_PROMPT_0df92f983a5f9a09_EOF
+          GH_AW_PROMPT_4f448fe4f579f978_EOF
           cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md"
           cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md"
           cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md"
@@ -256,16 +292,16 @@ jobs:
           cat "${RUNNER_TEMP}/gh-aw/prompts/agentic_workflows_guide.md"
           cat "${RUNNER_TEMP}/gh-aw/prompts/cache_memory_prompt.md"
           cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md"
-          cat << 'GH_AW_PROMPT_0df92f983a5f9a09_EOF'
+          cat << 'GH_AW_PROMPT_4f448fe4f579f978_EOF'
           <safe-output-tools>
           Tools: add_comment(max:2), create_issue, create_discussion, create_pull_request_review_comment(max:5), submit_pull_request_review, reply_to_pull_request_review_comment(max:5), add_labels, remove_labels, set_issue_type, dispatch_workflow, missing_tool, missing_data, noop, send_slack_message
-          GH_AW_PROMPT_0df92f983a5f9a09_EOF
+          GH_AW_PROMPT_4f448fe4f579f978_EOF
           cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_comment_memory.md"
-          cat << 'GH_AW_PROMPT_0df92f983a5f9a09_EOF'
+          cat << 'GH_AW_PROMPT_4f448fe4f579f978_EOF'
           </safe-output-tools>
-          GH_AW_PROMPT_0df92f983a5f9a09_EOF
+          GH_AW_PROMPT_4f448fe4f579f978_EOF
           cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md"
-          cat << 'GH_AW_PROMPT_0df92f983a5f9a09_EOF'
+          cat << 'GH_AW_PROMPT_4f448fe4f579f978_EOF'
           <github-context>
           The following GitHub context information is available for this workflow:
           {{#if __GH_AW_GITHUB_ACTOR__ }}
@@ -294,9 +330,9 @@ jobs:
           {{/if}}
           </github-context>
           
-          GH_AW_PROMPT_0df92f983a5f9a09_EOF
+          GH_AW_PROMPT_4f448fe4f579f978_EOF
           cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md"
-          cat << 'GH_AW_PROMPT_0df92f983a5f9a09_EOF'
+          cat << 'GH_AW_PROMPT_4f448fe4f579f978_EOF'
           </system>
           ## Serena Code Analysis
           
@@ -336,7 +372,7 @@ jobs:
           {{#runtime-import .github/workflows/shared/mcp/serena-go.md}}
           {{#runtime-import .github/workflows/shared/noop-reminder.md}}
           {{#runtime-import .github/workflows/smoke-copilot.md}}
-          GH_AW_PROMPT_0df92f983a5f9a09_EOF
+          GH_AW_PROMPT_4f448fe4f579f978_EOF
           } > "$GH_AW_PROMPT"
       - name: Interpolate variables and render templates
         uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9
@@ -347,6 +383,7 @@ jobs:
           GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
           GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
           GH_AW_GITHUB_SERVER_URL: ${{ github.server_url }}
+          GH_AW_EXPERIMENTS_CAVEMAN: ${{ steps.pick-experiment.outputs.caveman }}
         with:
           script: |
             const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
@@ -360,6 +397,7 @@ jobs:
           GH_AW_ALLOWED_EXTENSIONS: ''
           GH_AW_CACHE_DESCRIPTION: ''
           GH_AW_CACHE_DIR: '/tmp/gh-aw/cache-memory/'
+          GH_AW_EXPERIMENTS_CAVEMAN: ${{ steps.pick-experiment.outputs.caveman }}
           GH_AW_GITHUB_ACTOR: ${{ github.actor }}
           GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }}
           GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }}
@@ -385,6 +423,7 @@ jobs:
                 GH_AW_ALLOWED_EXTENSIONS: process.env.GH_AW_ALLOWED_EXTENSIONS,
                 GH_AW_CACHE_DESCRIPTION: process.env.GH_AW_CACHE_DESCRIPTION,
                 GH_AW_CACHE_DIR: process.env.GH_AW_CACHE_DIR,
+                GH_AW_EXPERIMENTS_CAVEMAN: process.env.GH_AW_EXPERIMENTS_CAVEMAN,
                 GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR,
                 GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID,
                 GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER,
@@ -633,9 +672,9 @@ jobs:
           mkdir -p /tmp/gh-aw/safeoutputs
           mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs
           mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts"
-          cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_412bfa2545520d0e_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_0514f1976a420feb_EOF'
           {"add_comment":{"allowed_repos":["github/gh-aw"],"hide_older_comments":true,"max":2},"add_labels":{"allowed":["smoke-copilot"],"allowed_repos":["github/gh-aw"]},"comment_memory":{"max":1,"memory_id":"default"},"create_discussion":{"category":"announcements","close_older_discussions":true,"close_older_key":"smoke-copilot","expires":2,"fallback_to_issue":true,"labels":["ai-generated"],"max":1},"create_issue":{"close_older_issues":true,"close_older_key":"smoke-copilot","expires":2,"group":true,"labels":["automation","testing"],"max":1},"create_pull_request_review_comment":{"max":5,"side":"RIGHT"},"create_report_incomplete_issue":{},"dispatch_workflow":{"max":1,"workflow_files":{"haiku-printer":".yml"},"workflows":["haiku-printer"]},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"remove_labels":{"allowed":["smoke"]},"reply_to_pull_request_review_comment":{"max":5},"report_incomplete":{},"send-slack-message":{"description":"Send a message to Slack (stub for testing)","inputs":{"message":{"description":"The message to send","required":false,"type":"string"}},"output":"Slack message stub executed!"},"set_issue_type":{},"submit_pull_request_review":{"max":1},"upload_artifact":{"max-size-bytes":104857600,"max-uploads":1,"retention-days":1,"skip-archive":true}}
-          GH_AW_SAFE_OUTPUTS_CONFIG_412bfa2545520d0e_EOF
+          GH_AW_SAFE_OUTPUTS_CONFIG_0514f1976a420feb_EOF
       - name: Write Safe Outputs Tools
         env:
           GH_AW_TOOLS_META_JSON: |
@@ -1061,7 +1100,7 @@ jobs:
       - name: Write MCP Scripts Config
         run: |
           mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-scripts/logs"
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/tools.json" << 'GH_AW_MCP_SCRIPTS_TOOLS_a22c58e5ddb027c3_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/tools.json" << 'GH_AW_MCP_SCRIPTS_TOOLS_6c8fc81f4a119af6_EOF'
           {
             "serverName": "mcpscripts",
             "version": "1.0.0",
@@ -1177,8 +1216,8 @@ jobs:
               }
             ]
           }
-          GH_AW_MCP_SCRIPTS_TOOLS_a22c58e5ddb027c3_EOF
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/mcp-server.cjs" << 'GH_AW_MCP_SCRIPTS_SERVER_051241934b769076_EOF'
+          GH_AW_MCP_SCRIPTS_TOOLS_6c8fc81f4a119af6_EOF
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/mcp-server.cjs" << 'GH_AW_MCP_SCRIPTS_SERVER_b2cf0aec0f7359a0_EOF'
             const path = require("path");
             const { startHttpServer } = require("./mcp_scripts_mcp_server_http.cjs");
             const configPath = path.join(__dirname, "tools.json");
@@ -1192,12 +1231,12 @@ jobs:
               console.error("Failed to start mcp-scripts HTTP server:", error);
               process.exit(1);
             });
-          GH_AW_MCP_SCRIPTS_SERVER_051241934b769076_EOF
+          GH_AW_MCP_SCRIPTS_SERVER_b2cf0aec0f7359a0_EOF
           chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/mcp-server.cjs"
           
       - name: Write MCP Scripts Tool Files
         run: |
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/gh.sh" << 'GH_AW_MCP_SCRIPTS_SH_GH_414a398ea60ba5ef_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/gh.sh" << 'GH_AW_MCP_SCRIPTS_SH_GH_3e80bf53460049f3_EOF'
           #!/bin/bash
           # Auto-generated mcp-script tool: gh
           # Execute any gh CLI command. This tool is accessible as 'mcpscripts-gh'. Provide the full command after 'gh' (e.g., args: 'pr list --limit 5'). The tool will run: gh <args>. Use single quotes ' for complex args to avoid shell interpretation issues.
@@ -1209,9 +1248,9 @@ jobs:
           GH_TOKEN="$GH_AW_GH_TOKEN" gh $INPUT_ARGS
           
           
-          GH_AW_MCP_SCRIPTS_SH_GH_414a398ea60ba5ef_EOF
+          GH_AW_MCP_SCRIPTS_SH_GH_3e80bf53460049f3_EOF
           chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/gh.sh"
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-discussion-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-DISCUSSION-QUERY_a515d529b75b5a36_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-discussion-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-DISCUSSION-QUERY_905d547b32af9cc7_EOF'
           #!/bin/bash
           # Auto-generated mcp-script tool: github-discussion-query
           # Query GitHub discussions with jq filtering support. Without --jq, returns schema and data size info. Use --jq '.' to get all data, or specific jq expressions to filter.
@@ -1347,9 +1386,9 @@ jobs:
           fi
           
           
-          GH_AW_MCP_SCRIPTS_SH_GITHUB-DISCUSSION-QUERY_a515d529b75b5a36_EOF
+          GH_AW_MCP_SCRIPTS_SH_GITHUB-DISCUSSION-QUERY_905d547b32af9cc7_EOF
           chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-discussion-query.sh"
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-issue-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-ISSUE-QUERY_55970a26050eef72_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-issue-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-ISSUE-QUERY_f1839fecf217b36b_EOF'
           #!/bin/bash
           # Auto-generated mcp-script tool: github-issue-query
           # Query GitHub issues with jq filtering support. Without --jq, returns schema and data size info. Use --jq '.' to get all data, or specific jq expressions to filter.
@@ -1428,9 +1467,9 @@ jobs:
           fi
           
           
-          GH_AW_MCP_SCRIPTS_SH_GITHUB-ISSUE-QUERY_55970a26050eef72_EOF
+          GH_AW_MCP_SCRIPTS_SH_GITHUB-ISSUE-QUERY_f1839fecf217b36b_EOF
           chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-issue-query.sh"
-          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-pr-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-PR-QUERY_5effbcb221526bb5_EOF'
+          cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-pr-query.sh" << 'GH_AW_MCP_SCRIPTS_SH_GITHUB-PR-QUERY_786c16707c3d8a34_EOF'
           #!/bin/bash
           # Auto-generated mcp-script tool: github-pr-query
           # Query GitHub pull requests with jq filtering support. Without --jq, returns schema and data size info. Use --jq '.' to get all data, or specific jq expressions to filter.
@@ -1515,7 +1554,7 @@ jobs:
           fi
           
           
-          GH_AW_MCP_SCRIPTS_SH_GITHUB-PR-QUERY_5effbcb221526bb5_EOF
+          GH_AW_MCP_SCRIPTS_SH_GITHUB-PR-QUERY_786c16707c3d8a34_EOF
           chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/github-pr-query.sh"
           
       - name: Generate MCP Scripts Server Config
@@ -1592,7 +1631,7 @@ jobs:
           
           mkdir -p /home/runner/.copilot
           GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node)
-          cat << GH_AW_MCP_CONFIG_086ff35fd544d263_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
+          cat << GH_AW_MCP_CONFIG_b5c741472fbdb3e1_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
           {
             "mcpServers": {
               "agenticworkflows": {
@@ -1693,7 +1732,7 @@ jobs:
               "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}"
             }
           }
-          GH_AW_MCP_CONFIG_086ff35fd544d263_EOF
+          GH_AW_MCP_CONFIG_b5c741472fbdb3e1_EOF
       - name: Mount MCP servers as CLIs
         id: mount-mcp-clis
         continue-on-error: true
diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md
index 55c5f0d901a..3f0df33ce2c 100644
--- a/.github/workflows/smoke-copilot.md
+++ b/.github/workflows/smoke-copilot.md
@@ -124,10 +124,16 @@ safe-outputs:
       run-failure: "📰 DEVELOPING STORY: [{workflow_name}]({run_url}) reports {status}. Our correspondents are investigating the incident..."
 timeout-minutes: 15
 strict: false
+experiments:
+  caveman: [yes, no]
 ---
 
 # Smoke Test: Copilot Engine Validation
 
+{{#if experiments.caveman }}
+Talk like a caveman in all your responses and outputs. Use short, broken sentences. Me test. You run.
+{{/if}}
+
 **IMPORTANT: Keep all outputs extremely short and concise. Use single-line responses where possible. No verbose explanations.**
 
 ## Tool Access Overview
diff --git a/actions/setup/js/interpolate_prompt.cjs b/actions/setup/js/interpolate_prompt.cjs
index 2097febc8a3..6238b2d035e 100644
--- a/actions/setup/js/interpolate_prompt.cjs
+++ b/actions/setup/js/interpolate_prompt.cjs
@@ -237,6 +237,33 @@ async function main() {
       core.info("No expression variables found, skipping interpolation");
     }
 
+    // Step 2.5: Substitute experiment placeholders BEFORE template rendering.
+    // When the runtime-import step processes {{#if experiments.name}} conditionals,
+    // it converts them to __GH_AW_EXPERIMENTS_NAME__ placeholders. These must be
+    // resolved with the actual variant value before renderMarkdownTemplate() runs,
+    // otherwise the placeholder string is truthy and the block is always kept.
+    // The activation job exposes GH_AW_EXPERIMENTS_* env vars (from the pick-experiment
+    // step output via the step's env: block), so we can substitute them here.
+    core.info("\n========================================");
+    core.info("[main] STEP 2.5: Experiment Placeholder Substitution");
+    core.info("========================================");
+    let experimentSubCount = 0;
+    for (const [key, value] of Object.entries(process.env)) {
+      if (key.startsWith("GH_AW_EXPERIMENTS_")) {
+        const placeholder = `__${key}__`;
+        if (content.includes(placeholder)) {
+          content = content.split(placeholder).join(value || "");
+          experimentSubCount++;
+          core.info(`  Substituted ${placeholder} → "${value || ""}"`);
+        }
+      }
+    }
+    if (experimentSubCount > 0) {
+      core.info(`Substituted ${experimentSubCount} experiment placeholder(s)`);
+    } else {
+      core.info("No experiment placeholders found in prompt");
+    }
+
     // Step 3: Render template conditionals
     core.info("\n========================================");
     core.info("[main] STEP 3: Template Rendering");
diff --git a/actions/setup/js/is_truthy.cjs b/actions/setup/js/is_truthy.cjs
index 84207526dac..a0464faa605 100644
--- a/actions/setup/js/is_truthy.cjs
+++ b/actions/setup/js/is_truthy.cjs
@@ -6,7 +6,7 @@
  */
 function isTruthy(expr) {
   const v = expr.trim().toLowerCase();
-  return !(v === "" || v === "false" || v === "0" || v === "null" || v === "undefined");
+  return !(v === "" || v === "false" || v === "no" || v === "0" || v === "null" || v === "undefined");
 }
 
 module.exports = { isTruthy };
diff --git a/actions/setup/js/is_truthy.test.cjs b/actions/setup/js/is_truthy.test.cjs
index fd9ea05a4fb..eb7ea89aa26 100644
--- a/actions/setup/js/is_truthy.test.cjs
+++ b/actions/setup/js/is_truthy.test.cjs
@@ -39,6 +39,12 @@ describe("is_truthy.cjs", () => {
       expect(isTruthy("hello")).toBe(true);
     });
 
+    it('should return false for "no" (case-insensitive)', () => {
+      expect(isTruthy("no")).toBe(false);
+      expect(isTruthy("NO")).toBe(false);
+      expect(isTruthy("No")).toBe(false);
+    });
+
     it("should trim whitespace", () => {
       expect(isTruthy("  false  ")).toBe(false);
       expect(isTruthy("  true  ")).toBe(true);
diff --git a/actions/setup/js/pick_experiment.cjs b/actions/setup/js/pick_experiment.cjs
index 925b5a26da2..1a6578a61c7 100644
--- a/actions/setup/js/pick_experiment.cjs
+++ b/actions/setup/js/pick_experiment.cjs
@@ -170,12 +170,18 @@ async function main() {
     recordVariant(name, selected, state);
     assignments[name] = selected;
 
-    // Export the selected variant so subsequent steps can read it via $GITHUB_ENV.
-    const envVarName = `GH_AW_EXPERIMENTS_${name.toUpperCase().replace(/[^A-Z0-9]/g, "_")}`;
-    core.exportVariable(envVarName, selected);
-    core.info(`Experiment "${name}": selected variant "${selected}" → ${envVarName}=${selected}`);
+    // Expose the selected variant as a step output (individual per experiment).
+    // Downstream jobs access this via needs.activation.outputs.experiment_<name>.
+    core.setOutput(name, selected);
+    core.info(`Experiment "${name}": selected variant "${selected}" (output: ${name}=${selected})`);
   }
 
+  // Expose the full assignments map as a serialized JSON step output.
+  // Downstream jobs access this via needs.activation.outputs.experiments.
+  const experimentsJSON = JSON.stringify(assignments);
+  core.setOutput("experiments", experimentsJSON);
+  core.info(`Experiment assignments (JSON): ${experimentsJSON}`);
+
   // Persist updated counts.
   saveState(stateFile, state);
   core.info(`Experiment state written to ${stateFile}`);
diff --git a/actions/setup/js/pick_experiment.test.cjs b/actions/setup/js/pick_experiment.test.cjs
index d69345a6d5f..7c4e39baa9c 100644
--- a/actions/setup/js/pick_experiment.test.cjs
+++ b/actions/setup/js/pick_experiment.test.cjs
@@ -10,7 +10,7 @@ const mockCore = {
   warning: vi.fn(),
   error: vi.fn(),
   setFailed: vi.fn(),
-  exportVariable: vi.fn(),
+  setOutput: vi.fn(),
   summary: {
     addRaw: vi.fn().mockReturnThis(),
     write: vi.fn().mockResolvedValue(undefined),
@@ -139,7 +139,7 @@ describe("pick_experiment", () => {
   // ── main ───────────────────────────────────────────────────────────────────
 
   describe("main", () => {
-    it("exports GITHUB_ENV variables for each experiment", async () => {
+    it("sets step outputs for each experiment and a combined JSON output", async () => {
       const stateFile = path.join(tmpDir, "state.json");
       process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({
         feature1: ["A", "B"],
@@ -149,7 +149,10 @@ describe("pick_experiment", () => {
 
       await main();
 
-      expect(mockCore.exportVariable).toHaveBeenCalledWith("GH_AW_EXPERIMENTS_FEATURE1", "A");
+      // Individual output per experiment
+      expect(mockCore.setOutput).toHaveBeenCalledWith("feature1", "A");
+      // Combined JSON output
+      expect(mockCore.setOutput).toHaveBeenCalledWith("experiments", JSON.stringify({ feature1: "A" }));
       expect(mockCore.setFailed).not.toHaveBeenCalled();
     });
 
@@ -163,14 +166,14 @@ describe("pick_experiment", () => {
 
       // First run → X
       await main();
-      const firstCall = mockCore.exportVariable.mock.calls.find(c => c[0] === "GH_AW_EXPERIMENTS_FEAT");
+      const firstCall = mockCore.setOutput.mock.calls.find(c => c[0] === "feat");
       expect(firstCall?.[1]).toBe("X");
 
       vi.clearAllMocks();
 
       // Second run → Y (state persisted from first call)
       await main();
-      const secondCall = mockCore.exportVariable.mock.calls.find(c => c[0] === "GH_AW_EXPERIMENTS_FEAT");
+      const secondCall = mockCore.setOutput.mock.calls.find(c => c[0] === "feat");
       expect(secondCall?.[1]).toBe("Y");
     });
 
@@ -181,7 +184,8 @@ describe("pick_experiment", () => {
 
       await main();
 
-      expect(mockCore.exportVariable).not.toHaveBeenCalled();
+      // When no experiments are declared, the function returns early and emits no outputs.
+      expect(mockCore.setOutput).not.toHaveBeenCalled();
       expect(mockCore.setFailed).not.toHaveBeenCalled();
     });
 
diff --git a/pkg/workflow/compiler_activation_job.go b/pkg/workflow/compiler_activation_job.go
index 98ca0179781..04e5140cde4 100644
--- a/pkg/workflow/compiler_activation_job.go
+++ b/pkg/workflow/compiler_activation_job.go
@@ -41,11 +41,14 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate
 	}
 
 	// Generate experiment selection steps when experiments are declared in the frontmatter.
-	// These steps run before the prompt is built so that ${{ experiments.name }} placeholders
+	// These steps run before the prompt is built so that experiments.name expressions
 	// can be resolved by the substitute_placeholders step.
 	if experimentSteps := c.generateExperimentSteps(data); len(experimentSteps) > 0 {
 		compilerActivationJobLog.Printf("Adding %d experiment step(s) for %d experiment(s)", len(experimentSteps), len(data.Experiments))
 		ctx.steps = append(ctx.steps, experimentSteps...)
+		// Expose the combined experiment JSON as a job output so downstream jobs can access
+		// the variant assignments via needs.activation.outputs.experiments.
+		ctx.outputs["experiments"] = "${{ steps.pick-experiment.outputs.experiments }}"
 	}
 
 	c.configureActivationNeedsAndCondition(ctx)
diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go
index ef565b2b218..01c0d7b58a4 100644
--- a/pkg/workflow/compiler_experiments.go
+++ b/pkg/workflow/compiler_experiments.go
@@ -59,7 +59,8 @@ func extractExperimentsFromFrontmatter(frontmatter map[string]any) map[string][]
 //
 // Steps generated (only when experiments are declared):
 //  1. Restore experiment cache   – actions/cache/restore keyed by workflow ID
-//  2. Pick variants              – pick_experiment.cjs (reads/writes state.json, sets GITHUB_ENV)
+//  2. Pick variants              – pick_experiment.cjs (reads/writes state.json, sets step outputs)
+//     Outputs: one per experiment (e.g. "caveman=yes") + "experiments" JSON blob
 //  3. Experiment step summary    – write a Markdown summary to GITHUB_STEP_SUMMARY
 //  4. Save experiment cache      – actions/cache/save keyed by workflow ID
 //  5. Upload experiment artifact – actions/upload-artifact named "experiment"
@@ -156,6 +157,38 @@ func buildExperimentSpecJSON(experiments map[string][]string, names []string) st
 	return sb.String()
 }
 
+// ExperimentExpressionMappings generates ExpressionMapping entries for all declared experiments.
+//
+// Each mapping maps the env-var name derived from "experiments.NAME"
+// (e.g. GH_AW_EXPERIMENTS_CAVEMAN) to the step output expression
+// "steps.pick-experiment.outputs.NAME".
+//
+// Adding these mappings to both expressionMappings and allExpressionMappings ensures:
+//   - The "Interpolate variables and render templates" step has
+//     GH_AW_EXPERIMENTS_NAME set from the step output, so that interpolate_prompt.cjs
+//     can substitute __GH_AW_EXPERIMENTS_NAME__ placeholders BEFORE template rendering.
+//   - The "Substitute placeholders" step can replace any remaining __GH_AW_EXPERIMENTS_NAME__
+//     occurrences that were produced by the runtime-import mechanism.
+func ExperimentExpressionMappings(experiments map[string][]string) []*ExpressionMapping {
+	names := sortedExperimentNames(experiments)
+	mappings := make([]*ExpressionMapping, 0, len(names))
+	for _, name := range names {
+		envVar := ExperimentEnvVarName(name) // e.g. GH_AW_EXPERIMENTS_CAVEMAN
+		// The step output expression resolves to the variant selected at runtime.
+		// The step ID "pick-experiment" is defined by generateExperimentSteps (the step with
+		// `id: pick-experiment` in the activation job).
+		content := "steps.pick-experiment.outputs." + name // e.g. steps.pick-experiment.outputs.caveman
+		original := "${{ experiments." + name + " }}"      // original expression in the markdown
+
+		mappings = append(mappings, &ExpressionMapping{
+			Original: original,
+			EnvVar:   envVar,
+			Content:  content,
+		})
+	}
+	return mappings
+}
+
 // sortedExperimentNames returns the experiment names in sorted order for deterministic output.
 func sortedExperimentNames(experiments map[string][]string) []string {
 	names := make([]string, 0, len(experiments))
diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go
index e78cbae95b0..7c96f4b3540 100644
--- a/pkg/workflow/compiler_experiments_test.go
+++ b/pkg/workflow/compiler_experiments_test.go
@@ -162,3 +162,27 @@ func TestGenerateExperimentSteps_SpecJSON(t *testing.T) {
 	joined := strings.Join(steps, "")
 	assert.Contains(t, joined, `{"style":["concise","detailed"]}`, "spec JSON should be embedded in the step")
 }
+
+func TestExperimentExpressionMappings(t *testing.T) {
+	experiments := map[string][]string{
+		"caveman": {"yes", "no"},
+		"style":   {"concise", "detailed"},
+	}
+	mappings := ExperimentExpressionMappings(experiments)
+	require.Len(t, mappings, 2, "one mapping per experiment")
+
+	// Build a lookup by EnvVar for easier assertions
+	byEnvVar := make(map[string]*ExpressionMapping, len(mappings))
+	for _, m := range mappings {
+		byEnvVar[m.EnvVar] = m
+	}
+
+	m := byEnvVar["GH_AW_EXPERIMENTS_CAVEMAN"]
+	require.NotNil(t, m, "mapping for GH_AW_EXPERIMENTS_CAVEMAN should exist")
+	assert.Equal(t, "steps.pick-experiment.outputs.caveman", m.Content, "content should be the step output expression")
+	assert.Equal(t, "${{ experiments.caveman }}", m.Original, "original should be the experiments expression")
+
+	m2 := byEnvVar["GH_AW_EXPERIMENTS_STYLE"]
+	require.NotNil(t, m2, "mapping for GH_AW_EXPERIMENTS_STYLE should exist")
+	assert.Equal(t, "steps.pick-experiment.outputs.style", m2.Content, "content should be the step output expression")
+}
diff --git a/pkg/workflow/compiler_yaml.go b/pkg/workflow/compiler_yaml.go
index 5300f7244fc..5b101743c65 100644
--- a/pkg/workflow/compiler_yaml.go
+++ b/pkg/workflow/compiler_yaml.go
@@ -526,6 +526,18 @@ func (c *Compiler) generatePrompt(yaml *strings.Builder, data *WorkflowData, pre
 	// referenced in activation's step env vars.
 	expressionMappings = filterExpressionsForActivation(expressionMappings, data.Jobs, beforeActivationJobs)
 
+	// Add expression mappings for declared experiments.
+	// These ensure the interpolation and substitution steps have GH_AW_EXPERIMENTS_* env vars
+	// set from pick-experiment step outputs, which is required for:
+	//   - Step 2.5 of interpolate_prompt.cjs: substitutes __GH_AW_EXPERIMENTS_*__ placeholders
+	//     produced by runtime_import.cjs from {{#if experiments.name}} template conditionals.
+	//   - The substitute_placeholders step: replaces any remaining occurrences.
+	if len(data.Experiments) > 0 {
+		experimentMappings := ExperimentExpressionMappings(data.Experiments)
+		compilerYamlLog.Printf("Adding %d experiment expression mapping(s)", len(experimentMappings))
+		expressionMappings = append(expressionMappings, experimentMappings...)
+	}
+
 	// Step 2: Add main workflow markdown content to the prompt
 	if c.inlinePrompt || data.InlinedImports {
 		// Inline mode (Wasm/browser): embed the markdown content directly in the YAML
diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go
index c0020c578e2..95bd76adc1d 100644
--- a/pkg/workflow/expression_extraction.go
+++ b/pkg/workflow/expression_extraction.go
@@ -210,14 +210,17 @@ func ExperimentEnvVarName(experimentName string) string {
 }
 
 // transformExperimentsExpression detects expressions of the form "experiments.<name>"
-// and rewrites them to "env.GH_AW_EXPERIMENTS_<NAME>" so that the placeholder
-// substitution step reads the value that the pick_experiment step wrote to GITHUB_ENV.
+// and rewrites them to "steps.pick-experiment.outputs.<name>" so that the placeholder
+// substitution step reads the value from the pick_experiment step output.
+// This is used for ${{ experiments.name }} expressions that appear directly in the prompt body
+// (mostly relevant in inline mode; in runtime-import mode the template conditional
+// {{#if experiments.name}} path is handled separately via ExperimentExpressionMappings).
 func transformExperimentsExpression(expr string) string {
 	m := experimentNameRegex.FindStringSubmatch(expr)
 	if m == nil {
 		return expr
 	}
-	return "env." + ExperimentEnvVarName(m[1])
+	return "steps.pick-experiment.outputs." + m[1]
 }
 
 // simpleIdentifierRegex matches simple JavaScript property access chains like
diff --git a/pkg/workflow/expression_extraction_test.go b/pkg/workflow/expression_extraction_test.go
index beb142c3333..a7d2f4d0155 100644
--- a/pkg/workflow/expression_extraction_test.go
+++ b/pkg/workflow/expression_extraction_test.go
@@ -55,6 +55,12 @@ func TestExpressionExtractor_ExtractExpressions(t *testing.T) {
 			wantCount:       1,
 			wantExpressions: []string{"steps.sanitized.outputs.text"},
 		},
+		{
+			name:            "experiments.name gets transformed to step output",
+			markdown:        "Value: ${{ experiments.caveman }}",
+			wantCount:       1,
+			wantExpressions: []string{"steps.pick-experiment.outputs.caveman"},
+		},
 		{
 			name:            "expression with whitespace",
 			markdown:        "Value: ${{  github.actor  }}",

From c0b7b392dbaf02c48c3aafa74a914085b76a4778 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 1 May 2026 16:23:53 +0000
Subject: [PATCH 4/7] docs(adr): add draft ADR-29534 for frontmatter A/B
 experiments feature

Auto-generated draft ADR documenting the decision to add an experiments
map to workflow frontmatter for balanced variant selection in A/B testing.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 ...matter-ab-experiments-variant-selection.md | 93 +++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 docs/adr/29534-frontmatter-ab-experiments-variant-selection.md

diff --git a/docs/adr/29534-frontmatter-ab-experiments-variant-selection.md b/docs/adr/29534-frontmatter-ab-experiments-variant-selection.md
new file mode 100644
index 00000000000..a70249b10ff
--- /dev/null
+++ b/docs/adr/29534-frontmatter-ab-experiments-variant-selection.md
@@ -0,0 +1,93 @@
+# ADR-29534: Frontmatter A/B Experiments with Balanced Variant Selection
+
+**Date**: 2026-05-01
+**Status**: Draft
+**Deciders**: pelikhan, copilot-swe-agent
+
+---
+
+## Part 1 — Narrative (Human-Friendly)
+
+### Context
+
+Agentic workflows compiled by gh-aw use a frontmatter-driven configuration model where the prompt template and runtime behaviour are declared in a Markdown header. Teams running these workflows wanted a first-class mechanism to test different prompt variants (e.g., tone, persona, feature flags embedded in the prompt) across successive workflow runs. Without such a mechanism, variant testing was ad-hoc, untracked, and statistically unbalanced. The system needed a solution that (1) could be declared entirely in the frontmatter, (2) required no external service dependency, and (3) guaranteed approximate balance across runs without user intervention.
+
+### Decision
+
+We will add an `experiments` map to the workflow frontmatter schema that associates named experiments with an ordered list of variant strings (e.g., `caveman: [yes, no]`). At runtime, the activation job selects the least-used variant for each experiment using a persisted counter stored in the GitHub Actions cache, sets individual step outputs (`steps.pick-experiment.outputs.<name>`), and exposes a combined JSON blob via the activation job's outputs. Prompt templates can reference experiment values via `${{ experiments.<name> }}` expression syntax and can guard blocks with `{{#if experiments.<name> }}` Handlebars conditionals, where the string `"no"` is treated as falsy. This approach is fully self-contained within the existing gh-aw compiler and Actions runtime.
+
+### Alternatives Considered
+
+#### Alternative 1: External Feature-Flag Service (e.g., LaunchDarkly or Unleash)
+
+An external feature-flag service would provide a mature A/B testing API with dashboards, targeting rules, and statistical analysis. However, it introduces a hard external dependency (network egress at runtime, credential management, billing) that conflicts with the firewall-isolated execution model of gh-aw. It was rejected as incompatible with the sandboxed runner environment.
+
+#### Alternative 2: Random Per-Run Variant Selection Without State Persistence
+
+Selecting a variant randomly on each run (e.g., `Math.random()`) requires no cache and no persistent state. This was rejected because it does not guarantee balance: over N runs, some variants may appear much more (or less) often than `N/K`, making statistically meaningful comparisons impossible without a large number of runs. The least-used counter approach achieves approximate balance in far fewer runs.
+
+#### Alternative 3: CI/CD Environment Variables Set Externally
+
+Teams could manually pass variant values as repository variables or dispatch inputs. This was considered too brittle: it requires human coordination across runs, does not scale to multiple simultaneous experiments, and cannot auto-rotate variants.
+
+### Consequences
+
+#### Positive
+- Experiments are declared alongside the workflow prompt — no external tooling required.
+- Balanced round-robin selection (least-used variant) ensures statistically even distribution with minimal runs.
+- Step summary and artifact upload provide per-run observability for downstream analysis (`gh aw audit --artifacts experiment`).
+- Template conditionals (`{{#if experiments.name }}`) allow prompt sections to be toggled based on the selected variant.
+
+#### Negative
+- Experiment state depends on the GitHub Actions cache, which has size limits, eviction policies, and is scoped per repository branch — cross-branch state is not shared.
+- The `"no"` string must be treated as falsy in `isTruthy`, introducing a special-case that differs from standard JavaScript truthiness and may surprise future contributors.
+- Expression rewriting (`${{ experiments.name }}` → `steps.pick-experiment.outputs.name`) adds a non-obvious indirection layer in the compiler's expression extraction phase.
+- Every workflow using experiments adds three steps to the activation job (restore cache, pick, save), increasing job duration and GitHub Actions billing slightly.
+
+#### Neutral
+- The `pick_experiment.cjs` script is bundled into the `actions/setup/js/` directory alongside other runtime helpers, following the existing pattern.
+- Experiment env vars (`GH_AW_EXPERIMENTS_*`) must be propagated explicitly to the interpolation and substitution steps; adding a new experiment to an existing workflow requires recompiling the lock file.
+- The feature requires no schema-breaking changes — `experiments` is an optional field; workflows without it are unaffected.
+
+---
+
+## Part 2 — Normative Specification (RFC 2119)
+
+> The key words **MUST**, **MUST NOT**, **REQUIRED**, **SHALL**, **SHALL NOT**, **SHOULD**, **SHOULD NOT**, **RECOMMENDED**, **MAY**, and **OPTIONAL** in this section are to be interpreted as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119).
+
+### Frontmatter Schema
+
+1. Workflow frontmatter **MAY** include an `experiments` field whose value is a YAML map from experiment name (string) to a non-empty list of variant strings.
+2. Each experiment name **MUST** be a valid YAML key composed of alphanumeric characters and underscores only.
+3. Each variant list **MUST** contain at least two entries; a single-element variant list **MUST NOT** be accepted by the compiler.
+4. Implementations **MUST NOT** require the `experiments` field — its absence **MUST** produce no change in compiled output.
+
+### Variant Selection
+
+5. Implementations **MUST** select the variant with the lowest cumulative invocation count across all previous runs (least-used selection).
+6. When two or more variants share the lowest count (including the initial state where all counts are zero), implementations **MUST** break ties by selecting the variant appearing earliest in the declared list.
+7. Variant counts **MUST** be persisted between workflow runs using the GitHub Actions cache, keyed by a combination of the sanitized workflow ID and the current run ID, with a restore-key prefix that matches any prior run for that workflow ID.
+8. Implementations **MUST** expose each selected variant as a named step output (`steps.pick-experiment.outputs.<experiment-name>`) and **MUST** also set a combined JSON output (`steps.pick-experiment.outputs.experiments`) containing all variant assignments.
+9. Implementations **MUST** upload the experiment state directory as an artifact named `experiment` (using `if: always()`) so that assignments are available for post-run analysis even when subsequent steps fail.
+
+### Expression and Template Integration
+
+10. The compiler **MUST** rewrite `${{ experiments.<name> }}` expressions in the frontmatter or prompt source to `steps.pick-experiment.outputs.<name>` during expression extraction, so the runtime value is injected by the Actions expression engine.
+11. Each experiment **MUST** be surfaced as an environment variable named `GH_AW_EXPERIMENTS_<NAME>` (uppercased) in every workflow step that performs prompt interpolation or template substitution.
+12. Implementations **MUST** substitute `__GH_AW_EXPERIMENTS_*__` placeholders in the raw prompt text before Handlebars template rendering (Step 2.5 of `interpolate_prompt.cjs`), so that `{{#if experiments.<name> }}` conditionals evaluate the actual runtime variant value.
+13. The `isTruthy` helper **MUST** treat the string `"no"` as falsy in addition to standard falsy values (`""`, `"false"`, `"0"`, `undefined`, `null`).
+14. Implementations **MUST NOT** evaluate `{{#if experiments.<name> }}` before the placeholder substitution step — raw placeholders **MUST NOT** be passed to the Handlebars engine.
+
+### Activation Job Structure
+
+15. When the `experiments` field is present in the frontmatter, the compiled activation job **MUST** include, in order: a cache restore step, the variant selection step, a cache save step (`if: always()`), and the artifact upload step (`if: always()`).
+16. The activation job **MUST** expose a `needs.activation.outputs.experiments` output containing the full JSON variant assignment object for use by downstream jobs.
+17. Implementations **MUST NOT** inject experiment steps into workflows that do not declare the `experiments` frontmatter field.
+
+### Conformance
+
+An implementation is considered conformant with this ADR if it satisfies all **MUST** and **MUST NOT** requirements above. Failure to meet any **MUST** or **MUST NOT** requirement constitutes non-conformance.
+
+---
+
+*This is a DRAFT ADR generated by the [Design Decision Gate](https://github.com/github/gh-aw/actions/runs/25222221666) workflow. The PR author must review, complete, and finalize this document before the PR can merge.*

From 532e6de26e5d420621360ae9d3c7c94514afc9e0 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 16:33:11 +0000
Subject: [PATCH 5/7] feat: display experiment artifact in logs/audit commands
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add pkg/cli/audit_report_experiments.go: parse state.json from the
  experiment artifact directory (both logsPath root and experiment/
  subdir), derive last-run variant assignments by max-count selection,
  expose as ExperimentData{Assignments, CumulativeCounts}
- Add pkg/cli/audit_report_experiments_test.go: 12 unit tests
- Add Experiments *ExperimentData to AuditData; populate in buildAuditData()
- Add renderExperimentData() and render "🧪 A/B Experiments" section in
  renderConsole() before the Metrics section
- Add Experiments *ExperimentData to RunData; populate in buildLogsData()

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/f8f310d4-097e-4b80-b86b-ff5b06132bfe

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 pkg/cli/audit_report.go                  |   2 +
 pkg/cli/audit_report_experiments.go      | 120 +++++++++++++++++
 pkg/cli/audit_report_experiments_test.go | 164 +++++++++++++++++++++++
 pkg/cli/audit_report_render.go           |  39 ++++++
 pkg/cli/logs_report.go                   |   2 +
 5 files changed, 327 insertions(+)
 create mode 100644 pkg/cli/audit_report_experiments.go
 create mode 100644 pkg/cli/audit_report_experiments_test.go

diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go
index 4fb318fa405..ccda25634b5 100644
--- a/pkg/cli/audit_report.go
+++ b/pkg/cli/audit_report.go
@@ -51,6 +51,7 @@ type AuditData struct {
 	ToolUsage               []ToolUsageInfo          `json:"tool_usage,omitempty"`
 	MCPToolUsage            *MCPToolUsageData        `json:"mcp_tool_usage,omitempty"`
 	CreatedItems            []CreatedItemReport      `json:"created_items,omitempty"`
+	Experiments             *ExperimentData          `json:"experiments,omitempty"`
 }
 
 // Finding represents a key insight discovered during audit
@@ -415,6 +416,7 @@ func buildAuditData(processedRun ProcessedRun, metrics LogMetrics, mcpToolUsage
 		ToolUsage:               toolUsage,
 		MCPToolUsage:            mcpToolUsage,
 		CreatedItems:            createdItems,
+		Experiments:             extractExperimentData(run.LogsPath),
 	}
 }
 
diff --git a/pkg/cli/audit_report_experiments.go b/pkg/cli/audit_report_experiments.go
new file mode 100644
index 00000000000..08f3581e220
--- /dev/null
+++ b/pkg/cli/audit_report_experiments.go
@@ -0,0 +1,120 @@
+// This file provides command-line interface functionality for gh-aw.
+// This file (audit_report_experiments.go) parses the experiment artifact uploaded by the
+// activation job and exposes the A/B experiment assignment data for display in the
+// audit and logs commands.
+
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/github/gh-aw/pkg/constants"
+)
+
+// ExperimentData represents the A/B experiment assignments for a single workflow run.
+type ExperimentData struct {
+	// Assignments maps each experiment name to the variant selected for this run.
+	// e.g. {"caveman": "yes", "style": "concise"}
+	Assignments map[string]string `json:"assignments"`
+
+	// CumulativeCounts maps each experiment name to a per-variant invocation counter.
+	// e.g. {"caveman": {"yes": 3, "no": 2}}
+	CumulativeCounts map[string]map[string]int `json:"cumulative_counts,omitempty"`
+}
+
+// experimentStateJSON matches the shape of the state.json written by pick_experiment.cjs:
+// { "counts": { "<name>": { "<variant>": <count> } } }
+type experimentStateJSON struct {
+	Counts map[string]map[string]int `json:"counts"`
+}
+
+// findExperimentStatePath returns the first existing state.json path inside the experiment
+// artifact directory. The file may be flattened to the run root or nested inside the
+// artifact subdirectory.
+func findExperimentStatePath(logsPath string) string {
+	candidates := []string{
+		filepath.Join(logsPath, "state.json"),
+		filepath.Join(logsPath, constants.ExperimentArtifactName, "state.json"),
+	}
+	for _, p := range candidates {
+		if _, err := os.Stat(p); err == nil {
+			return p
+		}
+	}
+	return ""
+}
+
+// extractExperimentData reads state.json from the experiment artifact directory under
+// logsPath and returns a populated ExperimentData or nil when no experiment artifact
+// is present.
+//
+// The state.json only contains the cumulative counters; it does not record which variant
+// was chosen for *this* run. The selected variant is derived by applying the same
+// least-used selection rule (lowest count wins; ties broken by the sorted variant order).
+func extractExperimentData(logsPath string) *ExperimentData {
+	if logsPath == "" {
+		return nil
+	}
+
+	statePath := findExperimentStatePath(logsPath)
+	if statePath == "" {
+		return nil
+	}
+
+	raw, err := os.ReadFile(statePath)
+	if err != nil {
+		return nil
+	}
+
+	var state experimentStateJSON
+	if err := json.Unmarshal(raw, &state); err != nil || len(state.Counts) == 0 {
+		return nil
+	}
+
+	// Derive this-run assignments: the variant selected on the most-recent run is
+	// the one with the maximum count (ties resolved by sorted order).
+	assignments := make(map[string]string, len(state.Counts))
+	names := make([]string, 0, len(state.Counts))
+	for name := range state.Counts {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		variantCounts := state.Counts[name]
+		selected := deriveLastSelectedVariant(variantCounts)
+		assignments[name] = selected
+	}
+
+	return &ExperimentData{
+		Assignments:      assignments,
+		CumulativeCounts: state.Counts,
+	}
+}
+
+// deriveLastSelectedVariant returns the variant selected on the last run based on the
+// highest count. Ties are broken by sorted order.
+func deriveLastSelectedVariant(variantCounts map[string]int) string {
+	if len(variantCounts) == 0 {
+		return ""
+	}
+
+	variants := make([]string, 0, len(variantCounts))
+	for v := range variantCounts {
+		variants = append(variants, v)
+	}
+	sort.Strings(variants)
+
+	selected := variants[0]
+	maxCount := variantCounts[selected]
+	for _, v := range variants[1:] {
+		if variantCounts[v] > maxCount {
+			maxCount = variantCounts[v]
+			selected = v
+		}
+	}
+	return selected
+}
diff --git a/pkg/cli/audit_report_experiments_test.go b/pkg/cli/audit_report_experiments_test.go
new file mode 100644
index 00000000000..7e1678738c6
--- /dev/null
+++ b/pkg/cli/audit_report_experiments_test.go
@@ -0,0 +1,164 @@
+//go:build !integration
+
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindExperimentStatePath(t *testing.T) {
+	t.Run("returns empty when logsPath is empty", func(t *testing.T) {
+		assert.Empty(t, findExperimentStatePath(""), "should return empty string for empty logsPath")
+	})
+
+	t.Run("finds state.json at root", func(t *testing.T) {
+		dir := t.TempDir()
+		statePath := filepath.Join(dir, "state.json")
+		require.NoError(t, os.WriteFile(statePath, []byte("{}"), 0o600))
+
+		got := findExperimentStatePath(dir)
+		assert.Equal(t, statePath, got, "should find state.json at logsPath root")
+	})
+
+	t.Run("finds state.json in experiment subdirectory", func(t *testing.T) {
+		dir := t.TempDir()
+		subDir := filepath.Join(dir, "experiment")
+		require.NoError(t, os.MkdirAll(subDir, 0o755))
+		statePath := filepath.Join(subDir, "state.json")
+		require.NoError(t, os.WriteFile(statePath, []byte("{}"), 0o600))
+
+		got := findExperimentStatePath(dir)
+		assert.Equal(t, statePath, got, "should find state.json in experiment subdirectory")
+	})
+
+	t.Run("returns empty when no state.json exists", func(t *testing.T) {
+		dir := t.TempDir()
+		got := findExperimentStatePath(dir)
+		assert.Empty(t, got, "should return empty string when no state.json found")
+	})
+}
+
+func TestExtractExperimentData(t *testing.T) {
+	t.Run("returns nil for empty logsPath", func(t *testing.T) {
+		assert.Nil(t, extractExperimentData(""), "should return nil for empty logsPath")
+	})
+
+	t.Run("returns nil when no state.json present", func(t *testing.T) {
+		dir := t.TempDir()
+		assert.Nil(t, extractExperimentData(dir), "should return nil when state.json missing")
+	})
+
+	t.Run("returns nil for invalid JSON", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "state.json"), []byte("not-json"), 0o600))
+		assert.Nil(t, extractExperimentData(dir), "should return nil for invalid JSON")
+	})
+
+	t.Run("returns nil for empty counts", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "state.json"), []byte(`{"counts":{}}`), 0o600))
+		assert.Nil(t, extractExperimentData(dir), "should return nil when counts map is empty")
+	})
+
+	t.Run("extracts single experiment with two variants", func(t *testing.T) {
+		dir := t.TempDir()
+		state := map[string]any{
+			"counts": map[string]any{
+				"caveman": map[string]int{"yes": 3, "no": 2},
+			},
+		}
+		raw, err := json.Marshal(state)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "state.json"), raw, 0o600))
+
+		got := extractExperimentData(dir)
+		require.NotNil(t, got, "should return non-nil ExperimentData")
+		assert.Equal(t, "yes", got.Assignments["caveman"], "variant with highest count should be selected")
+		assert.Equal(t, 3, got.CumulativeCounts["caveman"]["yes"], "cumulative count for yes should be 3")
+		assert.Equal(t, 2, got.CumulativeCounts["caveman"]["no"], "cumulative count for no should be 2")
+	})
+
+	t.Run("reads state.json from experiment subdirectory", func(t *testing.T) {
+		dir := t.TempDir()
+		subDir := filepath.Join(dir, "experiment")
+		require.NoError(t, os.MkdirAll(subDir, 0o755))
+		state := map[string]any{
+			"counts": map[string]any{
+				"style": map[string]int{"concise": 1, "detailed": 2},
+			},
+		}
+		raw, err := json.Marshal(state)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(filepath.Join(subDir, "state.json"), raw, 0o600))
+
+		got := extractExperimentData(dir)
+		require.NotNil(t, got, "should return non-nil ExperimentData from subdir")
+		assert.Equal(t, "detailed", got.Assignments["style"], "detailed has higher count so should be selected")
+	})
+
+	t.Run("extracts multiple experiments", func(t *testing.T) {
+		dir := t.TempDir()
+		state := map[string]any{
+			"counts": map[string]any{
+				"caveman": map[string]int{"yes": 1, "no": 0},
+				"style":   map[string]int{"concise": 2, "detailed": 1},
+			},
+		}
+		raw, err := json.Marshal(state)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "state.json"), raw, 0o600))
+
+		got := extractExperimentData(dir)
+		require.NotNil(t, got, "should return non-nil ExperimentData")
+		assert.Len(t, got.Assignments, 2, "should have 2 experiment assignments")
+		assert.Equal(t, "yes", got.Assignments["caveman"], "caveman should select yes (higher count)")
+		assert.Equal(t, "concise", got.Assignments["style"], "style should select concise (higher count)")
+	})
+}
+
+func TestDeriveLastSelectedVariant(t *testing.T) {
+	tests := []struct {
+		name     string
+		counts   map[string]int
+		expected string
+	}{
+		{
+			name:     "returns empty for nil map",
+			counts:   map[string]int{},
+			expected: "",
+		},
+		{
+			name:     "single variant",
+			counts:   map[string]int{"A": 5},
+			expected: "A",
+		},
+		{
+			name:     "highest count wins",
+			counts:   map[string]int{"A": 2, "B": 5},
+			expected: "B",
+		},
+		{
+			name:     "ties broken by sorted order",
+			counts:   map[string]int{"A": 3, "B": 3},
+			expected: "A",
+		},
+		{
+			name:     "three variants",
+			counts:   map[string]int{"yes": 4, "no": 3, "maybe": 2},
+			expected: "yes",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := deriveLastSelectedVariant(tt.counts)
+			assert.Equal(t, tt.expected, got, "deriveLastSelectedVariant result mismatch")
+		})
+	}
+}
diff --git a/pkg/cli/audit_report_render.go b/pkg/cli/audit_report_render.go
index 610adeec85b..3757ea74936 100644
--- a/pkg/cli/audit_report_render.go
+++ b/pkg/cli/audit_report_render.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/github/gh-aw/pkg/console"
@@ -131,6 +132,13 @@ func renderConsole(data AuditData, logsPath string) {
 		renderSafeOutputSummary(data.SafeOutputSummary)
 	}
 
+	// Experiments Section
+	if data.Experiments != nil {
+		fmt.Fprintln(os.Stderr, console.FormatSectionHeader("🧪 A/B Experiments"))
+		fmt.Fprintln(os.Stderr)
+		renderExperimentData(data.Experiments)
+	}
+
 	// Metrics Section - use new rendering system
 	fmt.Fprintln(os.Stderr, console.FormatSectionHeader("Metrics"))
 	fmt.Fprintln(os.Stderr)
@@ -240,6 +248,37 @@ func renderConsole(data AuditData, logsPath string) {
 	fmt.Fprintln(os.Stderr)
 }
 
+func renderExperimentData(exp *ExperimentData) {
+	if exp == nil {
+		return
+	}
+
+	names := make([]string, 0, len(exp.Assignments))
+	for name := range exp.Assignments {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		variant := exp.Assignments[name]
+		line := fmt.Sprintf("  • %s = %s", name, variant)
+		if counts, ok := exp.CumulativeCounts[name]; ok && len(counts) > 0 {
+			variants := make([]string, 0, len(counts))
+			for v := range counts {
+				variants = append(variants, v)
+			}
+			sort.Strings(variants)
+			parts := make([]string, 0, len(variants))
+			for _, v := range variants {
+				parts = append(parts, fmt.Sprintf("%s:%d", v, counts[v]))
+			}
+			line += fmt.Sprintf(" (cumulative: %s)", strings.Join(parts, ", "))
+		}
+		fmt.Fprintln(os.Stderr, line)
+	}
+	fmt.Fprintln(os.Stderr)
+}
+
 func renderAuditComparison(comparison *AuditComparisonData) {
 	if comparison == nil {
 		return
diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go
index db13779e25e..26b17d6b57a 100644
--- a/pkg/cli/logs_report.go
+++ b/pkg/cli/logs_report.go
@@ -138,6 +138,7 @@ type RunData struct {
 	TokenUsageSummary          *TokenUsageSummary     `json:"token_usage_summary,omitempty" console:"-"`                                            // Token usage from firewall proxy
 	GitHubAPICalls             int                    `json:"github_api_calls,omitempty" console:"header:GitHub API Calls,format:number,omitempty"` // GitHub API calls made during the run
 	AvgTimeBetweenTurns        string                 `json:"avg_time_between_turns,omitempty" console:"-"`                                         // Average time between consecutive LLM API calls (TBT)
+	Experiments                *ExperimentData        `json:"experiments,omitempty" console:"-"`                                                    // A/B experiment assignments for this run
 }
 
 // buildLogsData creates structured logs data from processed runs
@@ -291,6 +292,7 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation
 			AwContext:                  awContext,
 			TokenUsageSummary:          pr.TokenUsage,
 			GitHubAPICalls:             gitHubAPICalls,
+			Experiments:                extractExperimentData(run.LogsPath),
 		}
 		if awInfo != nil {
 			runData.Repository = awInfo.Repository

From bea5f0cbddbc0ff2ebd7d6fee340b9eff3b7367c Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 16:48:41 +0000
Subject: [PATCH 6/7] fix: address PR review comments for A/B experiments
 feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Fix step-list comment in compiler_experiments.go: remove non-existent
  "Experiment step summary" step (summary is inside pick_experiment.cjs)
- Add experiment name validation: names must match [a-zA-Z_][a-zA-Z0-9_]*;
  invalid names are logged and skipped (2 new test cases)
- Fix YAML single-quote escaping for GH_AW_EXPERIMENT_SPEC: escape ' as ''
  in YAML single-quoted scalar (new test: TestGenerateExperimentSteps_SingleQuoteEscaping)
- Fix expression_extraction.go comment: correct GITHUB_ENV → step output path
- Fix ExperimentEnvVarName comment: "replaced with underscore" → "dropped"
- Fix pick_experiment.cjs header: setOutput not GITHUB_ENV; fix downstream
  access expression needs.activation.outputs.<name>
- Add propertyNames pattern to experiments JSON schema (enforces identifiers)
- Create docs/src/content/docs/guides/experiments.md (how-to guide)
- Rebuild and recompile 205 workflow lock files

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/6e6bc500-04f5-47c0-a97b-ea46cb2d2517

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 actions/setup/js/pick_experiment.cjs         |  5 +-
 docs/src/content/docs/guides/experiments.md  | 92 ++++++++++++++++++++
 pkg/parser/schemas/main_workflow_schema.json |  4 +
 pkg/workflow/compiler_experiments.go         | 25 ++++--
 pkg/workflow/compiler_experiments_test.go    | 34 ++++++++
 pkg/workflow/expression_extraction.go        | 11 ++-
 6 files changed, 156 insertions(+), 15 deletions(-)
 create mode 100644 docs/src/content/docs/guides/experiments.md

diff --git a/actions/setup/js/pick_experiment.cjs b/actions/setup/js/pick_experiment.cjs
index 1a6578a61c7..642d394fd0f 100644
--- a/actions/setup/js/pick_experiment.cjs
+++ b/actions/setup/js/pick_experiment.cjs
@@ -22,7 +22,8 @@
  *   K is the number of variants, satisfying basic A/B statistical balance.
  *
  * Outputs:
- *   - Sets GH_AW_EXPERIMENTS_<NAME>=<variant> in GITHUB_ENV for each experiment.
+ *   - Sets core.setOutput(name, selected) for each experiment (e.g. caveman=yes).
+ *   - Sets core.setOutput('experiments', JSON.stringify(assignments)) for the full map.
  *   - Writes the updated counter state back to GH_AW_EXPERIMENT_STATE_FILE.
  *   - Appends a Markdown step summary with the assignment table and cumulative counts.
  */
@@ -171,7 +172,7 @@ async function main() {
     assignments[name] = selected;
 
     // Expose the selected variant as a step output (individual per experiment).
-    // Downstream jobs access this via needs.activation.outputs.experiment_<name>.
+    // Downstream jobs access this via needs.activation.outputs.<name>.
     core.setOutput(name, selected);
     core.info(`Experiment "${name}": selected variant "${selected}" (output: ${name}=${selected})`);
   }
diff --git a/docs/src/content/docs/guides/experiments.md b/docs/src/content/docs/guides/experiments.md
new file mode 100644
index 00000000000..80167544b7b
--- /dev/null
+++ b/docs/src/content/docs/guides/experiments.md
@@ -0,0 +1,92 @@
+---
+title: A/B Experiments
+description: Run A/B experiments in GitHub Agentic Workflows to test prompt variants and measure the effect of different instructions across runs.
+sidebar:
+  order: 7
+---
+
+The `experiments` section of the workflow frontmatter enables statistical A/B testing by defining named experiments, each with a set of variant values. At runtime the activation job selects one variant per experiment using a balanced round-robin counter and exposes the selection to the workflow prompt.
+
+## Declaring experiments
+
+Add an `experiments` map to the workflow frontmatter. Each key names an experiment; the value is an array of two or more variant strings.
+
+```aw wrap
+---
+on:
+  issues:
+    types: [opened]
+engine: copilot
+
+experiments:
+  style: [concise, detailed]
+---
+
+Summarize this issue in a **${{ experiments.style }}** way.
+```
+
+> [!NOTE]
+> Experiment names must be valid identifiers: start with a letter or underscore, followed by letters, digits, or underscores (e.g. `style`, `feature_1`). Names that do not match this pattern are ignored.
+
+## Using variants in the prompt
+
+Reference a variant with `${{ experiments.<name> }}`. At runtime this is substituted with the selected variant string (e.g. `concise`).
+
+Use the `{{#if experiments.<name> }}` block syntax for conditional prompt sections. A variant value of `no` is treated as falsy, enabling yes/no flag experiments:
+
+```aw wrap
+---
+experiments:
+  caveman: [yes, no]
+---
+
+{{#if experiments.caveman }}
+Talk like a caveman in all your responses. Me test. You run.
+{{/if}}
+
+Address the issue described above.
+```
+
+## Statistical balancing
+
+The activation job maintains a per-variant invocation counter in an `actions/cache` entry keyed by workflow ID. The variant with the lowest cumulative count is selected on each run; ties are broken by variant order. Over N runs every variant is used approximately N/K times (K = variant count), providing basic A/B balance with no configuration.
+
+The counter persists across workflow runs via the GitHub Actions cache. A fresh repository starts from zero counts.
+
+## Accessing assignments downstream
+
+Each experiment exposes its selected variant as an activation job output:
+
+| Expression | Description |
+|---|---|
+| `needs.activation.outputs.<name>` | Selected variant for experiment `<name>` |
+| `needs.activation.outputs.experiments` | All assignments as a JSON object |
+
+Use these expressions in downstream jobs defined in the `jobs:` frontmatter section.
+
+## Analyzing results
+
+The activation job uploads the counter state as an `experiment` artifact. Download and inspect it with the `gh aw` CLI:
+
+```bash
+# Download the experiment artifact for a specific run
+gh aw audit <run-id> --artifacts experiment
+
+# Display experiment assignments in the audit report
+gh aw audit <run-id>
+```
+
+The `🧪 A/B Experiments` section of the audit report shows the variant chosen on the most recent run and the cumulative counts across all runs:
+
+```
+🧪 A/B Experiments
+  • caveman = yes (cumulative: no:4, yes:5)
+  • style = concise (cumulative: concise:5, detailed:4)
+```
+
+## Frontmatter reference
+
+| Field | Type | Description |
+|---|---|---|
+| `experiments` | `object` | Map of experiment name → variant array |
+| `experiments.<name>` | `string[]` | Array of two or more variant strings for one experiment |
diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json
index 1a225a47e41..6a726acfdbe 100644
--- a/pkg/parser/schemas/main_workflow_schema.json
+++ b/pkg/parser/schemas/main_workflow_schema.json
@@ -2733,6 +2733,10 @@
     "experiments": {
       "description": "A/B testing experiments. Each key is an experiment name; the value is an array of two or more variant strings. At runtime the activation job picks a variant using actions/cache to maintain consistent assignment across runs. Use ${{ experiments.<name> }} in the workflow prompt to reference the selected variant. When multiple experiments are declared, assignments are statistically balanced using a counter that round-robins across variants.",
       "type": "object",
+      "propertyNames": {
+        "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$",
+        "description": "Experiment names must be valid identifiers: start with a letter or underscore, followed by letters, digits, or underscores."
+      },
       "additionalProperties": {
         "type": "array",
         "items": { "type": "string" },
diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go
index 01c0d7b58a4..3a56ef6cf89 100644
--- a/pkg/workflow/compiler_experiments.go
+++ b/pkg/workflow/compiler_experiments.go
@@ -3,6 +3,7 @@ package workflow
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"sort"
 	"strings"
 
@@ -21,6 +22,11 @@ const experimentStateFile = experimentsCacheDir + "/state.json"
 // extractExperimentsFromFrontmatter reads the "experiments" map from a raw frontmatter map.
 // Each key is an experiment name; each value must be a []string (or []any of strings) of
 // variant values.  Invalid entries are silently skipped.
+// Experiment names must match [a-zA-Z_][a-zA-Z0-9_]* (identifier style) so they can be used
+// as GitHub Actions step output names and in ${{ experiments.<name> }} expressions without
+// bracket notation.  Names that do not match are skipped with a warning.
+var experimentNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
+
 func extractExperimentsFromFrontmatter(frontmatter map[string]any) map[string][]string {
 	raw, ok := frontmatter["experiments"]
 	if !ok || raw == nil {
@@ -32,6 +38,10 @@ func extractExperimentsFromFrontmatter(frontmatter map[string]any) map[string][]
 	}
 	result := make(map[string][]string, len(rawMap))
 	for name, val := range rawMap {
+		if !experimentNamePattern.MatchString(name) {
+			experimentsLog.Printf("Skipping experiment %q: name must match [a-zA-Z_][a-zA-Z0-9_]*", name)
+			continue
+		}
 		switch v := val.(type) {
 		case []string:
 			if len(v) >= 2 {
@@ -59,11 +69,10 @@ func extractExperimentsFromFrontmatter(frontmatter map[string]any) map[string][]
 //
 // Steps generated (only when experiments are declared):
 //  1. Restore experiment cache   – actions/cache/restore keyed by workflow ID
-//  2. Pick variants              – pick_experiment.cjs (reads/writes state.json, sets step outputs)
-//     Outputs: one per experiment (e.g. "caveman=yes") + "experiments" JSON blob
-//  3. Experiment step summary    – write a Markdown summary to GITHUB_STEP_SUMMARY
-//  4. Save experiment cache      – actions/cache/save keyed by workflow ID
-//  5. Upload experiment artifact – actions/upload-artifact named "experiment"
+//  2. Pick variants              – pick_experiment.cjs (reads/writes state.json, sets step outputs,
+//     writes a Markdown step summary); outputs: one per experiment (e.g. "caveman=yes") + "experiments" JSON blob
+//  3. Save experiment cache      – actions/cache/save keyed by workflow ID
+//  4. Upload experiment artifact – actions/upload-artifact named "experiment"
 func (c *Compiler) generateExperimentSteps(data *WorkflowData) []string {
 	if len(data.Experiments) == 0 {
 		return nil
@@ -97,7 +106,7 @@ func (c *Compiler) generateExperimentSteps(data *WorkflowData) []string {
 		"        id: pick-experiment\n",
 		fmt.Sprintf("        uses: %s\n", getCachedActionPin("actions/github-script", data)),
 		"        env:\n",
-		fmt.Sprintf("          GH_AW_EXPERIMENT_SPEC: '%s'\n", specJSON),
+		fmt.Sprintf("          GH_AW_EXPERIMENT_SPEC: '%s'\n", strings.ReplaceAll(specJSON, "'", "''")),
 		fmt.Sprintf("          GH_AW_EXPERIMENT_STATE_FILE: %s\n", experimentStateFile),
 		fmt.Sprintf("          GH_AW_EXPERIMENT_STATE_DIR: %s\n", experimentsCacheDir),
 		"        with:\n",
@@ -136,7 +145,9 @@ func (c *Compiler) generateExperimentSteps(data *WorkflowData) []string {
 
 // buildExperimentSpecJSON builds a compact JSON object from the experiments map.
 // Uses encoding/json for proper escaping of all special characters.
-// The resulting JSON is safe to embed in a YAML single-quoted scalar (no single quotes in output).
+// Caller is responsible for escaping single quotes (” in YAML) when embedding the
+// result in a YAML single-quoted scalar, since JSON string values may contain literal
+// single quotes (e.g. "Bob's").
 func buildExperimentSpecJSON(experiments map[string][]string, names []string) string {
 	// Build JSON manually with encoding/json for individual values to ensure
 	// correct escaping of all special characters.  We iterate names (a sorted slice)
diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go
index 7c96f4b3540..27bfc5a6359 100644
--- a/pkg/workflow/compiler_experiments_test.go
+++ b/pkg/workflow/compiler_experiments_test.go
@@ -80,6 +80,26 @@ func TestExtractExperimentsFromFrontmatter(t *testing.T) {
 			},
 			want: map[string][]string{"feature1": {"A", "B"}},
 		},
+		{
+			name: "skips experiment with invalid name (hyphen)",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"my-flag": []any{"A", "B"},
+					"valid":   []any{"X", "Y"},
+				},
+			},
+			want: map[string][]string{"valid": {"X", "Y"}},
+		},
+		{
+			name: "skips experiment with name starting with digit",
+			frontmatter: map[string]any{
+				"experiments": map[string]any{
+					"1invalid": []any{"A", "B"},
+					"valid":    []any{"X", "Y"},
+				},
+			},
+			want: map[string][]string{"valid": {"X", "Y"}},
+		},
 	}
 
 	for _, tt := range tests {
@@ -163,6 +183,20 @@ func TestGenerateExperimentSteps_SpecJSON(t *testing.T) {
 	assert.Contains(t, joined, `{"style":["concise","detailed"]}`, "spec JSON should be embedded in the step")
 }
 
+func TestGenerateExperimentSteps_SingleQuoteEscaping(t *testing.T) {
+	c := &Compiler{}
+	data := &WorkflowData{
+		Experiments: map[string][]string{
+			"variant": {"Bob's choice", "Alice's choice"},
+		},
+	}
+	steps := c.generateExperimentSteps(data)
+	joined := strings.Join(steps, "")
+	// Single quotes in JSON string values must be doubled for YAML single-quoted scalar.
+	assert.Contains(t, joined, "Bob''s", "single quotes in variant values must be escaped as '' in YAML")
+	assert.Contains(t, joined, "Alice''s", "single quotes in variant values must be escaped as '' in YAML")
+}
+
 func TestExperimentExpressionMappings(t *testing.T) {
 	experiments := map[string][]string{
 		"caveman": {"yes", "no"},
diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go
index 95bd76adc1d..ce3156c3a9d 100644
--- a/pkg/workflow/expression_extraction.go
+++ b/pkg/workflow/expression_extraction.go
@@ -76,8 +76,8 @@ func (e *ExpressionExtractor) ExtractExpressions(markdown string) ([]*Expression
 			content = t
 		}
 
-		// Detect experiments.NAME expressions and remap them to env.GH_AW_EXPERIMENTS_NAME
-		// so the substitution step reads the value set by the experiment selection step via GITHUB_ENV.
+		// Detect experiments.NAME expressions and remap them to steps.pick-experiment.outputs.NAME
+		// so the substitution step reads the variant value from the pick_experiment step output.
 		if t := transformExperimentsExpression(content); t != content {
 			expressionExtractionLog.Printf("Transformed experiment expression: %s -> %s", content, t)
 			content = t
@@ -199,10 +199,9 @@ func transformActivationOutputs(expr string) string {
 // experimentNameRegex matches experiments.<name> expressions where name is a simple identifier.
 var experimentNameRegex = regexp.MustCompile(`^experiments\.([a-zA-Z_][a-zA-Z0-9_]*)$`)
 
-// ExperimentEnvVarName returns the GITHUB_ENV / GITHUB_OUTPUT variable name used by the
-// pick_experiment step for the given experiment name.
-// The name is uppercased and any character that is not A-Z, 0-9, or underscore is replaced
-// with underscore, matching the JavaScript normalization in pick_experiment.cjs.
+// ExperimentEnvVarName returns the env-var name used for the given experiment.
+// The name is uppercased; hyphens are converted to underscores; all other characters
+// that are not A-Z, 0-9, or underscore are dropped (not replaced).
 // Example: "feature1" → "GH_AW_EXPERIMENTS_FEATURE1"
 // Example: "my-flag"  → "GH_AW_EXPERIMENTS_MY_FLAG"
 func ExperimentEnvVarName(experimentName string) string {

From cb6a471f5111be1bdc83d511e3dd936550eb2ac7 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 1 May 2026 17:07:49 +0000
Subject: [PATCH 7/7] feat: restore experiment artifact in detection job

- Add buildExperimentArtifactDownloadSteps() in compiler_experiments.go:
  downloads the experiment artifact to /tmp/gh-aw/experiments/ so the
  detection agent can read current variant assignments from state.json.
  Returns nil when no experiments are declared (no-op for non-experiment
  workflows).
- Call the new helper in buildDetectionJob() (threat_detection.go) after
  the agent output download step, using the activation-downstream prefix.
- Add 3 unit tests for buildExperimentArtifactDownloadSteps and one
  integration test TestDetectionJobDownloadsExperimentArtifact verifying
  step presence/absence based on experiment declarations.
- Recompile 205 workflow lock files (smoke-copilot detection job now
  includes Download experiment artifact step).

Agent-Logs-Url: https://github.com/github/gh-aw/sessions/a8ddb085-cb12-4b28-9fd9-51760ccf429e

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 .github/workflows/smoke-copilot.lock.yml  |  6 ++
 pkg/workflow/compiler_experiments.go      | 18 +++++
 pkg/workflow/compiler_experiments_test.go | 31 +++++++++
 pkg/workflow/detection_success_test.go    | 81 +++++++++++++++++++++++
 pkg/workflow/threat_detection.go          |  5 ++
 5 files changed, 141 insertions(+)

diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml
index fa062cb9f64..b5240cad453 100644
--- a/.github/workflows/smoke-copilot.lock.yml
+++ b/.github/workflows/smoke-copilot.lock.yml
@@ -2209,6 +2209,12 @@ jobs:
           mkdir -p /tmp/gh-aw/
           find "/tmp/gh-aw/" -type f -print
           echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
+      - name: Download experiment artifact
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: experiment
+          path: /tmp/gh-aw/experiments/
       - name: Checkout repository for patch context
         if: needs.agent.outputs.has_patch == 'true'
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go
index 3a56ef6cf89..4600ff08726 100644
--- a/pkg/workflow/compiler_experiments.go
+++ b/pkg/workflow/compiler_experiments.go
@@ -209,3 +209,21 @@ func sortedExperimentNames(experiments map[string][]string) []string {
 	sort.Strings(names)
 	return names
 }
+
+// buildExperimentArtifactDownloadSteps creates a download step for the experiment artifact.
+// The artifact is downloaded to experimentsCacheDir so the detection agent can read the
+// current variant assignments from state.json.
+// prefix must be the artifact prefix expression for a job that directly depends on the
+// activation job (i.e. artifactPrefixExprForDownstreamJob).
+// The step is a no-op when no experiments are declared.
+func buildExperimentArtifactDownloadSteps(prefix string, experiments map[string][]string) []string {
+	if len(experiments) == 0 {
+		return nil
+	}
+	artifactName := prefix + constants.ExperimentArtifactName
+	return buildArtifactDownloadSteps(ArtifactDownloadConfig{
+		ArtifactName: artifactName,
+		DownloadPath: experimentsCacheDir + "/",
+		StepName:     "Download experiment artifact",
+	})
+}
diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go
index 27bfc5a6359..fa3d6e980ba 100644
--- a/pkg/workflow/compiler_experiments_test.go
+++ b/pkg/workflow/compiler_experiments_test.go
@@ -220,3 +220,34 @@ func TestExperimentExpressionMappings(t *testing.T) {
 	require.NotNil(t, m2, "mapping for GH_AW_EXPERIMENTS_STYLE should exist")
 	assert.Equal(t, "steps.pick-experiment.outputs.style", m2.Content, "content should be the step output expression")
 }
+
+// ── buildExperimentArtifactDownloadSteps ──────────────────────────────────
+
+func TestBuildExperimentArtifactDownloadStep_Empty(t *testing.T) {
+	steps := buildExperimentArtifactDownloadSteps("prefix-", nil)
+	assert.Empty(t, steps, "no steps when experiments is nil")
+
+	steps = buildExperimentArtifactDownloadSteps("prefix-", map[string][]string{})
+	assert.Empty(t, steps, "no steps when experiments is empty")
+}
+
+func TestBuildExperimentArtifactDownloadStep_Generated(t *testing.T) {
+	experiments := map[string][]string{"caveman": {"yes", "no"}}
+	steps := buildExperimentArtifactDownloadSteps("${{ needs.activation.outputs.artifact_prefix }}", experiments)
+	require.NotEmpty(t, steps, "steps should be generated when experiments are declared")
+	joined := strings.Join(steps, "")
+	assert.Contains(t, joined, "Download experiment artifact", "should include download step name")
+	assert.Contains(t, joined, "experiment", "should reference experiment artifact")
+	assert.Contains(t, joined, experimentsCacheDir, "should download to experiments cache dir")
+	assert.Contains(t, joined, "actions/download-artifact", "should use download-artifact action")
+}
+
+func TestBuildExperimentArtifactDownloadStep_NoPrefix(t *testing.T) {
+	// Non-workflow_call workflows use empty prefix
+	experiments := map[string][]string{"style": {"A", "B"}}
+	steps := buildExperimentArtifactDownloadSteps("", experiments)
+	require.NotEmpty(t, steps, "steps should be generated")
+	joined := strings.Join(steps, "")
+	// Artifact name should be just the base name (no prefix)
+	assert.Contains(t, joined, "          name: experiment\n", "artifact name should be unqualified for non-workflow_call")
+}
diff --git a/pkg/workflow/detection_success_test.go b/pkg/workflow/detection_success_test.go
index a38a525ce26..4b80d824f3c 100644
--- a/pkg/workflow/detection_success_test.go
+++ b/pkg/workflow/detection_success_test.go
@@ -325,3 +325,84 @@ Create an issue.
 		})
 	}
 }
+
+// TestDetectionJobDownloadsExperimentArtifact verifies that when experiments are declared,
+// the detection job includes a step to download the experiment artifact.
+func TestDetectionJobDownloadsExperimentArtifact(t *testing.T) {
+	tests := []struct {
+		name                   string
+		frontmatter            string
+		wantExperimentDownload bool
+	}{
+		{
+			name: "experiments declared — detection job downloads experiment artifact",
+			frontmatter: `---
+on: issues
+permissions:
+  contents: read
+engine: copilot
+experiments:
+  caveman: [yes, no]
+safe-outputs:
+  create-issue:
+---
+
+# Test
+
+Create an issue.
+`,
+			wantExperimentDownload: true,
+		},
+		{
+			name: "no experiments — detection job does not download experiment artifact",
+			frontmatter: `---
+on: issues
+permissions:
+  contents: read
+engine: copilot
+safe-outputs:
+  create-issue:
+---
+
+# Test
+
+Create an issue.
+`,
+			wantExperimentDownload: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpDir := testutil.TempDir(t, "test-*")
+			workflowPath := filepath.Join(tmpDir, "test-workflow.md")
+
+			if err := os.WriteFile(workflowPath, []byte(tt.frontmatter), 0644); err != nil {
+				t.Fatalf("Failed to write workflow file: %v", err)
+			}
+
+			compiler := NewCompiler()
+			if err := compiler.CompileWorkflow(workflowPath); err != nil {
+				t.Fatalf("Failed to compile: %v", err)
+			}
+
+			lockPath := stringutil.MarkdownToLockFile(workflowPath)
+			yamlBytes, err := os.ReadFile(lockPath)
+			if err != nil {
+				t.Fatalf("Failed to read compiled YAML: %v", err)
+			}
+			detectionSection := extractJobSection(string(yamlBytes), "detection")
+			if detectionSection == "" {
+				t.Fatal("Detection job not found in compiled YAML")
+			}
+
+			hasDownload := strings.Contains(detectionSection, "Download experiment artifact")
+			if tt.wantExperimentDownload && !hasDownload {
+				t.Error("Expected detection job to download experiment artifact when experiments are declared")
+			}
+			if !tt.wantExperimentDownload && hasDownload {
+				t.Error("Expected detection job NOT to download experiment artifact when no experiments are declared")
+			}
+		})
+	}
+}
diff --git a/pkg/workflow/threat_detection.go b/pkg/workflow/threat_detection.go
index ec73c16ef34..cbb856325ac 100644
--- a/pkg/workflow/threat_detection.go
+++ b/pkg/workflow/threat_detection.go
@@ -891,6 +891,11 @@ func (c *Compiler) buildDetectionJob(data *WorkflowData) (*Job, error) {
 	agentArtifactPrefix := artifactPrefixExprForAgentDownstreamJob(data)
 	steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix)...)
 
+	// Download experiment artifact so the detection agent can read the current variant assignments.
+	// The experiment artifact is uploaded by the activation job, so use the activation-downstream prefix.
+	experimentPrefix := artifactPrefixExprForDownstreamJob(data)
+	steps = append(steps, buildExperimentArtifactDownloadSteps(experimentPrefix, data.Experiments)...)
+
 	// Conditionally checkout the target repository so the detection engine can
 	// analyze patches in the context of the surrounding codebase.
 	steps = append(steps, c.buildWorkspaceCheckoutForDetectionStep(data)...)
