From 19cb3dac9a0e7f99cc4ad9ec5dfb9906c26251b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 16:32:12 +0000 Subject: [PATCH 1/4] Initial plan From 3f47611048a5ef4d018d6d3bb5126daf67d7320a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 17:05:58 +0000 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20add=20structured=20output=20mode=20?= =?UTF-8?q?=E2=80=94=20constrain=20agent=20responses=20to=20a=20declared?= =?UTF-8?q?=20JSON=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves: #structured-output-mode - New `structured-output` frontmatter field accepting `schema` (inline) or `schema-file` (file path) containing a JSON Schema draft-07 object - Compile-time validation of the schema using santhosh-tekuri/jsonschema/v6 - Pre-agent step writes schema to /tmp/gh-aw/structured-output-schema.json - GH_AW_STRUCTURED_OUTPUT_SCHEMA and GH_AW_STRUCTURED_OUTPUT_FILE env vars injected into all engine execution steps (copilot, claude, codex, crush, gemini, opencode) - Post-agent validation step (actions/github-script, always runs) checks /tmp/gh-aw/structured-output.json is well-formed JSON and sets the `validate_structured_output` step output - Agent job exposes `structured_output` output for downstream job consumption - JSON schema updated with `structured-output` property definition - StructuredOutputConfig added to FrontmatterConfig and WorkflowData - Unit tests covering all new functions Agent-Logs-Url: https://github.com/github/gh-aw/sessions/d1463ae7-cb32-4831-981b-39e1993c2cbc Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- pkg/parser/schemas/main_workflow_schema.json | 21 + pkg/workflow/claude_engine.go | 3 + pkg/workflow/codex_engine.go | 3 + pkg/workflow/compiler_main_job.go | 7 + .../compiler_orchestrator_workflow.go | 7 + pkg/workflow/compiler_types.go | 4 + pkg/workflow/compiler_yaml_main_job.go | 14 + pkg/workflow/copilot_engine_execution.go | 3 + pkg/workflow/crush_engine.go | 3 + pkg/workflow/frontmatter_types.go | 4 + pkg/workflow/gemini_engine.go | 3 + pkg/workflow/opencode_engine.go | 3 + pkg/workflow/structured_output.go | 254 ++++++++++++ pkg/workflow/structured_output_test.go | 378 ++++++++++++++++++ 14 files changed, 707 insertions(+) create mode 100644 pkg/workflow/structured_output.go create mode 100644 pkg/workflow/structured_output_test.go diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 8aed06e8750..eb5c8eb64d8 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -8831,6 +8831,27 @@ "description": "Allow npm pre/post install scripts to execute during package installation. By default, --ignore-scripts is added to all generated npm install commands to prevent supply chain attacks via malicious install hooks. Setting run-install-scripts: true disables this protection globally (all runtimes). A supply chain security warning is emitted at compile time; in strict mode this is an error. Per-runtime control is also available via runtimes..run-install-scripts. See: https://github.github.com/gh-aw/reference/frontmatter/#run-install-scripts", "examples": [false, true] }, + "structured-output": { + "type": "object", + "description": "Declares a JSON Schema (draft-07) that the agent's primary response must conform to. Enables structured output mode: the compiler validates the schema at compile time, writes it to disk before agent execution, and a post-agent step validates that the agent produced well-formed JSON matching the schema. The validated JSON is exposed as the `structured_output` job output for downstream jobs. Exactly one of 'schema' (inline) or 'schema-file' (file path) must be specified.", + "properties": { + "schema": { + "type": "object", + "description": "Inline JSON Schema object (draft-07 compatible) that the agent's response must conform to. Mutually exclusive with 'schema-file'.", + "additionalProperties": true + }, + "schema-file": { + "type": "string", + "description": "Path to a JSON Schema file (relative to the workflow file). The file must contain a valid JSON Schema object. Mutually exclusive with 'schema'.", + "examples": [".github/schemas/triage-output.schema.json"] + } + }, + "oneOf": [ + { "required": ["schema"], "not": { "required": ["schema-file"] } }, + { "required": ["schema-file"], "not": { "required": ["schema"] } } + ], + "additionalProperties": false + }, "mcp-scripts": { "type": "object", "description": "MCP Scripts configuration for defining custom lightweight MCP tools as JavaScript, shell scripts, or Python scripts. Tools are mounted in an MCP server and have access to secrets specified by the user. Only one of 'script' (JavaScript), 'run' (shell), or 'py' (Python) must be specified per tool.", diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index eeceae5f634..6a0876053f6 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -357,6 +357,9 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str // Add GH_AW_SAFE_OUTPUTS if output is needed applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + // Add GH_AW_STARTUP_TIMEOUT environment variable (in seconds) if startup-timeout is specified // Supports both literal integers and GitHub Actions expressions (e.g. "${{ inputs.startup-timeout }}") if workflowData.ToolsStartupTimeout != "" { diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index f4d529c93f2..9e0d0b0f8cf 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -302,6 +302,9 @@ mkdir -p "$CODEX_HOME/logs" // Add GH_AW_SAFE_OUTPUTS if output is needed applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + // In sandbox (AWF) mode, set git identity environment variables so the first git commit // succeeds inside the container. AWF's --env-all forwards these to the container, ensuring // git does not rely on the host-side ~/.gitconfig which is not visible in the sandbox. diff --git a/pkg/workflow/compiler_main_job.go b/pkg/workflow/compiler_main_job.go index 70d9184034d..a2aea7c7ad1 100644 --- a/pkg/workflow/compiler_main_job.go +++ b/pkg/workflow/compiler_main_job.go @@ -171,6 +171,13 @@ func (c *Compiler) buildMainJob(data *WorkflowData, activationJobCreated bool) ( outputs["has_patch"] = "${{ steps.collect_output.outputs.has_patch }}" } + // Add structured_output job output when structured-output mode is configured. + // The value is the validated, compact JSON string produced by the validate_structured_output step. + if HasStructuredOutput(data) { + outputs["structured_output"] = "${{ steps.validate_structured_output.outputs.structured_output }}" + compilerMainJobLog.Print("Added structured_output output (structured-output mode enabled)") + } + // Add checkout_pr_success output to track PR checkout status only if the checkout-pr step will be generated // This is used by the conclusion job to skip failure handling when checkout fails // (e.g., when PR is merged and branch is deleted) diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index c053f0d63d4..30689ff1703 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -295,6 +295,13 @@ func (c *Compiler) extractAdditionalConfigurations( workflowData.MCPScripts = c.mergeMCPScripts(workflowData.MCPScripts, importsResult.MergedMCPScripts) } + // Extract structured-output configuration (compile-time schema validation included) + structuredOutputConfig, err := extractStructuredOutputConfig(frontmatter, markdownDir) + if err != nil { + return fmt.Errorf("structured-output: %w", err) + } + workflowData.StructuredOutputConfig = structuredOutputConfig + // Extract safe-jobs from safe-outputs.jobs location topSafeJobs := extractSafeJobsFromFrontmatter(frontmatter) diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go index b46934438ef..a6295d0102e 100644 --- a/pkg/workflow/compiler_types.go +++ b/pkg/workflow/compiler_types.go @@ -516,6 +516,10 @@ type WorkflowData struct { CachedParsedToolsets []string // cached result of ParseGitHubToolsets for the GitHub tool (for performance optimization); populated by applyDefaults CachedAllowedDomainsStr string // cached allowed-domains string for sanitization (for performance optimization); computed once and reused across multiple compilation steps CachedAllowedDomainsComputed bool // true once CachedAllowedDomainsStr has been set; distinguishes "computed empty" from "not yet computed" + + // StructuredOutputConfig holds the resolved structured-output configuration. + // Set when the frontmatter declares a structured-output section. + StructuredOutputConfig *StructuredOutputConfig } // PinContext returns an actionpins.PinContext backed by this WorkflowData. diff --git a/pkg/workflow/compiler_yaml_main_job.go b/pkg/workflow/compiler_yaml_main_job.go index 6e0ee6a3921..ecdfb62f17d 100644 --- a/pkg/workflow/compiler_yaml_main_job.go +++ b/pkg/workflow/compiler_yaml_main_job.go @@ -397,6 +397,12 @@ func (c *Compiler) generateMainJobSteps(yaml *strings.Builder, data *WorkflowDat // connects to via host.docker.internal:18443. c.generateStartCliProxyStep(yaml, data) + // Write structured-output schema to disk before engine execution so the agent can + // discover the schema file via GH_AW_STRUCTURED_OUTPUT_SCHEMA and produce a conforming response. + if schemaStep := generateStructuredOutputSchemaStep(data); schemaStep != "" { + yaml.WriteString(schemaStep) + } + // Add AI execution step using the agentic engine compilerYamlLog.Printf("Generating engine execution steps for %s", engine.GetID()) c.generateEngineExecutionSteps(yaml, data, engine, logFileFull) @@ -459,6 +465,14 @@ func (c *Compiler) generateMainJobSteps(yaml *strings.Builder, data *WorkflowDat // secret redaction already scanned this file, so it is safe to append. c.generateAgentStepSummaryAppend(yaml) + // Validate structured output (if configured): reads the agent's structured output file, + // confirms it is well-formed JSON, and exposes it as the validate_structured_output step output. + if validationStep := generateStructuredOutputValidationStep(data, getCachedActionPin); validationStep != "" { + yaml.WriteString(validationStep) + // Include the structured output file in the agent artifact for traceability. + artifactPaths = append(artifactPaths, StructuredOutputFilePath) + } + // Add output collection step only if safe-outputs feature is used (GH_AW_SAFE_OUTPUTS functionality) if data.SafeOutputs != nil { if err := c.generateOutputCollectionStep(yaml, data); err != nil { diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 5b6e286456f..0c54a03d171 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -388,6 +388,9 @@ touch %s // Add GH_AW_SAFE_OUTPUTS if output is needed applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + // Add GH_AW_STARTUP_TIMEOUT environment variable (in seconds) if startup-timeout is specified // Supports both literal integers and GitHub Actions expressions (e.g. "${{ inputs.startup-timeout }}") if workflowData.ToolsStartupTimeout != "" { diff --git a/pkg/workflow/crush_engine.go b/pkg/workflow/crush_engine.go index c72e94cb762..9836a65bbf7 100644 --- a/pkg/workflow/crush_engine.go +++ b/pkg/workflow/crush_engine.go @@ -186,6 +186,9 @@ func (e *CrushEngine) GetExecutionSteps(workflowData *WorkflowData, logFile stri // Safe outputs env applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + // Model env var (only when explicitly configured) if modelConfigured { crushLog.Printf("Setting %s env var for model: %s", diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go index 46761b4d185..6ff30e00e28 100644 --- a/pkg/workflow/frontmatter_types.go +++ b/pkg/workflow/frontmatter_types.go @@ -207,6 +207,10 @@ type FrontmatterConfig struct { SecretMasking *SecretMaskingConfig `json:"secret-masking,omitempty"` Observability *ObservabilityConfig `json:"observability,omitempty"` + // Structured output configuration — declares a JSON Schema that the agent's primary + // response must conform to. Either 'schema' (inline) or 'schema-file' (file path) is required. + StructuredOutput *StructuredOutputConfig `json:"structured-output,omitempty"` + // Rate limiting configuration RateLimit *RateLimitConfig `json:"rate-limit,omitempty"` diff --git a/pkg/workflow/gemini_engine.go b/pkg/workflow/gemini_engine.go index ca004d08e49..1a97e9423dd 100644 --- a/pkg/workflow/gemini_engine.go +++ b/pkg/workflow/gemini_engine.go @@ -288,6 +288,9 @@ touch %s // Add safe outputs env applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + // Set the model environment variable only when explicitly configured. // When model is configured, use the native GEMINI_MODEL env var - the Gemini CLI reads it // directly, avoiding the need to embed the value in the shell command (which would fail diff --git a/pkg/workflow/opencode_engine.go b/pkg/workflow/opencode_engine.go index 196bf54f2ea..b8196a875e1 100644 --- a/pkg/workflow/opencode_engine.go +++ b/pkg/workflow/opencode_engine.go @@ -163,6 +163,9 @@ func (e *OpenCodeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile s applySafeOutputEnvToMap(env, workflowData) + // Add GH_AW_STRUCTURED_OUTPUT_SCHEMA / GH_AW_STRUCTURED_OUTPUT_FILE if structured output is configured + applyStructuredOutputEnvToMap(env, workflowData) + if modelConfigured { openCodeLog.Printf("Setting %s env var for model: %s", constants.OpenCodeCLIModelEnvVar, workflowData.EngineConfig.Model) diff --git a/pkg/workflow/structured_output.go b/pkg/workflow/structured_output.go new file mode 100644 index 00000000000..62cde83091a --- /dev/null +++ b/pkg/workflow/structured_output.go @@ -0,0 +1,254 @@ +// Package workflow provides support for structured output mode in agentic workflows. +// +// Structured output mode allows workflows to declare a JSON Schema that constrains +// the agent's primary response. This enables deterministic machine-readable output +// from agent jobs, making multi-agent pipelines and data extraction workflows reliable. +// +// The feature works as follows: +// 1. Compile-time: gh aw compile validates the schema is well-formed JSON Schema. +// 2. Pre-agent: The runtime writes the schema to /tmp/gh-aw/structured-output-schema.json +// and sets GH_AW_STRUCTURED_OUTPUT_SCHEMA so the agent knows to produce structured output. +// 3. Post-agent: A validation step checks the agent wrote a valid JSON file at +// /tmp/gh-aw/structured-output.json and exposes it as a typed job output. + +package workflow + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/github/gh-aw/pkg/logger" + "github.com/santhosh-tekuri/jsonschema/v6" +) + +var structuredOutputLog = logger.New("workflow:structured_output") + +const ( + // StructuredOutputSchemaPath is the runtime path for the compiled schema file. + StructuredOutputSchemaPath = "/tmp/gh-aw/structured-output-schema.json" + + // StructuredOutputFilePath is the runtime path where the agent must write its structured output. + StructuredOutputFilePath = "/tmp/gh-aw/structured-output.json" +) + +// StructuredOutputConfig holds configuration for structured output mode. +// Either Schema (inline) or SchemaFile (file reference) must be specified, not both. +type StructuredOutputConfig struct { + // Schema is an inline JSON Schema object (draft-07 compatible). + Schema map[string]any `json:"schema,omitempty" yaml:"schema,omitempty"` + + // SchemaFile is a repo-root-relative or workflow-relative path to a JSON Schema file. + SchemaFile string `json:"schema-file,omitempty" yaml:"schema-file,omitempty"` + + // resolvedSchemaJSON holds the serialized JSON string of the resolved schema, + // populated during compilation after reading schema-file if specified. + resolvedSchemaJSON string +} + +// GetResolvedSchemaJSON returns the resolved JSON schema as a compact string. +func (c *StructuredOutputConfig) GetResolvedSchemaJSON() string { + return c.resolvedSchemaJSON +} + +// HasStructuredOutput returns true when the workflow has structured output configured. +func HasStructuredOutput(data *WorkflowData) bool { + return data != nil && data.StructuredOutputConfig != nil +} + +// extractStructuredOutputConfig extracts and validates the structured-output frontmatter section. +// workflowDir is the directory containing the workflow file, used to resolve schema-file paths. +func extractStructuredOutputConfig(frontmatter map[string]any, workflowDir string) (*StructuredOutputConfig, error) { + raw, exists := frontmatter["structured-output"] + if !exists || raw == nil { + return nil, nil + } + + configMap, ok := raw.(map[string]any) + if !ok { + return nil, errors.New("structured-output must be an object") + } + + config := &StructuredOutputConfig{} + + // Parse inline schema + if schemaRaw, ok := configMap["schema"]; ok && schemaRaw != nil { + if schemaMap, ok := schemaRaw.(map[string]any); ok { + config.Schema = schemaMap + } else { + return nil, errors.New("structured-output.schema must be a JSON Schema object") + } + } + + // Parse schema-file reference + if schemaFileRaw, ok := configMap["schema-file"]; ok && schemaFileRaw != nil { + if schemaFileStr, ok := schemaFileRaw.(string); ok && schemaFileStr != "" { + config.SchemaFile = schemaFileStr + } else { + return nil, errors.New("structured-output.schema-file must be a non-empty string path") + } + } + + // Validate: exactly one of schema or schema-file must be specified + hasInline := config.Schema != nil + hasFile := config.SchemaFile != "" + if !hasInline && !hasFile { + return nil, errors.New("structured-output requires either 'schema' (inline object) or 'schema-file' (file path)") + } + if hasInline && hasFile { + return nil, errors.New("structured-output cannot specify both 'schema' and 'schema-file'") + } + + // Load schema-file contents at compile time + if hasFile { + schemaPath := config.SchemaFile + if !filepath.IsAbs(schemaPath) { + schemaPath = filepath.Join(workflowDir, schemaPath) + } + schemaBytes, err := os.ReadFile(schemaPath) + if err != nil { + return nil, fmt.Errorf("structured-output.schema-file: cannot read %q: %w", config.SchemaFile, err) + } + var schemaMap map[string]any + if err := json.Unmarshal(schemaBytes, &schemaMap); err != nil { + return nil, fmt.Errorf("structured-output.schema-file: %q is not valid JSON: %w", config.SchemaFile, err) + } + config.Schema = schemaMap + } + + // Validate that the schema is well-formed JSON Schema (draft-07) + if err := validateJSONSchema(config.Schema); err != nil { + return nil, fmt.Errorf("structured-output schema is not valid JSON Schema: %w", err) + } + + // Serialize the resolved schema for embedding in generated workflow steps + schemaJSON, err := json.Marshal(config.Schema) + if err != nil { + return nil, fmt.Errorf("structured-output: failed to serialize schema: %w", err) + } + config.resolvedSchemaJSON = string(schemaJSON) + + structuredOutputLog.Printf("Extracted structured-output config: schema=%d chars", len(config.resolvedSchemaJSON)) + return config, nil +} + +// validateJSONSchema checks that the provided map is a valid JSON Schema using the +// santhosh-tekuri/jsonschema compiler (draft-07 compatible). +func validateJSONSchema(schema map[string]any) error { + compiler := jsonschema.NewCompiler() + + // The compiler's AddResource accepts a pre-parsed document (any), not raw bytes. + const schemaURL = "urn:gh-aw:structured-output-schema" + if err := compiler.AddResource(schemaURL, schema); err != nil { + return fmt.Errorf("invalid JSON Schema: %w", err) + } + if _, err := compiler.Compile(schemaURL); err != nil { + return fmt.Errorf("invalid JSON Schema: %w", err) + } + return nil +} + +// applyStructuredOutputEnvToMap adds structured-output related environment variables +// to the provided env map. It is safe to call when structured output is not configured +// (the function is a no-op in that case). +// +// The following env vars are set for non-detection runs: +// - GH_AW_STRUCTURED_OUTPUT_SCHEMA: path to the JSON Schema file on disk +// - GH_AW_STRUCTURED_OUTPUT_FILE: path where the agent must write its JSON output +func applyStructuredOutputEnvToMap(env map[string]string, data *WorkflowData) { + if !HasStructuredOutput(data) || data.IsDetectionRun { + return + } + env["GH_AW_STRUCTURED_OUTPUT_SCHEMA"] = StructuredOutputSchemaPath + env["GH_AW_STRUCTURED_OUTPUT_FILE"] = StructuredOutputFilePath +} + +// generateStructuredOutputSchemaStep returns the YAML for a pre-agent step that writes +// the resolved JSON Schema to disk so the agent can discover it at runtime. +// Returns an empty string when structured output is not configured. +func generateStructuredOutputSchemaStep(data *WorkflowData) string { + if !HasStructuredOutput(data) { + return "" + } + + schemaJSON := data.StructuredOutputConfig.GetResolvedSchemaJSON() + if schemaJSON == "" { + return "" + } + + // Escape single quotes so the JSON can be safely embedded in a shell single-quoted argument. + // json.Marshal produces compact JSON (no newlines), so the entire schema is one line. + escaped := shellEscapeSingleQuote(schemaJSON) + + var sb strings.Builder + sb.WriteString(" - name: Set up structured output schema\n") + sb.WriteString(" run: |\n") + sb.WriteString(" mkdir -p /tmp/gh-aw\n") + fmt.Fprintf(&sb, " printf '%%s' '%s' > %s\n", escaped, StructuredOutputSchemaPath) + sb.WriteString("\n") + + structuredOutputLog.Printf("Generated structured-output schema write step (%d chars schema)", len(schemaJSON)) + return sb.String() +} + +// generateStructuredOutputValidationStep returns the YAML for a post-agent step that +// reads the agent's structured output file, validates it is well-formed JSON, and +// exposes it as the `structured_output` step output for downstream consumption. +// +// The step always runs (if: always()) so it reports validation failures even when the +// agent step itself has failed. Returns an empty string when structured output is not configured. +func generateStructuredOutputValidationStep(data *WorkflowData, actionPinFunc func(string, *WorkflowData) string) string { + if !HasStructuredOutput(data) { + return "" + } + + actionPin := "actions/github-script@v7" + if actionPinFunc != nil { + pin := actionPinFunc("actions/github-script", data) + if pin != "" { + actionPin = pin + } + } + + // Build JavaScript validation script + var script strings.Builder + fmt.Fprintf(&script, " const fs = require('fs');\n") + fmt.Fprintf(&script, " const outputPath = '%s';\n", StructuredOutputFilePath) + fmt.Fprintf(&script, " if (!fs.existsSync(outputPath)) {\n") + fmt.Fprintf(&script, " core.setFailed(`Structured output file not found: ${outputPath}. The agent must write its JSON response to this path.`);\n") + fmt.Fprintf(&script, " return;\n") + fmt.Fprintf(&script, " }\n") + fmt.Fprintf(&script, " const raw = fs.readFileSync(outputPath, 'utf8');\n") + fmt.Fprintf(&script, " let parsed;\n") + fmt.Fprintf(&script, " try {\n") + fmt.Fprintf(&script, " parsed = JSON.parse(raw);\n") + fmt.Fprintf(&script, " } catch (e) {\n") + fmt.Fprintf(&script, " core.setFailed(`Structured output is not valid JSON: ${e.message}`);\n") + fmt.Fprintf(&script, " return;\n") + fmt.Fprintf(&script, " }\n") + fmt.Fprintf(&script, " const compact = JSON.stringify(parsed);\n") + fmt.Fprintf(&script, " core.setOutput('structured_output', compact);\n") + fmt.Fprintf(&script, " core.info(`Structured output validated: ${compact.length} chars`);\n") + + var sb strings.Builder + sb.WriteString(" - name: Validate structured output\n") + sb.WriteString(" if: always()\n") + sb.WriteString(" id: validate_structured_output\n") + fmt.Fprintf(&sb, " uses: %s\n", actionPin) + sb.WriteString(" with:\n") + sb.WriteString(" script: |\n") + sb.WriteString(script.String()) + sb.WriteString("\n") + + return sb.String() +} + +// shellEscapeSingleQuote escapes a string for use inside single quotes in a shell command. +// In single-quoted strings the only character that requires special handling is the +// single quote itself, which is represented as: '\” +func shellEscapeSingleQuote(s string) string { + return strings.ReplaceAll(s, "'", `'\''`) +} diff --git a/pkg/workflow/structured_output_test.go b/pkg/workflow/structured_output_test.go new file mode 100644 index 00000000000..d70f9032660 --- /dev/null +++ b/pkg/workflow/structured_output_test.go @@ -0,0 +1,378 @@ +//go:build !integration + +package workflow + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestExtractStructuredOutputConfig tests the compile-time parsing and validation of +// the structured-output frontmatter section. +func TestExtractStructuredOutputConfig(t *testing.T) { + tests := []struct { + name string + frontmatter map[string]any + wantNil bool + wantErr bool + errContains string + }{ + { + name: "no structured-output field", + frontmatter: map[string]any{}, + wantNil: true, + }, + { + name: "structured-output is nil", + frontmatter: map[string]any{"structured-output": nil}, + wantNil: true, + }, + { + name: "valid inline schema - object type", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema": map[string]any{ + "type": "object", + "properties": map[string]any{ + "decision": map[string]any{"type": "string"}, + }, + "required": []any{"decision"}, + }, + }, + }, + wantNil: false, + }, + { + name: "schema with enum constraint", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema": map[string]any{ + "type": "object", + "properties": map[string]any{ + "decision": map[string]any{ + "type": "string", + "enum": []any{"APPROVE", "REQUEST_CHANGES", "ESCALATE"}, + }, + "confidence": map[string]any{ + "type": "number", + "minimum": 0, + "maximum": 1, + }, + }, + "required": []any{"decision", "confidence"}, + "additionalProperties": false, + }, + }, + }, + wantNil: false, + }, + { + name: "missing both schema and schema-file", + frontmatter: map[string]any{ + "structured-output": map[string]any{}, + }, + wantErr: true, + errContains: "requires either 'schema'", + }, + { + name: "both schema and schema-file specified", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema": map[string]any{"type": "object"}, + "schema-file": ".github/schemas/output.json", + }, + }, + wantErr: true, + errContains: "cannot specify both", + }, + { + name: "structured-output is not an object", + frontmatter: map[string]any{ + "structured-output": "invalid", + }, + wantErr: true, + errContains: "must be an object", + }, + { + name: "schema is not an object", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema": "not-an-object", + }, + }, + wantErr: true, + errContains: "must be a JSON Schema object", + }, + { + name: "schema-file is empty string", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema-file": "", + }, + }, + wantErr: true, + errContains: "must be a non-empty string path", + }, + { + name: "schema-file references non-existent file", + frontmatter: map[string]any{ + "structured-output": map[string]any{ + "schema-file": "nonexistent-schema.json", + }, + }, + wantErr: true, + errContains: "cannot read", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, err := extractStructuredOutputConfig(tt.frontmatter, "/tmp") + + if tt.wantErr { + require.Error(t, err, "expected an error") + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains, "error message should contain expected text") + } + return + } + + require.NoError(t, err, "expected no error") + if tt.wantNil { + assert.Nil(t, config, "expected nil config") + } else { + require.NotNil(t, config, "expected non-nil config") + assert.NotEmpty(t, config.GetResolvedSchemaJSON(), "expected resolved schema JSON to be set") + } + }) + } +} + +// TestHasStructuredOutput tests the HasStructuredOutput helper function. +func TestHasStructuredOutput(t *testing.T) { + assert.False(t, HasStructuredOutput(nil), "nil WorkflowData should return false") + + data := &WorkflowData{} + assert.False(t, HasStructuredOutput(data), "WorkflowData without StructuredOutputConfig should return false") + + data.StructuredOutputConfig = &StructuredOutputConfig{ + resolvedSchemaJSON: `{"type":"object"}`, + } + assert.True(t, HasStructuredOutput(data), "WorkflowData with StructuredOutputConfig should return true") +} + +// TestApplyStructuredOutputEnvToMap tests the environment variable injection. +func TestApplyStructuredOutputEnvToMap(t *testing.T) { + t.Run("no-op when structured output not configured", func(t *testing.T) { + env := make(map[string]string) + applyStructuredOutputEnvToMap(env, &WorkflowData{}) + assert.Empty(t, env, "env should be unchanged when no structured output configured") + }) + + t.Run("no-op for detection runs", func(t *testing.T) { + env := make(map[string]string) + data := &WorkflowData{ + StructuredOutputConfig: &StructuredOutputConfig{resolvedSchemaJSON: `{"type":"object"}`}, + IsDetectionRun: true, + } + applyStructuredOutputEnvToMap(env, data) + assert.Empty(t, env, "env should be unchanged for detection runs") + }) + + t.Run("sets env vars for configured workflow", func(t *testing.T) { + env := make(map[string]string) + data := &WorkflowData{ + StructuredOutputConfig: &StructuredOutputConfig{resolvedSchemaJSON: `{"type":"object"}`}, + } + applyStructuredOutputEnvToMap(env, data) + assert.Equal(t, StructuredOutputSchemaPath, env["GH_AW_STRUCTURED_OUTPUT_SCHEMA"], + "GH_AW_STRUCTURED_OUTPUT_SCHEMA should point to the schema file") + assert.Equal(t, StructuredOutputFilePath, env["GH_AW_STRUCTURED_OUTPUT_FILE"], + "GH_AW_STRUCTURED_OUTPUT_FILE should point to the output file") + }) +} + +// TestGenerateStructuredOutputSchemaStep tests the pre-agent schema write step generation. +func TestGenerateStructuredOutputSchemaStep(t *testing.T) { + t.Run("returns empty for unconfigured workflow", func(t *testing.T) { + step := generateStructuredOutputSchemaStep(&WorkflowData{}) + assert.Empty(t, step, "step should be empty when structured output is not configured") + }) + + t.Run("generates step for configured workflow", func(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "decision": map[string]any{"type": "string"}, + }, + } + config, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + require.NoError(t, err) + + data := &WorkflowData{StructuredOutputConfig: config} + step := generateStructuredOutputSchemaStep(data) + + assert.NotEmpty(t, step, "step should not be empty for configured workflow") + assert.Contains(t, step, "Set up structured output schema", "step should have correct name") + assert.Contains(t, step, StructuredOutputSchemaPath, "step should reference schema path") + assert.Contains(t, step, "mkdir -p /tmp/gh-aw", "step should create directory") + assert.Contains(t, step, "printf", "step should use printf to write the schema") + }) + + t.Run("schema JSON is embedded in step", func(t *testing.T) { + schema := map[string]any{"type": "string"} + config, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + require.NoError(t, err) + + data := &WorkflowData{StructuredOutputConfig: config} + step := generateStructuredOutputSchemaStep(data) + + // The step should contain "type" from the schema + assert.Contains(t, step, "type", "schema content should appear in step") + }) +} + +// TestGenerateStructuredOutputValidationStep tests the post-agent validation step generation. +func TestGenerateStructuredOutputValidationStep(t *testing.T) { + t.Run("returns empty for unconfigured workflow", func(t *testing.T) { + step := generateStructuredOutputValidationStep(&WorkflowData{}, nil) + assert.Empty(t, step, "step should be empty when structured output is not configured") + }) + + t.Run("generates validation step", func(t *testing.T) { + schema := map[string]any{"type": "object"} + config, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + require.NoError(t, err) + + data := &WorkflowData{StructuredOutputConfig: config} + step := generateStructuredOutputValidationStep(data, nil) + + assert.NotEmpty(t, step, "step should not be empty for configured workflow") + assert.Contains(t, step, "Validate structured output", "step should have correct name") + assert.Contains(t, step, "if: always()", "validation step should always run") + assert.Contains(t, step, "id: validate_structured_output", "step should have correct id") + assert.Contains(t, step, "actions/github-script", "step should use github-script") + assert.Contains(t, step, StructuredOutputFilePath, "step should reference output file path") + assert.Contains(t, step, "structured_output", "step should set structured_output output") + }) + + t.Run("uses custom action pin function", func(t *testing.T) { + schema := map[string]any{"type": "object"} + config, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + require.NoError(t, err) + + data := &WorkflowData{StructuredOutputConfig: config} + customPin := func(repo string, _ *WorkflowData) string { + return repo + "@abc123" + } + step := generateStructuredOutputValidationStep(data, customPin) + + assert.Contains(t, step, "actions/github-script@abc123", "step should use custom action pin") + }) +} + +// TestShellEscapeSingleQuote tests the single-quote escaping helper. +func TestShellEscapeSingleQuote(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"hello", "hello"}, + {"no quotes", "no quotes"}, + {"it's a test", `it'\''s a test`}, + {"a'b'c", `a'\''b'\''c`}, + {`{"key": "value"}`, `{"key": "value"}`}, + {`{"key": "it's here"}`, `{"key": "it'\''s here"}`}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := shellEscapeSingleQuote(tt.input) + assert.Equal(t, tt.expected, result, "single quote escaping should be correct") + }) + } +} + +// TestValidateJSONSchema tests the JSON Schema validation function. +func TestValidateJSONSchema(t *testing.T) { + t.Run("valid minimal schema", func(t *testing.T) { + schema := map[string]any{"type": "object"} + err := validateJSONSchema(schema) + assert.NoError(t, err, "minimal object schema should be valid") + }) + + t.Run("valid schema with properties", func(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "name": map[string]any{"type": "string"}, + "age": map[string]any{"type": "integer"}, + }, + "required": []any{"name"}, + } + err := validateJSONSchema(schema) + assert.NoError(t, err, "schema with properties should be valid") + }) + + t.Run("valid schema with enum", func(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "status": map[string]any{ + "type": "string", + "enum": []any{"PENDING", "ACTIVE", "CLOSED"}, + }, + }, + } + err := validateJSONSchema(schema) + assert.NoError(t, err, "schema with enum should be valid") + }) + + t.Run("empty schema is valid", func(t *testing.T) { + schema := map[string]any{} + err := validateJSONSchema(schema) + assert.NoError(t, err, "empty schema should be valid (matches anything)") + }) +} + +// TestStructuredOutputSchemaWriteEscaping tests that schemas with special characters +// are correctly escaped when embedded in the shell step. +func TestStructuredOutputSchemaWriteEscaping(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "message": map[string]any{ + "type": "string", + "description": "A message with 'single quotes' inside", + }, + }, + } + config, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + require.NoError(t, err, "schema with single quotes in description should parse") + + data := &WorkflowData{StructuredOutputConfig: config} + step := generateStructuredOutputSchemaStep(data) + + // Verify the generated step is well-formed YAML (starts correctly) + assert.True(t, strings.HasPrefix(step, " - name:"), "step should start with YAML list item") + // The escaped single quotes should appear as '\'' in the generated step + assert.Contains(t, step, `'\''`, "single quotes in schema should be escaped as '\\''") +} From a9173058b71456123c509c5e38b264a34fe5a354 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 17:12:28 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20address=20code=20review=20=E2=80=94?= =?UTF-8?q?=20fix=20comment=20typo,=20add=20edge=20case=20tests=20for=20sh?= =?UTF-8?q?ellEscapeSingleQuote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/github/gh-aw/sessions/d1463ae7-cb32-4831-981b-39e1993c2cbc Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- pkg/workflow/structured_output.go | 2 +- pkg/workflow/structured_output_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/workflow/structured_output.go b/pkg/workflow/structured_output.go index 62cde83091a..cf4c8862d2c 100644 --- a/pkg/workflow/structured_output.go +++ b/pkg/workflow/structured_output.go @@ -248,7 +248,7 @@ func generateStructuredOutputValidationStep(data *WorkflowData, actionPinFunc fu // shellEscapeSingleQuote escapes a string for use inside single quotes in a shell command. // In single-quoted strings the only character that requires special handling is the -// single quote itself, which is represented as: '\” +// single quote itself, which is represented as: '\” (close-quote, escaped-quote, open-quote) func shellEscapeSingleQuote(s string) string { return strings.ReplaceAll(s, "'", `'\''`) } diff --git a/pkg/workflow/structured_output_test.go b/pkg/workflow/structured_output_test.go index d70f9032660..a55d710850b 100644 --- a/pkg/workflow/structured_output_test.go +++ b/pkg/workflow/structured_output_test.go @@ -292,12 +292,16 @@ func TestShellEscapeSingleQuote(t *testing.T) { input string expected string }{ + {"", ""}, {"hello", "hello"}, {"no quotes", "no quotes"}, {"it's a test", `it'\''s a test`}, {"a'b'c", `a'\''b'\''c`}, + {"'''", `'\'''\'''\''`}, {`{"key": "value"}`, `{"key": "value"}`}, {`{"key": "it's here"}`, `{"key": "it'\''s here"}`}, + // very long string with no single quotes + {strings.Repeat("a", 10000), strings.Repeat("a", 10000)}, } for _, tt := range tests { From 5aa6e43de0ce14ee10b3639a2314b7d1b4613c78 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 22:37:06 +0000 Subject: [PATCH 4/4] feat(codex): pass --output-schema to codex exec when structured-output is configured When structured-output is configured and it is not a detection run, the Codex exec command now includes --output-schema so the model is constrained at the token-sampling level to produce schema-conformant JSON in addition to the file-write convention enforced by the post-agent validation step. Ref: https://openai.github.io/codex/cli/exec#param-output-schema Agent-Logs-Url: https://github.com/github/gh-aw/sessions/2811a83a-be95-4f17-a521-37792e0ae26d Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/codex_engine.go | 15 ++++++- pkg/workflow/codex_engine_test.go | 72 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 9e0d0b0f8cf..bf9abf53396 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -192,6 +192,17 @@ func (e *CodexEngine) GetExecutionSteps(workflowData *WorkflowData, logFile stri customArgsParam += customArgsParamSb.String() } + // Build output-schema parameter for structured output mode. + // When structured-output is configured, pass --output-schema to Codex so the model + // is constrained at the token-sampling level to produce schema-conformant JSON. + // The schema file is written by the "Set up structured output schema" pre-agent step. + // See: https://openai.github.io/codex/cli/exec#param-output-schema + var outputSchemaParam string + if HasStructuredOutput(workflowData) && !workflowData.IsDetectionRun { + outputSchemaParam = fmt.Sprintf("--output-schema %s ", StructuredOutputSchemaPath) + codexEngineLog.Printf("Adding --output-schema flag for structured output mode: %s", StructuredOutputSchemaPath) + } + // Build the Codex command // Determine which command to use var commandName string @@ -203,8 +214,8 @@ func (e *CodexEngine) GetExecutionSteps(workflowData *WorkflowData, logFile stri commandName = "codex" } - codexCommand := fmt.Sprintf("%s %sexec%s%s%s%s\"$INSTRUCTION\"", - commandName, modelParam, webSearchParam, webFetchParam, fullAutoParam, customArgsParam) + codexCommand := fmt.Sprintf("%s %sexec%s%s%s%s%s\"$INSTRUCTION\"", + commandName, modelParam, webSearchParam, webFetchParam, fullAutoParam, outputSchemaParam, customArgsParam) // Build the full command with agent file handling and AWF wrapping if enabled var command string diff --git a/pkg/workflow/codex_engine_test.go b/pkg/workflow/codex_engine_test.go index b6aa0fecf62..3c9b9a184f6 100644 --- a/pkg/workflow/codex_engine_test.go +++ b/pkg/workflow/codex_engine_test.go @@ -1007,3 +1007,75 @@ func TestCodexEngineWithExpressionVersion(t *testing.T) { t.Errorf("Expression should NOT be embedded directly in npm install command, got:\n%s", installStep) } } + +// TestCodexEngineStructuredOutputParam verifies that --output-schema is added to the codex +// exec command when structured-output is configured, and absent otherwise. +func TestCodexEngineStructuredOutputParam(t *testing.T) { + engine := NewCodexEngine() + + t.Run("no --output-schema without structured output config", func(t *testing.T) { + steps := engine.GetExecutionSteps(&WorkflowData{Name: "test-workflow"}, "/tmp/gh-aw/test.log") + if len(steps) == 0 { + t.Fatal("Expected at least one execution step") + } + stepContent := strings.Join([]string(steps[0]), "\n") + if strings.Contains(stepContent, "--output-schema") { + t.Errorf("Expected no --output-schema flag when structured output is not configured, got:\n%s", stepContent) + } + }) + + t.Run("--output-schema added with structured output config", func(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "decision": map[string]any{"type": "string"}, + }, + } + soConfig, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + if err != nil { + t.Fatalf("extractStructuredOutputConfig returned unexpected error: %v", err) + } + + data := &WorkflowData{ + Name: "test-workflow", + StructuredOutputConfig: soConfig, + } + steps := engine.GetExecutionSteps(data, "/tmp/gh-aw/test.log") + if len(steps) == 0 { + t.Fatal("Expected at least one execution step") + } + stepContent := strings.Join([]string(steps[0]), "\n") + expectedFlag := "--output-schema " + StructuredOutputSchemaPath + if !strings.Contains(stepContent, expectedFlag) { + t.Errorf("Expected %q in codex exec command for structured output, got:\n%s", expectedFlag, stepContent) + } + }) + + t.Run("no --output-schema for detection run", func(t *testing.T) { + schema := map[string]any{"type": "object"} + soConfig, err := extractStructuredOutputConfig( + map[string]any{"structured-output": map[string]any{"schema": schema}}, + "/tmp", + ) + if err != nil { + t.Fatalf("extractStructuredOutputConfig returned unexpected error: %v", err) + } + + data := &WorkflowData{ + Name: "test-workflow", + StructuredOutputConfig: soConfig, + IsDetectionRun: true, + } + steps := engine.GetExecutionSteps(data, "/tmp/gh-aw/test.log") + if len(steps) == 0 { + t.Fatal("Expected at least one execution step") + } + stepContent := strings.Join([]string(steps[0]), "\n") + if strings.Contains(stepContent, "--output-schema") { + t.Errorf("Expected no --output-schema flag for detection run, got:\n%s", stepContent) + } + }) +}