diff --git a/build_support/parser.rs b/build_support/parser.rs index 1d8bdfe..a108820 100644 --- a/build_support/parser.rs +++ b/build_support/parser.rs @@ -38,7 +38,20 @@ impl Level { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Applicability { Universal, - Conditional(String), + /// Conditional applicability. Either a prose `condition` (legacy `{ if: + /// "" }` shape, machine-opaque) or a machine-readable `antecedent` + /// (new `{ kind: conditional, antecedent: { check_id: ... } }` shape), or + /// both. The new shape is preferred; the legacy shape remains accepted for + /// rows where the verifier catalog has not yet grown a prerequisite check. + Conditional { + condition: Option, + antecedent: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Antecedent { + pub check_id: String, } #[derive(Debug, Clone)] @@ -91,6 +104,10 @@ pub enum ParseError { EmptyRequirements { file: String, }, + ConditionalEmpty { + file: String, + requirement_id: String, + }, } impl fmt::Display for ParseError { @@ -149,6 +166,13 @@ impl fmt::Display for ParseError { ParseError::EmptyRequirements { file } => { write!(f, "{file}: `requirements` list is empty") } + ParseError::ConditionalEmpty { + file, + requirement_id, + } => write!( + f, + "{file}: requirement `{requirement_id}` declares `kind: conditional` but provides neither `condition:` nor `antecedent.check_id` — at least one must be set" + ), } } } @@ -282,10 +306,23 @@ pub fn emit_rust(reqs: &[ParsedRequirement], spec_version: &str) -> String { Applicability::Universal => { out.push_str(" applicability: Applicability::Universal,\n"); } - Applicability::Conditional(cond) => { + Applicability::Conditional { + condition, + antecedent, + } => { + let cond_lit = match condition { + Some(c) => format!("Some(\"{}\")", escape_rust_str(c)), + None => "None".to_string(), + }; + let ante_lit = match antecedent { + Some(a) => format!( + "Some(Antecedent {{ check_id: \"{}\" }})", + escape_rust_str(&a.check_id) + ), + None => "None".to_string(), + }; out.push_str(&format!( - " applicability: Applicability::Conditional(\"{}\"),\n", - escape_rust_str(cond) + " applicability: Applicability::Conditional {{ condition: {cond_lit}, antecedent: {ante_lit} }},\n" )); } } @@ -397,6 +434,22 @@ fn parse_applicability( if let Some(map) = value.as_mapping() { let if_key = serde_yaml::Value::String("if".into()); + let kind_key = serde_yaml::Value::String("kind".into()); + + // Mixing the legacy `if:` and new `kind:` shape in one applicability + // block is ambiguous: the legacy branch only fires when `if:` is the + // single key, so a row carrying both would silently use the new shape + // and drop the legacy prose. Reject early with a pointer at the + // schema the author probably meant to use. + if map.contains_key(&if_key) && map.contains_key(&kind_key) { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`if:` and `kind:` cannot both be set; choose one (legacy `{ if: \"\" }` or new `{ kind: conditional, antecedent: { check_id: ... } }`)".into(), + }); + } + + // Legacy single-key `{ if: "" }` shape. if map.len() == 1 && let Some(if_val) = map.get(&if_key) { @@ -414,19 +467,121 @@ fn parse_applicability( hint: "`if:` value must be a non-empty string".into(), }); } - return Ok(Applicability::Conditional(cond.to_string())); + return Ok(Applicability::Conditional { + condition: Some(cond.to_string()), + antecedent: None, + }); } + + // New `{ kind: conditional, antecedent: { check_id: ... }, condition? }` + // shape. The `kind:` key gates the new branch so legacy maps with + // unrelated keys still surface as UnknownApplicability. + if let Some(kind_val) = map.get(&kind_key) { + let kind = kind_val + .as_str() + .ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`kind:` value must be the string `conditional`".into(), + })?; + if kind != "conditional" { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: format!("unknown `kind: {kind}` — only `conditional` is supported"), + }); + } + + let antecedent_key = serde_yaml::Value::String("antecedent".into()); + let condition_key = serde_yaml::Value::String("condition".into()); + + let antecedent = if let Some(ante_val) = map.get(&antecedent_key) { + let ante_map = + ante_val + .as_mapping() + .ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent:` must be a mapping like `{ check_id: }`" + .into(), + })?; + let check_id_key = serde_yaml::Value::String("check_id".into()); + // v1 schema is strict: only `check_id` is permitted inside + // `antecedent`. Compound antecedents (`op: any_of | all_of`) + // and any other key are explicitly deferred to a future + // schema bump per plan Sub-decision 2b. Silently ignoring + // unknown keys would let v2 syntax land in v1 vendored spec + // and behave subtly wrong. + for (k, _) in ante_map { + let k_str = k.as_str().unwrap_or(""); + if k_str != "check_id" { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: format!( + "`antecedent.{k_str}` is not part of the v1 schema (only `check_id` is permitted; compound antecedents deferred to v2 per plan Sub-decision 2b)" + ), + }); + } + } + let check_id = ante_map.get(&check_id_key).and_then(|v| v.as_str()).ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent.check_id` must be a non-empty string (v1 schema supports a single antecedent only)".into(), + })?; + if check_id.trim().is_empty() { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent.check_id` must be a non-empty string (whitespace-only is rejected)".into(), + }); + } + Some(Antecedent { + check_id: check_id.to_string(), + }) + } else { + None + }; + + let condition = match map.get(&condition_key) { + Some(v) => { + let s = v.as_str().ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`condition:` must be a string".into(), + })?; + if s.is_empty() { + None + } else { + Some(s.to_string()) + } + } + None => None, + }; + + if condition.is_none() && antecedent.is_none() { + return Err(ParseError::ConditionalEmpty { + file: file.to_string(), + requirement_id: req_id.to_string(), + }); + } + return Ok(Applicability::Conditional { + condition, + antecedent, + }); + } + return Err(ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), - hint: "expected `{ if: \"\" }`".into(), + hint: "expected `{ if: \"\" }` or `{ kind: conditional, antecedent: { check_id: } }`".into(), }); } Err(ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), - hint: "must be `universal` or `{ if: \"\" }`".into(), + hint: "must be `universal`, `{ if: \"\" }`, or `{ kind: conditional, antecedent: { check_id: } }`".into(), }) } diff --git a/coverage/matrix.json b/coverage/matrix.json index c576182..cdc8d53 100644 --- a/coverage/matrix.json +++ b/coverage/matrix.json @@ -192,7 +192,9 @@ "summary": "CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier.", "applicability": { "kind": "conditional", - "condition": "CLI emits structured output" + "antecedent": { + "check_id": "p2-json-output" + } }, "verifiers": [ { @@ -223,7 +225,9 @@ "summary": "Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool.", "applicability": { "kind": "conditional", - "condition": "CLI emits structured output" + "antecedent": { + "check_id": "p2-json-output" + } }, "verifiers": [ { @@ -885,7 +889,9 @@ "summary": "When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { @@ -920,7 +926,9 @@ "summary": "An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { @@ -936,7 +944,9 @@ "summary": "An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { diff --git a/docs/coverage-matrix.md b/docs/coverage-matrix.md index b096e6d..cf0abdb 100644 --- a/docs/coverage-matrix.md +++ b/docs/coverage-matrix.md @@ -33,9 +33,9 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | `p2-must-stdout-stderr-split` | MUST | Universal | `p2-output-module` (source) | Data goes to stdout; diagnostics/progress/warnings go to stderr, never interleaved. | | `p2-must-exit-codes` | MUST | Universal | `p2-structured-exit-codes` (behavioral) | Exit codes are structured and documented (0 success, 1 general, 2 usage, 77 auth, 78 config). | | `p2-must-json-errors` | MUST | Universal | `p2-json-errors` (behavioral) | When `--output json` is active, errors are emitted as JSON (to stderr) with at least `error`, `kind`, and `message` fields. | -| `p2-must-schema-print` | MUST | If: CLI emits structured output | `p2-schema-print` (behavioral) | CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier. | +| `p2-must-schema-print` | MUST | If: `p2-json-output` is present | `p2-schema-print` (behavioral) | CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier. | | `p2-should-consistent-envelope` | SHOULD | Universal | `p2-consistent-envelope` (behavioral) | JSON output uses a consistent envelope (a top-level object with predictable keys) across every command. | -| `p2-should-schema-file` | SHOULD | If: CLI emits structured output | `p2-schema-file` (project) | Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool. | +| `p2-should-schema-file` | SHOULD | If: `p2-json-output` is present | `p2-schema-file` (project) | Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool. | | `p2-should-json-aliases` | SHOULD | Universal | `p2-json-aliases` (behavioral) | `--json` and `--jsonl` are accepted as aliases for `--output json` and `--output jsonl`; the short forms work alongside the canonical enum. | | `p2-may-more-formats` | MAY | Universal | `p2-more-formats` (behavioral) | Additional output formats (CSV, TSV, YAML) beyond the core three. | | `p2-may-raw-flag` | MAY | Universal | `p2-raw-flag` (behavioral) | `--raw` flag for unformatted output suitable for piping to other tools. | @@ -107,8 +107,8 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | ID | Level | Applicability | Verifier(s) | Summary | | --- | --- | --- | --- | --- | -| `p8-must-bundle-install` | MUST | If: CLI ships an agent skill bundle | `p8-bundle-install` (behavioral) | When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes. | +| `p8-must-bundle-install` | MUST | If: `p8-bundle-exists` is present | `p8-bundle-install` (behavioral) | When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes. | | `p8-should-bundle-exists` | SHOULD | Universal | `p6-agents-md` (project)
`p8-bundle-exists` (project) | CLIs ship a top-level agent-discoverable markdown bundle (`AGENTS.md`, `SKILL.md`, or equivalent) with YAML frontmatter naming the tool and capability summary. | -| `p8-may-install-all` | MAY | If: CLI ships an agent skill bundle | `p8-install-all` (behavioral) | An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all. | -| `p8-may-bundle-update` | MAY | If: CLI ships an agent skill bundle | `p8-bundle-update` (behavioral) | An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version. | +| `p8-may-install-all` | MAY | If: `p8-bundle-exists` is present | `p8-install-all` (behavioral) | An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all. | +| `p8-may-bundle-update` | MAY | If: `p8-bundle-exists` is present | `p8-bundle-update` (behavioral) | An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version. | diff --git a/schema/scorecard.schema.json b/schema/scorecard.schema.json index 75a6e0f..6ca9df1 100644 --- a/schema/scorecard.schema.json +++ b/schema/scorecard.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://anc.dev/scorecard-v0.5.schema.json", + "$id": "https://anc.dev/scorecard-v0.6.schema.json", "title": "agentnative scorecard", - "description": "JSON Schema for `anc audit --output json` scorecards (schema version 0.5). Hand-written for v0.4.0. A schemars-derived version generated from src/scorecard/mod.rs is planned for a follow-up release; see docs/plans/2026-04-30-002-feat-scorecard-json-schema-plan.md on the `dev` branch.", + "description": "JSON Schema for `anc audit --output json` scorecards (schema version 0.6). Schema 0.6 introduces the 7-status taxonomy (`opt_out` and `n_a` added to `status`, with matching counters in `summary`), per-row emission (one result per requirement-row instead of per check_id; `tier` and `check_id` fields added to each result), and antecedent propagation for conditional requirements. See docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md in the agentnative-site repo for the full taxonomy rationale.", "type": "object", "required": [ "schema_version", @@ -23,11 +23,11 @@ "schema_version": { "type": "string", "description": "Scorecard schema version. Pre-launch additive — consumers feature-detect new fields rather than pin to an exact value.", - "examples": ["0.5"] + "examples": ["0.6"] }, "results": { "type": "array", - "description": "Per-check outcomes. One entry per check the runner produced.", + "description": "Per-requirement-row outcomes (schema 0.6+). One entry per requirement row covered by a check that ran in this invocation; a single probe whose `Check::covers()` lists multiple rows produces multiple entries (each entry's `check_id` carries the probe's id for provenance).", "items": { "$ref": "#/$defs/CheckResultView" } }, "summary": { "$ref": "#/$defs/Summary" }, @@ -61,12 +61,12 @@ "$defs": { "CheckResultView": { "type": "object", - "required": ["id", "label", "group", "layer", "status", "evidence", "confidence"], + "required": ["id", "label", "group", "layer", "status", "evidence", "confidence", "tier", "check_id"], "additionalProperties": false, "properties": { "id": { "type": "string", - "description": "Stable check identifier (e.g., `p1-non-interactive`). The site's `/score/` page and external leaderboards pin on these strings." + "description": "Requirement-row identifier (schema 0.6+; matches `coverage/matrix.json` row ids, e.g. `p1-must-no-interactive`). The site's `/score/` page and external leaderboards pin on these strings. Previously (≤0.5) this carried the probe id." }, "label": { "type": "string", @@ -84,30 +84,41 @@ }, "status": { "type": "string", - "description": "Outcome of the check. Skips and Errors are excluded from badge scoring.", - "enum": ["pass", "warn", "fail", "skip", "error"] + "description": "Outcome of the check under the 7-status taxonomy (schema 0.6+). `opt_out` = deliberate non-adoption; `n_a` = conditional antecedent unmet (set by antecedent propagation); `skip` = linter probe limitation. Skips, errors, opt_outs, and n_as are all excluded from the transitional badge denominator.", + "enum": ["pass", "warn", "fail", "opt_out", "n_a", "skip", "error"] }, "evidence": { "type": ["string", "null"], - "description": "Structured evidence for non-Pass statuses: the suppression-table marker, the unrecognized-flag list, the missing-file path. `null` for clean Pass." + "description": "Structured evidence for non-Pass statuses: the suppression-table marker, the unrecognized-flag list, the missing-file path, the antecedent propagation reason. `null` for clean Pass." }, "confidence": { "type": "string", "description": "How directly the check verifies its requirement. `high` for direct probes; `medium` for heuristics. Older consumers feature-detect.", "enum": ["high", "medium"] + }, + "tier": { + "type": ["string", "null"], + "description": "Requirement-row RFC 2119 level (schema 0.6+). `null` only when the row id is not in the registry — an internal inconsistency that should be loud.", + "enum": ["must", "should", "may", null] + }, + "check_id": { + "type": "string", + "description": "Probe that produced this row (schema 0.6+). One probe may produce multiple rows when its `Check::covers()` slice names multiple requirements (e.g. `p3-version` covers both `p3-must-version` and `p3-should-version-short`)." } } }, "Summary": { "type": "object", - "description": "Run-level outcome counts.", - "required": ["total", "pass", "warn", "fail", "skip", "error"], + "description": "Run-level outcome counts. Schema 0.6+ adds `opt_out` and `n_a` counters alongside the historic five.", + "required": ["total", "pass", "warn", "fail", "opt_out", "n_a", "skip", "error"], "additionalProperties": false, "properties": { "total": { "type": "integer", "minimum": 0 }, "pass": { "type": "integer", "minimum": 0 }, "warn": { "type": "integer", "minimum": 0 }, "fail": { "type": "integer", "minimum": 0 }, + "opt_out": { "type": "integer", "minimum": 0 }, + "n_a": { "type": "integer", "minimum": 0 }, "skip": { "type": "integer", "minimum": 0 }, "error": { "type": "integer", "minimum": 0 } } @@ -203,19 +214,21 @@ }, "examples": [ { - "schema_version": "0.5", + "schema_version": "0.6", "results": [ { - "id": "p1-non-interactive", + "id": "p1-must-no-interactive", "label": "Non-interactive by default", "group": "P1", "layer": "behavioral", "status": "pass", "evidence": null, - "confidence": "high" + "confidence": "high", + "tier": "must", + "check_id": "p1-non-interactive" } ], - "summary": { "total": 1, "pass": 1, "warn": 0, "fail": 0, "skip": 0, "error": 0 }, + "summary": { "total": 1, "pass": 1, "warn": 0, "fail": 0, "opt_out": 0, "n_a": 0, "skip": 0, "error": 0 }, "coverage_summary": { "must": { "total": 23, "verified": 17 }, "should": { "total": 14, "verified": 9 }, diff --git a/src/checks/behavioral/json_output.rs b/src/checks/behavioral/json_output.rs index 998e72b..3cce339 100644 --- a/src/checks/behavioral/json_output.rs +++ b/src/checks/behavioral/json_output.rs @@ -71,7 +71,12 @@ impl Check for JsonOutputCheck { fn probe_subcommands(runner: &BinaryRunner, help_output: &str) -> CheckStatus { let subcommands = parse_subcommand_names(help_output); if subcommands.is_empty() { - return CheckStatus::Skip("no --output/--format flag detected".into()); + return CheckStatus::OptOut( + "no --output/--format flag detected — tool does not ship structured output. \ + Schema-discovery requirements (p2-must-schema-print, p2-should-schema-file) \ + collapse to n/a via antecedent propagation." + .into(), + ); } for subcmd in &subcommands { @@ -90,7 +95,11 @@ fn probe_subcommands(runner: &BinaryRunner, help_output: &str) -> CheckStatus { } } - CheckStatus::Skip("no --output/--format flag detected in any subcommand".into()) + CheckStatus::OptOut( + "no --output/--format flag detected in any subcommand — tool does not ship \ + structured output." + .into(), + ) } /// Extract subcommand names from CLI --help output. @@ -293,12 +302,15 @@ esac } #[test] - fn json_output_skip_no_flag() { + fn json_output_opt_out_no_flag() { + // Schema 0.6: "no flag at all" is opt_out (deliberate non-adoption), + // not Skip (probe limitation). Distinguishes "tool doesn't ship + // structured output" from "we couldn't measure it". let project = test_project_with_sh_script("echo 'just some help text'"); let result = JsonOutputCheck.run(&project).expect("check should run"); match &result.status { - CheckStatus::Skip(msg) => assert!(msg.contains("no --output")), - other => panic!("expected Skip, got {other:?}"), + CheckStatus::OptOut(msg) => assert!(msg.contains("no --output")), + other => panic!("expected OptOut, got {other:?}"), } } diff --git a/src/checks/project/bundle_exists.rs b/src/checks/project/bundle_exists.rs index 0cb44e9..b76ee42 100644 --- a/src/checks/project/bundle_exists.rs +++ b/src/checks/project/bundle_exists.rs @@ -77,10 +77,17 @@ pub(crate) fn find_bundle(root: &Path) -> Option { None } -/// Core unit. SHOULD-tier: every miss is Warn, never Fail. +/// Core unit. SHOULD-tier: every miss is Warn, never Fail. The "no bundle +/// at all" case emits `OptOut` so antecedent propagation (Decision 2a) can +/// collapse the conditional MUSTs/MAYs that depend on `p8-bundle-exists` +/// (`p8-must-bundle-install`, `p8-may-install-all`, `p8-may-bundle-update`) +/// to `n_a` rather than dragging them through their own probe and emitting +/// double-penalty Skips. A bundle that exists but is malformed (missing +/// frontmatter or `name:` field) is a real SHOULD violation and stays +/// `Warn` — the feature is present, just incomplete. pub(crate) fn check_bundle_exists(root: &Path) -> CheckStatus { let Some(path) = find_bundle(root) else { - return CheckStatus::Warn( + return CheckStatus::OptOut( "no top-level AGENTS.md or SKILL.md found. Agents discover \ skill bundles via filesystem convention; ship one with YAML \ frontmatter naming the tool." @@ -179,12 +186,12 @@ mod tests { } #[test] - fn warn_no_bundle() { + fn opt_out_no_bundle() { let dir = temp_dir("nobundle"); fs::write(dir.join("README.md"), "# Tool\n").expect("write"); match check_bundle_exists(&dir) { - CheckStatus::Warn(msg) => assert!(msg.contains("AGENTS.md")), - other => panic!("expected Warn, got {other:?}"), + CheckStatus::OptOut(msg) => assert!(msg.contains("AGENTS.md")), + other => panic!("expected OptOut, got {other:?}"), } } diff --git a/src/principles/matrix.rs b/src/principles/matrix.rs index 2da5a24..fe114a9 100644 --- a/src/principles/matrix.rs +++ b/src/principles/matrix.rs @@ -267,7 +267,17 @@ pub fn render_markdown(matrix: &Matrix) -> String { }; let applicability = match row.applicability { Applicability::Universal => "Universal".to_string(), - Applicability::Conditional(cond) => format!("If: {cond}"), + Applicability::Conditional { + condition, + antecedent, + } => match (condition, antecedent) { + (Some(cond), Some(ante)) => { + format!("If: {cond} (antecedent: `{}`)", ante.check_id) + } + (Some(cond), None) => format!("If: {cond}"), + (None, Some(ante)) => format!("If: `{}` is present", ante.check_id), + (None, None) => "Conditional".to_string(), + }, }; let verifiers = if row.verifiers.is_empty() { "**UNCOVERED**".to_string() diff --git a/src/principles/registry.rs b/src/principles/registry.rs index 290fe1c..395b211 100644 --- a/src/principles/registry.rs +++ b/src/principles/registry.rs @@ -18,11 +18,34 @@ pub enum Level { } /// Whether a requirement applies to every CLI or only when a condition holds. +/// +/// `Conditional` carries an optional prose `condition` (legacy `{ if: "" +/// }` shape) and an optional machine-readable `antecedent` (new `{ kind: +/// conditional, antecedent: { check_id: ... } }` shape). The antecedent's check +/// status drives the propagation table documented in +/// `docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md` +/// Decision 2a: when the antecedent resolves to `opt_out` / `n_a`, this +/// requirement's row in the scorecard collapses to `n_a`; `skip` / `error` +/// inherit; `pass` / `warn` / `fail` let the consequent verifier's own status +/// stand. #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -#[serde(tag = "kind", content = "condition", rename_all = "lowercase")] +#[serde(tag = "kind", rename_all = "lowercase")] pub enum Applicability { Universal, - Conditional(&'static str), + Conditional { + #[serde(default, skip_serializing_if = "Option::is_none")] + condition: Option<&'static str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + antecedent: Option, + }, +} + +/// Machine-readable antecedent for a conditional requirement. The +/// `check_id` names the verifier whose status decides whether the consequent +/// row applies (see `Applicability` for the propagation rules). +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub struct Antecedent { + pub check_id: &'static str, } /// Categories under which a tool may be exempt from specific requirements. @@ -441,4 +464,85 @@ mod tests { } } } + + // ────────────────────────────────────────────────────────────────── + // U2 (schema 0.6): conditional applicability red-team guards. + // Each conditional row in the registry names an antecedent `check_id` + // that drives propagation. A typo or rename in the antecedent would + // silently mute propagation in production — the consequent row would + // forever look up `None` and pass through with its own probe status. + // The asserts below pin the contract loudly. + // ────────────────────────────────────────────────────────────────── + + #[test] + fn every_conditional_antecedent_resolves_to_a_real_check() { + use crate::check::Check; + use crate::checks::all_checks_catalog; + + let catalog: Vec> = all_checks_catalog(); + let catalog_ids: Vec<&str> = catalog.iter().map(|c| c.id()).collect(); + + let mut dangling: Vec<(&str, &str)> = Vec::new(); + for req in REQUIREMENTS { + if let Applicability::Conditional { + antecedent: Some(ante), + .. + } = req.applicability + && !catalog_ids.contains(&ante.check_id) + { + dangling.push((req.id, ante.check_id)); + } + } + assert!( + dangling.is_empty(), + "conditional requirements with dangling antecedent check_ids:\n{}\n\ + Fix the spec's `antecedent.check_id` or add the missing check to the catalog.", + dangling + .iter() + .map(|(req, ante)| format!( + " - row `{req}` → antecedent `{ante}` (not in catalog)" + )) + .collect::>() + .join("\n"), + ); + } + + #[test] + fn no_conditional_row_names_itself_as_antecedent() { + // Edge case: a conditional row's covering check is the same as its + // antecedent. The propagation table would then read the row's own + // probe status and could collapse the row to n_a based on itself — + // a logic loop that's never the right model. The spec should never + // produce this shape; this test catches it if it does. + use crate::check::Check; + use crate::checks::all_checks_catalog; + + let catalog: Vec> = all_checks_catalog(); + let mut covers_by_check: std::collections::HashMap<&'static str, &'static [&'static str]> = + std::collections::HashMap::new(); + for c in &catalog { + covers_by_check.insert(Box::leak(c.id().to_string().into_boxed_str()), c.covers()); + } + + for req in REQUIREMENTS { + let Applicability::Conditional { + antecedent: Some(ante), + .. + } = req.applicability + else { + continue; + }; + if let Some(covers) = covers_by_check.get(ante.check_id) { + assert!( + !covers.contains(&req.id), + "conditional row `{}` declares antecedent `{}`, but that \ + check already covers `{}` directly — the row would gate \ + its own status against itself.", + req.id, + ante.check_id, + req.id, + ); + } + } + } } diff --git a/src/principles/spec/principles/p1-non-interactive-by-default.md b/src/principles/spec/principles/p1-non-interactive-by-default.md index a4c6ee6..2cc1df7 100644 --- a/src/principles/spec/principles/p1-non-interactive-by-default.md +++ b/src/principles/spec/principles/p1-non-interactive-by-default.md @@ -1,7 +1,7 @@ --- id: p1 title: Non-Interactive by Default -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p1-must-env-var @@ -120,7 +120,7 @@ Measured by check IDs `p1-non-interactive` (behavioral) and `p1-non-interactive- ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p2-structured-parseable-output.md b/src/principles/spec/principles/p2-structured-parseable-output.md index 26098f4..4608c62 100644 --- a/src/principles/spec/principles/p2-structured-parseable-output.md +++ b/src/principles/spec/principles/p2-structured-parseable-output.md @@ -1,7 +1,7 @@ --- id: p2 title: Structured, Parseable Output -last-revised: 2026-05-06 +last-revised: 2026-05-21 status: active requirements: - id: p2-must-output-flag @@ -23,7 +23,9 @@ requirements: - id: p2-must-schema-print level: must applicability: - if: CLI emits structured output + kind: conditional + antecedent: + check_id: p2-json-output summary: "CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier." - id: p2-should-consistent-envelope level: should @@ -32,7 +34,9 @@ requirements: - id: p2-should-schema-file level: should applicability: - if: CLI emits structured output + kind: conditional + antecedent: + check_id: p2-json-output summary: "Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool." - id: p2-should-json-aliases level: should @@ -133,7 +137,7 @@ Measured by check IDs `p2-output-json`, `p2-output-format`, `p2-stderr-diagnosti ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p3-progressive-help-discovery.md b/src/principles/spec/principles/p3-progressive-help-discovery.md index 4504bc0..97e2773 100644 --- a/src/principles/spec/principles/p3-progressive-help-discovery.md +++ b/src/principles/spec/principles/p3-progressive-help-discovery.md @@ -69,9 +69,9 @@ trial-and-errors its way into a working call, burning tokens and sometimes landi - Short `about` for command-list summaries; `long_about` reserved for detailed descriptions visible with `--help` but not `-h`. - A short alias for `--version` SHOULD work: `-V` (clap default, `curl`, `wget`, `gzip`), `-v` (`npm`, `node`, `bun`, - `yarn`, `make`), or `-version` (Go's `flag` package). Any one is sufficient. Agents probing tool versions across many - CLIs save token cost when they can pin against a one- or two-character form; the long-only path forces an extra parse - step. + `yarn`, `make`), or `-version` (Go's `flag` package). Any of the three forms is sufficient. Agents probing tool + versions across many CLIs save token cost when they can pin against a one- or two-character flag; the long-only path + forces an extra parse step. **MAY:** diff --git a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md index dc7a46d..b18feef 100644 --- a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md +++ b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md @@ -1,7 +1,7 @@ --- id: p4 title: Fail Fast with Actionable Errors -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p4-must-try-parse @@ -124,7 +124,7 @@ Measured by check IDs `p4-bad-args`, `p4-process-exit`, `p4-unwrap`, `p4-exit-co ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md index 271775a..38a6508 100644 --- a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md +++ b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md @@ -1,7 +1,7 @@ --- id: p5 title: Safe Retries and Explicit Mutation Boundaries -last-revised: 2026-04-22 +last-revised: 2026-05-07 status: active requirements: - id: p5-must-force-yes @@ -79,7 +79,7 @@ under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p6-composable-predictable-command-structure.md b/src/principles/spec/principles/p6-composable-predictable-command-structure.md index 5407ee1..f67a437 100644 --- a/src/principles/spec/principles/p6-composable-predictable-command-structure.md +++ b/src/principles/spec/principles/p6-composable-predictable-command-structure.md @@ -1,7 +1,7 @@ --- id: p6 title: Composable and Predictable Command Structure -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p6-must-sigpipe @@ -162,7 +162,7 @@ check --principle 6 .` against the CLI under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p7-bounded-high-signal-responses.md b/src/principles/spec/principles/p7-bounded-high-signal-responses.md index d96e82e..7e21520 100644 --- a/src/principles/spec/principles/p7-bounded-high-signal-responses.md +++ b/src/principles/spec/principles/p7-bounded-high-signal-responses.md @@ -1,7 +1,7 @@ --- id: p7 title: Bounded, High-Signal Responses -last-revised: 2026-04-22 +last-revised: 2026-05-07 status: active requirements: - id: p7-must-quiet @@ -107,7 +107,7 @@ under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p8-discoverable-skill-bundle.md b/src/principles/spec/principles/p8-discoverable-skill-bundle.md index e35b93d..1479fdc 100644 --- a/src/principles/spec/principles/p8-discoverable-skill-bundle.md +++ b/src/principles/spec/principles/p8-discoverable-skill-bundle.md @@ -1,13 +1,15 @@ --- id: p8 title: Discoverable Through Agent Skill Bundles -last-revised: 2026-05-06 +last-revised: 2026-05-21 status: active requirements: - id: p8-must-bundle-install level: must applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes." - id: p8-should-bundle-exists level: should @@ -16,12 +18,16 @@ requirements: - id: p8-may-install-all level: may applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all." - id: p8-may-bundle-update level: may applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version." --- @@ -44,7 +50,7 @@ recognizes the tool's idioms across every subsequent invocation. **MUST:** -- When a CLI ships a skill bundle, the CLI MUST provide an install path that registers the bundle with installed agent +- If a CLI ships a skill bundle, then it MUST provide an install path that registers the bundle with installed agent runtimes. The canonical form is a `tool skill install []` subcommand that writes into the runtime's filesystem cascade (e.g., `~/.claude/skills/`, `~/.cursor/skills/`). Non-canonical alternatives (`tool init --skill`, `tool skills add`, `tool agents add`) are acceptable but SHOULD migrate toward `tool skill install`. A bundle without an @@ -60,11 +66,11 @@ recognizes the tool's idioms across every subsequent invocation. **MAY:** -- An `--all` mode MAY auto-detect installed agent runtimes (Claude Code, Cursor, Codex, OpenCode, and others as the - ecosystem evolves) and install the bundle across each. A user setting up a new machine with multiple coding agents - installs once and gets coverage across every runtime. -- An `update` (or `upgrade`) subcommand under `tool skill` MAY pull the latest bundle version, so agents stay current - with the CLI's evolving surface without a full reinstall. +- If a CLI ships a skill bundle, then an `--all` mode MAY auto-detect installed agent runtimes (Claude Code, Cursor, + Codex, OpenCode, and others as the ecosystem evolves) and install the bundle across each. A user setting up a new + machine with multiple coding agents installs once and gets coverage across every runtime. +- If a CLI ships a skill bundle, then an `update` (or `upgrade`) subcommand under `tool skill` MAY pull the latest + bundle version, so agents stay current with the CLI's evolving surface without a full reinstall. ## Evidence diff --git a/src/scorecard/mod.rs b/src/scorecard/mod.rs index aac31e7..e0e2456 100644 --- a/src/scorecard/mod.rs +++ b/src/scorecard/mod.rs @@ -20,8 +20,11 @@ use crate::types::{CheckGroup, CheckResult, CheckStatus}; /// target metadata blocks — self-describing scoring run), `0.5` (`badge` /// block — eligibility, embed snippet, and badge/scorecard URLs derived /// from the run, so authors learn about the badge from the CLI itself -/// rather than a round-trip to the site). -pub const SCHEMA_VERSION: &str = "0.5"; +/// rather than a round-trip to the site), `0.6` (7-status taxonomy: +/// `opt_out` + `n_a` added to `status`; matching counters in `summary`; +/// `tier` field on each result; one result per requirement-row instead of +/// per-check_id; antecedent propagation for conditional rows). +pub const SCHEMA_VERSION: &str = "0.6"; /// Eligibility floor for the agent-native badge, expressed as an integer /// percent. A score that meets or exceeds this floor qualifies a tool to @@ -186,11 +189,22 @@ pub fn compute_badge(results: &[CheckResult], tool_name: &str) -> BadgeInfo { } } -/// Compute the rounded integer percent score using the leaderboard's -/// denominator (`pass + warn + fail`). Skips and errors are excluded from -/// both sides of the ratio. Returns `0` when no checks contribute (every -/// status was Skip or Error, or no checks ran at all) — pairs with -/// `BadgeInfo::eligible == false` so a zero score never qualifies. +/// Compute the rounded integer percent score using the transitional +/// leaderboard denominator. Pass/Warn/Fail count in the denominator. Pass +/// counts in the numerator. Skip, Error, OptOut, and NotApplicable are +/// excluded from both sides of the ratio. +/// +/// **Transitional formula choice.** The 7-status taxonomy semantically +/// treats `opt_out` as in-denominator (deliberate non-adoption is a real +/// signal), but plan U2 explicitly keeps the U2 formula conservative: +/// "leave the existing formula but exclude `opt_out` from the denominator +/// and exclude `n_a` from both — minimal change that respects the new +/// semantics without committing to a new formula." Whether `opt_out` +/// re-enters the denominator (and at what weight) is the U3 spec issue, +/// after the disambiguated input has been rescored. +/// +/// Returns `0` when no checks contribute — pairs with `BadgeInfo::eligible +/// == false` so a zero score never qualifies. fn score_pct(results: &[CheckResult]) -> u32 { let mut pass = 0u32; let mut denom = 0u32; @@ -203,7 +217,10 @@ fn score_pct(results: &[CheckResult]) -> u32 { CheckStatus::Warn(_) | CheckStatus::Fail(_) => { denom += 1; } - CheckStatus::Skip(_) | CheckStatus::Error(_) => {} + CheckStatus::Skip(_) + | CheckStatus::Error(_) + | CheckStatus::OptOut(_) + | CheckStatus::NotApplicable(_) => {} } } if denom == 0 { @@ -292,16 +309,30 @@ pub struct CoverageSummary { pub may: LevelCounts, } -#[derive(Serialize)] +/// Run-level outcome counts. The 7-status taxonomy added `opt_out` and +/// `n_a` in schema 0.6; pre-0.6 consumers tolerate the new keys (additive +/// extension). +#[derive(Serialize, Debug)] pub struct Summary { pub total: usize, pub pass: usize, pub warn: usize, pub fail: usize, + pub opt_out: usize, + pub n_a: usize, pub skip: usize, pub error: usize, } +/// One row of `results[]` in the scorecard JSON. +/// +/// Schema 0.6 changed the unit of emission from "per check_id" to "per +/// requirement-row". `id` is now the requirement row id (matches +/// `coverage/matrix.json` row IDs). `tier` carries the row's RFC 2119 +/// level (`must`/`should`/`may`) so downstream scoring consumers do not +/// need a matrix join. `check_id` is the probe that produced this row, +/// preserved for provenance and so the site renderer / audience classifier +/// can find the originating probe without a registry walk. #[derive(Serialize)] pub struct CheckResultView { pub id: String, @@ -313,14 +344,42 @@ pub struct CheckResultView { /// `high` for direct probes, `medium` for heuristics. Older consumers /// feature-detect and tolerate missing keys. pub confidence: String, + /// Requirement tier (`must`/`should`/`may`). Pre-launch additive + /// (schema `0.6`). `null` only for results whose row id is not in the + /// registry — an internal inconsistency that should be loud. + pub tier: Option, + /// Underlying probe that produced this row (e.g., `p3-version` covers + /// both `p3-must-version` and `p3-should-version-short` — two rows + /// share one `check_id`). Pre-launch additive (schema `0.6`). Falls + /// back to the row `id` itself when no provenance was threaded in + /// (legacy test fixtures that hand-build a `CheckResult` without the + /// fan-out pipeline). + pub check_id: String, } impl CheckResultView { + /// Construct from a raw probe result (pre-fan-out callers and test + /// fixtures). `check_id` defaults to `r.id` and `tier` is looked up + /// from the registry (which fails to find anything for arbitrary test + /// IDs — surfaces as JSON null). Production code uses `from_row` + /// directly with the threaded probe provenance; this fallback exists + /// for tests and any future caller that builds a per-check view + /// without the fan-out pipeline. + #[allow(dead_code)] pub fn from_result(r: &CheckResult) -> Self { + Self::from_row(r, &r.id) + } + + /// Construct from a fanned-out per-row result with explicit probe + /// provenance. `check_id` is the probe's `Check::id()`; `r.id` is the + /// requirement row id. + pub fn from_row(r: &CheckResult, check_id: &str) -> Self { let (status, evidence) = match &r.status { CheckStatus::Pass => ("pass".to_string(), None), CheckStatus::Warn(e) => ("warn".to_string(), Some(e.clone())), CheckStatus::Fail(e) => ("fail".to_string(), Some(e.clone())), + CheckStatus::OptOut(e) => ("opt_out".to_string(), Some(e.clone())), + CheckStatus::NotApplicable(e) => ("n_a".to_string(), Some(e.clone())), CheckStatus::Skip(e) => ("skip".to_string(), Some(e.clone())), CheckStatus::Error(e) => ("error".to_string(), Some(e.clone())), }; @@ -338,6 +397,16 @@ impl CheckResultView { .ok() .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_else(|| format!("{:?}", r.confidence)); + // Look up tier from the registry. The row ID is the result ID under + // the per-row emission contract introduced in schema 0.6. Per-check + // results fed in by older callers (or test fixtures) won't find a + // match here — `tier` falls back to None, which surfaces as JSON + // null and is a visible sign of inconsistency. + let tier = crate::principles::registry::find(&r.id).map(|req| match req.level { + crate::principles::registry::Level::Must => "must".to_string(), + crate::principles::registry::Level::Should => "should".to_string(), + crate::principles::registry::Level::May => "may".to_string(), + }); CheckResultView { id: r.id.clone(), label: r.label.clone(), @@ -346,6 +415,8 @@ impl CheckResultView { status, evidence, confidence, + tier, + check_id: check_id.to_string(), } } } @@ -365,6 +436,14 @@ fn build_summary(results: &[CheckResult]) -> Summary { .iter() .filter(|r| matches!(r.status, CheckStatus::Fail(_))) .count(), + opt_out: results + .iter() + .filter(|r| matches!(r.status, CheckStatus::OptOut(_))) + .count(), + n_a: results + .iter() + .filter(|r| matches!(r.status, CheckStatus::NotApplicable(_))) + .count(), skip: results .iter() .filter(|r| matches!(r.status, CheckStatus::Skip(_))) @@ -461,6 +540,18 @@ pub fn format_text( } CheckStatus::Warn(_) => "WARN", CheckStatus::Fail(_) => "FAIL", + CheckStatus::OptOut(_) => { + if quiet { + continue; + } + "OPT " + } + CheckStatus::NotApplicable(_) => { + if quiet { + continue; + } + "N/A " + } CheckStatus::Skip(_) => { if quiet { continue; @@ -478,7 +569,11 @@ pub fn format_text( let _ = writeln!(out, " {line}"); } } - CheckStatus::Skip(reason) if !quiet => { + CheckStatus::Skip(reason) + | CheckStatus::OptOut(reason) + | CheckStatus::NotApplicable(reason) + if !quiet => + { let _ = writeln!(out, " {reason}"); } _ => {} @@ -490,8 +585,8 @@ pub fn format_text( let s = build_summary(results); let _ = writeln!( out, - "\n{} checks: {} pass, {} warn, {} fail, {} skip, {} error", - s.total, s.pass, s.warn, s.fail, s.skip, s.error + "\n{} checks: {} pass, {} warn, {} fail, {} opt_out, {} n_a, {} skip, {} error", + s.total, s.pass, s.warn, s.fail, s.opt_out, s.n_a, s.skip, s.error ); // Badge embed hint — appended only when eligible. Below the floor the @@ -505,9 +600,9 @@ pub fn format_text( } /// `--raw` rendering: one `idstatus` line per result, nothing else. -/// Status maps to the same five tokens the rich renderer uses (`PASS`, -/// `WARN`, `FAIL`, `SKIP`, `ERR`) so downstream pipelines see the same -/// vocabulary in both modes. +/// Status maps to one of the seven tokens (`PASS`, `WARN`, `FAIL`, +/// `OPT_OUT`, `N_A`, `SKIP`, `ERR`) so downstream pipelines see the same +/// vocabulary as the JSON `status` field (uppercased). fn format_text_raw(results: &[CheckResult]) -> String { let mut out = String::with_capacity(results.len() * 32); for r in results { @@ -515,6 +610,8 @@ fn format_text_raw(results: &[CheckResult]) -> String { CheckStatus::Pass => "PASS", CheckStatus::Warn(_) => "WARN", CheckStatus::Fail(_) => "FAIL", + CheckStatus::OptOut(_) => "OPT_OUT", + CheckStatus::NotApplicable(_) => "N_A", CheckStatus::Skip(_) => "SKIP", CheckStatus::Error(_) => "ERR", }; @@ -523,6 +620,101 @@ fn format_text_raw(results: &[CheckResult]) -> String { out } +/// Fan one probe-level result out into one entry per requirement-row in +/// the check's `Check::covers()` slice. The probe's status, label, group, +/// layer, and confidence propagate to every row; the `id` field is +/// replaced with the row id. Returns a pair `(row_result, check_id)` per +/// emitted row so downstream consumers (CheckResultView, propagation) know +/// the originating probe without a registry walk. +/// +/// Checks that declare no `covers()` rows produce a single passthrough +/// entry keyed by their own id — preserves the legacy per-check_id shape +/// for checks not yet wired into the requirement registry. +pub fn fan_out_per_row( + raw: &[CheckResult], + catalog: &[Box], +) -> Vec<(CheckResult, String)> { + let covers_by_id: HashMap<&str, &'static [&'static str]> = + catalog.iter().map(|c| (c.id(), c.covers())).collect(); + let mut out: Vec<(CheckResult, String)> = Vec::with_capacity(raw.len()); + for r in raw { + let covers = covers_by_id.get(r.id.as_str()).copied().unwrap_or(&[]); + if covers.is_empty() { + out.push((r.clone(), r.id.clone())); + continue; + } + for row_id in covers { + let mut row = r.clone(); + row.id = (*row_id).to_string(); + out.push((row, r.id.clone())); + } + } + out +} + +/// Apply the antecedent-status propagation table from plan Decision 2a. +/// +/// For each row whose registry entry has a conditional applicability with +/// an `antecedent.check_id`, look up the antecedent probe's raw status and +/// rewrite the row's status accordingly: +/// +/// | Antecedent status | Consequent row becomes | +/// | ----------------- | ----------------------------------- | +/// | `pass` / `warn` / `fail` | unchanged (evaluated normally) | +/// | `opt_out` / `n_a` | `n_a` (prerequisite absent) | +/// | `skip` | `skip` (inherited indeterminacy) | +/// | `error` | `error` (inherited indeterminacy) | +/// +/// Rows with no registry entry (legacy / unknown ids) are left untouched. +/// Rows whose antecedent did not produce a raw result (the antecedent +/// probe didn't run this invocation, e.g., source-only mode) are left +/// untouched — propagation needs an antecedent status to act on. +pub fn propagate_antecedents(rows: &mut [(CheckResult, String)], raw: &[CheckResult]) { + use crate::principles::registry::{Applicability, find}; + let raw_by_id: HashMap<&str, &CheckStatus> = + raw.iter().map(|r| (r.id.as_str(), &r.status)).collect(); + for (row, _check_id) in rows.iter_mut() { + let Some(req) = find(&row.id) else { continue }; + let Applicability::Conditional { antecedent, .. } = req.applicability else { + continue; + }; + let Some(ante) = antecedent else { continue }; + let Some(ante_status) = raw_by_id.get(ante.check_id) else { + continue; + }; + let new_status = match ante_status { + CheckStatus::Pass | CheckStatus::Warn(_) | CheckStatus::Fail(_) => continue, + CheckStatus::OptOut(reason) | CheckStatus::NotApplicable(reason) => { + CheckStatus::NotApplicable(format!( + "antecedent `{}` is {}: {reason}", + ante.check_id, + short_status_name(ante_status), + )) + } + CheckStatus::Skip(reason) => CheckStatus::Skip(format!( + "antecedent `{}` could not be measured: {reason}", + ante.check_id, + )), + CheckStatus::Error(reason) => { + CheckStatus::Error(format!("antecedent `{}` errored: {reason}", ante.check_id,)) + } + }; + row.status = new_status; + } +} + +fn short_status_name(s: &CheckStatus) -> &'static str { + match s { + CheckStatus::Pass => "pass", + CheckStatus::Warn(_) => "warn", + CheckStatus::Fail(_) => "fail", + CheckStatus::OptOut(_) => "opt_out", + CheckStatus::NotApplicable(_) => "n_a", + CheckStatus::Skip(_) => "skip", + CheckStatus::Error(_) => "error", + } +} + /// Bundle of run-level metadata captured by the runner around `Commands::Audit` /// and threaded into the scorecard. Grouped to keep `build_scorecard`'s /// signature manageable as schema `0.x` continues to add fields. The runner @@ -535,24 +727,32 @@ pub struct RunMetadata { } /// Build the scorecard. The `ran_checks` slice is the catalog of checks -/// that produced `results` — needed to translate check IDs back to the -/// requirement IDs they cover for `coverage_summary`. +/// that produced `raw_results`. +/// +/// Pipeline (schema 0.6): +/// raw probe results → fan out per requirement-row → antecedent +/// propagation → JSON view. Audience and coverage_summary still consume +/// raw probe results (signal classification keys on check_ids; coverage +/// counts requirements covered by the underlying probes). pub fn build_scorecard( - results: &[CheckResult], + raw_results: &[CheckResult], ran_checks: &[Box], audience: Option, audit_profile: Option, metadata: RunMetadata, ) -> Scorecard { - // `audience_reason` is derived from `results` rather than threaded + let mut row_results = fan_out_per_row(raw_results, ran_checks); + propagate_antecedents(&mut row_results, raw_results); + + // `audience_reason` is derived from `raw_results` rather than threaded // through as a caller parameter — the reason is a property of the - // result set, not a caller decision, and deriving it here keeps the - // label and its explanation in lock-step. When audience has a label - // the field is omitted from JSON (see Scorecard's serde skip rule). + // probe-level result set, not a caller decision, and deriving it here + // keeps the label and its explanation in lock-step. When audience has + // a label the field is omitted from JSON. let audience_reason = if audience.is_some() { None } else { - audience::classify_reason(results).map(|s| s.to_string()) + audience::classify_reason(raw_results).map(|s| s.to_string()) }; let RunMetadata { tool, @@ -560,16 +760,21 @@ pub fn build_scorecard( run, target, } = metadata; - // Compute the badge from the same `tool.name` the JSON emits, so the - // embed URL in `badge.embed_markdown` and the slug in `tool.name` can - // never disagree (a regression that diverges them would mislead any - // author copy-pasting from the JSON). - let badge = compute_badge(results, &tool.name); + + // Per-row results drive `summary` and `score_pct`. The badge uses the + // same per-row vector so the embed URL the JSON emits agrees with the + // post-summary text hint. + let per_row_only: Vec = row_results.iter().map(|(r, _)| r.clone()).collect(); + let badge = compute_badge(&per_row_only, &tool.name); + Scorecard { schema_version: SCHEMA_VERSION, - results: results.iter().map(CheckResultView::from_result).collect(), - summary: build_summary(results), - coverage_summary: build_coverage_summary(results, ran_checks), + results: row_results + .iter() + .map(|(r, check_id)| CheckResultView::from_row(r, check_id)) + .collect(), + summary: build_summary(&per_row_only), + coverage_summary: build_coverage_summary(raw_results, ran_checks), audience, audience_reason, audit_profile, @@ -583,13 +788,13 @@ pub fn build_scorecard( } pub fn format_json( - results: &[CheckResult], + raw_results: &[CheckResult], ran_checks: &[Box], audience: Option, audit_profile: Option, metadata: RunMetadata, ) -> String { - let scorecard = build_scorecard(results, ran_checks, audience, audit_profile, metadata); + let scorecard = build_scorecard(raw_results, ran_checks, audience, audit_profile, metadata); serde_json::to_string_pretty(&scorecard).unwrap_or_else(|e| format!("{{\"error\": \"{e}\"}}")) } @@ -732,7 +937,7 @@ mod tests { ]; let json = format_json(&results, &[], None, None, fixture_metadata()); let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); assert_eq!(parsed["summary"]["total"], 2); assert_eq!(parsed["summary"]["pass"], 1); assert_eq!(parsed["summary"]["fail"], 1); @@ -951,7 +1156,7 @@ mod tests { let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); assert_eq!(parsed["audience"], "agent-optimized"); assert!(parsed["audit_profile"].is_null()); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); } #[test] @@ -1227,7 +1432,7 @@ mod tests { } // 0.4 + 0.5 additions — every documented sub-key resolves. - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); for path in [ // 0.4 "tool.name", @@ -1550,4 +1755,545 @@ mod tests { ); assert_eq!(parsed["badge"]["convention_url"], "https://anc.dev/badge"); } + + // ────────────────────────────────────────────────────────────────── + // U2 (schema 0.6): per-row emission, tier, 7-status taxonomy, + // antecedent propagation. Plan reference: + // docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md + // in agentnative-site. + // ────────────────────────────────────────────────────────────────── + + fn make_raw(id: &str, status: CheckStatus) -> CheckResult { + make_result(id, status, CheckGroup::P2) + } + + /// Minimal `Check` impl that lets per-row fan-out tests express a + /// `covers()` slice without spinning up a real probe. + struct FakeCheck { + id: &'static str, + covers: &'static [&'static str], + } + + impl crate::check::Check for FakeCheck { + fn id(&self) -> &str { + self.id + } + fn label(&self) -> &'static str { + "fake" + } + fn group(&self) -> CheckGroup { + CheckGroup::P2 + } + fn layer(&self) -> CheckLayer { + CheckLayer::Behavioral + } + fn applicable(&self, _p: &crate::project::Project) -> bool { + true + } + fn run(&self, _p: &crate::project::Project) -> anyhow::Result { + unreachable!() + } + fn covers(&self) -> &'static [&'static str] { + self.covers + } + } + + #[test] + fn fan_out_emits_one_row_per_covered_requirement() { + // Single probe (`p3-version`) covers two requirement rows. Fan-out + // produces two entries with id = row_id and check_id = probe id. + let raw = vec![make_raw( + "p3-version", + CheckStatus::Warn("short alias missing".into()), + )]; + let catalog: Vec> = vec![Box::new(FakeCheck { + id: "p3-version", + covers: &["p3-must-version", "p3-should-version-short"], + })]; + let rows = fan_out_per_row(&raw, &catalog); + assert_eq!(rows.len(), 2); + let ids: Vec<&str> = rows.iter().map(|(r, _)| r.id.as_str()).collect(); + assert!(ids.contains(&"p3-must-version")); + assert!(ids.contains(&"p3-should-version-short")); + for (r, check_id) in &rows { + assert_eq!( + check_id, "p3-version", + "check_id provenance lost on row {}", + r.id + ); + assert!( + matches!(r.status, CheckStatus::Warn(_)), + "probe status must propagate to every covered row pre-propagation", + ); + } + } + + #[test] + fn fan_out_emits_passthrough_for_checks_without_covers() { + // Checks that don't declare any covers() pass through as a single + // row keyed by check.id() — preserves the legacy shape for any + // future check not yet wired into the registry. + let raw = vec![make_raw("orphan-check", CheckStatus::Pass)]; + let catalog: Vec> = vec![Box::new(FakeCheck { + id: "orphan-check", + covers: &[], + })]; + let rows = fan_out_per_row(&raw, &catalog); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0.id, "orphan-check"); + assert_eq!(rows[0].1, "orphan-check"); + } + + #[test] + fn propagation_passes_through_when_antecedent_is_pass_warn_fail() { + // Antecedent statuses that mean "feature present" (pass / warn / + // fail) leave the consequent row untouched. + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Pass), + make_raw("p2-schema-print", CheckStatus::Fail("missing".into())), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Fail("missing".into())), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Fail(_))); + } + + #[test] + fn propagation_collapses_consequent_when_antecedent_is_opt_out() { + // Antecedent OptOut → consequent becomes NotApplicable, regardless + // of what the consequent's own probe emitted. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::OptOut("no --output flag".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + match &rows[0].0.status { + CheckStatus::NotApplicable(reason) => { + assert!( + reason.contains("p2-json-output") && reason.contains("opt_out"), + "evidence should cite the antecedent + its status, got: {reason}", + ); + } + other => panic!("expected NotApplicable, got {other:?}"), + } + } + + #[test] + fn propagation_collapses_consequent_when_antecedent_is_n_a() { + // n_a antecedent (e.g., a chained conditional) propagates the same + // way as opt_out. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::NotApplicable("upstream n/a".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::NotApplicable(_))); + } + + #[test] + fn propagation_inherits_skip_from_antecedent() { + // Skip antecedent → consequent inherits Skip (couldn't measure + // upstream means can't meaningfully evaluate downstream). + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Skip("probe limit".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Skip(_))); + } + + #[test] + fn propagation_inherits_error_from_antecedent() { + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Error("probe crashed".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Error(_))); + } + + #[test] + fn propagation_leaves_universal_rows_untouched() { + // A row with applicability: universal must not be touched by + // propagation even if a check with the same id exists in `raw`. + let raw = vec![make_raw("p1-non-interactive", CheckStatus::Pass)]; + let mut rows = vec![( + make_raw("p1-must-no-interactive", CheckStatus::Pass), + "p1-non-interactive".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Pass)); + } + + #[test] + fn score_pct_excludes_opt_out_and_n_a_from_denominator() { + // Transitional formula (U2): pass / (pass + warn + fail). opt_out + // and n_a do not count in either side of the ratio. A run of + // 1 Pass + 1 OptOut + 1 NotApplicable scores 100% (1/1), not 33%. + let results = vec![ + make_raw("c1", CheckStatus::Pass), + make_raw("c2", CheckStatus::OptOut("deliberate".into())), + make_raw("c3", CheckStatus::NotApplicable("conditional unmet".into())), + ]; + assert_eq!(score_pct(&results), 100); + + // Adding one Fail pulls the score down to 50%; opt_out / n_a still + // do not contribute to the denominator. + let mixed = vec![ + make_raw("c1", CheckStatus::Pass), + make_raw("c2", CheckStatus::Fail("violates".into())), + make_raw("c3", CheckStatus::OptOut("deliberate".into())), + make_raw("c4", CheckStatus::NotApplicable("conditional unmet".into())), + ]; + assert_eq!(score_pct(&mixed), 50); + } + + #[test] + fn summary_counts_seven_statuses_independently() { + // build_summary surfaces opt_out and n_a alongside the historical + // five counters; total covers all seven. + let results = vec![ + make_raw("a", CheckStatus::Pass), + make_raw("b", CheckStatus::Warn("w".into())), + make_raw("c", CheckStatus::Fail("f".into())), + make_raw("d", CheckStatus::OptOut("o".into())), + make_raw("e", CheckStatus::NotApplicable("n".into())), + make_raw("f", CheckStatus::Skip("s".into())), + make_raw("g", CheckStatus::Error("e".into())), + ]; + let s = build_summary(&results); + assert_eq!(s.total, 7); + assert_eq!(s.pass, 1); + assert_eq!(s.warn, 1); + assert_eq!(s.fail, 1); + assert_eq!(s.opt_out, 1); + assert_eq!(s.n_a, 1); + assert_eq!(s.skip, 1); + assert_eq!(s.error, 1); + } + + #[test] + fn check_result_view_carries_tier_and_check_id() { + // Per-row CheckResultView built via from_row exposes the requirement + // tier (looked up from the registry) and the originating probe. + let r = make_raw("p3-must-version", CheckStatus::Pass); + let view = CheckResultView::from_row(&r, "p3-version"); + assert_eq!(view.id, "p3-must-version"); + assert_eq!(view.check_id, "p3-version"); + assert_eq!(view.tier.as_deref(), Some("must")); + } + + #[test] + fn check_result_view_tier_is_null_for_unknown_row_id() { + // Test fixtures with synthetic ids that don't exist in the registry + // surface as JSON null for tier — visible signal of inconsistency. + let r = make_raw("not-a-real-row-id", CheckStatus::Pass); + let view = CheckResultView::from_row(&r, "some-check"); + assert!(view.tier.is_none(), "got: {:?}", view.tier); + } + + #[test] + fn opt_out_status_serializes_as_opt_out_in_json() { + let r = make_raw("c1", CheckStatus::OptOut("test reason".into())); + let view = CheckResultView::from_result(&r); + assert_eq!(view.status, "opt_out"); + assert_eq!(view.evidence.as_deref(), Some("test reason")); + } + + #[test] + fn n_a_status_serializes_as_n_a_in_json() { + let r = make_raw("c1", CheckStatus::NotApplicable("antecedent unmet".into())); + let view = CheckResultView::from_result(&r); + assert_eq!(view.status, "n_a"); + assert_eq!(view.evidence.as_deref(), Some("antecedent unmet")); + } + + // ────────────────────────────────────────────────────────────────── + // U2 red team: adversarial inputs that try to break the per-row + + // propagation pipeline or the score formula. + // ────────────────────────────────────────────────────────────────── + + #[test] + fn rt_propagation_is_idempotent() { + // Propagation reads raw probe statuses (not row statuses), so a + // second pass over an already-propagated row vector must produce + // an identical result. Pins the no-feedback contract — a future + // refactor that reads `rows` in place would break this and + // potentially loop or oscillate on chained conditionals. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::OptOut("no --output flag".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + let after_first: Vec<(String, String)> = rows + .iter() + .map(|(r, c)| { + ( + serde_json::to_string(&r.status).expect("status serializes"), + c.clone(), + ) + }) + .collect(); + propagate_antecedents(&mut rows, &raw); + let after_second: Vec<(String, String)> = rows + .iter() + .map(|(r, c)| { + ( + serde_json::to_string(&r.status).expect("status serializes"), + c.clone(), + ) + }) + .collect(); + assert_eq!(after_first, after_second, "propagation must be idempotent"); + } + + #[test] + fn rt_propagation_no_op_when_antecedent_did_not_run() { + // Source-only or filtered run: the antecedent probe didn't produce + // a raw result. The row keeps its own status — propagation can't + // override what it can't read. This is the exact path tools using + // `--source` or `--principle ` exercise in production. + let raw = vec![make_raw("p2-schema-print", CheckStatus::Pass)]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!( + matches!(rows[0].0.status, CheckStatus::Pass), + "no antecedent in raw → row untouched, got: {:?}", + rows[0].0.status, + ); + } + + #[test] + fn rt_propagation_inherits_audit_profile_suppression_as_skip() { + // Adversarial case: a CLI runs with `--audit-profile ` that + // suppresses the antecedent probe. The suppressed Skip carries the + // SUPPRESSION_EVIDENCE_PREFIX sentinel. The consequent row should + // inherit Skip (cannot meaningfully evaluate). The new evidence + // string cites the antecedent so a reader can still trace the + // root cause back to the audit profile. + use crate::principles::registry::SUPPRESSION_EVIDENCE_PREFIX; + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::Skip(format!("{SUPPRESSION_EVIDENCE_PREFIX}human-tui")), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + match &rows[0].0.status { + CheckStatus::Skip(reason) => { + assert!( + reason.contains("p2-json-output"), + "propagated Skip must cite the antecedent, got: {reason}", + ); + assert!( + reason.contains("human-tui"), + "propagated Skip must preserve the suppression reason, got: {reason}", + ); + } + other => panic!("expected Skip, got {other:?}"), + } + } + + #[test] + fn rt_score_pct_only_n_a_returns_zero_without_panic() { + // Pathological: every result is NotApplicable. Denominator is zero; + // score must surface as 0 with no division-by-zero or NaN. + let results: Vec = (0..100) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::NotApplicable("conditional unmet".into()), + ) + }) + .collect(); + assert_eq!(score_pct(&results), 0); + } + + #[test] + fn rt_score_pct_only_opt_out_returns_zero_without_panic() { + // Mirror of the n_a case: opt_out also excluded from denominator. + let results: Vec = (0..50) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::OptOut("deliberate".into()), + ) + }) + .collect(); + assert_eq!(score_pct(&results), 0); + } + + #[test] + fn rt_score_pct_one_pass_amid_999_n_a_returns_100() { + // n_a must not dilute. One genuine pass against a thousand + // inapplicable rows is still 100%. + let mut results: Vec = (0..999) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::NotApplicable("conditional unmet".into()), + ) + }) + .collect(); + results.push(make_raw("row-last", CheckStatus::Pass)); + assert_eq!(score_pct(&results), 100); + } + + #[test] + fn rt_score_pct_skip_and_error_still_excluded() { + // Carry the legacy contract forward: Skip and Error contribute to + // neither side. A run of (1 Pass + 100 Skip + 100 Error) is 100%. + let mut results = vec![make_raw("good", CheckStatus::Pass)]; + for i in 0..100 { + results.push(make_raw( + &format!("s-{i}"), + CheckStatus::Skip("limit".into()), + )); + results.push(make_raw( + &format!("e-{i}"), + CheckStatus::Error("boom".into()), + )); + } + assert_eq!(score_pct(&results), 100); + } + + #[test] + fn rt_evidence_with_control_chars_roundtrips_through_json() { + // Evidence strings come from probe output and may contain quotes, + // backslashes, newlines, tabs. serde_json must escape them. The + // roundtrip parse must recover the exact byte sequence. + let hostile: &str = "line1\nline2\t\"quoted\"\\backslash\u{0007}bell"; + let r = make_raw("c1", CheckStatus::Warn(hostile.to_string())); + let view = CheckResultView::from_result(&r); + let json = serde_json::to_string(&view).expect("view serializes"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("re-parses"); + assert_eq!( + parsed["evidence"].as_str(), + Some(hostile), + "evidence must roundtrip through JSON without loss", + ); + } + + #[test] + fn rt_evidence_with_unicode_zero_width_and_rtl_roundtrips() { + // Zero-width joiner and RTL override are common smuggling vectors + // in display contexts. They must roundtrip through JSON unchanged; + // any sanitization belongs at the render layer (site), not here. + let hostile = "left\u{202e}right\u{200b}invisible"; + let r = make_raw("c1", CheckStatus::OptOut(hostile.to_string())); + let view = CheckResultView::from_result(&r); + let json = serde_json::to_string(&view).expect("view serializes"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("re-parses"); + assert_eq!(parsed["evidence"].as_str(), Some(hostile)); + assert_eq!(parsed["status"], "opt_out"); + } + + #[test] + fn rt_summary_total_equals_sum_of_per_status_counts() { + // Invariant: total == pass + warn + fail + opt_out + n_a + skip + error. + // A new variant added without updating build_summary would break this. + let statuses = vec![ + CheckStatus::Pass, + CheckStatus::Pass, + CheckStatus::Warn("w".into()), + CheckStatus::Fail("f".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::NotApplicable("n".into()), + CheckStatus::Skip("s".into()), + CheckStatus::Error("e".into()), + ]; + let results: Vec = statuses + .into_iter() + .enumerate() + .map(|(i, s)| make_raw(&format!("c{i}"), s)) + .collect(); + let s = build_summary(&results); + assert_eq!( + s.total, + s.pass + s.warn + s.fail + s.opt_out + s.n_a + s.skip + s.error, + "summary.total must equal the sum of every per-status counter", + ); + } + + #[test] + fn rt_full_pipeline_n_a_excluded_from_summary_n_a_and_score() { + // End-to-end: a probe emits OptOut for the antecedent. After + // fan-out + propagation, the consequent row carries n_a. The + // summary counts both. The score reflects the non-conditional + // pass-rate, untouched by the opt_out / n_a pair. + let raw = vec![ + make_raw("p2-json-output", CheckStatus::OptOut("no flag".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + make_raw("p1-non-interactive", CheckStatus::Pass), + ]; + let catalog: Vec> = vec![ + Box::new(FakeCheck { + id: "p2-json-output", + covers: &["p2-must-output-flag"], + }), + Box::new(FakeCheck { + id: "p2-schema-print", + covers: &["p2-must-schema-print"], + }), + Box::new(FakeCheck { + id: "p1-non-interactive", + covers: &["p1-must-no-interactive"], + }), + ]; + let mut rows = fan_out_per_row(&raw, &catalog); + propagate_antecedents(&mut rows, &raw); + let per_row: Vec = rows.into_iter().map(|(r, _)| r).collect(); + + let s = build_summary(&per_row); + assert_eq!(s.opt_out, 1, "p2-must-output-flag → opt_out: got {s:?}"); + assert_eq!( + s.n_a, 1, + "p2-must-schema-print → n_a via propagation: got {s:?}", + ); + // Score: 2 passes (p2-must-output-flag is opt_out, p2-must-schema-print + // is n_a; only 2 pass rows remain). Denominator = 2 (both passes), so + // 100%. The opt_out + n_a do not pull the score down. + assert_eq!(score_pct(&per_row), 100); + } } diff --git a/src/types.rs b/src/types.rs index 4a0bd21..0e3a1b8 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,13 @@ use serde::Serialize; /// The result of running a single check. +/// +/// The 7-status taxonomy splits the former `Skip` bucket into three distinct +/// outcomes so the scoring algorithm can tell "tool deliberately did not adopt +/// this" (`OptOut`) from "this check does not apply to this tool" +/// (`NotApplicable`) from "the linter could not measure" (`Skip`). See plan +/// `docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md` for +/// the taxonomy rationale and Decision 2a for antecedent propagation. #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "status", content = "evidence")] #[serde(rename_all = "snake_case")] @@ -8,6 +15,18 @@ pub enum CheckStatus { Pass, Warn(String), Fail(String), + /// Tool clearly has the capability surface but does not ship this feature + /// (deliberate non-adoption). Excluded from the numerator; whether it + /// counts in the denominator is the open formula choice deferred to U3. + OptOut(String), + /// Conditional antecedent unmet — the requirement does not apply to this + /// tool. Excluded from both numerator and denominator. Set either by a + /// verifier directly or by antecedent propagation in the scorecard module. + NotApplicable(String), + /// Linter probe limitation: the check could not measure. Excluded from + /// both numerator and denominator (preserved for backward compatibility; + /// pre-0.6 scorecards used this bucket for all of OptOut / NotApplicable + /// / Skip). Skip(String), Error(String), } @@ -53,6 +72,15 @@ pub enum CheckLayer { } /// A single check result with metadata. +/// +/// **Per-row emission (schema 0.6+).** A `Check::run()` produces one +/// probe-level `CheckResult` keyed by `id = check.id()`. The runner then +/// fans the probe out across every row in `Check::covers()`, producing one +/// scorecard row per requirement (with `id = row_id`). Antecedent +/// propagation runs after fan-out. The per-row result reuses this same +/// struct shape; `id` is then the requirement-row id, and the probe's +/// `check.id()` is recovered via the `check_id` field in +/// `scorecard::CheckResultView`. #[derive(Debug, Clone, Serialize)] pub struct CheckResult { pub id: String, diff --git a/tests/build_parser.rs b/tests/build_parser.rs index 7460f83..392a375 100644 --- a/tests/build_parser.rs +++ b/tests/build_parser.rs @@ -9,7 +9,8 @@ mod parser; use parser::{ - Applicability, Level, ParseError, ParsedRequirement, aggregate, emit_rust, parse_principle_file, + Antecedent, Applicability, Level, ParseError, ParsedRequirement, aggregate, emit_rust, + parse_principle_file, }; const VALID_P1: &str = r#"--- @@ -73,7 +74,10 @@ fn parses_valid_principle_file_in_source_order() { assert_eq!(parsed[1].id, "p1-must-bar"); assert_eq!( parsed[1].applicability, - Applicability::Conditional("condition holds".to_string()) + Applicability::Conditional { + condition: Some("condition holds".to_string()), + antecedent: None, + } ); assert_eq!(parsed[2].id, "p1-should-baz"); @@ -292,7 +296,22 @@ fn emit_rust_produces_well_formed_source() { principle: 1, level: Level::Must, summary: r#"Quotes "inside" and \ backslash."#.into(), - applicability: Applicability::Conditional("auth flow".into()), + applicability: Applicability::Conditional { + condition: Some("auth flow".into()), + antecedent: None, + }, + }, + ParsedRequirement { + id: "p1-must-baz".into(), + principle: 1, + level: Level::Must, + summary: "Conditional with check_id antecedent.".into(), + applicability: Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: "p1-prereq".into(), + }), + }, }, ]; let src = emit_rust(&reqs, "0.2.0"); @@ -302,7 +321,8 @@ fn emit_rust_produces_well_formed_source() { assert!(src.contains(r#"id: "p1-must-bar""#)); assert!(src.contains("Level::Must")); assert!(src.contains("Applicability::Universal")); - assert!(src.contains(r#"Applicability::Conditional("auth flow")"#)); + assert!(src.contains(r#"condition: Some("auth flow")"#)); + assert!(src.contains(r#"check_id: "p1-prereq""#)); assert!( src.contains(r#"Quotes \"inside\" and \\ backslash."#), "summary must escape quotes and backslashes for Rust string literal" @@ -310,6 +330,107 @@ fn emit_rust_produces_well_formed_source() { assert!(src.contains(r#"pub const SPEC_VERSION: &str = "0.2.0";"#)); } +#[test] +fn parses_new_conditional_antecedent_shape() { + let src = r#"--- +id: p2 +title: Conditional check_id shape +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-schema-when-json + level: must + applicability: + kind: conditional + antecedent: + check_id: p2-json-output + summary: If --output json is supported, the schema must be discoverable. +--- +"#; + let parsed = parse_principle_file("p2-cond.md", src).expect("valid input parses"); + assert_eq!(parsed.len(), 1); + assert_eq!( + parsed[0].applicability, + Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: "p2-json-output".to_string(), + }), + } + ); +} + +#[test] +fn rejects_unknown_kind_value() { + let src = r#"--- +id: p2 +title: Bad kind +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: optional + antecedent: + check_id: p2-something + summary: Bad kind. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("kind: optional"), + "should cite the bad kind: {msg}" + ); +} + +#[test] +fn rejects_conditional_kind_with_no_condition_or_antecedent() { + let src = r#"--- +id: p2 +title: Empty conditional +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: conditional + summary: Bare conditional. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("p2-must-foo")); + assert!(msg.contains("conditional")); +} + +#[test] +fn rejects_antecedent_missing_check_id() { + let src = r#"--- +id: p2 +title: Bad antecedent +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: conditional + antecedent: + kind: bundle-present + summary: antecedent without check_id. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("check_id"), + "should hint check_id is required: {msg}" + ); +} + #[test] fn vendored_spec_parses_to_expected_requirement_count() { // Drives the same content build.rs will see. This asserts the parser @@ -354,3 +475,200 @@ fn vendored_spec_parses_to_expected_requirement_count() { // p8-discoverable-skill-bundle.md). assert_eq!(combined.last().unwrap().id, "p8-may-bundle-update"); } + +// ───────────────────────────────────────────────────────────────────────── +// Red team: adversarial YAML inputs that try to slip past the parser. +// Every entry below tests a footgun a future spec author could fall into, +// or a subtle malformed value that the parser must surface loudly. +// ───────────────────────────────────────────────────────────────────────── + +const RT_FIXTURE_HEAD: &str = r#"--- +id: p2 +title: Red team +last-revised: 2026-01-01 +status: draft +requirements: +"#; + +fn rt_assert_error_mentions(content: &str, expected_substrings: &[&str]) { + // Append the closing frontmatter fence so each fixture is a complete + // markdown file. The body is intentionally empty — every fixture + // exercises the requirement-parsing path, not body content. + let full = format!("{content}---\n"); + let err = parse_principle_file("p2-rt.md", &full).unwrap_err(); + let msg = format!("{err}"); + for s in expected_substrings { + assert!( + msg.contains(s), + "error message must contain {s:?}, got: {msg}" + ); + } +} + +#[test] +fn rt_rejects_whitespace_only_check_id() { + // Subtle footgun — `is_empty` catches "" but not " ". The parser must + // trim before deciding "non-empty". + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: ' '\n\ + \x20\x20\x20\x20summary: bad check_id\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "check_id", "non-empty"]); +} + +#[test] +fn rt_rejects_antecedent_as_string() { + // YAML author wrote `antecedent: p2-json-output` instead of the mapping + // form. The parser must surface this with a hint at the expected shape. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: p2-json-output\n\ + \x20\x20\x20\x20summary: antecedent-as-string\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_antecedent_as_list() { + // Compound antecedents (`all_of` / `any_of`) are deferred to v2 per plan + // Sub-decision 2b. v1 must reject the list form. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20- check_id: p2-json-output\n\ + \x20\x20\x20\x20\x20\x20\x20\x20- check_id: p2-yaml-output\n\ + \x20\x20\x20\x20summary: antecedent-as-list\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_kind_with_wrong_case() { + // YAML is case-sensitive; "Conditional" is not "conditional". An author + // copying from a markdown example might capitalize and silently get a + // different code path. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: Conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20summary: capitalized kind\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "Conditional"]); +} + +#[test] +fn rt_rejects_mixed_legacy_if_and_new_kind() { + // Author mid-migration writes both shapes in the same row. The legacy + // `if:` branch only fires for `map.len() == 1`, so today both keys would + // silently pick the new shape and drop the legacy prose. The parser + // rejects the mixed shape outright. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20if: CLI emits structured output\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20summary: mixed legacy+new shape\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "if:", "kind:"]); +} + +#[test] +fn rt_rejects_antecedent_with_compound_op_key() { + // The v1 schema is strict about `antecedent` contents: only `check_id` + // is permitted. A v2-style `op: any_of` accidentally added to a v1 row + // must error so v2 syntax doesn't ship under v1 semantics. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20\x20\x20\x20\x20op: any_of\n\ + \x20\x20\x20\x20summary: smuggled v2 syntax\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "op", "v1 schema"]); +} + +#[test] +fn rt_rejects_antecedent_null() { + // `antecedent: null` slipped past an earlier draft; the parser must + // treat it the same as a non-mapping value and refuse the row. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: null\n\ + \x20\x20\x20\x20summary: null antecedent\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_empty_antecedent_mapping() { + // `antecedent: {}` is a mapping with no `check_id`. Surface the missing + // field by name. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: {{}}\n\ + \x20\x20\x20\x20summary: empty antecedent\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "check_id"]); +} + +#[test] +fn rt_emit_rust_escapes_quotes_in_antecedent_check_id() { + // Defense-in-depth: the parser already rejects whitespace-only + // check_ids, but emit_rust should still escape any string content + // that ends up in the generated source. A check_id containing a + // quote (impossible today but cheap to guard against) must not break + // the generated Rust literal. + let reqs = vec![ParsedRequirement { + id: "p2-must-quoted".into(), + principle: 2, + level: Level::Must, + summary: "row with a hostile check_id".into(), + applicability: Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: r#"p2-"injected""#.into(), + }), + }, + }]; + let src = emit_rust(&reqs, "0.0.0"); + // The check_id literal must escape the embedded quotes; the generated + // line should parse as valid Rust. + assert!( + src.contains(r#"check_id: "p2-\"injected\"""#), + "generated source must escape quotes in check_id, got:\n{src}", + ); +} diff --git a/tests/integration.rs b/tests/integration.rs index 656203f..14fa4df 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -745,7 +745,7 @@ fn test_audit_profile_echoed_in_json_output() { let json_str = String::from_utf8(output).expect("utf8 stdout"); let parsed: serde_json::Value = serde_json::from_str(&json_str).expect("valid JSON"); assert_eq!(parsed["audit_profile"], "human-tui"); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); } #[test] @@ -935,17 +935,16 @@ fn test_scorecard_json_has_stable_top_level_keys() { ); // Fixed enumerations also pin against the renderer contract. - assert_eq!(obj["schema_version"], "0.5"); + assert_eq!(obj["schema_version"], "0.6"); } #[test] fn test_audit_profile_diagnostic_does_not_panic_on_self() { // Dogfood edge case from the plan: `diagnostic-only` suppresses - // p5-dry-run on the self-target. A regression that drops the - // suppression (check runs normally) would still exit with a valid - // code, so a stronger assertion is required: `p5-dry-run` must - // appear in `results[]` as `status: "skip"` with the structured - // suppression evidence. + // p5-dry-run on the self-target. Schema 0.6 emits one result per + // requirement-row, so the suppressed probe surfaces under its row id + // `p5-must-dry-run` (covered by the `p5-dry-run` check) with + // `check_id: "p5-dry-run"` for provenance. let assert = cmd() .args([ "audit", @@ -964,15 +963,15 @@ fn test_audit_profile_diagnostic_does_not_panic_on_self() { let results = parsed["results"].as_array().expect("results is array"); let p5 = results .iter() - .find(|r| r["id"] == "p5-dry-run") - .expect("p5-dry-run check should appear in results[]"); + .find(|r| r["id"] == "p5-must-dry-run" && r["check_id"] == "p5-dry-run") + .expect("p5-must-dry-run row (from p5-dry-run probe) should appear in results[]"); assert_eq!( p5["status"], "skip", "diagnostic-only must suppress p5-dry-run to Skip (got {p5})", ); let evidence = p5["evidence"] .as_str() - .expect("suppressed p5-dry-run carries evidence string"); + .expect("suppressed p5-must-dry-run carries evidence string"); assert!( evidence.contains("suppressed by audit_profile: diagnostic-only"), "expected suppression evidence prefix, got {evidence:?}", diff --git a/tests/scorecard_metadata_security.rs b/tests/scorecard_metadata_security.rs index 1805b59..9d7990f 100644 --- a/tests/scorecard_metadata_security.rs +++ b/tests/scorecard_metadata_security.rs @@ -141,7 +141,7 @@ fn hostile_binary_nonzero_version_exit_yields_null() { ); // The scorecard itself must still emit — version probe failure is not // a scoring failure. - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); assert_eq!(parsed["target"]["kind"], "binary"); } diff --git a/tests/scorecard_schema_v05.rs b/tests/scorecard_schema_v05.rs index a9b5819..8f67a72 100644 --- a/tests/scorecard_schema_v05.rs +++ b/tests/scorecard_schema_v05.rs @@ -18,12 +18,14 @@ fn fixture_path(name: &str) -> String { format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) } -/// Assert every documented v0.5 key path resolves on the parsed JSON. The -/// segmented walk gives a precise failure message when a field is missing. +/// Assert every documented v0.5 key path resolves on the parsed JSON, plus +/// the v0.6 additions (per-row emission, `tier` + `check_id` on each +/// result, `opt_out` / `n_a` summary counters). The segmented walk gives a +/// precise failure message when a field is missing. fn assert_v05_shape(parsed: &Value) { assert_eq!( - parsed["schema_version"], "0.5", - "schema_version must be 0.5", + parsed["schema_version"], "0.6", + "schema_version must be 0.6 (per-row emission + 7-status taxonomy)", ); for path in [ @@ -54,6 +56,9 @@ fn assert_v05_shape(parsed: &Value) { "badge.scorecard_url", "badge.badge_url", "badge.convention_url", + // 0.6 additions — 7-status summary counters. + "summary.opt_out", + "summary.n_a", ] { let mut node = parsed; for segment in path.split('.') { @@ -63,6 +68,21 @@ fn assert_v05_shape(parsed: &Value) { } } + // 0.6: every result row carries `tier` and `check_id`. The shape is + // assertable as soon as `results[]` is non-empty. + if let Some(results) = parsed["results"].as_array() { + for (i, row) in results.iter().enumerate() { + assert!( + row.get("tier").is_some(), + "results[{i}] missing `tier` (schema 0.6): {row}", + ); + assert!( + row.get("check_id").is_some(), + "results[{i}] missing `check_id` (schema 0.6): {row}", + ); + } + } + // The convention URL is fixed and shared across every scored tool. A // regression that pointed it at a stale path would silently break the // pre-launch surface — pin it loudly here. @@ -305,3 +325,218 @@ fn schema_v05_badge_eligibility_flag_matches_score() { assert!(parsed["badge"]["embed_markdown"].is_null()); } } + +// ───────────────────────────────────────────────────────────────────────── +// Schema 0.6 red team: the committed scorecard.schema.json is the consumer +// contract for the site renderer and third-party leaderboards. A drift +// between the hand-written schema and the serde-derived live JSON would +// silently break those consumers. The tests below pin the shape contract +// from both directions. +// ───────────────────────────────────────────────────────────────────────── + +/// Read the committed schema once. Returns the parsed JSON value so each +/// test can assert against a specific shape concern in isolation. +fn schema_doc() -> Value { + let path = format!( + "{}/schema/scorecard.schema.json", + env!("CARGO_MANIFEST_DIR") + ); + let text = std::fs::read_to_string(&path).expect("schema file readable"); + serde_json::from_str(&text).expect("schema file is valid JSON") +} + +#[test] +fn rt_schema_id_pins_to_published_schema_version() { + // The schema's `$id` must match the SCHEMA_VERSION constant emitted by + // the runtime. A bump that updates one without the other ships a + // consumer contract that disagrees with itself. + let schema = schema_doc(); + let id = schema["$id"].as_str().expect("$id is a string"); + assert!( + id.contains("scorecard-v0.6"), + "schema $id must pin to the current SCHEMA_VERSION (0.6), got: {id}", + ); +} + +#[test] +fn rt_schema_status_enum_lists_all_seven_taxonomy_values() { + // The 7-status taxonomy is the load-bearing contract of schema 0.6. A + // drift here (missing `opt_out`, missing `n_a`, stray pre-0.6 value + // dropped, etc.) would either silently mute new statuses on the + // consumer side or fail validation against legitimate scorecards. + let schema = schema_doc(); + let enums = schema["$defs"]["CheckResultView"]["properties"]["status"]["enum"] + .as_array() + .expect("status.enum is an array"); + let values: Vec<&str> = enums.iter().filter_map(|v| v.as_str()).collect(); + for expected in ["pass", "warn", "fail", "opt_out", "n_a", "skip", "error"] { + assert!( + values.contains(&expected), + "status.enum missing `{expected}` — schema 0.6 contract violated. got: {values:?}", + ); + } + assert_eq!( + values.len(), + 7, + "status.enum must list exactly seven values; got {values:?}", + ); +} + +#[test] +fn rt_schema_summary_required_includes_opt_out_and_n_a() { + // Summary counters are an additive shape change: adding to `properties` + // without adding to `required` would let consumers omit them silently. + let schema = schema_doc(); + let required = schema["$defs"]["Summary"]["required"] + .as_array() + .expect("Summary.required is an array"); + let names: Vec<&str> = required.iter().filter_map(|v| v.as_str()).collect(); + for expected in [ + "total", "pass", "warn", "fail", "opt_out", "n_a", "skip", "error", + ] { + assert!( + names.contains(&expected), + "Summary.required missing `{expected}` — got: {names:?}", + ); + } +} + +#[test] +fn rt_schema_check_result_view_includes_tier_and_check_id() { + // Schema 0.6 added `tier` and `check_id` to every results[] entry. + // The drift guard pins both presence and the tier enum's three values + // (plus null for unknown row ids). + let schema = schema_doc(); + let props = &schema["$defs"]["CheckResultView"]["properties"]; + assert!(props["tier"].is_object(), "CheckResultView.tier missing"); + assert!( + props["check_id"].is_object(), + "CheckResultView.check_id missing", + ); + let tier_enum = props["tier"]["enum"] + .as_array() + .expect("tier.enum is array"); + let tier_values: Vec<&str> = tier_enum.iter().filter_map(|v| v.as_str()).collect(); + for expected in ["must", "should", "may"] { + assert!( + tier_values.contains(&expected), + "tier.enum missing `{expected}`, got: {tier_values:?}", + ); + } + // Also accepts null for rows whose id is not in the registry. + assert!( + tier_enum.iter().any(|v| v.is_null()), + "tier.enum must permit null for unknown row ids, got: {tier_enum:?}", + ); +} + +#[test] +fn rt_schema_example_block_passes_its_own_required_keys() { + // The schema's `examples[0]` is documentation surface — if it drifts + // from the actual `required` lists, agents copying it as a template + // will produce invalid scorecards. Walk the required[] tree and assert + // every key resolves on the example. + let schema = schema_doc(); + let example = &schema["examples"][0]; + assert!(example.is_object(), "examples[0] must be an object"); + + let top_required = schema["required"] + .as_array() + .expect("top-level required is array"); + for key_val in top_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + example.get(key).is_some(), + "examples[0] missing top-level required key `{key}`", + ); + } + + // Walk into results[0] and assert its required keys are present too. + let result_example = &example["results"][0]; + let result_required = schema["$defs"]["CheckResultView"]["required"] + .as_array() + .expect("CheckResultView.required is array"); + for key_val in result_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + result_example.get(key).is_some(), + "examples[0].results[0] missing required key `{key}`", + ); + } + + // And the summary block. + let summary_example = &example["summary"]; + let summary_required = schema["$defs"]["Summary"]["required"] + .as_array() + .expect("Summary.required is array"); + for key_val in summary_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + summary_example.get(key).is_some(), + "examples[0].summary missing required key `{key}`", + ); + } +} + +#[test] +fn rt_live_scorecard_top_level_keys_match_schema_required() { + // The strongest drift guard: spawn the real binary, produce a live + // scorecard, and assert every key in the schema's top-level `required` + // list is present. Any field added to the struct without a matching + // schema entry, or removed from the schema without removing from the + // struct, surfaces here. + let path = fixture_path("perfect-rust"); + let output = cmd() + .args(["audit", &path, "--output", "json"]) + .output() + .expect("anc spawn"); + let stdout = String::from_utf8(output.stdout).expect("utf-8 stdout"); + let live: Value = serde_json::from_str(&stdout).expect("live JSON parses"); + + let schema = schema_doc(); + let required = schema["required"] + .as_array() + .expect("top-level required is array"); + for key_val in required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + live.get(key).is_some(), + "live scorecard missing required top-level key `{key}` — \ + schema declares it but the live JSON omits it.", + ); + } +} + +#[test] +fn rt_live_results_rows_satisfy_check_result_view_required_keys() { + // Per-row contract: every row in results[] carries the keys declared + // required by CheckResultView. Catches a probe that hand-builds a + // CheckResult skipping a field, or a schema that lists a key the + // serializer dropped. + let path = fixture_path("perfect-rust"); + let output = cmd() + .args(["audit", &path, "--output", "json"]) + .output() + .expect("anc spawn"); + let stdout = String::from_utf8(output.stdout).expect("utf-8 stdout"); + let live: Value = serde_json::from_str(&stdout).expect("live JSON parses"); + + let schema = schema_doc(); + let required: Vec = schema["$defs"]["CheckResultView"]["required"] + .as_array() + .expect("CheckResultView.required is array") + .iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect(); + + let rows = live["results"].as_array().expect("results is array"); + assert!(!rows.is_empty(), "live run produced no rows"); + for (i, row) in rows.iter().enumerate() { + for key in &required { + assert!( + row.get(key).is_some(), + "results[{i}] missing required key `{key}`: row = {row}", + ); + } + } +}