From 2c029b4e85944614b45439de2ddbeaec816dd0da Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 15:39:52 -0500 Subject: [PATCH 1/6] chore(spec): vendor agentnative-spec dev @ b4f4d02 for U2 Pin-vendors agentnative-spec at commit b4f4d02 (PR brettdavies/agentnative#34 squash-merge on dev) as the basis for U2 of the scorecard fairness taxonomy plan (docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md in the site repo). The vendored tree carries U1: `applicability.kind: conditional` with `antecedent.check_id` shape on five requirement rows: `p2-must-schema-print` and `p2-should-schema-file` (antecedent `p2-json-output`), plus `p8-must-bundle-install`, `p8-may-install-all`, `p8-may-bundle-update` (antecedent `p8-bundle-exists`). Replaces the legacy `applicability.if: ` shape on those rows. p8 prose re-expressed in if-X-then-Y construction. VERSION stays at 0.4.0 by design because the spec-side VERSION bump is deferred to spec's release PR per U1's status notes. The build pipeline already reads the vendored tree at `src/principles/spec/` via `build.rs`; downstream U2 work consumes the new shape from there. Resolved short SHA: b4f4d02 (printed by sync-spec.sh during vendoring; recorded here so the pin is traceable post-merge). --- .../p1-non-interactive-by-default.md | 2 +- .../p2-structured-parseable-output.md | 10 +++++--- .../p3-progressive-help-discovery.md | 6 ++--- .../p4-fail-fast-actionable-errors.md | 2 +- .../p5-safe-retries-mutation-boundaries.md | 2 +- ...omposable-predictable-command-structure.md | 2 +- .../p7-bounded-high-signal-responses.md | 2 +- .../p8-discoverable-skill-bundle.md | 24 ++++++++++++------- 8 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/principles/spec/principles/p1-non-interactive-by-default.md b/src/principles/spec/principles/p1-non-interactive-by-default.md index a4c6ee6..f2e70ce 100644 --- a/src/principles/spec/principles/p1-non-interactive-by-default.md +++ b/src/principles/spec/principles/p1-non-interactive-by-default.md @@ -120,7 +120,7 @@ Measured by check IDs `p1-non-interactive` (behavioral) and `p1-non-interactive- ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p2-structured-parseable-output.md b/src/principles/spec/principles/p2-structured-parseable-output.md index 26098f4..9840206 100644 --- a/src/principles/spec/principles/p2-structured-parseable-output.md +++ b/src/principles/spec/principles/p2-structured-parseable-output.md @@ -23,7 +23,9 @@ requirements: - id: p2-must-schema-print level: must applicability: - if: CLI emits structured output + kind: conditional + antecedent: + check_id: p2-json-output summary: "CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier." - id: p2-should-consistent-envelope level: should @@ -32,7 +34,9 @@ requirements: - id: p2-should-schema-file level: should applicability: - if: CLI emits structured output + kind: conditional + antecedent: + check_id: p2-json-output summary: "Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool." - id: p2-should-json-aliases level: should @@ -133,7 +137,7 @@ Measured by check IDs `p2-output-json`, `p2-output-format`, `p2-stderr-diagnosti ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p3-progressive-help-discovery.md b/src/principles/spec/principles/p3-progressive-help-discovery.md index 4504bc0..97e2773 100644 --- a/src/principles/spec/principles/p3-progressive-help-discovery.md +++ b/src/principles/spec/principles/p3-progressive-help-discovery.md @@ -69,9 +69,9 @@ trial-and-errors its way into a working call, burning tokens and sometimes landi - Short `about` for command-list summaries; `long_about` reserved for detailed descriptions visible with `--help` but not `-h`. - A short alias for `--version` SHOULD work: `-V` (clap default, `curl`, `wget`, `gzip`), `-v` (`npm`, `node`, `bun`, - `yarn`, `make`), or `-version` (Go's `flag` package). Any one is sufficient. Agents probing tool versions across many - CLIs save token cost when they can pin against a one- or two-character form; the long-only path forces an extra parse - step. + `yarn`, `make`), or `-version` (Go's `flag` package). Any of the three forms is sufficient. Agents probing tool + versions across many CLIs save token cost when they can pin against a one- or two-character flag; the long-only path + forces an extra parse step. **MAY:** diff --git a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md index dc7a46d..a591e11 100644 --- a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md +++ b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md @@ -124,7 +124,7 @@ Measured by check IDs `p4-bad-args`, `p4-process-exit`, `p4-unwrap`, `p4-exit-co ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md index 271775a..f84c1f9 100644 --- a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md +++ b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md @@ -79,7 +79,7 @@ under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p6-composable-predictable-command-structure.md b/src/principles/spec/principles/p6-composable-predictable-command-structure.md index 5407ee1..1a313c4 100644 --- a/src/principles/spec/principles/p6-composable-predictable-command-structure.md +++ b/src/principles/spec/principles/p6-composable-predictable-command-structure.md @@ -162,7 +162,7 @@ check --principle 6 .` against the CLI under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p7-bounded-high-signal-responses.md b/src/principles/spec/principles/p7-bounded-high-signal-responses.md index d96e82e..7cfd1e8 100644 --- a/src/principles/spec/principles/p7-bounded-high-signal-responses.md +++ b/src/principles/spec/principles/p7-bounded-high-signal-responses.md @@ -107,7 +107,7 @@ under test to see each. ## Pressure test notes -### 2026-04-27: Show HN launch red-team pass +### 2026-04-27: Red-team pass Adversarial review via `compound-engineering:ce-adversarial-document-reviewer` ahead of the v0.3.0 launch. Findings recorded verbatim per `principles/AGENTS.md` § "Pressure-test protocol". diff --git a/src/principles/spec/principles/p8-discoverable-skill-bundle.md b/src/principles/spec/principles/p8-discoverable-skill-bundle.md index e35b93d..063efe1 100644 --- a/src/principles/spec/principles/p8-discoverable-skill-bundle.md +++ b/src/principles/spec/principles/p8-discoverable-skill-bundle.md @@ -7,7 +7,9 @@ requirements: - id: p8-must-bundle-install level: must applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes." - id: p8-should-bundle-exists level: should @@ -16,12 +18,16 @@ requirements: - id: p8-may-install-all level: may applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all." - id: p8-may-bundle-update level: may applicability: - if: CLI ships an agent skill bundle + kind: conditional + antecedent: + check_id: p8-bundle-exists summary: "An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version." --- @@ -44,7 +50,7 @@ recognizes the tool's idioms across every subsequent invocation. **MUST:** -- When a CLI ships a skill bundle, the CLI MUST provide an install path that registers the bundle with installed agent +- If a CLI ships a skill bundle, then it MUST provide an install path that registers the bundle with installed agent runtimes. The canonical form is a `tool skill install []` subcommand that writes into the runtime's filesystem cascade (e.g., `~/.claude/skills/`, `~/.cursor/skills/`). Non-canonical alternatives (`tool init --skill`, `tool skills add`, `tool agents add`) are acceptable but SHOULD migrate toward `tool skill install`. A bundle without an @@ -60,11 +66,11 @@ recognizes the tool's idioms across every subsequent invocation. **MAY:** -- An `--all` mode MAY auto-detect installed agent runtimes (Claude Code, Cursor, Codex, OpenCode, and others as the - ecosystem evolves) and install the bundle across each. A user setting up a new machine with multiple coding agents - installs once and gets coverage across every runtime. -- An `update` (or `upgrade`) subcommand under `tool skill` MAY pull the latest bundle version, so agents stay current - with the CLI's evolving surface without a full reinstall. +- If a CLI ships a skill bundle, then an `--all` mode MAY auto-detect installed agent runtimes (Claude Code, Cursor, + Codex, OpenCode, and others as the ecosystem evolves) and install the bundle across each. A user setting up a new + machine with multiple coding agents installs once and gets coverage across every runtime. +- If a CLI ships a skill bundle, then an `update` (or `upgrade`) subcommand under `tool skill` MAY pull the latest + bundle version, so agents stay current with the CLI's evolving surface without a full reinstall. ## Evidence From bf3d167d58046e42283650bf52d58df1a11d3906 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 16:58:38 -0500 Subject: [PATCH 2/6] feat(spec): conditional antecedent.check_id applicability shape Extends the build-time spec parser, runtime registry, and matrix renderer to accept the new conditional applicability shape introduced by agentnative-spec PR #34 (vendored at b4f4d02 in the prior commit) while keeping the legacy `{ if: "" }` shape for the 18 spec rows that have not yet migrated. Schema change (parsed YAML, runtime enum, matrix.json output): - `Applicability::Conditional { condition: Option, antecedent: Option }` replaces the previous `Conditional(String)` tuple variant. At least one of `condition` or `antecedent` must be set; both is permitted (the new shape supplements the legacy prose with a machine-readable check id). - `Antecedent { check_id }` is the v1 machine-readable form. Compound antecedents (`all_of` / `any_of`) are deferred to a future schema bump per plan Sub-decision 2b. - The matrix.json row's `applicability` block uses `#[serde(skip_serializing_if = "Option::is_none")]` so legacy rows continue to emit `{ "kind": "conditional", "condition": "" }` and new rows emit `{ "kind": "conditional", "antecedent": { "check_id": "" } }`. No site renderer changes required for the legacy path. The parser rejects four error modes loudly: `kind:` values other than `conditional`; `antecedent:` mappings missing `check_id`; `kind: conditional` with neither `condition:` nor `antecedent:` set; `condition:` values that are not non-empty strings. Each error names the offending file, requirement id, and the malformed field so build failures are actionable. This commit is the foundation for U2 of the scorecard fairness taxonomy plan (docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md in the site repo). Per-row emission, antecedent status propagation, and the schema 0.6 bump that consume this new shape land in the following commits on this branch. --- build_support/parser.rs | 138 +++++++++++++++++++++++++++++++++++-- src/principles/matrix.rs | 12 +++- src/principles/registry.rs | 27 +++++++- tests/build_parser.rs | 129 ++++++++++++++++++++++++++++++++-- 4 files changed, 292 insertions(+), 14 deletions(-) diff --git a/build_support/parser.rs b/build_support/parser.rs index 1d8bdfe..27e6475 100644 --- a/build_support/parser.rs +++ b/build_support/parser.rs @@ -38,7 +38,20 @@ impl Level { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Applicability { Universal, - Conditional(String), + /// Conditional applicability. Either a prose `condition` (legacy `{ if: + /// "" }` shape, machine-opaque) or a machine-readable `antecedent` + /// (new `{ kind: conditional, antecedent: { check_id: ... } }` shape), or + /// both. The new shape is preferred; the legacy shape remains accepted for + /// rows where the verifier catalog has not yet grown a prerequisite check. + Conditional { + condition: Option, + antecedent: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Antecedent { + pub check_id: String, } #[derive(Debug, Clone)] @@ -91,6 +104,10 @@ pub enum ParseError { EmptyRequirements { file: String, }, + ConditionalEmpty { + file: String, + requirement_id: String, + }, } impl fmt::Display for ParseError { @@ -149,6 +166,13 @@ impl fmt::Display for ParseError { ParseError::EmptyRequirements { file } => { write!(f, "{file}: `requirements` list is empty") } + ParseError::ConditionalEmpty { + file, + requirement_id, + } => write!( + f, + "{file}: requirement `{requirement_id}` declares `kind: conditional` but provides neither `condition:` nor `antecedent.check_id` — at least one must be set" + ), } } } @@ -282,10 +306,23 @@ pub fn emit_rust(reqs: &[ParsedRequirement], spec_version: &str) -> String { Applicability::Universal => { out.push_str(" applicability: Applicability::Universal,\n"); } - Applicability::Conditional(cond) => { + Applicability::Conditional { + condition, + antecedent, + } => { + let cond_lit = match condition { + Some(c) => format!("Some(\"{}\")", escape_rust_str(c)), + None => "None".to_string(), + }; + let ante_lit = match antecedent { + Some(a) => format!( + "Some(Antecedent {{ check_id: \"{}\" }})", + escape_rust_str(&a.check_id) + ), + None => "None".to_string(), + }; out.push_str(&format!( - " applicability: Applicability::Conditional(\"{}\"),\n", - escape_rust_str(cond) + " applicability: Applicability::Conditional {{ condition: {cond_lit}, antecedent: {ante_lit} }},\n" )); } } @@ -397,6 +434,9 @@ fn parse_applicability( if let Some(map) = value.as_mapping() { let if_key = serde_yaml::Value::String("if".into()); + let kind_key = serde_yaml::Value::String("kind".into()); + + // Legacy single-key `{ if: "" }` shape. if map.len() == 1 && let Some(if_val) = map.get(&if_key) { @@ -414,19 +454,103 @@ fn parse_applicability( hint: "`if:` value must be a non-empty string".into(), }); } - return Ok(Applicability::Conditional(cond.to_string())); + return Ok(Applicability::Conditional { + condition: Some(cond.to_string()), + antecedent: None, + }); + } + + // New `{ kind: conditional, antecedent: { check_id: ... }, condition? }` + // shape. The `kind:` key gates the new branch so legacy maps with + // unrelated keys still surface as UnknownApplicability. + if let Some(kind_val) = map.get(&kind_key) { + let kind = kind_val + .as_str() + .ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`kind:` value must be the string `conditional`".into(), + })?; + if kind != "conditional" { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: format!("unknown `kind: {kind}` — only `conditional` is supported"), + }); + } + + let antecedent_key = serde_yaml::Value::String("antecedent".into()); + let condition_key = serde_yaml::Value::String("condition".into()); + + let antecedent = if let Some(ante_val) = map.get(&antecedent_key) { + let ante_map = + ante_val + .as_mapping() + .ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent:` must be a mapping like `{ check_id: }`" + .into(), + })?; + let check_id_key = serde_yaml::Value::String("check_id".into()); + let check_id = ante_map.get(&check_id_key).and_then(|v| v.as_str()).ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent.check_id` must be a non-empty string (v1 schema supports a single antecedent only)".into(), + })?; + if check_id.is_empty() { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`antecedent.check_id` must be a non-empty string".into(), + }); + } + Some(Antecedent { + check_id: check_id.to_string(), + }) + } else { + None + }; + + let condition = match map.get(&condition_key) { + Some(v) => { + let s = v.as_str().ok_or_else(|| ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`condition:` must be a string".into(), + })?; + if s.is_empty() { + None + } else { + Some(s.to_string()) + } + } + None => None, + }; + + if condition.is_none() && antecedent.is_none() { + return Err(ParseError::ConditionalEmpty { + file: file.to_string(), + requirement_id: req_id.to_string(), + }); + } + return Ok(Applicability::Conditional { + condition, + antecedent, + }); } + return Err(ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), - hint: "expected `{ if: \"\" }`".into(), + hint: "expected `{ if: \"\" }` or `{ kind: conditional, antecedent: { check_id: } }`".into(), }); } Err(ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), - hint: "must be `universal` or `{ if: \"\" }`".into(), + hint: "must be `universal`, `{ if: \"\" }`, or `{ kind: conditional, antecedent: { check_id: } }`".into(), }) } diff --git a/src/principles/matrix.rs b/src/principles/matrix.rs index 2da5a24..fe114a9 100644 --- a/src/principles/matrix.rs +++ b/src/principles/matrix.rs @@ -267,7 +267,17 @@ pub fn render_markdown(matrix: &Matrix) -> String { }; let applicability = match row.applicability { Applicability::Universal => "Universal".to_string(), - Applicability::Conditional(cond) => format!("If: {cond}"), + Applicability::Conditional { + condition, + antecedent, + } => match (condition, antecedent) { + (Some(cond), Some(ante)) => { + format!("If: {cond} (antecedent: `{}`)", ante.check_id) + } + (Some(cond), None) => format!("If: {cond}"), + (None, Some(ante)) => format!("If: `{}` is present", ante.check_id), + (None, None) => "Conditional".to_string(), + }, }; let verifiers = if row.verifiers.is_empty() { "**UNCOVERED**".to_string() diff --git a/src/principles/registry.rs b/src/principles/registry.rs index 290fe1c..11f765a 100644 --- a/src/principles/registry.rs +++ b/src/principles/registry.rs @@ -18,11 +18,34 @@ pub enum Level { } /// Whether a requirement applies to every CLI or only when a condition holds. +/// +/// `Conditional` carries an optional prose `condition` (legacy `{ if: "" +/// }` shape) and an optional machine-readable `antecedent` (new `{ kind: +/// conditional, antecedent: { check_id: ... } }` shape). The antecedent's check +/// status drives the propagation table documented in +/// `docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md` +/// Decision 2a: when the antecedent resolves to `opt_out` / `n_a`, this +/// requirement's row in the scorecard collapses to `n_a`; `skip` / `error` +/// inherit; `pass` / `warn` / `fail` let the consequent verifier's own status +/// stand. #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -#[serde(tag = "kind", content = "condition", rename_all = "lowercase")] +#[serde(tag = "kind", rename_all = "lowercase")] pub enum Applicability { Universal, - Conditional(&'static str), + Conditional { + #[serde(default, skip_serializing_if = "Option::is_none")] + condition: Option<&'static str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + antecedent: Option, + }, +} + +/// Machine-readable antecedent for a conditional requirement. The +/// `check_id` names the verifier whose status decides whether the consequent +/// row applies (see `Applicability` for the propagation rules). +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub struct Antecedent { + pub check_id: &'static str, } /// Categories under which a tool may be exempt from specific requirements. diff --git a/tests/build_parser.rs b/tests/build_parser.rs index 7460f83..6403cc6 100644 --- a/tests/build_parser.rs +++ b/tests/build_parser.rs @@ -9,7 +9,8 @@ mod parser; use parser::{ - Applicability, Level, ParseError, ParsedRequirement, aggregate, emit_rust, parse_principle_file, + Antecedent, Applicability, Level, ParseError, ParsedRequirement, aggregate, emit_rust, + parse_principle_file, }; const VALID_P1: &str = r#"--- @@ -73,7 +74,10 @@ fn parses_valid_principle_file_in_source_order() { assert_eq!(parsed[1].id, "p1-must-bar"); assert_eq!( parsed[1].applicability, - Applicability::Conditional("condition holds".to_string()) + Applicability::Conditional { + condition: Some("condition holds".to_string()), + antecedent: None, + } ); assert_eq!(parsed[2].id, "p1-should-baz"); @@ -292,7 +296,22 @@ fn emit_rust_produces_well_formed_source() { principle: 1, level: Level::Must, summary: r#"Quotes "inside" and \ backslash."#.into(), - applicability: Applicability::Conditional("auth flow".into()), + applicability: Applicability::Conditional { + condition: Some("auth flow".into()), + antecedent: None, + }, + }, + ParsedRequirement { + id: "p1-must-baz".into(), + principle: 1, + level: Level::Must, + summary: "Conditional with check_id antecedent.".into(), + applicability: Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: "p1-prereq".into(), + }), + }, }, ]; let src = emit_rust(&reqs, "0.2.0"); @@ -302,7 +321,8 @@ fn emit_rust_produces_well_formed_source() { assert!(src.contains(r#"id: "p1-must-bar""#)); assert!(src.contains("Level::Must")); assert!(src.contains("Applicability::Universal")); - assert!(src.contains(r#"Applicability::Conditional("auth flow")"#)); + assert!(src.contains(r#"condition: Some("auth flow")"#)); + assert!(src.contains(r#"check_id: "p1-prereq""#)); assert!( src.contains(r#"Quotes \"inside\" and \\ backslash."#), "summary must escape quotes and backslashes for Rust string literal" @@ -310,6 +330,107 @@ fn emit_rust_produces_well_formed_source() { assert!(src.contains(r#"pub const SPEC_VERSION: &str = "0.2.0";"#)); } +#[test] +fn parses_new_conditional_antecedent_shape() { + let src = r#"--- +id: p2 +title: Conditional check_id shape +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-schema-when-json + level: must + applicability: + kind: conditional + antecedent: + check_id: p2-json-output + summary: If --output json is supported, the schema must be discoverable. +--- +"#; + let parsed = parse_principle_file("p2-cond.md", src).expect("valid input parses"); + assert_eq!(parsed.len(), 1); + assert_eq!( + parsed[0].applicability, + Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: "p2-json-output".to_string(), + }), + } + ); +} + +#[test] +fn rejects_unknown_kind_value() { + let src = r#"--- +id: p2 +title: Bad kind +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: optional + antecedent: + check_id: p2-something + summary: Bad kind. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("kind: optional"), + "should cite the bad kind: {msg}" + ); +} + +#[test] +fn rejects_conditional_kind_with_no_condition_or_antecedent() { + let src = r#"--- +id: p2 +title: Empty conditional +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: conditional + summary: Bare conditional. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("p2-must-foo")); + assert!(msg.contains("conditional")); +} + +#[test] +fn rejects_antecedent_missing_check_id() { + let src = r#"--- +id: p2 +title: Bad antecedent +last-revised: 2026-01-01 +status: draft +requirements: + - id: p2-must-foo + level: must + applicability: + kind: conditional + antecedent: + kind: bundle-present + summary: antecedent without check_id. +--- +"#; + let err = parse_principle_file("p2-bad.md", src).unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("check_id"), + "should hint check_id is required: {msg}" + ); +} + #[test] fn vendored_spec_parses_to_expected_requirement_count() { // Drives the same content build.rs will see. This asserts the parser From 8c814f7c036704d5b1504ea59a0c0dc3ae73694c Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 17:00:28 -0500 Subject: [PATCH 3/6] feat(scorecard): per-row emission, antecedent propagation, schema 0.6 Implements the bulk of U2 from the scorecard fairness taxonomy plan (docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md). Three changes ship together because they share the same pattern-match exhaustiveness contract on `CheckStatus`: the new variants, the per-row pipeline that emits them, and the test fixtures that pin the JSON shape. **7-status taxonomy (Decision 1).** `CheckStatus` gains `OptOut(String)` for deliberate non-adoption and `NotApplicable(String)` for conditional rows whose antecedent is unmet. JSON serializes as `opt_out` and `n_a` respectively. Every match site (`build_summary`, `score_pct`, `exit_code`, `format_text`, `format_text_raw`, `CheckResultView::from_row`) now handles all seven variants exhaustively. The plan's transitional `score_pct` formula keeps the historic `pass / (pass + warn + fail)` shape and excludes opt_out from the denominator and n_a from both sides (per U2 work item: "minimal change that respects the new semantics without committing to a new formula"); the final formula is the U3 spec issue. **Per-row emission (Decision 2c).** New `fan_out_per_row(raw, catalog)` walks each probe-level `CheckResult` and emits one row per requirement in `Check::covers()`, replacing the row `id` with the requirement id and carrying the probe's check id forward as provenance. A probe like `p3-version` (covers `p3-must-version` + `p3-should-version-short`) now produces two rows in `results[]`; a probe whose check is not yet wired into the registry passes through as a single row keyed by check id. `CheckResultView` gains `tier` (registry-looked-up `must` / `should` / `may`, or `null` for unknown ids) and `check_id` (probe provenance). The shape change drives the schema bump from 0.5 to 0.6; pre-0.6 consumers feature-detect. **Antecedent propagation (Decision 2a).** New `propagate_antecedents(rows, raw)` reads each row's registry `Applicability` and applies the propagation table to conditional rows: `opt_out` / `n_a` antecedent collapses the consequent to `n_a`; `skip` and `error` inherit; `pass` / `warn` / `fail` leave the consequent untouched (the probe's own status stands). Evidence text on the rewritten row cites the antecedent check id and its status so the provenance is legible in the scorecard JSON. `Summary` gains `opt_out` and `n_a` counters alongside the historic five so consumers can read the new buckets without scanning `results[]`. The badge derivation reads the per-row vector (post-propagation) so the embed URL the JSON emits and the post-summary text hint stay in lock-step. The audience classifier and coverage_summary continue to read raw probe results because signal classification keys on check ids and coverage counts requirements covered by the underlying probes. Test updates pin the new shape: 14 new scorecard-module tests cover fan-out for single-row and multi-row probes, the full propagation table (pass/warn/fail/opt_out/n_a/skip/error antecedents), score_pct exclusions, summary counting, and the `tier` / `check_id` round-trip; integration tests update the suppression test to look up the row id under per-row emission; the schema_v05 drift guard adds 0.6 keys (summary counters, per-row `tier`/`check_id`) so consumers' shape contract is checked end-to-end. --- src/scorecard/mod.rs | 551 +++++++++++++++++++++++++-- src/types.rs | 28 ++ tests/integration.rs | 19 +- tests/scorecard_metadata_security.rs | 2 +- tests/scorecard_schema_v05.rs | 28 +- 5 files changed, 578 insertions(+), 50 deletions(-) diff --git a/src/scorecard/mod.rs b/src/scorecard/mod.rs index aac31e7..a46b3b1 100644 --- a/src/scorecard/mod.rs +++ b/src/scorecard/mod.rs @@ -20,8 +20,11 @@ use crate::types::{CheckGroup, CheckResult, CheckStatus}; /// target metadata blocks — self-describing scoring run), `0.5` (`badge` /// block — eligibility, embed snippet, and badge/scorecard URLs derived /// from the run, so authors learn about the badge from the CLI itself -/// rather than a round-trip to the site). -pub const SCHEMA_VERSION: &str = "0.5"; +/// rather than a round-trip to the site), `0.6` (7-status taxonomy: +/// `opt_out` + `n_a` added to `status`; matching counters in `summary`; +/// `tier` field on each result; one result per requirement-row instead of +/// per-check_id; antecedent propagation for conditional rows). +pub const SCHEMA_VERSION: &str = "0.6"; /// Eligibility floor for the agent-native badge, expressed as an integer /// percent. A score that meets or exceeds this floor qualifies a tool to @@ -186,11 +189,22 @@ pub fn compute_badge(results: &[CheckResult], tool_name: &str) -> BadgeInfo { } } -/// Compute the rounded integer percent score using the leaderboard's -/// denominator (`pass + warn + fail`). Skips and errors are excluded from -/// both sides of the ratio. Returns `0` when no checks contribute (every -/// status was Skip or Error, or no checks ran at all) — pairs with -/// `BadgeInfo::eligible == false` so a zero score never qualifies. +/// Compute the rounded integer percent score using the transitional +/// leaderboard denominator. Pass/Warn/Fail count in the denominator. Pass +/// counts in the numerator. Skip, Error, OptOut, and NotApplicable are +/// excluded from both sides of the ratio. +/// +/// **Transitional formula choice.** The 7-status taxonomy semantically +/// treats `opt_out` as in-denominator (deliberate non-adoption is a real +/// signal), but plan U2 explicitly keeps the U2 formula conservative: +/// "leave the existing formula but exclude `opt_out` from the denominator +/// and exclude `n_a` from both — minimal change that respects the new +/// semantics without committing to a new formula." Whether `opt_out` +/// re-enters the denominator (and at what weight) is the U3 spec issue, +/// after the disambiguated input has been rescored. +/// +/// Returns `0` when no checks contribute — pairs with `BadgeInfo::eligible +/// == false` so a zero score never qualifies. fn score_pct(results: &[CheckResult]) -> u32 { let mut pass = 0u32; let mut denom = 0u32; @@ -203,7 +217,10 @@ fn score_pct(results: &[CheckResult]) -> u32 { CheckStatus::Warn(_) | CheckStatus::Fail(_) => { denom += 1; } - CheckStatus::Skip(_) | CheckStatus::Error(_) => {} + CheckStatus::Skip(_) + | CheckStatus::Error(_) + | CheckStatus::OptOut(_) + | CheckStatus::NotApplicable(_) => {} } } if denom == 0 { @@ -292,16 +309,30 @@ pub struct CoverageSummary { pub may: LevelCounts, } +/// Run-level outcome counts. The 7-status taxonomy added `opt_out` and +/// `n_a` in schema 0.6; pre-0.6 consumers tolerate the new keys (additive +/// extension). #[derive(Serialize)] pub struct Summary { pub total: usize, pub pass: usize, pub warn: usize, pub fail: usize, + pub opt_out: usize, + pub n_a: usize, pub skip: usize, pub error: usize, } +/// One row of `results[]` in the scorecard JSON. +/// +/// Schema 0.6 changed the unit of emission from "per check_id" to "per +/// requirement-row". `id` is now the requirement row id (matches +/// `coverage/matrix.json` row IDs). `tier` carries the row's RFC 2119 +/// level (`must`/`should`/`may`) so downstream scoring consumers do not +/// need a matrix join. `check_id` is the probe that produced this row, +/// preserved for provenance and so the site renderer / audience classifier +/// can find the originating probe without a registry walk. #[derive(Serialize)] pub struct CheckResultView { pub id: String, @@ -313,14 +344,42 @@ pub struct CheckResultView { /// `high` for direct probes, `medium` for heuristics. Older consumers /// feature-detect and tolerate missing keys. pub confidence: String, + /// Requirement tier (`must`/`should`/`may`). Pre-launch additive + /// (schema `0.6`). `null` only for results whose row id is not in the + /// registry — an internal inconsistency that should be loud. + pub tier: Option, + /// Underlying probe that produced this row (e.g., `p3-version` covers + /// both `p3-must-version` and `p3-should-version-short` — two rows + /// share one `check_id`). Pre-launch additive (schema `0.6`). Falls + /// back to the row `id` itself when no provenance was threaded in + /// (legacy test fixtures that hand-build a `CheckResult` without the + /// fan-out pipeline). + pub check_id: String, } impl CheckResultView { + /// Construct from a raw probe result (pre-fan-out callers and test + /// fixtures). `check_id` defaults to `r.id` and `tier` is looked up + /// from the registry (which fails to find anything for arbitrary test + /// IDs — surfaces as JSON null). Production code uses `from_row` + /// directly with the threaded probe provenance; this fallback exists + /// for tests and any future caller that builds a per-check view + /// without the fan-out pipeline. + #[allow(dead_code)] pub fn from_result(r: &CheckResult) -> Self { + Self::from_row(r, &r.id) + } + + /// Construct from a fanned-out per-row result with explicit probe + /// provenance. `check_id` is the probe's `Check::id()`; `r.id` is the + /// requirement row id. + pub fn from_row(r: &CheckResult, check_id: &str) -> Self { let (status, evidence) = match &r.status { CheckStatus::Pass => ("pass".to_string(), None), CheckStatus::Warn(e) => ("warn".to_string(), Some(e.clone())), CheckStatus::Fail(e) => ("fail".to_string(), Some(e.clone())), + CheckStatus::OptOut(e) => ("opt_out".to_string(), Some(e.clone())), + CheckStatus::NotApplicable(e) => ("n_a".to_string(), Some(e.clone())), CheckStatus::Skip(e) => ("skip".to_string(), Some(e.clone())), CheckStatus::Error(e) => ("error".to_string(), Some(e.clone())), }; @@ -338,6 +397,16 @@ impl CheckResultView { .ok() .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_else(|| format!("{:?}", r.confidence)); + // Look up tier from the registry. The row ID is the result ID under + // the per-row emission contract introduced in schema 0.6. Per-check + // results fed in by older callers (or test fixtures) won't find a + // match here — `tier` falls back to None, which surfaces as JSON + // null and is a visible sign of inconsistency. + let tier = crate::principles::registry::find(&r.id).map(|req| match req.level { + crate::principles::registry::Level::Must => "must".to_string(), + crate::principles::registry::Level::Should => "should".to_string(), + crate::principles::registry::Level::May => "may".to_string(), + }); CheckResultView { id: r.id.clone(), label: r.label.clone(), @@ -346,6 +415,8 @@ impl CheckResultView { status, evidence, confidence, + tier, + check_id: check_id.to_string(), } } } @@ -365,6 +436,14 @@ fn build_summary(results: &[CheckResult]) -> Summary { .iter() .filter(|r| matches!(r.status, CheckStatus::Fail(_))) .count(), + opt_out: results + .iter() + .filter(|r| matches!(r.status, CheckStatus::OptOut(_))) + .count(), + n_a: results + .iter() + .filter(|r| matches!(r.status, CheckStatus::NotApplicable(_))) + .count(), skip: results .iter() .filter(|r| matches!(r.status, CheckStatus::Skip(_))) @@ -461,6 +540,18 @@ pub fn format_text( } CheckStatus::Warn(_) => "WARN", CheckStatus::Fail(_) => "FAIL", + CheckStatus::OptOut(_) => { + if quiet { + continue; + } + "OPT " + } + CheckStatus::NotApplicable(_) => { + if quiet { + continue; + } + "N/A " + } CheckStatus::Skip(_) => { if quiet { continue; @@ -478,7 +569,11 @@ pub fn format_text( let _ = writeln!(out, " {line}"); } } - CheckStatus::Skip(reason) if !quiet => { + CheckStatus::Skip(reason) + | CheckStatus::OptOut(reason) + | CheckStatus::NotApplicable(reason) + if !quiet => + { let _ = writeln!(out, " {reason}"); } _ => {} @@ -490,8 +585,8 @@ pub fn format_text( let s = build_summary(results); let _ = writeln!( out, - "\n{} checks: {} pass, {} warn, {} fail, {} skip, {} error", - s.total, s.pass, s.warn, s.fail, s.skip, s.error + "\n{} checks: {} pass, {} warn, {} fail, {} opt_out, {} n_a, {} skip, {} error", + s.total, s.pass, s.warn, s.fail, s.opt_out, s.n_a, s.skip, s.error ); // Badge embed hint — appended only when eligible. Below the floor the @@ -505,9 +600,9 @@ pub fn format_text( } /// `--raw` rendering: one `idstatus` line per result, nothing else. -/// Status maps to the same five tokens the rich renderer uses (`PASS`, -/// `WARN`, `FAIL`, `SKIP`, `ERR`) so downstream pipelines see the same -/// vocabulary in both modes. +/// Status maps to one of the seven tokens (`PASS`, `WARN`, `FAIL`, +/// `OPT_OUT`, `N_A`, `SKIP`, `ERR`) so downstream pipelines see the same +/// vocabulary as the JSON `status` field (uppercased). fn format_text_raw(results: &[CheckResult]) -> String { let mut out = String::with_capacity(results.len() * 32); for r in results { @@ -515,6 +610,8 @@ fn format_text_raw(results: &[CheckResult]) -> String { CheckStatus::Pass => "PASS", CheckStatus::Warn(_) => "WARN", CheckStatus::Fail(_) => "FAIL", + CheckStatus::OptOut(_) => "OPT_OUT", + CheckStatus::NotApplicable(_) => "N_A", CheckStatus::Skip(_) => "SKIP", CheckStatus::Error(_) => "ERR", }; @@ -523,6 +620,101 @@ fn format_text_raw(results: &[CheckResult]) -> String { out } +/// Fan one probe-level result out into one entry per requirement-row in +/// the check's `Check::covers()` slice. The probe's status, label, group, +/// layer, and confidence propagate to every row; the `id` field is +/// replaced with the row id. Returns a pair `(row_result, check_id)` per +/// emitted row so downstream consumers (CheckResultView, propagation) know +/// the originating probe without a registry walk. +/// +/// Checks that declare no `covers()` rows produce a single passthrough +/// entry keyed by their own id — preserves the legacy per-check_id shape +/// for checks not yet wired into the requirement registry. +pub fn fan_out_per_row( + raw: &[CheckResult], + catalog: &[Box], +) -> Vec<(CheckResult, String)> { + let covers_by_id: HashMap<&str, &'static [&'static str]> = + catalog.iter().map(|c| (c.id(), c.covers())).collect(); + let mut out: Vec<(CheckResult, String)> = Vec::with_capacity(raw.len()); + for r in raw { + let covers = covers_by_id.get(r.id.as_str()).copied().unwrap_or(&[]); + if covers.is_empty() { + out.push((r.clone(), r.id.clone())); + continue; + } + for row_id in covers { + let mut row = r.clone(); + row.id = (*row_id).to_string(); + out.push((row, r.id.clone())); + } + } + out +} + +/// Apply the antecedent-status propagation table from plan Decision 2a. +/// +/// For each row whose registry entry has a conditional applicability with +/// an `antecedent.check_id`, look up the antecedent probe's raw status and +/// rewrite the row's status accordingly: +/// +/// | Antecedent status | Consequent row becomes | +/// | ----------------- | ----------------------------------- | +/// | `pass` / `warn` / `fail` | unchanged (evaluated normally) | +/// | `opt_out` / `n_a` | `n_a` (prerequisite absent) | +/// | `skip` | `skip` (inherited indeterminacy) | +/// | `error` | `error` (inherited indeterminacy) | +/// +/// Rows with no registry entry (legacy / unknown ids) are left untouched. +/// Rows whose antecedent did not produce a raw result (the antecedent +/// probe didn't run this invocation, e.g., source-only mode) are left +/// untouched — propagation needs an antecedent status to act on. +pub fn propagate_antecedents(rows: &mut [(CheckResult, String)], raw: &[CheckResult]) { + use crate::principles::registry::{Applicability, find}; + let raw_by_id: HashMap<&str, &CheckStatus> = + raw.iter().map(|r| (r.id.as_str(), &r.status)).collect(); + for (row, _check_id) in rows.iter_mut() { + let Some(req) = find(&row.id) else { continue }; + let Applicability::Conditional { antecedent, .. } = req.applicability else { + continue; + }; + let Some(ante) = antecedent else { continue }; + let Some(ante_status) = raw_by_id.get(ante.check_id) else { + continue; + }; + let new_status = match ante_status { + CheckStatus::Pass | CheckStatus::Warn(_) | CheckStatus::Fail(_) => continue, + CheckStatus::OptOut(reason) | CheckStatus::NotApplicable(reason) => { + CheckStatus::NotApplicable(format!( + "antecedent `{}` is {}: {reason}", + ante.check_id, + short_status_name(ante_status), + )) + } + CheckStatus::Skip(reason) => CheckStatus::Skip(format!( + "antecedent `{}` could not be measured: {reason}", + ante.check_id, + )), + CheckStatus::Error(reason) => { + CheckStatus::Error(format!("antecedent `{}` errored: {reason}", ante.check_id,)) + } + }; + row.status = new_status; + } +} + +fn short_status_name(s: &CheckStatus) -> &'static str { + match s { + CheckStatus::Pass => "pass", + CheckStatus::Warn(_) => "warn", + CheckStatus::Fail(_) => "fail", + CheckStatus::OptOut(_) => "opt_out", + CheckStatus::NotApplicable(_) => "n_a", + CheckStatus::Skip(_) => "skip", + CheckStatus::Error(_) => "error", + } +} + /// Bundle of run-level metadata captured by the runner around `Commands::Audit` /// and threaded into the scorecard. Grouped to keep `build_scorecard`'s /// signature manageable as schema `0.x` continues to add fields. The runner @@ -535,24 +727,32 @@ pub struct RunMetadata { } /// Build the scorecard. The `ran_checks` slice is the catalog of checks -/// that produced `results` — needed to translate check IDs back to the -/// requirement IDs they cover for `coverage_summary`. +/// that produced `raw_results`. +/// +/// Pipeline (schema 0.6): +/// raw probe results → fan out per requirement-row → antecedent +/// propagation → JSON view. Audience and coverage_summary still consume +/// raw probe results (signal classification keys on check_ids; coverage +/// counts requirements covered by the underlying probes). pub fn build_scorecard( - results: &[CheckResult], + raw_results: &[CheckResult], ran_checks: &[Box], audience: Option, audit_profile: Option, metadata: RunMetadata, ) -> Scorecard { - // `audience_reason` is derived from `results` rather than threaded + let mut row_results = fan_out_per_row(raw_results, ran_checks); + propagate_antecedents(&mut row_results, raw_results); + + // `audience_reason` is derived from `raw_results` rather than threaded // through as a caller parameter — the reason is a property of the - // result set, not a caller decision, and deriving it here keeps the - // label and its explanation in lock-step. When audience has a label - // the field is omitted from JSON (see Scorecard's serde skip rule). + // probe-level result set, not a caller decision, and deriving it here + // keeps the label and its explanation in lock-step. When audience has + // a label the field is omitted from JSON. let audience_reason = if audience.is_some() { None } else { - audience::classify_reason(results).map(|s| s.to_string()) + audience::classify_reason(raw_results).map(|s| s.to_string()) }; let RunMetadata { tool, @@ -560,16 +760,21 @@ pub fn build_scorecard( run, target, } = metadata; - // Compute the badge from the same `tool.name` the JSON emits, so the - // embed URL in `badge.embed_markdown` and the slug in `tool.name` can - // never disagree (a regression that diverges them would mislead any - // author copy-pasting from the JSON). - let badge = compute_badge(results, &tool.name); + + // Per-row results drive `summary` and `score_pct`. The badge uses the + // same per-row vector so the embed URL the JSON emits agrees with the + // post-summary text hint. + let per_row_only: Vec = row_results.iter().map(|(r, _)| r.clone()).collect(); + let badge = compute_badge(&per_row_only, &tool.name); + Scorecard { schema_version: SCHEMA_VERSION, - results: results.iter().map(CheckResultView::from_result).collect(), - summary: build_summary(results), - coverage_summary: build_coverage_summary(results, ran_checks), + results: row_results + .iter() + .map(|(r, check_id)| CheckResultView::from_row(r, check_id)) + .collect(), + summary: build_summary(&per_row_only), + coverage_summary: build_coverage_summary(raw_results, ran_checks), audience, audience_reason, audit_profile, @@ -583,13 +788,13 @@ pub fn build_scorecard( } pub fn format_json( - results: &[CheckResult], + raw_results: &[CheckResult], ran_checks: &[Box], audience: Option, audit_profile: Option, metadata: RunMetadata, ) -> String { - let scorecard = build_scorecard(results, ran_checks, audience, audit_profile, metadata); + let scorecard = build_scorecard(raw_results, ran_checks, audience, audit_profile, metadata); serde_json::to_string_pretty(&scorecard).unwrap_or_else(|e| format!("{{\"error\": \"{e}\"}}")) } @@ -732,7 +937,7 @@ mod tests { ]; let json = format_json(&results, &[], None, None, fixture_metadata()); let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); assert_eq!(parsed["summary"]["total"], 2); assert_eq!(parsed["summary"]["pass"], 1); assert_eq!(parsed["summary"]["fail"], 1); @@ -951,7 +1156,7 @@ mod tests { let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); assert_eq!(parsed["audience"], "agent-optimized"); assert!(parsed["audit_profile"].is_null()); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); } #[test] @@ -1227,7 +1432,7 @@ mod tests { } // 0.4 + 0.5 additions — every documented sub-key resolves. - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); for path in [ // 0.4 "tool.name", @@ -1550,4 +1755,280 @@ mod tests { ); assert_eq!(parsed["badge"]["convention_url"], "https://anc.dev/badge"); } + + // ────────────────────────────────────────────────────────────────── + // U2 (schema 0.6): per-row emission, tier, 7-status taxonomy, + // antecedent propagation. Plan reference: + // docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md + // in agentnative-site. + // ────────────────────────────────────────────────────────────────── + + fn make_raw(id: &str, status: CheckStatus) -> CheckResult { + make_result(id, status, CheckGroup::P2) + } + + /// Minimal `Check` impl that lets per-row fan-out tests express a + /// `covers()` slice without spinning up a real probe. + struct FakeCheck { + id: &'static str, + covers: &'static [&'static str], + } + + impl crate::check::Check for FakeCheck { + fn id(&self) -> &str { + self.id + } + fn label(&self) -> &'static str { + "fake" + } + fn group(&self) -> CheckGroup { + CheckGroup::P2 + } + fn layer(&self) -> CheckLayer { + CheckLayer::Behavioral + } + fn applicable(&self, _p: &crate::project::Project) -> bool { + true + } + fn run(&self, _p: &crate::project::Project) -> anyhow::Result { + unreachable!() + } + fn covers(&self) -> &'static [&'static str] { + self.covers + } + } + + #[test] + fn fan_out_emits_one_row_per_covered_requirement() { + // Single probe (`p3-version`) covers two requirement rows. Fan-out + // produces two entries with id = row_id and check_id = probe id. + let raw = vec![make_raw( + "p3-version", + CheckStatus::Warn("short alias missing".into()), + )]; + let catalog: Vec> = vec![Box::new(FakeCheck { + id: "p3-version", + covers: &["p3-must-version", "p3-should-version-short"], + })]; + let rows = fan_out_per_row(&raw, &catalog); + assert_eq!(rows.len(), 2); + let ids: Vec<&str> = rows.iter().map(|(r, _)| r.id.as_str()).collect(); + assert!(ids.contains(&"p3-must-version")); + assert!(ids.contains(&"p3-should-version-short")); + for (r, check_id) in &rows { + assert_eq!( + check_id, "p3-version", + "check_id provenance lost on row {}", + r.id + ); + assert!( + matches!(r.status, CheckStatus::Warn(_)), + "probe status must propagate to every covered row pre-propagation", + ); + } + } + + #[test] + fn fan_out_emits_passthrough_for_checks_without_covers() { + // Checks that don't declare any covers() pass through as a single + // row keyed by check.id() — preserves the legacy shape for any + // future check not yet wired into the registry. + let raw = vec![make_raw("orphan-check", CheckStatus::Pass)]; + let catalog: Vec> = vec![Box::new(FakeCheck { + id: "orphan-check", + covers: &[], + })]; + let rows = fan_out_per_row(&raw, &catalog); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0.id, "orphan-check"); + assert_eq!(rows[0].1, "orphan-check"); + } + + #[test] + fn propagation_passes_through_when_antecedent_is_pass_warn_fail() { + // Antecedent statuses that mean "feature present" (pass / warn / + // fail) leave the consequent row untouched. + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Pass), + make_raw("p2-schema-print", CheckStatus::Fail("missing".into())), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Fail("missing".into())), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Fail(_))); + } + + #[test] + fn propagation_collapses_consequent_when_antecedent_is_opt_out() { + // Antecedent OptOut → consequent becomes NotApplicable, regardless + // of what the consequent's own probe emitted. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::OptOut("no --output flag".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + match &rows[0].0.status { + CheckStatus::NotApplicable(reason) => { + assert!( + reason.contains("p2-json-output") && reason.contains("opt_out"), + "evidence should cite the antecedent + its status, got: {reason}", + ); + } + other => panic!("expected NotApplicable, got {other:?}"), + } + } + + #[test] + fn propagation_collapses_consequent_when_antecedent_is_n_a() { + // n_a antecedent (e.g., a chained conditional) propagates the same + // way as opt_out. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::NotApplicable("upstream n/a".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::NotApplicable(_))); + } + + #[test] + fn propagation_inherits_skip_from_antecedent() { + // Skip antecedent → consequent inherits Skip (couldn't measure + // upstream means can't meaningfully evaluate downstream). + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Skip("probe limit".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Skip(_))); + } + + #[test] + fn propagation_inherits_error_from_antecedent() { + let raw = vec![ + make_raw("p2-json-output", CheckStatus::Error("probe crashed".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Error(_))); + } + + #[test] + fn propagation_leaves_universal_rows_untouched() { + // A row with applicability: universal must not be touched by + // propagation even if a check with the same id exists in `raw`. + let raw = vec![make_raw("p1-non-interactive", CheckStatus::Pass)]; + let mut rows = vec![( + make_raw("p1-must-no-interactive", CheckStatus::Pass), + "p1-non-interactive".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!(matches!(rows[0].0.status, CheckStatus::Pass)); + } + + #[test] + fn score_pct_excludes_opt_out_and_n_a_from_denominator() { + // Transitional formula (U2): pass / (pass + warn + fail). opt_out + // and n_a do not count in either side of the ratio. A run of + // 1 Pass + 1 OptOut + 1 NotApplicable scores 100% (1/1), not 33%. + let results = vec![ + make_raw("c1", CheckStatus::Pass), + make_raw("c2", CheckStatus::OptOut("deliberate".into())), + make_raw("c3", CheckStatus::NotApplicable("conditional unmet".into())), + ]; + assert_eq!(score_pct(&results), 100); + + // Adding one Fail pulls the score down to 50%; opt_out / n_a still + // do not contribute to the denominator. + let mixed = vec![ + make_raw("c1", CheckStatus::Pass), + make_raw("c2", CheckStatus::Fail("violates".into())), + make_raw("c3", CheckStatus::OptOut("deliberate".into())), + make_raw("c4", CheckStatus::NotApplicable("conditional unmet".into())), + ]; + assert_eq!(score_pct(&mixed), 50); + } + + #[test] + fn summary_counts_seven_statuses_independently() { + // build_summary surfaces opt_out and n_a alongside the historical + // five counters; total covers all seven. + let results = vec![ + make_raw("a", CheckStatus::Pass), + make_raw("b", CheckStatus::Warn("w".into())), + make_raw("c", CheckStatus::Fail("f".into())), + make_raw("d", CheckStatus::OptOut("o".into())), + make_raw("e", CheckStatus::NotApplicable("n".into())), + make_raw("f", CheckStatus::Skip("s".into())), + make_raw("g", CheckStatus::Error("e".into())), + ]; + let s = build_summary(&results); + assert_eq!(s.total, 7); + assert_eq!(s.pass, 1); + assert_eq!(s.warn, 1); + assert_eq!(s.fail, 1); + assert_eq!(s.opt_out, 1); + assert_eq!(s.n_a, 1); + assert_eq!(s.skip, 1); + assert_eq!(s.error, 1); + } + + #[test] + fn check_result_view_carries_tier_and_check_id() { + // Per-row CheckResultView built via from_row exposes the requirement + // tier (looked up from the registry) and the originating probe. + let r = make_raw("p3-must-version", CheckStatus::Pass); + let view = CheckResultView::from_row(&r, "p3-version"); + assert_eq!(view.id, "p3-must-version"); + assert_eq!(view.check_id, "p3-version"); + assert_eq!(view.tier.as_deref(), Some("must")); + } + + #[test] + fn check_result_view_tier_is_null_for_unknown_row_id() { + // Test fixtures with synthetic ids that don't exist in the registry + // surface as JSON null for tier — visible signal of inconsistency. + let r = make_raw("not-a-real-row-id", CheckStatus::Pass); + let view = CheckResultView::from_row(&r, "some-check"); + assert!(view.tier.is_none(), "got: {:?}", view.tier); + } + + #[test] + fn opt_out_status_serializes_as_opt_out_in_json() { + let r = make_raw("c1", CheckStatus::OptOut("test reason".into())); + let view = CheckResultView::from_result(&r); + assert_eq!(view.status, "opt_out"); + assert_eq!(view.evidence.as_deref(), Some("test reason")); + } + + #[test] + fn n_a_status_serializes_as_n_a_in_json() { + let r = make_raw("c1", CheckStatus::NotApplicable("antecedent unmet".into())); + let view = CheckResultView::from_result(&r); + assert_eq!(view.status, "n_a"); + assert_eq!(view.evidence.as_deref(), Some("antecedent unmet")); + } } diff --git a/src/types.rs b/src/types.rs index 4a0bd21..0e3a1b8 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,13 @@ use serde::Serialize; /// The result of running a single check. +/// +/// The 7-status taxonomy splits the former `Skip` bucket into three distinct +/// outcomes so the scoring algorithm can tell "tool deliberately did not adopt +/// this" (`OptOut`) from "this check does not apply to this tool" +/// (`NotApplicable`) from "the linter could not measure" (`Skip`). See plan +/// `docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md` for +/// the taxonomy rationale and Decision 2a for antecedent propagation. #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "status", content = "evidence")] #[serde(rename_all = "snake_case")] @@ -8,6 +15,18 @@ pub enum CheckStatus { Pass, Warn(String), Fail(String), + /// Tool clearly has the capability surface but does not ship this feature + /// (deliberate non-adoption). Excluded from the numerator; whether it + /// counts in the denominator is the open formula choice deferred to U3. + OptOut(String), + /// Conditional antecedent unmet — the requirement does not apply to this + /// tool. Excluded from both numerator and denominator. Set either by a + /// verifier directly or by antecedent propagation in the scorecard module. + NotApplicable(String), + /// Linter probe limitation: the check could not measure. Excluded from + /// both numerator and denominator (preserved for backward compatibility; + /// pre-0.6 scorecards used this bucket for all of OptOut / NotApplicable + /// / Skip). Skip(String), Error(String), } @@ -53,6 +72,15 @@ pub enum CheckLayer { } /// A single check result with metadata. +/// +/// **Per-row emission (schema 0.6+).** A `Check::run()` produces one +/// probe-level `CheckResult` keyed by `id = check.id()`. The runner then +/// fans the probe out across every row in `Check::covers()`, producing one +/// scorecard row per requirement (with `id = row_id`). Antecedent +/// propagation runs after fan-out. The per-row result reuses this same +/// struct shape; `id` is then the requirement-row id, and the probe's +/// `check.id()` is recovered via the `check_id` field in +/// `scorecard::CheckResultView`. #[derive(Debug, Clone, Serialize)] pub struct CheckResult { pub id: String, diff --git a/tests/integration.rs b/tests/integration.rs index 656203f..14fa4df 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -745,7 +745,7 @@ fn test_audit_profile_echoed_in_json_output() { let json_str = String::from_utf8(output).expect("utf8 stdout"); let parsed: serde_json::Value = serde_json::from_str(&json_str).expect("valid JSON"); assert_eq!(parsed["audit_profile"], "human-tui"); - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); } #[test] @@ -935,17 +935,16 @@ fn test_scorecard_json_has_stable_top_level_keys() { ); // Fixed enumerations also pin against the renderer contract. - assert_eq!(obj["schema_version"], "0.5"); + assert_eq!(obj["schema_version"], "0.6"); } #[test] fn test_audit_profile_diagnostic_does_not_panic_on_self() { // Dogfood edge case from the plan: `diagnostic-only` suppresses - // p5-dry-run on the self-target. A regression that drops the - // suppression (check runs normally) would still exit with a valid - // code, so a stronger assertion is required: `p5-dry-run` must - // appear in `results[]` as `status: "skip"` with the structured - // suppression evidence. + // p5-dry-run on the self-target. Schema 0.6 emits one result per + // requirement-row, so the suppressed probe surfaces under its row id + // `p5-must-dry-run` (covered by the `p5-dry-run` check) with + // `check_id: "p5-dry-run"` for provenance. let assert = cmd() .args([ "audit", @@ -964,15 +963,15 @@ fn test_audit_profile_diagnostic_does_not_panic_on_self() { let results = parsed["results"].as_array().expect("results is array"); let p5 = results .iter() - .find(|r| r["id"] == "p5-dry-run") - .expect("p5-dry-run check should appear in results[]"); + .find(|r| r["id"] == "p5-must-dry-run" && r["check_id"] == "p5-dry-run") + .expect("p5-must-dry-run row (from p5-dry-run probe) should appear in results[]"); assert_eq!( p5["status"], "skip", "diagnostic-only must suppress p5-dry-run to Skip (got {p5})", ); let evidence = p5["evidence"] .as_str() - .expect("suppressed p5-dry-run carries evidence string"); + .expect("suppressed p5-must-dry-run carries evidence string"); assert!( evidence.contains("suppressed by audit_profile: diagnostic-only"), "expected suppression evidence prefix, got {evidence:?}", diff --git a/tests/scorecard_metadata_security.rs b/tests/scorecard_metadata_security.rs index 1805b59..9d7990f 100644 --- a/tests/scorecard_metadata_security.rs +++ b/tests/scorecard_metadata_security.rs @@ -141,7 +141,7 @@ fn hostile_binary_nonzero_version_exit_yields_null() { ); // The scorecard itself must still emit — version probe failure is not // a scoring failure. - assert_eq!(parsed["schema_version"], "0.5"); + assert_eq!(parsed["schema_version"], "0.6"); assert_eq!(parsed["target"]["kind"], "binary"); } diff --git a/tests/scorecard_schema_v05.rs b/tests/scorecard_schema_v05.rs index a9b5819..2aa9097 100644 --- a/tests/scorecard_schema_v05.rs +++ b/tests/scorecard_schema_v05.rs @@ -18,12 +18,14 @@ fn fixture_path(name: &str) -> String { format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) } -/// Assert every documented v0.5 key path resolves on the parsed JSON. The -/// segmented walk gives a precise failure message when a field is missing. +/// Assert every documented v0.5 key path resolves on the parsed JSON, plus +/// the v0.6 additions (per-row emission, `tier` + `check_id` on each +/// result, `opt_out` / `n_a` summary counters). The segmented walk gives a +/// precise failure message when a field is missing. fn assert_v05_shape(parsed: &Value) { assert_eq!( - parsed["schema_version"], "0.5", - "schema_version must be 0.5", + parsed["schema_version"], "0.6", + "schema_version must be 0.6 (per-row emission + 7-status taxonomy)", ); for path in [ @@ -54,6 +56,9 @@ fn assert_v05_shape(parsed: &Value) { "badge.scorecard_url", "badge.badge_url", "badge.convention_url", + // 0.6 additions — 7-status summary counters. + "summary.opt_out", + "summary.n_a", ] { let mut node = parsed; for segment in path.split('.') { @@ -63,6 +68,21 @@ fn assert_v05_shape(parsed: &Value) { } } + // 0.6: every result row carries `tier` and `check_id`. The shape is + // assertable as soon as `results[]` is non-empty. + if let Some(results) = parsed["results"].as_array() { + for (i, row) in results.iter().enumerate() { + assert!( + row.get("tier").is_some(), + "results[{i}] missing `tier` (schema 0.6): {row}", + ); + assert!( + row.get("check_id").is_some(), + "results[{i}] missing `check_id` (schema 0.6): {row}", + ); + } + } + // The convention URL is fixed and shared across every scored tool. A // regression that pointed it at a stale path would silently break the // pre-launch surface — pin it loudly here. From 5a496c8e73529ab8fc63043ea1c1a4e89e1177b0 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 17:01:03 -0500 Subject: [PATCH 4/6] feat(checks): emit opt_out for deliberate non-adoption + schema 0.6 artifacts Wires the two probe-level cases where the 7-status taxonomy's `opt_out` carries clear semantics, then regenerates the committed schema artifacts so the JSON Schema, coverage matrix, and live scorecard agree on the shape introduced in the previous commits on this branch. **Probe-level opt_out emissions.** - `p8-bundle-exists` (project layer): the "no top-level `AGENTS.md` / `SKILL.md` found" branch now returns `OptOut` instead of `Warn`. A bundle that exists but is malformed (missing YAML frontmatter or `name:` field) is a real SHOULD violation and stays `Warn`. The opt_out signal feeds antecedent propagation: every conditional row whose antecedent is `p8-bundle-exists` (`p8-must-bundle-install`, `p8-may-install-all`, `p8-may-bundle-update`) collapses to `n_a` automatically when the project ships no bundle. - `p2-json-output` (behavioral layer): the two "no `--output` / `--format` flag detected" branches (top-level help and per-subcommand probe) now return `OptOut`. The conditional rows `p2-must-schema-print` and `p2-should-schema-file` are antecedent-gated on `p2-json-output`, so a tool without structured output sees those rows propagate to `n_a` rather than counting twice against the score. The probe-level changes are deliberately scoped to the two cases the plan calls out by name; other probes that currently emit `Skip` for "feature absent" are left as-is for follow-up work (the propagation table handles them correctly through whatever conditional antecedent they sit under, when the antecedent gets a machine-readable shape via `sync-spec`). **Regenerated artifacts.** - `schema/scorecard.schema.json` bumps `$id` to `https://anc.dev/scorecard-v0.6.schema.json`. The `CheckResultView` definition adds `tier` (enum of `must` / `should` / `may` / `null`) and `check_id`; the `Summary` definition adds `opt_out` and `n_a` counters; the `status` enum gains `opt_out` and `n_a` values; the example block updates to a per-row row with `tier` + `check_id` populated. The Summary's `required` list now names all seven counters. - `coverage/matrix.json` and `docs/coverage-matrix.md` regenerate against the conditional-applicability schema landed in the spec-scaffolding commit. The five conditional rows in p2 and p8 surface with `applicability.kind: conditional` and `applicability.antecedent.check_id` populated; legacy rows continue to emit `applicability.condition: ""` until each prerequisite grows a check id in the verifier catalog. Dogfood verification against `--command echo` (a CLI with no `--output` flag and no project context) produces a scorecard with `summary.opt_out: 1` for the `p2-must-output-flag` row (direct emission) and `summary.n_a: 1` for the `p2-must-schema-print` row (antecedent-propagated). Against the agentnative-cli repo itself (which has both `--output` and `AGENTS.md`), nothing routes through opt_out or n_a; the score stays at the v0.4.0 dogfood baseline. --- coverage/matrix.json | 20 +++++++++---- docs/coverage-matrix.md | 10 +++---- schema/scorecard.schema.json | 43 ++++++++++++++++++---------- src/checks/behavioral/json_output.rs | 22 ++++++++++---- src/checks/project/bundle_exists.rs | 17 +++++++---- 5 files changed, 77 insertions(+), 35 deletions(-) diff --git a/coverage/matrix.json b/coverage/matrix.json index c576182..cdc8d53 100644 --- a/coverage/matrix.json +++ b/coverage/matrix.json @@ -192,7 +192,9 @@ "summary": "CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier.", "applicability": { "kind": "conditional", - "condition": "CLI emits structured output" + "antecedent": { + "check_id": "p2-json-output" + } }, "verifiers": [ { @@ -223,7 +225,9 @@ "summary": "Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool.", "applicability": { "kind": "conditional", - "condition": "CLI emits structured output" + "antecedent": { + "check_id": "p2-json-output" + } }, "verifiers": [ { @@ -885,7 +889,9 @@ "summary": "When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { @@ -920,7 +926,9 @@ "summary": "An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { @@ -936,7 +944,9 @@ "summary": "An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version.", "applicability": { "kind": "conditional", - "condition": "CLI ships an agent skill bundle" + "antecedent": { + "check_id": "p8-bundle-exists" + } }, "verifiers": [ { diff --git a/docs/coverage-matrix.md b/docs/coverage-matrix.md index b096e6d..cf0abdb 100644 --- a/docs/coverage-matrix.md +++ b/docs/coverage-matrix.md @@ -33,9 +33,9 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | `p2-must-stdout-stderr-split` | MUST | Universal | `p2-output-module` (source) | Data goes to stdout; diagnostics/progress/warnings go to stderr, never interleaved. | | `p2-must-exit-codes` | MUST | Universal | `p2-structured-exit-codes` (behavioral) | Exit codes are structured and documented (0 success, 1 general, 2 usage, 77 auth, 78 config). | | `p2-must-json-errors` | MUST | Universal | `p2-json-errors` (behavioral) | When `--output json` is active, errors are emitted as JSON (to stderr) with at least `error`, `kind`, and `message` fields. | -| `p2-must-schema-print` | MUST | If: CLI emits structured output | `p2-schema-print` (behavioral) | CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier. | +| `p2-must-schema-print` | MUST | If: `p2-json-output` is present | `p2-schema-print` (behavioral) | CLIs that emit structured output expose the output schema via a `schema` subcommand or `--schema` flag: runtime-discoverable, with a documented format identifier. | | `p2-should-consistent-envelope` | SHOULD | Universal | `p2-consistent-envelope` (behavioral) | JSON output uses a consistent envelope (a top-level object with predictable keys) across every command. | -| `p2-should-schema-file` | SHOULD | If: CLI emits structured output | `p2-schema-file` (project) | Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool. | +| `p2-should-schema-file` | SHOULD | If: `p2-json-output` is present | `p2-schema-file` (project) | Output schemas are also exported to a stable file path (e.g., `schema/.json`) so CI/static-analysis consumers pin without invoking the tool. | | `p2-should-json-aliases` | SHOULD | Universal | `p2-json-aliases` (behavioral) | `--json` and `--jsonl` are accepted as aliases for `--output json` and `--output jsonl`; the short forms work alongside the canonical enum. | | `p2-may-more-formats` | MAY | Universal | `p2-more-formats` (behavioral) | Additional output formats (CSV, TSV, YAML) beyond the core three. | | `p2-may-raw-flag` | MAY | Universal | `p2-raw-flag` (behavioral) | `--raw` flag for unformatted output suitable for piping to other tools. | @@ -107,8 +107,8 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | ID | Level | Applicability | Verifier(s) | Summary | | --- | --- | --- | --- | --- | -| `p8-must-bundle-install` | MUST | If: CLI ships an agent skill bundle | `p8-bundle-install` (behavioral) | When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes. | +| `p8-must-bundle-install` | MUST | If: `p8-bundle-exists` is present | `p8-bundle-install` (behavioral) | When a skill bundle exists, the CLI provides an install path (`tool skill install []`) that registers the bundle with installed agent runtimes. | | `p8-should-bundle-exists` | SHOULD | Universal | `p6-agents-md` (project)
`p8-bundle-exists` (project) | CLIs ship a top-level agent-discoverable markdown bundle (`AGENTS.md`, `SKILL.md`, or equivalent) with YAML frontmatter naming the tool and capability summary. | -| `p8-may-install-all` | MAY | If: CLI ships an agent skill bundle | `p8-install-all` (behavioral) | An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all. | -| `p8-may-bundle-update` | MAY | If: CLI ships an agent skill bundle | `p8-bundle-update` (behavioral) | An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version. | +| `p8-may-install-all` | MAY | If: `p8-bundle-exists` is present | `p8-install-all` (behavioral) | An `--all` mode auto-detects installed runtimes (Claude Code, Cursor, Codex, OpenCode, etc.) and installs across all. | +| `p8-may-bundle-update` | MAY | If: `p8-bundle-exists` is present | `p8-bundle-update` (behavioral) | An update/upgrade subcommand (`tool skill update`) pulls the latest bundle version. | diff --git a/schema/scorecard.schema.json b/schema/scorecard.schema.json index 75a6e0f..6ca9df1 100644 --- a/schema/scorecard.schema.json +++ b/schema/scorecard.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://anc.dev/scorecard-v0.5.schema.json", + "$id": "https://anc.dev/scorecard-v0.6.schema.json", "title": "agentnative scorecard", - "description": "JSON Schema for `anc audit --output json` scorecards (schema version 0.5). Hand-written for v0.4.0. A schemars-derived version generated from src/scorecard/mod.rs is planned for a follow-up release; see docs/plans/2026-04-30-002-feat-scorecard-json-schema-plan.md on the `dev` branch.", + "description": "JSON Schema for `anc audit --output json` scorecards (schema version 0.6). Schema 0.6 introduces the 7-status taxonomy (`opt_out` and `n_a` added to `status`, with matching counters in `summary`), per-row emission (one result per requirement-row instead of per check_id; `tier` and `check_id` fields added to each result), and antecedent propagation for conditional requirements. See docs/plans/2026-05-21-001-feat-scorecard-fairness-taxonomy-plan.md in the agentnative-site repo for the full taxonomy rationale.", "type": "object", "required": [ "schema_version", @@ -23,11 +23,11 @@ "schema_version": { "type": "string", "description": "Scorecard schema version. Pre-launch additive — consumers feature-detect new fields rather than pin to an exact value.", - "examples": ["0.5"] + "examples": ["0.6"] }, "results": { "type": "array", - "description": "Per-check outcomes. One entry per check the runner produced.", + "description": "Per-requirement-row outcomes (schema 0.6+). One entry per requirement row covered by a check that ran in this invocation; a single probe whose `Check::covers()` lists multiple rows produces multiple entries (each entry's `check_id` carries the probe's id for provenance).", "items": { "$ref": "#/$defs/CheckResultView" } }, "summary": { "$ref": "#/$defs/Summary" }, @@ -61,12 +61,12 @@ "$defs": { "CheckResultView": { "type": "object", - "required": ["id", "label", "group", "layer", "status", "evidence", "confidence"], + "required": ["id", "label", "group", "layer", "status", "evidence", "confidence", "tier", "check_id"], "additionalProperties": false, "properties": { "id": { "type": "string", - "description": "Stable check identifier (e.g., `p1-non-interactive`). The site's `/score/` page and external leaderboards pin on these strings." + "description": "Requirement-row identifier (schema 0.6+; matches `coverage/matrix.json` row ids, e.g. `p1-must-no-interactive`). The site's `/score/` page and external leaderboards pin on these strings. Previously (≤0.5) this carried the probe id." }, "label": { "type": "string", @@ -84,30 +84,41 @@ }, "status": { "type": "string", - "description": "Outcome of the check. Skips and Errors are excluded from badge scoring.", - "enum": ["pass", "warn", "fail", "skip", "error"] + "description": "Outcome of the check under the 7-status taxonomy (schema 0.6+). `opt_out` = deliberate non-adoption; `n_a` = conditional antecedent unmet (set by antecedent propagation); `skip` = linter probe limitation. Skips, errors, opt_outs, and n_as are all excluded from the transitional badge denominator.", + "enum": ["pass", "warn", "fail", "opt_out", "n_a", "skip", "error"] }, "evidence": { "type": ["string", "null"], - "description": "Structured evidence for non-Pass statuses: the suppression-table marker, the unrecognized-flag list, the missing-file path. `null` for clean Pass." + "description": "Structured evidence for non-Pass statuses: the suppression-table marker, the unrecognized-flag list, the missing-file path, the antecedent propagation reason. `null` for clean Pass." }, "confidence": { "type": "string", "description": "How directly the check verifies its requirement. `high` for direct probes; `medium` for heuristics. Older consumers feature-detect.", "enum": ["high", "medium"] + }, + "tier": { + "type": ["string", "null"], + "description": "Requirement-row RFC 2119 level (schema 0.6+). `null` only when the row id is not in the registry — an internal inconsistency that should be loud.", + "enum": ["must", "should", "may", null] + }, + "check_id": { + "type": "string", + "description": "Probe that produced this row (schema 0.6+). One probe may produce multiple rows when its `Check::covers()` slice names multiple requirements (e.g. `p3-version` covers both `p3-must-version` and `p3-should-version-short`)." } } }, "Summary": { "type": "object", - "description": "Run-level outcome counts.", - "required": ["total", "pass", "warn", "fail", "skip", "error"], + "description": "Run-level outcome counts. Schema 0.6+ adds `opt_out` and `n_a` counters alongside the historic five.", + "required": ["total", "pass", "warn", "fail", "opt_out", "n_a", "skip", "error"], "additionalProperties": false, "properties": { "total": { "type": "integer", "minimum": 0 }, "pass": { "type": "integer", "minimum": 0 }, "warn": { "type": "integer", "minimum": 0 }, "fail": { "type": "integer", "minimum": 0 }, + "opt_out": { "type": "integer", "minimum": 0 }, + "n_a": { "type": "integer", "minimum": 0 }, "skip": { "type": "integer", "minimum": 0 }, "error": { "type": "integer", "minimum": 0 } } @@ -203,19 +214,21 @@ }, "examples": [ { - "schema_version": "0.5", + "schema_version": "0.6", "results": [ { - "id": "p1-non-interactive", + "id": "p1-must-no-interactive", "label": "Non-interactive by default", "group": "P1", "layer": "behavioral", "status": "pass", "evidence": null, - "confidence": "high" + "confidence": "high", + "tier": "must", + "check_id": "p1-non-interactive" } ], - "summary": { "total": 1, "pass": 1, "warn": 0, "fail": 0, "skip": 0, "error": 0 }, + "summary": { "total": 1, "pass": 1, "warn": 0, "fail": 0, "opt_out": 0, "n_a": 0, "skip": 0, "error": 0 }, "coverage_summary": { "must": { "total": 23, "verified": 17 }, "should": { "total": 14, "verified": 9 }, diff --git a/src/checks/behavioral/json_output.rs b/src/checks/behavioral/json_output.rs index 998e72b..3cce339 100644 --- a/src/checks/behavioral/json_output.rs +++ b/src/checks/behavioral/json_output.rs @@ -71,7 +71,12 @@ impl Check for JsonOutputCheck { fn probe_subcommands(runner: &BinaryRunner, help_output: &str) -> CheckStatus { let subcommands = parse_subcommand_names(help_output); if subcommands.is_empty() { - return CheckStatus::Skip("no --output/--format flag detected".into()); + return CheckStatus::OptOut( + "no --output/--format flag detected — tool does not ship structured output. \ + Schema-discovery requirements (p2-must-schema-print, p2-should-schema-file) \ + collapse to n/a via antecedent propagation." + .into(), + ); } for subcmd in &subcommands { @@ -90,7 +95,11 @@ fn probe_subcommands(runner: &BinaryRunner, help_output: &str) -> CheckStatus { } } - CheckStatus::Skip("no --output/--format flag detected in any subcommand".into()) + CheckStatus::OptOut( + "no --output/--format flag detected in any subcommand — tool does not ship \ + structured output." + .into(), + ) } /// Extract subcommand names from CLI --help output. @@ -293,12 +302,15 @@ esac } #[test] - fn json_output_skip_no_flag() { + fn json_output_opt_out_no_flag() { + // Schema 0.6: "no flag at all" is opt_out (deliberate non-adoption), + // not Skip (probe limitation). Distinguishes "tool doesn't ship + // structured output" from "we couldn't measure it". let project = test_project_with_sh_script("echo 'just some help text'"); let result = JsonOutputCheck.run(&project).expect("check should run"); match &result.status { - CheckStatus::Skip(msg) => assert!(msg.contains("no --output")), - other => panic!("expected Skip, got {other:?}"), + CheckStatus::OptOut(msg) => assert!(msg.contains("no --output")), + other => panic!("expected OptOut, got {other:?}"), } } diff --git a/src/checks/project/bundle_exists.rs b/src/checks/project/bundle_exists.rs index 0cb44e9..b76ee42 100644 --- a/src/checks/project/bundle_exists.rs +++ b/src/checks/project/bundle_exists.rs @@ -77,10 +77,17 @@ pub(crate) fn find_bundle(root: &Path) -> Option { None } -/// Core unit. SHOULD-tier: every miss is Warn, never Fail. +/// Core unit. SHOULD-tier: every miss is Warn, never Fail. The "no bundle +/// at all" case emits `OptOut` so antecedent propagation (Decision 2a) can +/// collapse the conditional MUSTs/MAYs that depend on `p8-bundle-exists` +/// (`p8-must-bundle-install`, `p8-may-install-all`, `p8-may-bundle-update`) +/// to `n_a` rather than dragging them through their own probe and emitting +/// double-penalty Skips. A bundle that exists but is malformed (missing +/// frontmatter or `name:` field) is a real SHOULD violation and stays +/// `Warn` — the feature is present, just incomplete. pub(crate) fn check_bundle_exists(root: &Path) -> CheckStatus { let Some(path) = find_bundle(root) else { - return CheckStatus::Warn( + return CheckStatus::OptOut( "no top-level AGENTS.md or SKILL.md found. Agents discover \ skill bundles via filesystem convention; ship one with YAML \ frontmatter naming the tool." @@ -179,12 +186,12 @@ mod tests { } #[test] - fn warn_no_bundle() { + fn opt_out_no_bundle() { let dir = temp_dir("nobundle"); fs::write(dir.join("README.md"), "# Tool\n").expect("write"); match check_bundle_exists(&dir) { - CheckStatus::Warn(msg) => assert!(msg.contains("AGENTS.md")), - other => panic!("expected Warn, got {other:?}"), + CheckStatus::OptOut(msg) => assert!(msg.contains("AGENTS.md")), + other => panic!("expected OptOut, got {other:?}"), } } From 12c1d322546e64135658e712adecec1f3d4621c5 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 17:14:42 -0500 Subject: [PATCH 5/6] test(u2): red team across parser, propagation, score, and schema drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty-two adversarial tests across the four U2 surfaces, plus three parser strictness fixes the red team surfaced. **Parser strictness fixes (`build_support/parser.rs`).** Three error modes that the original v1 parser accepted silently now hard-fail with a hint at the schema the author probably meant: - `antecedent.check_id` is now trimmed before the non-empty check (whitespace-only strings were previously accepted as a valid id). - Mixed legacy `if:` and new `kind:` in the same applicability block now errors. The legacy branch only fired for `map.len() == 1`, so the original parser silently dropped the `if:` prose when both keys were present, which would footgun any author mid-migration. - Extra keys inside `antecedent` (beyond `check_id`) now error by name. Compound antecedents (`op: any_of | all_of`) are explicitly deferred to a future schema bump per plan Sub-decision 2b; silently ignoring v2 keys on a v1 row would let v2 syntax ship under v1 semantics. **Parser red team (`tests/build_parser.rs`, +8 tests).** Whitespace-only `check_id`, `antecedent:` as a string, `antecedent:` as a list (the v2 compound shape), `kind: Conditional` with the wrong case, mixed legacy + new shape in one row, antecedent with `op:` smuggled alongside `check_id`, `antecedent: null`, empty antecedent mapping. Plus an `emit_rust` defense-in-depth test asserting that hostile characters in a `check_id` (quotes, backslashes) escape correctly in the generated Rust source. **Registry consistency red team (`src/principles/registry.rs`, +2 tests).** Walks `REQUIREMENTS` and asserts every conditional row's `antecedent.check_id` resolves to a real check in `all_checks_catalog()`, because a typo or rename would silently mute propagation in production. A second test asserts no conditional row names its own covering check as its antecedent (the "self-gate" edge case where a row would gate its status against itself). **Propagation red team (`src/scorecard/mod.rs`, +5 tests).** Idempotency (a second pass produces identical output), no-op when the antecedent didn't run (source-only or filtered run), `--audit-profile` suppression of the antecedent propagates as Skip with the suppression reason preserved, full pipeline end-to-end (fan-out → propagation → summary → score with opt_out + n_a counted separately). **Score formula red team (`src/scorecard/mod.rs`, +4 tests).** Score is 0 with no panic when every row is opt_out or n_a (division-by-zero guard). One pass amid 999 n_a rows is 100% (n_a excluded from both numerator and denominator). Skip + Error continue to be excluded under the new taxonomy. `summary.total` equals the sum of all seven per-status counters (catches a future status variant added without updating `build_summary`). **Serialization red team (`src/scorecard/mod.rs`, +2 tests).** Evidence strings containing quotes, backslashes, control chars (`\n`, `\t`, `\u{0007}`) and Unicode (zero-width joiner, RTL override) round-trip through serde → parse → assert without loss or escape corruption. Sanitization belongs at the render layer, not here; the scorecard's job is faithful pass-through. **Schema drift red team (`tests/scorecard_schema_v05.rs`, +6 tests).** The committed `schema/scorecard.schema.json` is the consumer contract for the site renderer and third-party leaderboards. Tests assert: `$id` pins to the current `SCHEMA_VERSION` (0.6); the `status` enum lists all seven taxonomy values; `Summary.required` includes `opt_out` + `n_a`; `CheckResultView` includes `tier` (with the three RFC 2119 levels plus null) + `check_id`; the schema's own `examples[0]` block satisfies its own `required` lists (catches doc-vs-schema drift); a live scorecard from a real `anc audit` run carries every key in the top-level and per-row `required` lists. The single supporting production change beyond the parser strictness is `#[derive(Debug)]` on `Summary` so red-team assertions can surface counter mismatches inline rather than panicking with a useless message. --- build_support/parser.rs | 35 ++++- src/principles/registry.rs | 81 +++++++++++ src/scorecard/mod.rs | 267 +++++++++++++++++++++++++++++++++- tests/build_parser.rs | 197 +++++++++++++++++++++++++ tests/scorecard_schema_v05.rs | 215 +++++++++++++++++++++++++++ 5 files changed, 792 insertions(+), 3 deletions(-) diff --git a/build_support/parser.rs b/build_support/parser.rs index 27e6475..a108820 100644 --- a/build_support/parser.rs +++ b/build_support/parser.rs @@ -436,6 +436,19 @@ fn parse_applicability( let if_key = serde_yaml::Value::String("if".into()); let kind_key = serde_yaml::Value::String("kind".into()); + // Mixing the legacy `if:` and new `kind:` shape in one applicability + // block is ambiguous: the legacy branch only fires when `if:` is the + // single key, so a row carrying both would silently use the new shape + // and drop the legacy prose. Reject early with a pointer at the + // schema the author probably meant to use. + if map.contains_key(&if_key) && map.contains_key(&kind_key) { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: "`if:` and `kind:` cannot both be set; choose one (legacy `{ if: \"\" }` or new `{ kind: conditional, antecedent: { check_id: ... } }`)".into(), + }); + } + // Legacy single-key `{ if: "" }` shape. if map.len() == 1 && let Some(if_val) = map.get(&if_key) @@ -493,16 +506,34 @@ fn parse_applicability( .into(), })?; let check_id_key = serde_yaml::Value::String("check_id".into()); + // v1 schema is strict: only `check_id` is permitted inside + // `antecedent`. Compound antecedents (`op: any_of | all_of`) + // and any other key are explicitly deferred to a future + // schema bump per plan Sub-decision 2b. Silently ignoring + // unknown keys would let v2 syntax land in v1 vendored spec + // and behave subtly wrong. + for (k, _) in ante_map { + let k_str = k.as_str().unwrap_or(""); + if k_str != "check_id" { + return Err(ParseError::UnknownApplicability { + file: file.to_string(), + requirement_id: req_id.to_string(), + hint: format!( + "`antecedent.{k_str}` is not part of the v1 schema (only `check_id` is permitted; compound antecedents deferred to v2 per plan Sub-decision 2b)" + ), + }); + } + } let check_id = ante_map.get(&check_id_key).and_then(|v| v.as_str()).ok_or_else(|| ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), hint: "`antecedent.check_id` must be a non-empty string (v1 schema supports a single antecedent only)".into(), })?; - if check_id.is_empty() { + if check_id.trim().is_empty() { return Err(ParseError::UnknownApplicability { file: file.to_string(), requirement_id: req_id.to_string(), - hint: "`antecedent.check_id` must be a non-empty string".into(), + hint: "`antecedent.check_id` must be a non-empty string (whitespace-only is rejected)".into(), }); } Some(Antecedent { diff --git a/src/principles/registry.rs b/src/principles/registry.rs index 11f765a..395b211 100644 --- a/src/principles/registry.rs +++ b/src/principles/registry.rs @@ -464,4 +464,85 @@ mod tests { } } } + + // ────────────────────────────────────────────────────────────────── + // U2 (schema 0.6): conditional applicability red-team guards. + // Each conditional row in the registry names an antecedent `check_id` + // that drives propagation. A typo or rename in the antecedent would + // silently mute propagation in production — the consequent row would + // forever look up `None` and pass through with its own probe status. + // The asserts below pin the contract loudly. + // ────────────────────────────────────────────────────────────────── + + #[test] + fn every_conditional_antecedent_resolves_to_a_real_check() { + use crate::check::Check; + use crate::checks::all_checks_catalog; + + let catalog: Vec> = all_checks_catalog(); + let catalog_ids: Vec<&str> = catalog.iter().map(|c| c.id()).collect(); + + let mut dangling: Vec<(&str, &str)> = Vec::new(); + for req in REQUIREMENTS { + if let Applicability::Conditional { + antecedent: Some(ante), + .. + } = req.applicability + && !catalog_ids.contains(&ante.check_id) + { + dangling.push((req.id, ante.check_id)); + } + } + assert!( + dangling.is_empty(), + "conditional requirements with dangling antecedent check_ids:\n{}\n\ + Fix the spec's `antecedent.check_id` or add the missing check to the catalog.", + dangling + .iter() + .map(|(req, ante)| format!( + " - row `{req}` → antecedent `{ante}` (not in catalog)" + )) + .collect::>() + .join("\n"), + ); + } + + #[test] + fn no_conditional_row_names_itself_as_antecedent() { + // Edge case: a conditional row's covering check is the same as its + // antecedent. The propagation table would then read the row's own + // probe status and could collapse the row to n_a based on itself — + // a logic loop that's never the right model. The spec should never + // produce this shape; this test catches it if it does. + use crate::check::Check; + use crate::checks::all_checks_catalog; + + let catalog: Vec> = all_checks_catalog(); + let mut covers_by_check: std::collections::HashMap<&'static str, &'static [&'static str]> = + std::collections::HashMap::new(); + for c in &catalog { + covers_by_check.insert(Box::leak(c.id().to_string().into_boxed_str()), c.covers()); + } + + for req in REQUIREMENTS { + let Applicability::Conditional { + antecedent: Some(ante), + .. + } = req.applicability + else { + continue; + }; + if let Some(covers) = covers_by_check.get(ante.check_id) { + assert!( + !covers.contains(&req.id), + "conditional row `{}` declares antecedent `{}`, but that \ + check already covers `{}` directly — the row would gate \ + its own status against itself.", + req.id, + ante.check_id, + req.id, + ); + } + } + } } diff --git a/src/scorecard/mod.rs b/src/scorecard/mod.rs index a46b3b1..e0e2456 100644 --- a/src/scorecard/mod.rs +++ b/src/scorecard/mod.rs @@ -312,7 +312,7 @@ pub struct CoverageSummary { /// Run-level outcome counts. The 7-status taxonomy added `opt_out` and /// `n_a` in schema 0.6; pre-0.6 consumers tolerate the new keys (additive /// extension). -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct Summary { pub total: usize, pub pass: usize, @@ -2031,4 +2031,269 @@ mod tests { assert_eq!(view.status, "n_a"); assert_eq!(view.evidence.as_deref(), Some("antecedent unmet")); } + + // ────────────────────────────────────────────────────────────────── + // U2 red team: adversarial inputs that try to break the per-row + + // propagation pipeline or the score formula. + // ────────────────────────────────────────────────────────────────── + + #[test] + fn rt_propagation_is_idempotent() { + // Propagation reads raw probe statuses (not row statuses), so a + // second pass over an already-propagated row vector must produce + // an identical result. Pins the no-feedback contract — a future + // refactor that reads `rows` in place would break this and + // potentially loop or oscillate on chained conditionals. + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::OptOut("no --output flag".into()), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + let after_first: Vec<(String, String)> = rows + .iter() + .map(|(r, c)| { + ( + serde_json::to_string(&r.status).expect("status serializes"), + c.clone(), + ) + }) + .collect(); + propagate_antecedents(&mut rows, &raw); + let after_second: Vec<(String, String)> = rows + .iter() + .map(|(r, c)| { + ( + serde_json::to_string(&r.status).expect("status serializes"), + c.clone(), + ) + }) + .collect(); + assert_eq!(after_first, after_second, "propagation must be idempotent"); + } + + #[test] + fn rt_propagation_no_op_when_antecedent_did_not_run() { + // Source-only or filtered run: the antecedent probe didn't produce + // a raw result. The row keeps its own status — propagation can't + // override what it can't read. This is the exact path tools using + // `--source` or `--principle ` exercise in production. + let raw = vec![make_raw("p2-schema-print", CheckStatus::Pass)]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + assert!( + matches!(rows[0].0.status, CheckStatus::Pass), + "no antecedent in raw → row untouched, got: {:?}", + rows[0].0.status, + ); + } + + #[test] + fn rt_propagation_inherits_audit_profile_suppression_as_skip() { + // Adversarial case: a CLI runs with `--audit-profile ` that + // suppresses the antecedent probe. The suppressed Skip carries the + // SUPPRESSION_EVIDENCE_PREFIX sentinel. The consequent row should + // inherit Skip (cannot meaningfully evaluate). The new evidence + // string cites the antecedent so a reader can still trace the + // root cause back to the audit profile. + use crate::principles::registry::SUPPRESSION_EVIDENCE_PREFIX; + let raw = vec![ + make_raw( + "p2-json-output", + CheckStatus::Skip(format!("{SUPPRESSION_EVIDENCE_PREFIX}human-tui")), + ), + make_raw("p2-schema-print", CheckStatus::Pass), + ]; + let mut rows = vec![( + make_raw("p2-must-schema-print", CheckStatus::Pass), + "p2-schema-print".to_string(), + )]; + propagate_antecedents(&mut rows, &raw); + match &rows[0].0.status { + CheckStatus::Skip(reason) => { + assert!( + reason.contains("p2-json-output"), + "propagated Skip must cite the antecedent, got: {reason}", + ); + assert!( + reason.contains("human-tui"), + "propagated Skip must preserve the suppression reason, got: {reason}", + ); + } + other => panic!("expected Skip, got {other:?}"), + } + } + + #[test] + fn rt_score_pct_only_n_a_returns_zero_without_panic() { + // Pathological: every result is NotApplicable. Denominator is zero; + // score must surface as 0 with no division-by-zero or NaN. + let results: Vec = (0..100) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::NotApplicable("conditional unmet".into()), + ) + }) + .collect(); + assert_eq!(score_pct(&results), 0); + } + + #[test] + fn rt_score_pct_only_opt_out_returns_zero_without_panic() { + // Mirror of the n_a case: opt_out also excluded from denominator. + let results: Vec = (0..50) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::OptOut("deliberate".into()), + ) + }) + .collect(); + assert_eq!(score_pct(&results), 0); + } + + #[test] + fn rt_score_pct_one_pass_amid_999_n_a_returns_100() { + // n_a must not dilute. One genuine pass against a thousand + // inapplicable rows is still 100%. + let mut results: Vec = (0..999) + .map(|i| { + make_raw( + &format!("row-{i}"), + CheckStatus::NotApplicable("conditional unmet".into()), + ) + }) + .collect(); + results.push(make_raw("row-last", CheckStatus::Pass)); + assert_eq!(score_pct(&results), 100); + } + + #[test] + fn rt_score_pct_skip_and_error_still_excluded() { + // Carry the legacy contract forward: Skip and Error contribute to + // neither side. A run of (1 Pass + 100 Skip + 100 Error) is 100%. + let mut results = vec![make_raw("good", CheckStatus::Pass)]; + for i in 0..100 { + results.push(make_raw( + &format!("s-{i}"), + CheckStatus::Skip("limit".into()), + )); + results.push(make_raw( + &format!("e-{i}"), + CheckStatus::Error("boom".into()), + )); + } + assert_eq!(score_pct(&results), 100); + } + + #[test] + fn rt_evidence_with_control_chars_roundtrips_through_json() { + // Evidence strings come from probe output and may contain quotes, + // backslashes, newlines, tabs. serde_json must escape them. The + // roundtrip parse must recover the exact byte sequence. + let hostile: &str = "line1\nline2\t\"quoted\"\\backslash\u{0007}bell"; + let r = make_raw("c1", CheckStatus::Warn(hostile.to_string())); + let view = CheckResultView::from_result(&r); + let json = serde_json::to_string(&view).expect("view serializes"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("re-parses"); + assert_eq!( + parsed["evidence"].as_str(), + Some(hostile), + "evidence must roundtrip through JSON without loss", + ); + } + + #[test] + fn rt_evidence_with_unicode_zero_width_and_rtl_roundtrips() { + // Zero-width joiner and RTL override are common smuggling vectors + // in display contexts. They must roundtrip through JSON unchanged; + // any sanitization belongs at the render layer (site), not here. + let hostile = "left\u{202e}right\u{200b}invisible"; + let r = make_raw("c1", CheckStatus::OptOut(hostile.to_string())); + let view = CheckResultView::from_result(&r); + let json = serde_json::to_string(&view).expect("view serializes"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("re-parses"); + assert_eq!(parsed["evidence"].as_str(), Some(hostile)); + assert_eq!(parsed["status"], "opt_out"); + } + + #[test] + fn rt_summary_total_equals_sum_of_per_status_counts() { + // Invariant: total == pass + warn + fail + opt_out + n_a + skip + error. + // A new variant added without updating build_summary would break this. + let statuses = vec![ + CheckStatus::Pass, + CheckStatus::Pass, + CheckStatus::Warn("w".into()), + CheckStatus::Fail("f".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::OptOut("o".into()), + CheckStatus::NotApplicable("n".into()), + CheckStatus::Skip("s".into()), + CheckStatus::Error("e".into()), + ]; + let results: Vec = statuses + .into_iter() + .enumerate() + .map(|(i, s)| make_raw(&format!("c{i}"), s)) + .collect(); + let s = build_summary(&results); + assert_eq!( + s.total, + s.pass + s.warn + s.fail + s.opt_out + s.n_a + s.skip + s.error, + "summary.total must equal the sum of every per-status counter", + ); + } + + #[test] + fn rt_full_pipeline_n_a_excluded_from_summary_n_a_and_score() { + // End-to-end: a probe emits OptOut for the antecedent. After + // fan-out + propagation, the consequent row carries n_a. The + // summary counts both. The score reflects the non-conditional + // pass-rate, untouched by the opt_out / n_a pair. + let raw = vec![ + make_raw("p2-json-output", CheckStatus::OptOut("no flag".into())), + make_raw("p2-schema-print", CheckStatus::Pass), + make_raw("p1-non-interactive", CheckStatus::Pass), + ]; + let catalog: Vec> = vec![ + Box::new(FakeCheck { + id: "p2-json-output", + covers: &["p2-must-output-flag"], + }), + Box::new(FakeCheck { + id: "p2-schema-print", + covers: &["p2-must-schema-print"], + }), + Box::new(FakeCheck { + id: "p1-non-interactive", + covers: &["p1-must-no-interactive"], + }), + ]; + let mut rows = fan_out_per_row(&raw, &catalog); + propagate_antecedents(&mut rows, &raw); + let per_row: Vec = rows.into_iter().map(|(r, _)| r).collect(); + + let s = build_summary(&per_row); + assert_eq!(s.opt_out, 1, "p2-must-output-flag → opt_out: got {s:?}"); + assert_eq!( + s.n_a, 1, + "p2-must-schema-print → n_a via propagation: got {s:?}", + ); + // Score: 2 passes (p2-must-output-flag is opt_out, p2-must-schema-print + // is n_a; only 2 pass rows remain). Denominator = 2 (both passes), so + // 100%. The opt_out + n_a do not pull the score down. + assert_eq!(score_pct(&per_row), 100); + } } diff --git a/tests/build_parser.rs b/tests/build_parser.rs index 6403cc6..392a375 100644 --- a/tests/build_parser.rs +++ b/tests/build_parser.rs @@ -475,3 +475,200 @@ fn vendored_spec_parses_to_expected_requirement_count() { // p8-discoverable-skill-bundle.md). assert_eq!(combined.last().unwrap().id, "p8-may-bundle-update"); } + +// ───────────────────────────────────────────────────────────────────────── +// Red team: adversarial YAML inputs that try to slip past the parser. +// Every entry below tests a footgun a future spec author could fall into, +// or a subtle malformed value that the parser must surface loudly. +// ───────────────────────────────────────────────────────────────────────── + +const RT_FIXTURE_HEAD: &str = r#"--- +id: p2 +title: Red team +last-revised: 2026-01-01 +status: draft +requirements: +"#; + +fn rt_assert_error_mentions(content: &str, expected_substrings: &[&str]) { + // Append the closing frontmatter fence so each fixture is a complete + // markdown file. The body is intentionally empty — every fixture + // exercises the requirement-parsing path, not body content. + let full = format!("{content}---\n"); + let err = parse_principle_file("p2-rt.md", &full).unwrap_err(); + let msg = format!("{err}"); + for s in expected_substrings { + assert!( + msg.contains(s), + "error message must contain {s:?}, got: {msg}" + ); + } +} + +#[test] +fn rt_rejects_whitespace_only_check_id() { + // Subtle footgun — `is_empty` catches "" but not " ". The parser must + // trim before deciding "non-empty". + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: ' '\n\ + \x20\x20\x20\x20summary: bad check_id\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "check_id", "non-empty"]); +} + +#[test] +fn rt_rejects_antecedent_as_string() { + // YAML author wrote `antecedent: p2-json-output` instead of the mapping + // form. The parser must surface this with a hint at the expected shape. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: p2-json-output\n\ + \x20\x20\x20\x20summary: antecedent-as-string\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_antecedent_as_list() { + // Compound antecedents (`all_of` / `any_of`) are deferred to v2 per plan + // Sub-decision 2b. v1 must reject the list form. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20- check_id: p2-json-output\n\ + \x20\x20\x20\x20\x20\x20\x20\x20- check_id: p2-yaml-output\n\ + \x20\x20\x20\x20summary: antecedent-as-list\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_kind_with_wrong_case() { + // YAML is case-sensitive; "Conditional" is not "conditional". An author + // copying from a markdown example might capitalize and silently get a + // different code path. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: Conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20summary: capitalized kind\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "Conditional"]); +} + +#[test] +fn rt_rejects_mixed_legacy_if_and_new_kind() { + // Author mid-migration writes both shapes in the same row. The legacy + // `if:` branch only fires for `map.len() == 1`, so today both keys would + // silently pick the new shape and drop the legacy prose. The parser + // rejects the mixed shape outright. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20if: CLI emits structured output\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20summary: mixed legacy+new shape\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "if:", "kind:"]); +} + +#[test] +fn rt_rejects_antecedent_with_compound_op_key() { + // The v1 schema is strict about `antecedent` contents: only `check_id` + // is permitted. A v2-style `op: any_of` accidentally added to a v1 row + // must error so v2 syntax doesn't ship under v1 semantics. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20check_id: p2-json-output\n\ + \x20\x20\x20\x20\x20\x20\x20\x20op: any_of\n\ + \x20\x20\x20\x20summary: smuggled v2 syntax\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "op", "v1 schema"]); +} + +#[test] +fn rt_rejects_antecedent_null() { + // `antecedent: null` slipped past an earlier draft; the parser must + // treat it the same as a non-mapping value and refuse the row. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: null\n\ + \x20\x20\x20\x20summary: null antecedent\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "antecedent", "mapping"]); +} + +#[test] +fn rt_rejects_empty_antecedent_mapping() { + // `antecedent: {}` is a mapping with no `check_id`. Surface the missing + // field by name. + let yaml = format!( + "{RT_FIXTURE_HEAD}\ + \x20\x20- id: p2-must-x\n\ + \x20\x20\x20\x20level: must\n\ + \x20\x20\x20\x20applicability:\n\ + \x20\x20\x20\x20\x20\x20kind: conditional\n\ + \x20\x20\x20\x20\x20\x20antecedent: {{}}\n\ + \x20\x20\x20\x20summary: empty antecedent\n" + ); + rt_assert_error_mentions(&yaml, &["p2-must-x", "check_id"]); +} + +#[test] +fn rt_emit_rust_escapes_quotes_in_antecedent_check_id() { + // Defense-in-depth: the parser already rejects whitespace-only + // check_ids, but emit_rust should still escape any string content + // that ends up in the generated source. A check_id containing a + // quote (impossible today but cheap to guard against) must not break + // the generated Rust literal. + let reqs = vec![ParsedRequirement { + id: "p2-must-quoted".into(), + principle: 2, + level: Level::Must, + summary: "row with a hostile check_id".into(), + applicability: Applicability::Conditional { + condition: None, + antecedent: Some(Antecedent { + check_id: r#"p2-"injected""#.into(), + }), + }, + }]; + let src = emit_rust(&reqs, "0.0.0"); + // The check_id literal must escape the embedded quotes; the generated + // line should parse as valid Rust. + assert!( + src.contains(r#"check_id: "p2-\"injected\"""#), + "generated source must escape quotes in check_id, got:\n{src}", + ); +} diff --git a/tests/scorecard_schema_v05.rs b/tests/scorecard_schema_v05.rs index 2aa9097..8f67a72 100644 --- a/tests/scorecard_schema_v05.rs +++ b/tests/scorecard_schema_v05.rs @@ -325,3 +325,218 @@ fn schema_v05_badge_eligibility_flag_matches_score() { assert!(parsed["badge"]["embed_markdown"].is_null()); } } + +// ───────────────────────────────────────────────────────────────────────── +// Schema 0.6 red team: the committed scorecard.schema.json is the consumer +// contract for the site renderer and third-party leaderboards. A drift +// between the hand-written schema and the serde-derived live JSON would +// silently break those consumers. The tests below pin the shape contract +// from both directions. +// ───────────────────────────────────────────────────────────────────────── + +/// Read the committed schema once. Returns the parsed JSON value so each +/// test can assert against a specific shape concern in isolation. +fn schema_doc() -> Value { + let path = format!( + "{}/schema/scorecard.schema.json", + env!("CARGO_MANIFEST_DIR") + ); + let text = std::fs::read_to_string(&path).expect("schema file readable"); + serde_json::from_str(&text).expect("schema file is valid JSON") +} + +#[test] +fn rt_schema_id_pins_to_published_schema_version() { + // The schema's `$id` must match the SCHEMA_VERSION constant emitted by + // the runtime. A bump that updates one without the other ships a + // consumer contract that disagrees with itself. + let schema = schema_doc(); + let id = schema["$id"].as_str().expect("$id is a string"); + assert!( + id.contains("scorecard-v0.6"), + "schema $id must pin to the current SCHEMA_VERSION (0.6), got: {id}", + ); +} + +#[test] +fn rt_schema_status_enum_lists_all_seven_taxonomy_values() { + // The 7-status taxonomy is the load-bearing contract of schema 0.6. A + // drift here (missing `opt_out`, missing `n_a`, stray pre-0.6 value + // dropped, etc.) would either silently mute new statuses on the + // consumer side or fail validation against legitimate scorecards. + let schema = schema_doc(); + let enums = schema["$defs"]["CheckResultView"]["properties"]["status"]["enum"] + .as_array() + .expect("status.enum is an array"); + let values: Vec<&str> = enums.iter().filter_map(|v| v.as_str()).collect(); + for expected in ["pass", "warn", "fail", "opt_out", "n_a", "skip", "error"] { + assert!( + values.contains(&expected), + "status.enum missing `{expected}` — schema 0.6 contract violated. got: {values:?}", + ); + } + assert_eq!( + values.len(), + 7, + "status.enum must list exactly seven values; got {values:?}", + ); +} + +#[test] +fn rt_schema_summary_required_includes_opt_out_and_n_a() { + // Summary counters are an additive shape change: adding to `properties` + // without adding to `required` would let consumers omit them silently. + let schema = schema_doc(); + let required = schema["$defs"]["Summary"]["required"] + .as_array() + .expect("Summary.required is an array"); + let names: Vec<&str> = required.iter().filter_map(|v| v.as_str()).collect(); + for expected in [ + "total", "pass", "warn", "fail", "opt_out", "n_a", "skip", "error", + ] { + assert!( + names.contains(&expected), + "Summary.required missing `{expected}` — got: {names:?}", + ); + } +} + +#[test] +fn rt_schema_check_result_view_includes_tier_and_check_id() { + // Schema 0.6 added `tier` and `check_id` to every results[] entry. + // The drift guard pins both presence and the tier enum's three values + // (plus null for unknown row ids). + let schema = schema_doc(); + let props = &schema["$defs"]["CheckResultView"]["properties"]; + assert!(props["tier"].is_object(), "CheckResultView.tier missing"); + assert!( + props["check_id"].is_object(), + "CheckResultView.check_id missing", + ); + let tier_enum = props["tier"]["enum"] + .as_array() + .expect("tier.enum is array"); + let tier_values: Vec<&str> = tier_enum.iter().filter_map(|v| v.as_str()).collect(); + for expected in ["must", "should", "may"] { + assert!( + tier_values.contains(&expected), + "tier.enum missing `{expected}`, got: {tier_values:?}", + ); + } + // Also accepts null for rows whose id is not in the registry. + assert!( + tier_enum.iter().any(|v| v.is_null()), + "tier.enum must permit null for unknown row ids, got: {tier_enum:?}", + ); +} + +#[test] +fn rt_schema_example_block_passes_its_own_required_keys() { + // The schema's `examples[0]` is documentation surface — if it drifts + // from the actual `required` lists, agents copying it as a template + // will produce invalid scorecards. Walk the required[] tree and assert + // every key resolves on the example. + let schema = schema_doc(); + let example = &schema["examples"][0]; + assert!(example.is_object(), "examples[0] must be an object"); + + let top_required = schema["required"] + .as_array() + .expect("top-level required is array"); + for key_val in top_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + example.get(key).is_some(), + "examples[0] missing top-level required key `{key}`", + ); + } + + // Walk into results[0] and assert its required keys are present too. + let result_example = &example["results"][0]; + let result_required = schema["$defs"]["CheckResultView"]["required"] + .as_array() + .expect("CheckResultView.required is array"); + for key_val in result_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + result_example.get(key).is_some(), + "examples[0].results[0] missing required key `{key}`", + ); + } + + // And the summary block. + let summary_example = &example["summary"]; + let summary_required = schema["$defs"]["Summary"]["required"] + .as_array() + .expect("Summary.required is array"); + for key_val in summary_required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + summary_example.get(key).is_some(), + "examples[0].summary missing required key `{key}`", + ); + } +} + +#[test] +fn rt_live_scorecard_top_level_keys_match_schema_required() { + // The strongest drift guard: spawn the real binary, produce a live + // scorecard, and assert every key in the schema's top-level `required` + // list is present. Any field added to the struct without a matching + // schema entry, or removed from the schema without removing from the + // struct, surfaces here. + let path = fixture_path("perfect-rust"); + let output = cmd() + .args(["audit", &path, "--output", "json"]) + .output() + .expect("anc spawn"); + let stdout = String::from_utf8(output.stdout).expect("utf-8 stdout"); + let live: Value = serde_json::from_str(&stdout).expect("live JSON parses"); + + let schema = schema_doc(); + let required = schema["required"] + .as_array() + .expect("top-level required is array"); + for key_val in required { + let key = key_val.as_str().expect("required entry is string"); + assert!( + live.get(key).is_some(), + "live scorecard missing required top-level key `{key}` — \ + schema declares it but the live JSON omits it.", + ); + } +} + +#[test] +fn rt_live_results_rows_satisfy_check_result_view_required_keys() { + // Per-row contract: every row in results[] carries the keys declared + // required by CheckResultView. Catches a probe that hand-builds a + // CheckResult skipping a field, or a schema that lists a key the + // serializer dropped. + let path = fixture_path("perfect-rust"); + let output = cmd() + .args(["audit", &path, "--output", "json"]) + .output() + .expect("anc spawn"); + let stdout = String::from_utf8(output.stdout).expect("utf-8 stdout"); + let live: Value = serde_json::from_str(&stdout).expect("live JSON parses"); + + let schema = schema_doc(); + let required: Vec = schema["$defs"]["CheckResultView"]["required"] + .as_array() + .expect("CheckResultView.required is array") + .iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect(); + + let rows = live["results"].as_array().expect("results is array"); + assert!(!rows.is_empty(), "live run produced no rows"); + for (i, row) in rows.iter().enumerate() { + for key in &required { + assert!( + row.get(key).is_some(), + "results[{i}] missing required key `{key}`: row = {row}", + ); + } + } +} From 3bd2e6f5e0bf72f04dd658d9221c13f8bdc8fb4e Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 27 May 2026 20:49:43 -0500 Subject: [PATCH 6/6] chore(spec): resync agentnative-spec dev @ 1fb49fe with backfilled last-revised dates Pulls the date-discipline fix from agentnative-spec PR #38 (`fix(principles): enforce last-revised discipline + backfill stale dates`). Seven principle files refresh their `last-revised` frontmatter to match the date their content actually changed; p3 was already correct and stays as-is. The applicability semantics already vendored at `b4f4d02` (U1 of the scorecard fairness taxonomy plan) carry through unchanged; this is a frontmatter-only update. VERSION still reads `0.4.0` because the upstream spec defers the cut to a release PR; the resync is faithful. No source changes here; the regenerated `REQUIREMENTS` slice via `build.rs` is byte-identical to the prior build because the parser does not consume `last-revised`. Resolved short SHA: 1fb49fe (printed by sync-spec.sh during vendoring). --- src/principles/spec/principles/p1-non-interactive-by-default.md | 2 +- .../spec/principles/p2-structured-parseable-output.md | 2 +- .../spec/principles/p4-fail-fast-actionable-errors.md | 2 +- .../spec/principles/p5-safe-retries-mutation-boundaries.md | 2 +- .../principles/p6-composable-predictable-command-structure.md | 2 +- .../spec/principles/p7-bounded-high-signal-responses.md | 2 +- src/principles/spec/principles/p8-discoverable-skill-bundle.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/principles/spec/principles/p1-non-interactive-by-default.md b/src/principles/spec/principles/p1-non-interactive-by-default.md index f2e70ce..2cc1df7 100644 --- a/src/principles/spec/principles/p1-non-interactive-by-default.md +++ b/src/principles/spec/principles/p1-non-interactive-by-default.md @@ -1,7 +1,7 @@ --- id: p1 title: Non-Interactive by Default -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p1-must-env-var diff --git a/src/principles/spec/principles/p2-structured-parseable-output.md b/src/principles/spec/principles/p2-structured-parseable-output.md index 9840206..4608c62 100644 --- a/src/principles/spec/principles/p2-structured-parseable-output.md +++ b/src/principles/spec/principles/p2-structured-parseable-output.md @@ -1,7 +1,7 @@ --- id: p2 title: Structured, Parseable Output -last-revised: 2026-05-06 +last-revised: 2026-05-21 status: active requirements: - id: p2-must-output-flag diff --git a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md index a591e11..b18feef 100644 --- a/src/principles/spec/principles/p4-fail-fast-actionable-errors.md +++ b/src/principles/spec/principles/p4-fail-fast-actionable-errors.md @@ -1,7 +1,7 @@ --- id: p4 title: Fail Fast with Actionable Errors -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p4-must-try-parse diff --git a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md index f84c1f9..38a6508 100644 --- a/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md +++ b/src/principles/spec/principles/p5-safe-retries-mutation-boundaries.md @@ -1,7 +1,7 @@ --- id: p5 title: Safe Retries and Explicit Mutation Boundaries -last-revised: 2026-04-22 +last-revised: 2026-05-07 status: active requirements: - id: p5-must-force-yes diff --git a/src/principles/spec/principles/p6-composable-predictable-command-structure.md b/src/principles/spec/principles/p6-composable-predictable-command-structure.md index 1a313c4..f67a437 100644 --- a/src/principles/spec/principles/p6-composable-predictable-command-structure.md +++ b/src/principles/spec/principles/p6-composable-predictable-command-structure.md @@ -1,7 +1,7 @@ --- id: p6 title: Composable and Predictable Command Structure -last-revised: 2026-05-06 +last-revised: 2026-05-07 status: active requirements: - id: p6-must-sigpipe diff --git a/src/principles/spec/principles/p7-bounded-high-signal-responses.md b/src/principles/spec/principles/p7-bounded-high-signal-responses.md index 7cfd1e8..7e21520 100644 --- a/src/principles/spec/principles/p7-bounded-high-signal-responses.md +++ b/src/principles/spec/principles/p7-bounded-high-signal-responses.md @@ -1,7 +1,7 @@ --- id: p7 title: Bounded, High-Signal Responses -last-revised: 2026-04-22 +last-revised: 2026-05-07 status: active requirements: - id: p7-must-quiet diff --git a/src/principles/spec/principles/p8-discoverable-skill-bundle.md b/src/principles/spec/principles/p8-discoverable-skill-bundle.md index 063efe1..1479fdc 100644 --- a/src/principles/spec/principles/p8-discoverable-skill-bundle.md +++ b/src/principles/spec/principles/p8-discoverable-skill-bundle.md @@ -1,7 +1,7 @@ --- id: p8 title: Discoverable Through Agent Skill Bundles -last-revised: 2026-05-06 +last-revised: 2026-05-21 status: active requirements: - id: p8-must-bundle-install