From de9d8a7e26880657029451a5dcbe1fd9130263a8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:19:59 +0000 Subject: [PATCH 1/2] Plan AI credits specification Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/daily-syntax-error-quality.lock.yml | 2 +- docs/src/content/docs/reference/model-tables.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/daily-syntax-error-quality.lock.yml b/.github/workflows/daily-syntax-error-quality.lock.yml index 8a7566ebfbd..23e48754fbd 100644 --- a/.github/workflows/daily-syntax-error-quality.lock.yml +++ b/.github/workflows/daily-syntax-error-quality.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cb99cae9a9dad96aa918b578810de9c0d5fb6a88117b3a3392a4171d4bcd90a4","body_hash":"5fd73288a48f06a3c51a5eb67a84ab01210ab30f0f004cd7c4c766074ada9bd7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.57","copilot-sdk":"1.0.0"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cb99cae9a9dad96aa918b578810de9c0d5fb6a88117b3a3392a4171d4bcd90a4","body_hash":"b7aa366d7eb46464c93145ff04f4fd49c075bd10677d11bab4402c78966d96cb","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.57","copilot-sdk":"1.0.0"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.63"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.63"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.63"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22","digest":"sha256:ce5c6f5461b077af0d8e8eb1763436e85153f8e9531117d58a7bdb23de71f00a","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.22@sha256:ce5c6f5461b077af0d8e8eb1763436e85153f8e9531117d58a7bdb23de71f00a"},{"image":"ghcr.io/github/github-mcp-server:v1.1.2","digest":"sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c","pinned_image":"ghcr.io/github/github-mcp-server:v1.1.2@sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) diff --git a/docs/src/content/docs/reference/model-tables.md b/docs/src/content/docs/reference/model-tables.md index f260e488bc0..a097a071ea9 100644 --- a/docs/src/content/docs/reference/model-tables.md +++ b/docs/src/content/docs/reference/model-tables.md @@ -66,6 +66,7 @@ Meta-aliases reference other aliases by name. They are resolved recursively unti | `mini` | `haiku` → `gpt-5-mini` → `gpt-5-nano` → `gemini-flash-lite` | | `large` | `sonnet` → `gpt-5-pro` → `gpt-5` → `gemini-pro` | | `agent` | `sonnet-6x` → `gpt-5.4` → `gpt-5.3` → `gemini-pro` → `any` | +| `small-agent` | `haiku` → `gpt-5-mini` → `gemini-flash` → `any` | | `copilot` | `agent` | | `claude` | `agent` | | `codex` | `agent` | From 2793acffbf87ec2a48998c6ad8ee62cc4f68d24c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:22:34 +0000 Subject: [PATCH 2/2] Add AI Credits specification reference Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../reference/ai-credits-specification.md | 341 ++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 docs/src/content/docs/reference/ai-credits-specification.md diff --git a/docs/src/content/docs/reference/ai-credits-specification.md b/docs/src/content/docs/reference/ai-credits-specification.md new file mode 100644 index 00000000000..514ab1c1abc --- /dev/null +++ b/docs/src/content/docs/reference/ai-credits-specification.md @@ -0,0 +1,341 @@ +--- +title: AI Credits Specification +description: Formal W3C-style specification for AI Credits (AIC) calculation, model pricing catalog format, and Copilot billing reference requirements. +sidebar: + order: 1361 +--- + +# AI Credits Specification + +**Version**: 1.0.0 +**Status**: Draft +**Publication Date**: 2026-06-05 +**Editor**: GitHub Agentic Workflows Team +**This Version**: [ai-credits-specification](/gh-aw/reference/ai-credits-specification/) +**Latest Published Version**: This document + +--- + +## Abstract + +This specification defines AI Credits (AIC) as the normative inference-cost metric for GitHub Agentic Workflows (gh-aw). It specifies the required calculation model from token usage and provider pricing, the canonical `models.json` catalog format used to store per-model pricing inputs, and the required external references for GitHub Copilot model and billing alignment. + +## Status of This Document + +This section describes the status of this document at the time of publication. This is a draft specification and may be updated, replaced, or made obsolete by other documents at any time. + +This document is governed by the GitHub Agentic Workflows project specifications process. + +## Table of Contents + +1. [Introduction](#1-introduction) +2. [Conformance](#2-conformance) +3. [AI Credits Accounting Model](#3-ai-credits-accounting-model) +4. [Pricing Catalog Format (`models.json`)](#4-pricing-catalog-format-modelsjson) +5. [Catalog Provisioning and Synchronization](#5-catalog-provisioning-and-synchronization) +6. [Copilot Billing Reference Requirements](#6-copilot-billing-reference-requirements) +7. [Reporting Requirements](#7-reporting-requirements) +8. [Compliance Testing](#8-compliance-testing) +9. [Appendices](#appendices) +10. [References](#references) +11. [Change Log](#change-log) + +--- + +## 1. Introduction + +### 1.1 Purpose + +AIC provides a single monetary-normalized metric for inference cost across supported model providers. This specification defines how conforming implementations compute AIC, how pricing data is represented, and how Copilot-specific pricing alignment is governed. + +### 1.2 Scope + +This specification covers: + +- The normative AIC unit definition and conversion rules. +- The per-invocation and aggregated AIC calculation formulas. +- Required `models.json` data structure and field semantics. +- Requirements for how pricing catalog data is provided and mirrored in gh-aw. +- Required references to GitHub Copilot model and billing documentation. + +This specification does NOT cover: + +- GitHub Actions minutes billing. +- ET (Effective Tokens) normalization rules. +- Provider-side billing reconciliation and invoice dispute procedures. + +### 1.3 Design Goals + +The specification is designed to: + +1. Provide a testable and deterministic calculation contract. +2. Keep pricing inputs explicit and auditable through structured catalog files. +3. Support model-name drift through normalized lookup and prefix fallback matching. +4. Maintain compatibility between CLI and setup runtime pricing catalogs. + +--- + +## 2. Conformance + +### 2.1 Conformance Classes + +**Conforming implementation**: Satisfies all MUST/SHALL requirements in Sections 3 through 8. + +**Partially conforming implementation**: Computes core AIC from token usage and model pricing (Section 3) but omits one or more optional reporting or synchronization requirements. + +### 2.2 Requirements Notation + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +### 2.3 Compliance Levels + +- **Level 1 – Calculation**: Implements AIC formulas and provider token handling (Section 3). +- **Level 2 – Catalog**: Implements required `models.json` structure and lookup semantics (Section 4). +- **Level 3 – Operational**: Implements catalog provisioning, Copilot reference alignment, and reporting requirements (Sections 5–7). + +--- + +## 3. AI Credits Accounting Model + +### 3.1 Unit Definition + +A conforming implementation MUST define: + +- `1 AIC = 0.01 USD` +- `AIC = USD / 0.01` + +### 3.2 Token Classes + +Implementations MUST support the following token classes for cost calculation when available: + +- Input tokens +- Output tokens +- Cache read tokens +- Cache write tokens +- Reasoning tokens + +### 3.3 Per-Invocation Cost Formula + +For a single invocation, implementations MUST compute USD as: + +```text +cost_usd = + (input_tokens × input_price_per_token) + + (output_tokens × output_price_per_token) + + (cache_read_tokens × cache_read_price_per_token) + + (cache_write_tokens × cache_write_price_per_token) + + (reasoning_tokens × reasoning_price_per_token) +``` + +A conforming implementation MUST derive AIC as: + +```text +aic = cost_usd / 0.01 +``` + +### 3.4 Price Fallback Rules + +If a model entry omits optional price fields, implementations MUST apply the following fallback behavior: + +- `cache_read_price_per_token` defaults to `input_price_per_token` +- `cache_write_price_per_token` defaults to `input_price_per_token` +- `reasoning_price_per_token` defaults to `output_price_per_token` + +### 3.5 Provider-Specific Input Handling + +For providers that include cache-read tokens in total input tokens, implementations MUST subtract `cache_read_tokens` from `input_tokens` before applying input price and MUST NOT double-charge cache-read usage. + +### 3.6 Aggregation + +For grouped runs (for example, episodes), implementations MUST aggregate AIC by summing per-invocation AIC values. + +--- + +## 4. Pricing Catalog Format (`models.json`) + +### 4.1 Top-Level Structure + +A conforming catalog MUST be valid JSON with this structure: + +```json +{ + "providers": { + "provider-name": { + "models": { + "model-id": { + "cost": { + "input": "...", + "output": "...", + "cache_read": "...", + "cache_write": "...", + "reasoning": "..." + } + } + } + } + } +} +``` + +### 4.2 Required and Optional Fields + +For each model: + +- `cost.input` MUST be present. +- `cost.output` MUST be present. +- `cost.cache_read` MAY be present. +- `cost.cache_write` MAY be present. +- `cost.reasoning` MAY be present. + +Cost values MUST be decimal numbers encoded as strings and interpreted as USD per token. + +### 4.3 Provider Keys + +Provider keys MUST be lowercase identifiers. For Copilot-backed pricing, the canonical provider key MUST be `github-copilot`. + +### 4.4 Model Lookup Normalization + +A conforming implementation MUST normalize provider and model identifiers for lookup by trimming whitespace and applying case-insensitive comparison. An implementation SHOULD support compatibility matching between punctuation variants (for example, `.` and `_` compared to `-`) and provider-scoped prefix fallback. + +--- + +## 5. Catalog Provisioning and Synchronization + +### 5.1 Embedded Runtime Catalogs + +gh-aw implementations MUST provide synchronized pricing catalogs at: + +- `pkg/cli/data/models.json` +- `actions/setup/js/models.json` + +These files SHALL represent the same pricing dataset. + +### 5.2 Source and Refresh Expectations + +Catalog refresh processes SHOULD use normalized upstream model inventories and SHOULD validate Copilot entries against authoritative GitHub Copilot model and billing documentation. + +### 5.3 Change Control + +Catalog updates MUST preserve JSON validity and MUST maintain backward-safe handling for historical model IDs that remain in the catalog but are absent from current live inventories. + +--- + +## 6. Copilot Billing Reference Requirements + +Implementations and documentation that describe Copilot AIC behavior MUST reference: + +- GitHub Copilot models documentation: +- GitHub Copilot models and pricing reference: +- GitHub Copilot plan and billing documentation: + +These references SHOULD be treated as the external billing-alignment sources for Copilot model pricing validation. + +--- + +## 7. Reporting Requirements + +A conforming implementation MUST expose AIC in runtime reporting outputs where cost metrics are emitted. + +Implementations SHOULD provide: + +- Per-run AIC values. +- Aggregated AIC values for grouped executions. +- Structured output fields suitable for machine parsing. + +--- + +## 8. Compliance Testing + +### 8.1 Test Suite Requirements + +A conformance test suite MUST include at least the following test cases: + +- **T-AIC-001**: Verify `1 AIC = 0.01 USD` conversion. +- **T-AIC-002**: Verify per-invocation AIC computation using all token classes. +- **T-AIC-003**: Verify cache-read subtraction behavior for providers that include cache-read tokens in input totals. +- **T-AIC-004**: Verify fallback pricing when optional cost fields are omitted. +- **T-AIC-005**: Verify `models.json` rejects missing required fields (`input`, `output`). +- **T-AIC-006**: Verify provider key normalization and `github` to `github-copilot` mapping behavior. +- **T-AIC-007**: Verify catalog mirror consistency between CLI and setup runtime paths. +- **T-AIC-008**: Verify reporting outputs include per-run AIC values. + +### 8.2 Compliance Checklist + +| Requirement | Test ID | Level | Status | +|-------------|---------|-------|--------| +| Unit conversion (`1 AIC = 0.01 USD`) | T-AIC-001 | 1 | Required | +| Full token-class formula | T-AIC-002 | 1 | Required | +| Cache-read non-double-charge behavior | T-AIC-003 | 1 | Required | +| Optional price fallback behavior | T-AIC-004 | 1 | Required | +| Catalog schema conformance | T-AIC-005 | 2 | Required | +| Provider/model normalization behavior | T-AIC-006 | 2 | Required | +| Mirrored catalog consistency | T-AIC-007 | 3 | Required | +| AIC reporting visibility | T-AIC-008 | 3 | Required | + +--- + +## Appendices + +### Appendix A: Worked Example + +Given: + +- Input: 1000 at $0.000003/token +- Output: 200 at $0.000015/token +- Cache read: 400 at $0.0000003/token +- Cache write: 50 at $0.00000375/token +- Reasoning: 25 at $0.000015/token + +Result: + +```text +cost_usd = 0.0054825 +aic = 0.54825 +``` + +### Appendix B: Error Conditions + +Conforming implementations SHOULD surface explicit validation errors for: + +- Invalid `models.json` structure. +- Non-numeric cost values. +- Missing required model cost fields. +- Unknown provider/model pairs with no fallback match. + +### Appendix C: Security and Integrity Considerations + +Pricing catalogs are configuration inputs. Implementations SHOULD: + +- Treat catalog updates as controlled changes. +- Validate and review catalog source provenance. +- Avoid silently mutating cost values at runtime. + +--- + +## References + +### Normative References + +- **[RFC 2119]** Key words for use in RFCs to Indicate Requirement Levels. + +### Informative References + +- **[GH-AW-COST]** Cost Management reference. +- **[GH-COPILOT-MODELS]** About GitHub Copilot models. +- **[GH-COPILOT-BILLING-MODELS]** GitHub Copilot models and pricing. +- **[GH-COPILOT-BILLING-PLANS]** Subscription plans for GitHub Copilot. +- **[MODELS-DEV]** models.dev API index. + +--- + +## Change Log + +### Version 1.0.0 (Draft) + +- Added initial AI Credits (AIC) normative definition and formulas. +- Added canonical `models.json` format and synchronization requirements. +- Added Copilot billing reference requirements and compliance test matrix. + +--- + +Copyright © 2026 GitHub. All rights reserved.