diff --git a/README.md b/README.md index aaa0de8..eaf6e7f 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ docpull standards-pack rfc:9110 -o packs/standard docpull dataset-pack ./metrics.csv -o packs/dataset docpull dataset-pack 'https://data.example.org/rows.json?$limit=100' -o packs/dataset docpull relationship-pack example.com --subject "Example Brand" -o packs/relationships +docpull website-pack example.com -o packs/website docpull transcript-pack ./meeting.vtt -o packs/transcript docpull wiki-pack wiki:Web_scraping -o packs/wiki docpull brand-pack example.com -o packs/brand @@ -196,6 +197,9 @@ docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl docpull export packs/docs --format cursor-rules -o .cursor/rules --skill-name docs ``` +The canonical baseline-aware website acquisition contract is documented in +[Website snapshots](docs/website-snapshot.md). + Use `docpull ci --prepare` to validate a project or standalone pack in CI. Install optional extras as needed: @@ -648,7 +652,7 @@ part of the package release contract. `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`, `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`, `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`, - `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`, + `docpull wiki-pack`, `docpull website-pack`, `docpull brand-pack`, `docpull product-pack`, `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`, `docpull pack validate`, `docpull pack audit`, `docpull export`, diff --git a/bench/uv.lock b/bench/uv.lock index 5b0bac8..4627b49 100644 --- a/bench/uv.lock +++ b/bench/uv.lock @@ -584,7 +584,7 @@ wheels = [ [[package]] name = "docpull" -version = "6.3.0" +version = "6.4.0" source = { editable = "../" } dependencies = [ { name = "aiohttp" }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f7d7ae1..69312c7 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.4.0] - 2026-07-16 + +### Added +- Add `website-pack` across CLI, Python workflow API, and MCP. The workflow + emits the pinned `website.snapshot.v1` schema, stable URL-based document + identities, content-hash versions, page roles, source authority, OKF/raw + representations, bounded optional visuals, portable-v3 manifests, and + recursively verifiable artifact hashes. +- Add verified-baseline diff states for added, changed, unchanged, removed, + failed, and blocked pages, plus golden/tamper fixtures. + +### Changed +- Let brand, product, policy, styleguide, image, and relationship workflows + consume existing packs without network fallback. Product extraction now + keeps trial metadata separate from ordinary price fields and excludes + testimonial copy from feature evidence. + ### Removed - Remove the standalone DocPull website, its Next.js workspace, and web-only CI/release gates. GitHub, PyPI, and the MCP Registry remain the canonical diff --git a/docs/competitor-tracker-integration.md b/docs/competitor-tracker-integration.md index 022b8d0..3d4ffa7 100644 --- a/docs/competitor-tracker-integration.md +++ b/docs/competitor-tracker-integration.md @@ -1,6 +1,6 @@ # Competitor-tracker integration contract -Pin DocPull `6.3.0` and validate `intelligence.bundle.v1.json` against +Pin DocPull `6.4.0` and validate `website.snapshot.v1.json` and `intelligence.bundle.v1.json` against `intelligence-bundle.v1.schema.json` at import time. ## Producer flow diff --git a/docs/context-pack-contract-v3.md b/docs/context-pack-contract-v3.md index 5f40510..6280f1b 100644 --- a/docs/context-pack-contract-v3.md +++ b/docs/context-pack-contract-v3.md @@ -60,6 +60,11 @@ write the agent/eval sidecars after parsing. ## Typed Knowledge Lanes +`website-pack` is the canonical bounded website lane. In addition to the v3 +corpus, it writes `website.snapshot.v1.json`, explicit baseline states, OKF and +optional visual representations, and recursive artifact hashes. See +[Website snapshots](website-snapshot.md). + Known-source lanes use the same v3 raw contract: `openapi-pack`, `feed-pack`, `paper-pack`, `repo-pack`, `package-pack`, `standards-pack`, `dataset-pack`, `transcript-pack`, and `wiki-pack` all write diff --git a/docs/contracts.md b/docs/contracts.md index 0da4f78..8d62376 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -27,6 +27,13 @@ generic contracts. ## Cross-repository contracts +### `website.snapshot.v1` + +The canonical website-intelligence acquisition artifact. It binds stable +document/version identities, entity and authority metadata, page roles, +baseline states, OKF/raw/visual representations, portable-v3 manifests, and a +canonical snapshot hash. See [Website snapshots](website-snapshot.md). + ### `workflow.request.v1` A scheduler-neutral invocation containing a stable request ID, workflow name, diff --git a/docs/surface-contract.md b/docs/surface-contract.md index 4f61c48..b58d3c3 100644 --- a/docs/surface-contract.md +++ b/docs/surface-contract.md @@ -48,7 +48,7 @@ DocPull targets capability alignment, not 1:1 flag parity. MCP should not mirror | Refresh/score/diff/audit context packs | `docpull refresh`, `docpull pack score`, `docpull pack sources`, `docpull pack diff`, `docpull pack audit` | Pack helper modules, `refresh_pack`, `audit_pack` | `refresh_pack`, `pack_score`, `pack_diff`, `audit_pack` | Core-aligned | | Build pack citations/entities/search/briefs | `docpull pack citations`, `docpull pack entities`, `docpull pack search`, `docpull pack brief` | `build_citation_map`, `extract_pack_entities`, `search_pack`, `build_research_brief` | `pack_citations`, `pack_entities`, `pack_search`, `pack_brief` | Core-aligned | | Prepare full pack intelligence bundle | `docpull pack prepare` | `prepare_pack` in `docpull.pack_tools` | `pack_prepare` | Core-aligned | -| Evidence-pack workflow protocol | `brand-pack`, `product-pack`, `styleguide-pack`, `image-pack`, `screenshot-pack`, `policy-pack`, `relationship-pack`, `dataset-pack` | concrete `build_*_pack` builders plus `WorkflowRequest`, `run_workflow`, `async_run_workflow` | `workflow_run` plus dedicated evidence tools | Core-aligned | +| Evidence-pack workflow protocol | `website-pack`, `brand-pack`, `product-pack`, `styleguide-pack`, `image-pack`, `screenshot-pack`, `policy-pack`, `relationship-pack`, `dataset-pack` | concrete `build_*_pack` builders plus `WorkflowRequest`, `run_workflow`, `async_run_workflow` | `workflow_run` plus dedicated evidence tools including `website_pack` | Core-aligned | | Tracker import bundle | `docpull pack intelligence-bundle` (`company-brain` alias) | `build_intelligence_bundle` (`build_company_brain_bundle` alias) | `intelligence_bundle` | Core-aligned | | Context CI and eval-grade context | `docpull ci`, `docpull pack prepare --eval-grade`, `docpull pack validate --level eval` | `run_context_ci`, `validate_pack_contract`, pack preparation helpers | Not exposed yet | Core-aligned | | Build/query local source graphs | `docpull graph build`, `docpull graph status`, `docpull graph query`, `docpull graph neighbors`, `docpull graph refresh` | `build_graph`, `load_graph`, `graph_status`, `query_graph`, `graph_neighbors`, `refresh_graph` | `graph_build`, `graph_status`, `graph_query`, `graph_neighbors`, `graph_refresh` | Core-aligned | diff --git a/docs/website-snapshot.md b/docs/website-snapshot.md new file mode 100644 index 0000000..aab5180 --- /dev/null +++ b/docs/website-snapshot.md @@ -0,0 +1,58 @@ +# Website snapshots + +`website-pack` is DocPull's canonical acquisition workflow for downstream +competitive-intelligence systems. It writes a portable-v3 artifact whose entrypoint is +`website.snapshot.v1.json` and whose complete file set is pinned by +`artifact.manifest.json`. + +```bash +docpull website-pack https://example.com -o packs/example-website +docpull website-pack https://example.com -o packs/example-next \ + --baseline-pack packs/example-website +``` + +The default bounded crawl is 50 pages and depth 3. It classifies home, product, +pricing, documentation, trust, legal, changelog, support, and other pages. Every +active page has a stable canonical-URL `document_id`, a content-hash +`document_version`, source authority, entity reference, page role, and OKF +representation. Raw HTML is on by default. At most eight key-page screenshots +are included when `DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1` and a compatible +local browser runtime are available; missing browser capability produces a +typed warning and does not invalidate text evidence. + +## Baseline states + +A baseline must pass recursive manifest, reference, pack/run identity, schema, +and snapshot-hash verification before use. The new snapshot explicitly reports +`added`, `changed`, `unchanged`, `removed`, `failed`, and `blocked` document +states. Unchanged documents remain in `documents.ndjson`; removed documents +remain in the snapshot manifest without pretending they were acquired again. + +`validate_website_snapshot_pack(path)` verifies the current manifest and every +referenced representation. Files outside the manifest are ignored, so stale +files from an older output directory cannot alter current-run identity. + +## Artifact layout + +- `website.snapshot.v1.json` — canonical snapshot identity, baseline, options, + documents, coverage, and representation references. +- `documents.ndjson` — the active evidence corpus used by downstream analysis. +- `okf/` — one OKF concept per active document. +- `raw/` — selected source HTML when raw capture is enabled. +- `screenshots/` and `brand-assets/` — bounded optional visual evidence. +- `corpus.manifest.json`, `current-run.manifest.json`, + `coverage.manifest.json`, and `provenance.manifest.json` — portable-v3 + inventory, run, coverage, and provenance layers. +- `artifact.manifest.json` — sorted relative paths, byte counts, media types, + and SHA-256 hashes for the exact current artifact. + +The bundled schema is `website-snapshot.v1.schema.json`. Its file SHA-256 is +written into every snapshot as `schema_sha256`, allowing consumers to pin the +exact producer contract. + +## Local replay + +`brand-pack`, `product-pack`, `policy-pack`, `styleguide-pack`, `image-pack`, +and `relationship-pack` accept an existing DocPull pack path. Local-pack mode +does not fetch the network or download referenced assets. This makes one +website acquisition reusable across deterministic extraction lanes. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 73b5ba8..612aa0e 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "docpull", - "version": "6.3.0", + "version": "6.4.0", "description": "Pull public web sources into Claude Code. Indexes static and server-rendered sites as local Markdown with conditional-GET caching, then exposes them as MCP tools. Local, browser-free, no API keys.", "author": { "name": "Raintree Technology", diff --git a/plugin/.codex-plugin/plugin.json b/plugin/.codex-plugin/plugin.json index 13f7c44..b5acba2 100644 --- a/plugin/.codex-plugin/plugin.json +++ b/plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "id": "docpull", "name": "docpull", - "version": "6.3.0", + "version": "6.4.0", "description": "Pull public web sources into Codex as local, searchable Markdown through docpull's MCP server.", "skills": "./skills/", "mcpServers": "./.mcp.json", diff --git a/plugin/README.md b/plugin/README.md index d9c36bd..54e8822 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -17,9 +17,9 @@ for the boundary between the plugin's MCP tools and the broader CLI/SDK. ## What you get -- **MCP server** (35 tools): +- **MCP server** (36 tools): - Read: `fetch_url`, `list_sources`, `list_indexed`, `grep_docs`, `read_doc`, `pack_score`, `pack_diff`, `pack_citations`, `pack_entities`, `pack_search`, `pack_brief`, `graph_status`, `graph_query`, `graph_neighbors`, `validate_policy`, `serve_pack_status` - - Write: `render_url`, `ensure_docs`, `workflow_run`, `brand_pack`, `product_pack`, `styleguide_pack`, `image_pack`, `screenshot_pack`, `policy_pack`, `relationship_pack`, `intelligence_bundle`, `refresh_pack`, `audit_pack`, `pack_prepare`, `graph_build`, `graph_refresh`, `export_pack`, `add_source`, `remove_source` + - Write: `render_url`, `ensure_docs`, `workflow_run`, `website_pack`, `brand_pack`, `product_pack`, `styleguide_pack`, `image_pack`, `screenshot_pack`, `policy_pack`, `relationship_pack`, `intelligence_bundle`, `refresh_pack`, `audit_pack`, `pack_prepare`, `graph_build`, `graph_refresh`, `export_pack`, `add_source`, `remove_source` - All read tools advertise `readOnlyHint` so hosts that auto-approve safe tools won't prompt for them. - **Claude Code slash commands**: @@ -39,7 +39,7 @@ MCP server is available: ```bash pip install 'docpull[mcp]' # or: pipx install 'docpull[mcp]' # uv tool install 'docpull[mcp]' -docpull --version # should print 6.3.0 or newer +docpull --version # should print 6.4.0 or newer docpull mcp --help # confirm the MCP subcommand is wired ``` diff --git a/pyproject.toml b/pyproject.toml index 350e329..6edf81d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "docpull" -version = "6.3.0" +version = "6.4.0" dynamic = [] description = "Declare, sync, diff, and lock context dependencies for AI agents" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/server.json b/server.json index 9ca8705..d47eb49 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://github.com/raintree-technology/docpull", - "version": "6.3.0", + "version": "6.4.0", "packages": [ { "registryType": "pypi", "identifier": "docpull", - "version": "6.3.0", + "version": "6.4.0", "transport": { "type": "stdio" } diff --git a/src/docpull/__init__.py b/src/docpull/__init__.py index 15ec625..9c52a2d 100644 --- a/src/docpull/__init__.py +++ b/src/docpull/__init__.py @@ -15,7 +15,7 @@ from .surface import PUBLIC_SDK_EXPORTS -__version__ = "6.3.0" +__version__ = "6.4.0" _LAZY_EXPORTS: dict[str, tuple[str, str]] = { **{ @@ -41,9 +41,11 @@ "build_repo_pack", "build_standards_pack", "build_styleguide_pack", + "build_website_pack", "build_transcript_pack", "build_wiki_pack", "capture_screenshot_pack", + "validate_website_snapshot_pack", ) }, **{ @@ -59,6 +61,8 @@ "SourceAuthority", "WorkflowRequest", "WorkflowResult", + "WebsiteSnapshot", + "WebsiteSnapshotDocument", "bundled_schema_path", "write_contract_schemas", ) @@ -198,8 +202,10 @@ def __dir__() -> list[str]: build_standards_pack, build_styleguide_pack, build_transcript_pack, + build_website_pack, build_wiki_pack, capture_screenshot_pack, + validate_website_snapshot_pack, ) from .contracts import ( ArtifactManifest, @@ -210,6 +216,8 @@ def __dir__() -> list[str]: RelationshipCandidate, RelationshipPack, SourceAuthority, + WebsiteSnapshot, + WebsiteSnapshotDocument, WorkflowRequest, WorkflowResult, bundled_schema_path, diff --git a/src/docpull/cli.py b/src/docpull/cli.py index 1f30048..7116cec 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -1661,6 +1661,7 @@ def main(argv: list[str] | None = None) -> int: return run_monitor_cli(raw_argv[1:]) if raw_argv and raw_argv[0] in { "brand-pack", + "website-pack", "product-pack", "styleguide-pack", "image-pack", @@ -1676,9 +1677,11 @@ def main(argv: list[str] | None = None) -> int: run_relationship_pack_cli, run_screenshot_pack_cli, run_styleguide_pack_cli, + run_website_pack_cli, ) workflow_runners = { + "website-pack": run_website_pack_cli, "brand-pack": run_brand_pack_cli, "product-pack": run_product_pack_cli, "styleguide-pack": run_styleguide_pack_cli, diff --git a/src/docpull/context_packs/__init__.py b/src/docpull/context_packs/__init__.py index 3045fd4..3a80195 100644 --- a/src/docpull/context_packs/__init__.py +++ b/src/docpull/context_packs/__init__.py @@ -25,6 +25,8 @@ **{name: (".transcript", name) for name in ("async_build_transcript_pack", "build_transcript_pack")}, **{name: (".wiki", name) for name in ("async_build_wiki_pack", "build_wiki_pack")}, "capture_screenshot_pack": (".visuals", "capture_screenshot_pack"), + "build_website_pack": (".website", "build_website_pack"), + "validate_website_snapshot_pack": (".website", "validate_website_snapshot_pack"), } __all__ = list(PUBLIC_CONTEXT_PACK_EXPORTS) @@ -59,6 +61,7 @@ def __dir__() -> list[str]: from .styleguide import build_styleguide_pack from .transcript import async_build_transcript_pack, build_transcript_pack from .visuals import build_image_pack, capture_screenshot_pack + from .website import build_website_pack, validate_website_snapshot_pack from .wiki import async_build_wiki_pack, build_wiki_pack diff --git a/src/docpull/context_packs/brand.py b/src/docpull/context_packs/brand.py index ff5ddd4..c8c221a 100644 --- a/src/docpull/context_packs/brand.py +++ b/src/docpull/context_packs/brand.py @@ -29,6 +29,7 @@ homepage_url_for_domain, is_free_or_disposable_email_domain, likely_internal_pages, + pages_from_local_pack, public_url, quote_markdown, resolve_url, @@ -92,7 +93,14 @@ def build_brand_pack( max_pages: int = MAX_BRAND_PAGES, ) -> dict[str, Any]: """Build a local brand profile from public website evidence.""" - domain = _resolve_brand_domain(domain_or_url, email=email, allow_free_email=allow_free_email) + local_pages = pages_from_local_pack(domain_or_url) + domain = ( + domain_from_input(local_pages[0].url) + if local_pages + else _resolve_brand_domain(domain_or_url, email=email, allow_free_email=allow_free_email) + ) + if not domain: + raise ContextPackError("Could not resolve a domain from local brand-pack evidence.") policy = ensure_policy_for_domain(policy, domain) output_dir = output_dir.resolve() run = ContextPackRun( @@ -101,14 +109,20 @@ def build_brand_pack( policy=policy, input_value=email or domain_or_url, ) - home_url = homepage_url_for_domain(domain) - home_pages = fetch_pages_blocking([home_url], run=run, max_pages=1) - if not home_pages: - raise ContextPackError(f"Could not fetch homepage for {domain}.") - candidate_urls = likely_internal_pages(home_pages[0], domain, max_pages=max_pages) - pages = fetch_pages_blocking(candidate_urls, run=run, max_pages=max_pages) - if not pages: - pages = home_pages + if local_pages is not None: + pages = local_pages[:max_pages] + if not pages: + raise ContextPackError("Local brand-pack input contains no readable documents.") + download_assets = False + else: + home_url = homepage_url_for_domain(domain) + home_pages = fetch_pages_blocking([home_url], run=run, max_pages=1) + if not home_pages: + raise ContextPackError(f"Could not fetch homepage for {domain}.") + candidate_urls = likely_internal_pages(home_pages[0], domain, max_pages=max_pages) + pages = fetch_pages_blocking(candidate_urls, run=run, max_pages=max_pages) + if not pages: + pages = home_pages profile = _extract_brand_profile(pages, domain=domain, name_hint=name, ticker=ticker) assets = _extract_brand_assets( diff --git a/src/docpull/context_packs/common.py b/src/docpull/context_packs/common.py index 350c72e..f50b395 100644 --- a/src/docpull/context_packs/common.py +++ b/src/docpull/context_packs/common.py @@ -40,6 +40,7 @@ from ..models.events import SkipReason from ..models.run import RunIdentity from ..output_contract import OUTPUT_CONTRACT_SCHEMA_VERSION, write_raw_contract_sidecars +from ..pack_reader import PackReadError, load_pack, resolve_pack_path from ..pack_tools import build_citation_map from ..policy import PolicyConfig, policy_domain_matches from ..security.download_policy import SafeDownloadPolicy, UnsafeDownloadError, content_type_base @@ -118,6 +119,8 @@ class PageSnapshot: metadata: dict[str, Any] extraction: dict[str, Any] source_type: str | None = None + etag: str | None = None + last_modified: str | None = None @dataclass(frozen=True) @@ -214,6 +217,9 @@ class ContextPackRun: http_request_count: int = 0 cache_hit_count: int = 0 progress_events: list[dict[str, Any]] = field(default_factory=list) + fetch_output_dir: Path | None = None + cache_dir: Path | None = None + unchanged_urls: set[str] = field(default_factory=set) def __post_init__(self) -> None: self.progress("run", "started", message=f"Started {self.workflow}", timestamp=self.started_at) @@ -444,10 +450,17 @@ async def fetch_pages( run.progress("acquisition", "started", total=len(selected), message="Acquiring source pages") config = DocpullConfig(url=selected[0], profile=ProfileName.CUSTOM) config.network.log_retry_warnings = False + if run.fetch_output_dir is not None: + config.output.directory = run.fetch_output_dir + if run.cache_dir is not None: + config.cache.enabled = True + config.cache.directory = run.cache_dir + config.cache.ttl_days = None + config.cache.skip_unchanged = True snapshots: list[PageSnapshot] = [] async with Fetcher(config) as fetcher: for index, url in enumerate(selected, start=1): - ctx = await fetcher.fetch_one(url, save=False) + ctx = await fetcher.fetch_one(url, save=run.fetch_output_dir is not None) run.http_request_count += 1 if ctx.error: run.errors.append( @@ -462,6 +475,10 @@ async def fetch_pages( ) continue if ctx.should_skip: + if ctx.skip_code == SkipReason.CACHE_UNCHANGED: + run.cache_hit_count += 1 + run.unchanged_urls.add(public_url(url)) + continue if ctx.skip_code in { SkipReason.HTTP_ERROR, SkipReason.ROBOTS_DISALLOWED, @@ -503,6 +520,8 @@ async def fetch_pages( metadata=dict(ctx.metadata or {}), extraction=dict(ctx.extraction_info or {}), source_type=ctx.source_type, + etag=getattr(ctx, "etag", None), + last_modified=getattr(ctx, "last_modified", None), ) ) run.progress( @@ -531,6 +550,60 @@ def fetch_pages_blocking( return asyncio.run(fetch_pages(urls, run=run, max_pages=max_pages)) +def pages_from_local_pack(value: str | Path) -> list[PageSnapshot] | None: + """Load an existing DocPull pack without permitting any network fallback.""" + + path = Path(value).expanduser() + if not path.exists(): + return None + try: + pack = load_pack(path) + except PackReadError as err: + raise ContextPackError(f"Could not read local DocPull pack {path}: {err}") from err + raw_by_identity: dict[tuple[str, str], str] = {} + website_metadata_by_identity: dict[tuple[str, str], dict[str, Any]] = {} + records_path = path.resolve() / "documents.ndjson" + if records_path.is_file(): + for line in records_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(row, dict): + continue + identity = (str(row.get("document_id") or ""), str(row.get("content_hash") or "")) + website_metadata_by_identity[identity] = { + key: row[key] + for key in ("page_role", "entity_reference", "authority", "representations") + if row.get(key) is not None + } + representations = row.get("representations") + raw = representations.get("raw") if isinstance(representations, dict) else None + raw_path = resolve_pack_path(path.resolve(), raw.get("path") if isinstance(raw, dict) else None) + if raw_path and raw_path.is_file(): + raw_by_identity[identity] = raw_path.read_text(encoding="utf-8") + pages: list[PageSnapshot] = [] + for record in pack.documents: + identity = (record.document_id, record.content_hash) + pages.append( + PageSnapshot( + url=public_url(record.url), + title=record.title, + html=raw_by_identity.get(identity, ""), + markdown=record.content, + metadata={ + **dict(record.metadata), + **website_metadata_by_identity.get(identity, {}), + }, + extraction={**record.extraction, "local_pack": str(path.resolve())}, + source_type=record.source_type or "local-pack", + ) + ) + return pages + + def soup_for(page: PageSnapshot) -> BeautifulSoup: return BeautifulSoup(page.html or page.markdown, "html.parser") diff --git a/src/docpull/context_packs/policy_pack.py b/src/docpull/context_packs/policy_pack.py index 23dac9a..ad8642e 100644 --- a/src/docpull/context_packs/policy_pack.py +++ b/src/docpull/context_packs/policy_pack.py @@ -24,6 +24,7 @@ extract_links, fetch_pages_blocking, homepage_url_for_domain, + pages_from_local_pack, public_url, quote_markdown, same_policy_domain, @@ -73,7 +74,8 @@ def build_policy_pack( ) -> dict[str, Any]: """Discover policy pages and emit neutral, clause-level evidence records.""" - domain = domain_from_input(domain_or_url) + local_pages = pages_from_local_pack(domain_or_url) + domain = domain_from_input(local_pages[0].url) if local_pages else domain_from_input(domain_or_url) if not domain: raise ContextPackError("Could not resolve a domain from policy-pack input.") effective_policy = ensure_policy_for_domain(policy, domain) @@ -84,14 +86,21 @@ def build_policy_pack( input_value=domain_or_url, ) start_url = public_url(domain_or_url if "://" in domain_or_url else homepage_url_for_domain(domain)) - discovery_pages = fetch_pages_blocking([start_url], run=run, max_pages=1) + discovery_pages = ( + local_pages[:max_pages] + if local_pages is not None + else fetch_pages_blocking([start_url], run=run, max_pages=1) + ) if not discovery_pages: - raise ContextPackError(f"Could not fetch policy discovery target: {start_url}") + raise ContextPackError(f"Could not load policy discovery target: {start_url}") - urls = _policy_urls(discovery_pages[0], domain=domain, max_pages=max_pages) - if _policy_target_hint(discovery_pages[0]) and start_url not in urls: - urls.insert(0, start_url) - pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages) if urls else [] + if local_pages is not None: + pages = local_pages[:max_pages] + else: + urls = _policy_urls(discovery_pages[0], domain=domain, max_pages=max_pages) + if _policy_target_hint(discovery_pages[0]) and start_url not in urls: + urls.insert(0, start_url) + pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages) if urls else [] if not pages: run.warn( "policy_documents_not_found", diff --git a/src/docpull/context_packs/product.py b/src/docpull/context_packs/product.py index b5d521f..a135122 100644 --- a/src/docpull/context_packs/product.py +++ b/src/docpull/context_packs/product.py @@ -23,6 +23,7 @@ evidence_for_page, fetch_pages_blocking, homepage_url_for_domain, + pages_from_local_pack, public_url, quote_markdown, same_policy_domain, @@ -42,6 +43,10 @@ re.IGNORECASE, ) TRIAL_RE = re.compile(r"(?P\d{1,3})[-\s]day\s+(?:free\s+)?trial|free\s+trial", re.IGNORECASE) +TRIAL_WEEK_RE = re.compile( + r"(?:try\s+\S+\s+free\s+for\s+(?:7\s+days?|one\s+week)|(?:your\s+)?first\s+week\s+is\s+on\s+us)", + re.IGNORECASE, +) PRODUCT_LINK_KEYWORDS = ("pricing", "product", "products", "plans", "shop", "store") PRICING_CONTEXT_TERMS = ( "pricing", @@ -57,6 +62,13 @@ "buy", "checkout", ) +TESTIMONIAL_TERMS = ( + "testimonial", + "customer quote", + "what our customers say", + "case study", + "review", +) def build_product_pack( @@ -70,7 +82,8 @@ def build_product_pack( """Build cited product/pricing records from a page or bounded site discovery.""" if mode not in {"page", "site"}: raise ContextPackError("product-pack mode must be 'page' or 'site'.") - domain = domain_from_input(url_or_domain) + local_pages = pages_from_local_pack(url_or_domain) + domain = domain_from_input(local_pages[0].url) if local_pages else domain_from_input(url_or_domain) if not domain: raise ContextPackError("Could not resolve a domain from input.") policy = ensure_policy_for_domain(policy, domain) @@ -80,15 +93,24 @@ def build_product_pack( policy=policy, input_value=url_or_domain, ) - start_url = public_url(url_or_domain if "://" in url_or_domain else homepage_url_for_domain(domain)) - urls = [start_url] - if mode == "site": - home = fetch_pages_blocking([start_url], run=run, max_pages=1) - if home: - urls = _site_product_urls(home[0], domain, max_pages=max_pages) - pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages if mode == "site" else 1) - if not pages: - raise ContextPackError(f"Could not fetch product target: {start_url}") + if local_pages is not None: + if mode == "page": + pages = local_pages[:1] + else: + product_pages = [page for page in local_pages if _is_product_context_page(page)] + pages = (product_pages or local_pages[:1])[:max_pages] + if not pages: + raise ContextPackError("Local product-pack input contains no readable documents.") + else: + start_url = public_url(url_or_domain if "://" in url_or_domain else homepage_url_for_domain(domain)) + urls = [start_url] + if mode == "site": + home = fetch_pages_blocking([start_url], run=run, max_pages=1) + if home: + urls = _site_product_urls(home[0], domain, max_pages=max_pages) + pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages if mode == "site" else 1) + if not pages: + raise ContextPackError(f"Could not fetch product target: {start_url}") products: list[dict[str, Any]] = [] pricing_rows: list[dict[str, Any]] = [] @@ -188,7 +210,7 @@ def _extract_products_from_page(page: PageSnapshot, pages: list[PageSnapshot]) - "is_product_page": True, "evidence_status": "heuristic_pricing_evidence", "offers": pricing_rows, - "features": _feature_bullets(page), + "features": _feature_bullets(page, pages), "images": _image_candidates(page), "citations": [ evidence_for_page( @@ -220,7 +242,7 @@ def _jsonld_products(page: PageSnapshot, pages: list[PageSnapshot]) -> list[dict "is_product_page": True, "evidence_status": "jsonld_product", "offers": offers, - "features": _feature_bullets(page), + "features": _feature_bullets(page, pages), "images": _image_candidates(page, item.get("image")), "citations": [ evidence_for_page( @@ -241,16 +263,27 @@ def _offers_from_jsonld(raw: Any, page: PageSnapshot, pages: list[PageSnapshot]) for item in items: if not isinstance(item, dict): continue + serialized = json.dumps(item, ensure_ascii=False) price = _parse_price_value(item.get("price")) + trial = _trial_from_text(serialized) + if trial and trial.get("duration_days") is None: + page_trial = _trial_from_text(page.markdown) + if page_trial: + trial = page_trial + currency = _string_or_none(item.get("priceCurrency")) + if price == 0 and trial and not _explicit_free_plan(serialized): + price = None + currency = None + evidence_excerpt = _structured_offer_excerpt(page, item, trial=trial) offers.append( { "name": _string_or_none(item.get("name")), "price": price, - "currency": _string_or_none(item.get("priceCurrency")), - "billing_frequency": _billing_from_text(json.dumps(item, ensure_ascii=False)), - "billing_interval": _billing_interval(json.dumps(item, ensure_ascii=False)), - "trial": _trial_from_text(json.dumps(item, ensure_ascii=False)), - "feature_gates": _feature_gates(json.dumps(item, ensure_ascii=False)), + "currency": currency, + "billing_frequency": _billing_from_text(serialized), + "billing_interval": _billing_interval(serialized), + "trial": trial, + "feature_gates": _feature_gates(serialized), "price_source": {"medium": "structured_data", "context": "jsonld_offer"}, "availability": _string_or_none(item.get("availability")), "url": public_url(str(item.get("url") or page.url)), @@ -258,7 +291,7 @@ def _offers_from_jsonld(raw: Any, page: PageSnapshot, pages: list[PageSnapshot]) page, pages, field="offer", - excerpt=json.dumps(item)[:240], + excerpt=evidence_excerpt, ).to_dict(), } ) @@ -287,30 +320,37 @@ def _pricing_rows(page: PageSnapshot, pages: list[PageSnapshot]) -> list[dict[st ] seen: set[tuple[str, float | None, str | None]] = set() for block, context in text_blocks: - for match in PRICE_RE.finditer(block): + evidence_block = block if block in page.markdown else _matching_pricing_line(page.markdown, block) + if not evidence_block: + continue + for match in PRICE_RE.finditer(evidence_block): price = _parse_price_value(match.group("amount")) currency = _currency_code(match.group("currency")) - key = (block, price, currency) + trial = _trial_from_text(evidence_block) + if price == 0 and trial and not _explicit_free_plan(evidence_block): + price = None + currency = None + key = (evidence_block, price, currency) if key in seen: continue seen.add(key) rows.append( { - "plan_name": _plan_name_from_text(block), + "plan_name": _plan_name_from_text(evidence_block), "price": price, "currency": currency, - "billing_frequency": _billing_from_text(block) or match.group("period"), - "billing_interval": _billing_interval(block, fallback=match.group("period")), - "trial": _trial_from_text(block), - "feature_gates": _feature_gates(block), - "raw_text": block[:500], + "billing_frequency": _billing_from_text(evidence_block) or match.group("period"), + "billing_interval": _billing_interval(evidence_block, fallback=match.group("period")), + "trial": trial, + "feature_gates": _feature_gates(evidence_block), + "raw_text": evidence_block[:500], "source_url": page.url, "price_source": {"medium": "page_text", "context": context}, "evidence": evidence_for_page( page, pages, field="pricing", - excerpt=block[:500], + excerpt=evidence_block[:500], ).to_dict(), } ) @@ -423,9 +463,9 @@ def _currency_code(value: str | None) -> str | None: def _billing_from_text(text: str) -> str | None: lowered = text.lower() - if any(value in lowered for value in ("/mo", "monthly", "per month")): + if any(value in lowered for value in ("/mo", "monthly", "per month", "a month", "each month")): return "monthly" - if any(value in lowered for value in ("/yr", "yearly", "annually", "per year")): + if any(value in lowered for value in ("/yr", "yearly", "annually", "per year", "a year", "each year")): return "yearly" if "one time" in lowered or "one-time" in lowered: return "one_time" @@ -436,9 +476,9 @@ def _billing_from_text(text: str) -> str | None: def _billing_interval(text: str, *, fallback: str | None = None) -> dict[str, Any] | None: lowered = text.lower() - if any(value in lowered for value in ("/mo", "monthly", "per month")): + if any(value in lowered for value in ("/mo", "monthly", "per month", "a month", "each month")): return {"unit": "month", "count": 1} - if any(value in lowered for value in ("/yr", "yearly", "annually", "per year")): + if any(value in lowered for value in ("/yr", "yearly", "annually", "per year", "a year", "each year")): return {"unit": "year", "count": 1} if any(value in lowered for value in ("per user", "/user")): return {"unit": "user", "count": 1} @@ -451,12 +491,18 @@ def _billing_interval(text: str, *, fallback: str | None = None) -> dict[str, An def _trial_from_text(text: str) -> dict[str, Any] | None: match = TRIAL_RE.search(text) - if not match: + if match: + days = match.groupdict().get("days") + return { + "available": True, + "duration_days": int(days) if days else None, + "requires_payment_method": "unknown", + } + if not TRIAL_WEEK_RE.search(text): return None - days = match.groupdict().get("days") return { "available": True, - "duration_days": int(days) if days else None, + "duration_days": 7, "requires_payment_method": "unknown", } @@ -480,9 +526,61 @@ def _feature_gates(text: str) -> list[dict[str, str]]: def _looks_like_pricing_text(text: str) -> bool: lowered = text.lower() + if "data:image" in lowered or lowered.startswith("[![]("): + return False return any(term in lowered for term in PRICING_CONTEXT_TERMS) +def _is_product_context_page(page: PageSnapshot) -> bool: + role = str(page.metadata.get("page_role") or "").lower() + if role in {"home", "product", "pricing"}: + return True + path = (urlparse(page.url).path or "/").lower() + if path == "/": + return True + return any(keyword in path for keyword in (*PRODUCT_LINK_KEYWORDS, "feature", "solution", "platform")) + + +def _structured_offer_excerpt( + page: PageSnapshot, + item: dict[str, Any], + *, + trial: dict[str, Any] | None, +) -> str: + lines = [line.strip() for line in page.markdown.splitlines() if line.strip()] + if trial: + for terms in (("trial",), ("free", "7 days"), ("first week",)): + match = next((line for line in lines if all(term in line.lower() for term in terms)), None) + if match: + return match + price = _string_or_none(item.get("price")) + if price: + match = next((line for line in lines if price in line and _looks_like_pricing_text(line)), None) + if match: + return match + name = _string_or_none(item.get("name")) + if name: + match = next((line for line in lines if name.lower() in line.lower()), None) + if match: + return match + return page.markdown[: min(240, len(page.markdown))] + + +def _matching_pricing_line(markdown: str, html_block: str) -> str | None: + prices = {match.group(0).strip().casefold() for match in PRICE_RE.finditer(html_block)} + for line in markdown.splitlines(): + cleaned = line.strip() + lowered = cleaned.casefold() + if cleaned and _looks_like_pricing_text(cleaned) and any(price in lowered for price in prices): + return cleaned + return None + + +def _explicit_free_plan(text: str) -> bool: + lowered = text.lower() + return any(term in lowered for term in ("free plan", "free tier", "free forever")) + + def _plan_name_from_text(text: str) -> str | None: before_price = PRICE_RE.split(text, maxsplit=1)[0].strip(" :-|") if before_price: @@ -491,22 +589,52 @@ def _plan_name_from_text(text: str) -> str | None: return None -def _feature_bullets(page: PageSnapshot) -> list[dict[str, str]]: +def _feature_bullets(page: PageSnapshot, pages: list[PageSnapshot]) -> list[dict[str, Any]]: soup = soup_for(page) - features: list[dict[str, str]] = [] + features: list[dict[str, Any]] = [] seen: set[str] = set() for tag in soup.find_all(["li", "p"]): text = " ".join(tag.get_text(" ").split()) if len(text) < 8 or len(text) > 220 or text in seen: continue + if text not in page.markdown: + continue + if _is_testimonial_element(tag, text): + continue if any(keyword in text.lower() for keyword in ("feature", "include", "support", "unlimited", "api")): - features.append({"text": text, "source_url": page.url}) + features.append( + { + "text": text, + "source_url": page.url, + "evidence": evidence_for_page( + page, + pages, + field="feature", + excerpt=text, + ).to_dict(), + } + ) seen.add(text) if len(features) >= 30: break return features +def _is_testimonial_element(tag: Tag, text: str) -> bool: + if tag.find_parent("blockquote") is not None: + return True + for candidate in [tag, *tag.parents]: + if not isinstance(candidate, Tag): + continue + descriptor = " ".join( + [str(candidate.get("id") or ""), *[str(item) for item in candidate.get("class") or []]] + ).lower() + if any(term.replace(" ", "-") in descriptor or term in descriptor for term in TESTIMONIAL_TERMS): + return True + lowered = text.lower() + return any(term in lowered for term in TESTIMONIAL_TERMS) + + def _image_candidates(page: PageSnapshot, raw_image: Any = None) -> list[dict[str, str]]: soup = soup_for(page) urls: list[str] = [] diff --git a/src/docpull/context_packs/styleguide.py b/src/docpull/context_packs/styleguide.py index 8ab7287..ba50d66 100644 --- a/src/docpull/context_packs/styleguide.py +++ b/src/docpull/context_packs/styleguide.py @@ -24,6 +24,7 @@ fetch_context_asset_bytes, fetch_pages_blocking, homepage_url_for_domain, + pages_from_local_pack, public_url, quote_markdown, resolve_url, @@ -63,7 +64,8 @@ def build_styleguide_pack( "styleguide-pack rendering requires DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1. " "Run browser-free with render=False, or explicitly trust the target." ) - domain = domain_from_input(domain_or_url) + local_pages = pages_from_local_pack(domain_or_url) + domain = domain_from_input(local_pages[0].url) if local_pages else domain_from_input(domain_or_url) if not domain: raise ContextPackError("Could not resolve a domain from input.") policy = ensure_policy_for_domain(policy, domain) @@ -75,16 +77,21 @@ def build_styleguide_pack( input_value=domain_or_url, ) start_url = public_url(domain_or_url if "://" in domain_or_url else homepage_url_for_domain(domain)) - pages = fetch_pages_blocking([start_url], run=run, max_pages=1) + pages = ( + local_pages[:1] + if local_pages is not None + else fetch_pages_blocking([start_url], run=run, max_pages=1) + ) if not pages: - raise ContextPackError(f"Could not fetch styleguide target: {start_url}") + raise ContextPackError(f"Could not load styleguide target: {start_url}") page = pages[0] css_sources = _collect_css_sources(page, domain, max_stylesheets=max_stylesheets) css_texts: list[dict[str, str]] = [] - for source in css_sources: - text = _fetch_css_blocking(source["url"], domain=domain, run=run) - if text: - css_texts.append({"url": source["url"], "text": text, "source": source["source"]}) + if local_pages is None: + for source in css_sources: + text = _fetch_css_blocking(source["url"], domain=domain, run=run) + if text: + css_texts.append({"url": source["url"], "text": text, "source": source["source"]}) inline_css = _inline_css(page) css_texts.extend({"url": page.url, "text": text, "source": "inline"} for text in inline_css) diff --git a/src/docpull/context_packs/visuals.py b/src/docpull/context_packs/visuals.py index 11535d8..7b2a3dd 100644 --- a/src/docpull/context_packs/visuals.py +++ b/src/docpull/context_packs/visuals.py @@ -58,6 +58,7 @@ def build_image_pack( ) -> dict[str, Any]: """Extract a local image manifest from a URL or existing pack.""" output_dir = output_dir.resolve() + local_pack_mode = Path(url_or_pack).expanduser().exists() pages, run = _image_pages_from_input(url_or_pack, policy=policy, output_dir=output_dir) if not pages: raise ContextPackError("No image-pack evidence pages found.") @@ -67,7 +68,7 @@ def build_image_pack( assets: list[AssetRef] = [] for candidate in candidates[:max_assets]: asset = AssetRef(candidate["url"], candidate["kind"], candidate["source_url"]) - if download_assets and domain: + if download_assets and domain and not local_pack_mode: asset = fetch_asset_blocking( candidate["url"], output_dir=output_dir / "assets" / "images", @@ -96,7 +97,10 @@ def build_image_pack( "workflow": IMAGE_WORKFLOW, "provider": "local", "status": status_from_errors(run.errors), - "input": {"value": str(url_or_pack), "download_assets": download_assets}, + "input": { + "value": str(url_or_pack), + "download_assets": download_assets and not local_pack_mode, + }, "summary": { "candidate_count": len(candidates), "asset_count": len(image_records), diff --git a/src/docpull/context_packs/website.py b/src/docpull/context_packs/website.py new file mode 100644 index 0000000..ef7384d --- /dev/null +++ b/src/docpull/context_packs/website.py @@ -0,0 +1,1238 @@ +"""Portable, baseline-aware website snapshots for downstream intelligence systems.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlparse + +from ..contracts import ( + ArtifactEntry, + ArtifactManifest, + ArtifactReference, + BudgetUsage, + HashDigest, + ReplayConfiguration, + WebsiteSnapshot, + WebsiteSnapshotBaseline, + WebsiteSnapshotDocument, + WebsiteSnapshotManifests, + WebsiteSnapshotOptions, + WorkflowFailure, + WorkflowProgressEvent, + WorkflowRequest, + WorkflowResult, + WorkflowWarning, + bundled_schema_path, + canonical_sha256, + file_sha256, + stable_id, + verify_website_snapshot_hash, +) +from ..evidence import classify_source_authority +from ..policy import PolicyConfig +from ..time_utils import utc_now_iso +from .common import ( + ContextPackError, + ContextPackRun, + PageSnapshot, + asset_allowed_domains_for_domain, + domain_from_input, + ensure_policy_for_domain, + extract_links, + fetch_asset_blocking, + fetch_pages_blocking, + homepage_url_for_domain, + public_url, + same_policy_domain, + soup_for, + write_json, +) + +WEBSITE_WORKFLOW = "website-pack" +WEBSITE_SNAPSHOT_ENTRYPOINT = "website.snapshot.v1.json" +WEBSITE_SCHEMA_FILENAME = "website-snapshot.v1.schema.json" +DEFAULT_WEBSITE_OUTPUT_DIR = Path("packs/websites") +MAX_VISUALS = 8 +KEY_VISUAL_ROLES = {"home", "product", "pricing", "trust"} +IMAGE_MEDIA_TYPES = {"image/png", "image/jpeg", "image/gif", "image/webp", "image/svg+xml", "image/x-icon"} + + +def build_website_pack( + url_or_domain: str, + *, + output_dir: Path = DEFAULT_WEBSITE_OUTPUT_DIR, + request: WorkflowRequest, + max_pages: int = 50, + max_depth: int = 3, + raw_html: bool = True, + key_page_visuals: bool = True, + render_fallback: bool = True, + pdf_enabled: bool = False, + baseline_pack: Path | None = None, + baseline_snapshot_id: str | None = None, + baseline_snapshot_hash: str | None = None, + entity_reference: str | None = None, +) -> dict[str, Any]: + """Acquire a bounded website and emit one recursively verifiable portable-v3 pack.""" + + domain = domain_from_input(url_or_domain) + if not domain: + raise ContextPackError("Could not resolve a domain from website-pack input.") + policy = ensure_policy_for_domain(_request_policy(request), domain) + output = output_dir.resolve() + output.mkdir(parents=True, exist_ok=True) + run = ContextPackRun( + workflow=WEBSITE_WORKFLOW, + output_dir=output, + policy=policy, + input_value=url_or_domain, + ) + entity = (entity_reference or domain).strip() + captured_at = utc_now_iso() + options = WebsiteSnapshotOptions( + max_pages=max_pages, + max_depth=max_depth, + raw_html=raw_html, + key_page_visuals=key_page_visuals, + render_fallback=render_fallback, + pdf_enabled=pdf_enabled, + ) + if pdf_enabled: + run.warn( + "pdf_capture_unavailable", + "PDF extraction was requested but is not part of the website.snapshot.v1 producer lane.", + ) + + baseline, baseline_payload = _load_verified_baseline( + baseline_pack, + expected_snapshot_id=baseline_snapshot_id, + expected_snapshot_hash=baseline_snapshot_hash, + ) + _prepare_conditional_cache(baseline_pack, output) + run.fetch_output_dir = output / "acquired" + run.cache_dir = output / "http-cache" + start_url = public_url(url_or_domain if "://" in url_or_domain else homepage_url_for_domain(domain)) + baseline_urls = ( + [item.url for item in baseline.documents if item.state in {"added", "changed", "unchanged"}] + if baseline + else [] + ) + pages, failed_urls = _crawl_website( + start_url, + domain=domain, + run=run, + max_pages=max_pages, + max_depth=max_depth, + seed_urls=baseline_urls, + ) + baseline_by_id = {item.document_id: item for item in baseline.documents} if baseline else {} + + document_rows: list[dict[str, Any]] = [] + snapshot_documents: list[WebsiteSnapshotDocument] = [] + produced_paths: set[str] = set() + seen_document_ids: set[str] = set() + visual_count = 0 + downloaded_assets: set[str] = set() + + for page in pages: + canonical_url = _canonical_url(page.url) + document_id = stable_id("doc", {"canonical_url": canonical_url}) + seen_document_ids.add(document_id) + content = _page_content(page) + content_hash = _sha256_text(content) + document_version = content_hash + previous = baseline_by_id.get(document_id) + state = ( + "unchanged" + if previous and previous.content_hash == content_hash + else ("changed" if previous else "added") + ) + page_role = _page_role(page) + authority = classify_source_authority( + canonical_url, + official_domain=domain, + declared_role=_authority_role(page_role), + ) + okf_path = output / "okf" / f"{document_id}.md" + okf_path.parent.mkdir(parents=True, exist_ok=True) + okf_path.write_text( + _okf_document( + page=page, + content=content, + document_id=document_id, + document_version=document_version, + content_hash=content_hash, + entity_reference=entity, + page_role=page_role, + authority=authority.model_dump(mode="json"), + ), + encoding="utf-8", + ) + okf_ref = _reference(output, okf_path, media_type="text/markdown") + produced_paths.add(okf_ref.path) + + raw_ref: ArtifactReference | None = None + if raw_html: + raw_path = output / "raw" / f"{document_id}.html" + raw_path.parent.mkdir(parents=True, exist_ok=True) + raw_path.write_text(page.html, encoding="utf-8") + raw_ref = _reference(output, raw_path, media_type="text/html") + produced_paths.add(raw_ref.path) + + brand_assets = _capture_brand_assets( + page, + output=output, + domain=domain, + run=run, + downloaded=downloaded_assets, + enabled=page_role in KEY_VISUAL_ROLES, + ) + produced_paths.update(item.path for item in brand_assets) + + screenshot_ref: ArtifactReference | None = None + if key_page_visuals and page_role in KEY_VISUAL_ROLES and visual_count < MAX_VISUALS: + screenshot_ref = _capture_screenshot( + page, + output=output, + document_id=document_id, + run=run, + ) + if screenshot_ref: + visual_count += 1 + produced_paths.add(screenshot_ref.path) + + document = WebsiteSnapshotDocument( + document_id=document_id, + document_version=document_version, + content_hash=content_hash, + url=page.url, + canonical_url=canonical_url, + title=page.title, + entity_reference=entity, + authority=authority, + page_role=page_role, # type: ignore[arg-type] + state=state, # type: ignore[arg-type] + fetched_at=captured_at, + okf=okf_ref, + raw=raw_ref, + screenshot=screenshot_ref, + brand_assets=brand_assets, + ) + snapshot_documents.append(document) + document_rows.append( + { + "schema_version": 3, + "document_id": document_id, + "document_version": document_version, + "url": page.url, + "canonical_url": canonical_url, + "title": page.title, + "content": content, + "content_hash": content_hash, + "content_type": "text/markdown", + "mime_type": "text/markdown", + "fetched_at": captured_at, + "source_type": WEBSITE_WORKFLOW, + "entity_reference": entity, + "authority": authority.model_dump(mode="json"), + "page_role": page_role, + "representations": { + "okf": okf_ref.model_dump(mode="json", exclude_none=True), + "raw": raw_ref.model_dump(mode="json", exclude_none=True) if raw_ref else None, + "screenshot": ( + screenshot_ref.model_dump(mode="json", exclude_none=True) if screenshot_ref else None + ), + "brand_assets": [ + item.model_dump(mode="json", exclude_none=True) for item in brand_assets + ], + }, + "metadata": {"website_snapshot_state": state}, + "extraction": {"workflow": WEBSITE_WORKFLOW, "page_role": page_role}, + "route": {"name": "docpull.fetch"}, + "rights": {"status": "unknown"}, + } + ) + + failure_documents = _failure_documents( + failed_urls, + entity_reference=entity, + domain=domain, + captured_at=captured_at, + ) + snapshot_documents.extend(failure_documents) + seen_document_ids.update(item.document_id for item in failure_documents) + if baseline and baseline_pack: + baseline_rows = _baseline_document_rows(baseline_pack) + for previous in baseline.documents: + if ( + previous.document_id in seen_document_ids + or _canonical_url(previous.url) not in run.unchanged_urls + or previous.state not in {"added", "changed", "unchanged"} + ): + continue + carried = _carry_baseline_document( + previous, + baseline_root=baseline_pack.resolve(), + output=output, + captured_at=captured_at, + ) + snapshot_documents.append(carried) + seen_document_ids.add(carried.document_id) + for reference in [ + carried.okf, + carried.raw, + carried.screenshot, + *carried.brand_assets, + ]: + if reference is not None: + produced_paths.add(reference.path) + row = baseline_rows.get((carried.document_id, carried.document_version)) + if row: + raw_metadata = row.get("metadata") + metadata: dict[str, Any] = ( + {str(key): value for key, value in raw_metadata.items()} + if isinstance(raw_metadata, dict) + else {} + ) + document_rows.append( + { + **row, + "fetched_at": captured_at, + "metadata": { + **metadata, + "website_snapshot_state": "unchanged", + "conditional_reuse": True, + }, + } + ) + if carried.screenshot: + visual_count += 1 + if baseline: + for previous in baseline.documents: + if previous.document_id in seen_document_ids: + continue + snapshot_documents.append( + previous.model_copy( + update={ + "state": "removed", + "okf": None, + "raw": None, + "screenshot": None, + "brand_assets": [], + "warnings": [*previous.warnings, "Not present in the current bounded crawl."], + "failure": None, + } + ) + ) + + documents_path = output / "documents.ndjson" + documents_path.write_text( + "".join(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n" for item in document_rows), + encoding="utf-8", + ) + produced_paths.add("documents.ndjson") + for generated_dir in (output / "acquired", output / "http-cache"): + if generated_dir.is_dir(): + produced_paths.update( + str(path.relative_to(output)) for path in generated_dir.rglob("*") if path.is_file() + ) + + ordered_documents = sorted(snapshot_documents, key=lambda item: (item.url, item.document_id)) + okf_index_path = output / "okf" / "index.md" + okf_index_path.parent.mkdir(parents=True, exist_ok=True) + okf_index_path.write_text(_okf_index(ordered_documents), encoding="utf-8") + produced_paths.add("okf/index.md") + coverage = _coverage_payload(ordered_documents, run) + corpus_path = output / "corpus.manifest.json" + current_path = output / "current-run.manifest.json" + coverage_path = output / "coverage.manifest.json" + provenance_path = output / "provenance.manifest.json" + write_json(corpus_path, _corpus_manifest(ordered_documents, captured_at)) + write_json(current_path, _current_run_manifest(ordered_documents, request, captured_at)) + write_json(coverage_path, coverage) + write_json(provenance_path, _provenance_manifest(ordered_documents, entity, captured_at)) + source_policy_path = output / "source_policy.json" + write_json( + source_policy_path, + policy.to_source_policy_payload( + source=WEBSITE_WORKFLOW, + url=start_url, + metadata={"entity_reference": entity}, + ), + ) + accounting_path = output / "run.accounting.json" + write_json( + accounting_path, + { + "schema_version": 1, + "generated_at": captured_at, + "budget_limit_usd": policy.budget.maximum_paid_cost_usd, + "estimated_paid_cost_usd": 0.0, + "actual_paid_cost_usd": 0.0, + "paid_request_count": 0, + "http_request_count": run.http_request_count, + "cache_hit_count": run.cache_hit_count, + "local_browser_seconds": 0.0, + "blocked_actions": [], + }, + ) + produced_paths.update( + { + "corpus.manifest.json", + "current-run.manifest.json", + "coverage.manifest.json", + "provenance.manifest.json", + "source_policy.json", + "run.accounting.json", + } + ) + + schema_path = bundled_schema_path(WEBSITE_SCHEMA_FILENAME) + schema_sha256 = file_sha256(schema_path) + pack_id = stable_id( + "pack", + {"request_id": request.request_id, "entity_reference": entity, "captured_at": captured_at}, + ) + run_id = stable_id( + "run", + {"request_id": request.request_id, "pack_id": pack_id, "captured_at": captured_at}, + ) + baseline_contract = ( + WebsiteSnapshotBaseline( + snapshot_id=baseline.snapshot_id, + snapshot_hash=baseline.snapshot_hash, + verified=True, + ) + if baseline + else None + ) + manifests = WebsiteSnapshotManifests( + corpus=_reference(output, corpus_path, media_type="application/json"), + current_run=_reference(output, current_path, media_type="application/json"), + coverage=_reference(output, coverage_path, media_type="application/json"), + provenance=_reference(output, provenance_path, media_type="application/json"), + documents=_reference(output, documents_path, media_type="application/x-ndjson"), + ) + warnings = _warning_models(run.warnings) + snapshot_payload: dict[str, Any] = { + "contract_version": "website.snapshot.v1", + "schema_version": 1, + "schema_sha256": schema_sha256, + "pack_identity": { + "pack_id": pack_id, + "format": "portable-v3", + "workflow": WEBSITE_WORKFLOW, + }, + "run_identity": { + "run_id": run_id, + "request_id": request.request_id, + "scheduler": None, + }, + "entity_reference": entity, + "captured_at": captured_at, + "baseline": baseline_contract.model_dump(mode="json") if baseline_contract else None, + "options": options.model_dump(mode="json"), + "documents": [item.model_dump(mode="json", exclude_none=True) for item in ordered_documents], + "coverage": coverage, + "manifests": manifests.model_dump(mode="json", exclude_none=True), + "visual_count": visual_count, + "warnings": [item.model_dump(mode="json") for item in warnings], + } + snapshot_hash = canonical_sha256(snapshot_payload) + snapshot = WebsiteSnapshot( + snapshot_id=stable_id("snapshot", {"snapshot_hash": snapshot_hash}), + snapshot_hash=snapshot_hash, + **snapshot_payload, + ) + if not verify_website_snapshot_hash(snapshot): + raise ContextPackError("Internal website snapshot hash verification failed.") + entrypoint_path = output / WEBSITE_SNAPSHOT_ENTRYPOINT + serialized_snapshot = snapshot.model_dump(mode="json", exclude_none=True) + serialized_snapshot.setdefault("baseline", None) + write_json(entrypoint_path, serialized_snapshot) + produced_paths.add(WEBSITE_SNAPSHOT_ENTRYPOINT) + + manifest = _write_artifact_manifest( + output, + produced_paths, + pack_id=pack_id, + run_id=run_id, + ) + _write_workflow_result( + output=output, + request=request, + run=run, + snapshot=snapshot, + artifact_manifest=manifest, + baseline_payload=baseline_payload, + ) + return serialized_snapshot + + +def validate_website_snapshot_pack(pack_dir: Path | str) -> WebsiteSnapshot: + """Verify a website pack recursively while ignoring files outside its current manifest.""" + + root = Path(pack_dir).expanduser().resolve() + entrypoint = root / WEBSITE_SNAPSHOT_ENTRYPOINT + manifest_path = root / "artifact.manifest.json" + if not entrypoint.is_file() or not manifest_path.is_file(): + raise ContextPackError("Website pack is missing its entrypoint or artifact manifest.") + try: + snapshot = WebsiteSnapshot.model_validate_json(entrypoint.read_text(encoding="utf-8")) + manifest = ArtifactManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + except (ValueError, OSError) as err: + raise ContextPackError(f"Website pack contract is invalid: {err}") from err + if not verify_website_snapshot_hash(snapshot): + raise ContextPackError("Website snapshot hash verification failed.") + entries_by_path = {entry.path: entry for entry in manifest.entries} + if len(entries_by_path) != len(manifest.entries): + raise ContextPackError("Artifact manifest contains duplicate paths.") + for entry in manifest.entries: + path = _safe_pack_path(root, entry.path) + if not path.is_file(): + raise ContextPackError(f"Artifact is missing: {entry.path}") + if path.stat().st_size != entry.bytes or file_sha256(path) != entry.sha256: + raise ContextPackError(f"Artifact verification failed: {entry.path}") + aggregate = canonical_sha256( + [entry.model_dump(mode="json", exclude_none=True) for entry in manifest.entries] + ) + if aggregate != manifest.aggregate_sha256: + raise ContextPackError("Artifact manifest aggregate hash verification failed.") + if manifest.pack_id != str(snapshot.pack_identity.get("pack_id")): + raise ContextPackError("Artifact manifest pack identity does not match the snapshot.") + if manifest.run_id != str(snapshot.run_identity.get("run_id")): + raise ContextPackError("Artifact manifest run identity does not match the snapshot.") + references = [ + snapshot.manifests.corpus, + snapshot.manifests.current_run, + snapshot.manifests.coverage, + snapshot.manifests.provenance, + snapshot.manifests.documents, + *[ + reference + for document in snapshot.documents + for reference in [ + document.okf, + document.raw, + document.screenshot, + *document.brand_assets, + ] + if reference is not None + ], + ] + for reference in references: + manifest_entry = entries_by_path.get(reference.path) + if manifest_entry is None or manifest_entry.sha256 != reference.sha256: + raise ContextPackError(f"Snapshot reference is not pinned by the manifest: {reference.path}") + return snapshot + + +def _crawl_website( + start_url: str, + *, + domain: str, + run: ContextPackRun, + max_pages: int, + max_depth: int, + seed_urls: list[str] | None = None, +) -> tuple[list[PageSnapshot], list[dict[str, Any]]]: + pages: list[PageSnapshot] = [] + failures: list[dict[str, Any]] = [] + initial = list(dict.fromkeys([start_url, *(seed_urls or [])]))[:max_pages] + queued = set(initial) + visited: set[str] = set() + frontier = initial + depth = 0 + while frontier and len(visited) < max_pages and depth <= max_depth: + batch = [url for url in frontier if url not in visited][: max_pages - len(visited)] + frontier = [] + before_errors = len(run.errors) + acquired = fetch_pages_blocking(batch, run=run, max_pages=len(batch)) if batch else [] + visited.update(batch) + pages.extend(acquired) + failures.extend(run.errors[before_errors:]) + if depth >= max_depth: + break + candidates: list[tuple[int, str]] = [] + for page in acquired: + for link in extract_links(page): + url = _canonical_url(link["url"]) + if url in visited or url in queued or not same_policy_domain(url, domain): + continue + score = _link_score(url, link.get("text", "")) + candidates.append((score, url)) + queued.add(url) + candidates.sort(key=lambda item: (-item[0], item[1])) + frontier = [url for _, url in candidates[: max_pages - len(visited)]] + depth += 1 + unique_pages: dict[str, PageSnapshot] = {} + for page in pages: + unique_pages.setdefault(_canonical_url(page.url), page) + return list(unique_pages.values()), failures + + +def _link_score(url: str, text: str) -> int: + value = f"{urlparse(url).path} {text}".lower() + weighted = { + "pricing": 12, + "product": 10, + "solution": 9, + "docs": 8, + "documentation": 8, + "security": 8, + "trust": 8, + "changelog": 8, + "release": 7, + "privacy": 7, + "terms": 7, + "support": 6, + "about": 5, + } + return sum(score for keyword, score in weighted.items() if keyword in value) + + +def _page_role(page: PageSnapshot) -> str: + path = (urlparse(page.url).path or "/").lower() + title = (page.title or "").lower() + value = f"{path} {title}" + if path in {"", "/"}: + return "home" + rules = ( + ("pricing", ("pricing", "plans", "billing")), + ("documentation", ("/docs", "documentation", "api reference", "developer")), + ("trust", ("security", "trust", "compliance", "soc 2", "subprocessor")), + ("legal", ("privacy", "terms", "legal", "cookie", "dpa")), + ("changelog", ("changelog", "release-notes", "release notes", "updates")), + ("support", ("support", "help", "contact")), + ("product", ("product", "products", "solution", "features", "platform")), + ) + for role, keywords in rules: + if any(keyword in value for keyword in keywords): + return role + return "other" + + +def _authority_role(page_role: str) -> str: + if page_role == "legal" or page_role == "trust": + return "legal" + if page_role == "documentation": + return "documentation" + if page_role == "home": + return "official_corporate" + return "official_product" + + +def _page_content(page: PageSnapshot) -> str: + content = re.sub( + r'^crawled_at:\s*"[^"\r\n]*"\r?\n', + "", + page.markdown, + flags=re.MULTILINE, + ).strip() + if not content: + content = soup_for(page).get_text("\n").strip() + return content + ("\n" if content else "") + + +def _okf_document( + *, + page: PageSnapshot, + content: str, + document_id: str, + document_version: str, + content_hash: str, + entity_reference: str, + page_role: str, + authority: dict[str, Any], +) -> str: + metadata = { + "type": "Web Page", + "title": page.title or page.url, + "source": page.url, + "document_id": document_id, + "document_version": document_version, + "content_hash": content_hash, + "entity_reference": entity_reference, + "page_role": page_role, + "authority": authority, + } + frontmatter = json.dumps(metadata, ensure_ascii=False, sort_keys=True) + return f"---\njson: {frontmatter}\n---\n\n{content}" + + +def _okf_index(documents: list[WebsiteSnapshotDocument]) -> str: + lines = ["# Website concepts", ""] + for document in documents: + if document.okf is None: + continue + lines.append( + f"- [{document.title or document.url}]({Path(document.okf.path).name}) " + f"— `{document.document_id}@{document.document_version}` ({document.page_role}, " + f"{document.state})" + ) + return "\n".join(lines).rstrip() + "\n" + + +def _prepare_conditional_cache(baseline_pack: Path | None, output: Path) -> None: + if baseline_pack is None: + return + baseline_root = baseline_pack.resolve() + for name in ("acquired", "http-cache"): + source = baseline_root / name + if source.is_dir(): + shutil.copytree(source, output / name, dirs_exist_ok=True) + manifest_path = output / "http-cache" / "manifest.json" + if not manifest_path.is_file(): + return + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as err: + raise ContextPackError(f"Baseline conditional cache manifest is invalid: {err}") from err + if not isinstance(manifest, dict): + raise ContextPackError("Baseline conditional cache manifest must be an object.") + for entry in manifest.values(): + if not isinstance(entry, dict) or not isinstance(entry.get("file_path"), str): + continue + previous_path = Path(entry["file_path"]) + try: + acquired_index = previous_path.parts.index("acquired") + except ValueError: + entry.pop("file_path", None) + continue + relative = Path(*previous_path.parts[acquired_index + 1 :]) + current_path = (output / "acquired" / relative).resolve() + if current_path.is_file(): + entry["file_path"] = str(current_path) + else: + entry.pop("file_path", None) + write_json(manifest_path, manifest) + + +def _baseline_document_rows(root: Path) -> dict[tuple[str, str], dict[str, Any]]: + rows: dict[tuple[str, str], dict[str, Any]] = {} + path = root.resolve() / "documents.ndjson" + if not path.is_file(): + return rows + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as err: + raise ContextPackError(f"Baseline documents.ndjson is invalid: {err}") from err + if not isinstance(item, dict): + raise ContextPackError("Baseline documents.ndjson must contain objects.") + document_id = item.get("document_id") + document_version = item.get("document_version") or item.get("content_hash") + if isinstance(document_id, str) and isinstance(document_version, str): + rows[(document_id, document_version)] = item + return rows + + +def _carry_baseline_document( + document: WebsiteSnapshotDocument, + *, + baseline_root: Path, + output: Path, + captured_at: str, +) -> WebsiteSnapshotDocument: + references = [document.okf, document.raw, document.screenshot, *document.brand_assets] + for reference in references: + if reference is None: + continue + source = _safe_pack_path(baseline_root, reference.path) + destination = _safe_pack_path(output, reference.path) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + if file_sha256(destination) != reference.sha256: + raise ContextPackError(f"Carried baseline representation changed: {reference.path}") + return document.model_copy( + update={ + "state": "unchanged", + "fetched_at": captured_at, + "warnings": [*document.warnings, "Reused after conditional HTTP validation."], + "failure": None, + } + ) + + +def _candidate_asset_urls(page: PageSnapshot) -> list[str]: + soup = soup_for(page) + candidates: list[str] = [] + for tag in soup.find_all("link"): + rel = " ".join(str(item).lower() for item in (tag.get("rel") or [])) + href = str(tag.get("href") or "").strip() + if href and ("icon" in rel or "logo" in rel): + candidates.append(public_url(urljoin(page.url, href))) + for tag in soup.find_all("meta"): + name = str(tag.get("property") or tag.get("name") or "").lower() + content = str(tag.get("content") or "").strip() + if content and name in {"og:image", "twitter:image"}: + candidates.append(public_url(urljoin(page.url, content))) + for tag in soup.find_all("img")[:20]: + src = str(tag.get("src") or "").strip() + descriptor = f"{tag.get('alt') or ''} {tag.get('class') or ''}".lower() + if src and ("logo" in descriptor or "brand" in descriptor): + candidates.append(public_url(urljoin(page.url, src))) + return list(dict.fromkeys(candidates)) + + +def _capture_brand_assets( + page: PageSnapshot, + *, + output: Path, + domain: str, + run: ContextPackRun, + downloaded: set[str], + enabled: bool, +) -> list[ArtifactReference]: + if not enabled: + return [] + references: list[ArtifactReference] = [] + allowed_domains = asset_allowed_domains_for_domain(domain) + for url in _candidate_asset_urls(page): + if url in downloaded or len(downloaded) >= 12: + continue + downloaded.add(url) + asset = fetch_asset_blocking( + url, + output_dir=output / "brand-assets", + source_url=page.url, + kind="brand_asset", + allowed_domains=allowed_domains, + allowed_content_types=IMAGE_MEDIA_TYPES, + run=run, + ) + if asset.status == "downloaded" and asset.path and asset.sha256: + references.append( + ArtifactReference( + path=asset.path, + sha256=asset.sha256, + bytes=asset.bytes, + media_type=asset.content_type, + ) + ) + elif asset.warning: + run.warn( + "brand_asset_skipped", + f"Could not include a selected brand asset from {page.url}.", + reason=asset.warning, + ) + return references + + +def _capture_screenshot( + page: PageSnapshot, + *, + output: Path, + document_id: str, + run: ContextPackRun, +) -> ArtifactReference | None: + if os.environ.get("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS") != "1": + if not any(item.get("code") == "browser_capability_unavailable" for item in run.warnings): + run.warn( + "browser_capability_unavailable", + "Key-page visuals were requested, but trusted browser rendering is not enabled.", + ) + return None + from .visuals import _run_screenshot_command + + path = output / "screenshots" / f"{document_id}.png" + path.parent.mkdir(parents=True, exist_ok=True) + try: + _run_screenshot_command( + binary=os.environ.get("DOCPULL_AGENT_BROWSER_BIN", "agent-browser"), + url=page.url, + output_path=path, + viewport="1440x1000", + full_page=True, + wait_for="load", + ) + except (ContextPackError, OSError) as err: + run.warn( + "browser_capture_failed", + f"Could not capture key-page visual for {page.url}.", + reason=str(err)[:300], + ) + return None + return _reference(output, path, media_type="image/png") + + +def _failure_documents( + failures: list[dict[str, Any]], + *, + entity_reference: str, + domain: str, + captured_at: str, +) -> list[WebsiteSnapshotDocument]: + output: list[WebsiteSnapshotDocument] = [] + seen: set[str] = set() + empty_hash = _sha256_text("") + for item in failures: + raw_url = item.get("url") or item.get("source_url") + if not isinstance(raw_url, str) or not raw_url or raw_url in seen: + continue + seen.add(raw_url) + url = _canonical_url(raw_url) + blocked = bool(item.get("blocked")) + failure = WorkflowFailure( + code=str(item.get("code") or "page_acquisition_failed"), + message=str(item.get("error") or item.get("message") or "Page acquisition failed"), + stage=str(item.get("stage") or "fetch"), + retryable=not blocked and (item.get("http_status") in {408, 425, 429, 500, 502, 503, 504}), + source_url=url, + http_status=item.get("http_status") if isinstance(item.get("http_status"), int) else None, + attempts=item.get("attempts") if isinstance(item.get("attempts"), int) else None, + retry_after_seconds=( + float(item["retry_after_seconds"]) + if isinstance(item.get("retry_after_seconds"), (int, float)) + else None + ), + ) + role = _page_role(PageSnapshot(url, None, "", "", {}, {})) + output.append( + WebsiteSnapshotDocument( + document_id=stable_id("doc", {"canonical_url": url}), + document_version=empty_hash, + content_hash=empty_hash, + url=url, + canonical_url=url, + entity_reference=entity_reference, + authority=classify_source_authority( + url, + official_domain=domain, + declared_role=_authority_role(role), + ), + page_role=role, # type: ignore[arg-type] + state="blocked" if blocked else "failed", + fetched_at=captured_at, + failure=failure, + ) + ) + return output + + +def _coverage_payload( + documents: list[WebsiteSnapshotDocument], + run: ContextPackRun, +) -> dict[str, Any]: + states: dict[str, int] = {} + roles: dict[str, int] = {} + for document in documents: + states[document.state] = states.get(document.state, 0) + 1 + roles[document.page_role] = roles.get(document.page_role, 0) + 1 + return { + "schema_version": 1, + "status": ( + "usable" + if any(item.state in {"added", "changed", "unchanged"} for item in documents) + else "failed" + ), + "document_count": len(documents), + "active_document_count": sum(item.state in {"added", "changed", "unchanged"} for item in documents), + "states": dict(sorted(states.items())), + "page_roles": dict(sorted(roles.items())), + "failure_count": len(run.errors), + "warning_count": len(run.warnings), + } + + +def _corpus_manifest( + documents: list[WebsiteSnapshotDocument], + captured_at: str, +) -> dict[str, Any]: + return { + "schema_version": 3, + "generated_at": captured_at, + "output_format": "ndjson", + "document_count": len(documents), + "record_count": sum(item.state in {"added", "changed", "unchanged"} for item in documents), + "records": [ + { + "document_id": item.document_id, + "document_version": item.document_version, + "url": item.url, + "content_hash": item.content_hash, + "state": item.state, + "page_role": item.page_role, + "output_path": "documents.ndjson" + if item.state in {"added", "changed", "unchanged"} + else None, + } + for item in documents + ], + } + + +def _current_run_manifest( + documents: list[WebsiteSnapshotDocument], + request: WorkflowRequest, + captured_at: str, +) -> dict[str, Any]: + return { + "schema_version": 1, + "generated_at": captured_at, + "request_id": request.request_id, + "workflow": WEBSITE_WORKFLOW, + "documents": [ + { + "document_id": item.document_id, + "document_version": item.document_version, + "state": item.state, + } + for item in documents + ], + } + + +def _provenance_manifest( + documents: list[WebsiteSnapshotDocument], + entity_reference: str, + captured_at: str, +) -> dict[str, Any]: + return { + "schema_version": 1, + "generated_at": captured_at, + "nodes": [ + {"id": entity_reference, "type": "entity"}, + *[ + { + "id": item.document_id, + "version": item.document_version, + "type": "document", + "url": item.url, + "authority": item.authority.model_dump(mode="json"), + } + for item in documents + ], + ], + "edges": [ + {"from": item.document_id, "to": entity_reference, "type": "describes"} for item in documents + ], + } + + +def _load_verified_baseline( + baseline_pack: Path | None, + *, + expected_snapshot_id: str | None, + expected_snapshot_hash: str | None, +) -> tuple[WebsiteSnapshot | None, dict[str, Any] | None]: + if baseline_pack is None: + if expected_snapshot_id or expected_snapshot_hash: + raise ContextPackError("Baseline identity was supplied without a baseline pack.") + return None, None + snapshot = validate_website_snapshot_pack(baseline_pack) + if expected_snapshot_id and snapshot.snapshot_id != expected_snapshot_id: + raise ContextPackError("Verified baseline snapshot_id does not match the request.") + if expected_snapshot_hash and snapshot.snapshot_hash != expected_snapshot_hash.lower(): + raise ContextPackError("Verified baseline snapshot_hash does not match the request.") + return snapshot, { + "snapshot_id": snapshot.snapshot_id, + "snapshot_hash": snapshot.snapshot_hash, + "verified": True, + } + + +def _write_artifact_manifest( + output: Path, + relative_paths: set[str], + *, + pack_id: str, + run_id: str, +) -> ArtifactManifest: + entries: list[ArtifactEntry] = [] + for relative in sorted(relative_paths): + path = _safe_pack_path(output, relative) + if not path.is_file(): + raise ContextPackError(f"Expected website artifact was not written: {relative}") + entries.append( + ArtifactEntry( + name=_artifact_name(relative), + path=relative, + role=_artifact_role(relative), + media_type=_media_type(path), + bytes=path.stat().st_size, + sha256=file_sha256(path), + ) + ) + aggregate = canonical_sha256([entry.model_dump(mode="json", exclude_none=True) for entry in entries]) + manifest = ArtifactManifest( + pack_id=pack_id, + run_id=run_id, + entries=entries, + aggregate_sha256=aggregate, + ) + write_json(output / "artifact.manifest.json", manifest.model_dump(mode="json", exclude_none=True)) + return manifest + + +def _write_workflow_result( + *, + output: Path, + request: WorkflowRequest, + run: ContextPackRun, + snapshot: WebsiteSnapshot, + artifact_manifest: ArtifactManifest, + baseline_payload: dict[str, Any] | None, +) -> None: + active_count = sum(item.state in {"added", "changed", "unchanged"} for item in snapshot.documents) + failures = [item.failure for item in snapshot.documents if item.failure is not None] + warnings = _warning_models(run.warnings) + status = ( + "failed" + if active_count == 0 + else ("completed_with_warnings" if failures or warnings else "completed") + ) + finished_at = utc_now_iso() + progress = [WorkflowProgressEvent.model_validate(item) for item in run.progress_events] + result = WorkflowResult( + request_id=request.request_id, + workflow=WEBSITE_WORKFLOW, + status=status, # type: ignore[arg-type] + started_at=run.started_at, + finished_at=finished_at, + pack_identity=snapshot.pack_identity, + run_identity=snapshot.run_identity, + summary={ + "usable_output": active_count > 0, + "snapshot_id": snapshot.snapshot_id, + "snapshot_hash": snapshot.snapshot_hash, + "document_count": len(snapshot.documents), + "active_document_count": active_count, + "baseline": baseline_payload, + }, + data={"entrypoint": WEBSITE_SNAPSHOT_ENTRYPOINT}, + progress_events=progress, + warnings=warnings, + failures=failures, + budget_usage=BudgetUsage( + limit_usd=run.policy.budget.maximum_paid_cost_usd, + estimated_usd=0.0, + actual_usd=0.0, + paid_request_count=0, + http_request_count=run.http_request_count, + cache_hit_count=run.cache_hit_count, + ), + hashes={ + "artifact_manifest": HashDigest(digest=file_sha256(output / "artifact.manifest.json")), + "snapshot": HashDigest(digest=snapshot.snapshot_hash), + "pack": HashDigest(digest=artifact_manifest.aggregate_sha256), + }, + replay_configuration=ReplayConfiguration( + browser_enabled=request.replay.browser_enabled, + paid_routes_enabled=False, + configuration=snapshot.options.model_dump(mode="json"), + ), + compatibility_artifacts={ + "website_snapshot": WEBSITE_SNAPSHOT_ENTRYPOINT, + "documents": snapshot.manifests.documents.path, + }, + ) + write_json(output / "workflow.result.json", result.model_dump(mode="json", exclude_none=True)) + + +def _warning_models(items: list[dict[str, Any]]) -> list[WorkflowWarning]: + warnings: list[WorkflowWarning] = [] + for item in items: + raw_metadata = item.get("metadata") + metadata = ( + {str(key): value for key, value in raw_metadata.items()} if isinstance(raw_metadata, dict) else {} + ) + warnings.append( + WorkflowWarning( + code=str(item.get("code") or "warning"), + message=str(item.get("message") or "Website snapshot warning"), + metadata=metadata, + ) + ) + return warnings + + +def _reference(output: Path, path: Path, *, media_type: str) -> ArtifactReference: + relative = str(path.resolve().relative_to(output.resolve())) + return ArtifactReference( + path=relative, + sha256=file_sha256(path), + bytes=path.stat().st_size, + media_type=media_type, + ) + + +def _safe_pack_path(root: Path, relative: str) -> Path: + path = Path(relative) + if path.is_absolute() or ".." in path.parts: + raise ContextPackError(f"Unsafe website artifact path: {relative}") + resolved = (root / path).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError as err: + raise ContextPackError(f"Unsafe website artifact path: {relative}") from err + return resolved + + +def _canonical_url(url: str) -> str: + return public_url(url) + + +def _sha256_text(value: str) -> str: + import hashlib + + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _artifact_name(relative: str) -> str: + return relative.replace("/", "_").replace(".", "_") + + +def _artifact_role(relative: str) -> str: + if relative == WEBSITE_SNAPSHOT_ENTRYPOINT or "manifest" in relative: + return "manifest" + if relative.startswith("okf/") or relative == "documents.ndjson": + return "document" + if relative.startswith("screenshots/"): + return "visual" + if relative.startswith("brand-assets/"): + return "brand_asset" + if relative.startswith("raw/"): + return "raw_source" + if relative == "source_policy.json": + return "policy" + if relative == "run.accounting.json": + return "budget_usage" + return "workflow_output" + + +def _media_type(path: Path) -> str | None: + return { + ".json": "application/json", + ".ndjson": "application/x-ndjson", + ".md": "text/markdown", + ".html": "text/html", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + }.get(path.suffix.lower()) + + +def _request_policy(request: WorkflowRequest) -> PolicyConfig | None: + raw = request.options.get("policy") + return PolicyConfig.model_validate(raw) if isinstance(raw, dict) else None + + +__all__ = [ + "DEFAULT_WEBSITE_OUTPUT_DIR", + "WEBSITE_SNAPSHOT_ENTRYPOINT", + "WEBSITE_WORKFLOW", + "build_website_pack", + "validate_website_snapshot_pack", +] diff --git a/src/docpull/context_packs/workflow_cli.py b/src/docpull/context_packs/workflow_cli.py index 1b6e9b5..05f06c1 100644 --- a/src/docpull/context_packs/workflow_cli.py +++ b/src/docpull/context_packs/workflow_cli.py @@ -7,6 +7,7 @@ from pathlib import Path from ..policy import PolicyConfig +from ..workflows import create_workflow_request, run_workflow from ._legacy_cli import ( run_brand_pack_cli, run_image_pack_cli, @@ -17,6 +18,48 @@ from .cli import _positive_int, _run_and_print from .policy_pack import DEFAULT_POLICY_OUTPUT_DIR, build_policy_pack from .relationship import DEFAULT_RELATIONSHIP_OUTPUT_DIR, build_relationship_pack +from .website import DEFAULT_WEBSITE_OUTPUT_DIR + + +def run_website_pack_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="docpull website-pack", + description="Build a portable, baseline-aware website.snapshot.v1 artifact", + ) + parser.add_argument("url_or_domain") + parser.add_argument("--output-dir", "-o", type=Path, default=DEFAULT_WEBSITE_OUTPUT_DIR) + parser.add_argument("--policy", type=Path) + parser.add_argument("--max-pages", type=_positive_int, default=50) + parser.add_argument("--max-depth", type=int, default=3) + parser.add_argument("--baseline-pack", type=Path) + parser.add_argument("--no-raw-html", action="store_true") + parser.add_argument("--no-key-page-visuals", action="store_true") + parser.add_argument("--no-render-fallback", action="store_true") + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + if args.max_depth < 0: + parser.error("--max-depth must be non-negative") + options = { + "max_pages": args.max_pages, + "max_depth": args.max_depth, + "raw_html": not args.no_raw_html, + "key_page_visuals": not args.no_key_page_visuals, + "render_fallback": not args.no_render_fallback, + "baseline_pack": str(args.baseline_pack.resolve()) if args.baseline_pack else None, + } + options = {key: value for key, value in options.items() if value is not None} + request = create_workflow_request( + "website-pack", + args.url_or_domain, + output_dir=args.output_dir, + options=options, + policy=PolicyConfig.from_file(args.policy) if args.policy else None, + ) + return _run_and_print( + lambda: run_workflow(request), + json_output=args.json_output, + success_label="Website snapshot", + ) def run_policy_pack_cli(argv: list[str] | None = None) -> int: @@ -96,4 +139,5 @@ def run_relationship_pack_cli(argv: list[str] | None = None) -> int: "run_product_pack_cli", "run_screenshot_pack_cli", "run_styleguide_pack_cli", + "run_website_pack_cli", ] diff --git a/src/docpull/contracts.py b/src/docpull/contracts.py index a42aa37..fa83095 100644 --- a/src/docpull/contracts.py +++ b/src/docpull/contracts.py @@ -25,6 +25,7 @@ INTELLIGENCE_BUNDLE_CONTRACT: Final[Literal["intelligence.bundle.v1"]] = "intelligence.bundle.v1" CHANGE_EVENT_CONTRACT: Final[Literal["change.event.v1"]] = "change.event.v1" RELATIONSHIP_PACK_CONTRACT: Final[Literal["relationship.pack.v1"]] = "relationship.pack.v1" +WEBSITE_SNAPSHOT_CONTRACT: Final[Literal["website.snapshot.v1"]] = "website.snapshot.v1" class ContractModel(BaseModel): @@ -118,6 +119,13 @@ class ArtifactManifest(ContractModel): aggregate_sha256: str +class ArtifactReference(ContractModel): + path: str + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + bytes: int | None = Field(default=None, ge=0) + media_type: str | None = None + + class WorkflowResult(ContractModel): contract_version: Literal["workflow.result.v1"] = WORKFLOW_RESULT_CONTRACT schema_version: int = 1 @@ -140,6 +148,96 @@ class WorkflowResult(ContractModel): compatibility_artifacts: dict[str, str] = Field(default_factory=dict) +class WebsiteSnapshotDocument(ContractModel): + document_id: str + document_version: str + content_hash: str = Field(pattern=r"^[a-f0-9]{64}$") + url: str + canonical_url: str | None = None + title: str | None = None + entity_reference: str + authority: SourceAuthority + page_role: Literal[ + "home", + "product", + "pricing", + "documentation", + "trust", + "legal", + "changelog", + "support", + "other", + ] + state: Literal["added", "changed", "unchanged", "removed", "failed", "blocked"] + fetched_at: str | None = None + okf: ArtifactReference | None = None + raw: ArtifactReference | None = None + screenshot: ArtifactReference | None = None + brand_assets: list[ArtifactReference] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + failure: WorkflowFailure | None = None + + @model_validator(mode="after") + def _validate_representations(self) -> WebsiteSnapshotDocument: + if self.state in {"added", "changed", "unchanged"} and self.okf is None: + raise ValueError(f"{self.state} documents require an OKF representation") + if self.state in {"failed", "blocked"} and self.failure is None: + raise ValueError(f"{self.state} documents require a typed failure") + return self + + +class WebsiteSnapshotBaseline(ContractModel): + snapshot_id: str + snapshot_hash: str = Field(pattern=r"^[a-f0-9]{64}$") + verified: Literal[True] = True + + +class WebsiteSnapshotOptions(ContractModel): + max_pages: int = Field(default=50, ge=1) + max_depth: int = Field(default=3, ge=0) + raw_html: bool = True + key_page_visuals: bool = True + render_fallback: bool = True + pdf_enabled: bool = False + + +class WebsiteSnapshotManifests(ContractModel): + corpus: ArtifactReference + current_run: ArtifactReference + coverage: ArtifactReference + provenance: ArtifactReference + artifact: str = "artifact.manifest.json" + documents: ArtifactReference + + +class WebsiteSnapshot(ContractModel): + contract_version: Literal["website.snapshot.v1"] = WEBSITE_SNAPSHOT_CONTRACT + schema_version: Literal[1] = 1 + snapshot_id: str + snapshot_hash: str = Field(pattern=r"^[a-f0-9]{64}$") + schema_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + pack_identity: dict[str, Any] + run_identity: dict[str, Any] + entity_reference: str + captured_at: str + baseline: WebsiteSnapshotBaseline | None = None + options: WebsiteSnapshotOptions = Field(default_factory=WebsiteSnapshotOptions) + documents: list[WebsiteSnapshotDocument] = Field(default_factory=list) + coverage: dict[str, Any] = Field(default_factory=dict) + manifests: WebsiteSnapshotManifests + visual_count: int = Field(default=0, ge=0, le=8) + warnings: list[WorkflowWarning] = Field(default_factory=list) + + @model_validator(mode="after") + def _validate_snapshot(self) -> WebsiteSnapshot: + identities = [(item.document_id, item.document_version) for item in self.documents] + if len(identities) != len(set(identities)): + raise ValueError("snapshot document identities must be unique") + if sum(item.screenshot is not None for item in self.documents) != self.visual_count: + raise ValueError("visual_count must equal screenshot count") + return self + + class EvidenceSpan(ContractModel): citation_id: str record_citation_id: str | None = None @@ -357,6 +455,7 @@ class BasisContractV2(ContractModel): "intelligence-bundle.v1.schema.json": IntelligenceBundle, "change-event.v1.schema.json": ChangeEvent, "relationship-pack.v1.schema.json": RelationshipPack, + "website-snapshot.v1.schema.json": WebsiteSnapshot, "document.v3.schema.json": DocumentRecord, "run-identity.v1.schema.json": RunIdentity, "pack.v3.schema.json": PackContractV3, @@ -377,6 +476,39 @@ def canonical_sha256(value: Any) -> str: return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() +def website_snapshot_core(snapshot: WebsiteSnapshot | dict[str, Any]) -> dict[str, Any]: + """Return the frozen cross-runtime payload covered by ``snapshot_hash``.""" + + value = ( + snapshot.model_dump(mode="json", exclude_none=True) + if isinstance(snapshot, WebsiteSnapshot) + else dict(snapshot) + ) + value.setdefault("baseline", None) + keys = ( + "contract_version", + "schema_version", + "schema_sha256", + "pack_identity", + "run_identity", + "entity_reference", + "captured_at", + "baseline", + "options", + "documents", + "coverage", + "manifests", + "visual_count", + "warnings", + ) + return {key: value[key] for key in keys} + + +def verify_website_snapshot_hash(snapshot: WebsiteSnapshot | dict[str, Any]) -> bool: + value = snapshot if isinstance(snapshot, dict) else snapshot.model_dump(mode="json") + return canonical_sha256(website_snapshot_core(snapshot)) == str(value["snapshot_hash"]).lower() + + def stable_id(prefix: str, value: Any, *, length: int = 24) -> str: return f"{prefix}_{canonical_sha256(value)[:length]}" @@ -618,6 +750,8 @@ def _media_type(path: Path) -> str | None: "RELATIONSHIP_PACK_CONTRACT", "WORKFLOW_REQUEST_CONTRACT", "WORKFLOW_RESULT_CONTRACT", + "WEBSITE_SNAPSHOT_CONTRACT", + "ArtifactReference", "ArtifactEntry", "ArtifactManifest", "BasisContractV2", @@ -644,6 +778,11 @@ def _media_type(path: Path) -> str | None: "WorkflowRequest", "WorkflowResult", "WorkflowWarning", + "WebsiteSnapshot", + "WebsiteSnapshotBaseline", + "WebsiteSnapshotDocument", + "WebsiteSnapshotManifests", + "WebsiteSnapshotOptions", "artifact_entries", "build_workflow_request", "bundled_schema_path", @@ -652,6 +791,8 @@ def _media_type(path: Path) -> str | None: "file_sha256", "new_progress_event", "stable_id", + "verify_website_snapshot_hash", + "website_snapshot_core", "write_contract_schemas", "workflow_failure_from_mapping", ] diff --git a/src/docpull/mcp/server.py b/src/docpull/mcp/server.py index 74bb21d..f867009 100644 --- a/src/docpull/mcp/server.py +++ b/src/docpull/mcp/server.py @@ -859,6 +859,7 @@ async def _dispatch_tool( elif name in { "workflow_run", + "website_pack", "brand_pack", "product_pack", "styleguide_pack", @@ -871,6 +872,7 @@ async def _dispatch_tool( from ..workflows import create_workflow_request, run_workflow workflow_aliases = { + "website_pack": ("website-pack", "url_or_domain", "packs/websites"), "brand_pack": ("brand-pack", "domain_or_url", "packs/brand"), "product_pack": ("product-pack", "url_or_domain", "packs/products"), "styleguide_pack": ("styleguide-pack", "domain_or_url", "packs/styleguide"), @@ -1619,7 +1621,8 @@ async def _list_tools() -> list[Tool]: # type: ignore[no-any-unimported] name="workflow_run", description=( "Run a registered evidence-pack workflow through workflow.request.v1 and return " - "workflow.result.v1. Supported workflows are brand, product, styleguide, visual/image, " + "workflow.result.v1. Supported workflows are website, brand, product, " + "styleguide, visual/image, " "screenshot, policy, relationship, dataset, fetch, and crawl. Browser use remains " "explicitly gated." ), @@ -1636,6 +1639,7 @@ async def _list_tools() -> list[Tool]: # type: ignore[no-any-unimported] "workflow": { "type": "string", "enum": [ + "website-pack", "brand-pack", "product-pack", "styleguide-pack", @@ -1667,6 +1671,36 @@ async def _list_tools() -> list[Tool]: # type: ignore[no-any-unimported] }, outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, ), + Tool( + name="website_pack", + description=( + "Build a recursively verifiable portable-v3 website snapshot with optional " + "verified-baseline comparison." + ), + annotations=ToolAnnotations( + title="Build a website snapshot", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "url_or_domain": {"type": "string"}, + "output_dir": {"type": "string"}, + "max_pages": {"type": "integer", "minimum": 1, "default": 50}, + "max_depth": {"type": "integer", "minimum": 0, "default": 3}, + "raw_html": {"type": "boolean", "default": True}, + "key_page_visuals": {"type": "boolean", "default": True}, + "render_fallback": {"type": "boolean", "default": True}, + "baseline_pack": {"type": "string"}, + "policy": {"type": "string"}, + }, + "required": ["url_or_domain", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), Tool( name="brand_pack", description="Build an evidence-backed brand pack through the common workflow protocol.", diff --git a/src/docpull/schemas/website-snapshot.v1.schema.json b/src/docpull/schemas/website-snapshot.v1.schema.json new file mode 100644 index 0000000..7a2054a --- /dev/null +++ b/src/docpull/schemas/website-snapshot.v1.schema.json @@ -0,0 +1,567 @@ +{ + "$defs": { + "ArtifactReference": { + "additionalProperties": true, + "properties": { + "bytes": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bytes" + }, + "media_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Media Type" + }, + "path": { + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "path", + "sha256" + ], + "title": "ArtifactReference", + "type": "object" + }, + "SourceAuthority": { + "additionalProperties": true, + "properties": { + "rationale": { + "title": "Rationale", + "type": "string" + }, + "role": { + "enum": [ + "official_product", + "official_corporate", + "legal", + "documentation", + "social", + "marketplace", + "government_registry", + "regulatory_filing", + "press_release", + "local_reporting", + "third_party" + ], + "title": "Role", + "type": "string" + }, + "tier": { + "enum": [ + "tier_1_authoritative", + "tier_2_owned", + "tier_3_distribution", + "tier_4_external" + ], + "title": "Tier", + "type": "string" + } + }, + "required": [ + "role", + "tier", + "rationale" + ], + "title": "SourceAuthority", + "type": "object" + }, + "WebsiteSnapshotBaseline": { + "additionalProperties": true, + "properties": { + "snapshot_hash": { + "pattern": "^[a-f0-9]{64}$", + "title": "Snapshot Hash", + "type": "string" + }, + "snapshot_id": { + "title": "Snapshot Id", + "type": "string" + }, + "verified": { + "const": true, + "default": true, + "title": "Verified", + "type": "boolean" + } + }, + "required": [ + "snapshot_id", + "snapshot_hash" + ], + "title": "WebsiteSnapshotBaseline", + "type": "object" + }, + "WebsiteSnapshotDocument": { + "additionalProperties": true, + "properties": { + "authority": { + "$ref": "#/$defs/SourceAuthority" + }, + "brand_assets": { + "items": { + "$ref": "#/$defs/ArtifactReference" + }, + "title": "Brand Assets", + "type": "array" + }, + "canonical_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Canonical Url" + }, + "content_hash": { + "pattern": "^[a-f0-9]{64}$", + "title": "Content Hash", + "type": "string" + }, + "document_id": { + "title": "Document Id", + "type": "string" + }, + "document_version": { + "title": "Document Version", + "type": "string" + }, + "entity_reference": { + "title": "Entity Reference", + "type": "string" + }, + "failure": { + "anyOf": [ + { + "$ref": "#/$defs/WorkflowFailure" + }, + { + "type": "null" + } + ], + "default": null + }, + "fetched_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fetched At" + }, + "okf": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "page_role": { + "enum": [ + "home", + "product", + "pricing", + "documentation", + "trust", + "legal", + "changelog", + "support", + "other" + ], + "title": "Page Role", + "type": "string" + }, + "raw": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "screenshot": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "state": { + "enum": [ + "added", + "changed", + "unchanged", + "removed", + "failed", + "blocked" + ], + "title": "State", + "type": "string" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + }, + "url": { + "title": "Url", + "type": "string" + }, + "warnings": { + "items": { + "type": "string" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "document_id", + "document_version", + "content_hash", + "url", + "entity_reference", + "authority", + "page_role", + "state" + ], + "title": "WebsiteSnapshotDocument", + "type": "object" + }, + "WebsiteSnapshotManifests": { + "additionalProperties": true, + "properties": { + "artifact": { + "default": "artifact.manifest.json", + "title": "Artifact", + "type": "string" + }, + "corpus": { + "$ref": "#/$defs/ArtifactReference" + }, + "coverage": { + "$ref": "#/$defs/ArtifactReference" + }, + "current_run": { + "$ref": "#/$defs/ArtifactReference" + }, + "documents": { + "$ref": "#/$defs/ArtifactReference" + }, + "provenance": { + "$ref": "#/$defs/ArtifactReference" + } + }, + "required": [ + "corpus", + "current_run", + "coverage", + "provenance", + "documents" + ], + "title": "WebsiteSnapshotManifests", + "type": "object" + }, + "WebsiteSnapshotOptions": { + "additionalProperties": true, + "properties": { + "key_page_visuals": { + "default": true, + "title": "Key Page Visuals", + "type": "boolean" + }, + "max_depth": { + "default": 3, + "minimum": 0, + "title": "Max Depth", + "type": "integer" + }, + "max_pages": { + "default": 50, + "minimum": 1, + "title": "Max Pages", + "type": "integer" + }, + "pdf_enabled": { + "default": false, + "title": "Pdf Enabled", + "type": "boolean" + }, + "raw_html": { + "default": true, + "title": "Raw Html", + "type": "boolean" + }, + "render_fallback": { + "default": true, + "title": "Render Fallback", + "type": "boolean" + } + }, + "title": "WebsiteSnapshotOptions", + "type": "object" + }, + "WorkflowFailure": { + "additionalProperties": true, + "properties": { + "attempts": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attempts" + }, + "code": { + "default": "workflow_error", + "title": "Code", + "type": "string" + }, + "http_status": { + "anyOf": [ + { + "maximum": 599, + "minimum": 100, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Http Status" + }, + "message": { + "title": "Message", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "retry_after_seconds": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retry After Seconds" + }, + "retryable": { + "default": false, + "title": "Retryable", + "type": "boolean" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Url" + }, + "stage": { + "default": "workflow", + "title": "Stage", + "type": "string" + } + }, + "required": [ + "message" + ], + "title": "WorkflowFailure", + "type": "object" + }, + "WorkflowWarning": { + "additionalProperties": true, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "code", + "message" + ], + "title": "WorkflowWarning", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/website-snapshot.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "baseline": { + "anyOf": [ + { + "$ref": "#/$defs/WebsiteSnapshotBaseline" + }, + { + "type": "null" + } + ], + "default": null + }, + "captured_at": { + "title": "Captured At", + "type": "string" + }, + "contract_version": { + "const": "website.snapshot.v1", + "default": "website.snapshot.v1", + "title": "Contract Version", + "type": "string" + }, + "coverage": { + "additionalProperties": true, + "title": "Coverage", + "type": "object" + }, + "documents": { + "items": { + "$ref": "#/$defs/WebsiteSnapshotDocument" + }, + "title": "Documents", + "type": "array" + }, + "entity_reference": { + "title": "Entity Reference", + "type": "string" + }, + "manifests": { + "$ref": "#/$defs/WebsiteSnapshotManifests" + }, + "options": { + "$ref": "#/$defs/WebsiteSnapshotOptions" + }, + "pack_identity": { + "additionalProperties": true, + "title": "Pack Identity", + "type": "object" + }, + "run_identity": { + "additionalProperties": true, + "title": "Run Identity", + "type": "object" + }, + "schema_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Schema Sha256", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "snapshot_hash": { + "pattern": "^[a-f0-9]{64}$", + "title": "Snapshot Hash", + "type": "string" + }, + "snapshot_id": { + "title": "Snapshot Id", + "type": "string" + }, + "visual_count": { + "default": 0, + "maximum": 8, + "minimum": 0, + "title": "Visual Count", + "type": "integer" + }, + "warnings": { + "items": { + "$ref": "#/$defs/WorkflowWarning" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "snapshot_id", + "snapshot_hash", + "schema_sha256", + "pack_identity", + "run_identity", + "entity_reference", + "captured_at", + "manifests" + ], + "title": "WebsiteSnapshot", + "type": "object" +} diff --git a/src/docpull/surface.py b/src/docpull/surface.py index b755099..e9be981 100644 --- a/src/docpull/surface.py +++ b/src/docpull/surface.py @@ -44,6 +44,7 @@ class CliCommand: CliCommand("transcript-pack", "Build a local v3 pack from transcript files or URLs"), CliCommand("wiki-pack", "Build a local v3 pack from Wikimedia/MediaWiki REST page content"), CliCommand("brand-pack", "Build an evidence-backed brand knowledge pack"), + CliCommand("website-pack", "Build a portable baseline-aware website snapshot"), CliCommand("product-pack", "Build cited product and pricing evidence"), CliCommand("styleguide-pack", "Build a cited design-token and styleguide pack"), CliCommand("image-pack", "Build a bounded visual asset manifest"), @@ -118,6 +119,8 @@ class CliCommand: "CoverageResult", "RelationshipCandidate", "RelationshipPack", + "WebsiteSnapshot", + "WebsiteSnapshotDocument", "SourceAuthority", "WorkflowRequest", "WorkflowResult", @@ -142,10 +145,12 @@ class CliCommand: "build_repo_pack", "build_standards_pack", "build_styleguide_pack", + "build_website_pack", "build_transcript_pack", "build_wiki_pack", "build_image_pack", "capture_screenshot_pack", + "validate_website_snapshot_pack", "parse_documents", "parse_one_document", "DocumentParseError", @@ -245,10 +250,12 @@ class CliCommand: "build_repo_pack", "build_standards_pack", "build_styleguide_pack", + "build_website_pack", "build_transcript_pack", "build_wiki_pack", "build_image_pack", "capture_screenshot_pack", + "validate_website_snapshot_pack", ) PRUNED_SDK_EXPORTS = frozenset( diff --git a/src/docpull/workflows.py b/src/docpull/workflows.py index 4d69614..3e0c708 100644 --- a/src/docpull/workflows.py +++ b/src/docpull/workflows.py @@ -73,7 +73,11 @@ def create_workflow_request( source_policy=effective_policy.to_source_policy_payload(source=workflow), budget={"maximum_paid_cost_usd": effective_policy.budget.maximum_paid_cost_usd}, browser_enabled=_normalize_workflow(workflow) == "screenshot-pack" - or bool(merged_options.get("render")), + or bool(merged_options.get("render")) + or ( + _normalize_workflow(workflow) == "website-pack" + and bool(merged_options.get("key_page_visuals", True)) + ), paid_routes_enabled=False, ) @@ -142,6 +146,28 @@ def _run_product(request: WorkflowRequest) -> dict[str, Any]: ) +def _run_website(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.website import build_website_pack + + options = request.options + baseline = _optional_str(options, "baseline_pack") + return build_website_pack( + _request_value(request), + output_dir=_request_output_dir(request), + request=request, + max_pages=_positive_int(options, "max_pages", 50), + max_depth=_nonnegative_int(options, "max_depth", 3), + raw_html=bool(options.get("raw_html", True)), + key_page_visuals=bool(options.get("key_page_visuals", True)), + render_fallback=bool(options.get("render_fallback", True)), + pdf_enabled=bool(options.get("pdf_enabled", False)), + baseline_pack=Path(baseline) if baseline else None, + baseline_snapshot_id=_optional_str(options, "baseline_snapshot_id"), + baseline_snapshot_hash=_optional_str(options, "baseline_snapshot_hash"), + entity_reference=_optional_str(options, "entity_reference"), + ) + + def _run_styleguide(request: WorkflowRequest) -> dict[str, Any]: from .context_packs.styleguide import build_styleguide_pack @@ -246,6 +272,7 @@ def _run_crawl(request: WorkflowRequest) -> dict[str, Any]: WORKFLOW_REGISTRY: dict[str, PackWorkflow] = { + "website-pack": FunctionPackWorkflow("website-pack", _run_website), "brand-pack": FunctionPackWorkflow("brand-pack", _run_brand), "product-pack": FunctionPackWorkflow("product-pack", _run_product), "styleguide-pack": FunctionPackWorkflow("styleguide-pack", _run_styleguide), @@ -264,6 +291,7 @@ def _normalize_workflow(value: str) -> str: normalized = value.strip().lower().replace("_", "-") aliases = { "brand": "brand-pack", + "website": "website-pack", "product": "product-pack", "styleguide": "styleguide-pack", "visual": "visual-pack", @@ -314,6 +342,13 @@ def _positive_int(options: dict[str, Any], key: str, default: int) -> int: return value +def _nonnegative_int(options: dict[str, Any], key: str, default: int) -> int: + value = options.get(key, default) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise WorkflowExecutionError(f"WorkflowRequest.options.{key} must be a non-negative integer") + return value + + def _optional_str(options: dict[str, Any], key: str) -> str | None: value = options.get(key) if value is None: diff --git a/tests/fixtures/website_snapshot/golden-pages.json b/tests/fixtures/website_snapshot/golden-pages.json new file mode 100644 index 0000000..db9cfdc --- /dev/null +++ b/tests/fixtures/website_snapshot/golden-pages.json @@ -0,0 +1,15 @@ +{ + "contract_version": "website.snapshot.v1", + "description": "Golden page-role coverage expected from the bounded website workflow.", + "expected_roles": [ + "home", + "product", + "pricing", + "documentation", + "trust", + "legal", + "changelog", + "support", + "other" + ] +} diff --git a/tests/test_local_pack_workflows.py b/tests/test_local_pack_workflows.py new file mode 100644 index 0000000..cfe2348 --- /dev/null +++ b/tests/test_local_pack_workflows.py @@ -0,0 +1,151 @@ +"""Local-pack replay must never silently fall back to network acquisition.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from docpull.context_packs import brand, policy_pack, product, relationship, styleguide, visuals +from docpull.context_packs.common import PageSnapshot + + +def _local_pack(root: Path) -> Path: + root.mkdir() + records = [ + { + "schema_version": 3, + "document_id": "doc_home", + "url": "https://example.com/", + "title": "Example Product", + "content": "# Example Product\n\nOfficial API support and unlimited monitoring.\n", + "content_hash": hashlib.sha256( + b"# Example Product\n\nOfficial API support and unlimited monitoring.\n" + ).hexdigest(), + "metadata": {"entity_name": "Example"}, + "extraction": {}, + }, + { + "schema_version": 3, + "document_id": "doc_pricing", + "url": "https://example.com/pricing", + "title": "Pricing", + "content": "# Pricing\n\nPro plan $20 per month. 14-day free trial.\n", + "content_hash": hashlib.sha256( + b"# Pricing\n\nPro plan $20 per month. 14-day free trial.\n" + ).hexdigest(), + "metadata": {}, + "extraction": {}, + }, + ] + (root / "documents.ndjson").write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + (root / "corpus.manifest.json").write_text( + json.dumps({"schema_version": 3, "records": []}), + encoding="utf-8", + ) + return root + + +def _network_forbidden(*args, **kwargs): # type: ignore[no-untyped-def] + raise AssertionError("local pack mode attempted network access") + + +def test_all_compatible_pack_workflows_replay_without_network( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pack = _local_pack(tmp_path / "input") + monkeypatch.setattr(brand, "fetch_pages_blocking", _network_forbidden) + monkeypatch.setattr(brand, "fetch_asset_blocking", _network_forbidden) + monkeypatch.setattr(product, "fetch_pages_blocking", _network_forbidden) + monkeypatch.setattr(styleguide, "fetch_pages_blocking", _network_forbidden) + monkeypatch.setattr(styleguide, "_fetch_css_blocking", _network_forbidden) + monkeypatch.setattr(policy_pack, "fetch_pages_blocking", _network_forbidden) + monkeypatch.setattr(visuals, "fetch_pages_blocking", _network_forbidden) + monkeypatch.setattr(visuals, "fetch_asset_blocking", _network_forbidden) + monkeypatch.setattr(relationship, "fetch_pages_blocking", _network_forbidden) + + results = [ + brand.build_brand_pack(str(pack), output_dir=tmp_path / "brand"), + product.build_product_pack(str(pack), mode="site", output_dir=tmp_path / "product"), + policy_pack.build_policy_pack(str(pack), output_dir=tmp_path / "policy"), + styleguide.build_styleguide_pack(str(pack), output_dir=tmp_path / "styleguide"), + visuals.build_image_pack(str(pack), output_dir=tmp_path / "visual"), + relationship.build_relationship_pack([str(pack)], output_dir=tmp_path / "relationship"), + ] + assert all(result["provider"] == "local" for result in results) + assert all( + (tmp_path / name / "workflow.result.json").is_file() + for name in ( + "brand", + "product", + "policy", + "styleguide", + "visual", + "relationship", + ) + ) + + +def test_product_trial_and_testimonial_guards_are_deterministic() -> None: + page = PageSnapshot( + url="https://example.com/pricing", + title="Pricing", + html=( + '

14-day free trial $0

' + '

Pro $25 per month

' + '

We love the unlimited API support feature.

' + '

Includes unlimited API monitoring.

' + ), + markdown=( + "14-day free trial $0\n\nPro $25 per month\n\n" + "We love the unlimited API support feature.\n\n" + "Includes unlimited API monitoring.\n" + ), + metadata={}, + extraction={}, + ) + rows = product._pricing_rows(page, [page]) + trial = next(row for row in rows if row["trial"]) + assert trial["price"] is None + assert trial["currency"] is None + assert any(row["price"] == 25.0 and row["currency"] == "USD" for row in rows) + features = product._feature_bullets(page, [page]) + assert [item["text"] for item in features] == ["Includes unlimited API monitoring."] + + +def test_structured_zero_price_trial_is_not_a_subscription_price() -> None: + page = PageSnapshot( + url="https://example.com/", + title="Example", + html=( + '" + ), + markdown="# Example\n\nTry Example free for 7 days.\n", + metadata={"page_role": "home"}, + extraction={}, + ) + + offer = product._jsonld_products(page, [page])[0]["offers"][0] + + assert offer["price"] is None + assert offer["currency"] is None + assert offer["trial"]["duration_days"] == 7 + assert offer["evidence"]["excerpt"] == "Try Example free for 7 days." + assert offer["evidence"]["evidence_span"]["exact_text"] == offer["evidence"]["excerpt"] + + +def test_plain_language_annual_cadence_is_normalized() -> None: + assert product._billing_from_text("The plan costs $99 a year.") == "yearly" + assert product._billing_interval("The plan costs $99 each year.") == { + "unit": "year", + "count": 1, + } diff --git a/tests/test_website_snapshot.py b/tests/test_website_snapshot.py new file mode 100644 index 0000000..4e461c0 --- /dev/null +++ b/tests/test_website_snapshot.py @@ -0,0 +1,352 @@ +"""Golden contract and baseline-diff tests for website.snapshot.v1.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from docpull.context_packs import website as website_module +from docpull.context_packs.common import ContextPackError, ContextPackRun, PageSnapshot +from docpull.context_packs.website import ( + WEBSITE_SNAPSHOT_ENTRYPOINT, + validate_website_snapshot_pack, +) +from docpull.contracts import WebsiteSnapshot +from docpull.workflows import create_workflow_request, run_workflow + + +def _page( + url: str, + *, + title: str, + body: str, + links: list[tuple[str, str]] | None = None, +) -> PageSnapshot: + anchors = "".join(f'{label}' for href, label in (links or [])) + return PageSnapshot( + url=url, + title=title, + html=f"{title}{anchors}
{body}
", + markdown=f"# {title}\n\n{body}\n", + metadata={}, + extraction={}, + source_type="html", + ) + + +def _fake_fetcher(site: dict[str, PageSnapshot], failures: dict[str, dict[str, Any]] | None = None): + failure_map = failures or {} + + def fetch( + urls: list[str], + *, + run: ContextPackRun, + max_pages: int, + ) -> list[PageSnapshot]: + output: list[PageSnapshot] = [] + for url in urls[:max_pages]: + run.http_request_count += 1 + if url in failure_map: + run.errors.append({"url": url, **failure_map[url]}) + elif url in site: + output.append(site[url]) + return output + + return fetch + + +def _run_snapshot( + output: Path, + *, + options: dict[str, Any] | None = None, +) -> WebsiteSnapshot: + request = create_workflow_request( + "website-pack", + "https://example.com/", + output_dir=output, + options={ + "max_pages": 10, + "max_depth": 1, + "raw_html": True, + "key_page_visuals": True, + **(options or {}), + }, + ) + result = run_workflow(request) + assert result["workflow"] == "website-pack" + return WebsiteSnapshot.model_validate_json( + (output / WEBSITE_SNAPSHOT_ENTRYPOINT).read_text(encoding="utf-8") + ) + + +def test_page_content_excludes_volatile_capture_time() -> None: + def captured_page(captured_at: str) -> PageSnapshot: + return PageSnapshot( + url="https://example.com/", + title="Example", + html="

Example

Stable body.

", + markdown=( + f'---\ntitle: "Example"\ncrawled_at: "{captured_at}"\n---\n\n# Example\n\nStable body.\n' + ), + metadata={}, + extraction={}, + ) + + first = captured_page("2026-07-17T05:00:00Z") + second = captured_page("2026-07-17T06:00:00Z") + + first_content = website_module._page_content(first) + second_content = website_module._page_content(second) + + assert first_content == second_content + assert "crawled_at" not in first_content + + +def test_website_snapshot_is_recursive_portable_and_tamper_evident( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS", raising=False) + site = { + "https://example.com/": _page( + "https://example.com/", + title="Example", + body="The official home page.", + links=[("/pricing", "Pricing"), ("/product", "Product")], + ), + "https://example.com/pricing": _page( + "https://example.com/pricing", + title="Pricing", + body="Pro costs $20 per month.", + ), + "https://example.com/product": _page( + "https://example.com/product", + title="Product", + body="Automated competitive monitoring.", + ), + } + monkeypatch.setattr(website_module, "fetch_pages_blocking", _fake_fetcher(site)) + output = tmp_path / "first" + snapshot = _run_snapshot(output) + + assert snapshot.pack_identity["format"] == "portable-v3" + assert {document.state for document in snapshot.documents} == {"added"} + assert {document.page_role for document in snapshot.documents} >= {"home", "pricing", "product"} + assert snapshot.visual_count == 0 + assert any(warning.code == "browser_capability_unavailable" for warning in snapshot.warnings) + assert validate_website_snapshot_pack(output).snapshot_hash == snapshot.snapshot_hash + assert (output / "okf" / "index.md").is_file() + rows = [ + json.loads(line) + for line in (output / "documents.ndjson").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_identity = {(row["document_id"], row["document_version"]): row for row in rows} + for document in snapshot.documents: + row = by_identity[(document.document_id, document.document_version)] + assert row["content_hash"] == document.content_hash + assert document.document_id in (output / document.okf.path).read_text( # type: ignore[union-attr] + encoding="utf-8" + ) + + stale = output / "stale-from-prior-run.txt" + stale.write_text("ignored", encoding="utf-8") + assert validate_website_snapshot_pack(output).snapshot_hash == snapshot.snapshot_hash + + okf_path = output / snapshot.documents[0].okf.path # type: ignore[union-attr] + okf_path.write_text(okf_path.read_text(encoding="utf-8") + "tampered\n", encoding="utf-8") + with pytest.raises(ContextPackError, match="verification failed"): + validate_website_snapshot_pack(output) + + +def test_tampered_raw_ndjson_manifest_and_document_version_are_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS", raising=False) + site = { + "https://example.com/": _page( + "https://example.com/", + title="Example", + body="Evidence body.", + ) + } + monkeypatch.setattr(website_module, "fetch_pages_blocking", _fake_fetcher(site)) + source = tmp_path / "source" + snapshot = _run_snapshot(source) + mutations = { + "raw": lambda root: (root / snapshot.documents[0].raw.path).write_text( # type: ignore[union-attr] + "tampered", encoding="utf-8" + ), + "ndjson": lambda root: (root / "documents.ndjson").write_text("{}\n", encoding="utf-8"), + "manifest": lambda root: (root / "artifact.manifest.json").write_text( + '{"contract_version":"artifact.manifest.v1"}', encoding="utf-8" + ), + "document-version": lambda root: _tamper_document_version(root), + } + for name, mutate in mutations.items(): + target = tmp_path / name + shutil.copytree(source, target) + mutate(target) + with pytest.raises(ContextPackError): + validate_website_snapshot_pack(target) + + +def _tamper_document_version(root: Path) -> None: + path = root / WEBSITE_SNAPSHOT_ENTRYPOINT + payload = json.loads(path.read_text(encoding="utf-8")) + payload["documents"][0]["document_version"] = "0" * 64 + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_conditional_not_modified_reuses_verified_baseline_documents( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS", raising=False) + site = { + "https://example.com/": _page( + "https://example.com/", + title="Example", + body="Stable home.", + links=[("/pricing", "Pricing")], + ), + "https://example.com/pricing": _page( + "https://example.com/pricing", + title="Pricing", + body="Pro $25 per month.", + ), + } + monkeypatch.setattr(website_module, "fetch_pages_blocking", _fake_fetcher(site)) + baseline_dir = tmp_path / "baseline-conditional" + baseline = _run_snapshot(baseline_dir) + + def not_modified( + urls: list[str], + *, + run: ContextPackRun, + max_pages: int, + ) -> list[PageSnapshot]: + for url in urls[:max_pages]: + run.http_request_count += 1 + run.cache_hit_count += 1 + run.unchanged_urls.add(url) + return [] + + monkeypatch.setattr(website_module, "fetch_pages_blocking", not_modified) + current_dir = tmp_path / "current-conditional" + current = _run_snapshot( + current_dir, + options={ + "baseline_pack": str(baseline_dir), + "baseline_snapshot_id": baseline.snapshot_id, + "baseline_snapshot_hash": baseline.snapshot_hash, + }, + ) + assert {document.state for document in current.documents} == {"unchanged"} + rows = (current_dir / "documents.ndjson").read_text(encoding="utf-8") + assert len([line for line in rows.splitlines() if line.strip()]) == 2 + assert validate_website_snapshot_pack(current_dir).snapshot_id == current.snapshot_id + + +def test_verified_baseline_emits_unchanged_changed_added_removed_and_blocked( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS", raising=False) + first_site = { + "https://example.com/": _page( + "https://example.com/", + title="Example", + body="Stable home copy.", + links=[("/pricing", "Pricing"), ("/product", "Product")], + ), + "https://example.com/pricing": _page( + "https://example.com/pricing", + title="Pricing", + body="Pro costs $20 per month.", + ), + "https://example.com/product": _page( + "https://example.com/product", + title="Product", + body="Legacy feature page.", + ), + } + monkeypatch.setattr(website_module, "fetch_pages_blocking", _fake_fetcher(first_site)) + baseline_dir = tmp_path / "baseline" + baseline = _run_snapshot(baseline_dir) + + second_site = { + "https://example.com/": _page( + "https://example.com/", + title="Example", + body="Stable home copy.", + links=[ + ("/pricing", "Pricing"), + ("/new-product", "Product"), + ("/trust", "Trust"), + ], + ), + "https://example.com/pricing": _page( + "https://example.com/pricing", + title="Pricing", + body="Pro costs $25 per month.", + ), + "https://example.com/new-product": _page( + "https://example.com/new-product", + title="New Product", + body="A newly launched feature.", + ), + } + failures = { + "https://example.com/trust": { + "error": "robots.txt denied acquisition", + "code": "robots_disallowed", + "stage": "fetch", + "blocked": True, + "attempts": 1, + } + } + monkeypatch.setattr( + website_module, + "fetch_pages_blocking", + _fake_fetcher(second_site, failures), + ) + current_dir = tmp_path / "current" + current = _run_snapshot( + current_dir, + options={ + "baseline_pack": str(baseline_dir), + "baseline_snapshot_id": baseline.snapshot_id, + "baseline_snapshot_hash": baseline.snapshot_hash, + }, + ) + by_url = {document.url: document for document in current.documents} + assert by_url["https://example.com/"].state == "unchanged" + assert by_url["https://example.com/pricing"].state == "changed" + assert by_url["https://example.com/new-product"].state == "added" + assert by_url["https://example.com/product"].state == "removed" + assert by_url["https://example.com/trust"].state == "blocked" + assert by_url["https://example.com/trust"].failure is not None + assert current.baseline is not None and current.baseline.verified is True + assert validate_website_snapshot_pack(current_dir).snapshot_id == current.snapshot_id + + +def test_checked_in_golden_roles_cover_intelligence_surface() -> None: + fixture = Path(__file__).parent / "fixtures" / "website_snapshot" / "golden-pages.json" + payload = json.loads(fixture.read_text(encoding="utf-8")) + assert set(payload["expected_roles"]) == { + "home", + "product", + "pricing", + "documentation", + "trust", + "legal", + "changelog", + "support", + "other", + } diff --git a/uv.lock b/uv.lock index db849bd..1ec3d7a 100644 --- a/uv.lock +++ b/uv.lock @@ -1302,7 +1302,7 @@ wheels = [ [[package]] name = "docpull" -version = "6.3.0" +version = "6.4.0" source = { editable = "." } dependencies = [ { name = "aiohttp" },