Pathfinder gap-report remediation: hybrid search + docs repoint + ag-ui sources + chunker/durability

deploy/copilotkit-docs.yaml

@@ -14,10 +14,10 @@ sources:
   - name: docs
     type: markdown
     repo: https://github.com/CopilotKit/CopilotKit.git
-    path: docs/content/docs/
+    path: showcase/shell-docs/src/content/docs/
     base_url: https://docs.copilotkit.ai/
     url_derivation:
-      strip_prefix: "docs/content/docs/"
+      strip_prefix: "showcase/shell-docs/src/content/docs/"
       strip_suffix: ".mdx"
       strip_route_groups: true
       strip_index: true
@@ -45,6 +45,12 @@ sources:
       - "**/__tests__/**"
       - "**/*.test.*"
       - "**/*.spec.*"
+      # Demo/example apps and the docs site Next.js build are not library code
+      - "examples/**"
+      - "showcase/**"
+      - "**/.next/**"
+      # Generated type declarations duplicate the .ts sources they describe
+      - "**/*.d.ts"
     skip_dirs:
       - node_modules
       - dist
@@ -66,27 +72,51 @@ sources:
       strip_route_groups: true
       strip_index: true
     file_patterns:
+      # .mdx only — the docs/ tree's two .md files (README.md, ag_ui.md) are a
+      # folder README and a raw spec, not rendered product pages, and keep their
+      # extension under a .mdx-only strip_suffix, deriving 404 URLs.
       - "**/*.mdx"
-      - "**/*.md"
     chunk:
       target_tokens: 600
       overlap_tokens: 50
 
   - name: ag-ui-code
     type: code
     repo: https://github.com/ag-ui-protocol/ag-ui.git
-    path: sdks/typescript/packages/
+    # Walk from the repo root so the TS SDK, the Python adapters under
+    # integrations/, the Python SDK, and the JVM/multi-language community SDK
+    # are all reachable. file_patterns (matched against repo-root-relative
+    # paths) scope which subtrees actually get indexed.
+    path: "."
     file_patterns:
-      - "**/*.ts"
-      - "**/*.tsx"
-      - "**/*.js"
-      - "**/*.jsx"
+      # TypeScript SDK packages
+      - "sdks/typescript/packages/**/*.ts"
+      - "sdks/typescript/packages/**/*.tsx"
+      - "sdks/typescript/packages/**/*.js"
+      - "sdks/typescript/packages/**/*.jsx"
+      # Framework integrations (Python adapters + TS bindings)
+      - "integrations/**/*.py"
+      - "integrations/**/*.ts"
+      # Python SDK
+      - "sdks/python/**/*.py"
+      # Community SDK — JVM (Kotlin/Java) plus the other native ports that
+      # live under sdks/community/ (go, rust, dart, ruby)
+      - "sdks/community/**/*.kt"
+      - "sdks/community/**/*.java"
+      - "sdks/community/**/*.go"
+      - "sdks/community/**/*.rs"
+      - "sdks/community/**/*.dart"
+      - "sdks/community/**/*.rb"
     exclude_patterns:
       - "**/test/**"
       - "**/tests/**"
       - "**/__tests__/**"
       - "**/*.test.*"
       - "**/*.spec.*"
+      # Codegen output — protobuf codec + generated stubs dominate and add no
+      # hand-written signal
+      - "**/generated/**"
+      - "**/*.pb.*"
     skip_dirs:
       - node_modules
       - dist
@@ -100,35 +130,43 @@ sources:
 tools:
   - name: search-docs
     type: search
-    description: "Search the server's documentation to retrieve relevant information. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, and how-tos for building with CopilotKit. Use this for CopilotKit usage and configuration questions. NOT for AG-UI protocol docs (use search-ag-ui-docs) and NOT for source code (use search-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: docs
     default_limit: 5
     max_limit: 20
     result_format: docs
+    search_mode: hybrid
+    min_score: 0.3
 
   - name: search-code
     type: search
-    description: "Search the server's indexed codebase to find relevant code snippets and implementations. Use this tool when you need to understand how something is implemented, find code examples, or locate specific functionality in the codebase. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the CopilotKit/CopilotKit source code (React UI, hooks, runtime, CSS, framework adapters) to understand how something is implemented or find code examples. Covers library/package source only — NOT example or showcase apps, NOT generated .d.ts type declarations, and NOT the AG-UI protocol SDK (use search-ag-ui-code for that). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: code
     default_limit: 10
     max_limit: 20
     result_format: code
+    search_mode: hybrid
+    min_score: 0.3
 
   - name: search-ag-ui-docs
     type: search
-    description: "Search the AG-UI protocol documentation to retrieve relevant information about the Agent-User Interaction protocol, events, types, and integration guides. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: ag-ui-docs
     default_limit: 5
     max_limit: 20
     result_format: docs
+    search_mode: hybrid
+    min_score: 0.3
 
   - name: search-ag-ui-code
     type: search
-    description: "Search the AG-UI TypeScript SDK codebase to find relevant code snippets and implementations. Use this tool to understand AG-UI protocol types, events, encoders, and client implementations. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the AG-UI protocol SDK source code across languages — TypeScript SDK packages, the Python SDK and framework integration adapters (LangGraph, CrewAI, etc.), and the JVM/multi-language community SDK. Use this to understand AG-UI protocol types, events, encoders, and client/adapter implementations. This is AG-UI SDK code only — NOT CopilotKit React/CSS/hooks (use search-code) and NOT prose docs (use search-ag-ui-docs). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: ag-ui-code
     default_limit: 10
     max_limit: 20
     result_format: code
+    search_mode: hybrid
+    min_score: 0.3
 
   - name: explore-docs
     type: bash
@@ -194,9 +232,12 @@ webhook:
       - ag-ui-code
   path_triggers:
     docs:
-      - "docs/"
+      - "showcase/shell-docs/src/content/"
     code: []
     ag-ui-docs:
       - "docs/"
     ag-ui-code:
       - "sdks/typescript/packages/"
+      - "integrations/"
+      - "sdks/python/"
+      - "sdks/community/"

docs/analytics.html

@@ -741,8 +741,11 @@ <h2 style="margin-bottom: 16px; font-size: 1.2rem">Analytics Token</h2>
       })();
 
       // --- Constants / helpers ---
-      // Upper bound used by the "All time" preset. Server-side MAX_DAYS must
-      // stay >= this value.
+      // Sentinel "days" value the "All time" preset sends. MUST match
+      // ALL_TIME_DAYS in src/db/analytics.ts (the canonical source) — the DB
+      // layer treats a window >= this value as "no lower time bound" so the
+      // totals span every row. Server-side MAX_DAYS stays strictly above this
+      // value so the preset is admitted rather than 400'd.
       var ALL_TIME_DAYS = 99999;
 
       // Dedicated error so load()'s catch block can distinguish auth failures

scripts/seed-analytics.ts

@@ -97,9 +97,20 @@ interface SeedRow {
   latency_ms: number;
   source_name: string;
   session_id: string | null;
+  request_source: string | null;
   created_at: Date;
 }
 
+// Request-origin mix: mostly real users, a slice of synthetic/analysis traffic
+// plus some untagged (null) rows to mimic historical data predating the column.
+function pickRequestSource(): string | null {
+  const r = Math.random();
+  if (r < 0.7) return "user";
+  if (r < 0.82) return "synthetic";
+  if (r < 0.9) return "analysis";
+  return null; // untagged historical row
+}
+
 function generateRow(): SeedRow {
   const isEmptyResult = Math.random() < 0.15; // ~15% empty
   const resultCount = isEmptyResult ? 0 : randomInt(1, 20);
@@ -115,6 +126,7 @@ function generateRow(): SeedRow {
     latency_ms: randomInt(50, 500),
     source_name: pick(SOURCE_NAMES),
     session_id: Math.random() < 0.6 ? `sess_${randomInt(1000, 9999)}` : null,
+    request_source: pickRequestSource(),
     created_at: randomTimestamp(),
   };
 }
@@ -139,8 +151,8 @@ async function main() {
 
   for (const row of rows) {
     await pool.query(
-      `INSERT INTO query_log (tool_name, query_text, result_count, top_score, latency_ms, source_name, session_id, created_at)
-             VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+      `INSERT INTO query_log (tool_name, query_text, result_count, top_score, latency_ms, source_name, session_id, request_source, created_at)
+             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
       [
         row.tool_name,
         row.query_text,
@@ -149,6 +161,7 @@ async function main() {
         row.latency_ms,
         row.source_name,
         row.session_id,
+        row.request_source,
         row.created_at,
       ],
     );

scripts/test-path-filter.ts

@@ -37,7 +37,11 @@ function makeFileSourceConfig(
     };
 }
 
-// Code source config — matches the real pathfinder.yaml code source patterns
+// Code source config — mirrors the real deploy/copilotkit-docs.yaml `code`
+// source patterns (file_patterns + exclude_patterns). Keep in sync with that
+// file: the excludes drop the v1 packages, every flavor of test file, the
+// demo/example and showcase apps, the Next.js build output, and generated
+// .d.ts declarations.
 const codeConfig = makeFileSourceConfig(
     ['**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.py'],
     [
@@ -48,10 +52,16 @@ const codeConfig = makeFileSourceConfig(
         '**/__tests__/**',
         '**/*.test.*',
         '**/*.spec.*',
+        'examples/**',
+        'showcase/**',
+        '**/.next/**',
+        '**/*.d.ts',
     ],
 );
 
-// Docs source config — only *.mdx, no excludes
+// Docs source config — mirrors the real `docs` source: only *.mdx, no excludes.
+// (The source path is showcase/shell-docs/src/content/docs/, so derived paths
+// live under that tree.)
 const docsConfig = makeFileSourceConfig(['**/*.mdx']);
 
 console.log('=== Path Filter Tests ===\n');
@@ -79,17 +89,27 @@ assert(!matchesPatterns('packages/v2/runtime/src/runtime.spec.ts', codeConfig),
 assert(!matchesPatterns('packages/v2/runtime/src/runtime.test.tsx', codeConfig), '*.test.tsx excluded');
 assert(!matchesPatterns('examples/with-agno/tests/test_agent.py', codeConfig), 'Python test dir excluded');
 
+// --- demo/example, showcase, build output, and generated decls excluded ---
+console.log('\n--- examples/showcase/.next/.d.ts exclusion ---');
+assert(!matchesPatterns('examples/with-agno/src/agent.py', codeConfig), 'examples/ src excluded');
+assert(!matchesPatterns('examples/coagents-research-canvas/ui/src/page.tsx', codeConfig), 'examples/ deep path excluded');
+assert(!matchesPatterns('showcase/shell-docs/src/lib/util.ts', codeConfig), 'showcase/ excluded');
+assert(!matchesPatterns('packages/v2/react/.next/server/app/page.js', codeConfig), '**/.next/** build output excluded');
+assert(!matchesPatterns('packages/v2/runtime/dist/index.d.ts', codeConfig), '**/*.d.ts generated decls excluded');
+assert(!matchesPatterns('packages/v2/runtime/src/types.d.ts', codeConfig), '.d.ts excluded even alongside sources');
+
 // --- non-test files included ---
 console.log('\n--- non-test files included ---');
 assert(matchesPatterns('packages/v2/runtime/src/runtime.ts', codeConfig), 'normal src file included');
-assert(matchesPatterns('examples/with-agno/src/agent.py', codeConfig), 'example src file included');
+assert(matchesPatterns('packages/shared/src/utils.ts', codeConfig), 'library package src included');
 assert(matchesPatterns('src/index.ts', codeConfig), 'root src file included');
 
 // --- docs (*.mdx only) ---
+// Paths reflect the current docs source tree: showcase/shell-docs/src/content/docs/
 console.log('\n--- docs (*.mdx only) ---');
-assert(matchesPatterns('docs/content/docs/(root)/quickstart.mdx', docsConfig), 'docs included');
-assert(matchesPatterns('docs/content/docs/reference/v1/hooks.mdx', docsConfig), 'v1 docs included (no code excludes on docs)');
-assert(!matchesPatterns('docs/content/docs/reference/v1/hooks.ts', docsConfig), 'ts file not matched by docs config');
+assert(matchesPatterns('showcase/shell-docs/src/content/docs/(root)/quickstart.mdx', docsConfig), 'docs included');
+assert(matchesPatterns('showcase/shell-docs/src/content/docs/reference/hooks.mdx', docsConfig), 'nested docs included (no code excludes on docs)');
+assert(!matchesPatterns('showcase/shell-docs/src/content/docs/reference/hooks.ts', docsConfig), 'ts file not matched by docs config');
 
 // --- edge cases ---
 console.log('\n--- edge cases ---');