From 8fefc11e0f1228f57deb32a1976d92bb35ba3be8 Mon Sep 17 00:00:00 2001
From: Muzaffer Mahi Can <141342610+muzaffermahi@users.noreply.github.com>
Date: Fri, 29 May 2026 16:16:07 +0300
Subject: [PATCH] Wire RAG into harness and document UI workflow

---
 README.md                 |  20 ++++++
 bin/rag-index.js          |  47 ++++++++++++++
 bin/smallcode.js          |  24 ++++++-
 docs/rag-harness.md       | 133 ++++++++++++++++++++++++++++++++++++++
 package-lock.json         |  57 +++++++++++++---
 package.json              |  14 ++--
 src/rag/github_scraper.js |  57 ++++++++++++++++
 src/rag/index_store.js    |  82 +++++++++++++++++++++++
 src/rag/retriever.js      |  80 +++++++++++++++++++++++
 test/rag.test.js          |  32 +++++++++
 10 files changed, 532 insertions(+), 14 deletions(-)
 create mode 100755 bin/rag-index.js
 create mode 100644 docs/rag-harness.md
 create mode 100644 src/rag/github_scraper.js
 create mode 100644 src/rag/index_store.js
 create mode 100644 src/rag/retriever.js
 create mode 100644 test/rag.test.js

diff --git a/README.md b/README.md
index e6d86006..bf134fc8 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,26 @@ The install script downloads the correct tarball for your platform, extracts it
 
 SmallCode includes [BoneScript](https://github.com/Doorman11991/BoneScript) and [budget-aware-mcp](https://github.com/Doorman11991/budget-aware-mcp) as dependencies — everything installs in one go.
 
+### RAG harness quick run
+
+SmallCode runs as a terminal UI harness by default:
+
+```bash
+smallcode                 # fullscreen TUI
+smallcode --classic       # readline UI fallback
+node bin/smallcode.js     # from a repo checkout
+```
+
+To build the local GitHub RAG database, create `.smallcode/rag/repos.json` with a `repos` array, then run:
+
+```bash
+npm run rag:index
+# or, after install:
+smallcode-rag-index
+```
+
+See [docs/rag-harness.md](docs/rag-harness.md) for the full LM Studio/llama.cpp setup, UI walkthrough, RAG config, indexing, and web-fallback flow.
+
 ### Requirements
 
 - Node.js 18+ (LTS recommended — 20.x or 22.x have prebuilt binaries for SQLite)
diff --git a/bin/rag-index.js b/bin/rag-index.js
new file mode 100755
index 00000000..cfc274ac
--- /dev/null
+++ b/bin/rag-index.js
@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { RagIndexStore } = require('../src/rag/index_store');
+const { ensureRepo, collectSnippets } = require('../src/rag/github_scraper');
+
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`Usage: smallcode-rag-index
+
+Reads .smallcode/rag/repos.json (or SMALLCODE_RAG_REPOS) and writes .smallcode/rag/index.json.
+
+Example repos.json:
+{
+  "repos": ["https://github.com/owner/repo.git"]
+}`);
+  process.exit(0);
+}
+
+function loadConfig() {
+  const cfgPath = process.env.SMALLCODE_RAG_REPOS || path.join(process.cwd(), '.smallcode', 'rag', 'repos.json');
+  if (!fs.existsSync(cfgPath)) {
+    console.error(`Missing repo list: ${cfgPath}`);
+    process.exit(1);
+  }
+  return JSON.parse(fs.readFileSync(cfgPath, 'utf-8'));
+}
+
+(function main() {
+  const cfg = loadConfig();
+  const repoDir = cfg.cacheDir || path.join(process.cwd(), '.smallcode', 'rag', 'repos');
+  fs.mkdirSync(repoDir, { recursive: true });
+  const store = new RagIndexStore({ path: cfg.indexPath });
+  store.load();
+
+  let total = 0;
+  for (const url of (cfg.repos || [])) {
+    const local = ensureRepo(url, repoDir);
+    const snippets = collectSnippets(local, url);
+    total += snippets.length;
+    store.upsertMany(snippets);
+    process.stdout.write(`indexed ${snippets.length} snippets from ${url}\n`);
+  }
+  store.save();
+  process.stdout.write(`done. total snippets: ${total}\n`);
+})();
diff --git a/bin/smallcode.js b/bin/smallcode.js
index 91b688a0..3cdf2317 100755
--- a/bin/smallcode.js
+++ b/bin/smallcode.js
@@ -245,6 +245,7 @@ let _planTracker = null;
 let _bootstrapDetector = null;
 let _testRunnerDetector = null;
 let _knowledgeLoader = null;
+let _ragRetriever = null;
 const improvementAttempts = {}; // filePath → attempt count
 
 async function runTUI(config) {
@@ -684,6 +685,12 @@ async function runAgentLoop(userMessage, config) {
     const { KnowledgeLoader } = require('../src/knowledge/loader');
     _knowledgeLoader = new KnowledgeLoader({ rootDir: process.cwd() });
   } catch { _knowledgeLoader = null; }
+  // RAG retriever: local code examples indexed from GitHub repos or other source trees.
+  try {
+    const { RagRetriever } = require('../src/rag/retriever');
+    _ragRetriever = new RagRetriever();
+    _ragRetriever.load();
+  } catch { _ragRetriever = null; }
   // Trust decay (Feature 13) resets per agent loop turn so TUI sessions
   // don't accumulate decay from unrelated prior requests.
   try {
@@ -1942,7 +1949,7 @@ CRITICAL — large file rule: write_file calls are limited to 60 lines / ~8KB. l
   }
 
   // Legacy behavior: everything in the system prompt
-  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getActivePlanContext() + getTestRunnerContext();
+  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getRagContext(messages) + getActivePlanContext() + getTestRunnerContext();
 
   return prompt;
 }
@@ -1957,6 +1964,7 @@ function buildDynamicContext(messages) {
     getMemoryContext(messages),
     getSkillContext(messages),
     getKnowledgeContext(messages),
+    getRagContext(messages),
     // Note: getPluginPrompts() stays in the system prompt — plugin instructions
     //   are authoritative and should come from the system role.
     // Note: getActivePlanContext() stays in the system prompt — the model needs
@@ -2006,6 +2014,20 @@ function getKnowledgeContext(messages) {
   }
 }
 
+// Auto-load similar code snippets from the local RAG index. The index is
+// created with `npm run rag:index` / `smallcode-rag-index` and stays fully local.
+function getRagContext(messages) {
+  try {
+    if (!_ragRetriever) return '';
+    const lastUser = [...messages].reverse().find(m => m.role === 'user');
+    if (!lastUser || typeof lastUser.content !== 'string') return '';
+    const maxChars = Math.min(6000, Math.floor(((config?.context?.detected_window || 32768) * 0.06) * 4));
+    return _ragRetriever.formatForPrompt(lastUser.content, { limit: 6, maxChars });
+  } catch {
+    return '';
+  }
+}
+
 // Auto-load relevant memory for the current task (injected into system prompt)
 // Fix #15: Only inject memory objects with score >= 2 (at least 2 word matches)
 // to avoid burning tokens on low-relevance hits.
diff --git a/docs/rag-harness.md b/docs/rag-harness.md
new file mode 100644
index 00000000..112efb63
--- /dev/null
+++ b/docs/rag-harness.md
@@ -0,0 +1,133 @@
+# RAG Coding Harness Guide
+
+This repo runs as a terminal UI coding harness. The default interface is the fullscreen TUI; use `--classic` if your terminal does not support alternate-screen rendering.
+
+## 1. Start a local model server
+
+SmallCode talks to any OpenAI-compatible local endpoint.
+
+### LM Studio
+
+1. Open LM Studio.
+2. Download/load a coder model such as Qwen Coder or another 8B-35B local coding model.
+3. Start the **Local Server**.
+4. Note the model name shown by LM Studio and the base URL, usually `http://localhost:1234/v1`.
+
+Create `.env` in the project you want to edit:
+
+```bash
+SMALLCODE_MODEL=your-loaded-model-name
+SMALLCODE_BASE_URL=http://localhost:1234/v1
+```
+
+### llama.cpp server
+
+Start llama.cpp with its OpenAI-compatible server, then point SmallCode at it:
+
+```bash
+SMALLCODE_MODEL=local-model
+SMALLCODE_BASE_URL=http://localhost:8080/v1
+```
+
+## 2. Run the UI harness
+
+From the project directory you want the agent to edit:
+
+```bash
+smallcode
+```
+
+If you are developing from this repository checkout instead of a global install:
+
+```bash
+node bin/smallcode.js
+```
+
+Useful launch modes:
+
+```bash
+smallcode --classic                    # readline UI instead of fullscreen UI
+smallcode -P "fix the parser bug"      # one-shot prompt
+smallcode --non-interactive "refactor" # stdin/script friendly mode
+smallcode --resume                     # continue previous session
+```
+
+Inside the UI:
+
+- Type your task and press Enter.
+- Use `/help` for commands.
+- Use `/plan` to inspect the active plan.
+- Use `/undo` to revert the last edit.
+- Use `/quit` to exit.
+
+## 3. Create the local GitHub RAG database
+
+Create `.smallcode/rag/repos.json` in the workspace where you run SmallCode:
+
+```json
+{
+  "repos": [
+    "https://github.com/owner/framework-example.git",
+    "https://github.com/owner/language-examples.git"
+  ]
+}
+```
+
+Optional fields:
+
+```json
+{
+  "cacheDir": ".smallcode/rag/repos",
+  "indexPath": ".smallcode/rag/index.json",
+  "repos": ["https://github.com/owner/repo.git"]
+}
+```
+
+Build/update the index:
+
+```bash
+npm run rag:index
+```
+
+After package installation, the same command is also available as:
+
+```bash
+smallcode-rag-index
+```
+
+The indexer shallow-clones or fast-forwards each repo, chunks source files, computes lightweight local embeddings, and saves `.smallcode/rag/index.json`.
+
+## 4. Use RAG in the harness
+
+Once `.smallcode/rag/index.json` exists, start the UI normally:
+
+```bash
+smallcode
+```
+
+For each user turn, SmallCode now:
+
+1. plans/classifies the request,
+2. retrieves similar snippets from the local RAG index,
+3. injects the best snippets into the model context,
+4. asks the local model to do one step at a time through the normal tool loop.
+
+No cloud embedding service is required. The default embedding path is dependency-free and optimized for fast local startup.
+
+## 5. Optional web fallback when RAG is weak
+
+Web search is disabled by default. Enable it only when you want the model to search externally after local RAG confidence is low:
+
+```bash
+SMALLCODE_WEB_BROWSE=true smallcode
+```
+
+When enabled, low-confidence RAG context tells the model to use `web_search` with a GitHub/code-example query if it gets blocked.
+
+## 6. Speed tips for local models
+
+- Prefer the fullscreen UI (`smallcode`) for normal work; use `--classic` only for terminal compatibility issues.
+- Keep `SMALLCODE_CACHE_SPLIT` at its default (`true`) so llama.cpp-style KV cache reuse is not invalidated by dynamic context.
+- Keep RAG repos focused: index examples for the frameworks/languages you actually use instead of huge monorepos.
+- Use 8B-35B coder models; very small models often fail multi-step tool use.
+- If the model struggles, ask for a smaller concrete task first, then continue with follow-up prompts.
diff --git a/package-lock.json b/package-lock.json
index 7012edc7..8c11189b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "smallcode",
-  "version": "0.9.6",
+  "version": "1.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "smallcode",
-      "version": "0.9.6",
+      "version": "1.3.0",
       "license": "MIT",
       "dependencies": {
         "bonescript-compiler": "0.14.0",
@@ -18,7 +18,8 @@
       },
       "bin": {
         "smallcode": "bin/smallcode.js",
-        "smallcode-init": "bin/init.js"
+        "smallcode-init": "bin/init.js",
+        "smallcode-rag-index": "bin/rag-index.js"
       },
       "devDependencies": {
         "@types/node": "^25.9.0",
@@ -30,9 +31,19 @@
         "node": ">=18.0.0"
       },
       "optionalDependencies": {
-        "budget-aware-mcp": "0.6.1",
-        "playwright-extra": "4.3.0",
-        "puppeteer-extra-plugin-stealth": "2.11.0"
+        "budget-aware-mcp": "0.6.1"
+      },
+      "peerDependencies": {
+        "playwright-extra": "^4.3.0",
+        "puppeteer-extra-plugin-stealth": "^2.11.0"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra-plugin-stealth": {
+          "optional": true
+        }
       }
     },
     "node_modules/@babel/code-frame": {
@@ -1226,6 +1237,7 @@
       "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "@types/ms": "*"
       }
@@ -1262,7 +1274,8 @@
       "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",
       "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",
       "license": "MIT",
-      "optional": true
+      "optional": true,
+      "peer": true
     },
     "node_modules/@types/node": {
       "version": "25.9.0",
@@ -1747,6 +1760,7 @@
       "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -2257,6 +2271,7 @@
       "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "for-own": "^0.1.3",
         "is-plain-object": "^2.0.1",
@@ -2912,6 +2927,7 @@
       "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -2922,6 +2938,7 @@
       "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "for-in": "^1.0.1"
       },
@@ -2960,6 +2977,7 @@
       "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -3316,7 +3334,8 @@
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
       "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
       "license": "MIT",
-      "optional": true
+      "optional": true,
+      "peer": true
     },
     "node_modules/is-extendable": {
       "version": "0.1.1",
@@ -3324,6 +3343,7 @@
       "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -3353,6 +3373,7 @@
       "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "isobject": "^3.0.1"
       },
@@ -3392,6 +3413,7 @@
       "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -4201,6 +4223,7 @@
       "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "universalify": "^2.0.0"
       },
@@ -4214,6 +4237,7 @@
       "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "is-buffer": "^1.1.5"
       },
@@ -4227,6 +4251,7 @@
       "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -4401,6 +4426,7 @@
       "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "arr-union": "^3.1.0",
         "clone-deep": "^0.2.4",
@@ -4519,6 +4545,7 @@
       "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "for-in": "^0.1.3",
         "is-extendable": "^0.1.1"
@@ -4533,6 +4560,7 @@
       "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -4796,7 +4824,8 @@
       "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
       "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
       "license": "BlueOak-1.0.0",
-      "optional": true
+      "optional": true,
+      "peer": true
     },
     "node_modules/parse-json": {
       "version": "5.2.0",
@@ -4963,6 +4992,7 @@
       "integrity": "sha512-/Hec3BmYMY/GznBo0ZPsSqm6IHil7jInxLz+9/UnBBC5Ozh2abVNnv+vYNJ+JKKRVtiKyHCrJpKRibJ9uribZw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "debug": "^4.3.4"
       },
@@ -5057,6 +5087,7 @@
       "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "@types/debug": "^4.1.0",
         "debug": "^4.1.1",
@@ -5084,6 +5115,7 @@
       "integrity": "sha512-BqckPV95MHP25quZgzBnZJD8S38ZYP4B3HJ3Kr/vibqxJxhK6L1VQ6jnu/JcFKV0wzCIQPrCiiavZnwE5u1C2A==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "debug": "^4.1.1",
         "puppeteer-extra-plugin": "^3.2.2",
@@ -5111,6 +5143,7 @@
       "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "debug": "^4.1.1",
         "fs-extra": "^10.0.0",
@@ -5139,6 +5172,7 @@
       "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "debug": "^4.1.1",
         "deepmerge": "^4.2.2",
@@ -5322,6 +5356,7 @@
       "integrity": "sha512-LKg+Cr2ZF61fkcaK1UdkH2yEBBKnYjTyWzTJT6KNPcSPaiT7HSdhtMXQuN5wkTX0Xu72KQ1l8S42rlmexS2hSA==",
       "license": "BlueOak-1.0.0",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "glob": "^13.0.3",
         "package-json-from-dist": "^1.0.1"
@@ -5446,6 +5481,7 @@
       "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "is-extendable": "^0.1.1",
         "kind-of": "^2.0.1",
@@ -5462,6 +5498,7 @@
       "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "dependencies": {
         "is-buffer": "^1.0.2"
       },
@@ -5475,6 +5512,7 @@
       "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -6104,6 +6142,7 @@
       "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
       "license": "MIT",
       "optional": true,
+      "peer": true,
       "engines": {
         "node": ">= 10.0.0"
       }
diff --git a/package.json b/package.json
index d9ca38d9..a31023de 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
   "main": "src/api/index.js",
   "bin": {
     "smallcode": "bin/smallcode.js",
-    "smallcode-init": "bin/init.js"
+    "smallcode-init": "bin/init.js",
+    "smallcode-rag-index": "bin/rag-index.js"
   },
   "scripts": {
     "build": "node build.js",
@@ -21,7 +22,8 @@
     "bench:diff": "node bench/diff.js",
     "test": "node --test test/*.test.js",
     "test:e2e": "node test/e2e_smoke.js",
-    "test:e2e:offline": "node test/e2e_offline.js"
+    "test:e2e:offline": "node test/e2e_offline.js",
+    "rag:index": "node bin/rag-index.js"
   },
   "dependencies": {
     "bonescript-compiler": "0.14.0",
@@ -39,8 +41,12 @@
     "puppeteer-extra-plugin-stealth": "^2.11.0"
   },
   "peerDependenciesMeta": {
-    "playwright-extra": { "optional": true },
-    "puppeteer-extra-plugin-stealth": { "optional": true }
+    "playwright-extra": {
+      "optional": true
+    },
+    "puppeteer-extra-plugin-stealth": {
+      "optional": true
+    }
   },
   "keywords": [
     "ai",
diff --git a/src/rag/github_scraper.js b/src/rag/github_scraper.js
new file mode 100644
index 00000000..2bccafd2
--- /dev/null
+++ b/src/rag/github_scraper.js
@@ -0,0 +1,57 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const cp = require('child_process');
+
+const CODE_EXTS = new Set(['.js', '.ts', '.tsx', '.jsx', '.py', '.go', '.rs', '.java', '.cpp', '.c', '.cs', '.php', '.rb']);
+
+function runGit(args, cwd) {
+  cp.execFileSync('git', args, { cwd, stdio: 'pipe' });
+}
+
+function ensureRepo(url, targetDir) {
+  const name = url.replace(/\.git$/, '').split('/').slice(-2).join('__');
+  const out = path.join(targetDir, name);
+  if (fs.existsSync(path.join(out, '.git'))) runGit(['pull', '--ff-only'], out);
+  else runGit(['clone', '--depth=1', url, out], targetDir);
+  return out;
+}
+
+function* walk(dir) {
+  const list = fs.readdirSync(dir, { withFileTypes: true });
+  for (const ent of list) {
+    if (ent.name === '.git' || ent.name === 'node_modules' || ent.name === 'dist' || ent.name === 'build') continue;
+    const full = path.join(dir, ent.name);
+    if (ent.isDirectory()) yield* walk(full);
+    else if (ent.isFile()) yield full;
+  }
+}
+
+function chunkCode(text, chunkLines = 60, overlap = 15) {
+  const lines = text.split('\n');
+  const out = [];
+  for (let i = 0; i < lines.length; i += (chunkLines - overlap)) {
+    const piece = lines.slice(i, i + chunkLines).join('\n').trim();
+    if (piece.length >= 80) out.push({ startLine: i + 1, code: piece });
+  }
+  return out;
+}
+
+function collectSnippets(repoRoot, sourceUrl) {
+  const snippets = [];
+  for (const file of walk(repoRoot)) {
+    const ext = path.extname(file).toLowerCase();
+    if (!CODE_EXTS.has(ext)) continue;
+    const rel = path.relative(repoRoot, file);
+    const body = fs.readFileSync(file, 'utf-8');
+    for (const c of chunkCode(body)) {
+      const id = crypto.createHash('sha1').update(`${sourceUrl}:${rel}:${c.startLine}:${c.code}`).digest('hex');
+      snippets.push({ id, repo: sourceUrl, path: rel, startLine: c.startLine, code: c.code, lang: ext.slice(1) });
+    }
+  }
+  return snippets;
+}
+
+module.exports = { ensureRepo, collectSnippets, chunkCode };
diff --git a/src/rag/index_store.js b/src/rag/index_store.js
new file mode 100644
index 00000000..8fcf41f4
--- /dev/null
+++ b/src/rag/index_store.js
@@ -0,0 +1,82 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const DIMS = parseInt(process.env.SMALLCODE_RAG_DIMS || '512', 10);
+
+function tokenize(text) {
+  return String(text || '')
+    .toLowerCase()
+    .split(/[^a-z0-9_#]+/)
+    .filter(t => t.length >= 2);
+}
+
+function hashToken(token, dims = DIMS) {
+  let h = 2166136261;
+  for (let i = 0; i < token.length; i++) {
+    h ^= token.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return Math.abs(h) % dims;
+}
+
+function embed(text, dims = DIMS) {
+  const vec = new Float32Array(dims);
+  const toks = tokenize(text);
+  for (const t of toks) vec[hashToken(t, dims)] += 1;
+  let norm = 0;
+  for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i];
+  norm = Math.sqrt(norm) || 1;
+  for (let i = 0; i < vec.length; i++) vec[i] /= norm;
+  return Array.from(vec);
+}
+
+function cosine(a, b) {
+  const n = Math.min(a.length, b.length);
+  let s = 0;
+  for (let i = 0; i < n; i++) s += a[i] * b[i];
+  return s;
+}
+
+class RagIndexStore {
+  constructor(options = {}) {
+    this.path = options.path || path.join(process.cwd(), '.smallcode', 'rag', 'index.json');
+    this.docs = [];
+  }
+
+  load() {
+    try {
+      const raw = JSON.parse(fs.readFileSync(this.path, 'utf-8'));
+      this.docs = Array.isArray(raw.docs) ? raw.docs : [];
+    } catch {
+      this.docs = [];
+    }
+    return this.docs.length;
+  }
+
+  save() {
+    fs.mkdirSync(path.dirname(this.path), { recursive: true });
+    fs.writeFileSync(this.path, JSON.stringify({ version: 1, dims: DIMS, docs: this.docs }, null, 2));
+  }
+
+  upsertMany(snippets) {
+    const byId = new Map(this.docs.map(d => [d.id, d]));
+    for (const s of snippets) {
+      byId.set(s.id, { ...s, embedding: embed(`${s.path}\n${s.code}`) });
+    }
+    this.docs = [...byId.values()];
+    return this.docs.length;
+  }
+
+  search(query, limit = 8) {
+    if (!this.docs.length) return [];
+    const q = embed(query);
+    return this.docs
+      .map(d => ({ ...d, score: cosine(q, d.embedding || []) }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+}
+
+module.exports = { RagIndexStore, tokenize, embed, cosine };
diff --git a/src/rag/retriever.js b/src/rag/retriever.js
new file mode 100644
index 00000000..99498cea
--- /dev/null
+++ b/src/rag/retriever.js
@@ -0,0 +1,80 @@
+'use strict';
+
+const { RagIndexStore } = require('./index_store');
+
+function planQuery(query) {
+  const q = String(query || '').trim();
+  const intent = /bug|error|fix|failing|stack/.test(q.toLowerCase()) ? 'debug' : 'implement';
+  const focus = q.split(/\s+/).filter(w => w.length > 3).slice(0, 8);
+  return { intent, focus };
+}
+
+function googleFallbackUrl(query) {
+  return `https://www.google.com/search?q=${encodeURIComponent(query + ' github code example')}`;
+}
+
+class RagRetriever {
+  constructor(options = {}) {
+    this.index = options.index || new RagIndexStore(options.store || {});
+    this.maxLoops = options.maxLoops || 3;
+    this.loaded = false;
+  }
+
+  load() {
+    const count = this.index.load();
+    this.loaded = true;
+    return count;
+  }
+
+  ensureLoaded() {
+    if (!this.loaded) return this.load();
+    return this.index.docs.length;
+  }
+
+  retrieve(query, opts = {}) {
+    this.ensureLoaded();
+    const plan = planQuery(query);
+    const loops = [];
+    let hits = [];
+    for (let i = 0; i < (opts.maxLoops || this.maxLoops); i++) {
+      const subquery = i === 0 ? query : `${query} ${plan.focus.slice(0, i + 2).join(' ')}`;
+      hits = this.index.search(subquery, opts.limit || 8);
+      loops.push({ loop: i + 1, subquery, hitCount: hits.length, topScore: hits[0]?.score || 0 });
+      if ((hits[0]?.score || 0) >= 0.3) break;
+    }
+    return {
+      plan,
+      loops,
+      hits,
+      stuck: (hits[0]?.score || 0) < 0.2,
+      googleFallback: (hits[0]?.score || 0) < 0.2 ? googleFallbackUrl(query) : null,
+    };
+  }
+
+  formatForPrompt(query, opts = {}) {
+    if (process.env.SMALLCODE_RAG_DISABLE === 'true') return '';
+    const result = this.retrieve(query, opts);
+    if (!result.hits.length) return '';
+
+    const maxChars = opts.maxChars || 6000;
+    let used = 0;
+    const parts = [];
+    for (const hit of result.hits.slice(0, opts.limit || 6)) {
+      const code = String(hit.code || '').slice(0, opts.snippetChars || 1200);
+      const lang = String(hit.lang || '').replace(/[^a-z0-9+#-]/gi, '');
+      const header = `### ${hit.repo || 'local'}:${hit.path}:${hit.startLine || 1} score=${hit.score.toFixed(3)}`;
+      const block = `${header}\n` + '```' + `${lang}\n${code}\n` + '```';
+      if (used + block.length > maxChars) break;
+      used += block.length;
+      parts.push(block);
+    }
+    if (!parts.length) return '';
+
+    const stuckHint = result.stuck && process.env.SMALLCODE_WEB_BROWSE === 'true'
+      ? `\nRAG confidence is low. If blocked, use web_search with: ${query} github code example`
+      : '';
+    return `\n[RAG_CODE_CONTEXT] Retrieved similar code snippets. Use these as examples, not as authoritative project files.${stuckHint}\n${parts.join('\n\n')}\n[/RAG_CODE_CONTEXT]\n`;
+  }
+}
+
+module.exports = { RagRetriever, planQuery, googleFallbackUrl };
diff --git a/test/rag.test.js b/test/rag.test.js
new file mode 100644
index 00000000..636b5ca9
--- /dev/null
+++ b/test/rag.test.js
@@ -0,0 +1,32 @@
+'use strict';
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { RagIndexStore } = require('../src/rag/index_store');
+const { RagRetriever, planQuery } = require('../src/rag/retriever');
+
+test('RAG index ranks related snippets first', () => {
+  const store = new RagIndexStore({ path: '/tmp/nonexistent.json' });
+  store.upsertMany([
+    { id: '1', path: 'a.py', code: 'def binary_search(arr, x):\n  pass' },
+    { id: '2', path: 'b.js', code: 'function renderButton() { return "ok" }' },
+  ]);
+  const hits = store.search('how to implement binary search in python', 1);
+  assert.equal(hits[0].id, '1');
+});
+
+test('query planning detects debug intent', () => {
+  const p = planQuery('fix stack error in parser');
+  assert.equal(p.intent, 'debug');
+});
+
+test('retriever formats snippets for prompt injection', () => {
+  const store = new RagIndexStore({ path: '/tmp/nonexistent.json' });
+  store.upsertMany([
+    { id: '1', repo: 'local', path: 'router.ts', startLine: 7, lang: 'ts', code: 'export function routeRequest() { return true }' },
+  ]);
+  const retriever = new RagRetriever({ index: store });
+  retriever.loaded = true;
+  const prompt = retriever.formatForPrompt('route request in typescript', { limit: 1 });
+  assert.match(prompt, /RAG_CODE_CONTEXT/);
+  assert.match(prompt, /router\.ts:7/);
+});
