Add local RAG harness: indexer, retriever, prompt injection and docs

README.md

@@ -50,6 +50,26 @@ The install script downloads the correct tarball for your platform, extracts it
 
 SmallCode includes [BoneScript](https://github.com/Doorman11991/BoneScript) and [budget-aware-mcp](https://github.com/Doorman11991/budget-aware-mcp) as dependencies — everything installs in one go.
 
+### RAG harness quick run
+
+SmallCode runs as a terminal UI harness by default:
+
+```bash
+smallcode                 # fullscreen TUI
+smallcode --classic       # readline UI fallback
+node bin/smallcode.js     # from a repo checkout
+```
+
+To build the local GitHub RAG database, create `.smallcode/rag/repos.json` with a `repos` array, then run:
+
+```bash
+npm run rag:index
+# or, after install:
+smallcode-rag-index
+```
+
+See [docs/rag-harness.md](docs/rag-harness.md) for the full LM Studio/llama.cpp setup, UI walkthrough, RAG config, indexing, and web-fallback flow.
+
 ### Requirements
 
 - Node.js 18+ (LTS recommended — 20.x or 22.x have prebuilt binaries for SQLite)

bin/rag-index.js

@@ -6,6 +6,18 @@ const path = require('path');
 const { RagIndexStore } = require('../src/rag/index_store');
 const { ensureRepo, collectSnippets } = require('../src/rag/github_scraper');
 
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`Usage: smallcode-rag-index
+
+Reads .smallcode/rag/repos.json (or SMALLCODE_RAG_REPOS) and writes .smallcode/rag/index.json.
+
+Example repos.json:
+{
+  "repos": ["https://github.com/owner/repo.git"]
+}`);
+  process.exit(0);
+}
+
 function loadConfig() {
   const cfgPath = process.env.SMALLCODE_RAG_REPOS || path.join(process.cwd(), '.smallcode', 'rag', 'repos.json');
   if (!fs.existsSync(cfgPath)) {

bin/smallcode.js

@@ -245,6 +245,7 @@ let _planTracker = null;
 let _bootstrapDetector = null;
 let _testRunnerDetector = null;
 let _knowledgeLoader = null;
+let _ragRetriever = null;
 const improvementAttempts = {}; // filePath → attempt count
 
 async function runTUI(config) {
@@ -684,6 +685,12 @@ async function runAgentLoop(userMessage, config) {
     const { KnowledgeLoader } = require('../src/knowledge/loader');
     _knowledgeLoader = new KnowledgeLoader({ rootDir: process.cwd() });
   } catch { _knowledgeLoader = null; }
+  // RAG retriever: local code examples indexed from GitHub repos or other source trees.
+  try {
+    const { RagRetriever } = require('../src/rag/retriever');
+    _ragRetriever = new RagRetriever();
+    _ragRetriever.load();
+  } catch { _ragRetriever = null; }
   // Trust decay (Feature 13) resets per agent loop turn so TUI sessions
   // don't accumulate decay from unrelated prior requests.
   try {
@@ -1942,7 +1949,7 @@ CRITICAL — large file rule: write_file calls are limited to 60 lines / ~8KB. l
   }
 
   // Legacy behavior: everything in the system prompt
-  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getActivePlanContext() + getTestRunnerContext();
+  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getRagContext(messages) + getActivePlanContext() + getTestRunnerContext();
 
   return prompt;
 }
@@ -1957,6 +1964,7 @@ function buildDynamicContext(messages) {
     getMemoryContext(messages),
     getSkillContext(messages),
     getKnowledgeContext(messages),
+    getRagContext(messages),
     // Note: getPluginPrompts() stays in the system prompt — plugin instructions
     //   are authoritative and should come from the system role.
     // Note: getActivePlanContext() stays in the system prompt — the model needs
@@ -2006,6 +2014,20 @@ function getKnowledgeContext(messages) {
   }
 }
 
+// Auto-load similar code snippets from the local RAG index. The index is
+// created with `npm run rag:index` / `smallcode-rag-index` and stays fully local.
+function getRagContext(messages) {
+  try {
+    if (!_ragRetriever) return '';
+    const lastUser = [...messages].reverse().find(m => m.role === 'user');
+    if (!lastUser || typeof lastUser.content !== 'string') return '';
+    const maxChars = Math.min(6000, Math.floor(((config?.context?.detected_window || 32768) * 0.06) * 4));
+    return _ragRetriever.formatForPrompt(lastUser.content, { limit: 6, maxChars });
+  } catch {
+    return '';
+  }
+}
+
 // Auto-load relevant memory for the current task (injected into system prompt)
 // Fix #15: Only inject memory objects with score >= 2 (at least 2 word matches)
 // to avoid burning tokens on low-relevance hits.

docs/rag-harness.md

@@ -0,0 +1,133 @@
+# RAG Coding Harness Guide
+
+This repo runs as a terminal UI coding harness. The default interface is the fullscreen TUI; use `--classic` if your terminal does not support alternate-screen rendering.
+
+## 1. Start a local model server
+
+SmallCode talks to any OpenAI-compatible local endpoint.
+
+### LM Studio
+
+1. Open LM Studio.
+2. Download/load a coder model such as Qwen Coder or another 8B-35B local coding model.
+3. Start the **Local Server**.
+4. Note the model name shown by LM Studio and the base URL, usually `http://localhost:1234/v1`.
+
+Create `.env` in the project you want to edit:
+
+```bash
+SMALLCODE_MODEL=your-loaded-model-name
+SMALLCODE_BASE_URL=http://localhost:1234/v1
+```
+
+### llama.cpp server
+
+Start llama.cpp with its OpenAI-compatible server, then point SmallCode at it:
+
+```bash
+SMALLCODE_MODEL=local-model
+SMALLCODE_BASE_URL=http://localhost:8080/v1
+```
+
+## 2. Run the UI harness
+
+From the project directory you want the agent to edit:
+
+```bash
+smallcode
+```
+
+If you are developing from this repository checkout instead of a global install:
+
+```bash
+node bin/smallcode.js
+```
+
+Useful launch modes:
+
+```bash
+smallcode --classic                    # readline UI instead of fullscreen UI
+smallcode -P "fix the parser bug"      # one-shot prompt
+smallcode --non-interactive "refactor" # stdin/script friendly mode
+smallcode --resume                     # continue previous session
+```
+
+Inside the UI:
+
+- Type your task and press Enter.
+- Use `/help` for commands.
+- Use `/plan` to inspect the active plan.
+- Use `/undo` to revert the last edit.
+- Use `/quit` to exit.
+
+## 3. Create the local GitHub RAG database
+
+Create `.smallcode/rag/repos.json` in the workspace where you run SmallCode:
+
+```json
+{
+  "repos": [
+    "https://github.com/owner/framework-example.git",
+    "https://github.com/owner/language-examples.git"
+  ]
+}
+```
+
+Optional fields:
+
+```json
+{
+  "cacheDir": ".smallcode/rag/repos",
+  "indexPath": ".smallcode/rag/index.json",
+  "repos": ["https://github.com/owner/repo.git"]
+}
+```
+
+Build/update the index:
+
+```bash
+npm run rag:index
+```
+
+After package installation, the same command is also available as:
+
+```bash
+smallcode-rag-index
+```
+
+The indexer shallow-clones or fast-forwards each repo, chunks source files, computes lightweight local embeddings, and saves `.smallcode/rag/index.json`.
+
+## 4. Use RAG in the harness
+
+Once `.smallcode/rag/index.json` exists, start the UI normally:
+
+```bash
+smallcode
+```
+
+For each user turn, SmallCode now:
+
+1. plans/classifies the request,
+2. retrieves similar snippets from the local RAG index,
+3. injects the best snippets into the model context,
+4. asks the local model to do one step at a time through the normal tool loop.
+
+No cloud embedding service is required. The default embedding path is dependency-free and optimized for fast local startup.
+
+## 5. Optional web fallback when RAG is weak
+
+Web search is disabled by default. Enable it only when you want the model to search externally after local RAG confidence is low:
+
+```bash
+SMALLCODE_WEB_BROWSE=true smallcode
+```
+
+When enabled, low-confidence RAG context tells the model to use `web_search` with a GitHub/code-example query if it gets blocked.
+
+## 6. Speed tips for local models
+
+- Prefer the fullscreen UI (`smallcode`) for normal work; use `--classic` only for terminal compatibility issues.
+- Keep `SMALLCODE_CACHE_SPLIT` at its default (`true`) so llama.cpp-style KV cache reuse is not invalidated by dynamic context.
+- Keep RAG repos focused: index examples for the frameworks/languages you actually use instead of huge monorepos.
+- Use 8B-35B coder models; very small models often fail multi-step tool use.
+- If the model struggles, ask for a smaller concrete task first, then continue with follow-up prompts.