Codex/create rag coding harness 6vogkz

README.md

@@ -50,9 +50,59 @@ The install script downloads the correct tarball for your platform, extracts it
 
 SmallCode includes [BoneScript](https://github.com/Doorman11991/BoneScript) and [budget-aware-mcp](https://github.com/Doorman11991/budget-aware-mcp) as dependencies — everything installs in one go.
 
+### Fresh GitHub checkout quick start
+
+If you just cloned/pulled this repository, run it directly from the checkout first:
+
+```bash
+cd smallcode
+npm install
+
+# Start your local model server first (LM Studio, llama.cpp, Ollama, etc.)
+cat > .env <<'EOF'
+SMALLCODE_MODEL=your-local-model-name
+SMALLCODE_BASE_URL=http://localhost:1234/v1
+EOF
+
+node bin/smallcode.js
+```
+
+Optional: make the `smallcode` and `smallcode-rag-index` commands available globally from this checkout:
+
+```bash
+npm link
+smallcode --help
+```
+
+If the fullscreen UI has display issues in your terminal, start with `node bin/smallcode.js --classic`.
+
+### RAG harness quick run
+
+SmallCode runs as a terminal UI harness by default:
+
+```bash
+smallcode                 # fullscreen TUI
+smallcode --classic       # readline UI fallback
+node bin/smallcode.js     # from a repo checkout
+```
+
+To build the local GitHub RAG database, run the Python scraper/indexer with the curated starter corpus, or use the broader preset for a larger multi-language corpus:
+
+```bash
+npm run rag:index
+npm run rag:index -- --preset broad
+# or, after install:
+smallcode-rag-index --preset broad
+```
+
+For custom repos, create `.smallcode/rag/repos.json` with `preset`, `repos`, and chunking limits.
+
+See [docs/rag-harness.md](docs/rag-harness.md) for the full LM Studio/llama.cpp setup, UI walkthrough, RAG config, indexing, and web-fallback flow.
+
 ### Requirements
 
 - Node.js 18+ (LTS recommended — 20.x or 22.x have prebuilt binaries for SQLite)
+- Python 3 + Git for the RAG scraper/indexer (`npm run rag:index`)
 - A local LLM server (LM Studio, Ollama, or any OpenAI-compatible endpoint)
 
 **Optional** (for code graph + FTS5 memory search):

bin/rag-index.js

@@ -0,0 +1,110 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const cp = require('child_process');
+const readline = require('readline');
+const { RagIndexStore } = require('../src/rag/index_store');
+
+function usage() {
+  console.log(`Usage: smallcode-rag-index [--config PATH] [--preset starter|broad|none] [--repo URL_OR_PATH]
+
+Builds .smallcode/rag/index.json from snippet-sized code chunks scraped by scripts/rag_scraper.py.
+
+Config example (.smallcode/rag/repos.json):
+{
+  "preset": "starter",
+  "repos": ["https://github.com/owner/repo.git"],
+  "maxFilesPerRepo": 1000,
+  "maxSnippetsPerRepo": 4000,
+  "chunkLines": 80
+}
+
+Presets:
+  starter  curated multi-language framework/library set (default when no config exists)
+  broad    larger curated corpus across Python, JS/TS, Go, Rust, Java, C#, Ruby, PHP, C/C++
+  none     only scrape repos listed in config or --repo
+`);
+}
+
+function parseArgs(argv) {
+  const args = { repos: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--help' || a === '-h') args.help = true;
+    else if (a === '--config') args.config = argv[++i];
+    else if (a === '--preset') args.preset = argv[++i];
+    else if (a === '--repo') args.repos.push(argv[++i]);
+    else if (a === '--index-path') args.indexPath = argv[++i];
+    else throw new Error(`Unknown argument: ${a}`);
+  }
+  return args;
+}
+
+function readConfig(configPath) {
+  if (!fs.existsSync(configPath)) return {};
+  return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+}
+
+function findPython() {
+  for (const bin of ['python3', 'python']) {
+    const r = cp.spawnSync(bin, ['--version'], { encoding: 'utf-8' });
+    if (r.status === 0) return bin;
+  }
+  throw new Error('Python 3 is required for the RAG scraper (tried python3 and python).');
+}
+
+async function loadJsonl(file, onRecord) {
+  const rl = readline.createInterface({ input: fs.createReadStream(file), crlfDelay: Infinity });
+  let count = 0;
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    onRecord(JSON.parse(line));
+    count++;
+  }
+  return count;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) { usage(); return; }
+
+  const configPath = args.config || process.env.SMALLCODE_RAG_REPOS || path.join(process.cwd(), '.smallcode', 'rag', 'repos.json');
+  const cfg = readConfig(configPath);
+  const indexPath = args.indexPath || cfg.indexPath || path.join(process.cwd(), '.smallcode', 'rag', 'index.json');
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'smallcode-rag-'));
+  const jsonlPath = path.join(tmpDir, 'snippets.jsonl');
+  const scraper = path.join(__dirname, '..', 'scripts', 'rag_scraper.py');
+
+  const pyArgs = [scraper, '--out', jsonlPath];
+  if (fs.existsSync(configPath)) pyArgs.push('--config', configPath);
+  if (args.preset) pyArgs.push('--preset', args.preset);
+  for (const repo of args.repos) pyArgs.push('--repo', repo);
+
+  const py = findPython();
+  const scrape = cp.spawnSync(py, pyArgs, { cwd: process.cwd(), stdio: ['ignore', 'inherit', 'inherit'] });
+  if (scrape.status !== 0) process.exit(scrape.status || 1);
+
+  const store = new RagIndexStore({ path: indexPath });
+  store.load();
+  const batch = [];
+  let total = 0;
+  await loadJsonl(jsonlPath, (rec) => {
+    batch.push(rec);
+    if (batch.length >= 1000) {
+      store.upsertMany(batch.splice(0, batch.length));
+    }
+    total++;
+  });
+  if (batch.length) store.upsertMany(batch);
+  store.save();
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+  process.stdout.write(`indexed ${total} snippets into ${indexPath}\n`);
+}
+
+main().catch((err) => {
+  console.error(err.stack || err.message || String(err));
+  process.exit(1);
+});

bin/smallcode.js

@@ -245,6 +245,7 @@ let _planTracker = null;
 let _bootstrapDetector = null;
 let _testRunnerDetector = null;
 let _knowledgeLoader = null;
+let _ragRetriever = null;
 const improvementAttempts = {}; // filePath → attempt count
 
 async function runTUI(config) {
@@ -684,6 +685,12 @@ async function runAgentLoop(userMessage, config) {
     const { KnowledgeLoader } = require('../src/knowledge/loader');
     _knowledgeLoader = new KnowledgeLoader({ rootDir: process.cwd() });
   } catch { _knowledgeLoader = null; }
+  // RAG retriever: local code examples indexed from GitHub repos or other source trees.
+  try {
+    const { RagRetriever } = require('../src/rag/retriever');
+    _ragRetriever = new RagRetriever();
+    _ragRetriever.load();
+  } catch { _ragRetriever = null; }
   // Trust decay (Feature 13) resets per agent loop turn so TUI sessions
   // don't accumulate decay from unrelated prior requests.
   try {
@@ -1942,7 +1949,7 @@ CRITICAL — large file rule: write_file calls are limited to 60 lines / ~8KB. l
   }
 
   // Legacy behavior: everything in the system prompt
-  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getActivePlanContext() + getTestRunnerContext();
+  prompt += getMemoryContext(messages) + getSkillContext(messages) + getPluginPrompts() + getKnowledgeContext(messages) + getRagContext(messages) + getActivePlanContext() + getTestRunnerContext();
 
   return prompt;
 }
@@ -1957,6 +1964,7 @@ function buildDynamicContext(messages) {
     getMemoryContext(messages),
     getSkillContext(messages),
     getKnowledgeContext(messages),
+    getRagContext(messages),
     // Note: getPluginPrompts() stays in the system prompt — plugin instructions
     //   are authoritative and should come from the system role.
     // Note: getActivePlanContext() stays in the system prompt — the model needs
@@ -2006,6 +2014,20 @@ function getKnowledgeContext(messages) {
   }
 }
 
+// Auto-load similar code snippets from the local RAG index. The index is
+// created with `npm run rag:index` / `smallcode-rag-index` and stays fully local.
+function getRagContext(messages) {
+  try {
+    if (!_ragRetriever) return '';
+    const lastUser = [...messages].reverse().find(m => m.role === 'user');
+    if (!lastUser || typeof lastUser.content !== 'string') return '';
+    const maxChars = Math.min(6000, Math.floor(((config?.context?.detected_window || 32768) * 0.06) * 4));
+    return _ragRetriever.formatForPrompt(lastUser.content, { limit: 6, maxChars });
+  } catch {
+    return '';
+  }
+}
+
 // Auto-load relevant memory for the current task (injected into system prompt)
 // Fix #15: Only inject memory objects with score >= 2 (at least 2 word matches)
 // to avoid burning tokens on low-relevance hits.