feat: recursive directory scanning for API spec discovery

src/core/SpecScanner.ts

@@ -33,7 +33,36 @@ export class DefaultSpecScanner implements ISpecScanner {
   constructor(private readonly specProcessor: ISpecProcessor) {}
 
   /**
-   * Scans a directory for OpenAPI specification files and yields processed results
+   * Recursively walks a directory tree, yielding file entries.
+   * Skips hidden directories, underscore-prefixed directories (e.g. _catalog, _dereferenced),
+   * and node_modules.
+   */
+  private async *walkDirectory(
+    basePath: string,
+    currentPath: string = basePath
+  ): AsyncGenerator<{ relativePath: string; fullPath: string }> {
+    const entries = await fs.readdir(currentPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(currentPath, entry.name);
+      if (entry.isDirectory()) {
+        if (
+          entry.name.startsWith(".") ||
+          entry.name.startsWith("_") ||
+          entry.name === "node_modules"
+        ) {
+          continue;
+        }
+        yield* this.walkDirectory(basePath, fullPath);
+      } else {
+        const relativePath = path.relative(basePath, fullPath);
+        yield { relativePath, fullPath };
+      }
+    }
+  }
+
+  /**
+   * Scans a directory for OpenAPI specification files and yields processed results.
+   * Recursively walks subdirectories.
    * @param folderPath - Path to the directory containing OpenAPI specs
    * @throws {SpecScanError} If the folder doesn't exist or isn't readable
    */
@@ -46,18 +75,18 @@ export class DefaultSpecScanner implements ISpecScanner {
     }
 
     try {
-      const files = await fs.readdir(folderPath);
-
-      for (const file of files) {
+      for await (const { relativePath, fullPath } of this.walkDirectory(
+        folderPath
+      )) {
         try {
-          const result = await this.processFile(folderPath, file);
+          const result = await this.processFile(fullPath, relativePath);
           if (result) {
             yield result;
           }
         } catch (error) {
           yield {
-            filename: file,
-            specId: file,
+            filename: relativePath,
+            specId: relativePath,
             spec: {} as OpenAPIV3.Document, // Empty spec for error cases
             error: error instanceof Error ? error : new Error(String(error)),
           };
@@ -74,46 +103,44 @@ export class DefaultSpecScanner implements ISpecScanner {
 
   /**
    * Processes a single OpenAPI specification file
-   * @param folderPath - Path to the directory containing the file
-   * @param filename - Name of the file to process
+   * @param fullPath - Absolute path to the file
+   * @param relativePath - Path relative to the scanned base directory (used as filename and specId fallback)
    * @returns The processed spec result or null if the file type is invalid
    * @throws {SpecScanError} If there's an error processing the file
    */
   private async processFile(
-    folderPath: string,
-    filename: string
+    fullPath: string,
+    relativePath: string
   ): Promise<SpecScanResult | null> {
-    const fileType = this.getFileType(filename);
+    const fileType = this.getFileType(fullPath);
     if (fileType === "invalid") {
       return null;
     }
 
-    const filePath = path.join(folderPath, filename);
-
     try {
-      const content = await fs.readFile(filePath, "utf-8");
+      const content = await fs.readFile(fullPath, "utf-8");
       const specObject = await this.parseSpec(content, fileType);
 
       // Validate basic spec structure
       if (!this.isValidSpecObject(specObject)) {
         throw new SpecScanError(
           "Invalid OpenAPI specification format",
-          filename
+          relativePath
         );
       }
 
-      const specId = this.extractSpecId(specObject, filename);
+      const specId = this.extractSpecId(specObject, relativePath);
       const processedSpec = await this.specProcessor.process(specObject);
 
       return {
-        filename,
+        filename: relativePath,
         spec: processedSpec,
         specId,
       };
     } catch (error) {
       throw new SpecScanError(
-        `Failed to process spec file: ${filename}`,
-        filename,
+        `Failed to process spec file: ${relativePath}`,
+        relativePath,
         error instanceof Error ? error : new Error(String(error))
       );
     }

src/core/SpecService.ts

@@ -246,8 +246,8 @@ export class FileSystemSpecService implements ISpecStore, ISpecExplorer {
   async saveSpec(spec: OpenAPIV3.Document, specId: string): Promise<void> {
     this.logger.debug({ specId }, "Persisting specification");
     try {
-      await this.ensureDirectory(this.dereferencedPath);
       const specPath = path.join(this.dereferencedPath, `${specId}.json`);
+      await fs.mkdir(path.dirname(specPath), { recursive: true });
       await fs.writeFile(specPath, JSON.stringify(spec, null, 2));
       this.specs[specId] = spec;
       this.specCache.set(specId, spec);

src/core/__tests__/SpecScanner.test.ts

@@ -1,6 +1,7 @@
 import fs from 'fs/promises';
+import os from 'os';
 import path from 'path';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { ISpecProcessor } from '../interfaces/ISpecProcessor';
 import { SpecScanResult } from '../interfaces/ISpecScanner';
 import { DefaultSpecScanner } from '../SpecScanner';
@@ -94,4 +95,104 @@ describe('DefaultSpecScanner', () => {
       }
     });
   });
+
+  describe('recursive scanning', () => {
+    let tmpDir: string;
+
+    const minimalSpec = JSON.stringify({
+      openapi: '3.0.0',
+      info: { title: 'Nested API', version: '1.0.0' },
+      paths: {},
+    });
+
+    const minimalSpecWithId = JSON.stringify({
+      openapi: '3.0.0',
+      info: { title: 'Custom ID API', version: '1.0.0', 'x-spec-id': 'my-custom-id' },
+      paths: {},
+    });
+
+    beforeAll(async () => {
+      tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'spec-scanner-test-'));
+      // Create nested directory structure
+      await fs.mkdir(path.join(tmpDir, 'payments'), { recursive: true });
+      await fs.mkdir(path.join(tmpDir, 'users', 'v2'), { recursive: true });
+      await fs.mkdir(path.join(tmpDir, '.hidden'), { recursive: true });
+      await fs.mkdir(path.join(tmpDir, '_dereferenced'), { recursive: true });
+      await fs.mkdir(path.join(tmpDir, 'node_modules', 'pkg'), { recursive: true });
+
+      // Top-level spec
+      await fs.writeFile(path.join(tmpDir, 'root-api.json'), minimalSpec);
+      // Nested specs
+      await fs.writeFile(path.join(tmpDir, 'payments', 'stripe.yaml'),
+        'openapi: "3.0.0"\ninfo:\n  title: Stripe API\n  version: "1.0.0"\npaths: {}\n');
+      await fs.writeFile(path.join(tmpDir, 'users', 'v2', 'api.json'), minimalSpecWithId);
+      // Specs in dirs that should be skipped
+      await fs.writeFile(path.join(tmpDir, '.hidden', 'secret.json'), minimalSpec);
+      await fs.writeFile(path.join(tmpDir, '_dereferenced', 'cached.json'), minimalSpec);
+      await fs.writeFile(path.join(tmpDir, 'node_modules', 'pkg', 'spec.json'), minimalSpec);
+      // Non-spec file in nested dir
+      await fs.writeFile(path.join(tmpDir, 'payments', 'README.md'), '# Payments');
+    });
+
+    afterAll(async () => {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    });
+
+    it('should discover specs in nested subdirectories', async () => {
+      const results: SpecScanResult[] = [];
+      for await (const result of specScanner.scan(tmpDir)) {
+        results.push(result);
+      }
+
+      const filenames = results.map(r => r.filename);
+      expect(filenames).toContain('root-api.json');
+      expect(filenames).toContain(path.join('payments', 'stripe.yaml'));
+      expect(filenames).toContain(path.join('users', 'v2', 'api.json'));
+    });
+
+    it('should use relative path as specId fallback for nested specs', async () => {
+      const results: SpecScanResult[] = [];
+      for await (const result of specScanner.scan(tmpDir)) {
+        results.push(result);
+      }
+
+      const stripeResult = results.find(r => r.filename === path.join('payments', 'stripe.yaml'));
+      expect(stripeResult).toBeDefined();
+      expect(stripeResult?.specId).toBe(path.join('payments', 'stripe.yaml'));
+    });
+
+    it('should use x-spec-id when present in nested specs', async () => {
+      const results: SpecScanResult[] = [];
+      for await (const result of specScanner.scan(tmpDir)) {
+        results.push(result);
+      }
+
+      const customIdResult = results.find(r => r.filename === path.join('users', 'v2', 'api.json'));
+      expect(customIdResult).toBeDefined();
+      expect(customIdResult?.specId).toBe('my-custom-id');
+    });
+
+    it('should skip hidden, underscore-prefixed, and node_modules directories', async () => {
+      const results: SpecScanResult[] = [];
+      for await (const result of specScanner.scan(tmpDir)) {
+        results.push(result);
+      }
+
+      const filenames = results.map(r => r.filename);
+      // Should NOT contain specs from skipped dirs
+      expect(filenames).not.toContain(path.join('.hidden', 'secret.json'));
+      expect(filenames).not.toContain(path.join('_dereferenced', 'cached.json'));
+      expect(filenames).not.toContain(path.join('node_modules', 'pkg', 'spec.json'));
+    });
+
+    it('should skip non-spec files in nested directories', async () => {
+      const results: SpecScanResult[] = [];
+      for await (const result of specScanner.scan(tmpDir)) {
+        results.push(result);
+      }
+
+      const filenames = results.map(r => r.filename);
+      expect(filenames).not.toContain(path.join('payments', 'README.md'));
+    });
+  });
 });