test(e2e): scaffold tests/e2e harness + first smoke case

.github/workflows/ci.yml

@@ -118,8 +118,9 @@ jobs:
     name: e2e (vitest) + coverage
     needs: [build-bin, build-ui]
     runs-on: ubuntu-latest
-    # Scaffold-phase soft gate: the harness lands with PR #2+.
-    # Flip `continue-on-error` to false at PR #5 per plan §5.2 ratchet.
+    # Soft gate while the harness stabilises across CI runners (etcd
+    # service startup, port detection, undici quirks). Flip to `false`
+    # once we have ~5 consecutive green runs on main.
     continue-on-error: true
     services:
       etcd:

tests/e2e/.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+coverage/
+*.tsbuildinfo
+package-lock.json
+pnpm-lock.yaml

tests/e2e/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "aisix-e2e",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "description": "End-to-end test harness for the aisix AI Gateway (spec §12).",
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "coverage": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "@types/node": "20.11.30",
+    "@vitest/coverage-v8": "2.1.9",
+    "typescript": "5.5.4",
+    "vitest": "2.1.9"
+  },
+  "dependencies": {
+    "openai": "4.65.0",
+    "undici": "6.19.8",
+    "yaml": "2.5.1"
+  }
+}

tests/e2e/src/cases/smoke.test.ts

@@ -0,0 +1,97 @@
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import {
+  AdminClient,
+  EtcdClient,
+  ProxyClient,
+  spawnApp,
+  startOpenAiUpstream,
+  waitConfigPropagation,
+  type OpenAiUpstream,
+  type SpawnedApp,
+} from "../harness/index.js";
+
+describe("smoke: admin write → proxy read", () => {
+  let app: SpawnedApp | undefined;
+  let upstream: OpenAiUpstream | undefined;
+  let admin: AdminClient | undefined;
+  let etcdReachable = false;
+
+  beforeAll(async () => {
+    etcdReachable = await new EtcdClient().ping();
+    if (!etcdReachable) return;
+    upstream = await startOpenAiUpstream();
+    app = await spawnApp();
+    admin = new AdminClient(app.adminUrl, app.adminKey);
+  });
+
+  afterAll(async () => {
+    await app?.exit();
+    await upstream?.close();
+  });
+
+  test("a Model + ApiKey written via Admin API are visible to /v1/models", async (ctx) => {
+    if (!etcdReachable || !app || !admin || !upstream) {
+      ctx.skip();
+      return;
+    }
+
+    await admin.createModel({
+      name: "smoke-gpt",
+      model: "openai/gpt-4o-mini",
+      // The OpenAI bridge appends `/chat/completions`, so the api_base
+      // already needs the `/v1` segment to land on `/v1/chat/completions`.
+      provider_config: { api_key: "sk-mock", api_base: `${upstream.baseUrl}/v1` },
+    });
+    await admin.createApiKey({
+      key: "sk-smoke-caller",
+      allowed_models: ["smoke-gpt"],
+    });
+
+    await waitConfigPropagation();
+
+    const proxy = new ProxyClient(app.proxyUrl, "sk-smoke-caller");
+    const { status, body } = await proxy.listModels();
+
+    expect(status).toBe(200);
+    expect(body).toMatchObject({
+      object: "list",
+      data: expect.arrayContaining([expect.objectContaining({ id: "smoke-gpt" })]),
+    });
+  });
+
+  test("a chat completion forwards to the mock upstream", async (ctx) => {
+    if (!etcdReachable || !app || !upstream) {
+      ctx.skip();
+      return;
+    }
+
+    const proxy = new ProxyClient(app.proxyUrl, "sk-smoke-caller");
+    const { status, body } = await proxy.chat({
+      model: "smoke-gpt",
+      messages: [{ role: "user", content: "hello" }],
+    });
+
+    if (status !== 200) {
+      throw new Error(
+        `chat returned ${status}: ${JSON.stringify(body)}\n  upstream paths: ${JSON.stringify(upstream.receivedRequests.map((r) => r.path))}`,
+      );
+    }
+    expect(body).toMatchObject({
+      object: "chat.completion",
+      choices: expect.arrayContaining([
+        expect.objectContaining({
+          message: expect.objectContaining({ role: "assistant" }),
+        }),
+      ]),
+    });
+
+    const seen = upstream.receivedRequests.some((r) =>
+      r.path.startsWith("/v1/chat/completions"),
+    );
+    if (!seen) {
+      throw new Error(
+        `upstream did not receive /v1/chat/completions; saw paths: ${JSON.stringify(upstream.receivedRequests.map((r) => `${r.method} ${r.path}`))}`,
+      );
+    }
+  });
+});

tests/e2e/src/harness/admin.ts

@@ -0,0 +1,59 @@
+import { harnessRequest } from "./http.js";
+
+/**
+ * Thin typed wrapper over the Admin API. Keeps the test surface readable
+ * — `await admin.createModel({...})` instead of inlined fetch boilerplate.
+ */
+export class AdminClient {
+  constructor(
+    private readonly baseUrl: string,
+    private readonly adminKey: string,
+  ) {}
+
+  async createModel(
+    model: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    return this.json("POST", "/admin/v1/models", model);
+  }
+
+  async createApiKey(
+    key: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    return this.json("POST", "/admin/v1/apikeys", key);
+  }
+
+  async listModels(): Promise<Array<Record<string, unknown>>> {
+    const res = await this.json<{ items?: Array<{ value: Record<string, unknown> }> }>(
+      "GET",
+      "/admin/v1/models",
+    );
+    return (res.items ?? []).map((entry) => entry.value);
+  }
+
+  async json<T = Record<string, unknown>>(
+    method: string,
+    path: string,
+    body?: unknown,
+  ): Promise<T> {
+    const res = await harnessRequest(`${this.baseUrl}${path}`, {
+      method,
+      headers: {
+        authorization: `Bearer ${this.adminKey}`,
+        "content-type": "application/json",
+      },
+      body: body === undefined ? undefined : JSON.stringify(body),
+    });
+    const text = await res.body.text();
+    if (res.statusCode >= 300) {
+      throw new Error(
+        `admin ${method} ${path} → ${res.statusCode}: ${text.slice(0, 512)}`,
+      );
+    }
+    return text ? (JSON.parse(text) as T) : ({} as T);
+  }
+}
+
+/** Convenience: wait the spec-mandated 500ms for snapshot propagation. */
+export function waitConfigPropagation(): Promise<void> {
+  return new Promise((r) => setTimeout(r, 500));
+}

tests/e2e/src/harness/app.ts

@@ -0,0 +1,180 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { stringify as yamlStringify } from "yaml";
+
+import { pickFreePorts } from "./ports.js";
+import { EtcdClient } from "./etcd.js";
+import { harnessRequest } from "./http.js";
+
+export interface AppOverrides {
+  /** Inserted into `admin.admin_keys`. Defaults to a fresh random key. */
+  adminKey?: string;
+  /** Whether to enable Prometheus on `/metrics`. Defaults to true. */
+  prometheus?: boolean;
+  /** Extra raw config keys merged into the YAML at the top level. */
+  extra?: Record<string, unknown>;
+}
+
+export interface SpawnedApp {
+  proxyUrl: string;
+  adminUrl: string;
+  adminKey: string;
+  etcdPrefix: string;
+  exit(): Promise<void>;
+}
+
+const BIN_PATH =
+  process.env.AISIX_BIN ?? join(process.cwd(), "..", "..", "target", "debug", "aisix");
+const READY_TIMEOUT_MS = 10_000;
+const SHUTDOWN_GRACE_MS = 3_000;
+
+/**
+ * Per-test handle to a spawned `aisix` binary. Each call writes a fresh
+ * config YAML into a tmp dir, picks two free ports, picks a unique etcd
+ * prefix, and waits up to 10s for `/health` on both ports to respond
+ * 200. `exit()` issues SIGTERM and waits up to 3s, escalating to SIGKILL.
+ */
+export async function spawnApp(overrides: AppOverrides = {}): Promise<SpawnedApp> {
+  const etcd = new EtcdClient();
+  if (!(await etcd.ping())) {
+    throw new Error(
+      `etcd not reachable at ${process.env.AISIX_E2E_ETCD ?? "http://127.0.0.1:2379"} ` +
+        "(set AISIX_E2E_ETCD or run `docker run --rm -p 2379:2379 quay.io/coreos/etcd:v3.5.15`)",
+    );
+  }
+
+  const [proxyPort, adminPort] = await pickFreePorts(2);
+  const adminKey = overrides.adminKey ?? `admin-${randomUUID()}`;
+  const etcdPrefix = `/aisix-e2e-${randomUUID()}`;
+
+  const cfg = {
+    etcd: {
+      endpoints: [process.env.AISIX_E2E_ETCD ?? "http://127.0.0.1:2379"],
+      prefix: etcdPrefix,
+      dial_timeout_ms: 5000,
+      request_timeout_ms: 5000,
+    },
+    proxy: { addr: `127.0.0.1:${proxyPort}`, request_body_limit_bytes: 10485760 },
+    admin: { addr: `127.0.0.1:${adminPort}`, admin_keys: [adminKey] },
+    observability: {
+      service_name: "aisix-e2e",
+      log_level: "warn",
+      access_log: false,
+      metrics: {
+        prometheus: { enabled: overrides.prometheus ?? true, path: "/metrics" },
+        otlp: { enabled: false, endpoint: "http://127.0.0.1:4317" },
+      },
+      tracing: { otlp: { enabled: false, endpoint: "http://127.0.0.1:4317", sample_ratio: 1 } },
+      langfuse: { enabled: false },
+    },
+    cache: { backend: "memory" },
+    ...(overrides.extra ?? {}),
+  };
+
+  const dir = await mkdtemp(join(tmpdir(), "aisix-e2e-"));
+  const cfgPath = join(dir, "config.yaml");
+  await writeFile(cfgPath, yamlStringify(cfg), "utf8");
+
+  const child = spawn(BIN_PATH, ["--config", cfgPath], {
+    stdio: ["ignore", "pipe", "pipe"],
+    env: { ...process.env, RUST_LOG: process.env.RUST_LOG ?? "warn" },
+  });
+
+  let stderrBuf = "";
+  child.stderr?.on("data", (c: Buffer) => {
+    stderrBuf += c.toString("utf8");
+  });
+  child.stdout?.on("data", (c: Buffer) => {
+    stderrBuf += c.toString("utf8");
+  });
+  let exitErr: string | undefined;
+  child.once("exit", (code, signal) => {
+    if (code !== 0 && code !== null) {
+      exitErr = `aisix exited early with code=${code} signal=${signal}`;
+    }
+  });
+
+  const proxyUrl = `http://127.0.0.1:${proxyPort}`;
+  const adminUrl = `http://127.0.0.1:${adminPort}`;
+
+  try {
+    await Promise.all([
+      waitForReady(`${proxyUrl}/health`, READY_TIMEOUT_MS),
+      waitForReady(`${adminUrl}/admin/v1/health`, READY_TIMEOUT_MS, adminKey),
+    ]);
+  } catch (err) {
+    const detail = exitErr ?? "still running";
+    const stderr = stderrBuf.slice(-2000);
+    await terminate(child);
+    await cleanup(etcd, etcdPrefix, dir);
+    throw new Error(
+      `${(err as Error).message}\n  binary state: ${detail}\n  stderr tail:\n${stderr}`,
+    );
+  }
+
+  return {
+    proxyUrl,
+    adminUrl,
+    adminKey,
+    etcdPrefix,
+    async exit() {
+      await terminate(child);
+      await cleanup(etcd, etcdPrefix, dir);
+    },
+  };
+}
+
+async function waitForReady(url: string, timeoutMs: number, bearer?: string): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  let lastErr: unknown;
+  let lastStatus: number | undefined;
+  let attempts = 0;
+  while (Date.now() < deadline) {
+    attempts++;
+    try {
+      const headers: Record<string, string> = {};
+      if (bearer) headers.authorization = `Bearer ${bearer}`;
+      const res = await harnessRequest(url, { method: "GET", headers });
+      lastStatus = res.statusCode;
+      if (res.statusCode === 200) {
+        await res.body.dump();
+        return;
+      }
+      await res.body.dump();
+    } catch (err) {
+      lastErr = err;
+    }
+    await sleep(100);
+  }
+  throw new Error(
+    `timed out waiting for ${url} after ${attempts} attempts (lastStatus=${lastStatus ?? "n/a"}): ${lastErr ?? "no response"}`,
+  );
+}
+
+async function terminate(child: ChildProcess): Promise<void> {
+  if (child.exitCode !== null) return;
+  child.kill("SIGTERM");
+  const exited = await Promise.race([
+    new Promise<boolean>((r) => child.once("exit", () => r(true))),
+    sleep(SHUTDOWN_GRACE_MS).then(() => false),
+  ]);
+  if (!exited && child.exitCode === null) {
+    child.kill("SIGKILL");
+    await new Promise<void>((r) => child.once("exit", () => r()));
+  }
+}
+
+async function cleanup(etcd: EtcdClient, prefix: string, dir: string): Promise<void> {
+  // Best-effort — never throw from cleanup.
+  await Promise.allSettled([
+    etcd.deletePrefix(prefix),
+    rm(dir, { recursive: true, force: true }),
+  ]);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}