diff --git a/Makefile b/Makefile index 9711dc1a4..4acc79008 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,12 @@ endif .PHONY: all help clean test cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) -all: ds4 ds4-server ds4-bench ds4-eval ds4-agent +all: ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor help: @echo "DS4 build targets:" - @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" - @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" + @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, ./ds4-agent, and ./ds4-doctor" + @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, ./ds4-agent, and ./ds4-doctor" @echo " make test Build and run tests" @echo " make clean Remove build outputs" @@ -67,12 +67,16 @@ ds4-eval: ds4_eval.o ds4_help.o $(CORE_OBJS) ds4-agent: ds4_agent.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) $(CC) $(CFLAGS) -o $@ ds4_agent.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) $(METAL_LDLIBS) -cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o ds4_eval_cpu.o ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o rax.o $(CPU_CORE_OBJS) +ds4-doctor: ds4_doctor.o ds4_help.o + $(CC) $(CFLAGS) -o $@ ds4_doctor.o ds4_help.o $(LDLIBS) + +cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o ds4_eval_cpu.o ds4_agent_cpu.o ds4_doctor_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o rax.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o ds4 ds4_cli_cpu.o ds4_help.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-server ds4_server_cpu.o ds4_help.o ds4_kvstore.o rax.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-bench ds4_bench_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-eval ds4_eval_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-agent ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS) + $(CC) $(CFLAGS) -o ds4-doctor ds4_doctor_cpu.o ds4_help.o $(LDLIBS) cuda-regression: @echo "cuda-regression requires a CUDA build" @@ -86,15 +90,15 @@ help: @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" @echo " make strix-halo Build ROCm for Strix Halo / gfx1151" @echo " make rocm Alias for make strix-halo" - @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" + @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, ./ds4-agent, and ./ds4-doctor" @echo " make test Build and run tests" @echo " make clean Remove build outputs" cuda-spark: - $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent CUDA_ARCH= + $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor CUDA_ARCH= cuda-generic: - $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent CUDA_ARCH=native + $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor CUDA_ARCH=native cuda: @if [ -z "$(strip $(CUDA_ARCH))" ]; then \ @@ -102,10 +106,10 @@ cuda: echo " or use make cuda-spark / make cuda-generic"; \ exit 2; \ fi - $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent CUDA_ARCH="$(CUDA_ARCH)" + $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor CUDA_ARCH="$(CUDA_ARCH)" strix-halo: - $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent \ + $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor \ CORE_OBJS="ds4.o ds4_distributed.o ds4_ssd.o ds4_rocm.o" \ CFLAGS="$(CFLAGS) -DDS4_ROCM_BUILD" \ DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \ @@ -128,12 +132,16 @@ ds4-eval: ds4_eval.o ds4_help.o $(CORE_OBJS) ds4-agent: ds4_agent.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) $(DS4_LINK) -o $@ $^ $(DS4_LINK_LIBS) -cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o ds4_eval_cpu.o ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o rax.o $(CPU_CORE_OBJS) +ds4-doctor: ds4_doctor.o ds4_help.o + $(CC) $(CFLAGS) -o $@ ds4_doctor.o ds4_help.o $(LDLIBS) + +cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o ds4_eval_cpu.o ds4_agent_cpu.o ds4_doctor_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o rax.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o ds4 ds4_cli_cpu.o ds4_help.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-server ds4_server_cpu.o ds4_help.o ds4_kvstore.o rax.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-bench ds4_bench_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-eval ds4_eval_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(LDLIBS) $(CC) $(CFLAGS) -o ds4-agent ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS) + $(CC) $(CFLAGS) -o ds4-doctor ds4_doctor_cpu.o ds4_help.o $(LDLIBS) cuda-regression: tests/cuda_long_context_smoke ./tests/cuda_long_context_smoke @@ -154,6 +162,9 @@ ds4_distributed.o: ds4_distributed.c ds4_distributed.h ds4.h ds4_ssd.h ds4_help.o: ds4_help.c ds4_help.h $(CC) $(CFLAGS) -c -o $@ ds4_help.c +ds4_doctor.o: ds4_doctor.c ds4_help.h + $(CC) $(CFLAGS) -c -o $@ ds4_doctor.c + ds4_server.o: ds4_server.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_kvstore.h rax.h $(CC) $(CFLAGS) -c -o $@ ds4_server.c @@ -205,6 +216,9 @@ ds4_eval_cpu.o: ds4_eval.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_agent_cpu.o: ds4_agent.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_kvstore.h ds4_web.h linenoise.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_agent.c +ds4_doctor_cpu.o: ds4_doctor.c ds4_help.h + $(CC) $(CFLAGS) -c -o $@ ds4_doctor.c + ds4_metal.o: ds4_metal.m ds4_gpu.h $(METAL_SRCS) $(CC) $(OBJCFLAGS) -c -o $@ ds4_metal.m @@ -231,7 +245,7 @@ else $(NVCC) $(NVCCFLAGS) -o $@ ds4_agent_test.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) $(CUDA_LDLIBS) endif -test: ds4_test ds4_agent_test ds4-eval q4k-dot-test +test: ds4_test ds4_agent_test ds4-eval ds4-doctor q4k-dot-test ./ds4-eval --self-test-extractors ./ds4_agent_test ./ds4_test @@ -241,4 +255,4 @@ q4k-dot-test: tests/test_q4k_dot.c ./tests/test_q4k_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test tests/test_q4k_dot *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4-doctor ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test tests/test_q4k_dot *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/README.md b/README.md index 785695284..1ca701b0f 100644 --- a/README.md +++ b/README.md @@ -667,6 +667,45 @@ DeepSeek V4 Flash still solve a representative mix of hard science, broad knowledge, exact math, and security-code problems while using the same inference path users run? +## Self-Diagnostics + +`ds4-doctor` is a small read-only diagnostic command that checks the local +installation before a real inference run. It does not load the model, does not +touch the network, and does not write to disk. The default run walks five +checks: the compile-time backend (`Metal` / `CUDA` / `ROCm` / `CPU`), the GGUF +header of the model file you point it at, whether the default server port +(`127.0.0.1:8000`) is bindable, total system RAM, and the optional KV-disk +directory. It is safe to run inside CI on machines without a GPU. + +```sh +./ds4-doctor +./ds4-doctor -m ./ds4flash.gguf +./ds4-doctor -m ./ds4flash.gguf --kv-disk-dir /tmp/ds4-kv --json +``` + +Each check produces one of four states: `ok`, `warn`, `fail`, `skip` (a check +that did not run, e.g. the model path was not given). Human-readable output +includes an ANSI-coloured status column when stdout is a TTY; pass `--color` +to force it on. `--json` emits one self-describing line with a top-level +`status`, a `summary` counter, and a `checks` array: + +```json +{"status":"ok","summary":"5 checks: 4 ok, 0 warn, 0 fail, 1 skip","checks":[ + {"id":"backend","title":"Backend compile","status":"ok","duration_ms":0,"message":"Metal backend compiled in (Apple Silicon)"}, + {"id":"model","title":"Model file integrity","status":"ok","duration_ms":3,"message":"81.32 GiB, v3, 715 tensors, 24 metadata kvs"}, + {"id":"port","title":"Default server port","status":"ok","duration_ms":0,"message":"127.0.0.1:8000 bindable (default for ds4-server)"}, + {"id":"memory","title":"Total system RAM","status":"ok","duration_ms":0,"message":"128.00 GiB total RAM"}, + {"id":"kv-disk","title":"KV-disk directory","status":"skip","duration_ms":0,"message":"no --kv-disk-dir provided"} +]} +``` + +Exit codes match the worst severity, so scripts can branch without parsing +output: `0` if every check is `ok` or `skip`, `1` if at least one check is +`warn` and none is `fail`, `2` if any check is `fail`. `--timeout N` bounds +individual checks to `N` milliseconds (default 5000, max 600000). + +Run `./ds4-doctor --help` for the full option list. + ## CLI One-shot prompt: diff --git a/ds4_doctor.c b/ds4_doctor.c new file mode 100644 index 000000000..b82dbe35f --- /dev/null +++ b/ds4_doctor.c @@ -0,0 +1,599 @@ +#include "ds4.h" +#include "ds4_help.h" + +/* ds4-doctor self-diagnostic command. + * + * Runs read-only checks against the local installation: backend compile, + * model file integrity, environment sanity. No GPU loads beyond what the + * probe functions already do, no network calls, no filesystem writes. + * Intended to be safe to run in CI without GPU hardware. + * + * Units: + * U1 skeleton + help dispatch + option parsing + * U2 doctor_check_backend (compile-time backend identity) + * U3 doctor_check_model (GGUF v3 header walk) + * U4 doctor_check_port, doctor_check_memory, doctor_check_kv_disk + * U5 print_json, print_human, exit-code policy + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + const char *model_path; + const char *kv_disk_dir; + bool json_output; + bool color_output; + int timeout_ms; +} doctor_config; + +static doctor_config parse_options(int argc, char **argv) { + doctor_config c = {0}; + c.timeout_ms = 5000; + + for (int i = 1; i < argc; i++) { + const char *arg = argv[i]; + if (!strcmp(arg, "-h") || !strcmp(arg, "--help")) { + ds4_help_print(stdout, DS4_HELP_DOCTOR, NULL); + exit(0); + } else if (!strcmp(arg, "--json")) { + c.json_output = true; + } else if (!strcmp(arg, "--color")) { + c.color_output = true; + } else if (!strcmp(arg, "-m") || !strcmp(arg, "--model")) { + if (i + 1 >= argc) { + fprintf(stderr, "ds4-doctor: %s requires a path argument\n", arg); + exit(2); + } + c.model_path = argv[++i]; + } else if (!strcmp(arg, "--kv-disk-dir")) { + if (i + 1 >= argc) { + fprintf(stderr, "ds4-doctor: %s requires a path argument\n", arg); + exit(2); + } + c.kv_disk_dir = argv[++i]; + } else if (!strcmp(arg, "--timeout")) { + if (i + 1 >= argc) { + fprintf(stderr, "ds4-doctor: --timeout requires a value in ms\n"); + exit(2); + } + char *end = NULL; + long v = strtol(argv[++i], &end, 10); + if (end == argv[i] || v < 0 || v > 600000) { + fprintf(stderr, "ds4-doctor: --timeout must be 0..600000 ms\n"); + exit(2); + } + c.timeout_ms = (int)v; + } else { + fprintf(stderr, "ds4-doctor: unknown option: %s\n", arg); + ds4_help_print(stderr, DS4_HELP_DOCTOR, NULL); + exit(2); + } + } + return c; +} + +/* Canonical check status. Mirrors KTD-2 in the plan: + * OK check passed + * SKIP not run (deferred, disabled, missing dependency) + * WARN check ran but flagged a soft problem + * FAIL check ran and found a hard problem + * + * Exit-code mapping (per R4): + * 0 = OK or SKIP only + * 1 = at least one WARN, no FAIL + * 2 = at least one FAIL + */ +typedef enum { + DOCTOR_OK, + DOCTOR_SKIP, + DOCTOR_WARN, + DOCTOR_FAIL, +} doctor_status; + +static const char *doctor_status_name(doctor_status s) { + switch (s) { + case DOCTOR_OK: return "ok"; + case DOCTOR_WARN: return "warn"; + case DOCTOR_FAIL: return "fail"; + case DOCTOR_SKIP: return "skip"; + } + return "skip"; +} + +typedef struct { + const char *id; /* short stable identifier, e.g. "backend" */ + const char *title; /* human-readable label */ + doctor_status status; /* canonical status */ + int duration_ms; /* wall time spent in the check */ + char message[256]; /* one-line explanation or hint */ +} doctor_check; + +static uint64_t now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL; +} + +/* Forward declarations for the five checks. */ +static doctor_check doctor_check_backend(void); +static doctor_check doctor_check_model(const char *model_path, int timeout_ms); +static doctor_check doctor_check_port(void); +static doctor_check doctor_check_memory(void); +static doctor_check doctor_check_kv_disk(const char *kv_disk_dir); + +static doctor_check doctor_check_backend(void) { + doctor_check c = {0}; + c.id = "backend"; + c.title = "Backend compile"; + c.status = DOCTOR_OK; +#ifdef DS4_NO_GPU + snprintf(c.message, sizeof(c.message), "CPU-only build (DS4_NO_GPU)"); +#elif defined(__APPLE__) + snprintf(c.message, sizeof(c.message), "Metal backend compiled in (Apple Silicon)"); +#elif defined(DS4_ROCM_BUILD) + snprintf(c.message, sizeof(c.message), "ROCm/HIP backend compiled in (Strix Halo / gfx1151)"); +#else + snprintf(c.message, sizeof(c.message), "CUDA backend compiled in (Linux/Windows CUDA)"); +#endif + return c; +} + +/* Read a little-endian u32 / u64 with strict short-read detection. + * Returns false if the stream ends before the requested width. */ +static bool read_u32_le(FILE *fp, uint32_t *out) { + uint8_t buf[4]; + if (fread(buf, 1, 4, fp) != 4) return false; + *out = (uint32_t)buf[0] + | ((uint32_t)buf[1] << 8) + | ((uint32_t)buf[2] << 16) + | ((uint32_t)buf[3] << 24); + return true; +} + +static bool read_u64_le(FILE *fp, uint64_t *out) { + uint8_t buf[8]; + if (fread(buf, 1, 8, fp) != 8) return false; + *out = (uint64_t)buf[0] + | ((uint64_t)buf[1] << 8) + | ((uint64_t)buf[2] << 16) + | ((uint64_t)buf[3] << 24) + | ((uint64_t)buf[4] << 32) + | ((uint64_t)buf[5] << 40) + | ((uint64_t)buf[6] << 48) + | ((uint64_t)buf[7] << 56); + return true; +} + +#define DOCTOR_GGUF_MAGIC 0x46554747u /* "GGUF" little-endian; matches + DS4_GGUF_MAGIC in ds4.c. */ +#define DOCTOR_GGUF_VERSION 3u + +static doctor_check doctor_check_model(const char *model_path, int timeout_ms) { + (void)timeout_ms; /* 24-byte header read is bounded by syscall latency; + no async timeout needed at this scale */ + doctor_check c = {0}; + c.id = "model"; + c.title = "Model file integrity"; + c.status = DOCTOR_OK; + + if (model_path == NULL) { + c.status = DOCTOR_SKIP; + snprintf(c.message, sizeof(c.message), + "no model path provided (use -m FILE or DS4_GGUF)"); + return c; + } + + FILE *fp = fopen(model_path, "rb"); + if (!fp) { + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "cannot open %s", model_path); + return c; + } + + struct stat st; + if (fstat(fileno(fp), &st) != 0) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "cannot stat %s", model_path); + return c; + } + + uint32_t magic = 0; + if (!read_u32_le(fp, &magic)) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: header too short to read magic (< 4 bytes)", model_path); + return c; + } + + if (magic != DOCTOR_GGUF_MAGIC) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: bad magic 0x%08x (expected GGUF)", model_path, magic); + return c; + } + + uint32_t version = 0; + if (!read_u32_le(fp, &version)) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: header truncated at version field", model_path); + return c; + } + + if (version != DOCTOR_GGUF_VERSION) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: unsupported GGUF version %u (expected %u)", + model_path, version, DOCTOR_GGUF_VERSION); + return c; + } + + uint64_t n_tensors = 0; + if (!read_u64_le(fp, &n_tensors)) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: header truncated at tensor count", model_path); + return c; + } + + uint64_t n_kv = 0; + if (!read_u64_le(fp, &n_kv)) { + fclose(fp); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: header truncated at metadata kv count", model_path); + return c; + } + + fclose(fp); + + double gib = (double)st.st_size / (1024.0 * 1024.0 * 1024.0); + snprintf(c.message, sizeof(c.message), + "%.2f GiB, v%u, %llu tensors, %llu metadata kvs", + gib, version, (unsigned long long)n_tensors, + (unsigned long long)n_kv); + return c; +} + +/* Default port ds4-server binds to when --port is not provided. Mirrors + * ds4_server.c; if the project changes the default, update both. */ +#define DOCTOR_DEFAULT_PORT 8000 +#define DOCTOR_DEFAULT_HOST "127.0.0.1" + +static doctor_check doctor_check_port(void) { + doctor_check c = {0}; + c.id = "port"; + c.title = "Default server port"; + c.status = DOCTOR_OK; + + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "cannot create socket: %s", strerror(errno)); + return c; + } + + int yes = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + struct sockaddr_in sa; + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_port = htons((uint16_t)DOCTOR_DEFAULT_PORT); + if (inet_pton(AF_INET, DOCTOR_DEFAULT_HOST, &sa.sin_addr) != 1) { + close(fd); + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "inet_pton failed for %s", DOCTOR_DEFAULT_HOST); + return c; + } + + if (bind(fd, (struct sockaddr *)&sa, sizeof(sa)) != 0) { + int e = errno; + close(fd); + if (e == EADDRINUSE) { + c.status = DOCTOR_WARN; + snprintf(c.message, sizeof(c.message), + "port %d is in use (ds4-server may already be running)", + DOCTOR_DEFAULT_PORT); + return c; + } + if (e == EACCES) { + c.status = DOCTOR_WARN; + snprintf(c.message, sizeof(c.message), + "port %d requires elevated privileges", + DOCTOR_DEFAULT_PORT); + return c; + } + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "bind to %s:%d failed: %s", + DOCTOR_DEFAULT_HOST, DOCTOR_DEFAULT_PORT, strerror(e)); + return c; + } + + close(fd); + snprintf(c.message, sizeof(c.message), + "%s:%d bindable (default for ds4-server)", + DOCTOR_DEFAULT_HOST, DOCTOR_DEFAULT_PORT); + return c; +} + +#define DOCTOR_MEM_WARN_GIB 8.0 +#define DOCTOR_MEM_FAIL_GIB 4.0 + +static doctor_check doctor_check_memory(void) { + doctor_check c = {0}; + c.id = "memory"; + c.title = "Total system RAM"; + c.status = DOCTOR_OK; + + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + if (pages <= 0 || page_size <= 0) { + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "sysconf(_SC_PHYS_PAGES/_SC_PAGESIZE) unavailable"); + return c; + } + + double gib = (double)pages * (double)page_size + / (1024.0 * 1024.0 * 1024.0); + + if (gib < DOCTOR_MEM_FAIL_GIB) { + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%.2f GiB total RAM (below %.0f GiB minimum)", + gib, DOCTOR_MEM_FAIL_GIB); + return c; + } + if (gib < DOCTOR_MEM_WARN_GIB) { + c.status = DOCTOR_WARN; + snprintf(c.message, sizeof(c.message), + "%.2f GiB total RAM (below %.0f GiB recommended)", + gib, DOCTOR_MEM_WARN_GIB); + return c; + } + snprintf(c.message, sizeof(c.message), + "%.2f GiB total RAM", gib); + return c; +} + +static doctor_check doctor_check_kv_disk(const char *kv_disk_dir) { + doctor_check c = {0}; + c.id = "kv-disk"; + c.title = "KV-disk directory"; + c.status = DOCTOR_OK; + + if (kv_disk_dir == NULL) { + c.status = DOCTOR_SKIP; + snprintf(c.message, sizeof(c.message), + "no --kv-disk-dir provided"); + return c; + } + + struct stat st; + if (stat(kv_disk_dir, &st) != 0) { + int e = errno; + c.status = DOCTOR_WARN; + snprintf(c.message, sizeof(c.message), + "%s: %s (server will create it on demand)", + kv_disk_dir, strerror(e)); + return c; + } + if (!S_ISDIR(st.st_mode)) { + c.status = DOCTOR_FAIL; + snprintf(c.message, sizeof(c.message), + "%s: exists but is not a directory", kv_disk_dir); + return c; + } + + struct statvfs vfs; + double free_gib = 0.0; + if (statvfs(kv_disk_dir, &vfs) == 0 && vfs.f_frsize > 0) { + free_gib = (double)vfs.f_bavail * (double)vfs.f_frsize + / (1024.0 * 1024.0 * 1024.0); + } + + snprintf(c.message, sizeof(c.message), + "%s: %.2f GiB free", kv_disk_dir, free_gib); + return c; +} + +/* Aggregate counts: drives both the JSON summary field, the human + * "Status: ..." footer line, and the exit-code policy. */ +typedef struct { + int ok; + int warn; + int fail; + int skip; + doctor_status worst; /* highest severity observed */ +} doctor_summary; + +static void doctor_summary_update(doctor_summary *s, doctor_status st) { + switch (st) { + case DOCTOR_OK: s->ok++; break; + case DOCTOR_WARN: s->warn++; break; + case DOCTOR_FAIL: s->fail++; break; + case DOCTOR_SKIP: s->skip++; break; + } + if ((int)st > (int)s->worst) s->worst = st; +} + +static doctor_status doctor_summary_status(const doctor_summary *s) { + if (s->fail > 0) return DOCTOR_FAIL; + if (s->warn > 0) return DOCTOR_WARN; + return DOCTOR_OK; +} + +static const char *doctor_summary_name(const doctor_summary *s) { + return doctor_status_name(doctor_summary_status(s)); +} + +/* Minimal JSON string escaper. Every emitted message comes from + * known C literals or snprintf into fixed-size buffers with %s %d %u %f + * format specifiers, so the only character we expect to escape is the + * double quote and the backslash. */ +static void print_json_escaped(FILE *fp, const char *s) { + fputc('"', fp); + for (; *s; s++) { + unsigned char c = (unsigned char)*s; + switch (c) { + case '"': fputs("\\\"", fp); break; + case '\\': fputs("\\\\", fp); break; + case '\n': fputs("\\n", fp); break; + case '\r': fputs("\\r", fp); break; + case '\t': fputs("\\t", fp); break; + default: + if (c < 0x20) fprintf(fp, "\\u%04x", c); + else fputc((char)c, fp); + } + } + fputc('"', fp); +} + +static void print_json(const doctor_check *checks, size_t n, + const doctor_summary *s) { + fputs("{\"status\":", stdout); + print_json_escaped(stdout, doctor_summary_name(s)); + fprintf(stdout, + ",\"summary\":\"%d checks: %d ok, %d warn, %d fail, %d skip\"," + "\"checks\":[", + (int)n, s->ok, s->warn, s->fail, s->skip); + for (size_t i = 0; i < n; i++) { + const doctor_check *c = &checks[i]; + if (i) fputc(',', stdout); + fputc('{', stdout); + fputs("\"id\":", stdout); print_json_escaped(stdout, c->id ? c->id : ""); + fputs(",\"title\":", stdout); print_json_escaped(stdout, c->title ? c->title : ""); + fputs(",\"status\":", stdout); print_json_escaped(stdout, doctor_status_name(c->status)); + fprintf(stdout, + ",\"duration_ms\":%d,\"message\":", + c->duration_ms < 0 ? 0 : c->duration_ms); + print_json_escaped(stdout, c->message); + fputc('}', stdout); + } + fputs("]}\n", stdout); +} + +/* ANSI color codes (used when --color is requested or stdout is a TTY + * and cfg.color_output is unset). Color output is intentionally minimal: + * just the status column. */ +static const char *status_color(doctor_status st, bool color) { + if (!color) return ""; + switch (st) { + case DOCTOR_OK: return "\x1b[32m"; /* green */ + case DOCTOR_WARN: return "\x1b[33m"; /* yellow */ + case DOCTOR_FAIL: return "\x1b[31m"; /* red */ + case DOCTOR_SKIP: return "\x1b[90m"; /* grey */ + } + return ""; +} + +static const char *color_off(bool color) { return color ? "\x1b[0m" : ""; } + +static void print_human(const doctor_check *checks, size_t n, + const doctor_summary *s, bool color) { + printf("%sds4-doctor%s\n\n", color ? "\x1b[1m" : "", color_off(color)); + printf("%-9s %-5s %-7s %s\n", + "ID", "STATE", "TIME", "MESSAGE"); + for (size_t i = 0; i < n; i++) { + const doctor_check *c = &checks[i]; + char tbuf[16]; + if (c->duration_ms < 0) snprintf(tbuf, sizeof(tbuf), "0ms"); + else if (c->duration_ms >= 1000) + snprintf(tbuf, sizeof(tbuf), "1s+"); + else snprintf(tbuf, sizeof(tbuf), "%dms", c->duration_ms); + printf("%-9s %s%-5s%s %-7s %s\n", + c->id ? c->id : "", + status_color(c->status, color), + doctor_status_name(c->status), + color_off(color), + tbuf, + c->message); + } + printf("\nStatus: %s%s%s (%d checks: %d ok, %d warn, %d fail, %d skip)\n", + status_color(doctor_summary_status(s), color), + doctor_summary_name(s), + color_off(color), + (int)n, s->ok, s->warn, s->fail, s->skip); +} + +int main(int argc, char **argv) { + doctor_config cfg = parse_options(argc, argv); + + doctor_check checks[5]; + doctor_summary summary = {0}; + summary.worst = DOCTOR_OK; + uint64_t t0; + + t0 = now_ms(); + checks[0] = doctor_check_backend(); + checks[0].duration_ms = (int)(now_ms() - t0); + doctor_summary_update(&summary, checks[0].status); + + t0 = now_ms(); + checks[1] = doctor_check_model(cfg.model_path, cfg.timeout_ms); + checks[1].duration_ms = (int)(now_ms() - t0); + doctor_summary_update(&summary, checks[1].status); + + t0 = now_ms(); + checks[2] = doctor_check_port(); + checks[2].duration_ms = (int)(now_ms() - t0); + doctor_summary_update(&summary, checks[2].status); + + t0 = now_ms(); + checks[3] = doctor_check_memory(); + checks[3].duration_ms = (int)(now_ms() - t0); + doctor_summary_update(&summary, checks[3].status); + + t0 = now_ms(); + checks[4] = doctor_check_kv_disk(cfg.kv_disk_dir); + checks[4].duration_ms = (int)(now_ms() - t0); + doctor_summary_update(&summary, checks[4].status); + + size_t n = sizeof(checks) / sizeof(checks[0]); + + if (cfg.json_output) { + print_json(checks, n, &summary); + } else { + bool color = cfg.color_output || isatty(fileno(stdout)); + print_human(checks, n, &summary, color); + } + + /* Exit-code policy (per R4 in the plan): + * 0 = no FAIL, no WARN (everything OK or SKIP) + * 1 = at least one WARN, no FAIL + * 2 = at least one FAIL + */ + switch (doctor_summary_status(&summary)) { + case DOCTOR_OK: return 0; + case DOCTOR_WARN: return 1; + case DOCTOR_FAIL: return 2; + case DOCTOR_SKIP: return 0; + } + return 0; +} diff --git a/ds4_help.c b/ds4_help.c index d32e088cf..85d6fe832 100644 --- a/ds4_help.c +++ b/ds4_help.c @@ -107,6 +107,7 @@ static const char *tool_name(ds4_help_tool tool) { case DS4_HELP_AGENT: return "ds4-agent"; case DS4_HELP_BENCH: return "ds4-bench"; case DS4_HELP_EVAL: return "ds4-eval"; + case DS4_HELP_DOCTOR: return "ds4-doctor"; } return "ds4"; } @@ -123,6 +124,8 @@ static const char *tool_usage(ds4_help_tool tool) { return "Usage: ds4-bench (--prompt-file FILE | --chat-prompt-file FILE) [options]"; case DS4_HELP_EVAL: return "Usage: ds4-eval [options]"; + case DS4_HELP_DOCTOR: + return "Usage: ds4-doctor [options]"; } return "Usage: ds4 [options]"; } @@ -139,6 +142,8 @@ static const char *tool_summary(ds4_help_tool tool) { return "Measure prefill, decode, context growth, and KV-cache size across repeatable context frontiers."; case DS4_HELP_EVAL: return "Run the built-in reasoning, math, science, and security evaluation harness with a live terminal UI."; + case DS4_HELP_DOCTOR: + return "Run read-only self-diagnostics: backend compile, model file integrity, and environment sanity."; } return ""; } @@ -369,6 +374,18 @@ static void print_eval_specific(FILE *fp, const help_colors *c) { fputc('\n', fp); } +static void print_doctor_specific(FILE *fp, const help_colors *c) { + title(fp, c, "Doctor Checks"); + para(fp, c, "ds4-doctor runs read-only self-diagnostics against the local installation. No GPU loads beyond what the probe functions already do, no network calls, no filesystem writes. Intended to be safe in CI without GPU hardware."); + fputc('\n', fp); + opt(fp, c, "--json", "Emit a machine-readable JSON report on stdout."); + opt(fp, c, "--color", "Force colored output even when stdout is not a TTY."); + opt(fp, c, "-m, --model FILE", "GGUF model to validate. Default: ds4flash.gguf"); + opt(fp, c, "--kv-disk-dir DIR", "KV disk directory to stat. Default: unset (skip disk check)."); + opt(fp, c, "--timeout N", "Per-check timeout in ms. 0..600000. Default: 5000"); + fputc('\n', fp); +} + static bool tool_has_topic(ds4_help_tool tool, const char *topic) { if (!topic) return true; if (streq(topic, "all")) return true; @@ -388,6 +405,8 @@ static bool tool_has_topic(ds4_help_tool tool, const char *topic) { return streq(topic, "benchmark"); case DS4_HELP_EVAL: return streq(topic, "evaluation"); + case DS4_HELP_DOCTOR: + return false; } return false; } @@ -449,6 +468,9 @@ static void print_examples(FILE *fp, const help_colors *c, ds4_help_tool tool, c } else if (tool == DS4_HELP_EVAL) { opt(fp, c, "eval", "./ds4-eval --questions 10 --ctx 100000"); opt(fp, c, "CPU debug", "./ds4-eval --cpu --questions 1 --tokens 32"); + } else if (tool == DS4_HELP_DOCTOR) { + opt(fp, c, "human", "./ds4-doctor"); + opt(fp, c, "json", "./ds4-doctor --json | jq .status"); } else { opt(fp, c, "Metal", "./ds4 -m ds4flash.gguf --metal -c 100000"); opt(fp, c, "quiet thermals", "./ds4 -p \"Summarize README\" --power 50"); @@ -467,6 +489,10 @@ static void print_examples(FILE *fp, const help_colors *c, ds4_help_tool tool, c } else if (tool == DS4_HELP_EVAL || topic_is(topic, "evaluation")) { opt(fp, c, "first 10", "./ds4-eval --questions 10 --trace eval.trace"); opt(fp, c, "plain", "./ds4-eval --plain --nothink --tokens 512"); + } else if (tool == DS4_HELP_DOCTOR) { + opt(fp, c, "human", "./ds4-doctor"); + opt(fp, c, "json", "./ds4-doctor --json"); + opt(fp, c, "model check", "./ds4-doctor -m /path/to/model.gguf"); } else { opt(fp, c, "chat", "./ds4"); opt(fp, c, "one shot", "./ds4 -p \"Explain mmap in C\""); @@ -495,6 +521,8 @@ static void print_topic(FILE *fp, const help_colors *c, ds4_help_tool tool, cons print_bench_specific(fp, c); } else if (tool == DS4_HELP_EVAL) { print_eval_specific(fp, c); + } else if (tool == DS4_HELP_DOCTOR) { + print_doctor_specific(fp, c); } return; } @@ -536,6 +564,8 @@ static void print_default(FILE *fp, const help_colors *c, ds4_help_tool tool) { print_bench_specific(fp, c); } else if (tool == DS4_HELP_EVAL) { print_eval_specific(fp, c); + } else if (tool == DS4_HELP_DOCTOR) { + print_doctor_specific(fp, c); } } diff --git a/ds4_help.h b/ds4_help.h index f4fa400b6..9c4dab9f0 100644 --- a/ds4_help.h +++ b/ds4_help.h @@ -9,6 +9,7 @@ typedef enum { DS4_HELP_AGENT, DS4_HELP_BENCH, DS4_HELP_EVAL, + DS4_HELP_DOCTOR, } ds4_help_tool; void ds4_help_print(FILE *fp, ds4_help_tool tool, const char *topic); diff --git a/tests/ds4_test.c b/tests/ds4_test.c index ea1e52487..5b26e62de 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -4,6 +4,8 @@ #ifndef DS4_NO_GPU #include "../ds4_gpu.h" #include +#endif +#include static ds4_engine *test_engine_fast; static ds4_engine *test_engine_quality; @@ -2180,6 +2182,125 @@ static void test_server_unit_group(void) { ds4_server_unit_tests_run(); } +/* ds4-doctor is a separate binary; tests shell out to it via popen(). + * The default binary path is "./ds4-doctor" relative to the test CWD; + * DS4_TEST_DOCTOR_BIN overrides it (handy for cross-build CI). */ +static const char *test_doctor_bin(void) { + const char *env = getenv("DS4_TEST_DOCTOR_BIN"); + return (env && env[0]) ? env : "./ds4-doctor"; +} + +/* Run `bin ` and capture stdout in `out` (size `out_cap`). Returns + * the child exit status, or -1 on popen failure. Stderr is left for the + * child to surface — the test relies on the JSON output, not on stderr. */ +static int test_doctor_run(const char *bin, const char *args, char *out, + size_t out_cap) { + if (out_cap == 0) return -1; + out[0] = '\0'; + char cmd[1024]; + int n = snprintf(cmd, sizeof(cmd), "%s %s 2>/dev/null", bin, args); + if (n < 0 || (size_t)n >= sizeof(cmd)) return -1; + FILE *fp = popen(cmd, "r"); + if (!fp) return -1; + size_t got = 0; + while (got + 1 < out_cap) { + size_t r = fread(out + got, 1, out_cap - 1 - got, fp); + if (r == 0) break; + got += r; + } + out[got] = '\0'; + int status = pclose(fp); + if (status == -1) return -1; + /* pclose returns the wait(2) status; the child's exit code is in the + * upper byte for a normal exit. */ + if (WIFEXITED(status)) return WEXITSTATUS(status); + return -1; +} + +/* Write a temp file with the given bytes; returns the path on success or + * NULL on failure. The path uses mkstemp so it's race-safe across CI. */ +static const char *test_doctor_write_temp(const uint8_t *bytes, size_t n_bytes, + char *path_buf, size_t path_cap) { + if (path_cap < 32) return NULL; + strcpy(path_buf, "/tmp/ds4_test_doctor_XXXXXX"); + int fd = mkstemp(path_buf); + if (fd < 0) return NULL; + size_t written = 0; + while (written < n_bytes) { + ssize_t w = write(fd, bytes + written, n_bytes - written); + if (w <= 0) { close(fd); unlink(path_buf); return NULL; } + written += (size_t)w; + } + close(fd); + return path_buf; +} + +static void test_doctor_json_shape(void) { + char out[8192]; + int rc = test_doctor_run(test_doctor_bin(), "--json", out, sizeof(out)); + TEST_ASSERT(rc >= 0); + TEST_ASSERT(out[0] == '{'); + TEST_ASSERT(strstr(out, "\"status\":\"ok\"") || + strstr(out, "\"status\":\"warn\"") || + strstr(out, "\"status\":\"fail\"")); + TEST_ASSERT(strstr(out, "\"id\":\"backend\"")); + TEST_ASSERT(strstr(out, "\"id\":\"model\"")); + TEST_ASSERT(strstr(out, "\"id\":\"port\"")); + TEST_ASSERT(strstr(out, "\"id\":\"memory\"")); + TEST_ASSERT(strstr(out, "\"id\":\"kv-disk\"")); +} + +static void test_doctor_bad_model_file(void) { + char path_buf[64]; + const char *path = test_doctor_write_temp( + (const uint8_t *)"this is not a gguf file at all", + 30, path_buf, sizeof(path_buf)); + TEST_ASSERT(path != NULL); + + char args[128]; + snprintf(args, sizeof(args), "-m %s --json", path); + char out[8192]; + int rc = test_doctor_run(test_doctor_bin(), args, out, sizeof(out)); + TEST_ASSERT(rc >= 0); + /* Bad model -> at least one FAIL, so rc must be 2 per R4. */ + TEST_ASSERT(rc == 2); + TEST_ASSERT(strstr(out, "\"id\":\"model\"")); + TEST_ASSERT(strstr(out, "\"status\":\"fail\"")); + + unlink(path); +} + +static void test_doctor_good_model_header(void) { + /* 24-byte GGUF v3 header: magic + version=3 + n_tensors=1 + n_kv=0. */ + uint8_t hdr[24] = {0}; + hdr[0] = 'G'; hdr[1] = 'G'; hdr[2] = 'U'; hdr[3] = 'F'; + hdr[4] = 3; /* version = 3 (LE u32) */ + hdr[8] = 1; /* n_tensors = 1 */ + char path_buf[64]; + const char *path = test_doctor_write_temp(hdr, sizeof(hdr), + path_buf, sizeof(path_buf)); + TEST_ASSERT(path != NULL); + + char args[128]; + snprintf(args, sizeof(args), "-m %s --json", path); + char out[8192]; + int rc = test_doctor_run(test_doctor_bin(), args, out, sizeof(out)); + TEST_ASSERT(rc >= 0); + /* 24 valid header bytes -> model check is OK, and nothing else fails + * by default -> overall status must be ok. */ + TEST_ASSERT(rc == 0); + TEST_ASSERT(strstr(out, "\"id\":\"model\",\"title\":\"Model file integrity\"")); + TEST_ASSERT(strstr(out, "\"status\":\"ok\"")); + + unlink(path); +} + +static void test_doctor_group(void) { + test_doctor_json_shape(); + test_doctor_bad_model_file(); + test_doctor_good_model_header(); +} + typedef void (*test_fn)(void); typedef struct { @@ -2204,6 +2325,7 @@ static const ds4_test_entry test_entries[] = { {"--mtp-verify-depth", "mtp-verify-depth", "MTP speculative verify commits autoregressive-identical tokens at draft depth > 2", test_mtp_verify_depth}, #endif {"--server", "server", "server parser/rendering/cache unit tests", test_server_unit_group}, + {"--doctor", "doctor", "ds4-doctor JSON shape, bad/valid GGUF checks, exit codes", test_doctor_group}, }; static void test_print_help(const char *prog) {