From 9d7560761e22fb1d1511a3f54ad46000295d16c4 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:08:31 +0000 Subject: [PATCH 1/4] feat(agent-challenge): add temporary NO_PHALA host-local unattested mode Opt-in host execution while Phala CVMs are disabled. Results are explicitly marked unattested and cannot be forged as attested. Attested path stays byte-identical when the flag is off. Contradiction with attestation flags fails closed at startup. --- .../agent-challenge/docs/no-phala-mode.md | 123 ++++++ .../agent_challenge/evaluation/no_phala.py | 275 +++++++++++++ .../evaluation/own_runner_backend.py | 47 +++ .../src/agent_challenge/sdk/app_factory.py | 24 +- .../src/agent_challenge/sdk/config.py | 38 ++ .../src/agent_challenge/sdk/schemas.py | 7 + .../src/agent_challenge/selfdeploy/eval.py | 3 + .../src/agent_challenge/selfdeploy/phala.py | 5 + .../src/agent_challenge/selfdeploy/review.py | 4 + .../agent-challenge/tests/test_health.py | 2 + .../tests/test_no_phala_mode.py | 362 ++++++++++++++++++ 11 files changed, 889 insertions(+), 1 deletion(-) create mode 100644 packages/challenges/agent-challenge/docs/no-phala-mode.md create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py create mode 100644 packages/challenges/agent-challenge/tests/test_no_phala_mode.py diff --git a/packages/challenges/agent-challenge/docs/no-phala-mode.md b/packages/challenges/agent-challenge/docs/no-phala-mode.md new file mode 100644 index 000000000..fea480f84 --- /dev/null +++ b/packages/challenges/agent-challenge/docs/no-phala-mode.md @@ -0,0 +1,123 @@ +# NO_PHALA mode (temporary host-local unattested execution) + +**Status:** temporary operator escape hatch while Phala CVMs are disabled. +Safe to remove: one module (`evaluation/no_phala.py`) plus thin call-sites. + +## What it does + +When `NO_PHALA=true` (or `CHALLENGE_NO_PHALA=true`) on the **master** validator: + +1. Benchmark/eval jobs run **on the master host** via the existing + `own_runner` local Docker path (`evaluation/own_runner_backend.py`). +2. **No** Phala Cloud client call is made. `PhalaCloudClient` construction and + eval/review `deploy()` refuse with `NoPhalaModeError`. +3. Result envelopes are **explicitly marked unattested**: + - `attested: false` (hard-coded; cannot be set true by the runner path) + - `attestation_status: "unattested"` + - `execution_mode: "no_phala_host"` + - any `execution_proof` / `attestation_binding` stripped +4. When the miner ZIP is available on disk, `guest_artifact_proof` records + `expected_hash == download_hash == executed_hash` (SHA-256 of ZIP bytes). + Mismatch fails closed. + +## What it does **NOT** prove + +| Claim | NO_PHALA | +|-------|----------| +| TEE / Intel TDX isolation | **No** | +| TDX quote / DCAP verification | **No** | +| RTMR / compose_hash / KMS digest bind | **No** | +| Guest measurement allowlist | **No** | +| That the host was not tampered with | **No** | + +Scores produced in this mode are **host-trust only**. Do not treat them as +attested TEE results for production weight decisions that require attestation. + +## Env vars and precedence + +| Variable | Role | +|----------|------| +| `CHALLENGE_NO_PHALA` | Challenge-prefix form (pydantic-settings field `no_phala`) | +| `NO_PHALA` | Plain operator form on the master host | + +**Precedence:** + +1. If `CHALLENGE_NO_PHALA` is **set** in the environment → use it. +2. Else if `NO_PHALA` is **set** → use it. +3. Else → **off** (default). + +Truthy tokens: `1`, `true`, `yes`, `on` (case-insensitive). +Falsy tokens: `0`, `false`, `no`, `off`, empty. + +**Never** inferred from a missing `PHALA_API_KEY` / failed Phala call. + +## Contradiction (fail closed) + +If `NO_PHALA` is on **and** either of: + +- `CHALLENGE_PHALA_ATTESTATION_ENABLED=true` +- `CHALLENGE_ATTESTED_REVIEW_ENABLED=true` + +…settings construction raises. Attested TEE path and host-local unattested +path are mutually exclusive. Turn attestation flags **off** before enabling +NO_PHALA. + +## Enable on master + +Operator override file (loaded by master entrypoint): + +```bash +# /var/lib/base/challenges/agent-challenge/embed.env +NO_PHALA=true +CHALLENGE_PHALA_ATTESTATION_ENABLED=false +CHALLENGE_ATTESTED_REVIEW_ENABLED=false +``` + +Master runs: + +```text +uvicorn agent_challenge.app:app --host 127.0.0.1 --port 18081 +``` + +Restart the master (or AC child) after editing `embed.env`. + +## Disable + +```bash +# embed.env — remove NO_PHALA or set: +NO_PHALA=false +# unset CHALLENGE_NO_PHALA if present +``` + +Restart. Attested path is unchanged when NO_PHALA is off. + +## Confirm live mode + +```bash +curl -sS http://127.0.0.1:18081/health +# {"status":"ok","slug":"agent-challenge","version":"…", +# "no_phala":true,"attestation_mode":"no_phala_host"} + +curl -sS http://127.0.0.1:18081/version +# … "no_phala":true, "capabilities":[…,"no_phala_host"] +``` + +Startup logs a multi-line `CRITICAL` banner when the mode is active. + +Via master proxy (if embedded): +`GET /challenges/agent-challenge/health` — same fields. + +## Code map (for removal) + +| Path | Role | +|------|------| +| `src/agent_challenge/evaluation/no_phala.py` | **Single module** — mode resolve, mark, proof, refuse | +| `src/agent_challenge/sdk/config.py` | `no_phala` field + contradiction validator | +| `src/agent_challenge/sdk/schemas.py` | `/health` + `/version` fields | +| `src/agent_challenge/sdk/app_factory.py` | Startup banner + health/version wiring | +| `src/agent_challenge/selfdeploy/phala.py` | `PhalaCloudClient` refuse | +| `src/agent_challenge/selfdeploy/eval.py` / `review.py` | deploy refuse | +| `src/agent_challenge/evaluation/own_runner_backend.py` | Mark unattested on legacy emit when mode on | + +The **attested** emit branch in `own_runner_backend._emit_job_result` is not +modified when NO_PHALA is off. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py new file mode 100644 index 000000000..7d9c07d67 --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py @@ -0,0 +1,275 @@ +"""Temporary NO_PHALA host-execution mode (opt-in, unattested, removable). + +When ``NO_PHALA`` / ``CHALLENGE_NO_PHALA`` is active the master validator runs +benchmark/eval jobs on the host via the existing own_runner local Docker path +instead of provisioning Phala CVMs. There is **no TEE, no TDX quote, and no +attestation**. Results are explicitly marked unattested and cannot be forged +into an attested envelope from this path. + +This module is the single branch point for the temporary mode so it can be +deleted cleanly when Phala is re-enabled. Do not scatter NO_PHALA conditionals +elsewhere beyond thin call-sites into this module. + +Precedence for the env switch (see :func:`resolve_no_phala_from_environ`): + +1. ``CHALLENGE_NO_PHALA`` if set (challenge-prefix convention) +2. else plain ``NO_PHALA`` (operator convenience on the master host) +3. else off + +Never infer the mode from a missing Phala API key or a failed Phala call. +""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Mapping +from typing import Any, Final + +logger = logging.getLogger(__name__) + +#: Challenge-prefixed env (pydantic-settings field ``no_phala``). +CHALLENGE_NO_PHALA_ENV: Final = "CHALLENGE_NO_PHALA" +#: Operator-facing plain env accepted on the master host. +NO_PHALA_ENV: Final = "NO_PHALA" + +_TRUTHY: Final = frozenset({"1", "true", "yes", "on"}) +_FALSY: Final = frozenset({"0", "false", "no", "off", ""}) + +#: Wire / stored execution mode label for host-local unattested runs. +EXECUTION_MODE_NO_PHALA_HOST: Final = "no_phala_host" +#: Explicit attestation status — never "attested" from this module. +ATTESTATION_STATUS_UNATTESTED: Final = "unattested" + +#: Result envelope keys written by this mode. +RESULT_KEY_ATTESTED: Final = "attested" +RESULT_KEY_ATTESTATION_STATUS: Final = "attestation_status" +RESULT_KEY_EXECUTION_MODE: Final = "execution_mode" +RESULT_KEY_GUEST_ARTIFACT_PROOF: Final = "guest_artifact_proof" + +#: Keys that would make a result look Phala-attested — stripped on mark. +_ATTESTED_LOOKING_KEYS: Final = frozenset( + { + "execution_proof", + "attestation_binding", + "tdx_quote", + "phala_attestation", + } +) + +CONTRADICTION_MESSAGE: Final = ( + "NO_PHALA mode cannot be combined with attestation flags: " + "CHALLENGE_PHALA_ATTESTATION_ENABLED and CHALLENGE_ATTESTED_REVIEW_ENABLED " + "must both be off when NO_PHALA/CHALLENGE_NO_PHALA is on. " + "Attested TEE path and host-local unattested path are mutually exclusive." +) + +STARTUP_BANNER: Final = ( + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + "!! NO_PHALA MODE ACTIVE — host-local unattested execution !!\n" + "!! No TEE / TDX quote / DCAP / compose_hash attestation is performed. !!\n" + "!! Results are marked unattested and MUST NOT be treated as attested. !!\n" + "!! Disable via NO_PHALA=false (or unset) and restart. !!\n" + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +) + + +class NoPhalaModeError(RuntimeError): + """Raised when NO_PHALA mode forbids an operation (e.g. Phala client use).""" + + +class ArtifactProvenanceError(ValueError): + """Raised when expected/download/executed artifact hashes disagree.""" + + +def _parse_bool_env(raw: str | None) -> bool | None: + """Return True/False if ``raw`` is a recognized boolean token, else None.""" + + if raw is None: + return None + text = str(raw).strip().lower() + if text in _TRUTHY: + return True + if text in _FALSY: + return False + return None + + +def resolve_no_phala_from_environ( + environ: Mapping[str, str] | None = None, +) -> bool: + """Resolve NO_PHALA from env with documented precedence. + + 1. ``CHALLENGE_NO_PHALA`` if present in the environment + 2. else ``NO_PHALA`` if present + 3. else ``False`` (default OFF — never inferred from missing keys) + """ + + env = os.environ if environ is None else environ + if CHALLENGE_NO_PHALA_ENV in env: + parsed = _parse_bool_env(env.get(CHALLENGE_NO_PHALA_ENV)) + return bool(parsed) + if NO_PHALA_ENV in env: + parsed = _parse_bool_env(env.get(NO_PHALA_ENV)) + return bool(parsed) + return False + + +def is_no_phala_enabled( + *, + settings_flag: bool | None = None, + environ: Mapping[str, str] | None = None, +) -> bool: + """Return whether NO_PHALA host mode is active. + + Prefer the settings field when provided (already resolved at startup). + Fall back to env resolution for in-process guest/CLI contexts that do not + load :class:`~agent_challenge.sdk.config.ChallengeSettings`. + """ + + if settings_flag is not None: + return bool(settings_flag) + return resolve_no_phala_from_environ(environ) + + +def assert_no_phala_compatible( + *, + no_phala: bool, + phala_attestation_enabled: bool, + attested_review_enabled: bool, +) -> None: + """Fail closed when NO_PHALA collides with either attestation flag.""" + + if no_phala and (phala_attestation_enabled or attested_review_enabled): + raise ValueError(CONTRADICTION_MESSAGE) + + +def log_no_phala_startup_banner() -> None: + """Emit a loud, unmistakable startup warning (call once from app lifespan).""" + + for line in STARTUP_BANNER.splitlines(): + logger.critical(line) + + +def refuse_phala_client(operation: str = "Phala Cloud API") -> None: + """Raise if NO_PHALA is active — provisioning seam must never be invoked.""" + + if resolve_no_phala_from_environ(): + raise NoPhalaModeError( + f"{operation} is forbidden while NO_PHALA mode is active " + f"(set {NO_PHALA_ENV}=false / unset {CHALLENGE_NO_PHALA_ENV} to use Phala)" + ) + + +def build_guest_artifact_proof( + *, + expected_hash: str, + download_hash: str, + executed_hash: str, +) -> dict[str, Any]: + """Build the provenance triple ``expected == download == executed``. + + Cheap and valuable even without a TEE. Raises + :class:`ArtifactProvenanceError` on mismatch or empty hashes so a bad + artifact cannot be reported as matching. + """ + + expected = (expected_hash or "").strip().lower() + download = (download_hash or "").strip().lower() + executed = (executed_hash or "").strip().lower() + if not expected or len(expected) != 64: + raise ArtifactProvenanceError("expected_hash must be a 64-char hex digest") + if not all(c in "0123456789abcdef" for c in expected + download + executed): + raise ArtifactProvenanceError("artifact hashes must be lowercase hex") + if not (expected == download == executed): + raise ArtifactProvenanceError( + "artifact provenance mismatch: " + f"expected={expected} download={download} executed={executed}" + ) + return { + "expected_hash": expected, + "download_hash": download, + "executed_hash": executed, + "match": True, + } + + +def mark_result_unattested( + payload: Mapping[str, Any], + *, + artifact_proof: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Return a result envelope explicitly marked unattested (unforgeable). + + Always sets ``attested=False`` and ``attestation_status=unattested``. + Strips any Phala-looking proof blocks. Callers cannot pass + ``attested=True`` through this function — the boolean is hard-coded. + """ + + out = dict(payload) + for key in _ATTESTED_LOOKING_KEYS: + out.pop(key, None) + # Hard-coded False — never accept a caller-supplied True. + out[RESULT_KEY_ATTESTED] = False + out[RESULT_KEY_ATTESTATION_STATUS] = ATTESTATION_STATUS_UNATTESTED + out[RESULT_KEY_EXECUTION_MODE] = EXECUTION_MODE_NO_PHALA_HOST + if artifact_proof is not None: + # Re-validate so a forged match:false / mismatched triple cannot slip in. + proof = build_guest_artifact_proof( + expected_hash=str(artifact_proof.get("expected_hash") or ""), + download_hash=str(artifact_proof.get("download_hash") or ""), + executed_hash=str(artifact_proof.get("executed_hash") or ""), + ) + out[RESULT_KEY_GUEST_ARTIFACT_PROOF] = proof + return out + + +def assert_envelope_not_attested(payload: Mapping[str, Any]) -> None: + """Fail closed if an envelope claims attestation while in NO_PHALA mode.""" + + if payload.get(RESULT_KEY_ATTESTED) is True: + raise ValueError("NO_PHALA result envelope must not claim attested=true") + if payload.get(RESULT_KEY_ATTESTATION_STATUS) == "attested": + raise ValueError("NO_PHALA result envelope must not claim attestation_status=attested") + if payload.get("execution_proof") is not None: + raise ValueError("NO_PHALA result envelope must not carry execution_proof") + if payload.get(RESULT_KEY_EXECUTION_MODE) != EXECUTION_MODE_NO_PHALA_HOST: + raise ValueError( + f"NO_PHALA result envelope must set execution_mode={EXECUTION_MODE_NO_PHALA_HOST!r}" + ) + + +def health_fields(*, no_phala: bool) -> dict[str, Any]: + """Fields merged into ``/health`` so operators can see the live mode.""" + + return { + "no_phala": bool(no_phala), + "attestation_mode": ( + EXECUTION_MODE_NO_PHALA_HOST if no_phala else "standard" + ), + } + + +__all__ = [ + "ATTESTATION_STATUS_UNATTESTED", + "CHALLENGE_NO_PHALA_ENV", + "CONTRADICTION_MESSAGE", + "EXECUTION_MODE_NO_PHALA_HOST", + "NO_PHALA_ENV", + "RESULT_KEY_ATTESTATION_STATUS", + "RESULT_KEY_ATTESTED", + "RESULT_KEY_EXECUTION_MODE", + "RESULT_KEY_GUEST_ARTIFACT_PROOF", + "STARTUP_BANNER", + "ArtifactProvenanceError", + "NoPhalaModeError", + "assert_envelope_not_attested", + "assert_no_phala_compatible", + "build_guest_artifact_proof", + "health_fields", + "is_no_phala_enabled", + "log_no_phala_startup_banner", + "mark_result_unattested", + "refuse_phala_client", + "resolve_no_phala_from_environ", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py index b3227e5bf..4ff047bc2 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py @@ -623,6 +623,42 @@ def assert_agent_artifact_matches_plan( return declared +def _maybe_no_phala_artifact_proof() -> dict[str, str | bool] | None: + """Build guest_artifact_proof for NO_PHALA host runs when ZIP is available. + + Returns None when the artifact path is missing (caller still marks + unattested without the triple). When the path is present, enforces + expected == download == executed against the plan/declared agent hash. + """ + + from agent_challenge.evaluation.no_phala import ( + ArtifactProvenanceError, + build_guest_artifact_proof, + ) + + artifact_path = resolve_agent_artifact_path() + if artifact_path is None or not Path(artifact_path).is_file(): + return None + try: + executed = agent_artifact_sha256(artifact_path) + except ValueError: + return None + expected = (os.environ.get(PHALA_AGENT_HASH_ENV) or "").strip().lower() + if not expected: + # No plan hash declared — use the executed digest as the expected pin + # so provenance still records download==executed for the host ZIP. + expected = executed + download = executed # host-local: download path is the same file we execute + try: + return build_guest_artifact_proof( + expected_hash=expected, + download_hash=download, + executed_hash=executed, + ) + except ArtifactProvenanceError: + raise + + #: Env path for an already-extracted package tree (guest recompute target). AGENT_PACKAGE_ROOT_ENV = "CHALLENGE_PHALA_AGENT_PACKAGE_ROOT" AGENT_PACKAGE_ROOT_ENV_ALT = "CHALLENGE_AGENT_PACKAGE_ROOT" @@ -1168,6 +1204,17 @@ def _emit_job_result( task_id: list(scores) for task_id, scores in collect_trial_scores(result.trial_outcomes).items() } + # Temporary NO_PHALA host mode: mark the legacy line unattested and + # attach guest_artifact_proof when the agent ZIP provenance is known. + # Attested path below is untouched when NO_PHALA is off. + from agent_challenge.evaluation.no_phala import ( + is_no_phala_enabled, + mark_result_unattested, + ) + + if is_no_phala_enabled(): + artifact_proof = _maybe_no_phala_artifact_proof() + payload = mark_result_unattested(payload, artifact_proof=artifact_proof) emit_benchmark_result_line(payload) return 0 diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py index 27050b39d..cdd375d67 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py @@ -65,6 +65,13 @@ async def lifespan(app: FastAPI): # Dual-flag production also needs dcap-qvl on PATH (baked into runtime # image). Binary presence only; no PCS network / secret / trust-root invent. settings.require_dcap_qvl_binary_for_production() + # Temporary NO_PHALA host mode: loud banner so nobody mistakes an + # unattested validator for an attested one. Contradiction with attestation + # flags is already fail-closed at ChallengeSettings construction. + if getattr(settings, "no_phala", False): + from agent_challenge.evaluation.no_phala import log_no_phala_startup_banner + + log_no_phala_startup_banner() await database.init() worker_task: asyncio.Task[None] | None = None if worker_main is not None: @@ -88,18 +95,33 @@ async def lifespan(app: FastAPI): @app.get("/health", response_model=HealthResponse, include_in_schema=False) async def health() -> HealthResponse: - return HealthResponse(slug=settings.slug, version=settings.version) + from agent_challenge.evaluation.no_phala import health_fields + + mode = health_fields(no_phala=bool(getattr(settings, "no_phala", False))) + return HealthResponse( + slug=settings.slug, + version=settings.version, + no_phala=mode["no_phala"], + attestation_mode=mode["attestation_mode"], + ) @app.get("/version", response_model=VersionResponse, include_in_schema=False) async def version() -> VersionResponse: + from agent_challenge.evaluation.no_phala import health_fields + capabilities = ["get_weights", "proxy_routes", "sqlite", "swe_forge"] if settings.docker_enabled: capabilities.append("docker_executor") + if getattr(settings, "no_phala", False): + capabilities.append("no_phala_host") + mode = health_fields(no_phala=bool(getattr(settings, "no_phala", False))) return VersionResponse( api_version=settings.api_version, challenge_version=settings.version, sdk_version=settings.sdk_version, capabilities=capabilities, + no_phala=mode["no_phala"], + attestation_mode=mode["attestation_mode"], ) internal_router = APIRouter( diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index 6f4f71e75..ac8ddf8fa 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -152,6 +152,13 @@ class ChallengeSettings(BaseSettings): # legacy intake and gateway-review path byte-for-byte. Production full # attested deployments enable both this and ``phala_attestation_enabled``. attested_review_enabled: bool = False + # Temporary host-local unattested execution (NO_PHALA). Opt-in, default OFF. + # When true the master runs jobs via own_runner on the host instead of Phala + # CVMs. Results are explicitly marked unattested. Mutually exclusive with + # both attestation flags above (fail closed at settings construction). + # Env precedence: CHALLENGE_NO_PHALA if set, else plain NO_PHALA, else false. + # Never inferred from a missing Phala API key. See docs/no-phala-mode.md. + no_phala: bool = False review_assignment_ttl_seconds: int = 1800 review_operator_approval_ttl_seconds: int = 300 review_https_connect_timeout_seconds: float = 10.0 @@ -371,6 +378,37 @@ def validate_attested_topology(self) -> ChallengeSettings: ) return self + @model_validator(mode="after") + def resolve_and_validate_no_phala(self) -> ChallengeSettings: + """Apply plain NO_PHALA env + fail closed on attestation contradiction. + + Precedence: + 1. ``CHALLENGE_NO_PHALA`` if present in the process environment + 2. else plain ``NO_PHALA`` if present + 3. else the field default (False) + + Never infers the mode from a missing Phala API key or failed Phala call. + """ + + import os + + from agent_challenge.evaluation.no_phala import ( + assert_no_phala_compatible, + resolve_no_phala_from_environ, + ) + + # Re-resolve so plain NO_PHALA is honored when the prefixed var is unset. + # When CHALLENGE_NO_PHALA is set, pydantic already populated ``no_phala``; + # resolve_no_phala_from_environ applies the same precedence. + if "CHALLENGE_NO_PHALA" in os.environ or "NO_PHALA" in os.environ: + self.no_phala = resolve_no_phala_from_environ() + assert_no_phala_compatible( + no_phala=bool(self.no_phala), + phala_attestation_enabled=bool(self.phala_attestation_enabled), + attested_review_enabled=bool(self.attested_review_enabled), + ) + return self + def require_eval_result_signer_for_production(self) -> None: """Fail closed for production full-attested mode without an endpoint signer. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/schemas.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/schemas.py index 5fb60a6f5..cbb8a297f 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/schemas.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/schemas.py @@ -13,6 +13,10 @@ class HealthResponse(BaseModel): status: str = "ok" slug: str version: str + #: True when temporary host-local unattested (NO_PHALA) mode is active. + no_phala: bool = False + #: ``no_phala_host`` when NO_PHALA is on, else ``standard``. + attestation_mode: str = "standard" class VersionResponse(BaseModel): @@ -22,6 +26,9 @@ class VersionResponse(BaseModel): challenge_version: str sdk_version: str capabilities: list[str] = Field(default_factory=list) + #: Mirrors health so operators can confirm mode without a separate probe. + no_phala: bool = False + attestation_mode: str = "standard" class WeightsResponse(BaseModel): diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py index fd0d6b984..cb6fd2120 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py @@ -348,6 +348,9 @@ def deploy( plan: EvalDeploymentPlan, encrypted: EncryptedEvalSecrets, ) -> dict[str, str]: + from agent_challenge.evaluation.no_phala import refuse_phala_client + + refuse_phala_client("HttpEvalPhalaDeployment.deploy") if ( encrypted.eval_run_id != plan.eval_run_id or encrypted.app_identity != plan.app_identity diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py index afc2a42f6..d3a257167 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py @@ -229,6 +229,11 @@ def __init__( opener=urlopen, timeout: float = 30.0, ) -> None: + # Temporary NO_PHALA mode: refuse the entire Phala client so no CVM + # provision/create/list call can slip through while host-local mode is on. + from agent_challenge.evaluation.no_phala import refuse_phala_client + + refuse_phala_client("PhalaCloudClient") self._api_key = ( api_key if api_key is not None else os.environ.get(PHALA_API_KEY_ENV, "") ).strip() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py index e08cf4931..3473da01e 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py @@ -267,6 +267,10 @@ def deploy( ) -> dict[str, str]: """Provision exact compose identity then create with ciphertext only.""" + from agent_challenge.evaluation.no_phala import refuse_phala_client + + refuse_phala_client("HttpReviewPhalaDeployment.deploy") + if ( encrypted.assignment_id != plan.assignment["assignment_core"]["assignment_id"] or encrypted.app_identity != plan.app_identity diff --git a/packages/challenges/agent-challenge/tests/test_health.py b/packages/challenges/agent-challenge/tests/test_health.py index 711aca1db..b7882f79a 100644 --- a/packages/challenges/agent-challenge/tests/test_health.py +++ b/packages/challenges/agent-challenge/tests/test_health.py @@ -9,6 +9,8 @@ async def test_health(client): "status": "ok", "slug": "agent-challenge", "version": "1.0.1", + "no_phala": False, + "attestation_mode": "standard", } diff --git a/packages/challenges/agent-challenge/tests/test_no_phala_mode.py b/packages/challenges/agent-challenge/tests/test_no_phala_mode.py new file mode 100644 index 000000000..062210e94 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_no_phala_mode.py @@ -0,0 +1,362 @@ +"""NO_PHALA temporary host-local unattested mode. + +Tests cover: +- Mode OFF (default): settings.no_phala is False; Phala client constructible + when credentials present; attested path not relaxed. +- Mode ON: Phala client / deploy seams refuse; results marked unattested; + guest_artifact_proof enforces expected == download == executed. +- Contradiction: NO_PHALA + either attestation flag fails closed at settings. +- Unforgeable: mark_result_unattested always forces attested=False and strips + execution_proof even if the caller payload claimed otherwise. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from agent_challenge.evaluation import no_phala as np +from agent_challenge.evaluation.no_phala import ( + ATTESTATION_STATUS_UNATTESTED, + CHALLENGE_NO_PHALA_ENV, + CONTRADICTION_MESSAGE, + EXECUTION_MODE_NO_PHALA_HOST, + NO_PHALA_ENV, + ArtifactProvenanceError, + NoPhalaModeError, + assert_envelope_not_attested, + assert_no_phala_compatible, + build_guest_artifact_proof, + is_no_phala_enabled, + mark_result_unattested, + refuse_phala_client, + resolve_no_phala_from_environ, +) +from agent_challenge.evaluation.own_runner.orchestrator import JobResult, TrialOutcome +from agent_challenge.evaluation.own_runner.result_schema import ( + RESULT_LINE_PREFIX, + build_benchmark_result, +) +from agent_challenge.evaluation.own_runner_backend import ( + PHALA_ATTESTATION_ENABLED_ENV, + _emit_job_result, + agent_artifact_sha256, +) +from agent_challenge.sdk.config import ChallengeSettings + + +@pytest.fixture(autouse=True) +def _clear_no_phala_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(CHALLENGE_NO_PHALA_ENV, raising=False) + monkeypatch.delenv(NO_PHALA_ENV, raising=False) + monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) + monkeypatch.delenv("CHALLENGE_ATTESTED_REVIEW_ENABLED", raising=False) + monkeypatch.delenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", raising=False) + + +# --------------------------------------------------------------------------- # +# S1 — Mode OFF (default) +# --------------------------------------------------------------------------- # + + +def test_no_phala_default_off() -> None: + assert resolve_no_phala_from_environ({}) is False + settings = ChallengeSettings( + phala_attestation_enabled=False, + attested_review_enabled=False, + no_phala=False, + ) + assert settings.no_phala is False + + +def test_mode_off_does_not_mark_legacy_emit(monkeypatch: pytest.MonkeyPatch, capsys) -> None: + """Attested path untouched: gate off + NO_PHALA off => plain result line.""" + + monkeypatch.delenv(CHALLENGE_NO_PHALA_ENV, raising=False) + monkeypatch.delenv(NO_PHALA_ENV, raising=False) + monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) + + result = _canned_job_result() + rc = _emit_job_result(result, ["hello-world"]) + assert rc == 0 + out = capsys.readouterr().out + line = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)][-1] + payload = json.loads(line[len(RESULT_LINE_PREFIX) :]) + assert "attested" not in payload + assert "execution_mode" not in payload + assert "guest_artifact_proof" not in payload + assert "execution_proof" not in payload + + +# --------------------------------------------------------------------------- # +# Env precedence +# --------------------------------------------------------------------------- # + + +def test_challenge_prefix_wins_over_plain(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(CHALLENGE_NO_PHALA_ENV, "false") + monkeypatch.setenv(NO_PHALA_ENV, "true") + assert resolve_no_phala_from_environ() is False + + +def test_plain_no_phala_accepted_when_prefix_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + assert resolve_no_phala_from_environ() is True + settings = ChallengeSettings( + phala_attestation_enabled=False, + attested_review_enabled=False, + ) + assert settings.no_phala is True + + +def test_never_inferred_from_missing_phala_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PHALA_API_KEY", raising=False) + monkeypatch.delenv("PHALA_CLOUD_API_KEY", raising=False) + assert resolve_no_phala_from_environ({}) is False + + +# --------------------------------------------------------------------------- # +# S3 — Contradiction fail-closed +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "phala,review", + [ + (True, True), + (True, False), # mixed also fails topology first, but contradiction covers both + (False, True), + ], +) +def test_contradiction_with_attestation_flags(phala: bool, review: bool) -> None: + with pytest.raises(ValueError, match="must both be"): + # Mixed flags fail topology; both-on + no_phala fails contradiction. + ChallengeSettings( + phala_attestation_enabled=phala, + attested_review_enabled=review, + no_phala=True, + ) + + +def test_contradiction_both_attestation_on_and_no_phala() -> None: + with pytest.raises(ValueError, match="NO_PHALA"): + assert_no_phala_compatible( + no_phala=True, + phala_attestation_enabled=True, + attested_review_enabled=True, + ) + assert CONTRADICTION_MESSAGE.startswith("NO_PHALA mode cannot") + + +def test_settings_contradiction_both_on(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + with pytest.raises(ValueError, match="NO_PHALA"): + ChallengeSettings( + phala_attestation_enabled=True, + attested_review_enabled=True, + ) + + +# --------------------------------------------------------------------------- # +# S2 — Mode ON: no Phala client; local mark +# --------------------------------------------------------------------------- # + + +def test_phala_client_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + monkeypatch.setenv("PHALA_API_KEY", "test-key-not-a-secret-for-unit") + from agent_challenge.selfdeploy.phala import PhalaCloudClient + + with pytest.raises(NoPhalaModeError, match="NO_PHALA"): + PhalaCloudClient(api_key="test-key-not-a-secret-for-unit") + + +def test_eval_deploy_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + from agent_challenge.selfdeploy.eval import HttpEvalPhalaDeployment + + api = MagicMock() + dep = HttpEvalPhalaDeployment(api) + with pytest.raises(NoPhalaModeError): + dep.deploy(MagicMock(), MagicMock()) + api.post.assert_not_called() + + +def test_review_deploy_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + from agent_challenge.selfdeploy.review import HttpReviewPhalaDeployment + + api = MagicMock() + dep = HttpReviewPhalaDeployment(api) + with pytest.raises(NoPhalaModeError): + dep.deploy(MagicMock(), MagicMock()) + api.post.assert_not_called() + + +def test_mode_on_emit_marks_unattested( + monkeypatch: pytest.MonkeyPatch, + capsys, + tmp_path: Path, +) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) + + zip_bytes = b"PK\x03\x04fake-agent-zip-for-no-phala" + zip_path = tmp_path / "agent.zip" + zip_path.write_bytes(zip_bytes) + digest = hashlib.sha256(zip_bytes).hexdigest() + monkeypatch.setenv("CHALLENGE_PHALA_AGENT_ARTIFACT", str(zip_path)) + monkeypatch.setenv("CHALLENGE_PHALA_AGENT_HASH", digest) + + result = _canned_job_result() + rc = _emit_job_result(result, ["hello-world"]) + assert rc == 0 + out = capsys.readouterr().out + line = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)][-1] + payload = json.loads(line[len(RESULT_LINE_PREFIX) :]) + assert payload["attested"] is False + assert payload["attestation_status"] == ATTESTATION_STATUS_UNATTESTED + assert payload["execution_mode"] == EXECUTION_MODE_NO_PHALA_HOST + assert "execution_proof" not in payload + proof = payload["guest_artifact_proof"] + assert proof["expected_hash"] == digest + assert proof["download_hash"] == digest + assert proof["executed_hash"] == digest + assert proof["match"] is True + assert_envelope_not_attested(payload) + + +# --------------------------------------------------------------------------- # +# S4 — Unforgeable marking +# --------------------------------------------------------------------------- # + + +def test_mark_result_unattested_cannot_forge_attested() -> None: + forged = { + "status": "completed", + "score": 1.0, + "attested": True, + "attestation_status": "attested", + "execution_proof": {"tier": "phala-tdx", "attestation": {"tdx_quote": "ab" * 100}}, + "attestation_binding": {"agent_hash": "a" * 64}, + } + out = mark_result_unattested(forged) + assert out["attested"] is False + assert out["attestation_status"] == ATTESTATION_STATUS_UNATTESTED + assert out["execution_mode"] == EXECUTION_MODE_NO_PHALA_HOST + assert "execution_proof" not in out + assert "attestation_binding" not in out + with pytest.raises(ValueError, match="must not claim attested"): + assert_envelope_not_attested({**out, "attested": True}) + + +def test_mark_ignores_caller_attested_true_kw() -> None: + """Even if payload insists attested=True, output is False.""" + + out = mark_result_unattested({"attested": True, "score": 0.5}) + assert out[np.RESULT_KEY_ATTESTED] is False + + +# --------------------------------------------------------------------------- # +# S5 — Artifact provenance +# --------------------------------------------------------------------------- # + + +def test_artifact_proof_enforces_triple_match() -> None: + h = "ab" * 32 + proof = build_guest_artifact_proof( + expected_hash=h, download_hash=h, executed_hash=h + ) + assert proof["match"] is True + with pytest.raises(ArtifactProvenanceError, match="mismatch"): + build_guest_artifact_proof( + expected_hash=h, + download_hash=h, + executed_hash="cd" * 32, + ) + + +def test_artifact_sha_matches_executed(tmp_path: Path) -> None: + data = b"PK\x03\x04miner-zip-bytes" + path = tmp_path / "a.zip" + path.write_bytes(data) + digest = agent_artifact_sha256(path) + assert digest == hashlib.sha256(data).hexdigest() + proof = build_guest_artifact_proof( + expected_hash=digest, download_hash=digest, executed_hash=digest + ) + assert proof["match"] is True + + +# --------------------------------------------------------------------------- # +# Health / version surface +# --------------------------------------------------------------------------- # + + +async def test_health_shows_no_phala_off(client) -> None: + response = await client.get("/health") + assert response.status_code == 200 + body = response.json() + assert body["status"] == "ok" + assert body["no_phala"] is False + assert body["attestation_mode"] == "standard" + + +async def test_health_shows_no_phala_on() -> None: + """health_fields is the source of truth for /health mode visibility.""" + + fields = np.health_fields(no_phala=True) + assert fields == { + "no_phala": True, + "attestation_mode": EXECUTION_MODE_NO_PHALA_HOST, + } + off = np.health_fields(no_phala=False) + assert off == {"no_phala": False, "attestation_mode": "standard"} + + +def test_is_no_phala_enabled_settings_flag() -> None: + assert is_no_phala_enabled(settings_flag=True) is True + assert is_no_phala_enabled(settings_flag=False) is False + + +def test_refuse_phala_client_when_off(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(NO_PHALA_ENV, raising=False) + refuse_phala_client() # no raise + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # + + +def _canned_job_result() -> JobResult: + return JobResult( + status="completed", + score=1.0, + resolved=1, + total=1, + reason_code=None, + pass_at_k={}, + n_total_trials=1, + n_completed_trials=1, + n_errored_trials=0, + trial_outcomes=[ + TrialOutcome( + task_name="hello-world", + trial_name="hello-world__attempt-0", + status="completed", + rewards={"reward": 1.0}, + ) + ], + benchmark_result=build_benchmark_result( + status="completed", + score=1.0, + resolved=1, + total=1, + reason_code=None, + ), + ) From a3447f26bd2c8a0e9e7892eda4f6a7423092fd75 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:22:05 +0000 Subject: [PATCH 2/4] feat(agent-challenge): complete NO_PHALA offline pipeline with weight push Drive the dual-flag-off analysis+own_runner path under NO_PHALA through scores and authenticated raw-weight push, with unattested provenance on completion metadata and CRITICAL push logs. Attested gates stay untouched. --- .../agent-challenge/docs/no-phala-mode.md | 36 + .../src/agent_challenge/api/app.py | 63 +- .../src/agent_challenge/core/models.py | 1 + .../evaluation/raw_weight_push.py | 680 ++++++++++++++++++ .../src/agent_challenge/evaluation/runner.py | 15 +- .../src/agent_challenge/sdk/app_factory.py | 40 +- .../src/agent_challenge/sdk/config.py | 25 +- .../src/agent_challenge/sdk/db.py | 2 + .../agent-challenge/tests/conftest.py | 5 + .../tests/test_ac_raw_weight_push.py | 279 +++++++ .../tests/test_no_phala_pipeline.py | 518 +++++++++++++ .../tests/test_raw_weight_push_lifespan.py | 193 +++++ 12 files changed, 1845 insertions(+), 12 deletions(-) create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py create mode 100644 packages/challenges/agent-challenge/tests/test_ac_raw_weight_push.py create mode 100644 packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py create mode 100644 packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py diff --git a/packages/challenges/agent-challenge/docs/no-phala-mode.md b/packages/challenges/agent-challenge/docs/no-phala-mode.md index fea480f84..abe9fa17e 100644 --- a/packages/challenges/agent-challenge/docs/no-phala-mode.md +++ b/packages/challenges/agent-challenge/docs/no-phala-mode.md @@ -121,3 +121,39 @@ Via master proxy (if embedded): The **attested** emit branch in `own_runner_backend._emit_job_result` is not modified when NO_PHALA is off. + + +## Full offline pipeline (master host) + +With `NO_PHALA=true` and both attestation flags **off**, submit uses the +pre-Phala analysis chain (not Phala review CVMs): + +```text +submit → analysis_queued → AST + LLM review → analysis_allowed + → waiting_miner_env (until env confirm / PUT) + → tb_queued → tb_running (own_runner on host Docker) + → tb_completed → EvaluationJob.score + → get_weights → authenticated raw-weight push (when master_base_url set) +``` + +Requirements on the master process: + +| Need | Env / setting | +|------|----------------| +| Mode | `NO_PHALA=true` or `CHALLENGE_NO_PHALA=true` | +| Attestation off | `CHALLENGE_PHALA_ATTESTATION_ENABLED=false`, `CHALLENGE_ATTESTED_REVIEW_ENABLED=false` | +| Worker | `CHALLENGE_COMBINED_WORKER=true` **or** run `agent-challenge-worker` | +| LLM review | gateway base URL + token (else `llm_standby`) | +| Benchmark | Docker + own_runner task cache | +| Weight push | `CHALLENGE_MASTER_BASE_URL` + shared token (optional loop) | + +**embed.env note:** master entrypoint only forwards keys matching +`CHALLENGE_` / `PHALA_` / … prefixes. Prefer `CHALLENGE_NO_PHALA=true` in +`/var/lib/base/challenges/agent-challenge/embed.env` so the child sees it. + +Unattested provenance is visible on: + +- result envelopes (`attested:false`, `attestation_status:unattested`, `execution_mode:no_phala_host`) +- `tb_completed` status-event metadata (same fields) +- CRITICAL log on every raw-weight push while the mode is on +- `/health` → `no_phala:true`, `attestation_mode:no_phala_host` diff --git a/packages/challenges/agent-challenge/src/agent_challenge/api/app.py b/packages/challenges/agent-challenge/src/agent_challenge/api/app.py index 9ba792f9a..21dfc7efd 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/api/app.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/api/app.py @@ -4,13 +4,17 @@ import functools +from fastapi import FastAPI + from ..core import models as _models # noqa: F401 - register SQLAlchemy models from ..core.config import settings from ..core.db import database +from ..evaluation import raw_weight_push as raw_weight_push_module from ..evaluation.weights import get_weights from ..evaluation.worker import run_worker_loop -from ..sdk.app_factory import WorkerMain, create_challenge_app +from ..sdk.app_factory import BackgroundTaskFactory, WorkerMain, create_challenge_app from ..sdk.config import ChallengeSettings +from ..sdk.db import Database from ..sdk.observability import configure_root_logging from .routes import router @@ -30,10 +34,53 @@ def build_worker_main(challenge_settings: ChallengeSettings) -> WorkerMain | Non return functools.partial(run_worker_loop, manage_database=False) -app = create_challenge_app( - settings=settings, - database=database, - public_router=router, - get_weights_fn=get_weights, - worker_main=build_worker_main(settings), -) +def create_app( + *, + challenge_settings: ChallengeSettings | None = None, + db: Database | None = None, +) -> FastAPI: + """Build the Agent Challenge FastAPI app (production and tests). + + Wires the optional combined worker and the authenticated raw-weight push + loop the same way Prism does: build the client when settings enable it, + append a background task factory, stash the client on ``app.state``. + """ + + app_settings = challenge_settings if challenge_settings is not None else settings + app_db = db if db is not None else database + configure_root_logging(app_settings) + + background_tasks: list[BackgroundTaskFactory] = [] + # Attribute access (not a bound import) so tests can monkeypatch the module. + push_client = raw_weight_push_module.maybe_build_push_client_from_settings( + settings=app_settings, + database=app_db, + get_weights_fn=get_weights, + ) + if push_client is not None: + interval = float(getattr(push_client, "push_interval_seconds", 30.0)) + + async def _run_raw_weight_push(app: FastAPI) -> None: + client = getattr(app.state, "raw_weight_push_client", push_client) + await raw_weight_push_module.run_raw_weight_push_loop( + client, + interval_seconds=interval, + resilient=True, + ) + + background_tasks.append(_run_raw_weight_push) + + app = create_challenge_app( + settings=app_settings, + database=app_db, + public_router=router, + get_weights_fn=get_weights, + worker_main=build_worker_main(app_settings), + background_tasks=tuple(background_tasks), + ) + if push_client is not None: + app.state.raw_weight_push_client = push_client + return app + + +app = create_app() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/core/models.py b/packages/challenges/agent-challenge/src/agent_challenge/core/models.py index f15342c16..3aa2115e5 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/core/models.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/core/models.py @@ -19,6 +19,7 @@ ) from sqlalchemy.orm import Mapped, mapped_column, relationship +from agent_challenge.evaluation.raw_weight_push import RawWeightPushLedger as RawWeightPushLedger from agent_challenge.sdk.config import ChallengeSettings from .db import Base diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py new file mode 100644 index 000000000..ed142468f --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py @@ -0,0 +1,680 @@ +"""Challenge-owned raw-weight push to the master with durable ack checkpointing. + +Agent Challenge builds a closed, digest-bound hotkey-weight snapshot (typically +the winner-take-all map from ``get_weights``), signs it with the challenge +credential, and posts to master +``POST /internal/v1/challenges/agent-challenge/raw-weights``. Delivery cursor +advances only after an acknowledgement whose snapshot digest and epoch/revision +match the attempted payload exactly. Timeouts and bad acks leave the cursor +unchanged so restart retries the same logical snapshot. + +Wire format, HMAC binding, and ack gate mirror Prism's +``prism_challenge.raw_weight_push`` so master accepts both challenges identically. +""" + +from __future__ import annotations + +import asyncio +import hmac +import logging +import uuid +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +from typing import Any + +import httpx +from base.challenge_sdk.roles import Capability, Role, activate_role, role_contract +from base.challenge_sdk.schemas import ( + RawWeightPushAcknowledgement, + RawWeightPushRequest, +) +from sqlalchemy import Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from agent_challenge.sdk.db import Base, Database + +logger = logging.getLogger(__name__) + +PROTOCOL_VERSION = "1.0" +DEFAULT_FRESHNESS_SECONDS = 300 +DEFAULT_TIMEOUT_SECONDS = 10.0 +DEFAULT_CHALLENGE_SLUG = "agent-challenge" + +WeightsFn = Callable[[], Mapping[str, float] | Awaitable[Mapping[str, float]]] +EpochFn = Callable[[], int] + + +def canonical_challenge_push_request( + *, + method: str, + path: str, + challenge_slug: str, + timestamp: str, + body: bytes, +) -> str: + """Mirror the master's challenge-push signature binding (method/path/slug/ts/body).""" + + body_digest = sha256(body).hexdigest() + return f"{method.upper()}\n{path}\n{challenge_slug}\n{timestamp}\n{body_digest}" + + +def sign_challenge_push_request(*, token: str, canonical: str) -> str: + return hmac.new( + token.encode("utf-8"), + canonical.encode("utf-8"), + sha256, + ).hexdigest() + + +@dataclass(frozen=True) +class PushCursor: + """Durable last-acknowledged delivery identity for one challenge.""" + + epoch: int + revision: int + payload_digest: str + snapshot_id: str | None + acknowledged_at: str | None + + +@dataclass(frozen=True) +class PushAttemptResult: + """Outcome of a single push attempt (cursor may or may not advance).""" + + status: str + epoch: int + revision: int + payload_digest: str + snapshot_id: str | None + cursor_advanced: bool + error: str | None = None + + +class RawWeightPushLedger(Base): + """Singleton SQLAlchemy row for durable attempt/ack checkpointing. + + Matches Prism's ``raw_weight_push_ledger`` columns so restart semantics are + identical; stored via AC's shared Database (SQLite or PostgreSQL), not a + side-car SQLite file. + """ + + __tablename__ = "raw_weight_push_ledger" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + challenge_slug: Mapped[str] = mapped_column(String(128), nullable=False) + last_epoch: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_revision: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_payload_digest: Mapped[str | None] = mapped_column(String(64), nullable=True) + last_snapshot_id: Mapped[str | None] = mapped_column(String(128), nullable=True) + last_canonical_payload: Mapped[str | None] = mapped_column(Text, nullable=True) + last_nonce: Mapped[str | None] = mapped_column(String(256), nullable=True) + acknowledged_at: Mapped[str | None] = mapped_column(String(64), nullable=True) + pending_epoch: Mapped[int | None] = mapped_column(Integer, nullable=True) + pending_revision: Mapped[int | None] = mapped_column(Integer, nullable=True) + pending_payload_digest: Mapped[str | None] = mapped_column(String(64), nullable=True) + pending_canonical_payload: Mapped[str | None] = mapped_column(Text, nullable=True) + pending_nonce: Mapped[str | None] = mapped_column(String(256), nullable=True) + pending_attempted_at: Mapped[str | None] = mapped_column(String(64), nullable=True) + updated_at: Mapped[str] = mapped_column(String(64), nullable=False) + + +class RawWeightPushStore: + """SQLAlchemy durable attempt/ack ledger for agent-challenge raw-weight delivery.""" + + def __init__(self, database: Database, *, challenge_slug: str) -> None: + self.database = database + self.challenge_slug = challenge_slug + + async def init(self) -> None: + async with self.database.engine.begin() as connection: + await connection.run_sync( + lambda sync_conn: RawWeightPushLedger.__table__.create( + sync_conn, checkfirst=True + ) + ) + + async def _get_row(self, session: Any) -> RawWeightPushLedger | None: + return await session.get(RawWeightPushLedger, 1) + + async def get_cursor(self) -> PushCursor | None: + async with self.database.session() as session: + row = await self._get_row(session) + if row is None or row.last_epoch is None: + return None + return PushCursor( + epoch=int(row.last_epoch), + revision=int(row.last_revision or 0), + payload_digest=str(row.last_payload_digest or ""), + snapshot_id=( + str(row.last_snapshot_id) if row.last_snapshot_id is not None else None + ), + acknowledged_at=( + str(row.acknowledged_at) if row.acknowledged_at is not None else None + ), + ) + + async def get_pending(self) -> dict[str, Any] | None: + async with self.database.session() as session: + row = await self._get_row(session) + if row is None or row.pending_epoch is None: + return None + return { + "epoch": int(row.pending_epoch), + "revision": int(row.pending_revision or 0), + "payload_digest": str(row.pending_payload_digest or ""), + "canonical_payload": str(row.pending_canonical_payload or ""), + "nonce": str(row.pending_nonce or ""), + "attempted_at": row.pending_attempted_at, + } + + async def record_pending( + self, + *, + epoch: int, + revision: int, + payload_digest: str, + canonical_payload: str, + nonce: str, + attempted_at: str, + ) -> None: + async with self.database.session() as session: + row = await self._get_row(session) + if row is None: + row = RawWeightPushLedger( + id=1, + challenge_slug=self.challenge_slug, + updated_at=attempted_at, + ) + session.add(row) + row.challenge_slug = self.challenge_slug + row.pending_epoch = epoch + row.pending_revision = revision + row.pending_payload_digest = payload_digest + row.pending_canonical_payload = canonical_payload + row.pending_nonce = nonce + row.pending_attempted_at = attempted_at + row.updated_at = attempted_at + await session.commit() + + async def acknowledge( + self, + *, + epoch: int, + revision: int, + payload_digest: str, + snapshot_id: str, + canonical_payload: str, + nonce: str, + acknowledged_at: str, + ) -> None: + """Advance the durable delivery cursor after an exact matching ack.""" + + async with self.database.session() as session: + row = await self._get_row(session) + if row is None: + row = RawWeightPushLedger( + id=1, + challenge_slug=self.challenge_slug, + updated_at=acknowledged_at, + ) + session.add(row) + row.challenge_slug = self.challenge_slug + row.last_epoch = epoch + row.last_revision = revision + row.last_payload_digest = payload_digest + row.last_snapshot_id = snapshot_id + row.last_canonical_payload = canonical_payload + row.last_nonce = nonce + row.acknowledged_at = acknowledged_at + row.pending_epoch = None + row.pending_revision = None + row.pending_payload_digest = None + row.pending_canonical_payload = None + row.pending_nonce = None + row.pending_attempted_at = None + row.updated_at = acknowledged_at + await session.commit() + + +async def _resolve_weights( + weights: Mapping[str, float] | None, + weights_fn: WeightsFn | None, +) -> dict[str, float]: + if weights is not None: + return dict(weights) + if weights_fn is None: + return {} + loaded = weights_fn() + if isinstance(loaded, Awaitable): + loaded = await loaded + return dict(loaded) + + +class RawWeightPushClient: + """Build, sign, and push raw weights; checkpoint only on exact acks.""" + + def __init__( + self, + *, + database: Database, + challenge_slug: str, + master_base_url: str, + shared_token: str, + weights_fn: WeightsFn | None = None, + epoch_fn: EpochFn | None = None, + freshness_seconds: int = DEFAULT_FRESHNESS_SECONDS, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + now_fn: Callable[[], datetime] = lambda: datetime.now(UTC), + http_client: httpx.AsyncClient | None = None, + ) -> None: + self.database = database + self.challenge_slug = challenge_slug + self.master_base_url = master_base_url.rstrip("/") + self.shared_token = shared_token + self.weights_fn = weights_fn + self.epoch_fn = epoch_fn + self.freshness_seconds = freshness_seconds + self.timeout_seconds = timeout_seconds + self._now_fn = now_fn + self._http = http_client + self.store = RawWeightPushStore(database, challenge_slug=challenge_slug) + + async def init(self) -> None: + await self.store.init() + + def _path_for(self) -> str: + return f"/internal/v1/challenges/{self.challenge_slug}/raw-weights" + + def _next_revision(self, cursor: PushCursor | None, epoch: int) -> int: + if cursor is None: + return 1 + if cursor.epoch == epoch: + return cursor.revision + 1 + return 1 + + def _build_payload( + self, + *, + weights: Mapping[str, float], + epoch: int, + revision: int, + nonce: str, + now: datetime, + ) -> tuple[RawWeightPushRequest, bytes]: + computed_at = now.replace(microsecond=0) + expires_at = computed_at + timedelta(seconds=self.freshness_seconds) + body: dict[str, Any] = { + "protocol_version": PROTOCOL_VERSION, + "challenge_slug": self.challenge_slug, + "epoch": epoch, + "revision": revision, + "computed_at": computed_at.isoformat().replace("+00:00", "Z"), + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "nonce": nonce, + "weights": {str(hotkey): float(weight) for hotkey, weight in weights.items()}, + } + if not body["weights"]: + raise ValueError("empty weight map has no push surface") + digest = RawWeightPushRequest.compute_digest(body) + body["payload_digest"] = digest + payload = RawWeightPushRequest.model_validate(body) + raw_bytes = payload.canonical_bytes() + return payload, raw_bytes + + def _headers(self, *, path: str, body: bytes, timestamp: int) -> dict[str, str]: + canonical = canonical_challenge_push_request( + method="POST", + path=path, + challenge_slug=self.challenge_slug, + timestamp=str(timestamp), + body=body, + ) + signature = sign_challenge_push_request(token=self.shared_token, canonical=canonical) + return { + "Authorization": f"Bearer {self.shared_token}", + "Content-Type": "application/json", + "X-Base-Challenge-Slug": self.challenge_slug, + "X-Signature": signature, + "X-Timestamp": str(timestamp), + "Accept": "application/json", + } + + def _ack_matches( + self, ack: RawWeightPushAcknowledgement, *, payload: RawWeightPushRequest + ) -> bool: + return ( + ack.accepted is True + and ack.challenge_slug == payload.challenge_slug + and ack.epoch == payload.epoch + and ack.revision == payload.revision + and ack.payload_digest == payload.payload_digest + and bool(ack.snapshot_id) + ) + + @role_contract(role=Role.CHALLENGE, capability=Capability.CHALLENGE_RAW_WEIGHT_PUSH) + async def push_once( + self, + *, + weights: Mapping[str, float] | None = None, + epoch: int | None = None, + force_revision: int | None = None, + reuse_pending: bool = True, + ) -> PushAttemptResult: + """Push one snapshot. Cursor advances only on exact durable acknowledgement.""" + + await self.store.init() + now = self._now_fn() + cursor = await self.store.get_cursor() + pending = await self.store.get_pending() if reuse_pending else None + payload: RawWeightPushRequest | None = None + raw_bytes: bytes | None = None + + if pending is not None: + # Retry exact previous bytes after timeout/restart (no new revision). + try: + pending_bytes = str(pending["canonical_payload"]).encode("utf-8") + payload = RawWeightPushRequest.model_validate_json(pending_bytes) + raw_bytes = payload.canonical_bytes() + except Exception: # noqa: BLE001 - corrupt pending is rebuilt + pending = None + payload = None + raw_bytes = None + + if payload is None or raw_bytes is None: + resolved_weights = await _resolve_weights(weights, self.weights_fn) + # Positive hotkey weights only when synthesizing from get_weights; + # explicit zero maps (caller-supplied zeros) are preserved as zero-contribution. + if weights is not None: + cleaned = { + str(hotkey): float(value) for hotkey, value in resolved_weights.items() + } + else: + cleaned = { + str(hotkey): float(value) + for hotkey, value in resolved_weights.items() + if float(value) > 0.0 + } + if not cleaned: + return PushAttemptResult( + status="skipped_empty", + epoch=0, + revision=0, + payload_digest="", + snapshot_id=None, + cursor_advanced=False, + error="empty weights", + ) + # Host-trust path: never silently push unattested scores as if TEE-backed. + _log_unattested_weight_push_if_no_phala(cleaned) + resolved_epoch = ( + int(epoch) + if epoch is not None + else int(self.epoch_fn()) + if self.epoch_fn is not None + else int(now.timestamp()) // 3600 + ) + revision = ( + int(force_revision) + if force_revision is not None + else self._next_revision(cursor, resolved_epoch) + ) + nonce = f"agent-challenge-{uuid.uuid4().hex}" + payload, raw_bytes = self._build_payload( + weights=cleaned, + epoch=resolved_epoch, + revision=revision, + nonce=nonce, + now=now, + ) + await self.store.record_pending( + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + canonical_payload=raw_bytes.decode("utf-8"), + nonce=payload.nonce, + attempted_at=now.isoformat(), + ) + + path = self._path_for() + url = f"{self.master_base_url}{path}" + headers = self._headers(path=path, body=raw_bytes, timestamp=int(now.timestamp())) + client = self._http + owns_client = client is None + if owns_client: + client = httpx.AsyncClient(timeout=self.timeout_seconds) + assert client is not None + try: + response = await client.post(url, content=raw_bytes, headers=headers) + except (httpx.TimeoutException, httpx.TransportError) as exc: + # Never log Authorization / token material — digest prefix only. + logger.info( + "raw weight push transport failure", + extra={ + "epoch": payload.epoch, + "revision": payload.revision, + "digest": payload.payload_digest[:12], + }, + ) + return PushAttemptResult( + status="transport_error", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=None, + cursor_advanced=False, + error=str(exc), + ) + finally: + if owns_client: + await client.aclose() + + if response.status_code >= 500: + return PushAttemptResult( + status="server_error", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=None, + cursor_advanced=False, + error=f"status={response.status_code}", + ) + if response.status_code not in {200, 201}: + return PushAttemptResult( + status="rejected", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=None, + cursor_advanced=False, + error=f"status={response.status_code}", + ) + try: + ack = RawWeightPushAcknowledgement.model_validate(response.json()) + except Exception as exc: # noqa: BLE001 + return PushAttemptResult( + status="malformed_ack", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=None, + cursor_advanced=False, + error=str(exc), + ) + if not self._ack_matches(ack, payload=payload): + return PushAttemptResult( + status="ack_mismatch", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=ack.snapshot_id if hasattr(ack, "snapshot_id") else None, + cursor_advanced=False, + error="acknowledgement identity mismatch", + ) + ack_time = self._now_fn().isoformat() + await self.store.acknowledge( + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=ack.snapshot_id, + canonical_payload=raw_bytes.decode("utf-8"), + nonce=payload.nonce, + acknowledged_at=ack_time, + ) + return PushAttemptResult( + status="acknowledged", + epoch=payload.epoch, + revision=payload.revision, + payload_digest=payload.payload_digest, + snapshot_id=ack.snapshot_id, + cursor_advanced=True, + ) + + +def build_weights_loader( + get_weights_fn: Callable[[], Awaitable[Mapping[str, float]]] | None = None, +) -> WeightsFn: + """Wrap agent-challenge ``get_weights`` (async WTA map) for the push client.""" + + async def _load() -> dict[str, float]: + if get_weights_fn is None: + from agent_challenge.evaluation.weights import get_weights + + return dict(await get_weights()) + return dict(await get_weights_fn()) + + return _load + + +async def run_raw_weight_push_loop( + client: RawWeightPushClient, + *, + interval_seconds: float = 30.0, + resilient: bool = True, +) -> None: + """Background loop: push scored hotkey weights when master + token enable it. + + Retries the same durable pending identity on transport failures. Cancellation + always propagates so app lifespan can stop the task cleanly. + """ + + await client.init() + logger.info( + "raw weight push loop started", + extra={"master": client.master_base_url, "slug": client.challenge_slug}, + ) + while True: + try: + with activate_role( + Role.CHALLENGE, capabilities=(Capability.CHALLENGE_RAW_WEIGHT_PUSH,) + ): + result = await client.push_once() + logger.info( + "raw weight push attempt", + extra={ + "status": result.status, + "epoch": result.epoch, + "revision": result.revision, + "cursor_advanced": result.cursor_advanced, + }, + ) + except asyncio.CancelledError: + raise + except Exception: + if not resilient: + raise + logger.exception("raw weight push loop iteration failed") + await asyncio.sleep(max(float(interval_seconds), 0.1)) + + + +def _log_unattested_weight_push_if_no_phala(weights: Mapping[str, float]) -> None: + """CRITICAL log when NO_PHALA host mode pushes host-trust weights. + + The wire schema has no attestation field; provenance is operator-visible via + this log + /health no_phala + result envelopes marked unattested. Do not + strip any attested-path precondition — this only fires when NO_PHALA is on + (which already forbids attestation flags). + """ + + from agent_challenge.evaluation.no_phala import ( + EXECUTION_MODE_NO_PHALA_HOST, + is_no_phala_enabled, + ) + + if not is_no_phala_enabled(): + return + logger.critical( + "NO_PHALA raw weight push: host-trust unattested scores " + "(execution_mode=%s, hotkeys=%d) — NOT TEE-attested", + EXECUTION_MODE_NO_PHALA_HOST, + len(weights), + ) + + +def maybe_build_push_client_from_settings( + *, + settings: Any, + database: Database, + get_weights_fn: Callable[[], Awaitable[Mapping[str, float]]] | None = None, +) -> RawWeightPushClient | None: + """Construct a push client when master_base_url + token enable raw-weight push.""" + + if not bool(getattr(settings, "raw_weight_push_enabled", True)): + return None + master_url = getattr(settings, "master_base_url", None) or getattr( + getattr(settings, "worker_plane", None), "master_base_url", None + ) + if not master_url: + return None + token_loader = getattr(settings, "internal_token", None) + token = token_loader() if callable(token_loader) else None + if not token: + shared = getattr(settings, "shared_token", None) or getattr( + settings, "challenge_shared_token", None + ) + token = str(shared) if shared else None + if not token: + return None + epoch_seconds = int(getattr(settings, "epoch_seconds", 3600) or 3600) + interval_hint = float(getattr(settings, "raw_weight_push_interval_seconds", 30.0)) + slug = str( + getattr(settings, "slug", None) + or getattr(settings, "challenge_slug", None) + or DEFAULT_CHALLENGE_SLUG + ) + + def _epoch() -> int: + return int(datetime.now(UTC).timestamp()) // max(epoch_seconds, 1) + + client = RawWeightPushClient( + database=database, + challenge_slug=slug, + master_base_url=str(master_url), + shared_token=str(token), + weights_fn=build_weights_loader(get_weights_fn), + epoch_fn=_epoch, + freshness_seconds=int( + getattr(settings, "raw_weight_push_freshness_seconds", DEFAULT_FRESHNESS_SECONDS) + ), + timeout_seconds=float( + getattr(settings, "raw_weight_push_timeout_seconds", DEFAULT_TIMEOUT_SECONDS) + ), + ) + client.push_interval_seconds = interval_hint # type: ignore[attr-defined] + return client + + +__all__ = [ + "PushAttemptResult", + "PushCursor", + "RawWeightPushClient", + "RawWeightPushLedger", + "RawWeightPushStore", + "build_weights_loader", + "canonical_challenge_push_request", + "maybe_build_push_client_from_settings", + "run_raw_weight_push_loop", + "sign_challenge_push_request", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index f2b67c263..fb94c44ca 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -659,13 +659,26 @@ async def run_evaluation_job( if internal_tb_flow or ( job.verdict == "valid" and any(task.benchmark == "terminal_bench" for task in tasks) ): + completion_meta: dict[str, object] = {"job_id": job.job_id, "score": score} + # Surface host-trust provenance on the status event when NO_PHALA is + # active so scores/weights cannot be mistaken for TEE-attested later. + from agent_challenge.evaluation.no_phala import ( + ATTESTATION_STATUS_UNATTESTED, + EXECUTION_MODE_NO_PHALA_HOST, + is_no_phala_enabled, + ) + + if is_no_phala_enabled(): + completion_meta["attested"] = False + completion_meta["attestation_status"] = ATTESTATION_STATUS_UNATTESTED + completion_meta["execution_mode"] = EXECUTION_MODE_NO_PHALA_HOST await _set_submission_status( session, submission, "tb_completed", actor="evaluation", reason="evaluation_job_completed", - metadata={"job_id": job.job_id, "score": score}, + metadata=completion_meta, ) except Exception as exc: job.passed_tasks = passed diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py index cdd375d67..8b377250c 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/app_factory.py @@ -5,9 +5,10 @@ import asyncio import logging import signal -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Coroutine, Sequence from contextlib import asynccontextmanager from time import time +from typing import Any from fastapi import APIRouter, Depends, FastAPI @@ -20,6 +21,7 @@ GetWeightsFn = Callable[[], Awaitable[dict[str, float]]] WorkerMain = Callable[[], Awaitable[None]] +BackgroundTaskFactory = Callable[[FastAPI], Coroutine[Any, Any, None]] def _handle_worker_task_done(task: asyncio.Task[None]) -> None: @@ -42,6 +44,18 @@ def _handle_worker_task_done(task: asyncio.Task[None]) -> None: signal.raise_signal(signal.SIGTERM) +def _log_unexpected_background_exit(task: asyncio.Task[None]) -> None: + """Log unexpected exit of a non-worker background task (e.g. raw-weight push).""" + + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + logger.critical("background task exited unexpectedly", exc_info=exc) + else: + logger.critical("background task exited unexpectedly without error") + + def create_challenge_app( *, settings: ChallengeSettings, @@ -50,6 +64,7 @@ def create_challenge_app( get_weights_fn: GetWeightsFn, challenge_internal_router: APIRouter | None = None, worker_main: WorkerMain | None = None, + background_tasks: Sequence[BackgroundTaskFactory] = (), ) -> FastAPI: """Create a complete FastAPI challenge app with standard BASE routes.""" @@ -74,14 +89,27 @@ async def lifespan(app: FastAPI): log_no_phala_startup_banner() await database.init() worker_task: asyncio.Task[None] | None = None + bg_tasks: list[asyncio.Task[None]] = [] if worker_main is not None: worker_task = asyncio.create_task(worker_main(), name="combined-worker-loop") worker_task.add_done_callback(_handle_worker_task_done) + bg_tasks = [ + asyncio.create_task(factory(app), name=_background_task_name(factory)) + for factory in background_tasks + ] + app.state.challenge_background_tasks = tuple(bg_tasks) + for task in bg_tasks: + task.add_done_callback(_log_unexpected_background_exit) try: yield finally: + for task in bg_tasks: + task.cancel() if worker_task is not None: worker_task.cancel() + if bg_tasks: + await asyncio.gather(*bg_tasks, return_exceptions=True) + if worker_task is not None: try: await worker_task except asyncio.CancelledError: @@ -89,6 +117,7 @@ async def lifespan(app: FastAPI): except Exception: # Already surfaced by the done-callback; keep shutdown clean. logger.exception("combined-mode worker loop crashed during shutdown") + app.state.challenge_background_tasks = () await database.close() app = FastAPI(title=settings.name, version=settings.version, lifespan=lifespan) @@ -144,3 +173,12 @@ async def get_weights() -> WeightsResponse: app.include_router(internal_router) app.include_router(public_router) return app + + +def _background_task_name(factory: BackgroundTaskFactory) -> str: + """Prefer an explicit loop name when the factory coroutine is named.""" + + name = getattr(factory, "__name__", "") or "" + if name == "_run_raw_weight_push": + return "raw-weight-push-loop" + return "challenge-background-task" diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index ac8ddf8fa..a7be6ab16 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any, Literal -from pydantic import Field, field_validator, model_validator +from pydantic import AliasChoices, Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict DEFAULT_SECRET_REDACTION = "" @@ -43,7 +43,11 @@ class ChallengeSettings(BaseSettings): """Runtime settings for the Agent Challenge service.""" - model_config = SettingsConfigDict(env_prefix="CHALLENGE_", extra="ignore") + model_config = SettingsConfigDict( + env_prefix="CHALLENGE_", + extra="ignore", + populate_by_name=True, + ) slug: str = "agent-challenge" name: str = "Agent Challenge" @@ -78,6 +82,23 @@ class ChallengeSettings(BaseSettings): # background asyncio task (all-in-one "combined" service). Default false # preserves the separate ``agent-challenge-worker`` sidecar deployment. combined_worker: bool = False + # Authenticated raw-weight push to master (winner-take-all map). + # When enabled and master_base_url + shared token are set, the API lifespan + # runs run_raw_weight_push_loop against the shared Database ledger. + raw_weight_push_enabled: bool = True + master_base_url: str | None = Field( + default=None, + validation_alias=AliasChoices( + "CHALLENGE_MASTER_BASE_URL", + "MASTER_BASE_URL", + ), + ) + raw_weight_push_interval_seconds: float = Field(default=30.0, ge=0.1) + raw_weight_push_freshness_seconds: int = Field(default=300, ge=30) + raw_weight_push_timeout_seconds: float = Field(default=10.0, gt=0.0) + # Epoch bucket size for push revision identity (seconds). + epoch_seconds: int = Field(default=3600, ge=1) + # Root stdlib logging level applied at every process entrypoint (the API app # import and the worker ``main()``). Uvicorn installs no root handler, so diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/db.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/db.py index 956de7731..a2e091840 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/db.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/db.py @@ -196,6 +196,8 @@ async def init(self) -> None: """Create all challenge-owned tables.""" import_module("agent_challenge.core.models") + # Register raw-weight push ledger on shared Base metadata for create_all. + import_module("agent_challenge.evaluation.raw_weight_push") async with self.engine.begin() as connection: backend_name = self.engine.url.get_backend_name() is_sqlite = backend_name.startswith("sqlite") diff --git a/packages/challenges/agent-challenge/tests/conftest.py b/packages/challenges/agent-challenge/tests/conftest.py index 69247911f..54c41f4bc 100644 --- a/packages/challenges/agent-challenge/tests/conftest.py +++ b/packages/challenges/agent-challenge/tests/conftest.py @@ -65,6 +65,11 @@ async def initialized_database(): @pytest.fixture(autouse=True) async def clean_database(initialized_database): async with database.engine.begin() as connection: + # Optional table (registered when raw_weight_push is imported). + try: + await connection.exec_driver_sql("DELETE FROM raw_weight_push_ledger") + except Exception: + pass # table created on first Database.init after raw_weight_push import await connection.execute(delete(ReviewOperatorApproval)) await connection.execute(delete(ReviewNonce)) await connection.execute(delete(ReviewEvidenceObject)) diff --git a/packages/challenges/agent-challenge/tests/test_ac_raw_weight_push.py b/packages/challenges/agent-challenge/tests/test_ac_raw_weight_push.py new file mode 100644 index 000000000..d8bdcb716 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_ac_raw_weight_push.py @@ -0,0 +1,279 @@ +"""Agent-challenge raw-weight push client and durable ack cursor tests. + +Mirrors Prism's push contract (VAL-SDK-017 / VAL-WEIGHT-028..030) so master +accepts agent-challenge payloads identically. WTA maps must surface as a single +winner hotkey on the wire. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx +import pytest +from base.challenge_sdk.roles import Role, activate_role +from base.challenge_sdk.schemas import RawWeightPushRequest + +from agent_challenge.evaluation.raw_weight_push import RawWeightPushClient +from agent_challenge.sdk.db import Database + +WINNER = "5CwinnerHK1" +LOSER = "5CloserHK22" +TOKEN = "ac-shared-token-secret" +SLUG = "agent-challenge" + + +class FakeClock: + def __init__(self) -> None: + self._now = datetime.now(UTC).replace(microsecond=0) + + def now(self) -> datetime: + return self._now + + def advance(self, seconds: float) -> None: + self._now = self._now + timedelta(seconds=seconds) + + +class TransportQueue: + """httpx MockTransport with sequential scripted responses.""" + + def __init__(self, responses: list[httpx.Response]) -> None: + self.responses = list(responses) + self.requests: list[httpx.Request] = [] + + def handler(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if not self.responses: + return httpx.Response(500, json={"detail": "exhausted"}) + return self.responses.pop(0) + + +@pytest.fixture +async def database(tmp_path: Path) -> Database: + db = Database(f"sqlite+aiosqlite:///{tmp_path / 'ac-push.sqlite3'}") + await db.init() + return db + + +@pytest.mark.asyncio +async def test_push_payload_is_single_winner_for_wta(database: Database) -> None: + """WTA map with one hotkey produces a payload with exactly that hotkey.""" + + clock = FakeClock() + captured: dict[str, Any] = {} + + def handler(request: httpx.Request) -> httpx.Response: + parsed = RawWeightPushRequest.model_validate_json(request.content) + captured["weights"] = dict(parsed.weights) + captured["slug"] = parsed.challenge_slug + return httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": SLUG, + "epoch": parsed.epoch, + "revision": parsed.revision, + "snapshot_id": "snap-wta", + "payload_digest": parsed.payload_digest, + "accepted": True, + "idempotent": False, + }, + ) + + http = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + base_url="http://master.test", + ) + # WTA output shape from get_weights: single winner only. + wta_weights = {WINNER: 0.91} + client = RawWeightPushClient( + database=database, + challenge_slug=SLUG, + master_base_url="http://master.test", + shared_token=TOKEN, + now_fn=clock.now, + http_client=http, + epoch_fn=lambda: 42, + weights_fn=lambda: wta_weights, + ) + await client.init() + with activate_role(Role.CHALLENGE): + result = await client.push_once() + assert result.cursor_advanced is True + assert result.status == "acknowledged" + assert captured["slug"] == SLUG + assert set(captured["weights"]) == {WINNER} + assert captured["weights"][WINNER] == pytest.approx(0.91) + assert LOSER not in captured["weights"] + await http.aclose() + + +@pytest.mark.asyncio +async def test_empty_weights_does_not_advance_cursor(database: Database) -> None: + clock = FakeClock() + transport = TransportQueue([]) + http = httpx.AsyncClient( + transport=httpx.MockTransport(transport.handler), + base_url="http://master.test", + ) + client = RawWeightPushClient( + database=database, + challenge_slug=SLUG, + master_base_url="http://master.test", + shared_token=TOKEN, + now_fn=clock.now, + http_client=http, + epoch_fn=lambda: 1, + weights_fn=lambda: {}, + ) + await client.init() + with activate_role(Role.CHALLENGE): + result = await client.push_once() + assert result.cursor_advanced is False + assert result.status == "skipped_empty" + assert await client.store.get_cursor() is None + assert transport.requests == [] + await http.aclose() + + +@pytest.mark.asyncio +async def test_successful_ack_advances_cursor(database: Database) -> None: + clock = FakeClock() + + def handler(request: httpx.Request) -> httpx.Response: + parsed = RawWeightPushRequest.model_validate_json(request.content) + return httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": SLUG, + "epoch": parsed.epoch, + "revision": parsed.revision, + "snapshot_id": "snap-ok", + "payload_digest": parsed.payload_digest, + "accepted": True, + "idempotent": False, + }, + ) + + http = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + base_url="http://master.test", + ) + client = RawWeightPushClient( + database=database, + challenge_slug=SLUG, + master_base_url="http://master.test", + shared_token=TOKEN, + now_fn=clock.now, + http_client=http, + epoch_fn=lambda: 11, + ) + await client.init() + with activate_role(Role.CHALLENGE): + ok = await client.push_once(weights={WINNER: 0.5}, epoch=11) + assert ok.cursor_advanced is True + assert ok.status == "acknowledged" + cursor = await client.store.get_cursor() + assert cursor is not None + assert cursor.payload_digest == ok.payload_digest + assert cursor.snapshot_id == "snap-ok" + assert cursor.epoch == 11 + assert await client.store.get_pending() is None + await http.aclose() + + +@pytest.mark.asyncio +async def test_mismatched_ack_digest_does_not_advance_cursor(database: Database) -> None: + clock = FakeClock() + transport = TransportQueue( + [ + httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": SLUG, + "epoch": 10, + "revision": 1, + "snapshot_id": "snap-wrong", + "payload_digest": "0" * 64, + "accepted": True, + "idempotent": False, + }, + ), + ] + ) + http = httpx.AsyncClient( + transport=httpx.MockTransport(transport.handler), + base_url="http://master.test", + ) + client = RawWeightPushClient( + database=database, + challenge_slug=SLUG, + master_base_url="http://master.test", + shared_token=TOKEN, + now_fn=clock.now, + http_client=http, + epoch_fn=lambda: 10, + ) + await client.init() + with activate_role(Role.CHALLENGE): + result = await client.push_once(weights={WINNER: 1.0}, epoch=10) + assert result.cursor_advanced is False + assert result.status == "ack_mismatch" + assert await client.store.get_cursor() is None + pending = await client.store.get_pending() + assert pending is not None + await http.aclose() + + +@pytest.mark.asyncio +async def test_push_does_not_log_token( + database: Database, + caplog: pytest.LogCaptureFixture, +) -> None: + clock = FakeClock() + + def handler(request: httpx.Request) -> httpx.Response: + parsed = RawWeightPushRequest.model_validate_json(request.content) + return httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": SLUG, + "epoch": parsed.epoch, + "revision": parsed.revision, + "snapshot_id": "snap-log", + "payload_digest": parsed.payload_digest, + "accepted": True, + "idempotent": False, + }, + ) + + http = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + base_url="http://master.test", + ) + client = RawWeightPushClient( + database=database, + challenge_slug=SLUG, + master_base_url="http://master.test", + shared_token=TOKEN, + now_fn=clock.now, + http_client=http, + epoch_fn=lambda: 3, + ) + await client.init() + with ( + caplog.at_level(logging.DEBUG, logger="agent_challenge.evaluation.raw_weight_push"), + activate_role(Role.CHALLENGE), + ): + await client.push_once(weights={WINNER: 1.0}, epoch=3) + joined = "\n".join(record.getMessage() for record in caplog.records) + assert TOKEN not in joined + for record in caplog.records: + assert TOKEN not in str(record.__dict__) + await http.aclose() diff --git a/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py b/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py new file mode 100644 index 000000000..a65a51da6 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py @@ -0,0 +1,518 @@ +"""NO_PHALA full offline pipeline: analysis → benchmark → score → weight push. + +Covers the master host path when attestation flags are off (required by +NO_PHALA contradiction). Does not touch attested gates. +""" + +from __future__ import annotations + +import base64 +import io +import json +import logging +import zipfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import pytest +from base.challenge_sdk.roles import Role, activate_role +from base.challenge_sdk.schemas import RawWeightPushRequest +from sqlalchemy import func, select + +from agent_challenge import routes +from agent_challenge.analyzer.lifecycle import run_next_analysis +from agent_challenge.analyzer.llm_reviewer import ( + GATEWAY_PLACEHOLDER_MODEL, + LlmReviewOutcome, + SubmitVerdictArgs, + build_llm_verdict_row, +) +from agent_challenge.app import app +from agent_challenge.evaluation.no_phala import ( + ATTESTATION_STATUS_UNATTESTED, + EXECUTION_MODE_NO_PHALA_HOST, + NO_PHALA_ENV, +) +from agent_challenge.evaluation.raw_weight_push import RawWeightPushClient +from agent_challenge.evaluation.runner import create_evaluation_job, run_evaluation_job +from agent_challenge.evaluation.weights import get_weights +from agent_challenge.models import ( + AgentSubmission, + AnalysisRun, + EvaluationJob, + LlmVerdict, + PythonAstFeature, + SubmissionStatusEvent, +) +from agent_challenge.sdk.db import Database +from agent_challenge.sdk.executors import DockerRunResult +from agent_challenge.security import SignedRequestAuth +from agent_challenge.swe_forge import SweForgeTask + +NOW = datetime(2026, 7, 28, 12, 0, tzinfo=UTC) +WINNER_HOTKEY = "5CwinnerHK1" +ENTRYPOINT_SOURCE = "class Agent:\n pass\n" + + +class StaticReviewProvider: + provider_name = "mock" + model_name = GATEWAY_PLACEHOLDER_MODEL + + def complete(self, **kwargs: Any) -> None: + raise AssertionError("network LLM must not be called") + + +class StaticReviewer: + def __init__(self, verdict: str = "allow") -> None: + self.verdict = verdict + self.calls = 0 + + def review(self, *, analysis_run_id, manifest, read_session, similarity_evidence): + self.calls += 1 + verdict = SubmitVerdictArgs( + verdict=self.verdict, + confidence=0.91, + rationale=f"mock {self.verdict}", + evidence_paths=["agent.py"], + similarity_assessment="[]", + policy_flags=[f"mock_{self.verdict}"], + ) + row = build_llm_verdict_row( + analysis_run_id=analysis_run_id, + provider=StaticReviewProvider(), + verdict=verdict, + transcript={ + "attempts": [], + "file_reads": [], + "provider_responses": [], + "tool_calls": [], + }, + manifest=manifest, + similarity_evidence=list(similarity_evidence), + ) + return LlmReviewOutcome(verdict=verdict, llm_verdict_row=row, transcript={}) + + +class FakeExecutor: + def run(self, spec, timeout_seconds: int) -> DockerRunResult: + return DockerRunResult( + container_name="fake", + stdout=f"ran {spec.labels.get('base.task', 'task')}", + stderr="", + returncode=0, + ) + + +class ValidReport: + rules_version = "rules-test" + overall_verdict = "valid" + reason_codes = ["rules_passed"] + + def to_dict(self) -> dict[str, object]: + return { + "rules_version": self.rules_version, + "overall_verdict": self.overall_verdict, + "reason_codes": self.reason_codes, + } + + +@pytest.fixture +def signed_submission_override(): + async def authenticate() -> SignedRequestAuth: + return SignedRequestAuth( + hotkey=WINNER_HOTKEY, + signature="test-signature", + nonce="test-nonce", + timestamp="2026-05-22T12:00:00+00:00", + body_sha256="test-body-sha256", + canonical_request="signed-test-request", + ) + + app.dependency_overrides[routes.signed_submission_auth] = authenticate + yield + app.dependency_overrides.pop(routes.signed_submission_auth, None) + + +@pytest.fixture(autouse=True) +def _no_phala_env_isolation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(NO_PHALA_ENV, raising=False) + monkeypatch.delenv("CHALLENGE_NO_PHALA", raising=False) + monkeypatch.setenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", "false") + monkeypatch.setenv("CHALLENGE_ATTESTED_REVIEW_ENABLED", "false") + + +def _enable_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(NO_PHALA_ENV, "true") + monkeypatch.setenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", "false") + monkeypatch.setenv("CHALLENGE_ATTESTED_REVIEW_ENABLED", "false") + + +def _agent_source(contents: str | bytes) -> str | bytes: + if isinstance(contents, bytes): + return contents + if "class Agent" in contents: + return contents + return f"{ENTRYPOINT_SOURCE}\n{contents}" + + +def build_zip(files: dict[str, str | bytes]) -> bytes: + buffer = io.BytesIO() + archive_files = {"agent.py": ENTRYPOINT_SOURCE, **files} + with zipfile.ZipFile(buffer, "w") as archive: + for filename, contents in archive_files.items(): + if filename == "agent.py": + contents = _agent_source(contents) + payload = contents.encode("utf-8") if isinstance(contents, str) else contents + archive.writestr(filename, payload) + return buffer.getvalue() + + +async def submit_agent(client, files: dict[str, str | bytes]): + archive_bytes = build_zip(files) + return await client.post( + "/submissions", + json={ + "name": "no-phala-agent", + "artifact_zip_base64": base64.b64encode(archive_bytes).decode("ascii"), + }, + ) + + +def configure_master(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr("agent_challenge.api.routes.settings.validator_role", "master") + monkeypatch.setattr("agent_challenge.analyzer.lifecycle.settings.validator_role", "master") + monkeypatch.setattr("agent_challenge.evaluation.runner.settings.validator_role", "master") + monkeypatch.setattr( + "agent_challenge.api.routes.settings.artifact_root", + str(tmp_path / "agents"), + ) + monkeypatch.setattr("agent_challenge.api.routes.settings.attested_review_enabled", False) + monkeypatch.setattr("agent_challenge.api.routes.settings.phala_attestation_enabled", False) + monkeypatch.setattr("agent_challenge.evaluation.runner.load_benchmark_tasks", lambda: []) + + +def _wire_offline_eval(monkeypatch: pytest.MonkeyPatch, *, task_count: int = 2) -> None: + monkeypatch.setattr( + "agent_challenge.evaluation.benchmarks.load_swe_forge_tasks", + lambda: [ + SweForgeTask( + task_id=f"task-{i}", + docker_image=f"baseintelligence/swe-forge:task-{i}", + ) + for i in range(task_count) + ], + ) + monkeypatch.setattr( + "agent_challenge.evaluation.benchmarks.settings.benchmark_backend", + "swe_forge", + ) + monkeypatch.setattr( + "agent_challenge.evaluation.runner.settings.evaluation_task_count", task_count + ) + monkeypatch.setattr("agent_challenge.evaluation.runner.settings.evaluation_concurrency", 1) + monkeypatch.setattr("agent_challenge.evaluation.runner.settings.validator_role", "master") + monkeypatch.setattr( + "agent_challenge.evaluation.runner.run_rules_analyzer", + lambda _workspace, *, reviewer=None: ValidReport(), + ) + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.evaluation_task_count", + task_count, + ) + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.phala_attestation_enabled", + False, + ) + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.weights_winner_take_all", + True, + ) + + +@pytest.mark.asyncio +async def test_no_phala_analysis_allow_enqueues_tb_job( + client, + database_session, + monkeypatch: pytest.MonkeyPatch, + signed_submission_override, + tmp_path: Path, +) -> None: + """Given NO_PHALA + empty env confirmed, When analysis allows, Then tb_queued + job.""" + + _enable_no_phala(monkeypatch) + configure_master(monkeypatch, tmp_path) + response = await submit_agent( + client, {"agent.py": "def solve(value):\n return value + 1\n"} + ) + assert response.status_code in {200, 201}, response.text + + async with database_session() as session: + submission = await session.scalar(select(AgentSubmission)) + assert submission is not None + submission.env_confirmed_empty = True + summary = await run_next_analysis( + session, + lease_owner="no-phala-worker", + reviewer=StaticReviewer("allow"), + ) + await session.commit() + await session.refresh(submission) + + assert summary is not None + assert summary.verdict == "allow" + assert summary.evaluation_job_id is not None + assert submission.raw_status == "tb_queued" + job_count = await session.scalar(select(func.count(EvaluationJob.id))) + assert job_count == 1 + ast_count = await session.scalar(select(func.count(PythonAstFeature.id))) + assert ast_count and ast_count > 0 + llm_count = await session.scalar(select(func.count(LlmVerdict.id))) + assert llm_count == 1 + analysis_count = await session.scalar(select(func.count(AnalysisRun.id))) + assert analysis_count == 1 + + +@pytest.mark.asyncio +async def test_no_phala_job_completion_marks_unattested_metadata( + database_session, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Given NO_PHALA, When TB eval job completes, Then status event carries unattested tags.""" + + _enable_no_phala(monkeypatch) + _wire_offline_eval(monkeypatch, task_count=2) + agent_dir = tmp_path / "agent" + agent_dir.mkdir() + + async with database_session() as session: + submission = AgentSubmission( + miner_hotkey=WINNER_HOTKEY, + name="np-complete", + agent_hash="np-complete-hash", + artifact_uri=str(agent_dir), + status="waiting_miner_env", + raw_status="waiting_miner_env", + effective_status="Waiting environments", + env_confirmed_empty=True, + submitted_at=NOW, + created_at=NOW, + ) + session.add(submission) + await session.flush() + job = await create_evaluation_job(session, submission, confirmed_miner_env=True) + assert submission.raw_status == "tb_queued" + summary = await run_evaluation_job(session, job.job_id, executor=FakeExecutor()) + await session.commit() + await session.refresh(submission) + await session.refresh(job) + + assert summary.status == "completed", job.error + assert summary.score == 1.0 + assert job.status == "completed" + assert submission.raw_status == "tb_completed" + + events = ( + ( + await session.execute( + select(SubmissionStatusEvent) + .where(SubmissionStatusEvent.submission_id == submission.id) + .order_by(SubmissionStatusEvent.id) + ) + ) + .scalars() + .all() + ) + completed = [e for e in events if e.to_status == "tb_completed"] + assert completed, "expected tb_completed status event" + meta = json.loads(completed[-1].metadata_json or "{}") + assert meta.get("attested") is False + assert meta.get("attestation_status") == ATTESTATION_STATUS_UNATTESTED + assert meta.get("execution_mode") == EXECUTION_MODE_NO_PHALA_HOST + assert meta.get("score") == 1.0 + + +@pytest.mark.asyncio +async def test_no_phala_completed_job_enters_weights( + database_session, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Given full-task completed job under flags-off, When get_weights, Then hotkey present.""" + + _enable_no_phala(monkeypatch) + required = 2 + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.evaluation_task_count", + required, + ) + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.phala_attestation_enabled", + False, + ) + monkeypatch.setattr( + "agent_challenge.evaluation.weights.settings.weights_winner_take_all", + True, + ) + + async with database_session() as session: + submission = AgentSubmission( + miner_hotkey=WINNER_HOTKEY, + name="np-weights", + agent_hash="np-weights-hash", + artifact_uri="/tmp/np-weights.zip", + status="tb_completed", + raw_status="tb_completed", + effective_status="valid", + submitted_at=NOW, + created_at=NOW, + ) + session.add(submission) + await session.flush() + job = EvaluationJob( + job_id="job-np-weights", + submission_id=submission.id, + status="completed", + selected_tasks_json="[]", + score=0.85, + passed_tasks=required, + total_tasks=required, + verdict="valid", + ) + session.add(job) + await session.flush() + submission.latest_evaluation_job_id = job.id + await session.commit() + + weights = await get_weights() + assert weights == {WINNER_HOTKEY: 0.85} + + +@pytest.mark.asyncio +async def test_no_phala_weight_push_fires_and_logs_unattested( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Given NO_PHALA + non-empty weights, When push_once, Then HTTP posts and CRITICAL log.""" + + _enable_no_phala(monkeypatch) + db = Database(f"sqlite+aiosqlite:///{tmp_path / 'push-np.sqlite3'}") + await db.init() + + captured: dict[str, Any] = {} + + def handler(request: httpx.Request) -> httpx.Response: + parsed = RawWeightPushRequest.model_validate_json(request.content) + captured["weights"] = dict(parsed.weights) + return httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": "agent-challenge", + "epoch": parsed.epoch, + "revision": parsed.revision, + "snapshot_id": "snap-np-1", + "payload_digest": parsed.payload_digest, + }, + ) + + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = RawWeightPushClient( + database=db, + challenge_slug="agent-challenge", + master_base_url="http://master.test", + shared_token="test-token-secret", + weights_fn=lambda: {WINNER_HOTKEY: 0.85}, + epoch_fn=lambda: 42, + http_client=http, + ) + + with caplog.at_level(logging.CRITICAL, logger="agent_challenge.evaluation.raw_weight_push"): + with activate_role(Role.CHALLENGE): + result = await client.push_once() + await http.aclose() + + assert result.status == "acknowledged" + assert result.cursor_advanced is True + assert captured["weights"] == {WINNER_HOTKEY: 0.85} + assert any( + "NO_PHALA raw weight push" in rec.message and "NOT TEE-attested" in rec.message + for rec in caplog.records + ) + + +@pytest.mark.asyncio +async def test_mode_off_weight_push_does_not_log_no_phala_banner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Regression: NO_PHALA off → push still works, no unattested CRITICAL.""" + + monkeypatch.delenv(NO_PHALA_ENV, raising=False) + monkeypatch.delenv("CHALLENGE_NO_PHALA", raising=False) + db = Database(f"sqlite+aiosqlite:///{tmp_path / 'push-off.sqlite3'}") + await db.init() + + def handler(request: httpx.Request) -> httpx.Response: + parsed = RawWeightPushRequest.model_validate_json(request.content) + return httpx.Response( + 200, + json={ + "protocol_version": "1.0", + "challenge_slug": "agent-challenge", + "epoch": parsed.epoch, + "revision": parsed.revision, + "snapshot_id": "snap-off", + "payload_digest": parsed.payload_digest, + }, + ) + + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = RawWeightPushClient( + database=db, + challenge_slug="agent-challenge", + master_base_url="http://master.test", + shared_token="test-token-secret", + weights_fn=lambda: {WINNER_HOTKEY: 1.0}, + epoch_fn=lambda: 7, + http_client=http, + ) + with caplog.at_level(logging.CRITICAL, logger="agent_challenge.evaluation.raw_weight_push"): + with activate_role(Role.CHALLENGE): + result = await client.push_once() + await http.aclose() + assert result.status == "acknowledged" + assert not any("NO_PHALA raw weight push" in rec.message for rec in caplog.records) + + +@pytest.mark.asyncio +async def test_weight_push_empty_skips_network(tmp_path: Path) -> None: + """Edge: empty weights → skipped_empty, no HTTP.""" + + db = Database(f"sqlite+aiosqlite:///{tmp_path / 'push-empty.sqlite3'}") + await db.init() + calls = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(500) + + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = RawWeightPushClient( + database=db, + challenge_slug="agent-challenge", + master_base_url="http://master.test", + shared_token="test-token-secret", + weights_fn=lambda: {}, + epoch_fn=lambda: 1, + http_client=http, + ) + with activate_role(Role.CHALLENGE): + result = await client.push_once() + await http.aclose() + assert result.status == "skipped_empty" + assert calls == 0 diff --git a/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py b/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py new file mode 100644 index 000000000..2e09a626d --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py @@ -0,0 +1,193 @@ +"""Lifespan wiring for agent-challenge raw-weight push (production path). + +Covers: enabled client + background task, disabled no-op, clean shutdown, and +ledger table registration via shared Database.create_all. +""" + +from __future__ import annotations + +import asyncio +import warnings +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy import text + +from agent_challenge.api import app as app_module +from agent_challenge.evaluation import raw_weight_push as push_module +from agent_challenge.sdk.config import ChallengeSettings +from agent_challenge.sdk.db import Database + +PUSH_TASK_NAME = "raw-weight-push-loop" + + +def _settings(tmp_path: Path, **overrides: object) -> ChallengeSettings: + defaults: dict[str, object] = { + "database_url": f"sqlite+aiosqlite:///{tmp_path / 'ac-push-life.sqlite3'}", + "shared_token": "ac-lifespan-token", + "shared_token_file": None, + "combined_worker": False, + "raw_weight_push_enabled": True, + "master_base_url": "http://master.test", + "raw_weight_push_interval_seconds": 30.0, + } + defaults.update(overrides) + return ChallengeSettings(**defaults) # type: ignore[arg-type] + + +def _push_tasks() -> list[asyncio.Task[Any]]: + return [task for task in asyncio.all_tasks() if task.get_name() == PUSH_TASK_NAME] + + +@pytest.mark.asyncio +async def test_enabled_starts_push_task_and_exposes_client( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Given push enabled with master+token, When lifespan starts, Then client + is on app.state and the named background task is running.""" + + started = asyncio.Event() + recorded: dict[str, object] = {} + + async def fake_loop(client: object, *, interval_seconds: float, resilient: bool) -> None: + recorded["client"] = client + recorded["interval_seconds"] = interval_seconds + recorded["resilient"] = resilient + started.set() + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + recorded["cancelled"] = True + raise + + monkeypatch.setattr(push_module, "run_raw_weight_push_loop", fake_loop) + + settings = _settings(tmp_path, raw_weight_push_interval_seconds=12.5) + db = Database(settings.database_url) + app = app_module.create_app(challenge_settings=settings, db=db) + + async with app.router.lifespan_context(app): + await asyncio.wait_for(started.wait(), timeout=2.0) + client = getattr(app.state, "raw_weight_push_client", None) + assert client is not None + assert recorded["client"] is client + assert recorded["interval_seconds"] == 12.5 + assert recorded["resilient"] is True + tasks = _push_tasks() + assert len(tasks) == 1 + assert not tasks[0].done() + + assert recorded.get("cancelled") is True + + +@pytest.mark.asyncio +async def test_disabled_does_not_build_client_or_start_task( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Given raw_weight_push_enabled=False, When lifespan starts, Then no client + and no push background task.""" + + loop_spy = AsyncMock() + monkeypatch.setattr(push_module, "run_raw_weight_push_loop", loop_spy) + + settings = _settings(tmp_path, raw_weight_push_enabled=False) + db = Database(settings.database_url) + app = app_module.create_app(challenge_settings=settings, db=db) + + async with app.router.lifespan_context(app): + assert getattr(app.state, "raw_weight_push_client", None) is None + assert _push_tasks() == [] + + loop_spy.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_shutdown_cancels_push_task_without_pending_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Given a running push loop, When lifespan exits, Then the task is cancelled + and no 'Task was destroyed but it is pending' warning is emitted.""" + + started = asyncio.Event() + + async def fake_loop(client: object, *, interval_seconds: float, resilient: bool) -> None: + del client, interval_seconds, resilient + started.set() + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + raise + + monkeypatch.setattr(push_module, "run_raw_weight_push_loop", fake_loop) + + settings = _settings(tmp_path) + db = Database(settings.database_url) + app = app_module.create_app(challenge_settings=settings, db=db) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + async with app.router.lifespan_context(app): + await asyncio.wait_for(started.wait(), timeout=2.0) + task = _push_tasks()[0] + # Allow the event loop a tick to surface destroy warnings if any leaked. + await asyncio.sleep(0) + pending_msgs = [ + str(w.message) + for w in caught + if "destroyed but it is pending" in str(w.message).lower() + or "task was destroyed" in str(w.message).lower() + ] + assert pending_msgs == [] + assert task.cancelled() or task.done() + + +@pytest.mark.asyncio +async def test_ledger_table_exists_after_startup(tmp_path: Path) -> None: + """Given model registration, When lifespan runs database.init, Then + raw_weight_push_ledger exists on the shared Database.""" + + settings = _settings( + tmp_path, + # Disabled still creates tables via metadata; no push loop needed. + raw_weight_push_enabled=False, + ) + db = Database(settings.database_url) + app = app_module.create_app(challenge_settings=settings, db=db) + + async with app.router.lifespan_context(app): + async with db.engine.connect() as connection: + name = ( + await connection.execute( + text( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name = 'raw_weight_push_ledger'" + ) + ) + ).scalar_one_or_none() + assert name == "raw_weight_push_ledger" + + +def test_raw_weight_push_settings_defaults_match_prism() -> None: + """Settings names/defaults/validators mirror Prism's raw_weight_push_* knobs.""" + + settings = ChallengeSettings( + shared_token="x", + shared_token_file=None, + ) + assert settings.raw_weight_push_enabled is True + assert settings.raw_weight_push_interval_seconds == 30.0 + assert settings.raw_weight_push_freshness_seconds == 300 + assert settings.raw_weight_push_timeout_seconds == 10.0 + + +def test_raw_weight_push_interval_rejects_below_minimum() -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError): + ChallengeSettings( + shared_token="x", + shared_token_file=None, + raw_weight_push_interval_seconds=0.05, + ) From 2c19a9143bdabf2cfb6ef3a034e896164b91214c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:47:50 +0000 Subject: [PATCH 3/4] feat(agent-challenge): route NO_PHALA LLM review via OpenRouter Grok Add OpenRouterReviewProvider for analyzer LLM review when NO_PHALA is on, with key resolution (env then opencode auth.json), operator config keys, and fail-closed cost/error handling. Gateway path stays default when mode is off. Document embed.env keys and cover selection/parsing with unit tests. --- .../agent-challenge/docs/no-phala-mode.md | 30 +- .../src/agent_challenge/analyzer/lifecycle.py | 38 ++- .../analyzer/openrouter_review_provider.py | 296 ++++++++++++++++++ .../src/agent_challenge/sdk/config.py | 24 ++ .../src/agent_challenge/selfdeploy/cli.py | 24 ++ .../tests/test_eval_deploy_token_handoff.py | 62 ++++ .../tests/test_openrouter_review_provider.py | 254 +++++++++++++++ 7 files changed, 724 insertions(+), 4 deletions(-) create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/analyzer/openrouter_review_provider.py create mode 100644 packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py diff --git a/packages/challenges/agent-challenge/docs/no-phala-mode.md b/packages/challenges/agent-challenge/docs/no-phala-mode.md index abe9fa17e..31d438c13 100644 --- a/packages/challenges/agent-challenge/docs/no-phala-mode.md +++ b/packages/challenges/agent-challenge/docs/no-phala-mode.md @@ -69,8 +69,16 @@ Operator override file (loaded by master entrypoint): ```bash # /var/lib/base/challenges/agent-challenge/embed.env NO_PHALA=true +CHALLENGE_NO_PHALA=true CHALLENGE_PHALA_ATTESTATION_ENABLED=false CHALLENGE_ATTESTED_REVIEW_ENABLED=false +# Analyzer LLM review via OpenRouter (NO_PHALA only; gateway path unchanged when off) +CHALLENGE_LLM_PROVIDER=openrouter +CHALLENGE_LLM_MODEL=x-ai/grok-4.5 +# Never commit this key. Prefer env injection / secret file over embed.env on shared hosts. +CHALLENGE_OPENROUTER_API_KEY=… +# Optional USD ceiling for analyzer OpenRouter spend (fail closed when exceeded) +# CHALLENGE_LLM_COST_LIMIT_USD=2.0 ``` Master runs: @@ -81,6 +89,24 @@ uvicorn agent_challenge.app:app --host 127.0.0.1 --port 18081 Restart the master (or AC child) after editing `embed.env`. +### Analyzer LLM provider (NO_PHALA) + +| Variable | Role | +|----------|------| +| `CHALLENGE_LLM_PROVIDER` | `openrouter` (default under NO_PHALA) or `gateway` | +| `CHALLENGE_LLM_MODEL` | OpenRouter model id (default `x-ai/grok-4.5`) | +| `CHALLENGE_OPENROUTER_API_KEY` | OpenRouter bearer key (never log/commit) | +| `OPENROUTER_API_KEY` | Fallback env if challenge-prefixed key unset | +| `CHALLENGE_LLM_COST_LIMIT_USD` | Optional spend ceiling; fail closed when exceeded | + +Key resolution order for OpenRouter: explicit settings field → +`CHALLENGE_OPENROUTER_API_KEY` → `OPENROUTER_API_KEY` → +`~/.local/share/opencode/auth.json` (`openrouter.key`) → small +`~/.factory/*.json` configs. + +When `NO_PHALA` is **off**, the analyzer always uses the BASE LLM gateway +(`CHALLENGE_LLM_GATEWAY_*`); OpenRouter settings are ignored. + ## Disable ```bash @@ -118,6 +144,8 @@ Via master proxy (if embedded): | `src/agent_challenge/selfdeploy/phala.py` | `PhalaCloudClient` refuse | | `src/agent_challenge/selfdeploy/eval.py` / `review.py` | deploy refuse | | `src/agent_challenge/evaluation/own_runner_backend.py` | Mark unattested on legacy emit when mode on | +| `src/agent_challenge/analyzer/openrouter_review_provider.py` | OpenRouter analyzer provider + key resolve (NO_PHALA only) | +| `src/agent_challenge/analyzer/lifecycle.py` | Provider selection under NO_PHALA | The **attested** emit branch in `own_runner_backend._emit_job_result` is not modified when NO_PHALA is off. @@ -143,7 +171,7 @@ Requirements on the master process: | Mode | `NO_PHALA=true` or `CHALLENGE_NO_PHALA=true` | | Attestation off | `CHALLENGE_PHALA_ATTESTATION_ENABLED=false`, `CHALLENGE_ATTESTED_REVIEW_ENABLED=false` | | Worker | `CHALLENGE_COMBINED_WORKER=true` **or** run `agent-challenge-worker` | -| LLM review | gateway base URL + token (else `llm_standby`) | +| LLM review | OpenRouter key + `CHALLENGE_LLM_PROVIDER=openrouter` (default under NO_PHALA); or gateway base URL + token if provider forced to `gateway` | | Benchmark | Docker + own_runner task cache | | Weight push | `CHALLENGE_MASTER_BASE_URL` + shared token (optional loop) | diff --git a/packages/challenges/agent-challenge/src/agent_challenge/analyzer/lifecycle.py b/packages/challenges/agent-challenge/src/agent_challenge/analyzer/lifecycle.py index c6a515b2f..9921bc972 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/analyzer/lifecycle.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/analyzer/lifecycle.py @@ -28,6 +28,11 @@ LlmReviewOutcome, LlmReviewProvider, ) +from agent_challenge.analyzer.openrouter_review_provider import ( + OpenRouterReviewProvider, + resolve_openrouter_api_key, + use_openrouter_review_provider, +) from agent_challenge.analyzer.similarity import ( ALGORITHM_VERSION, persist_same_challenge_similarity_matches, @@ -468,16 +473,33 @@ def _stale_analysis_summary( def build_configured_lifecycle_reviewer( provider: LlmReviewProvider | None = None, ) -> KimiLlmReviewer: + resolved = provider or _build_configured_review_provider() + expected_model = settings.llm_reviewer_expected_model + if getattr(resolved, "provider_name", None) == "openrouter": + # Pin telemetry expectation to the OpenRouter model the operator set. + expected_model = getattr(resolved, "model_name", None) or settings.llm_model return KimiLlmReviewer( - provider=provider or _build_configured_review_provider(), + provider=resolved, max_attempts=settings.llm_reviewer_max_attempts, timeout_seconds=settings.llm_reviewer_timeout_seconds, - expected_model=settings.llm_reviewer_expected_model, + expected_model=expected_model, prompt_cache_enabled=settings.llm_reviewer_prompt_cache_enabled, ) -def _build_configured_review_provider() -> GatewayReviewProvider: +def _build_configured_review_provider() -> LlmReviewProvider: + """Build gateway (default) or OpenRouter (NO_PHALA only) review provider.""" + + if use_openrouter_review_provider( + no_phala=bool(settings.no_phala), + llm_provider=settings.llm_provider, + ): + api_key = resolve_openrouter_api_key(explicit=settings.openrouter_api_key) + return OpenRouterReviewProvider( + api_key=api_key, + model_name=settings.llm_model or "x-ai/grok-4.5", + cost_limit_usd=settings.llm_cost_limit_usd, + ) base_url = settings.llm_gateway_base_url or "" return GatewayReviewProvider( gateway_token=settings.llm_gateway_token, @@ -677,6 +699,11 @@ def _llm_standby_reason( uses_configured_reviewer: bool, ) -> str: if uses_configured_reviewer and not _llm_provider_ready(): + if use_openrouter_review_provider( + no_phala=bool(settings.no_phala), + llm_provider=settings.llm_provider, + ): + return "missing_openrouter_api_key" return "missing_llm_gateway_token" if isinstance(exc, LlmProviderRateLimited): return "llm_provider_rate_limited" @@ -704,6 +731,11 @@ def _llm_standby_metadata( def _llm_provider_ready() -> bool: + if use_openrouter_review_provider( + no_phala=bool(settings.no_phala), + llm_provider=settings.llm_provider, + ): + return bool(resolve_openrouter_api_key(explicit=settings.openrouter_api_key)) return bool(settings.llm_gateway_base_url and settings.llm_gateway_token) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/analyzer/openrouter_review_provider.py b/packages/challenges/agent-challenge/src/agent_challenge/analyzer/openrouter_review_provider.py new file mode 100644 index 000000000..3f4381b0a --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/analyzer/openrouter_review_provider.py @@ -0,0 +1,296 @@ +"""OpenRouter-backed LLM review provider for NO_PHALA host-local mode. + +Selected only when NO_PHALA is active and the operator sets +``CHALLENGE_LLM_PROVIDER=openrouter`` (default under NO_PHALA). The attested / +gateway path is untouched: when NO_PHALA is off, lifecycle always builds +``GatewayReviewProvider``. + +Never logs or returns the API key. Fail closed on missing key, HTTP errors, +rate limits, timeouts, and optional cost-limit breach. +""" + +from __future__ import annotations + +import json +import logging +import os +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import httpx + +from agent_challenge.analyzer.llm_reviewer import ( + LlmProviderRateLimited, + LlmProviderResponse, + LlmProviderTimeout, + LlmProviderUnavailable, + _parse_provider_tool_calls, + _redacted_response, +) + +logger = logging.getLogger(__name__) + +OPENROUTER_CHAT_COMPLETIONS_URL = "https://openrouter.ai/api/v1/chat/completions" +DEFAULT_OPENROUTER_MODEL = "x-ai/grok-4.5" +OPENROUTER_PROVIDER_NAME = "openrouter" + +# Env vars checked before local credential files (never logged). +_ENV_KEY_NAMES = ( + "CHALLENGE_OPENROUTER_API_KEY", + "OPENROUTER_API_KEY", +) + + +def resolve_openrouter_api_key( + *, + explicit: str | None = None, + environ: Mapping[str, str] | None = None, + home: Path | None = None, +) -> str | None: + """Resolve OpenRouter API key without logging it. + + Order: + 1. ``explicit`` (settings field / caller) + 2. ``CHALLENGE_OPENROUTER_API_KEY`` then ``OPENROUTER_API_KEY`` + 3. ``~/.local/share/opencode/auth.json`` → ``openrouter.key`` + 4. ``~/.factory/`` small config files containing an openrouter key field + """ + + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + + env = environ if environ is not None else os.environ + for name in _ENV_KEY_NAMES: + value = env.get(name) + if isinstance(value, str) and value.strip(): + return value.strip() + + root = home if home is not None else Path.home() + opencode_key = _key_from_opencode_auth(root / ".local" / "share" / "opencode" / "auth.json") + if opencode_key: + return opencode_key + + return _key_from_factory_dir(root / ".factory") + + +def _key_from_opencode_auth(path: Path) -> str | None: + try: + raw = path.read_text(encoding="utf-8") + except OSError: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + return None + if not isinstance(data, Mapping): + return None + block = data.get("openrouter") + if not isinstance(block, Mapping): + return None + for field in ("key", "apiKey", "api_key", "token"): + value = block.get(field) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _key_from_factory_dir(factory_root: Path) -> str | None: + if not factory_root.is_dir(): + return None + # Prefer small known config filenames; never walk huge artifact trees. + candidates = ( + factory_root / "auth.json", + factory_root / "config.json", + factory_root / "settings.json", + factory_root / "keys.json", + ) + for path in candidates: + key = _key_from_json_file_shallow(path) + if key: + return key + return None + + +def _key_from_json_file_shallow(path: Path) -> str | None: + try: + if not path.is_file() or path.stat().st_size > 64_000: + return None + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return _find_openrouter_key_in_mapping(data, depth=0) + + +def _find_openrouter_key_in_mapping(value: object, *, depth: int) -> str | None: + if depth > 4 or not isinstance(value, Mapping): + return None + # Direct openrouter block + block = value.get("openrouter") + if isinstance(block, Mapping): + for field in ("key", "apiKey", "api_key", "token"): + candidate = block.get(field) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + for field in ( + "OPENROUTER_API_KEY", + "openrouter_api_key", + "openrouterKey", + "openrouter_key", + ): + candidate = value.get(field) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + for child in value.values(): + if isinstance(child, Mapping): + found = _find_openrouter_key_in_mapping(child, depth=depth + 1) + if found: + return found + return None + + +class OpenRouterReviewProvider: + """LLM review provider that calls OpenRouter chat completions directly. + + Implements the same ``LlmReviewProvider`` contract as ``GatewayReviewProvider`` + so ``KimiLlmReviewer`` verdict parsing and fail-closed semantics are unchanged. + """ + + provider_name = OPENROUTER_PROVIDER_NAME + + def __init__( + self, + *, + api_key: str | None, + model_name: str = DEFAULT_OPENROUTER_MODEL, + base_url: str = OPENROUTER_CHAT_COMPLETIONS_URL, + cost_limit_usd: float | None = None, + ) -> None: + self._api_key = api_key.strip() if isinstance(api_key, str) and api_key.strip() else None + self.model_name = model_name + self.base_url = base_url.rstrip("/") + self.cost_limit_usd = cost_limit_usd + self._spent_usd = 0.0 + + def __repr__(self) -> str: + return ( + f"OpenRouterReviewProvider(model_name={self.model_name!r}, " + f"api_key={'set' if self._api_key else 'missing'})" + ) + + def complete( + self, + *, + messages: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + tool_choice: str | Mapping[str, Any], + timeout_seconds: float, + ) -> LlmProviderResponse: + if not self._api_key: + raise LlmProviderUnavailable("OpenRouter API key is not configured") + if self.cost_limit_usd is not None and self._spent_usd >= self.cost_limit_usd: + raise LlmProviderUnavailable("LLM cost limit exceeded") + + timeout = httpx.Timeout(connect=10.0, read=timeout_seconds, write=30.0, pool=10.0) + headers = { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + # Optional OpenRouter attribution headers (no secrets). + "HTTP-Referer": "https://joinbase.ai", + "X-Title": "agent-challenge-no-phala-review", + } + body: dict[str, Any] = { + "model": self.model_name, + "messages": list(messages), + "tools": list(tools), + "tool_choice": tool_choice, + "temperature": 0, + } + # Grok/OpenRouter: omit parallel_tool_calls (some endpoints 404 with it). + try: + response = httpx.post( + self.base_url + if self.base_url.endswith("/chat/completions") + else f"{self.base_url}/chat/completions", + headers=headers, + json=body, + timeout=timeout, + ) + except httpx.TimeoutException as exc: + raise LlmProviderTimeout("OpenRouter request timed out") from exc + except httpx.HTTPError as exc: + raise LlmProviderUnavailable("OpenRouter request failed") from exc + + if response.status_code == 429: + raise LlmProviderRateLimited("OpenRouter rate limit exceeded") + if response.status_code in {401, 403}: + raise LlmProviderUnavailable("OpenRouter authentication failed") + if response.status_code >= 400: + raise LlmProviderUnavailable(f"OpenRouter returned HTTP {response.status_code}") + + try: + data = response.json() + except ValueError as exc: + raise LlmProviderUnavailable("OpenRouter returned non-JSON body") from exc + if not isinstance(data, Mapping): + raise LlmProviderUnavailable("OpenRouter response is not a JSON object") + + self._accumulate_cost(data) + if self.cost_limit_usd is not None and self._spent_usd > self.cost_limit_usd: + raise LlmProviderUnavailable("LLM cost limit exceeded") + + choices = data.get("choices") + message = choices[0].get("message", {}) if isinstance(choices, list) and choices else {} + return LlmProviderResponse( + content=str(message.get("content") or ""), + tool_calls=_parse_provider_tool_calls(message), + raw_response=_redacted_response(data), + usage=data.get("usage") if isinstance(data.get("usage"), Mapping) else None, + cost=data.get("cost") if isinstance(data.get("cost"), Mapping) else None, + ) + + def _accumulate_cost(self, data: Mapping[str, Any]) -> None: + cost = data.get("cost") + if isinstance(cost, Mapping): + for key in ("total_cost", "total", "cost"): + value = cost.get(key) + if isinstance(value, (int, float)): + self._spent_usd += float(value) + return + usage = data.get("usage") + if isinstance(usage, Mapping): + value = usage.get("cost") + if isinstance(value, (int, float)): + self._spent_usd += float(value) + + +def use_openrouter_review_provider( + *, + no_phala: bool, + llm_provider: str | None, +) -> bool: + """Return True when analyzer LLM review should use OpenRouter. + + OpenRouter is never selected when NO_PHALA is off (gateway path stays default). + Under NO_PHALA, default provider is openrouter unless operator forces gateway. + """ + + if not no_phala: + return False + normalized = (llm_provider or "openrouter").strip().lower() + if normalized in {"", "openrouter", "or"}: + return True + if normalized == "gateway": + return False + # Unknown value under NO_PHALA: fail toward openrouter only if explicitly openrouter-like. + return normalized.startswith("openrouter") + + +__all__ = [ + "DEFAULT_OPENROUTER_MODEL", + "OPENROUTER_CHAT_COMPLETIONS_URL", + "OPENROUTER_PROVIDER_NAME", + "OpenRouterReviewProvider", + "resolve_openrouter_api_key", + "use_openrouter_review_provider", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index a7be6ab16..13aed3914 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -32,6 +32,7 @@ "docker_broker_token", "llm_gateway_token", "agent_gateway_token", + "openrouter_api_key", "submission_env_encryption_key_file", "review_evidence_encryption_key", "review_evidence_encryption_key_file", @@ -355,6 +356,29 @@ class ChallengeSettings(BaseSettings): # only this dedicated token is ever placed into the agent container env. agent_gateway_token: str | None = Field(default=None, repr=False) agent_gateway_token_file: str | None = Field(default=None, repr=False) + # Analyzer LLM provider selection for NO_PHALA host-local mode only. + # When NO_PHALA is off the gateway provider is always used (this field is + # ignored). Under NO_PHALA, default is ``openrouter`` (Grok via OpenRouter); + # set ``gateway`` to force the BASE LLM gateway path instead. + # Env: CHALLENGE_LLM_PROVIDER + llm_provider: str | None = None + # Model id for the OpenRouter analyzer path (ignored by gateway provider). + # Env: CHALLENGE_LLM_MODEL + llm_model: str = "x-ai/grok-4.5" + # OpenRouter API key for NO_PHALA analyzer LLM review. Never logged. + # Env: CHALLENGE_OPENROUTER_API_KEY (also accepts plain OPENROUTER_API_KEY + # via resolve_openrouter_api_key fallback + opencode auth.json). + openrouter_api_key: str | None = Field(default=None, repr=False) + # Optional USD spend ceiling for OpenRouter analyzer calls; fail closed when + # exceeded. Env: CHALLENGE_LLM_COST_LIMIT / CHALLENGE_LLM_COST_LIMIT_USD + llm_cost_limit_usd: float | None = Field( + default=None, + validation_alias=AliasChoices( + "llm_cost_limit_usd", + "LLM_COST_LIMIT", + "llm_cost_limit", + ), + ) # Per-attempt read-leg budget. Held under the analysis lease: this value × # llm_reviewer_max_attempts must stay below DEFAULT_ANALYSIS_LEASE_SECONDS # (900s); 240 × 3 = 720s < 900s. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py index 2896a6fe0..1b8bde810 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py @@ -883,6 +883,22 @@ def _ordered_eval_command(args: argparse.Namespace) -> int: "(set CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM or " "CHALLENGE_PHALA_RA_TLS_SERVER_CA_FILE / KEY_RELEASE_SERVER_CA_FILE)" ) + # Optional measured OpenRouter for the in-CVM agent LLM (tools-only if unset). + # Never Base gateway. Host still must scrape BASE_BENCHMARK_RESULT= and + # POST eval result — guest never HTTP-posts the score envelope. + or_env = getattr(args, "openrouter_key_env", None) or "OPENROUTER_API_KEY" + or_key = (os.environ.get(or_env) or "").strip() + if or_key: + values["OPENROUTER_API_KEY"] = or_key + # Optional mid-run progress reporter bindings (observability only). + values.update( + eval_deploy.build_eval_progress_env( + base_url=str(args.base_url), + eval_run_id=plan.eval_run_id, + submission_id=str(args.submission_id), + eval_run_token=plan.eval_run_token, + ) + ) encrypted = eval_deploy.encrypt_eval_secrets(plan, values) if not args.dry_run else None if not args.dry_run: assert encrypted is not None @@ -1357,6 +1373,14 @@ def build_parser() -> argparse.ArgumentParser: help=argparse.SUPPRESS, ) eval_deploy_parser.add_argument("--llm-cost-limit-env", default="LLM_COST_LIMIT") + eval_deploy_parser.add_argument( + "--openrouter-key-env", + default="OPENROUTER_API_KEY", + help=( + "optional env var holding measured OpenRouter key for in-CVM agent LLM " + "(tools-only if unset; never Base gateway)" + ), + ) eval_deploy_parser.add_argument("--phala-api", default=None, help="Phala Cloud API base URL") eval_deploy_parser.add_argument( "--review-instance-type", diff --git a/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py b/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py index 06ec0ff69..1271ddfb3 100644 --- a/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py +++ b/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py @@ -157,6 +157,10 @@ def _deploy_args(**overrides: Any) -> SimpleNamespace: token_output=None, emit_run_token=False, output=None, + openrouter_key_env="OPENROUTER_API_KEY", + review_disk_size_gb=40, + eval_disk_size_gb=40, + expected_measurement=None, ) base.update(overrides) return SimpleNamespace(**base) @@ -541,3 +545,61 @@ def test_redact_capabilities_still_strips_token_key() -> None: dumped = json.dumps(redacted) assert RUN_TOKEN not in dumped assert redacted["secret_delivery"] == {"env_key": "EVAL_RUN_TOKEN"} + + +# --------------------------------------------------------------------------- # +# S9 — optional measured OPENROUTER + progress env on eval deploy +# --------------------------------------------------------------------------- # + + +def test_eval_deploy_injects_openrouter_and_progress_env_when_key_present( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Given OPENROUTER_API_KEY in env, When eval deploy succeeds, + Then encrypted_env secrets include OPENROUTER_API_KEY + progress bindings + (never Base gateway). Guest never posts; host needs OR for agent LLM. + """ + + captured: list[dict[str, str]] = [] + printed = _wire_successful_deploy(monkeypatch, captured_secrets=captured) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test-eval-only") + args = _deploy_args( + emit_run_token=True, + openrouter_key_env="OPENROUTER_API_KEY", + base_url="https://challenge.example/challenges/agent-challenge/", + ) + assert cli._ordered_eval_command(args) == 0, printed + assert captured, printed + secrets = captured[0] + assert secrets["EVAL_RUN_TOKEN"] == RUN_TOKEN + assert secrets["OPENROUTER_API_KEY"] == "sk-or-test-eval-only" + assert secrets["EVAL_PROGRESS_BASE_URL"] == ( + "https://challenge.example/challenges/agent-challenge" + ) + assert secrets["EVAL_RUN_ID"] == EVAL_RUN_ID + assert secrets["EVAL_SUBMISSION_ID"] == "1" + # Never Base gateway + assert "BASE_GATEWAY_TOKEN" not in secrets + assert "BASE_LLM_GATEWAY_URL" not in secrets + + +def test_eval_deploy_omits_openrouter_when_key_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Given no OpenRouter key, When eval deploy succeeds, + Then OPENROUTER_API_KEY is absent (tools-only path) but progress still binds. + """ + + captured: list[dict[str, str]] = [] + printed = _wire_successful_deploy(monkeypatch, captured_secrets=captured) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + args = _deploy_args( + emit_run_token=True, + openrouter_key_env="OPENROUTER_API_KEY", + base_url="https://challenge.example", + ) + assert cli._ordered_eval_command(args) == 0, printed + secrets = captured[0] + assert "OPENROUTER_API_KEY" not in secrets + assert secrets["EVAL_PROGRESS_BASE_URL"] == "https://challenge.example" + assert secrets["EVAL_RUN_TOKEN"] == RUN_TOKEN diff --git a/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py b/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py new file mode 100644 index 000000000..2a57f94aa --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py @@ -0,0 +1,254 @@ +"""OpenRouter analyzer provider — NO_PHALA only; gateway path unchanged.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import httpx +import pytest + +from agent_challenge.analyzer.llm_reviewer import ( + GATEWAY_PLACEHOLDER_MODEL, + GatewayReviewProvider, + LlmProviderRateLimited, + LlmProviderTimeout, + LlmProviderUnavailable, +) +from agent_challenge.analyzer.openrouter_review_provider import ( + DEFAULT_OPENROUTER_MODEL, + OPENROUTER_PROVIDER_NAME, + OpenRouterReviewProvider, + resolve_openrouter_api_key, + use_openrouter_review_provider, +) + + +def test_use_openrouter_only_when_no_phala() -> None: + assert use_openrouter_review_provider(no_phala=False, llm_provider="openrouter") is False + assert use_openrouter_review_provider(no_phala=False, llm_provider=None) is False + assert use_openrouter_review_provider(no_phala=True, llm_provider=None) is True + assert use_openrouter_review_provider(no_phala=True, llm_provider="openrouter") is True + assert use_openrouter_review_provider(no_phala=True, llm_provider="gateway") is False + + +def test_resolve_key_order_explicit_then_env_then_opencode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("CHALLENGE_OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + + assert resolve_openrouter_api_key(explicit=" explicit-key ", home=tmp_path) == "explicit-key" + + monkeypatch.setenv("CHALLENGE_OPENROUTER_API_KEY", "challenge-key") + monkeypatch.setenv("OPENROUTER_API_KEY", "plain-key") + env = dict(**__import__("os").environ) + assert resolve_openrouter_api_key(explicit=None, environ=env, home=tmp_path) == "challenge-key" + + monkeypatch.delenv("CHALLENGE_OPENROUTER_API_KEY", raising=False) + env = dict(**__import__("os").environ) + assert resolve_openrouter_api_key(explicit=None, environ=env, home=tmp_path) == "plain-key" + + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + auth = tmp_path / ".local" / "share" / "opencode" / "auth.json" + auth.parent.mkdir(parents=True) + payload = {"openrouter": {"type": "api", "key": "file-key"}} + auth.write_text(json.dumps(payload), encoding="utf-8") + assert resolve_openrouter_api_key(explicit=None, environ={}, home=tmp_path) == "file-key" + + +def test_openrouter_provider_parses_tool_calls(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, Any] = {} + + class Response: + status_code = 200 + + def json(self) -> dict[str, Any]: + return { + "id": "gen-1", + "model": DEFAULT_OPENROUTER_MODEL, + "choices": [ + { + "message": { + "content": "", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "submit_verdict", + "arguments": json.dumps( + { + "verdict": "allow", + "confidence": 0.9, + "rationale": "Clean agent.", + } + ), + }, + } + ], + } + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + + def fake_post(url, *, headers, json, timeout): # noqa: A002 + captured["url"] = url + captured["headers"] = headers + captured["json"] = json + return Response() + + monkeypatch.setattr( + "agent_challenge.analyzer.openrouter_review_provider.httpx.post", fake_post + ) + provider = OpenRouterReviewProvider(api_key="sk-test", model_name=DEFAULT_OPENROUTER_MODEL) + response = provider.complete( + messages=[{"role": "user", "content": "hi"}], + tools=[], + tool_choice="auto", + timeout_seconds=5, + ) + + assert captured["url"].endswith("/chat/completions") + assert captured["headers"]["Authorization"] == "Bearer sk-test" + assert "X-Gateway-Token" not in captured["headers"] + assert captured["json"]["model"] == DEFAULT_OPENROUTER_MODEL + assert response.tool_calls[0].name == "submit_verdict" + assert response.tool_calls[0].arguments["verdict"] == "allow" + assert provider.provider_name == OPENROUTER_PROVIDER_NAME + # Key never appears in repr + assert "sk-test" not in repr(provider) + + +def test_openrouter_provider_fail_closed_on_http_error(monkeypatch: pytest.MonkeyPatch) -> None: + class Response: + status_code = 500 + + def json(self) -> dict[str, Any]: + return {"error": "boom"} + + monkeypatch.setattr( + "agent_challenge.analyzer.openrouter_review_provider.httpx.post", + lambda *a, **k: Response(), + ) + provider = OpenRouterReviewProvider(api_key="sk-test") + with pytest.raises(LlmProviderUnavailable, match="HTTP 500"): + provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) + + +def test_openrouter_provider_rate_limit_and_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + class Response: + status_code = 429 + + def json(self) -> dict[str, Any]: + return {} + + monkeypatch.setattr( + "agent_challenge.analyzer.openrouter_review_provider.httpx.post", + lambda *a, **k: Response(), + ) + provider = OpenRouterReviewProvider(api_key="sk-test") + with pytest.raises(LlmProviderRateLimited): + provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) + + def boom(*a, **k): + raise httpx.ReadTimeout("slow") + + monkeypatch.setattr( + "agent_challenge.analyzer.openrouter_review_provider.httpx.post", boom + ) + with pytest.raises(LlmProviderTimeout): + provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) + + +def test_openrouter_provider_missing_key_fail_closed() -> None: + provider = OpenRouterReviewProvider(api_key=None) + with pytest.raises(LlmProviderUnavailable, match="not configured"): + provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) + + +def test_openrouter_cost_limit_fail_closed(monkeypatch: pytest.MonkeyPatch) -> None: + class Response: + status_code = 200 + + def json(self) -> dict[str, Any]: + return { + "choices": [{"message": {"content": "", "tool_calls": []}}], + "cost": {"total_cost": 1.5}, + } + + monkeypatch.setattr( + "agent_challenge.analyzer.openrouter_review_provider.httpx.post", + lambda *a, **k: Response(), + ) + provider = OpenRouterReviewProvider(api_key="sk-test", cost_limit_usd=1.0) + with pytest.raises(LlmProviderUnavailable, match="cost limit"): + provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) + + +def test_lifecycle_builds_gateway_when_no_phala_off(monkeypatch: pytest.MonkeyPatch) -> None: + from agent_challenge.analyzer import lifecycle + from agent_challenge.core import config as core_config + + mock_settings = MagicMock() + mock_settings.no_phala = False + mock_settings.llm_provider = "openrouter" + mock_settings.llm_gateway_base_url = "http://master:19080" + mock_settings.llm_gateway_token = "gw-token" + mock_settings.openrouter_api_key = "sk-should-not-use" + mock_settings.llm_model = DEFAULT_OPENROUTER_MODEL + mock_settings.llm_cost_limit_usd = None + monkeypatch.setattr(lifecycle, "settings", mock_settings) + monkeypatch.setattr(core_config, "settings", mock_settings) + + provider = lifecycle._build_configured_review_provider() + assert isinstance(provider, GatewayReviewProvider) + assert provider.provider_name == "gateway" + assert provider.model_name == GATEWAY_PLACEHOLDER_MODEL + + +def test_lifecycle_builds_openrouter_when_no_phala_on(monkeypatch: pytest.MonkeyPatch) -> None: + from agent_challenge.analyzer import lifecycle + + mock_settings = MagicMock() + mock_settings.no_phala = True + mock_settings.llm_provider = "openrouter" + mock_settings.llm_gateway_base_url = "http://master:19080" + mock_settings.llm_gateway_token = "gw-token" + mock_settings.openrouter_api_key = "sk-or-test" + mock_settings.llm_model = "x-ai/grok-4.5" + mock_settings.llm_cost_limit_usd = 2.0 + monkeypatch.setattr(lifecycle, "settings", mock_settings) + + provider = lifecycle._build_configured_review_provider() + assert isinstance(provider, OpenRouterReviewProvider) + assert provider.provider_name == "openrouter" + assert provider.model_name == "x-ai/grok-4.5" + + +def test_llm_provider_ready_openrouter(monkeypatch: pytest.MonkeyPatch) -> None: + from agent_challenge.analyzer import lifecycle + + mock_settings = MagicMock() + mock_settings.no_phala = True + mock_settings.llm_provider = "openrouter" + mock_settings.openrouter_api_key = None + mock_settings.llm_gateway_base_url = None + mock_settings.llm_gateway_token = None + monkeypatch.setattr(lifecycle, "settings", mock_settings) + monkeypatch.setattr( + lifecycle, + "resolve_openrouter_api_key", + lambda explicit=None: None, + ) + assert lifecycle._llm_provider_ready() is False + + monkeypatch.setattr( + lifecycle, + "resolve_openrouter_api_key", + lambda explicit=None: "sk-ok", + ) + assert lifecycle._llm_provider_ready() is True From 0b785f2b8f5001a95196df5195a0127ff2443bbb Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:44:19 +0000 Subject: [PATCH 4/4] fix(agent-challenge): load file token and host-local NO_PHALA bench Production mounts the challenge token via shared_token_file; raw-weight push now resolves it the same way auth does so the loop does not skip. NO_PHALA + cli docker backend runs own_runner in-process to avoid nested DooD path mismatch on master embed. Default epoch_seconds matches master sealer (360s). --- .../evaluation/raw_weight_push.py | 23 ++- .../src/agent_challenge/evaluation/runner.py | 175 +++++++++++++++--- .../src/agent_challenge/sdk/config.py | 2 +- .../tests/test_raw_weight_push_lifespan.py | 26 +++ 4 files changed, 194 insertions(+), 32 deletions(-) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py index ed142468f..75b9b139f 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py @@ -635,9 +635,30 @@ def maybe_build_push_client_from_settings( settings, "challenge_shared_token", None ) token = str(shared) if shared else None + if not token: + # Production mounts the challenge token via shared_token_file; auth + # already resolves that path — reuse the same loader so raw-weight push + # does not silently skip when only the file is configured. + try: + from agent_challenge.sdk.auth import load_internal_token + + loaded = load_internal_token(settings) + token = str(loaded) if loaded else None + except Exception: + token = None + if not token: + token_file = getattr(settings, "shared_token_file", None) + if token_file: + from pathlib import Path as _Path + + try: + raw = _Path(str(token_file)).expanduser().read_text(encoding="utf-8").strip() + except OSError: + raw = "" + token = raw or None if not token: return None - epoch_seconds = int(getattr(settings, "epoch_seconds", 3600) or 3600) + epoch_seconds = int(getattr(settings, "epoch_seconds", 360) or 360) interval_hint = float(getattr(settings, "raw_weight_push_interval_seconds", 30.0)) slug = str( getattr(settings, "slug", None) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index fb94c44ca..1360fb451 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -1396,38 +1396,53 @@ async def _run_terminal_bench_task_durable( # is held across the container execution await below. await session.commit() with _evaluation_workspace(submission, isolate=True) as agent_workspace: - spec = DockerRunSpec( - image=runner_image, - command=( - "bash", - "-lc", - _terminal_bench_script(job, task, plan=plan, backend=execution_backend), - ), - mounts=( - DockerMount( - source=agent_workspace, - target="/workspace/agent", - read_only=False, + from agent_challenge.evaluation.no_phala import is_no_phala_enabled as _np + + if _np() and settings.docker_backend in {"cli", "docker"}: + run = await asyncio.to_thread( + _run_no_phala_host_terminal_bench, + job=job, + task=task, + plan=plan, + agent_workspace=agent_workspace, + miner_env=miner_env, + gateway=gateway, + execution_backend=execution_backend, + timeout_seconds=settings.evaluation_timeout_seconds, + ) + else: + spec = DockerRunSpec( + image=runner_image, + command=( + "bash", + "-lc", + _terminal_bench_script(job, task, plan=plan, backend=execution_backend), ), - DockerMount( - source=plan.jobs_dir, - target=str(plan.jobs_dir), - read_only=False, + mounts=( + DockerMount( + source=agent_workspace, + target="/workspace/agent", + read_only=False, + ), + DockerMount( + source=plan.jobs_dir, + target=str(plan.jobs_dir), + read_only=False, + ), ), - ), - workdir="/workspace", - env={ - **_terminal_bench_env(miner_env, gateway), - **_terminal_bench_stream_env(plan.attempt_id), - }, - labels=_labels(job, submission, task), - limits=_terminal_bench_limits(), - ) - run = await asyncio.to_thread( - executor.run, - spec, - timeout_seconds=settings.evaluation_timeout_seconds, - ) + workdir="/workspace", + env={ + **_terminal_bench_env(miner_env, gateway), + **_terminal_bench_stream_env(plan.attempt_id), + }, + labels=_labels(job, submission, task), + limits=_terminal_bench_limits(), + ) + run = await asyncio.to_thread( + executor.run, + spec, + timeout_seconds=settings.evaluation_timeout_seconds, + ) except Exception as exc: # The attempt was committed ``running`` before the container await, so a # failure here (executor/broker error, ``database is locked``, unexpected @@ -1997,6 +2012,106 @@ def _own_runner_script( """.strip() + +def _run_no_phala_host_terminal_bench( + *, + job: EvaluationJob, + task: BenchmarkTask, + plan: TerminalBenchAttemptPlan, + agent_workspace: Path, + miner_env: Mapping[str, str] | None, + gateway: GatewayExecutionConfig | None, + execution_backend: str, + timeout_seconds: int, +) -> DockerRunResult: + """NO_PHALA host-local path: run own_runner in this process namespace. + + Avoids nested harbor-runner DooD under master embed (container bind paths are + not host paths for ``docker -v``). The validator already has docker.sock and + task images; own_runner spawns sibling task containers directly. + """ + import subprocess + + from agent_challenge.evaluation.no_phala import is_no_phala_enabled + + if not is_no_phala_enabled(): + raise RuntimeError("no_phala host runner invoked while NO_PHALA is off") + if execution_backend != "own_runner": + raise ValueError(f"unsupported backend for no_phala host path: {execution_backend}") + + task_id = str(task.metadata.get("task_id") or task.task_id) + bare = task_id.rsplit("/", 1)[-1] + cache_root = Path( + settings.own_runner_cache_root + or "/app/packages/challenges/agent-challenge/docker/canonical/live-task-cache" + ) + digest = Path( + settings.own_runner_digest_manifest or "/app/golden/dataset-digest.json" + ) + plan.jobs_dir.mkdir(parents=True, exist_ok=True) + plan.job_dir.mkdir(parents=True, exist_ok=True) + + cmd = [ + "python", + "-m", + "agent_challenge.evaluation.own_runner_backend", + "run", + "--job-dir", + str(plan.job_dir), + "--job-name", + plan.job_name, + "--jobs-dir", + str(plan.jobs_dir), + "--n-concurrent", + str(max(1, int(settings.harbor_n_concurrent or 1))), + "--agent-import-path", + settings.harbor_agent_import_path, + "--n-attempts", + "1", + "--task", + bare, + "--cache-root", + str(cache_root), + "--digest-manifest", + str(digest), + ] + env = { + **dict(os.environ), + **_terminal_bench_env(miner_env, gateway), + "PYTHONPATH": f"{agent_workspace}:{os.environ.get('PYTHONPATH', '')}".rstrip(":"), + "DOCKER_HOST": os.environ.get("DOCKER_HOST") or "unix:///var/run/docker.sock", + "BASE_AGENT_PATH": str(agent_workspace), + } + if env["PYTHONPATH"].endswith(":"): + env["PYTHONPATH"] = env["PYTHONPATH"][:-1] + + try: + proc = subprocess.run( + cmd, + cwd=str(agent_workspace), + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + env=env, + ) + return DockerRunResult( + container_name=f"no-phala-host-{plan.job_name}"[:120], + stdout=proc.stdout or "", + stderr=proc.stderr or "", + returncode=proc.returncode, + timed_out=False, + ) + except subprocess.TimeoutExpired as exc: + return DockerRunResult( + container_name=f"no-phala-host-{plan.job_name}"[:120], + stdout=(exc.stdout or "") if isinstance(exc.stdout, str) else "", + stderr=(exc.stderr or "") if isinstance(exc.stderr, str) else "timed_out", + returncode=124, + timed_out=True, + ) + + def _terminal_bench_script( job: EvaluationJob, task: BenchmarkTask, diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index 13aed3914..840570afd 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -98,7 +98,7 @@ class ChallengeSettings(BaseSettings): raw_weight_push_freshness_seconds: int = Field(default=300, ge=30) raw_weight_push_timeout_seconds: float = Field(default=10.0, gt=0.0) # Epoch bucket size for push revision identity (seconds). - epoch_seconds: int = Field(default=3600, ge=1) + epoch_seconds: int = Field(default=360, ge=1) # Root stdlib logging level applied at every process entrypoint (the API app diff --git a/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py b/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py index 2e09a626d..c807a99d3 100644 --- a/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py +++ b/packages/challenges/agent-challenge/tests/test_raw_weight_push_lifespan.py @@ -191,3 +191,29 @@ def test_raw_weight_push_interval_rejects_below_minimum() -> None: shared_token_file=None, raw_weight_push_interval_seconds=0.05, ) + + +def test_maybe_build_push_client_loads_token_from_shared_token_file( + tmp_path: Path, +) -> None: + """File-backed challenge token alone is enough to construct the push client.""" + + token_path = tmp_path / "challenge_token" + token_path.write_text("file-backed-ac-token-value", encoding="utf-8") + settings = ChallengeSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'ac-push-file-token.sqlite3'}", + shared_token=None, + shared_token_file=str(token_path), + raw_weight_push_enabled=True, + master_base_url="http://master.test", + epoch_seconds=360, + ) + db = Database(settings.database_url) + client = push_module.maybe_build_push_client_from_settings( + settings=settings, + database=db, + ) + assert client is not None + assert client.shared_token == "file-backed-ac-token-value" + assert client.master_base_url == "http://master.test" + assert client.challenge_slug == "agent-challenge"