fix(agent-challenge): stop forwarding operator OpenRouter key after cutoff

packages/challenges/agent-challenge/config.example.yaml

@@ -137,6 +137,10 @@ challenge:
   harbor_agent_import_path: agent:Agent
   # Empty by default. Operators must explicitly opt in before forwarding provider credentials.
   harbor_forward_env_vars: []
+  # ISO-8601 UTC. Submissions with created_at >= this do not receive operator
+  # harbor_forward_env_vars; miners must supply their own keys. null = always forward.
+  # Env override: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT
+  operator_env_forward_cutoff_at: null
   harbor_n_concurrent: 1
   harbor_output_dir: /tmp/harbor-runs
   swe_forge_tree_url: https://huggingface.co/api/datasets/CortexLM/swe-forge/tree/main?recursive=true

packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py

@@ -1211,6 +1211,7 @@ def _run_terminal_bench_task(
                 gateway,
                 replay_audit=replay_audit,
                 replay_eval_plan=replay_eval_plan,
+                submission_created_at=submission.created_at,
             ),
             labels=_labels(job, submission, task),
             limits=_terminal_bench_limits(),
@@ -1428,6 +1429,7 @@ async def _run_terminal_bench_task_durable(
                         gateway=gateway,
                         execution_backend=execution_backend,
                         timeout_seconds=settings.evaluation_timeout_seconds,
+                        submission_created_at=submission.created_at,
                     )
                 else:
                     spec = DockerRunSpec(
@@ -1451,7 +1453,11 @@ async def _run_terminal_bench_task_durable(
                         ),
                         workdir="/workspace",
                         env={
-                            **_terminal_bench_env(miner_env, gateway),
+                            **_terminal_bench_env(
+                                miner_env,
+                                gateway,
+                                submission_created_at=submission.created_at,
+                            ),
                             **_terminal_bench_stream_env(plan.attempt_id),
                         },
                         labels=_labels(job, submission, task),
@@ -1625,6 +1631,7 @@ def _terminal_bench_env(
     *,
     replay_audit: bool = False,
     replay_eval_plan: Mapping[str, Any] | None = None,
+    submission_created_at: datetime | None = None,
 ) -> dict[str, str]:
     env = {
         "BASE_AGENT_PATH": "/workspace/agent",
@@ -1639,11 +1646,17 @@ def _terminal_bench_env(
     # agent still fails loudly rather than running an unmeasured default.
     if settings.llm_model:
         env["LLM_MODEL"] = settings.llm_model
-    for name in settings.harbor_forward_env_vars:
-        value = os.environ.get(name)
-        if value and name not in TERMINAL_BENCH_CONTROL_ENV_KEYS:
-            env[name] = value
-    operator_env_names = set(settings.harbor_forward_env_vars)
+    forward_operator = _should_forward_operator_env(submission_created_at)
+    if forward_operator:
+        for name in settings.harbor_forward_env_vars:
+            value = os.environ.get(name)
+            if value and name not in TERMINAL_BENCH_CONTROL_ENV_KEYS:
+                env[name] = value
+        operator_env_names = set(settings.harbor_forward_env_vars)
+    else:
+        # Post-cutoff: do not inject operator keys and do not shadow miner keys
+        # that share those names (e.g. miner OPENROUTER_API_KEY).
+        operator_env_names = set()
     # VAL-ACLOCK: only keys/tokens; drop URL/proxy/host/gateway/stream injection.
     # sanitize_miner_env_for_job is fail-closed even if an older stored env row
     # still contains a rejected key from before the lock landed.
@@ -1663,6 +1676,19 @@ def _terminal_bench_env(
     # (BASE_LLM_GATEWAY_URL / BASE_GATEWAY_TOKEN), even if a residual
     # GatewayExecutionConfig is passed by legacy call sites.
     _ = gateway
+    if not forward_operator:
+        missing = [
+            name
+            for name in settings.harbor_forward_env_vars
+            if name not in TERMINAL_BENCH_CONTROL_ENV_KEYS and not str(env.get(name) or "").strip()
+        ]
+        if missing:
+            names = ", ".join(missing)
+            raise ValueError(
+                "miner submission env missing required var(s) after operator "
+                f"forward cutoff: {names}. Set these on the submission env; "
+                "the operator no longer injects them for new submissions."
+            )
     if replay_audit:
         env["BASE_REPLAY_AUDIT"] = "1"
     if replay_eval_plan is not None:
@@ -1674,6 +1700,28 @@ def _terminal_bench_env(
     return env
 
 
+def _should_forward_operator_env(submission_created_at: datetime | None) -> bool:
+    """Return True when operator harbor_forward_env_vars should still be injected.
+
+    Grandfathers submissions created *before* ``operator_env_forward_cutoff_at``.
+    The boundary instant itself is post-cutoff (operator key not forwarded).
+    Comparison uses submission.created_at so a later re-evaluation of an old
+    submission stays grandfathered. When the cutoff is unset, keep legacy
+    always-forward behavior. When cutoff is set but created_at is missing,
+    fail closed (do not forward).
+    """
+    cutoff = settings.operator_env_forward_cutoff_at
+    if cutoff is None:
+        return True
+    if submission_created_at is None:
+        return False
+    created = submission_created_at
+    if created.tzinfo is None:
+        created = created.replace(tzinfo=UTC)
+    cutoff_aware = cutoff if cutoff.tzinfo is not None else cutoff.replace(tzinfo=UTC)
+    return created < cutoff_aware
+
+
 def _is_provider_key_env(name: str) -> bool:
     """Return ``True`` when ``name`` is a raw provider API key/token env var."""
 
@@ -1759,11 +1807,17 @@ async def _terminal_bench_env_for_submission(
     session: AsyncSession,
     submission: AgentSubmission,
 ) -> dict[str, str]:
-    return _terminal_bench_env(await _locked_miner_env_for_submission(session, submission))
+    return _terminal_bench_env(
+        await _locked_miner_env_for_submission(session, submission),
+        submission_created_at=submission.created_at,
+    )
 
 
 def _terminal_bench_env_for_loaded_submission(submission: AgentSubmission) -> dict[str, str]:
-    return _terminal_bench_env(_locked_miner_env_from_loaded_submission(submission))
+    return _terminal_bench_env(
+        _locked_miner_env_from_loaded_submission(submission),
+        submission_created_at=submission.created_at,
+    )
 
 
 async def _locked_miner_env_for_submission(
@@ -2046,6 +2100,7 @@ def _run_no_phala_host_terminal_bench(
     gateway: GatewayExecutionConfig | None,
     execution_backend: str,
     timeout_seconds: int,
+    submission_created_at: datetime | None = None,
 ) -> DockerRunResult:
     """NO_PHALA host-local path: run own_runner in this process namespace.
 
@@ -2098,7 +2153,7 @@ def _run_no_phala_host_terminal_bench(
     ]
     env = {
         **dict(os.environ),
-        **_terminal_bench_env(miner_env, gateway),
+        **_terminal_bench_env(miner_env, gateway, submission_created_at=submission_created_at),
         "PYTHONPATH": f"{agent_workspace}:{os.environ.get('PYTHONPATH', '')}".rstrip(":"),
         "DOCKER_HOST": os.environ.get("DOCKER_HOST") or "unix:///var/run/docker.sock",
         "BASE_AGENT_PATH": str(agent_workspace),

packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import math
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Literal
 
@@ -144,6 +145,11 @@ class ChallengeSettings(BaseSettings):
     harbor_agent_import_path: str = "agent:Agent"
     harbor_model: str | None = None
     harbor_forward_env_vars: tuple[str, ...] = ()
+    # ISO-8601 UTC instant. Submissions with created_at >= this do not receive
+    # operator harbor_forward_env_vars (miners must supply their own keys, e.g.
+    # OPENROUTER_API_KEY). None = legacy always-forward until ops sets a cutoff.
+    # Env: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT
+    operator_env_forward_cutoff_at: datetime | None = None
     harbor_n_concurrent: int = 1
     harbor_output_dir: str = "/tmp/harbor-runs"
     # Control-plane paths for the own_runner job's task cache + frozen digest

packages/challenges/agent-challenge/tests/test_config.py

@@ -87,6 +87,7 @@ def test_normal_validator_defaults():
     assert settings.terminal_bench_label == "terminal-bench@2.1"
     assert settings.terminal_bench_execution_backend == "own_runner"
     assert settings.harbor_forward_env_vars == ()
+    assert settings.operator_env_forward_cutoff_at is None
     assert settings.harbor_n_concurrent == 1
     assert "ghcr.io/baseintelligence/agent-challenge-analyzer:1.0" in settings.docker_allowed_images
     assert (

packages/challenges/agent-challenge/tests/test_operator_env_forward_cutoff.py

@@ -0,0 +1,131 @@
+"""Operator harbor_forward_env_vars grandfathering by submission.created_at.
+
+Pre-cutoff submissions keep receiving the operator-injected OPENROUTER_API_KEY
+(and any other harbor_forward_env_vars). Post-cutoff submissions must supply
+their own key via miner env; the operator key is not forwarded and must not
+shadow the miner value. Missing keys fail with an attributable error.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from agent_challenge.evaluation.runner import _terminal_bench_env
+from agent_challenge.sdk.config import ChallengeSettings
+
+CUTOFF = datetime(2026, 7, 30, 12, 0, 0, tzinfo=UTC)
+PRE_CUTOFF = CUTOFF - timedelta(hours=1)
+POST_CUTOFF = CUTOFF + timedelta(hours=1)
+OPERATOR_KEY = "sk-or-v1-operator-forward-test-value"
+MINER_KEY = "sk-or-v1-miner-supplied-test-value"
+
+
+def _patch_forward_settings(monkeypatch, *, cutoff: datetime | None = CUTOFF) -> None:
+    monkeypatch.setattr(
+        "agent_challenge.evaluation.runner.settings.harbor_forward_env_vars",
+        ("OPENROUTER_API_KEY",),
+    )
+    monkeypatch.setattr(
+        "agent_challenge.evaluation.runner.settings.operator_env_forward_cutoff_at",
+        cutoff,
+    )
+    monkeypatch.setenv("OPENROUTER_API_KEY", OPERATOR_KEY)
+
+
+def test_pre_cutoff_submission_forwards_operator_key(monkeypatch) -> None:
+    """Given: submission created before cutoff, operator has OPENROUTER_API_KEY.
+    When: _terminal_bench_env builds job env
+    Then: operator key is present (grandfathered).
+    """
+    _patch_forward_settings(monkeypatch)
+
+    env = _terminal_bench_env(
+        {"LLM_COST_LIMIT": "1.0"},
+        submission_created_at=PRE_CUTOFF,
+    )
+
+    assert env["OPENROUTER_API_KEY"] == OPERATOR_KEY
+
+
+def test_post_cutoff_with_miner_key_uses_miner_not_operator(monkeypatch) -> None:
+    """Given: post-cutoff submission with miner OPENROUTER_API_KEY
+    When: _terminal_bench_env builds job env
+    Then: miner key is present, operator key is absent (not shadowed).
+    """
+    _patch_forward_settings(monkeypatch)
+
+    env = _terminal_bench_env(
+        {"OPENROUTER_API_KEY": MINER_KEY},
+        submission_created_at=POST_CUTOFF,
+    )
+
+    assert env["OPENROUTER_API_KEY"] == MINER_KEY
+    assert env["OPENROUTER_API_KEY"] != OPERATOR_KEY
+
+
+def test_post_cutoff_without_key_raises_attributable_error(monkeypatch) -> None:
+    """Given: post-cutoff submission with no OPENROUTER_API_KEY
+    When: _terminal_bench_env builds job env
+    Then: ValueError names OPENROUTER_API_KEY (not silent zero / crash elsewhere).
+    """
+    _patch_forward_settings(monkeypatch)
+
+    with pytest.raises(ValueError, match="OPENROUTER_API_KEY") as exc_info:
+        _terminal_bench_env(
+            {"LLM_COST_LIMIT": "1.0"},
+            submission_created_at=POST_CUTOFF,
+        )
+
+    message = str(exc_info.value)
+    assert "OPENROUTER_API_KEY" in message
+    assert "cutoff" in message.lower() or "operator" in message.lower()
+
+
+def test_cutoff_boundary_instant_is_post_cutoff(monkeypatch) -> None:
+    """Given: submission.created_at exactly equal to cutoff
+    When: _terminal_bench_env builds job env without miner key
+    Then: boundary is post-cutoff (operator not forwarded; missing key fails).
+    """
+    _patch_forward_settings(monkeypatch)
+
+    with pytest.raises(ValueError, match="OPENROUTER_API_KEY"):
+        _terminal_bench_env(
+            {},
+            submission_created_at=CUTOFF,
+        )
+
+    env = _terminal_bench_env(
+        {"OPENROUTER_API_KEY": MINER_KEY},
+        submission_created_at=CUTOFF,
+    )
+    assert env["OPENROUTER_API_KEY"] == MINER_KEY
+
+
+def test_no_cutoff_configured_keeps_legacy_operator_forward(monkeypatch) -> None:
+    """Given: operator_env_forward_cutoff_at is None (unset)
+    When: any submission builds env
+    Then: operator key is still forwarded (safe default until ops sets cutoff).
+    """
+    _patch_forward_settings(monkeypatch, cutoff=None)
+
+    env = _terminal_bench_env(
+        {},
+        submission_created_at=POST_CUTOFF,
+    )
+
+    assert env["OPENROUTER_API_KEY"] == OPERATOR_KEY
+
+
+def test_operator_env_forward_cutoff_at_env_override(monkeypatch) -> None:
+    """Given: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT is set
+    When: ChallengeSettings loads
+    Then: operator_env_forward_cutoff_at parses as aware UTC datetime.
+    """
+    monkeypatch.setenv(
+        "CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT",
+        "2026-07-30T12:00:00+00:00",
+    )
+    settings = ChallengeSettings()
+    assert settings.operator_env_forward_cutoff_at == CUTOFF