diff --git a/packages/challenges/agent-challenge/config.example.yaml b/packages/challenges/agent-challenge/config.example.yaml index 8dc540fd6..717324e15 100644 --- a/packages/challenges/agent-challenge/config.example.yaml +++ b/packages/challenges/agent-challenge/config.example.yaml @@ -137,6 +137,10 @@ challenge: harbor_agent_import_path: agent:Agent # Empty by default. Operators must explicitly opt in before forwarding provider credentials. harbor_forward_env_vars: [] + # ISO-8601 UTC. Submissions with created_at >= this do not receive operator + # harbor_forward_env_vars; miners must supply their own keys. null = always forward. + # Env override: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT + operator_env_forward_cutoff_at: null harbor_n_concurrent: 1 harbor_output_dir: /tmp/harbor-runs swe_forge_tree_url: https://huggingface.co/api/datasets/CortexLM/swe-forge/tree/main?recursive=true diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index 5f35a5791..61551312c 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -1211,6 +1211,7 @@ def _run_terminal_bench_task( gateway, replay_audit=replay_audit, replay_eval_plan=replay_eval_plan, + submission_created_at=submission.created_at, ), labels=_labels(job, submission, task), limits=_terminal_bench_limits(), @@ -1428,6 +1429,7 @@ async def _run_terminal_bench_task_durable( gateway=gateway, execution_backend=execution_backend, timeout_seconds=settings.evaluation_timeout_seconds, + submission_created_at=submission.created_at, ) else: spec = DockerRunSpec( @@ -1451,7 +1453,11 @@ async def _run_terminal_bench_task_durable( ), workdir="/workspace", env={ - **_terminal_bench_env(miner_env, gateway), + **_terminal_bench_env( + miner_env, + gateway, + submission_created_at=submission.created_at, + ), **_terminal_bench_stream_env(plan.attempt_id), }, labels=_labels(job, submission, task), @@ -1625,6 +1631,7 @@ def _terminal_bench_env( *, replay_audit: bool = False, replay_eval_plan: Mapping[str, Any] | None = None, + submission_created_at: datetime | None = None, ) -> dict[str, str]: env = { "BASE_AGENT_PATH": "/workspace/agent", @@ -1639,11 +1646,17 @@ def _terminal_bench_env( # agent still fails loudly rather than running an unmeasured default. if settings.llm_model: env["LLM_MODEL"] = settings.llm_model - for name in settings.harbor_forward_env_vars: - value = os.environ.get(name) - if value and name not in TERMINAL_BENCH_CONTROL_ENV_KEYS: - env[name] = value - operator_env_names = set(settings.harbor_forward_env_vars) + forward_operator = _should_forward_operator_env(submission_created_at) + if forward_operator: + for name in settings.harbor_forward_env_vars: + value = os.environ.get(name) + if value and name not in TERMINAL_BENCH_CONTROL_ENV_KEYS: + env[name] = value + operator_env_names = set(settings.harbor_forward_env_vars) + else: + # Post-cutoff: do not inject operator keys and do not shadow miner keys + # that share those names (e.g. miner OPENROUTER_API_KEY). + operator_env_names = set() # VAL-ACLOCK: only keys/tokens; drop URL/proxy/host/gateway/stream injection. # sanitize_miner_env_for_job is fail-closed even if an older stored env row # still contains a rejected key from before the lock landed. @@ -1663,6 +1676,19 @@ def _terminal_bench_env( # (BASE_LLM_GATEWAY_URL / BASE_GATEWAY_TOKEN), even if a residual # GatewayExecutionConfig is passed by legacy call sites. _ = gateway + if not forward_operator: + missing = [ + name + for name in settings.harbor_forward_env_vars + if name not in TERMINAL_BENCH_CONTROL_ENV_KEYS and not str(env.get(name) or "").strip() + ] + if missing: + names = ", ".join(missing) + raise ValueError( + "miner submission env missing required var(s) after operator " + f"forward cutoff: {names}. Set these on the submission env; " + "the operator no longer injects them for new submissions." + ) if replay_audit: env["BASE_REPLAY_AUDIT"] = "1" if replay_eval_plan is not None: @@ -1674,6 +1700,28 @@ def _terminal_bench_env( return env +def _should_forward_operator_env(submission_created_at: datetime | None) -> bool: + """Return True when operator harbor_forward_env_vars should still be injected. + + Grandfathers submissions created *before* ``operator_env_forward_cutoff_at``. + The boundary instant itself is post-cutoff (operator key not forwarded). + Comparison uses submission.created_at so a later re-evaluation of an old + submission stays grandfathered. When the cutoff is unset, keep legacy + always-forward behavior. When cutoff is set but created_at is missing, + fail closed (do not forward). + """ + cutoff = settings.operator_env_forward_cutoff_at + if cutoff is None: + return True + if submission_created_at is None: + return False + created = submission_created_at + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + cutoff_aware = cutoff if cutoff.tzinfo is not None else cutoff.replace(tzinfo=UTC) + return created < cutoff_aware + + def _is_provider_key_env(name: str) -> bool: """Return ``True`` when ``name`` is a raw provider API key/token env var.""" @@ -1759,11 +1807,17 @@ async def _terminal_bench_env_for_submission( session: AsyncSession, submission: AgentSubmission, ) -> dict[str, str]: - return _terminal_bench_env(await _locked_miner_env_for_submission(session, submission)) + return _terminal_bench_env( + await _locked_miner_env_for_submission(session, submission), + submission_created_at=submission.created_at, + ) def _terminal_bench_env_for_loaded_submission(submission: AgentSubmission) -> dict[str, str]: - return _terminal_bench_env(_locked_miner_env_from_loaded_submission(submission)) + return _terminal_bench_env( + _locked_miner_env_from_loaded_submission(submission), + submission_created_at=submission.created_at, + ) async def _locked_miner_env_for_submission( @@ -2046,6 +2100,7 @@ def _run_no_phala_host_terminal_bench( gateway: GatewayExecutionConfig | None, execution_backend: str, timeout_seconds: int, + submission_created_at: datetime | None = None, ) -> DockerRunResult: """NO_PHALA host-local path: run own_runner in this process namespace. @@ -2098,7 +2153,7 @@ def _run_no_phala_host_terminal_bench( ] env = { **dict(os.environ), - **_terminal_bench_env(miner_env, gateway), + **_terminal_bench_env(miner_env, gateway, submission_created_at=submission_created_at), "PYTHONPATH": f"{agent_workspace}:{os.environ.get('PYTHONPATH', '')}".rstrip(":"), "DOCKER_HOST": os.environ.get("DOCKER_HOST") or "unix:///var/run/docker.sock", "BASE_AGENT_PATH": str(agent_workspace), diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index 53587c5ef..ee28b6525 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +from datetime import datetime from pathlib import Path from typing import Any, Literal @@ -144,6 +145,11 @@ class ChallengeSettings(BaseSettings): harbor_agent_import_path: str = "agent:Agent" harbor_model: str | None = None harbor_forward_env_vars: tuple[str, ...] = () + # ISO-8601 UTC instant. Submissions with created_at >= this do not receive + # operator harbor_forward_env_vars (miners must supply their own keys, e.g. + # OPENROUTER_API_KEY). None = legacy always-forward until ops sets a cutoff. + # Env: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT + operator_env_forward_cutoff_at: datetime | None = None harbor_n_concurrent: int = 1 harbor_output_dir: str = "/tmp/harbor-runs" # Control-plane paths for the own_runner job's task cache + frozen digest diff --git a/packages/challenges/agent-challenge/tests/test_config.py b/packages/challenges/agent-challenge/tests/test_config.py index 4452c467c..97fc84343 100644 --- a/packages/challenges/agent-challenge/tests/test_config.py +++ b/packages/challenges/agent-challenge/tests/test_config.py @@ -87,6 +87,7 @@ def test_normal_validator_defaults(): assert settings.terminal_bench_label == "terminal-bench@2.1" assert settings.terminal_bench_execution_backend == "own_runner" assert settings.harbor_forward_env_vars == () + assert settings.operator_env_forward_cutoff_at is None assert settings.harbor_n_concurrent == 1 assert "ghcr.io/baseintelligence/agent-challenge-analyzer:1.0" in settings.docker_allowed_images assert ( diff --git a/packages/challenges/agent-challenge/tests/test_operator_env_forward_cutoff.py b/packages/challenges/agent-challenge/tests/test_operator_env_forward_cutoff.py new file mode 100644 index 000000000..71fd46e49 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_operator_env_forward_cutoff.py @@ -0,0 +1,131 @@ +"""Operator harbor_forward_env_vars grandfathering by submission.created_at. + +Pre-cutoff submissions keep receiving the operator-injected OPENROUTER_API_KEY +(and any other harbor_forward_env_vars). Post-cutoff submissions must supply +their own key via miner env; the operator key is not forwarded and must not +shadow the miner value. Missing keys fail with an attributable error. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from agent_challenge.evaluation.runner import _terminal_bench_env +from agent_challenge.sdk.config import ChallengeSettings + +CUTOFF = datetime(2026, 7, 30, 12, 0, 0, tzinfo=UTC) +PRE_CUTOFF = CUTOFF - timedelta(hours=1) +POST_CUTOFF = CUTOFF + timedelta(hours=1) +OPERATOR_KEY = "sk-or-v1-operator-forward-test-value" +MINER_KEY = "sk-or-v1-miner-supplied-test-value" + + +def _patch_forward_settings(monkeypatch, *, cutoff: datetime | None = CUTOFF) -> None: + monkeypatch.setattr( + "agent_challenge.evaluation.runner.settings.harbor_forward_env_vars", + ("OPENROUTER_API_KEY",), + ) + monkeypatch.setattr( + "agent_challenge.evaluation.runner.settings.operator_env_forward_cutoff_at", + cutoff, + ) + monkeypatch.setenv("OPENROUTER_API_KEY", OPERATOR_KEY) + + +def test_pre_cutoff_submission_forwards_operator_key(monkeypatch) -> None: + """Given: submission created before cutoff, operator has OPENROUTER_API_KEY. + When: _terminal_bench_env builds job env + Then: operator key is present (grandfathered). + """ + _patch_forward_settings(monkeypatch) + + env = _terminal_bench_env( + {"LLM_COST_LIMIT": "1.0"}, + submission_created_at=PRE_CUTOFF, + ) + + assert env["OPENROUTER_API_KEY"] == OPERATOR_KEY + + +def test_post_cutoff_with_miner_key_uses_miner_not_operator(monkeypatch) -> None: + """Given: post-cutoff submission with miner OPENROUTER_API_KEY + When: _terminal_bench_env builds job env + Then: miner key is present, operator key is absent (not shadowed). + """ + _patch_forward_settings(monkeypatch) + + env = _terminal_bench_env( + {"OPENROUTER_API_KEY": MINER_KEY}, + submission_created_at=POST_CUTOFF, + ) + + assert env["OPENROUTER_API_KEY"] == MINER_KEY + assert env["OPENROUTER_API_KEY"] != OPERATOR_KEY + + +def test_post_cutoff_without_key_raises_attributable_error(monkeypatch) -> None: + """Given: post-cutoff submission with no OPENROUTER_API_KEY + When: _terminal_bench_env builds job env + Then: ValueError names OPENROUTER_API_KEY (not silent zero / crash elsewhere). + """ + _patch_forward_settings(monkeypatch) + + with pytest.raises(ValueError, match="OPENROUTER_API_KEY") as exc_info: + _terminal_bench_env( + {"LLM_COST_LIMIT": "1.0"}, + submission_created_at=POST_CUTOFF, + ) + + message = str(exc_info.value) + assert "OPENROUTER_API_KEY" in message + assert "cutoff" in message.lower() or "operator" in message.lower() + + +def test_cutoff_boundary_instant_is_post_cutoff(monkeypatch) -> None: + """Given: submission.created_at exactly equal to cutoff + When: _terminal_bench_env builds job env without miner key + Then: boundary is post-cutoff (operator not forwarded; missing key fails). + """ + _patch_forward_settings(monkeypatch) + + with pytest.raises(ValueError, match="OPENROUTER_API_KEY"): + _terminal_bench_env( + {}, + submission_created_at=CUTOFF, + ) + + env = _terminal_bench_env( + {"OPENROUTER_API_KEY": MINER_KEY}, + submission_created_at=CUTOFF, + ) + assert env["OPENROUTER_API_KEY"] == MINER_KEY + + +def test_no_cutoff_configured_keeps_legacy_operator_forward(monkeypatch) -> None: + """Given: operator_env_forward_cutoff_at is None (unset) + When: any submission builds env + Then: operator key is still forwarded (safe default until ops sets cutoff). + """ + _patch_forward_settings(monkeypatch, cutoff=None) + + env = _terminal_bench_env( + {}, + submission_created_at=POST_CUTOFF, + ) + + assert env["OPENROUTER_API_KEY"] == OPERATOR_KEY + + +def test_operator_env_forward_cutoff_at_env_override(monkeypatch) -> None: + """Given: CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT is set + When: ChallengeSettings loads + Then: operator_env_forward_cutoff_at parses as aware UTC datetime. + """ + monkeypatch.setenv( + "CHALLENGE_OPERATOR_ENV_FORWARD_CUTOFF_AT", + "2026-07-30T12:00:00+00:00", + ) + settings = ChallengeSettings() + assert settings.operator_env_forward_cutoff_at == CUTOFF