diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index 958780258..a8ef63715 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -211,18 +211,6 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive e.name === evalName); - if (evaluator?.config.codeBased) { - throw new Error( - `Code-based evaluator "${evalName}" cannot be used in online eval configs. Only LLM-as-a-Judge evaluators are supported for online evaluation.` - ); - } - } - const config: OnlineEvalConfig = { name: options.name, agent: options.agent, diff --git a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx index d5322e9ed..92c56d90e 100644 --- a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx +++ b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx @@ -48,17 +48,12 @@ export function AddOnlineEvalFlow({ isInteractive = true, onExit, onBack, onDev, const result = await listEvaluators({ region }); if (cancelled) return; - // Filter out code-based evaluators — not supported for online evaluation. - // Check both the API response type ('CustomCode') and local config (codeBased). - const codeBasedNames = new Set(projectSpec.evaluators.filter(e => e.config.codeBased).map(e => e.name)); - const items: EvaluatorItem[] = result.evaluators - .filter(e => e.evaluatorType !== 'CustomCode' && !codeBasedNames.has(e.evaluatorName)) - .map(e => ({ - arn: e.evaluatorArn, - name: e.evaluatorName, - type: e.evaluatorType, - description: e.description, - })); + const items: EvaluatorItem[] = result.evaluators.map(e => ({ + arn: e.evaluatorArn, + name: e.evaluatorName, + type: e.evaluatorType, + description: e.description, + })); const agentNames = projectSpec.runtimes.map(a => a.name); diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts index 8d0f35f1c..dd407f2f3 100644 --- a/src/schema/schemas/agentcore-project.ts +++ b/src/schema/schemas/agentcore-project.ts @@ -337,15 +337,6 @@ export const AgentCoreProjectSpecSchema = z message: `Online eval config "${config.name}" references unknown evaluator "${evalName}"`, }); } - - // Block code-based evaluators in online eval configs - const evaluator = spec.evaluators.find(e => e.name === evalName); - if (evaluator?.config.codeBased) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Online eval config "${config.name}" references code-based evaluator "${evalName}". Code-based evaluators are not supported for online evaluation.`, - }); - } } } });