From 2a7159d7c5b19d6782ea36ccad0130ccfe4e83eb Mon Sep 17 00:00:00 2001
From: "T.J Ariyawansa" <tjariy@amazon.com>
Date: Fri, 24 Apr 2026 13:18:10 -0400
Subject: [PATCH 1/2] fix: allow code-based evaluators in online eval configs

Remove restrictions that blocked code-based evaluators from being used
in online evaluation configs. The service now supports code-based
evaluators for online evaluation.

Changes:
- Remove code-based evaluator block in OnlineEvalConfigPrimitive
- Remove code-based evaluator validation in schema superRefine
- Remove code-based evaluator filter in TUI evaluator picker
---
 src/cli/primitives/OnlineEvalConfigPrimitive.ts      | 12 ------------
 .../tui/screens/online-eval/AddOnlineEvalFlow.tsx    |  4 ----
 src/schema/schemas/agentcore-project.ts              |  8 --------
 3 files changed, 24 deletions(-)

diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts
index 958780258..a8ef63715 100644
--- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts
+++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts
@@ -211,18 +211,6 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive<AddOnlineEvalConfig
 
     this.checkDuplicate(project.onlineEvalConfigs, options.name, 'Online eval config');
 
-    // Block code-based evaluators — only LLM-as-a-Judge evaluators are supported for online evaluation.
-    // Checks local project config. ARN-based evaluators are filtered in the TUI by API evaluatorType.
-    // TODO: For ARN-based evaluators in non-interactive mode, call getEvaluator to check type.
-    for (const evalName of options.evaluators) {
-      const evaluator = project.evaluators.find(e => e.name === evalName);
-      if (evaluator?.config.codeBased) {
-        throw new Error(
-          `Code-based evaluator "${evalName}" cannot be used in online eval configs. Only LLM-as-a-Judge evaluators are supported for online evaluation.`
-        );
-      }
-    }
-
     const config: OnlineEvalConfig = {
       name: options.name,
       agent: options.agent,
diff --git a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
index d5322e9ed..25110de0d 100644
--- a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
+++ b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
@@ -48,11 +48,7 @@ export function AddOnlineEvalFlow({ isInteractive = true, onExit, onBack, onDev,
         const result = await listEvaluators({ region });
         if (cancelled) return;
 
-        // Filter out code-based evaluators — not supported for online evaluation.
-        // Check both the API response type ('CustomCode') and local config (codeBased).
-        const codeBasedNames = new Set(projectSpec.evaluators.filter(e => e.config.codeBased).map(e => e.name));
         const items: EvaluatorItem[] = result.evaluators
-          .filter(e => e.evaluatorType !== 'CustomCode' && !codeBasedNames.has(e.evaluatorName))
           .map(e => ({
             arn: e.evaluatorArn,
             name: e.evaluatorName,
diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts
index 8d0f35f1c..40c0fc260 100644
--- a/src/schema/schemas/agentcore-project.ts
+++ b/src/schema/schemas/agentcore-project.ts
@@ -338,14 +338,6 @@ export const AgentCoreProjectSpecSchema = z
           });
         }
 
-        // Block code-based evaluators in online eval configs
-        const evaluator = spec.evaluators.find(e => e.name === evalName);
-        if (evaluator?.config.codeBased) {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `Online eval config "${config.name}" references code-based evaluator "${evalName}". Code-based evaluators are not supported for online evaluation.`,
-          });
-        }
       }
     }
   });

From 7c1025c44ae1961c4eb8b8c9a64958a989fdfa81 Mon Sep 17 00:00:00 2001
From: "T.J Ariyawansa" <tjariy@amazon.com>
Date: Fri, 24 Apr 2026 14:00:21 -0400
Subject: [PATCH 2/2] style: fix prettier formatting

---
 .../tui/screens/online-eval/AddOnlineEvalFlow.tsx   | 13 ++++++-------
 src/schema/schemas/agentcore-project.ts             |  1 -
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
index 25110de0d..92c56d90e 100644
--- a/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
+++ b/src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx
@@ -48,13 +48,12 @@ export function AddOnlineEvalFlow({ isInteractive = true, onExit, onBack, onDev,
         const result = await listEvaluators({ region });
         if (cancelled) return;
 
-        const items: EvaluatorItem[] = result.evaluators
-          .map(e => ({
-            arn: e.evaluatorArn,
-            name: e.evaluatorName,
-            type: e.evaluatorType,
-            description: e.description,
-          }));
+        const items: EvaluatorItem[] = result.evaluators.map(e => ({
+          arn: e.evaluatorArn,
+          name: e.evaluatorName,
+          type: e.evaluatorType,
+          description: e.description,
+        }));
 
         const agentNames = projectSpec.runtimes.map(a => a.name);
 
diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts
index 40c0fc260..dd407f2f3 100644
--- a/src/schema/schemas/agentcore-project.ts
+++ b/src/schema/schemas/agentcore-project.ts
@@ -337,7 +337,6 @@ export const AgentCoreProjectSpecSchema = z
             message: `Online eval config "${config.name}" references unknown evaluator "${evalName}"`,
           });
         }
-
       }
     }
   });