feat(eval): support session trace filters and log group prefixes

package.json

@@ -64,7 +64,7 @@
   "dependencies": {
     "@aws-cdk/toolkit-lib": "1.38.2",
     "@aws-sdk/client-bedrock-agent": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore": "^3.1129.0",
+    "@aws-sdk/client-bedrock-agentcore": "^3.1135.0",
     "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0",
     "@aws-sdk/client-cloudformation": "^3.1092.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",

src/core/eval.tsx

@@ -1003,7 +1003,10 @@ export class EvalClient implements CoreEvalClient {
           await evaluatorKmsKeys(input.evaluatorIds ?? [], control),
           // Read only to widen the write scope to the chosen destination; the
           // request object below still gets the caller's object untouched.
-          { outputConfig: input.outputConfig },
+          {
+            outputConfig: input.outputConfig,
+            logGroupNamePrefixes: logGroupNamePrefixesOf(dataSourceConfig),
+          },
         )
       ).roleArn;
 
@@ -2116,6 +2119,12 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] {
     : [];
 }
 
+function logGroupNamePrefixesOf(dataSourceConfig: DataSourceConfig): string[] {
+  return "cloudWatchLogs" in dataSourceConfig
+    ? (dataSourceConfig.cloudWatchLogs?.logGroupNamePrefixes ?? [])
+    : [];
+}
+
 // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path
 // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a
 // new --endpoint without the caller passing --agent again. Returns undefined for

src/core/onlineEvalExecutionRole.tsx

@@ -112,14 +112,18 @@ export function executionPolicy(
   logGroupNames: string[],
   kmsKeyArns: string[],
   outputConfig?: OnlineEvalResultDestination,
+  logGroupNamePrefixes: string[] = [],
 ): string {
   const logs = `arn:aws:logs:${region}:${accountId}:log-group`;
   const spansArn = `${logs}:aws/spans`;
   // Scope to the runtime prefix rather than the exact endpoint log group: the
   // service validates query access at the runtime level (all of a runtime's
   // endpoints share the `...-<runtimeId>-<endpoint>` naming), and a policy
   // pinned to one endpoint is rejected as insufficient.
-  const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`);
+  const sampledArns = [
+    ...logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`),
+    ...logGroupNamePrefixes.map((prefix) => `${logs}:${prefix}*`),
+  ];
   return JSON.stringify({
     Version: "2012-10-17",
     Statement: [
@@ -220,6 +224,7 @@ export function scopePolicyName(policyDocument: string): string {
 export type GrantScopeOptions = {
   roleName?: string;
   outputConfig?: OnlineEvalResultDestination;
+  logGroupNamePrefixes?: string[];
 };
 
 // grantOnlineEvalScope creates the execution role for `configName` if it does not
@@ -232,7 +237,11 @@ export async function grantOnlineEvalScope(
   region: string,
   logGroupNames: string[],
   kmsKeyArns: string[] = [],
-  { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {},
+  {
+    roleName = onlineEvalExecutionRoleName(configName),
+    outputConfig,
+    logGroupNamePrefixes,
+  }: GrantScopeOptions = {},
 ): Promise<{ roleArn: string; policyName: string }> {
   let roleArn: string;
   try {
@@ -256,6 +265,7 @@ export async function grantOnlineEvalScope(
     logGroupNames,
     kmsKeyArns,
     outputConfig,
+    logGroupNamePrefixes,
   );
   const policyName = scopePolicyName(policyDocument);
   await iam.send(

src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.2a1e0be72c84e19a.json → src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.5a0199bc9738f03e.json

@@ -18,4 +18,4 @@
       "resultDestination": "SOURCE_LOG_GROUP"
     }
   }
-}
+}

src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx

@@ -227,12 +227,15 @@ describe("eval batch-evaluation (fixture-backed)", () => {
   }, 180_000);
 
   test("evaluate submits a job with a customer-supplied output config", async () => {
+    // The StartBatchEvaluation fixture is keyed by the full SDK request, so this
+    // also verifies that raw prefix selectors and session trace filters pass
+    // through the command handler unchanged.
     const stdout = await run([
       "eval",
       "batch-evaluation",
       "evaluate",
-      "--agent",
-      FIXTURE_EVAL_AGENT,
+      "--data-source-config",
+      '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/support_agent-"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionTraceIds":[{"sessionId":"session-123","traceIds":["4bf92f3577b34da6a3ce929d0e0e4736"]}]}}}',
       "--evaluators",
       "Builtin.Helpfulness",
       "--name",

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json → src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.3d4753441b668fca.json

@@ -11,4 +11,4 @@
       "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-15ciiv2UoV"
     }
   }
-}
+}

src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json

@@ -12,8 +12,8 @@
       "serviceNames": [
         "testAgent_Agent.DEFAULT"
       ],
-      "logGroupNames": [
-        "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT"
+      "logGroupNamePrefixes": [
+        "/aws/bedrock-agentcore/runtimes/testAgent_Agent-"
       ]
     }
   },
@@ -40,4 +40,4 @@
     }
   },
   "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_kms"
-}
+}

src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f81a24c09995fc6c.json

@@ -0,0 +1 @@
+{}

src/handlers/eval/online-eval/online-eval.test.tsx

@@ -454,8 +454,10 @@ describe("execution role KMS scoping", () => {
       "create",
       "--name",
       KMS_CONFIG_NAME,
-      "--agent",
-      FIXTURE_AGENT_ID,
+      // The PutRolePolicy fixture is keyed by the complete generated policy.
+      // Using a prefix source here verifies the default role grants that prefix.
+      "--data-source-config",
+      '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/testAgent_Agent-"],"serviceNames":["testAgent_Agent.DEFAULT"]}}',
       // Builtin.Correctness is backed by the hand-authored fixture carrying a
       // kmsKeyArn; Builtin.Helpfulness carries none, so this covers both arms of
       // the resolution in one create.

src/handlers/eval/sessionSource.tsx

@@ -19,10 +19,18 @@ Accepts inline JSON, file://<path>, or - to read stdin.
 JSON syntax:
   {
     "cloudWatchLogs": {
-      "logGroupNames": ["string", ...],  // [required] groups holding the traces
+      "logGroupNames": ["string", ...],         // exact names; maximum 10
+      "logGroupNamePrefixes": ["string", ...],  // or match by prefix; maximum 5
       "serviceNames": ["string", ...],   // e.g. "my_agent.DEFAULT"
       "filterConfig": {
         "sessionIds": ["string", ...],
+        "sessionTraceIds": [
+          {
+            "sessionId": "string",       // [required] session containing the traces
+            "traceIds": ["string", ...]  // [required] up to 100 trace IDs
+          },
+          ...
+        ],
         "timeRange": {
           "startTime": "timestamp",
           "endTime": "timestamp"
@@ -35,7 +43,7 @@ API reference:
   https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_DataSourceConfig.html
 
 Example:
-  --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/support_agent-AbC123XyZ9-DEFAULT"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionIds":["session-123"]}}}'`;
+  --data-source-config '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/support_agent-"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionTraceIds":[{"sessionId":"session-123","traceIds":["4bf92f3577b34da6a3ce929d0e0e4736"]}]}}}'`;
 
 export class SessionSource {
   static readonly flags = [