From 576581fa1e1d21ec660475402b664e5b2ac1be14 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 18 Sep 2026 21:37:27 +0000 Subject: [PATCH] feat(eval): support session trace filters in batch evaluations --- bun.lock | 4 ++-- package.json | 2 +- src/core/eval.tsx | 11 ++++++++++- src/core/onlineEvalExecutionRole.tsx | 14 ++++++++++++-- ...rtBatchEvaluationCommand.5a0199bc9738f03e.json} | 2 +- .../batch-evaluation.fixture.test.tsx | 7 +++++-- ...eEvaluationConfigCommand.3d4753441b668fca.json} | 2 +- ...neEvaluationConfigCommand.d1856366f3636123.json | 6 +++--- .../PutRolePolicyCommand.cb047843e517ce33.json | 1 - .../PutRolePolicyCommand.f81a24c09995fc6c.json | 1 + src/handlers/eval/online-eval/online-eval.test.tsx | 6 ++++-- src/handlers/eval/sessionSource.tsx | 12 ++++++++++-- 12 files changed, 50 insertions(+), 18 deletions(-) rename src/handlers/eval/batch-evaluation/__fixtures__/{StartBatchEvaluationCommand.2a1e0be72c84e19a.json => StartBatchEvaluationCommand.5a0199bc9738f03e.json} (99%) rename src/handlers/eval/online-eval/__fixtures__/{CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json => CreateOnlineEvaluationConfigCommand.3d4753441b668fca.json} (99%) delete mode 100644 src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f81a24c09995fc6c.json diff --git a/bun.lock b/bun.lock index 8cb03228d3..7d83dd693e 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,7 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1135.0", "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", @@ -96,7 +96,7 @@ "@aws-sdk/client-bedrock-agent": ["@aws-sdk/client-bedrock-agent@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-RAjn1g6X+u4WwnuwxYU8sKna5q1waTBCM+kzXWAvxSaNbgRsXFL6ZylPjOl0qCDZ7x6aVTj7Jb7Q0z6DZifP3w=="], - "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1131.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-xyPxp33VylkesNriC2WOJMkYmj0BX7borAUMojWg/sFcoTyEU0KVs0U6bElZkq7X46/1w6w3hhSGrMVEti7RcA=="], + "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1136.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-fpyMhUO8FMqtsJv4+2b6iE/vkSI2arraWdqo8FOYGnkeLfwtikao3QOwqiNj1ihZ0wk7xr+Sscg5M1wigQEMFw=="], "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1131.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-syVIB7YMmMm2LAfpCNh03geBsRh/XBbeTXurDP0a/y5FcKw35yAdsYiZgfH4LPBAiGI7CbxqyZx8mtROXcAptg=="], diff --git a/package.json b/package.json index 8142bacf9c..e46219d457 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1135.0", "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 4e2a061e15..73b681968d 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -1003,7 +1003,10 @@ export class EvalClient implements CoreEvalClient { await evaluatorKmsKeys(input.evaluatorIds ?? [], control), // Read only to widen the write scope to the chosen destination; the // request object below still gets the caller's object untouched. - { outputConfig: input.outputConfig }, + { + outputConfig: input.outputConfig, + logGroupNamePrefixes: logGroupNamePrefixesOf(dataSourceConfig), + }, ) ).roleArn; @@ -2116,6 +2119,12 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] { : []; } +function logGroupNamePrefixesOf(dataSourceConfig: DataSourceConfig): string[] { + return "cloudWatchLogs" in dataSourceConfig + ? (dataSourceConfig.cloudWatchLogs?.logGroupNamePrefixes ?? []) + : []; +} + // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a // new --endpoint without the caller passing --agent again. Returns undefined for diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx index 5e9d76edea..448881a89e 100644 --- a/src/core/onlineEvalExecutionRole.tsx +++ b/src/core/onlineEvalExecutionRole.tsx @@ -112,6 +112,7 @@ export function executionPolicy( logGroupNames: string[], kmsKeyArns: string[], outputConfig?: OnlineEvalResultDestination, + logGroupNamePrefixes: string[] = [], ): string { const logs = `arn:aws:logs:${region}:${accountId}:log-group`; const spansArn = `${logs}:aws/spans`; @@ -119,7 +120,10 @@ export function executionPolicy( // service validates query access at the runtime level (all of a runtime's // endpoints share the `...--` naming), and a policy // pinned to one endpoint is rejected as insufficient. - const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`); + const sampledArns = [ + ...logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`), + ...logGroupNamePrefixes.map((prefix) => `${logs}:${prefix}*`), + ]; return JSON.stringify({ Version: "2012-10-17", Statement: [ @@ -220,6 +224,7 @@ export function scopePolicyName(policyDocument: string): string { export type GrantScopeOptions = { roleName?: string; outputConfig?: OnlineEvalResultDestination; + logGroupNamePrefixes?: string[]; }; // grantOnlineEvalScope creates the execution role for `configName` if it does not @@ -232,7 +237,11 @@ export async function grantOnlineEvalScope( region: string, logGroupNames: string[], kmsKeyArns: string[] = [], - { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {}, + { + roleName = onlineEvalExecutionRoleName(configName), + outputConfig, + logGroupNamePrefixes, + }: GrantScopeOptions = {}, ): Promise<{ roleArn: string; policyName: string }> { let roleArn: string; try { @@ -256,6 +265,7 @@ export async function grantOnlineEvalScope( logGroupNames, kmsKeyArns, outputConfig, + logGroupNamePrefixes, ); const policyName = scopePolicyName(policyDocument); await iam.send( diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.2a1e0be72c84e19a.json b/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.5a0199bc9738f03e.json similarity index 99% rename from src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.2a1e0be72c84e19a.json rename to src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.5a0199bc9738f03e.json index 868682e886..e17bf0e2a3 100644 --- a/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.2a1e0be72c84e19a.json +++ b/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.5a0199bc9738f03e.json @@ -18,4 +18,4 @@ "resultDestination": "SOURCE_LOG_GROUP" } } -} \ No newline at end of file +} diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx index a191994374..2f2e5a290a 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx @@ -227,12 +227,15 @@ describe("eval batch-evaluation (fixture-backed)", () => { }, 180_000); test("evaluate submits a job with a customer-supplied output config", async () => { + // The StartBatchEvaluation fixture is keyed by the full SDK request, so this + // also verifies that raw prefix selectors and session trace filters pass + // through the command handler unchanged. const stdout = await run([ "eval", "batch-evaluation", "evaluate", - "--agent", - FIXTURE_EVAL_AGENT, + "--data-source-config", + '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/support_agent-"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionTraceIds":[{"sessionId":"session-123","traceIds":["4bf92f3577b34da6a3ce929d0e0e4736"]}]}}}', "--evaluators", "Builtin.Helpfulness", "--name", diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.3d4753441b668fca.json similarity index 99% rename from src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json rename to src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.3d4753441b668fca.json index 724eb49bcf..3eec2796bb 100644 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.3d4753441b668fca.json @@ -11,4 +11,4 @@ "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-15ciiv2UoV" } } -} \ No newline at end of file +} diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json index 97ab5d9fcd..dc920ceb08 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json @@ -12,8 +12,8 @@ "serviceNames": [ "testAgent_Agent.DEFAULT" ], - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + "logGroupNamePrefixes": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-" ] } }, @@ -40,4 +40,4 @@ } }, "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_kms" -} \ No newline at end of file +} diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f81a24c09995fc6c.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f81a24c09995fc6c.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f81a24c09995fc6c.json @@ -0,0 +1 @@ +{} diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index 1e0c3b7343..10d6230019 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -454,8 +454,10 @@ describe("execution role KMS scoping", () => { "create", "--name", KMS_CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, + // The PutRolePolicy fixture is keyed by the complete generated policy. + // Using a prefix source here verifies the default role grants that prefix. + "--data-source-config", + '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/testAgent_Agent-"],"serviceNames":["testAgent_Agent.DEFAULT"]}}', // Builtin.Correctness is backed by the hand-authored fixture carrying a // kmsKeyArn; Builtin.Helpfulness carries none, so this covers both arms of // the resolution in one create. diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx index 8933efa346..15393728ac 100644 --- a/src/handlers/eval/sessionSource.tsx +++ b/src/handlers/eval/sessionSource.tsx @@ -19,10 +19,18 @@ Accepts inline JSON, file://, or - to read stdin. JSON syntax: { "cloudWatchLogs": { - "logGroupNames": ["string", ...], // [required] groups holding the traces + "logGroupNames": ["string", ...], // exact names; maximum 10 + "logGroupNamePrefixes": ["string", ...], // or match by prefix; maximum 5 "serviceNames": ["string", ...], // e.g. "my_agent.DEFAULT" "filterConfig": { "sessionIds": ["string", ...], + "sessionTraceIds": [ + { + "sessionId": "string", // [required] session containing the traces + "traceIds": ["string", ...] // [required] up to 100 trace IDs + }, + ... + ], "timeRange": { "startTime": "timestamp", "endTime": "timestamp" @@ -35,7 +43,7 @@ API reference: https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_DataSourceConfig.html Example: - --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/support_agent-AbC123XyZ9-DEFAULT"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionIds":["session-123"]}}}'`; + --data-source-config '{"cloudWatchLogs":{"logGroupNamePrefixes":["/aws/bedrock-agentcore/runtimes/support_agent-"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionTraceIds":[{"sessionId":"session-123","traceIds":["4bf92f3577b34da6a3ce929d0e0e4736"]}]}}}'`; export class SessionSource { static readonly flags = [