From 59b9945089675224777b2147e8bc2d5a159f425a Mon Sep 17 00:00:00 2001 From: Ayushi Ahjolia Date: Thu, 30 Jul 2026 12:30:32 -0700 Subject: [PATCH] feat(otel): add no-arg InvocationOtelPlugin constructor with ADOT Java agent SPI --- .../java/plugin/ConformanceLoggingPlugin.java | 8 +- .../java/plugin/FaultyConformancePlugin.java | 24 +-- .../java/plugin/PluginErrorIsolation.java | 4 +- .../plugin/PluginInvocationLifecycle.java | 4 +- .../java/plugin/PluginMultiplePlugins.java | 7 +- .../java/plugin/PluginOperationLifecycle.java | 4 +- .../java/plugin/PluginTerminalFailure.java | 6 +- examples/generate-template.py | 55 +++++- examples/pom.xml | 36 ++-- .../durable/examples/ExampleTemplate.java | 2 + .../OtelXRayDefaultConstructorExample.java | 37 ++++ .../examples/otel/OtelXRayExamples.java | 13 +- .../examples/otel/OtelXRayStepExample.java | 18 +- .../examples/otel/OtelXRayWaitExample.java | 34 ++-- examples/src/main/resources/log4j2.xml | 2 +- .../otel/OtelXRayExampleTestSupport.java | 30 +++ .../otel/OtelXRayStepExampleTest.java | 12 ++ .../otel/OtelXRayWaitExampleTest.java | 12 ++ otel-plugin/README.md | 128 ++++++------ otel-plugin/pom.xml | 7 + .../otel/DeterministicIdGenerator.java | 45 ++++- .../durable/otel/InvocationOtelPlugin.java | 186 +++++++++++++++--- .../lambda/durable/otel/MdcSpanEnricher.java | 10 +- ...inAutoConfigurationCustomizerProvider.java | 23 +++ .../OtelPluginAutoConfigurationState.java | 23 +++ ...re.spi.AutoConfigurationCustomizerProvider | 1 + .../testing/FakeJavaAgentTracerProvider.java | 31 +++ .../otel/DeterministicIdGeneratorTest.java | 19 ++ .../InvocationOtelPluginIntegrationTest.java | 82 ++++++++ .../otel/InvocationOtelPluginTest.java | 185 +++++++++++++++++ .../durable/otel/MdcSpanEnricherTest.java | 13 +- 31 files changed, 880 insertions(+), 181 deletions(-) create mode 100644 examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java create mode 100644 examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java create mode 100644 otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider create mode 100644 otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java diff --git a/conformance-tests/src/main/java/plugin/ConformanceLoggingPlugin.java b/conformance-tests/src/main/java/plugin/ConformanceLoggingPlugin.java index 0315128de..54ea51660 100644 --- a/conformance-tests/src/main/java/plugin/ConformanceLoggingPlugin.java +++ b/conformance-tests/src/main/java/plugin/ConformanceLoggingPlugin.java @@ -13,10 +13,10 @@ /** * Shared instrumentation plugin for the plugin conformance suite. * - *

Emits lifecycle log lines with a configurable prefix (e.g. {@code CONFPLUGIN}, {@code CONFPLUGIN-A}) so one - * plugin — or two, for the multiple-plugins case — can be registered on a handler. Operation- and attempt-level hooks - * are filtered to step-type operations to match the requirement vocabulary. All lines are emitted from the real SDK - * plugin hooks; nothing is hand-rolled. + *

Emits lifecycle log lines with a configurable prefix (e.g. {@code CONFPLUGIN}, {@code CONFPLUGIN-A}) so one plugin + * — or two, for the multiple-plugins case — can be registered on a handler. Operation- and attempt-level hooks are + * filtered to step-type operations to match the requirement vocabulary. All lines are emitted from the real SDK plugin + * hooks; nothing is hand-rolled. */ @SuppressWarnings("deprecation") public class ConformanceLoggingPlugin implements DurableExecutionPlugin { diff --git a/conformance-tests/src/main/java/plugin/FaultyConformancePlugin.java b/conformance-tests/src/main/java/plugin/FaultyConformancePlugin.java index 2b5e2306c..59811aae3 100644 --- a/conformance-tests/src/main/java/plugin/FaultyConformancePlugin.java +++ b/conformance-tests/src/main/java/plugin/FaultyConformancePlugin.java @@ -35,23 +35,23 @@ private String arnField() { @Override public void onInvocationStart(InvocationInfo info) { this.executionArn = info.durableExecutionArn(); - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"invocation-start\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"invocation-start\"%s}", arnField())); throw new RuntimeException("faulty invocation-start"); } @Override public void onInvocationEnd(InvocationEndInfo info) { - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"invocation-end\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"invocation-end\"%s}", arnField())); throw new RuntimeException("faulty invocation-end"); } @Override public void onOperationStart(OperationInfo info) { if (isStep(info.type())) { - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"operation-start\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"operation-start\"%s}", arnField())); throw new RuntimeException("faulty operation-start"); } } @@ -59,8 +59,8 @@ public void onOperationStart(OperationInfo info) { @Override public void onOperationEnd(OperationEndInfo info) { if (isStep(info.type())) { - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"operation-end\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"operation-end\"%s}", arnField())); throw new RuntimeException("faulty operation-end"); } } @@ -68,8 +68,8 @@ public void onOperationEnd(OperationEndInfo info) { @Override public void onUserFunctionStart(UserFunctionStartInfo info) { if (isStep(info.type())) { - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"attempt-start\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"attempt-start\"%s}", arnField())); throw new RuntimeException("faulty attempt-start"); } } @@ -77,8 +77,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { if (isStep(info.type())) { - System.out.println(String.format( - "{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"attempt-end\"%s}", arnField())); + System.out.println( + String.format("{\"plugin\": \"CONFPLUGIN-FAULTY\", \"hook\": \"attempt-end\"%s}", arnField())); throw new RuntimeException("faulty attempt-end"); } } diff --git a/conformance-tests/src/main/java/plugin/PluginErrorIsolation.java b/conformance-tests/src/main/java/plugin/PluginErrorIsolation.java index 59aeb80ce..0ff7aea7d 100644 --- a/conformance-tests/src/main/java/plugin/PluginErrorIsolation.java +++ b/conformance-tests/src/main/java/plugin/PluginErrorIsolation.java @@ -18,7 +18,9 @@ public class PluginErrorIsolation extends DurableHandler { @Override protected DurableConfig createConfiguration() { - return DurableConfig.builder().withPlugins(new FaultyConformancePlugin()).build(); + return DurableConfig.builder() + .withPlugins(new FaultyConformancePlugin()) + .build(); } @Override diff --git a/conformance-tests/src/main/java/plugin/PluginInvocationLifecycle.java b/conformance-tests/src/main/java/plugin/PluginInvocationLifecycle.java index a407f2674..fc59beffa 100644 --- a/conformance-tests/src/main/java/plugin/PluginInvocationLifecycle.java +++ b/conformance-tests/src/main/java/plugin/PluginInvocationLifecycle.java @@ -10,8 +10,8 @@ * 10-1: Plugin invocation lifecycle hooks (start and end on a single invocation). * *

A single greeting step configured with {@link ConformanceLoggingPlugin}. The plugin's invocation-start hook logs - * {@code first=true} before the handler runs and its invocation-end hook logs {@code status=SUCCEEDED} after the - * result is finalized. The step body logs its running line via the context logger. + * {@code first=true} before the handler runs and its invocation-end hook logs {@code status=SUCCEEDED} after the result + * is finalized. The step body logs its running line via the context logger. */ @SuppressWarnings("deprecation") public class PluginInvocationLifecycle extends DurableHandler { diff --git a/conformance-tests/src/main/java/plugin/PluginMultiplePlugins.java b/conformance-tests/src/main/java/plugin/PluginMultiplePlugins.java index 4dde9f6b3..619d34b55 100644 --- a/conformance-tests/src/main/java/plugin/PluginMultiplePlugins.java +++ b/conformance-tests/src/main/java/plugin/PluginMultiplePlugins.java @@ -38,8 +38,8 @@ private String arnField() { @Override public void onInvocationStart(InvocationInfo info) { this.executionArn = info.durableExecutionArn(); - System.out.println(String.format( - "{\"plugin\": \"%s\", \"hook\": \"invocation-start\"%s}", prefix, arnField())); + System.out.println( + String.format("{\"plugin\": \"%s\", \"hook\": \"invocation-start\"%s}", prefix, arnField())); } @Override @@ -53,8 +53,7 @@ public void onInvocationEnd(InvocationEndInfo info) { @Override protected DurableConfig createConfiguration() { return DurableConfig.builder() - .withPlugins( - new InvocationLoggingPlugin("CONFPLUGIN-A"), new InvocationLoggingPlugin("CONFPLUGIN-B")) + .withPlugins(new InvocationLoggingPlugin("CONFPLUGIN-A"), new InvocationLoggingPlugin("CONFPLUGIN-B")) .build(); } diff --git a/conformance-tests/src/main/java/plugin/PluginOperationLifecycle.java b/conformance-tests/src/main/java/plugin/PluginOperationLifecycle.java index ecbb39bb9..4de3f93b0 100644 --- a/conformance-tests/src/main/java/plugin/PluginOperationLifecycle.java +++ b/conformance-tests/src/main/java/plugin/PluginOperationLifecycle.java @@ -10,8 +10,8 @@ * 10-2: Plugin operation lifecycle hooks (step start and terminal end). * *

A single greeting step configured with {@link ConformanceLoggingPlugin}. The plugin, filtering to step-type - * operations, logs {@code operation-start} when the step's STARTED checkpoint is observed and - * {@code operation-end status=SUCCEEDED} when the step reaches its terminal status. + * operations, logs {@code operation-start} when the step's STARTED checkpoint is observed and {@code operation-end + * status=SUCCEEDED} when the step reaches its terminal status. */ @SuppressWarnings("deprecation") public class PluginOperationLifecycle extends DurableHandler { diff --git a/conformance-tests/src/main/java/plugin/PluginTerminalFailure.java b/conformance-tests/src/main/java/plugin/PluginTerminalFailure.java index 9ba9cf7da..8b6db8e5f 100644 --- a/conformance-tests/src/main/java/plugin/PluginTerminalFailure.java +++ b/conformance-tests/src/main/java/plugin/PluginTerminalFailure.java @@ -11,9 +11,9 @@ /** * 10-7: Plugin invocation-end hook receives FAILED status when execution fails. * - *

A single step that always throws, configured with no retries and {@link ConformanceLoggingPlugin}. The plugin - * logs {@code invocation-start first=true}; the step throws, no retry is attempted, and the plugin's invocation-end - * hook fires with {@code status=FAILED}. + *

A single step that always throws, configured with no retries and {@link ConformanceLoggingPlugin}. The plugin logs + * {@code invocation-start first=true}; the step throws, no retry is attempted, and the plugin's invocation-end hook + * fires with {@code status=FAILED}. */ @SuppressWarnings("deprecation") public class PluginTerminalFailure extends DurableHandler { diff --git a/examples/generate-template.py b/examples/generate-template.py index 75e6cce09..d223d37d4 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -4,6 +4,7 @@ import argparse import re +import xml.etree.ElementTree as ET from dataclasses import dataclass from pathlib import Path @@ -13,6 +14,17 @@ EXAMPLE_PACKAGE_ROOT = SOURCE_ROOT / "software/amazon/lambda/durable/examples" DEFAULT_OUTPUT = EXAMPLES_DIR / "template.yaml" TEMPLATE_ANNOTATION = "ExampleTemplate" +POM_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} + + +def read_otel_plugin_jar_path() -> str: + root = ET.parse(EXAMPLES_DIR / "pom.xml").getroot() + version = root.findtext("m:version", namespaces=POM_NAMESPACE) + if version is None: + version = root.findtext("m:parent/m:version", namespaces=POM_NAMESPACE) + if version is None: + raise ValueError("Unable to read examples version from pom.xml") + return f"/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-{version}.jar" @dataclass(frozen=True) @@ -22,6 +34,7 @@ class ExampleFunction: suffix: str condition: str | None tracing: bool + java_agent: bool @property def logical_id(self) -> str: @@ -58,24 +71,26 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool]: +def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None, False + return None, False, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None, False + return None, False, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) tracing_match = re.search(r"tracing\s*=\s*(true|false)", body) + java_agent_match = re.search(r"javaAgent\s*=\s*(true|false)", body) condition = condition_match.group(1) if condition_match else None tracing = tracing_match.group(1) == "true" if tracing_match else False - return condition, tracing + java_agent = java_agent_match.group(1) == "true" if java_agent_match else False + return condition, tracing, java_agent def discover_examples() -> list[ExampleFunction]: @@ -86,7 +101,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition, tracing = read_template_annotation(source, class_name) + condition, tracing, java_agent = read_template_annotation(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -95,12 +110,13 @@ def discover_examples() -> list[ExampleFunction]: suffix=kebab_case(class_name), condition=condition, tracing=tracing, + java_agent=java_agent, ) ) return examples -def emit_function(lines: list[str], example: ExampleFunction) -> None: +def emit_function(lines: list[str], example: ExampleFunction, java_agent_extension_path: str) -> None: lines.extend( [ f" {example.logical_id}:", @@ -123,9 +139,19 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: [ " Tracing: Active", " Layers:", - " - !Sub", - " - arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-agent-${AdotArch}-ver-1-32-0:6", - " - AdotArch: amd64", + " - !Ref AdotLayerArn", + ] + ) + if example.java_agent: + lines.extend( + [ + " LoggingConfig:", + " LogFormat: JSON", + " Environment:", + " Variables:", + " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument", + f' JAVA_TOOL_OPTIONS: "-Dotel.javaagent.extensions={java_agent_extension_path}"', + f" OTEL_JAVAAGENT_EXTENSIONS: {java_agent_extension_path}", ] ) lines.append("") @@ -151,6 +177,7 @@ def emit_log_group(lines: list[str], example: ExampleFunction) -> None: def render_template(examples: list[ExampleFunction]) -> str: + java_agent_extension_path = read_otel_plugin_jar_path() lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", 'AWSTemplateFormatVersion: "2010-09-09"', @@ -176,6 +203,14 @@ def render_template(examples: list[ExampleFunction]) -> str: " RoleArn:", " Type: String", " Description: IAM Role ARN for Lambda function execution", + " AdotLayerArn:", + " Type: String", + " Default: arn:aws:lambda:us-west-2:615299751070:layer:AWSOpenTelemetryDistroJava:16", + " Description: >-", + " ARN of the ADOT (AWS Distro for OpenTelemetry) Lambda layer used by the tracing examples.", + " The layer is regional: its account ID and version vary by region, so override this per", + " deployment region. The default targets us-west-2 (the region used by e2e tests); CI", + " resolves the latest ARN for its region.", "", "Conditions:", " IsJava21OrLater:", @@ -202,7 +237,7 @@ def render_template(examples: list[ExampleFunction]) -> str: for example in examples: emit_log_group(lines, example) - emit_function(lines, example) + emit_function(lines, example, java_agent_extension_path) lines.append("Outputs:") for index, example in enumerate(examples): diff --git a/examples/pom.xml b/examples/pom.xml index 7c3530cb2..51566f3e1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -50,18 +50,6 @@ 1.64.0 - - - io.opentelemetry - opentelemetry-exporter-otlp - 1.64.0 - - - io.grpc - grpc-netty-shaded - 1.82.2 - - com.amazonaws @@ -154,6 +142,30 @@ + + org.apache.maven.plugins + maven-dependency-plugin + 3.9.0 + + + copy-otel-javaagent-extension + prepare-package + + copy + + + + + ${project.groupId} + aws-durable-execution-sdk-java-plugin-otel + ${project.version} + ${project.build.outputDirectory}/lib + + + + + + org.apache.maven.plugins maven-source-plugin diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index 898634884..bd7d92bc0 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -14,4 +14,6 @@ String condition() default ""; boolean tracing() default false; + + boolean javaAgent() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java new file mode 100644 index 000000000..9dee0c7bf --- /dev/null +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.otel; + +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.examples.ExampleTemplate; +import software.amazon.lambda.durable.examples.types.GreetingRequest; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; + +/** + * OTel + X-Ray example that uses the no-arg plugin constructor. + * + *

{@link InvocationOtelPlugin#InvocationOtelPlugin()} uses the global provider initialized by the ADOT Java agent + * with deterministic span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI. + */ +@ExampleTemplate(tracing = true, javaAgent = true) +public class OtelXRayDefaultConstructorExample extends DurableHandler { + + @Override + protected DurableConfig createConfiguration() { + return DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + } + + @Override + public String handleRequest(GreetingRequest input, DurableContext context) { + context.getLogger().info("Starting OTel X-Ray default constructor example for {}", input.getName()); + + var greeting = context.step("default-create-greeting", String.class, stepCtx -> "Hello, " + input.getName()); + + var result = context.step("default-transform", String.class, stepCtx -> greeting.toUpperCase() + "!"); + + context.getLogger().info("OTel X-Ray default constructor example complete: {}", result); + return result; + } +} diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java index dcd6a875b..625fb0aad 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.util.List; @@ -20,10 +20,9 @@ public final class OtelXRayExamples { private OtelXRayExamples() {} - private static DurableConfig otelConfig() { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); + private static DurableConfig localOtelConfig() { var otelPlugin = new InvocationOtelPlugin( - SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create()))); return DurableConfig.builder().withPlugins(otelPlugin).build(); } @@ -32,7 +31,7 @@ public static class MapExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - return otelConfig(); + return localOtelConfig(); } @Override @@ -56,7 +55,7 @@ public static class ParallelExample extends DurableHandlerExports spans via OTLP to the ADOT Lambda Layer collector, which forwards them to X-Ray. Requires: + *

Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires: * *

* *

Expected trace structure in X-Ray: @@ -32,18 +30,12 @@ * └── transform attempt 1 * */ -@ExampleTemplate(tracing = true) +@ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayStepExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = new InvocationOtelPlugin( - SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); } @Override diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java index f58fea837..aac7c95d8 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java @@ -2,9 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Duration; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; @@ -23,39 +20,34 @@ *

  • Invocation 2: replays "before-wait" (no-op) → wait completes → "after-wait" step runs * * - *

    Exports spans via OTLP to the ADOT Lambda Layer collector. Requires: + *

    Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires: * *

    * *

    Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root): * * 

      * Trace (single trace ID across both invocations)
- * ├── durable.invocation (invocation 1)
- * │   ├── durable.step:before-wait
- * │   │   └── durable.step:before-wait [attempt 1]
- * │   └── durable.wait:pause (ended as PENDING)
- * └── durable.invocation (invocation 2)
- *     ├── durable.wait:pause (completed)
- *     └── durable.step:after-wait
- *         └── durable.step:after-wait [attempt 1]
+ * ├── invocation (invocation 1)
+ * │   ├── before-wait
+ * │   │   └── before-wait attempt 1
+ * │   └── pause (ended as PENDING)
+ * └── invocation (invocation 2)
+ *     ├── pause (completed)
+ *     └── after-wait
+ *         └── after-wait attempt 1
  *
    */ -@ExampleTemplate(tracing = true) +@ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayWaitExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = new InvocationOtelPlugin( - SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); } @Override diff --git a/examples/src/main/resources/log4j2.xml b/examples/src/main/resources/log4j2.xml index 827132a4c..7bf78fae6 100644 --- a/examples/src/main/resources/log4j2.xml +++ b/examples/src/main/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java new file mode 100644 index 000000000..b1f1a5744 --- /dev/null +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.otel; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; + +final class OtelXRayExampleTestSupport { + + private static final String SPI_INSTALLED_PROPERTY = + "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; + + private OtelXRayExampleTestSupport() {} + + static void installGlobalOpenTelemetry() { + System.setProperty(SPI_INSTALLED_PROPERTY, Boolean.TRUE.toString()); + var tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + } + + static void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + System.clearProperty(SPI_INSTALLED_PROPERTY); + } +} diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java index 1bae61686..dffe99080 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.examples.types.GreetingRequest; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -11,6 +13,16 @@ class OtelXRayStepExampleTest { + @BeforeEach + void setUp() { + OtelXRayExampleTestSupport.installGlobalOpenTelemetry(); + } + + @AfterEach + void tearDown() { + OtelXRayExampleTestSupport.resetGlobalOpenTelemetry(); + } + @Test void testSimpleSteps_succeeds() { var handler = new OtelXRayStepExample(); diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java index 4d46dc2bb..ac42d31d9 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.examples.types.GreetingRequest; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -11,6 +13,16 @@ class OtelXRayWaitExampleTest { + @BeforeEach + void setUp() { + OtelXRayExampleTestSupport.installGlobalOpenTelemetry(); + } + + @AfterEach + void tearDown() { + OtelXRayExampleTestSupport.resetGlobalOpenTelemetry(); + } + @Test void testFirstInvocation_suspendsOnWait() { var handler = new OtelXRayWaitExample(); diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 8b2a06ea5..328855dfc 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -9,8 +9,8 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **Deterministic Trace IDs**: All invocations of the same durable execution share a single trace, derived from the X-Ray trace header or execution ARN - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries -- **Log Correlation**: Injects `traceId`, `spanId`, and `traceSampled` into SLF4J MDC for end-to-end observability -- **Self-Contained Setup**: No manual TracerProvider configuration required beyond the exporter +- **Log Correlation**: Injects `trace_id`, `span_id`, and `traceSampled` into SLF4J MDC for end-to-end observability +- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` uses the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization ## Installation @@ -22,40 +22,43 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo ``` -You also need the OpenTelemetry SDK and an exporter: +For the no-arg constructor (`new InvocationOtelPlugin()`), no additional OpenTelemetry dependencies are needed — the ADOT Java agent layer provides them. + +If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK and an exporter: ```xml io.opentelemetry opentelemetry-sdk - 1.63.0 + 1.64.0 io.opentelemetry - opentelemetry-exporter-otlp - 1.63.0 + opentelemetry-exporter-logging + 1.64.0 ``` -## Quick Start using X-Ray/CloudWatch Tracing +## Quick Start using X-Ray/CloudWatch Tracing (ADOT Java Agent) 1. Add the ADOT Lambda Layer to your function 2. Enable X-Ray Active Tracing on the function -3. Register `InvocationOtelPlugin` in your handler's `DurableConfig` -4. Grant X-Ray write permissions +3. Configure environment variables +4. Register `InvocationOtelPlugin` in your handler's `DurableConfig` +5. Grant X-Ray write permissions ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The layer provides an OTLP collector that receives spans from the plugin and exports them to X-Ray. - -> **Note:** Do NOT set `AWS_LAMBDA_EXEC_WRAPPER`. The ADOT layer's collector extension runs independently as a Lambda External Extension. The wrapper would attach the auto-instrumentation agent which creates a competing TracerProvider, causing disconnected service nodes in the X-Ray trace map. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor uses the global provider initialized by the ADOT Java agent with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. The layer ARN follows the format: ``` -arn:aws:lambda::901920570463:layer:aws-otel-java-agent--ver-1-32-0:6 +arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava: ``` +> **Note:** The layer is regional — the account ID and version vary by region. Find the current per-region ARN in the [ADOT Java instrumentation releases](https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest). + **CloudFormation / SAM:** ```yaml @@ -63,10 +66,14 @@ MyFunction: Type: AWS::Serverless::Function Properties: Tracing: Active + LoggingConfig: + LogFormat: JSON Layers: - - !Sub - - arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-agent-${Arch}-ver-1-32-0:6 - - Arch: amd64 + - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:16 + Environment: + Variables: + AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument + OTEL_JAVAAGENT_EXTENSIONS: /var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar ``` **AWS CLI:** @@ -74,39 +81,30 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::901920570463:layer:aws-otel-java-agent-amd64-ver-1-32-0:6" + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:16" \ + --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument,OTEL_JAVAAGENT_EXTENSIONS=/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar}" ``` +Set `OTEL_JAVAAGENT_EXTENSIONS` to the deployed OTel plugin jar that contains this plugin's `META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider` entry. + ### 2. AWS X-Ray Active Tracing Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution. **AWS Console:** Lambda > Configuration > Monitoring and operations tools > Active tracing > Enable -**AWS CLI:** - -```bash -aws lambda update-function-configuration \ - --function-name your-function-name \ - --tracing-config Mode=Active -``` - **CloudFormation / SAM:** ```yaml MyFunction: - Type: AWS::Lambda::Function + Type: AWS::Serverless::Function Properties: - TracingConfig: - Mode: Active + Tracing: Active ``` ### 3. In Your Lambda Handler ```java -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -116,13 +114,7 @@ public class MyHandler extends DurableHandler { @Override protected DurableConfig createConfiguration() { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = new InvocationOtelPlugin( - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); } @Override @@ -149,10 +141,11 @@ The function's execution role needs the `AWSXRayDaemonWriteAccess` managed polic ## Trace Structure -The plugin creates spans at three levels: +The plugin creates spans at four levels: ``` -invocation +Workflow (deterministic ID, exported once on terminal invocation) +Invocation ├── fetch-data │ └── fetch-data attempt 1 ├── cool-down @@ -160,7 +153,8 @@ invocation └── process attempt 1 ``` -- **Invocation span** (`SpanKind.SERVER`) — one per Lambda invocation, creates a distinct X-Ray service node +- **Workflow span** — one logical span per durable execution with a deterministic ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED). Serves as a correlation anchor across invocations. +- **Invocation span** — one per Lambda invocation - **Operation span** — one per durable operation, named after your step/wait names - **Attempt span** — one per user function execution (retries produce additional attempt spans) @@ -203,18 +197,31 @@ When `enableMdc` is true (default), the plugin injects these fields into SLF4J M | MDC Key | Description | |---------|-------------| -| `traceId` | W3C trace ID (32 hex chars) | -| `spanId` | Current span ID (16 hex chars) | +| `trace_id` | W3C trace ID (32 hex chars) | +| `span_id` | Current span ID (16 hex chars) | | `traceSampled` | Whether the trace is sampled (true/false) | -These appear automatically in structured log output (Log4j2 JSON, Logback JSON) for log-trace correlation. +The `trace_id` is also injected at invocation start so handler-level logs (between steps) include it. + +Configure your logging framework (e.g., Log4j2) to include MDC fields in the output. For example, using `JsonLayout`: + +```xml + + + +``` + +With Lambda's `LoggingConfig: JSON` (required for durable functions), CloudWatch parses the JSON and X-Ray correlates logs via `requestId` (injected by the core SDK's `DurableLogger`). ## Configuration ### Constructor Options ```java -// Default: X-Ray context extraction, MDC enabled +// Default: ADOT Java agent global provider, X-Ray context extraction, MDC enabled +new InvocationOtelPlugin(); + +// Custom tracer provider pipeline new InvocationOtelPlugin(tracerProviderBuilder); // Custom context extractor, MDC enabled @@ -226,9 +233,19 @@ new InvocationOtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Required | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Not used by `new InvocationOtelPlugin()`; the default constructor uses the ADOT Java agent provider | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | -| `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | +| `enableMdc` | If true, injects `trace_id`/`span_id`/`traceSampled` into SLF4J MDC | `true` | + +## Known Limitations + +### X-Ray Segments Timeline (ungrouped view) + +The plugin's spans do not appear as nested subsegments of the Lambda platform segment in the ungrouped "Segments Timeline" view. This is because the ADOT collector's OTLP-to-X-Ray conversion cannot attach exported spans as subsegments of the Lambda service's native X-Ray segment (created outside the OTLP pipeline). Use the **"Group by nodes"** view to see the full span hierarchy. + +### Workflow Span + +The Workflow span appears as a separate root segment in the X-Ray trace because it uses `setNoParent()` with a deterministic span ID. This is expected — it serves as a correlation anchor across invocations. ## Verification @@ -236,24 +253,24 @@ After deploying your function with the plugin configured: 1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle. -2. **Check CloudWatch console** — Navigate to CloudWatch > Traces. You should see a trace with: - - An invocation span per Lambda invocation +2. **Check CloudWatch console** — Navigate to CloudWatch > Traces. Enable "Group by nodes" to see: + - A Workflow span covering the entire execution + - An Invocation span per Lambda invocation - Child spans for each durable operation (named after your step names) - All invocations of the same execution grouped under one trace ID -3. **Check log correlation** — Verify that your logs include `traceId` and `spanId` fields matching the spans in the trace view. +3. **Check log correlation** — Verify that the Logs section at the bottom of the trace view shows both platform logs and application logs correlated with the trace. ### Troubleshooting | Symptom | Likely Cause | |---------|-------------| -| No traces appear | ADOT layer not added to the function | +| No traces appear | ADOT layer not added, or `AWS_LAMBDA_EXEC_WRAPPER` not set | | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Two service nodes in trace map | `AWS_LAMBDA_EXEC_WRAPPER` is set — remove it (see note above) | - -> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER`. The wrapper attaches the auto-instrumentation agent which creates a separate TracerProvider. Since the plugin needs its own TracerProvider (for deterministic ID generation), having two providers causes X-Ray to render disconnected service nodes. +| Plugin spans missing but Lambda/runtime spans appear | Plugin jar not configured in `OTEL_JAVAAGENT_EXTENSIONS` | +| Logs not correlated | Ensure `LoggingConfig: JSON` is set and logging framework outputs MDC fields | ## Local Development @@ -271,7 +288,8 @@ var otelPlugin = new InvocationOtelPlugin( - Java 17+ - AWS Durable Execution SDK for Java 2.0.0+ -- OpenTelemetry SDK 1.30.0+ +- OpenTelemetry SDK 1.64.0+ (only for custom TracerProvider path) +- ADOT Lambda Layer `AWSOpenTelemetryDistroJava` (for the no-arg constructor path) ## License diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml index 3c759ab20..f145e1cc1 100644 --- a/otel-plugin/pom.xml +++ b/otel-plugin/pom.xml @@ -40,6 +40,13 @@ ${opentelemetry.version} provided + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + ${opentelemetry.version} + provided + diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java index b74ef845b..fa1794eeb 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java @@ -30,6 +30,10 @@ public class DeterministicIdGenerator implements IdGenerator { private static final IdGenerator RANDOM = IdGenerator.random(); + private static final String PROPERTY_PREFIX = "software.amazon.lambda.durable.otel."; + private static final String EXTRACTED_TRACE_ID_PROPERTY = PROPERTY_PREFIX + "extractedTraceId"; + private static final String DURABLE_EXECUTION_ARN_PROPERTY = PROPERTY_PREFIX + "durableExecutionArn"; + private static final String PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingSpanOperationId."; private final AtomicReference extractedTraceId = new AtomicReference<>(null); private final AtomicReference arnDerivedTraceId = new AtomicReference<>(null); @@ -45,6 +49,7 @@ public class DeterministicIdGenerator implements IdGenerator { */ public void setExtractedTraceId(String traceId) { this.extractedTraceId.set(traceId); + setOrClearProperty(EXTRACTED_TRACE_ID_PROPERTY, traceId); } /** @@ -55,7 +60,8 @@ public void setExtractedTraceId(String traceId) { */ public void setDurableExecutionArn(String arn) { this.durableExecutionArn.set(arn); - this.arnDerivedTraceId.set(generateTraceIdFromArn(arn)); + this.arnDerivedTraceId.set(arn != null ? generateTraceIdFromArn(arn) : null); + setOrClearProperty(DURABLE_EXECUTION_ARN_PROPERTY, arn); } /** @@ -65,6 +71,7 @@ public void setDurableExecutionArn(String arn) { */ public void setNextSpanOperationId(String operationId) { this.pendingSpanOperationId.set(operationId); + setOrClearProperty(pendingSpanOperationIdProperty(), operationId); } /** @@ -110,11 +117,18 @@ public String generateWorkflowSpanId() { public String generateTraceId() { // Priority 1: extracted from X-Ray header (backend propagates same Root across invocations) var extracted = extractedTraceId.get(); + if (extracted == null) { + extracted = System.getProperty(EXTRACTED_TRACE_ID_PROPERTY); + } if (extracted != null) { return extracted; } // Priority 2: deterministic from execution ARN (local tests, non-Lambda) var arnDerived = arnDerivedTraceId.get(); + if (arnDerived == null) { + var arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY); + arnDerived = arn != null ? generateTraceIdFromArn(arn) : null; + } if (arnDerived != null) { return arnDerived; } @@ -130,8 +144,12 @@ public String generateSpanId() { return raw; } var operationId = pendingSpanOperationId.get(); + if (operationId == null) { + operationId = System.getProperty(pendingSpanOperationIdProperty()); + } if (operationId != null) { pendingSpanOperationId.remove(); + System.clearProperty(pendingSpanOperationIdProperty()); return generateSpanIdFromOperation(operationId); } return RANDOM.generateSpanId(); @@ -148,6 +166,9 @@ private String generateTraceIdFromArn(String arn) { */ private String generateSpanIdFromOperation(String operationId) { var arn = durableExecutionArn.get(); + if (arn == null) { + arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY); + } var input = arn != null ? arn + ":" + operationId : operationId; var hash = sha256(input); return hash.substring(0, 16); @@ -166,4 +187,26 @@ private static String sha256(String input) { throw new IllegalStateException("SHA-256 not available", e); } } + + static void clearSharedStateForTest() { + System.clearProperty(EXTRACTED_TRACE_ID_PROPERTY); + System.clearProperty(DURABLE_EXECUTION_ARN_PROPERTY); + System.getProperties().stringPropertyNames().stream() + .filter(name -> name.startsWith(PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX)) + .toList() + .forEach(System::clearProperty); + } + + private static String pendingSpanOperationIdProperty() { + return PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX + + Thread.currentThread().getId(); + } + + private static void setOrClearProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 9111b80d2..e8224e77b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -4,8 +4,8 @@ import static software.amazon.lambda.durable.otel.SpanAttributes.*; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanContext; @@ -14,16 +14,19 @@ import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; -import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.semconv.ServiceAttributes; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; @@ -46,6 +49,18 @@ *
  • Attempt span — one per user function execution (step attempt, child context run) * * + *

    Workflow span behavior by provider: + * + *

    + * *

    Trace ID resolution: * *

      @@ -58,12 +73,20 @@ *

      Requires the ADOT Lambda Layer for trace export. Configure with: * *

        - *
      • Lambda Layer: {@code aws-otel-java-agent} (provides the OTLP collector extension) + *
      • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the ADOT Java agent and export pipeline) *
      • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) *
      * - *

      Note: Do NOT set {@code AWS_LAMBDA_EXEC_WRAPPER}. The collector extension runs independently. The wrapper would - * attach the auto-instrumentation agent which creates a competing TracerProvider. + *

      When using {@link #InvocationOtelPlugin()}, the plugin requires + * {@code OtelPluginAutoConfigurationCustomizerProvider} to have been installed by the OpenTelemetry Java agent and uses + * the global provider directly. + * + *

      X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation, + * operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of + * the OTLP-to-X-Ray conversion: the ADOT collector cannot attach OTLP-exported spans as subsegments of the Lambda + * service's native X-Ray segment because that segment is created outside the OTLP pipeline. Use the "Group by nodes" + * view to see the full span hierarchy correctly — it stitches all spans together by trace ID and parent-child + * relationships regardless of segment boundaries. * *

      Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -77,7 +100,7 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { private static final String INSTRUMENTATION_NAME = "aws-durable-execution-sdk-java"; private static final String DEFAULT_WORKFLOW_SPAN_NAME = "Workflow"; - private final SdkTracerProvider tracerProvider; + private final SdkTracerProvider sdkTracerProvider; private final Tracer tracer; private final DeterministicIdGenerator idGenerator; private final ContextExtractor contextExtractor; @@ -105,12 +128,13 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { *

      Uses the provided tracer provider builder. Customers configure exporters and span processors on the builder — * the plugin handles ID generation. * - *

      For ADOT layer usage, configure with an OTLP exporter: + *

      For ADOT Java agent usage, prefer {@link #InvocationOtelPlugin()} with the plugin jar configured through + * {@code OTEL_JAVAAGENT_EXTENSIONS}. Use this builder constructor when you want to own the exporter pipeline: * * 

      {@code
-     * var otlpExporter = OtlpGrpcSpanExporter.getDefault(); // sends to localhost:4317
+     * var exporter = LoggingSpanExporter.create();
      * var plugin = new InvocationOtelPlugin(
-     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
+     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
      * }
      * * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) @@ -119,6 +143,16 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { this(tracerProviderBuilder, new XRayContextExtractor(), true); } + /** + * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled. + * + *

      Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by + * {@code OtelPluginAutoConfigurationCustomizerProvider}. + */ + public InvocationOtelPlugin() { + this(getDefaultTracerProvider(), createDefaultIdGenerator()); + } + /** * Creates an OTel plugin with a custom context extractor, MDC enabled. * @@ -156,17 +190,24 @@ public InvocationOtelPlugin( String workflowSpanName) { this.idGenerator = new DeterministicIdGenerator(); - // Set service.name to "invocation" for the exported spans' resource. - var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); - tracerProviderBuilder.addResource(resource); - - this.tracerProvider = tracerProviderBuilder.setIdGenerator(idGenerator).build(); - this.tracer = tracerProvider.get(INSTRUMENTATION_NAME); + this.sdkTracerProvider = + tracerProviderBuilder.setIdGenerator(idGenerator).build(); + this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); this.contextExtractor = contextExtractor; this.enableMdc = enableMdc; this.workflowSpanName = workflowSpanName != null ? workflowSpanName : DEFAULT_WORKFLOW_SPAN_NAME; } + private InvocationOtelPlugin(TracerProvider tracerProvider, DeterministicIdGenerator idGenerator) { + this.idGenerator = idGenerator; + this.sdkTracerProvider = getSdkTracerProviderForFlush(tracerProvider); + this.tracer = tracerProvider.get(INSTRUMENTATION_NAME); + + this.contextExtractor = new XRayContextExtractor(); + this.enableMdc = true; + this.workflowSpanName = DEFAULT_WORKFLOW_SPAN_NAME; + } + // ─── Invocation hooks ──────────────────────────────────────────────── @Override @@ -178,10 +219,15 @@ public void onInvocationStart(InvocationInfo info) { // Extract trace context from environment (X-Ray header) var extractedContext = contextExtractor.extract(); + if (extractedContext == null) { + extractedContext = extractCurrentSpanContext(); + } if (extractedContext != null) { // Use the X-Ray trace ID — backend propagates same Root across all invocations idGenerator.setExtractedTraceId(extractedContext.traceId()); + } else { + idGenerator.setExtractedTraceId(null); } // If no extracted context, idGenerator falls back to ARN-derived trace ID @@ -200,8 +246,8 @@ public void onInvocationStart(InvocationInfo info) { // Determine parent context for the invocation span. Context parentContext; if (extractedContext != null && extractedContext.parentSpanId() != null) { - // X-Ray header has parent — create the invocation span as a child of that segment. - // This connects our OTLP-exported spans to the Lambda service's X-Ray segments. + // Reconstruct a remote parent from the extracted trace context (X-Ray header or current span). + // This connects plugin spans to the Lambda service's X-Ray segments. var parentSpanContext = SpanContext.createFromRemoteParent( extractedContext.traceId(), extractedContext.parentSpanId(), @@ -224,10 +270,22 @@ public void onInvocationStart(InvocationInfo info) { } invocationSpan = spanBuilder.startSpan(); + + // Inject MDC on the handler thread so handler-level logs (between steps) have trace context. + // This runs on the same thread as context.getLogger() calls in the handler. + if (enableMdc) { + var traceId = idGenerator.generateTraceId(); + MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId); + } } @Override public void onInvocationEnd(InvocationEndInfo info) { + // Clear invocation-level MDC (set in onInvocationStart on the handler thread) + if (enableMdc) { + MdcSpanEnricher.clear(); + } + if (invocationSpan == null) return; // End still-open operation spans without stamping a status — no terminal @@ -301,10 +359,12 @@ public void onInvocationEnd(InvocationEndInfo info) { workflowSpan = null; } - // Flush spans before Lambda freezes - var flushResult = tracerProvider.forceFlush().join(5, java.util.concurrent.TimeUnit.SECONDS); - if (!flushResult.isSuccess()) { - logger.warn("OTel span flush failed or timed out — some spans may be lost"); + if (sdkTracerProvider != null) { + // Flush spans before Lambda freezes + var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) { + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } } } @@ -489,9 +549,9 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { scope.close(); } - // Clear MDC after user function completes + // Clear span-level MDC after user function completes (keep trace_id for handler-level logs between steps) if (enableMdc) { - MdcSpanEnricher.clear(); + MDC.remove(MdcSpanEnricher.MDC_SPAN_ID); } // CONTEXT operations don't have attempt spans — scope cleanup is all we need @@ -572,4 +632,82 @@ private static String attemptSpanName(String type, String subType, String name, private static String attemptKey(String operationId, Integer attempt) { return operationId + "-" + (attempt != null ? attempt : "ctx"); } + + private static ExtractedContext extractCurrentSpanContext() { + var spanContext = Span.current().getSpanContext(); + if (!spanContext.isValid()) { + return null; + } + return new ExtractedContext(spanContext.getTraceId(), spanContext.getSpanId()); + } + + private static TracerProvider getDefaultTracerProvider() { + validateAutoConfigurationCustomizerProviderInstalled(); + + var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); + if (globalTracerProvider == TracerProvider.noop()) { + throw new IllegalStateException("InvocationOtelPlugin() requires GlobalOpenTelemetry to be initialized by " + + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent."); + } + logger.info( + "InvocationOtelPlugin initialized from existing GlobalOpenTelemetry tracer provider {}; assuming " + + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider", + globalTracerProvider.getClass().getName()); + return globalTracerProvider; + } + + private static DeterministicIdGenerator createDefaultIdGenerator() { + // This is intentionally a separate instance from the SPI provider's generator. The Java agent extension and + // application may load this plugin in different class loaders, so DeterministicIdGenerator bridges invocation + // state through system properties that the SPI-installed generator can read when spans are started. + return new DeterministicIdGenerator(); + } + + private static void validateAutoConfigurationCustomizerProviderInstalled() { + if (OtelPluginAutoConfigurationState.isInstalled()) { + return; + } + throw new IllegalStateException( + "InvocationOtelPlugin() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the " + + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set " + + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing " + + "InvocationOtelPlugin(). " + + javaAgentExtensionsDiagnostic()); + } + + private static String javaAgentExtensionsDiagnostic() { + var propertyValue = System.getProperty("otel.javaagent.extensions"); + var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS"); + var configuredPath = propertyValue != null ? propertyValue : environmentValue; + return "otel.javaagent.extensions=" + + valueOrUnset(propertyValue) + + ", OTEL_JAVAAGENT_EXTENSIONS=" + + valueOrUnset(environmentValue) + + ", configured extension path exists=" + + extensionPathExists(configuredPath); + } + + private static String valueOrUnset(String value) { + return value != null ? value : ""; + } + + private static boolean extensionPathExists(String configuredPath) { + if (configuredPath == null || configuredPath.isBlank()) { + return false; + } + var firstPath = configuredPath.split(",", 2)[0]; + return Files.exists(Path.of(firstPath)); + } + + private static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvider) { + if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { + return sdkTracerProvider; + } + logger.info( + "InvocationOtelPlugin forceFlush is not available because GlobalOpenTelemetry provider {} is not an " + + "SdkTracerProvider visible to the application class loader; spans will rely on the " + + "provider's own flushing.", + tracerProvider.getClass().getName()); + return null; + } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java index 2b769e308..e5a49e1cc 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java @@ -9,13 +9,13 @@ * Injects OTel trace/span IDs into SLF4J MDC for log-trace correlation. * *

      When used with structured logging (Log4j2 JSON, Logback JSON), these MDC fields appear in every log line, enabling - * tools like CloudWatch Logs Insights and Datadog to correlate logs with traces. + * tools like CloudWatch Application Signals and Datadog to correlate logs with traces. * *

      MDC keys injected: * *

        - *
      • {@code traceId} — the W3C trace ID (32 hex chars) - *
      • {@code spanId} — the current span ID (16 hex chars) + *
      • {@code trace_id} — the W3C trace ID (32 hex chars) + *
      • {@code span_id} — the current span ID (16 hex chars) *
      • {@code traceSampled} — whether the trace is sampled (true/false) *
      * @@ -28,8 +28,8 @@ @Deprecated public final class MdcSpanEnricher { - public static final String MDC_TRACE_ID = "traceId"; - public static final String MDC_SPAN_ID = "spanId"; + public static final String MDC_TRACE_ID = "trace_id"; + public static final String MDC_SPAN_ID = "span_id"; public static final String MDC_TRACE_SAMPLED = "traceSampled"; private MdcSpanEnricher() {} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java new file mode 100644 index 000000000..23ac19ecd --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; + +/** + * Installs the durable-execution ID generator when the OpenTelemetry Java agent auto-configures the SDK. + * + * @deprecated This is a preview API that is experimental and may be changed or removed in future releases. + */ +@Deprecated +public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider { + + private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator(); + + @Override + public void customize(AutoConfigurationCustomizer autoConfiguration) { + OtelPluginAutoConfigurationState.markInstalled(); + autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); + } +} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java new file mode 100644 index 000000000..765489b69 --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +final class OtelPluginAutoConfigurationState { + + private static final String INSTALLED_PROPERTY = + "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; + + private OtelPluginAutoConfigurationState() {} + + static boolean isInstalled() { + return Boolean.getBoolean(INSTALLED_PROPERTY); + } + + static void markInstalled() { + System.setProperty(INSTALLED_PROPERTY, Boolean.TRUE.toString()); + } + + static void resetInstalledForTest() { + System.clearProperty(INSTALLED_PROPERTY); + } +} diff --git a/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider b/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider new file mode 100644 index 000000000..3aa2fb591 --- /dev/null +++ b/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider @@ -0,0 +1 @@ +software.amazon.lambda.durable.otel.OtelPluginAutoConfigurationCustomizerProvider diff --git a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java new file mode 100644 index 000000000..db99b5bdb --- /dev/null +++ b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package io.opentelemetry.javaagent.testing; + +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerBuilder; +import io.opentelemetry.api.trace.TracerProvider; + +public final class FakeJavaAgentTracerProvider implements TracerProvider { + + private final TracerProvider delegate; + + public FakeJavaAgentTracerProvider(TracerProvider agentTracerProvider) { + this.delegate = agentTracerProvider; + } + + @Override + public Tracer get(String instrumentationName) { + return delegate.get(instrumentationName); + } + + @Override + public Tracer get(String instrumentationName, String instrumentationVersion) { + return delegate.get(instrumentationName, instrumentationVersion); + } + + @Override + public TracerBuilder tracerBuilder(String instrumentationScopeName) { + return delegate.tracerBuilder(instrumentationScopeName); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java index bc1a5a90d..6133a3ddb 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -13,9 +14,15 @@ class DeterministicIdGeneratorTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); generator = new DeterministicIdGenerator(); } + @AfterEach + void tearDown() { + DeterministicIdGenerator.clearSharedStateForTest(); + } + @Test void generateTraceId_withoutArn_returnsRandom() { var id1 = generator.generateTraceId(); @@ -120,6 +127,18 @@ void generateSpanIdForOperation_isDeterministic() { assertEquals(id1, id2); } + @Test + void generatedIds_areSharedAcrossGeneratorInstances() { + var pluginGenerator = new DeterministicIdGenerator(); + var agentGenerator = new DeterministicIdGenerator(); + + pluginGenerator.setDurableExecutionArn("arn:exec1"); + pluginGenerator.setNextSpanOperationId("op-1"); + + assertEquals(pluginGenerator.generateTraceId(), agentGenerator.generateTraceId()); + assertEquals(pluginGenerator.generateSpanIdForOperation("op-1"), agentGenerator.generateSpanId()); + } + @Test void traceId_isValidHex() { generator.setDurableExecutionArn("arn:exec1"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java index c6961a827..d4e85c241 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java @@ -4,7 +4,12 @@ import static org.junit.jupiter.api.Assertions.*; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; @@ -13,6 +18,7 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -34,6 +40,8 @@ class InvocationOtelPluginIntegrationTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); var plugin = new InvocationOtelPlugin( @@ -44,6 +52,13 @@ void setUp() { otelConfig = DurableConfig.builder().withPlugins(plugin).build(); } + @AfterEach + void tearDown() { + GlobalOpenTelemetry.resetForTest(); + DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + @Test void simpleStep_producesInvocationAndOperationAndAttemptSpans() { var runner = LocalDurableTestRunner.create( @@ -511,4 +526,71 @@ void waitForCondition_producesSpansWithAttempts() { "Should have waitForCondition span. Got: " + spans.stream().map(SpanData::getName).toList()); } + + @Test + void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + var defaultConfig = + DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input), + defaultConfig); + + var result = runner.runUntilComplete("World"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + + var spans = globalExporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 4, "Expected at least 4 spans, got " + spans.size()); + assertSpanExists(spans, "Invocation"); + assertSpanExists(spans, "global-step"); + assertSpanExists(spans, "global-step attempt 1"); + } + + @Test + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var javaAgentIdGenerator = new DeterministicIdGenerator(); + var sdkTracerProvider = SdkTracerProvider.builder() + .setIdGenerator(javaAgentIdGenerator) + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + GlobalOpenTelemetry.set(new OpenTelemetry() { + @Override + public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { + return javaAgentTracerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + + var defaultConfig = + DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> ctx.step("javaagent-step", String.class, stepCtx -> "Hello " + input), + defaultConfig); + + var result = runner.runUntilComplete("World"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + + var spans = globalExporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 4, "Expected at least 4 spans, got " + spans.size()); + assertSpanExists(spans, "Invocation"); + assertSpanExists(spans, "javaagent-step"); + assertSpanExists(spans, "javaagent-step attempt 1"); + } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 2536d0ad3..39971f667 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -3,16 +3,38 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; +import java.util.ServiceLoader; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.plugin.*; @@ -23,6 +45,8 @@ class InvocationOtelPluginTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); plugin = new InvocationOtelPlugin( @@ -31,6 +55,167 @@ void setUp() { false); } + @AfterEach + void tearDown() { + GlobalOpenTelemetry.resetForTest(); + DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + + @Test + void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() { + GlobalOpenTelemetry.resetForTest(); + + var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new); + + assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + assertTrue(error.getMessage().contains("OTEL_JAVAAGENT_EXTENSIONS")); + } + + @Test + void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + + var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new); + + assertTrue(error.getMessage().contains("GlobalOpenTelemetry")); + assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + } + + @Test + void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + var defaultPlugin = new InvocationOtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", null, false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var spans = globalExporter.getFinishedSpanItems(); + // Plugin creates Workflow + Invocation + operation spans + assertEquals(3, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + } + + @Test + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var javaAgentIdGenerator = new DeterministicIdGenerator(); + var sdkTracerProvider = SdkTracerProvider.builder() + .setIdGenerator(javaAgentIdGenerator) + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + GlobalOpenTelemetry.set(new OpenTelemetry() { + @Override + public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { + return javaAgentTracerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + + var defaultPlugin = new InvocationOtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", null, false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var spans = globalExporter.getFinishedSpanItems(); + // Plugin creates Workflow + Invocation + operation spans + assertEquals(3, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + var expectedIds = new DeterministicIdGenerator(); + expectedIds.setDurableExecutionArn("arn:exec1"); + var stepSpan = spans.stream() + .filter(span -> span.getName().equals("step")) + .findFirst() + .orElseThrow(); + assertEquals(expectedIds.generateSpanIdForOperation("op-1"), stepSpan.getSpanId()); + } + + @Test + void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() { + OtelPluginAutoConfigurationState.resetInstalledForTest(); + var exporter = InMemorySpanExporter.create(); + var autoConfiguration = mock(AutoConfigurationCustomizer.class); + when(autoConfiguration.addTracerProviderCustomizer(any())).thenReturn(autoConfiguration); + + new OtelPluginAutoConfigurationCustomizerProvider().customize(autoConfiguration); + assertTrue(OtelPluginAutoConfigurationState.isInstalled()); + + @SuppressWarnings("unchecked") + var customizer = ArgumentCaptor.forClass(BiFunction.class); + verify(autoConfiguration).addTracerProviderCustomizer(customizer.capture()); + + var pluginGenerator = new DeterministicIdGenerator(); + pluginGenerator.setDurableExecutionArn("arn:spi"); + pluginGenerator.setNextSpanOperationId("op-spi"); + + @SuppressWarnings("unchecked") + var tracerProviderCustomizer = + (BiFunction) + customizer.getValue(); + var tracerProvider = tracerProviderCustomizer + .apply(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), null) + .build(); + + var span = tracerProvider.get("test").spanBuilder("step").startSpan(); + span.end(); + tracerProvider.forceFlush().join(5, TimeUnit.SECONDS); + + var spans = exporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + assertEquals( + pluginGenerator.generateSpanIdForOperation("op-spi"), + spans.get(0).getSpanId()); + } + + @Test + void autoConfigurationCustomizerProvider_isRegisteredAsServiceProvider() { + assertTrue(ServiceLoader.load(AutoConfigurationCustomizerProvider.class).stream() + .anyMatch(provider -> provider.type().equals(OtelPluginAutoConfigurationCustomizerProvider.class))); + } + + @Test + void invocationStart_usesCurrentSpanContext_whenExtractorReturnsNull() { + var traceId = "5759e988bd862e3fe1be46a994272793"; + var parentSpanId = "53995c3f42cd8ad8"; + var parentSpanContext = + SpanContext.create(traceId, parentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + + try (var ignored = Span.wrap(parentSpanContext).makeCurrent()) { + plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + } + plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var invocationSpan = spanExporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow(); + assertEquals(traceId, invocationSpan.getTraceId()); + assertEquals(parentSpanId, invocationSpan.getParentSpanId()); + } + @Test void invocationStart_and_end_createsSpan() { plugin.onInvocationStart(new InvocationInfo( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java index 0c896e141..f7e7174a3 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java @@ -64,12 +64,17 @@ void plugin_withMdcEnabled_setsFieldsInMdc() { plugin.onUserFunctionEnd(new UserFunctionEndInfo( "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), false, 1, true, null)); - // MDC should be cleared after onUserFunctionEnd - assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); - assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID)); - assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED)); + // After onUserFunctionEnd: span_id is cleared, but trace_id remains for handler-level logs between steps + assertNotNull(MDC.get(MdcSpanEnricher.MDC_TRACE_ID), "trace_id should persist between steps"); + assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID), "span_id should be cleared after step"); + assertNotNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED), "trace_flags should persist between steps"); plugin.onInvocationEnd( new InvocationEndInfo("req-1", "arn:exec-mdc-test", true, InvocationStatus.SUCCEEDED, null)); + + // After onInvocationEnd: all MDC fields are cleared + assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); + assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID)); + assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED)); } }