diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a2a4f..05e2bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `Langfuse::Testing` module (`require "langfuse/testing"`) — include in test classes to capture spans in memory without network calls or private ivar access; provides `emitted_langfuse_spans` and `reset_langfuse` instance methods + ## [0.10.1] - 2026-05-05 ### Changed diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..15cd1e3 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,108 @@ +# Testing Guide + +This guide covers how to safely incorporate Langfuse into your application test suite. + +## Overview + +Langfuse can be configured to use an in-memory exporter in your test environemnt so that you can run real code end-to-end without making real network requests. This allows you to exercise the full Langfuse observation pipeline and write assertions against produced spans. Stubs break as the SDK evolves and they test the wrong thing — that *a method was called*, not that *the right telemetry was produced*. By asserting against real spans your tests can catch regressions iinvolving span naming, attribute values, and export filtering that stubs would silently miss. + +## Setup + +Add `require "langfuse/testing"` to your test helper. Requiring the file automatically wires an in-memory exporter into the SDK — no extra configuration beyond the usual keys. + +```ruby +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end +``` + +Then include `Langfuse::Testing` wherever you need span assertions. It provides two instance methods: + +| Method | Description | +|---|---| +| `emitted_langfuse_spans` | Instance method — flushes pending batches and returns all recorded spans | +| `reset_langfuse` | Module method — flushes and discards all recorded spans; call this in test setup | + +## RSpec + +```ruby +# spec/support/langfuse_helpers.rb +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end + +RSpec.configure do |config| + config.include Langfuse::Testing, :langfuse + config.before(:each, :langfuse) { reset_langfuse } +end +``` + +Tag examples that need span assertions with `:langfuse`: + +```ruby +RSpec.describe SummarizationService, :langfuse do + it "emits a generation span" do + SummarizationService.new.call("hello world") + + span = emitted_langfuse_spans.find { |s| s.name == "summarize" } + expect(span).not_to be_nil + expect(span.attributes["langfuse.observation.type"]).to eq("generation") + end +end +``` + +## Minitest + +```ruby +# test/support/langfuse_test_helper.rb +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end + +module LangfuseTestHelper + include Langfuse::Testing + + def setup + super + reset_langfuse + end +end +``` + +Include it in any test class that needs it: + +```ruby +class SummarizationServiceTest < ActiveSupport::TestCase + include LangfuseTestHelper + + test "emits a generation span" do + SummarizationService.new.call("hello world") + + span = emitted_langfuse_spans.find { |s| s.name == "summarize" } + assert span, "expected a summarize span" + assert_equal "generation", span.attributes["langfuse.observation.type"] + end +end +``` + +## Available Span Data + +`emitted_langfuse_spans` returns an array of [`OpenTelemetry::SDK::Trace::SpanData`](https://opentelemetry.io/docs/specs/otel/trace/sdk/) values. Useful fields: + +| Field | Description | +|---|---| +| `span.name` | Span name passed to `Langfuse.observe` | +| `span.attributes` | Hash of all attributes set on the span | +| `span.status.code` | `:ok`, `:error`, or `:unset` | +| `span.events` | Array of timed events recorded on the span | +| `span.parent_span_id` | Non-zero for child spans | +| `span.start_timestamp` / `span.end_timestamp` | Monotonic timestamps | diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index e4bfaa1..ce2aefd 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -26,6 +26,19 @@ class << self # @return [OpenTelemetry::SDK::Trace::TracerProvider, nil] The configured internal tracer provider attr_reader :tracer_provider + # @return [OpenTelemetry::Exporter::OTLP::Exporter, OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter, nil] + # The active span exporter for the current provider — +OTLP::Exporter+ in production, + # +InMemorySpanExporter+ when +test_mode+ is enabled. +nil+ until the first +setup+ + # call completes and cleared on +shutdown+. + # @api private + attr_reader :span_exporter + + # When true, +build_exporter+ returns a fresh +InMemorySpanExporter+ on each provider + # build instead of the OTLP exporter. Set by +Langfuse::Testing+ at require time and + # survives +shutdown+ cycles. + # @api private + attr_accessor :test_mode + # Initialize Langfuse's internal tracer provider without mutating global OpenTelemetry state. # # @param config [Langfuse::Config] The Langfuse configuration @@ -37,8 +50,9 @@ def setup(config) candidate_provider = nil provider = nil created = false - candidate_provider = build_tracer_provider(config) - provider, created = publish_provider(candidate_provider, tracing_config_snapshot(config)) + candidate_exporter = build_exporter(config) + candidate_provider = build_tracer_provider(config, candidate_exporter) + provider, created = publish_provider(candidate_provider, candidate_exporter, tracing_config_snapshot(config)) unless created candidate_provider.shutdown(timeout: 30) return existing_provider_for(config) @@ -61,6 +75,7 @@ def shutdown(timeout: 30) provider = @tracer_provider @tracer_provider = nil @config_snapshot = nil + @span_exporter = nil end provider&.shutdown(timeout: timeout) end @@ -95,7 +110,7 @@ def existing_provider_for(config) @tracer_provider end - def publish_provider(provider, snapshot) + def publish_provider(provider, exporter, snapshot) created = false current = nil @@ -106,6 +121,7 @@ def publish_provider(provider, snapshot) else @tracer_provider = provider @config_snapshot = snapshot + @span_exporter = exporter current = provider created = true end @@ -120,23 +136,24 @@ def rollback_provider(provider) @tracer_provider = nil @config_snapshot = nil + @span_exporter = nil end provider.shutdown(timeout: 1) rescue StandardError nil end - def build_tracer_provider(config) + def build_tracer_provider(config, exporter) provider = OpenTelemetry::SDK::Trace::TracerProvider.new( sampler: build_sampler(config.sample_rate) ) - provider.add_span_processor( - SpanProcessor.new(config: config, exporter: build_exporter(config)) - ) + provider.add_span_processor(SpanProcessor.new(config: config, exporter: exporter)) provider end def build_exporter(config) + return OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new if @test_mode + OpenTelemetry::Exporter::OTLP::Exporter.new( endpoint: "#{config.base_url}/api/public/otel/v1/traces", headers: build_headers(config.public_key, config.secret_key), diff --git a/lib/langfuse/testing.rb b/lib/langfuse/testing.rb new file mode 100644 index 0000000..c581d9c --- /dev/null +++ b/lib/langfuse/testing.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "langfuse" +require "opentelemetry/sdk" + +module Langfuse + # Mixin for capturing Langfuse spans in tests without real network calls. + # See docs/TESTING.md for setup and usage examples. + module Testing + # Set test_mode once at require time. It survives shutdown cycles so every + # provider build gets a fresh InMemorySpanExporter automatically. + OtelSetup.test_mode = true + + # Discard all recorded spans so the next test starts clean. + # Flushes pending batches first to avoid cross-test span leakage. + # + # @return [void] + def reset_langfuse + Langfuse.force_flush + OtelSetup.span_exporter&.reset + end + + # Return all spans emitted since the last reset, flushing any pending batches first. + # + # @return [Array] + def emitted_langfuse_spans + Langfuse.force_flush + OtelSetup.span_exporter&.finished_spans || [] + end + end +end diff --git a/spec/langfuse/otel_setup_spec.rb b/spec/langfuse/otel_setup_spec.rb index 7f76798..ece2c9c 100644 --- a/spec/langfuse/otel_setup_spec.rb +++ b/spec/langfuse/otel_setup_spec.rb @@ -79,6 +79,18 @@ expect(described_class.setup(config)).to equal(existing_provider) end + it "does not overwrite span_exporter when losing the publish race" do + described_class.setup(config) + winning_exporter = described_class.span_exporter + + losing_exporter = instance_double(OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter) + losing_provider = instance_double(OpenTelemetry::SDK::Trace::TracerProvider) + _, created = described_class.send(:publish_provider, losing_provider, losing_exporter, {}) + + expect(created).to be false + expect(described_class.span_exporter).to equal(winning_exporter) + end + it "validates should_export_span in setup" do config.should_export_span = "bad" @@ -88,6 +100,22 @@ ) end + it "uses an in-memory exporter when test_mode is set" do + original_test_mode = described_class.test_mode + described_class.test_mode = true + allow(described_class).to receive(:build_exporter).and_call_original + + described_class.setup(config) + described_class.tracer_provider.tracer(Langfuse::LANGFUSE_TRACER_NAME).start_span("test-span").finish + described_class.force_flush(timeout: 1) + + expect(described_class.span_exporter).to be_a(OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter) + expect(described_class.span_exporter.finished_spans.map(&:name)).to eq(["test-span"]) + ensure + described_class.test_mode = original_test_mode + described_class.shutdown(timeout: 1) + end + context "with sample_rate below 1.0" do before do config.sample_rate = 0.1 diff --git a/spec/langfuse/testing_spec.rb b/spec/langfuse/testing_spec.rb new file mode 100644 index 0000000..26955c8 --- /dev/null +++ b/spec/langfuse/testing_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "spec_helper" +require "langfuse/testing" + +RSpec.describe Langfuse::Testing do + include described_class + + before { reset_langfuse } + + it "sets test_mode at require time" do + expect(Langfuse::OtelSetup.test_mode).to be true + end + + describe "#emitted_langfuse_spans" do + it "returns finished spans after flushing" do + Langfuse.observe("test-span").end + + expect(emitted_langfuse_spans.map(&:name)).to eq(["test-span"]) + end + + it "returns an empty array when no spans have been recorded" do + expect(emitted_langfuse_spans).to eq([]) + end + end + + describe "#reset_langfuse" do + it "discards all recorded spans" do + Langfuse.observe("test-span").end + + emitted_langfuse_spans # flush so span lands in exporter + + reset_langfuse + + expect(Langfuse::OtelSetup.span_exporter.finished_spans).to be_empty + end + end +end