From 120e8d6c52a6a3fe062c611f95ffc3ea75d234d8 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:11:25 +0300 Subject: [PATCH 01/13] feat: the shared retry policy core, the baked flag and the day widening Add Dexpace::Resilience -- Policy (the two-axis classifier consult, the backoff calculator, the total pacing-header parser over the private PacingParsers, the retry-count resolver, the recovery-only budget and RETRY-12's constants), Resend.eligible? (the re-sendability gate over phase 1's idempotent set and phase 3b's replayability), and the frozen RetrySettings both retry stacks build from, the first reader of 5a's Keys::MAX_RETRY_ATTEMPTS -- plus the flat Dexpace::RetryPredicateError. Widen two earlier-phase files in place: ProtocolError bakes XCUT-5's retryable_by_status? from 5a's Retryability at construction (phase 4b's postponement; deliberately not #retryable?, P6-10), and HTTPDate's day group becomes (\d{1,2}) so RETRY-15's single-digit day parses through the one RFC 1123 parser (R1); 5a's rejection loop swaps that element for the bare-date row CFG-31 still refuses. One calculator, two budget policies: backoff_delay takes no total-timeout and budget_remaining is a separate function only the recovery engine names (R6, P6-5). A zero initial delay answers 0.0 before the power is taken, because 0.0 * Infinity is NaN and [NaN, 8.0].min raises (P6-53). --- gems/dexpace-core/lib/dexpace.rb | 14 + .../lib/dexpace/error/protocol_error.rb | 22 +- .../dexpace/error/retry_predicate_error.rb | 21 ++ gems/dexpace-core/lib/dexpace/http_date.rb | 26 +- .../lib/dexpace/resilience/pacing_parsers.rb | 77 +++++ .../lib/dexpace/resilience/policy.rb | 280 ++++++++++++++++++ .../lib/dexpace/resilience/resend.rb | 40 +++ .../lib/dexpace/resilience/retry_settings.rb | 189 ++++++++++++ .../sig/dexpace/error/protocol_error.rbs | 5 + .../dexpace/error/retry_predicate_error.rbs | 5 + .../sig/dexpace/resilience/pacing_parsers.rbs | 18 ++ .../sig/dexpace/resilience/policy.rbs | 36 +++ .../sig/dexpace/resilience/resend.rbs | 9 + .../sig/dexpace/resilience/retry_settings.rbs | 56 ++++ .../test/dexpace/http_date_test.rb | 2 +- gems/dexpace-core/test/dexpace_test.rb | 13 +- 16 files changed, 795 insertions(+), 18 deletions(-) create mode 100644 gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/policy.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/resend.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb create mode 100644 gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/policy.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/resend.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs diff --git a/gems/dexpace-core/lib/dexpace.rb b/gems/dexpace-core/lib/dexpace.rb index b8b0cde..16119fd 100644 --- a/gems/dexpace-core/lib/dexpace.rb +++ b/gems/dexpace-core/lib/dexpace.rb @@ -192,6 +192,20 @@ require_relative "dexpace/instrumentation/step" require_relative "dexpace/instrumentation/async_step" +# Phase 6a: the retry layer, in dependency order -- after 5b's block, because the settings read +# the clock and the configuration slot (5a) and the steps emit through the logging facade (5b) +# and the HTTP-tracer vocabulary (5c). The one flat error first, then the shared policy core +# (the private parsers, the policy over them, the re-sendability gate, the settings), then the +# private helpers the two stage drivers share, the two drivers, and the recovery-stack engine. +# Three earlier files gained a method or a keyword in place and appear above: http_date.rb (the +# day group), error/protocol_error.rb (the baked flag) and pipeline/cursor.rb with the two +# runtimes and the two instrumentation steps (the context-bundle widening). +require_relative "dexpace/error/retry_predicate_error" +require_relative "dexpace/resilience/pacing_parsers" +require_relative "dexpace/resilience/policy" +require_relative "dexpace/resilience/resend" +require_relative "dexpace/resilience/retry_settings" + # The dexpace Ruby SDK: an HTTP-client toolkit, not an HTTP client. # # This file issues explicit `require_relative`s for the whole tree rather than using an diff --git a/gems/dexpace-core/lib/dexpace/error/protocol_error.rb b/gems/dexpace-core/lib/dexpace/error/protocol_error.rb index d75cea9..dc53546 100644 --- a/gems/dexpace-core/lib/dexpace/error/protocol_error.rb +++ b/gems/dexpace-core/lib/dexpace/error/protocol_error.rb @@ -4,6 +4,7 @@ require_relative "../error" require_relative "invalid_argument_error" require_relative "../http/response" +require_relative "../retryability" module Dexpace # XCUT-4's branch (a): a protocol error carries a fully-received response -- status, headers and @@ -24,9 +25,14 @@ module Dexpace # a customer identifier, and OBS-11..OBS-19's redaction is phase 5's. #response is how a caller # who wants the body reads it (the plan's open question 5). # - # No #retryable_by_status?. XCUT-5 requires the baked flag to be computed once at construction - # from a SINGLE shared status classifier, and that classifier is phase 5a's; phase 6a (Task 6) - # adds the predicate to this class. Adding a method widens a signature, which NFR-4 permits. + # #retryable_by_status? is XCUT-5's baked flag (RETRY-3), computed ONCE at construction from + # the SINGLE shared status classifier -- phase 5a's Dexpace::Retryability, which phase 4b + # could not build and phase 6a reads rather than rebuilds (its Task 6). It is deliberately + # NOT named #retryable?, the open capability XCUT-6's throwable query looks for (P6-10): a + # ProtocolError answering that query would let the baked set override the CONFIGURED set + # RETRY-37 makes authoritative whenever the error is wrapped in another -- including a set + # that deliberately narrows -- so the two questions carry two names, design §6.1's own, and + # the retry drivers consult Resilience::Policy.retry_eligible?(status, set:) and never this. class ProtocolError < ::StandardError include Dexpace::Error @@ -73,9 +79,19 @@ def initialize(response) @response = response @status = response.status + @retryable_by_status = Retryability.retryable_status?(@status) super(describe(@status)) end + # XCUT-5 / RETRY-3: whether the status is one the built-in classifier calls retryable, + # baked at construction and never per-subclass -- a queryable property of the error, and not + # what the retry step consults (XCUT-5's closing NOTE; see the class comment). + # + # @return [Boolean] + def retryable_by_status? + @retryable_by_status + end + private def describe(status) diff --git a/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb b/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb new file mode 100644 index 0000000..fa288bb --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../error" + +module Dexpace + # RETRY-40's well-typed abort: raised by the two stage-based retry drivers when a + # caller-supplied `should_retry:` predicate itself raises. The predicate's own failure is + # carried as #cause -- a genuine wrap of a fresh failure, raised with `cause:` at the site, + # and not a re-raise of a carried error, so pipeline/7ce4431d's `cause: nil` rule does not + # apply to it. A StandardError including Dexpace::Error, flat under Dexpace:: on the tree's + # one-error-per-file convention (PipelineError, ContextConflictError, ProtocolError), and + # numbered P6-11: RETRY-40 requires the well-typed error and names none. + # + # Only a predicate that raises a StandardError is wrapped. The fatal family -- NoMemoryError, + # SystemStackError -- propagates through an absent rescue arm, unchanged and unwrapped + # (RETRY-25, RETRY-40's "fatal errors rethrown unchanged"). + class RetryPredicateError < ::StandardError + include Dexpace::Error + end +end diff --git a/gems/dexpace-core/lib/dexpace/http_date.rb b/gems/dexpace-core/lib/dexpace/http_date.rb index 57b8b9c..43b95d6 100644 --- a/gems/dexpace-core/lib/dexpace/http_date.rb +++ b/gems/dexpace-core/lib/dexpace/http_date.rb @@ -35,10 +35,17 @@ module HTTPDate # Time.httpdate rejects a bare "06 Nov 1994 ..." too, so the port and the stdlib agree. No # normalisation runs before the match, which is how CFG-31's blank-input failure is a # property of the pattern rather than of whatever a normaliser did to the input. The timeout - # is per-pattern (never Regexp.timeout), on a fixed-width grammar that cannot backtrack, by - # the same rule phase 1 applied to a two-character hex pattern. + # is per-pattern (never Regexp.timeout), on a grammar whose one variable-width group is a + # bounded digit run that cannot backtrack, by the same rule phase 1 applied to a + # two-character hex pattern. The day group is (\d{1,2}), not (\d{2}): RETRY-15 requires the + # Retry-After HTTP-date form to be parsed "tolerant of an informational weekday and + # single-digit day", design §6.1 forbids a second parser, and phase 6a widened this group in + # place (its R1) -- "Sun, 6 Nov 1994 08:49:37 GMT" parses as 6 November. Nothing else moved: + # the weekday is still REQUIRED and still not validated (CFG-30's "informational only" and + # RETRY-15's "informational weekday" are one tolerance), and RFC 850, asctime, a leading + # space, a two-digit year and a trailing token all still fail (CFG-31). GRAMMAR = ::Regexp.new( - '\A[A-Za-z]{3}, (\d{2}) ([A-Za-z]{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) ' \ + '\A[A-Za-z]{3}, (\d{1,2}) ([A-Za-z]{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) ' \ '(GMT|UTC|\+0000|\+00:00)\z', ::Regexp::IGNORECASE, timeout: 1.0, @@ -46,8 +53,8 @@ module HTTPDate # Neither is public surface -- a month table and one pattern with a single reader each, and # P5-1 enumerates HTTPDate alone. A bare reference from inside `module Dexpace; module - # HTTPDate` resolves whatever the visibility, so phase 6a widening the day group edits this - # file at no cost. + # HTTPDate` resolves whatever the visibility, which is what let phase 6a widen the day group + # above in place at no cost to the public surface. private_constant :MONTHS, :GRAMMAR # The canonical HTTP-date form, rendered in UTC with a literal GMT and a zero-padded day @@ -67,10 +74,11 @@ def format(time) time.getutc.httpdate end - # The UTC instant an RFC 1123 date denotes, with CFG-30's four tolerances and CFG-31's - # strictness: the four zone tokens all mean the zero offset, month and zone case do not - # matter, the weekday is stripped and never compared against the date; blank input, a missing - # comma, RFC 850, asctime, a leading space, a single-digit day and a trailing token all fail. + # The UTC instant an RFC 1123 date denotes, with CFG-30's four tolerances, RETRY-15's + # single-digit day (phase 6a's widening) and CFG-31's strictness: the four zone tokens all + # mean the zero offset, month and zone case do not matter, the weekday is stripped and never + # compared against the date, a one-digit day is read as that day; blank input, a missing + # comma, an absent weekday, RFC 850, asctime, a leading space and a trailing token all fail. # So does a syntactically well-formed but impossible date -- Time.utc silently normalises # 31 November, 29 February 1995, hour 24 and second 60 into the next day, month, day or # minute, so every component is checked against what Time.utc built rather than trusted. diff --git a/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb new file mode 100644 index 0000000..00a0df8 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../http_date" +require_relative "../error/invalid_argument_error" + +module Dexpace + module Resilience + # RETRY-15, RETRY-17, RETRY-19, RECOV-24, RECOV-25: the per-form pacing-header value parsers, + # one method per wire form, each TOTAL over a String -- nil for anything it cannot read, never + # a raise and never a zero delay it did not earn (RETRY-16, RECOV-23). Policy.pacing_delay is + # the public dispatch over them and nothing outside this file names a form directly, which is + # why this is a private_constant on 5a's ConfigParsers precedent (the plan's open question 2): + # Policy's public surface stays the methods the design names and the parsing internals carry + # no NFR-4 lock. It has a sig/ mirror because the strict `core` Steep target types its call + # sites, and no test/ mirror -- every branch is asserted through Policy.pacing_delay. + # + # Three grammars, each anchored \A..\z, each compiled once with its own timeout (never + # Regexp.timeout), and none can backtrack: digit runs and one optional fraction. RETRY-19's + # screen is the DECIMAL grammar, applied BEFORE String#to_f -- "30d", "0x10", "1e3", "5_0", + # "Infinity", "NaN" and " 5" all fail it and fall through to the HTTP-date attempt, which + # rejects them too, so the answer is nil. Nothing here calls Float() or Integer() on an + # unscreened value: Integer("5_0", 10) is 50 on every supported Ruby (verified 3.2.11, + # 3.4.10, 4.0.6), which is exactly the mis-parse the screen exists to prevent. + module PacingParsers + extend self + + DECIMAL_GRAMMAR = ::Regexp.new('\A\d+(\.\d+)?\z', timeout: 1.0).freeze + INTEGER_GRAMMAR = ::Regexp.new('\A\d+\z', timeout: 1.0).freeze + private_constant :DECIMAL_GRAMMAR, :INTEGER_GRAMMAR + + # The upper bound of RECOV-25's positive jitter: [100%, 120%] of the computed delta. + RESET_JITTER_CEILING = 1.2 + private_constant :RESET_JITTER_CEILING + + # Retry-After: delta-seconds first (integer and fractional, sub-second honoured), then an + # RFC 1123 HTTP-date through the ONE parser, whose past instants floor to zero (RETRY-17). + # An unparseable value is nil, distinct from a past date's 0.0. + def parse_retry_after(value, now:) + return nil unless value.is_a?(::String) + return value.to_f if DECIMAL_GRAMMAR.match?(value) + + instant = Dexpace::HTTPDate.parse(value) + [(instant - now).to_f, 0.0].max + rescue Dexpace::InvalidArgumentError + nil + end + + # retry-after-ms / x-ms-retry-after-ms: a non-negative integer count of milliseconds. + def parse_millis(value) + return nil unless value.is_a?(::String) && INTEGER_GRAMMAR.match?(value) + + value.to_i / 1000.0 + end + + # X-RateLimit-Reset: Unix epoch seconds. A reset already past floors to zero (RETRY-17); a + # future one is jittered UPWARD to [100%, 120%] of the delta (RECOV-25), inside the parser, + # so the caller applies no second jitter on top (RETRY-20, RECOV-22). The jitter is drawn + # only over a finite range: a digit run long enough to convert to Infinity is out of range + # and answers nil (RETRY-16), and a finite delta whose 120% would overflow is returned + # unjittered for the dispatcher's 365-day clamp to bound -- Random#rand raises EDOM on an + # infinite bound (verified on 4.0.6), and RECOV-26's "never surface an arithmetic overflow + # mid-retry" covers a server's hint before any clamp reaches it. + def parse_epoch_reset(value, now:, random:) + return nil unless value.is_a?(::String) && INTEGER_GRAMMAR.match?(value) + + delta = [(value.to_i - now.to_i).to_f, 0.0].max + return nil unless delta.finite? + return 0.0 if delta.zero? + + ceiling = delta * RESET_JITTER_CEILING + ceiling.finite? ? random.rand(delta..ceiling) : delta + end + end + private_constant :PacingParsers + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/policy.rb b/gems/dexpace-core/lib/dexpace/resilience/policy.rb new file mode 100644 index 0000000..e7c3993 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/policy.rb @@ -0,0 +1,280 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../each_cause" +require_relative "../error/invalid_argument_error" +require_relative "../error/protocol_error" +require_relative "../instrumentation/contain" +require_relative "../instrumentation/keys" +require_relative "../instrumentation/logger" +require_relative "pacing_parsers" + +module Dexpace + # The resilience layer: phase 6's namespace, design §6.1's. Retry is 6a's; redirect (6b) and + # authentication (6c) add their own constants beside these. + module Resilience + # The ONE shared retry policy core (RETRY-13, RECOV-30): the tuning constants, the backoff + # calculator, the pacing-header dispatcher, the two-branch retryability consult and the + # effective-retry-count resolver. Both retry stacks -- RecoveryRetry beneath phase 4b's + # orchestrator and RetryStep/AsyncRetryStep at Stages::RETRY -- call these exact methods and + # carry no formula or constant of their own; a text scan over lib/dexpace/resilience/ for a + # literal delay constant outside this file is the mechanised form of that claim. + # + # A module with no instance side, frozen constants and pure functions, so RETRY-42's + # "immutable and stateless after construction and safe for concurrent invocation" is + # structural: nothing here has state to share. `extend self` and never module_function + # (Style/ModuleFunction), on 5a's HTTPDate and Retryability precedent. + # + # Public (R5): a generated SDK assembling its own retry driver needs to call the SAME + # calculator, and XCUT-6's capability query is the extension point a third-party transport + # uses by defining #retryable? on its own error class -- both only work if the consulting + # code is public and stable. It coexists with 5a's Dexpace::Retryability without touching + # it: that module is XCUT-5's fixed baked classifier, read once by ProtocolError at + # construction; .retry_eligible? below is XCUT-7's CONFIGURABLE set, consulted by the retry + # drivers, and neither reads the other (XCUT-5's closing NOTE, P6-10). + # + # Two budget policies, one calculator (R6, P6-5): .backoff_delay takes no total-timeout + # parameter at all, and RECOV-20's budget is the separate .budget_remaining that only + # RecoveryRetry calls. RETRY-28's prohibition on the stage stack is then a fact about which + # files name which method, checkable by a text scan, rather than a keyword every future edit + # must remember not to pass. + module Policy + extend self + + # RETRY-12's first default: the wait before the first retry, 200 ms in seconds. The five + # defaults are one schedule -- 200 ms, x2, 8 s, 0.2 and three sends -- and every one is + # read from here by RetrySettings and re-typed nowhere (RETRY-13). + DEFAULT_INITIAL_DELAY = 0.2 + # RETRY-12: the exponential factor between successive waits. + DEFAULT_MULTIPLIER = 2.0 + # RETRY-12: the cap on any one wait, 8 s in seconds. + DEFAULT_MAX_DELAY = 8.0 + # RETRY-12: the symmetric jitter fraction, a fifth of the delay either way. + DEFAULT_JITTER = 0.2 + # RETRY-12's budget of three sends, spelled in the STAGE vocabulary as two retries after + # the initial send; the recovery stack's max-attempts is this plus one (RETRY-14, P6-6). + DEFAULT_MAX_RETRIES = 2 + + # XCUT-7's configurable retryable-status set, at its default: a SUBSET of Retryability's + # baked classification (which also admits 500-599 less 501 and 505 in full), enumerable + # because it is data and not a rule. + DEFAULT_RETRYABLE_STATUSES = ::Set[408, 429, 500, 502, 503, 504].freeze + + # RETRY-21's fixed recovery-stack precedence, and the stage stack's default when a caller + # configures no order of its own: Retry-After (numeric, then date), the two millisecond + # variants, then the epoch reset. + DEFAULT_PACING_HEADER_ORDER = %w[ + Retry-After retry-after-ms x-ms-retry-after-ms X-RateLimit-Reset + ].freeze + + # RETRY-18 / RECOV-26: the finite ceiling every pacing delta is clamped to, 365 days in + # seconds, applied before any nanosecond conversion could overflow. + MAX_PACING_DELAY_SECONDS = 365 * 24 * 60 * 60 + + # RECOV-34's representability ceiling: the largest signed 64-bit nanosecond count, ~292 + # years, which design §10.18 substitutes for the reference's Duration bound. + MAX_DURATION_NANOSECONDS = (1 << 63) - 1 + + # Sub-nanosecond: RETRY-10's degenerate jitter range, below which the base delay is + # returned unperturbed. + NANOSECOND = 1e-9 + private_constant :NANOSECOND + + # RETRY-37 / XCUT-7 / RECOV-17: whether `status` is in the CONFIGURED retryable-status + # set, consulted alone -- authoritative-contains, never AND-ed with XCUT-5's baked flag. + # There is no baked-flag parameter on this method by design, so the intersection RETRY-37 + # forbids is a method that does not exist. A set that NARROWS narrows: 500 is retryable + # under Retryability and refused here when the set omits it. + # + # @param status [Integer] the response status code + # @param set [Set] the configured retryable-status set + # @return [Boolean] + def retry_eligible?(status, set:) + set.include?(status) + end + + # RETRY-2 / XCUT-6 / CFG-35's throwable half: the retryable-throwable set, defined here and + # nowhere else, as the CAPABILITY query -- `respond_to?(:retryable?) && retryable?` -- + # walked over the error and its whole cause chain through Dexpace.each_cause, which is + # already cycle-safe by reference identity (XCUT-9), so RETRY-2's termination clause needs + # no second walk. Never a concrete-type match: `is_a?(::IOError)` is wrong in both + # directions (5a's R1) -- it would call phase 3a's StreamError retryable and a bare + # Errno::ETIMEDOUT not. The residual that comes with the capability form is P6-4: a bare, + # unwrapped stdlib I/O or timeout error answers no capability and classifies not + # retryable, which is why phase 8's adapters must wrap what they let escape. + # + # @param error [Exception] + # @return [Boolean] + def throwable_retryable?(error) + Dexpace.each_cause(error).any? do |cause| + # The capability is a duck type the signature cannot name; probed untyped, as 5a's + # Retryability probes #code. + probe = cause #: untyped + probe.respond_to?(:retryable?) && probe.retryable? + end + end + + # The one dispatch point both stacks call: a ProtocolError is classified by its status + # against the configured set (RETRY-37), anything else by the capability query (RETRY-2). + # A ProtocolError deliberately does not answer #retryable? (P6-10), so a wrapped one buried + # in a cause chain can never smuggle the baked set past the configured one. + # + # @param error [Exception] + # @param retryable_statuses [Set] the configured set + # @return [Boolean] + def retryable?(error, retryable_statuses:) + if error.is_a?(Dexpace::ProtocolError) + retry_eligible?(error.status.code, set: retryable_statuses) + else + throwable_retryable?(error) + end + end + + # RETRY-9, RETRY-10, RETRY-11, RECOV-21: `initial_delay * multiplier**(attempt - 1)`, + # capped at `max_delay`, then symmetric jitter drawn uniformly from + # [d * (1 - j/2), d * (1 + j/2)] -- midpoint d, j = 0 returning d exactly, a degenerate + # sub-nanosecond spread returning d, a negative sample floored to zero. `attempt` is + # 1-based, 1 being the wait before the first retry, and anything below is refused. + # + # Overflow-safe by Ruby's own float arithmetic plus ONE guard: `2.0**9999` is Infinity + # (never a raise, verified on 3.2.11, 3.4.10 and 4.0.6) and `[Infinity, max_delay].min` is + # `max_delay`, so a large attempt saturates to the cap -- but `0.0 * Infinity` is NaN, and + # `[NaN, 8.0].min` RAISES ("comparison of Float with NaN failed"), so a zero initial delay + # answers zero without touching the power (found by the 2000-attempt trampoline test; + # P6-53). No budget parameter: the total-timeout is RecoveryRetry's alone, applied through + # .budget_remaining (R6). + # + # @param attempt [Integer] 1-based + # @param initial_delay [Float] seconds + # @param multiplier [Float] >= 1.0 + # @param max_delay [Float] seconds, the cap + # @param jitter [Float] in [0.0, 1.0] + # @param random [#rand] the generator the jitter sample is drawn from + # @return [Float] seconds + # @raise [Dexpace::InvalidArgumentError] for an attempt below 1 + def backoff_delay(attempt, initial_delay:, multiplier:, max_delay:, jitter:, random:) + unless attempt.is_a?(::Integer) && attempt >= 1 + raise InvalidArgumentError, "attempt must be an Integer >= 1, got #{attempt.inspect}" + end + + initial = seconds(initial_delay) + return 0.0 if initial.zero? + + raw = initial * (seconds(multiplier)**(attempt - 1)) + jittered([raw, seconds(max_delay)].min, jitter, random) + end + + # RETRY-15 through RETRY-22, RECOV-22 through RECOV-26, RECOV-29: the total pacing + # dispatch. Walks `header_order` -- the recovery stack's fixed precedence or the stage + # stack's caller-configurable list (RETRY-21) -- and returns the first usable value, + # clamped to MAX_PACING_DELAY_SECONDS, or nil for no hint. Each form's parser is total and + # every parser call is fenced besides, so a malformed value in one header falls through to + # the next rather than aborting the scan, and nothing here ever raises: RETRY-22's "a + # failure while parsing a pacing header MUST NOT mask the real upstream failure" is honoured + # one level up, by the caller falling back to .backoff_delay on nil, and one level down, + # by no single value poisoning the scan. A result that is not finite is no hint (RETRY-16's + # out-of-range clause); a finite one past the ceiling is the ceiling (RETRY-18). + # + # @param headers [Dexpace::Headers] the response's headers + # @param header_order [Array] the names to consult, in precedence order + # @param now [Time] the instant absolute forms are measured from; the driver passes its + # clock's so a fake clock drives the arithmetic + # @param random [#rand] the generator RECOV-25's reset jitter draws from + # @return [Float, nil] seconds, or nil for no hint + def pacing_delay(headers, header_order:, now: ::Time.now, random: ::Random) + header_order.each do |name| + value = headers[name]&.first + next if value.nil? + + delay = parse_form(name, value, now, random) + next unless delay.is_a?(::Float) && delay.finite? && delay >= 0.0 + + return [delay, MAX_PACING_DELAY_SECONDS.to_f].min + end + nil + end + + # RETRY-41: the stage stack's effective retry count -- a present per-call override wins + # (validated non-negative), else the configured value, with a negative configured value + # clamped to DEFAULT_MAX_RETRIES and the clamp logged; zero means no retries. The log is a + # WARNING `http.instrumentation.config` diagnostic through 5b's facade, contained (OBS-20) + # so a raising sink cannot fail the request, and Logger::NULL by default so the + # non-clamping path allocates nothing and touches no sink (P6-12). + # + # @param override [Integer, nil] the per-call RequestOptions#max_retries + # @param configured [Integer] the settings' max_retries + # @param logger [Dexpace::Instrumentation::Logger] where the clamp is reported + # @return [Integer] >= 0 + # @raise [Dexpace::InvalidArgumentError] for a negative override + def effective_max_retries(override:, configured:, logger: Instrumentation::Logger::NULL) + unless override.nil? + unless override.is_a?(::Integer) && override >= 0 + raise InvalidArgumentError, "max_retries override must be a non-negative Integer" + end + + return override + end + return configured if configured >= 0 + + Instrumentation.diagnostic( + logger, event: Instrumentation::Events::INSTRUMENTATION_CONFIG, + message: "max_retries #{configured} is negative and was clamped to " \ + "#{DEFAULT_MAX_RETRIES} (RETRY-41)", + ) + DEFAULT_MAX_RETRIES + end + + # RECOV-20 / RETRY-27: the recovery stack's budget, as the time REMAINING -- never the + # elapsed figure -- so every caller compares one number against a delay: Float::INFINITY + # when the budget is zero ("unbounded, deadline disabled"), else `total_timeout - elapsed`, + # which is negative once the budget is spent. Recovery-driver only: RetryStep and + # AsyncRetryStep never call this, which is what makes RETRY-28 a property of the source + # (R6, P6-5). + # + # @param elapsed [Float] seconds since the engine's own start instant + # @param total_timeout [Numeric] seconds; zero disables the budget + # @return [Float] seconds remaining, possibly negative, or Float::INFINITY + def budget_remaining(elapsed:, total_timeout:) + return ::Float::INFINITY if total_timeout.zero? + + seconds(total_timeout) - seconds(elapsed) + end + + private + + # A Numeric as a Float. rbs's Numeric declares no #to_f (its subclasses do), and Kernel#Float + # is typed as answering nil, so the conversion reads the value untyped -- the same spelling + # 5a's Clock::Guard uses. + def seconds(value) + number = value #: untyped + number.to_f + end + + # RETRY-10's symmetric draw over [d(1 - j/2), d(1 + j/2)]: d itself for j = 0 or a + # sub-nanosecond spread, and a negative sample floored at zero. + def jittered(unjittered, jitter, random) + return unjittered if jitter.zero? + + spread = unjittered * jitter + return unjittered if spread < NANOSECOND + + half = spread / 2.0 + [random.rand((unjittered - half)..(unjittered + half)), 0.0].max + end + + # The per-form dispatch, fenced: a parser is total by contract and this rescue is the + # belt behind it. Header names are matched case-insensitively with a bare #downcase + # (HTTP-13); Headers#[] already folded the lookup. + def parse_form(name, value, now, random) + case name.downcase + when "retry-after" then PacingParsers.parse_retry_after(value, now: now) + when "retry-after-ms", "x-ms-retry-after-ms" then PacingParsers.parse_millis(value) + when "x-ratelimit-reset" + PacingParsers.parse_epoch_reset(value, now: now, random: random) + end + rescue ::StandardError + nil + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/resend.rb b/gems/dexpace-core/lib/dexpace/resilience/resend.rb new file mode 100644 index 0000000..4e1068d --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/resend.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../http/method" +require_relative "../http/body" + +module Dexpace + module Resilience + # RETRY-5, RETRY-6, RETRY-7, RETRY-8 and RECOV-18: the re-sendability gate, the second of + # the two independent axes retry needs, single-sourced. Both retry stacks call this exact + # method and neither restates the rule; 6b's REDIR-6 and 6c's AUTH-31 are its other two + # callers, as phase 3b's forward table anticipated (R5, confirmed unchanged). + # + # The rule in one line: a body-less request is re-sendable iff its method is idempotent -- + # HTTP-9's single set, phase 1's Method::IDEMPOTENT read through #idempotent? and never + # re-listed here, which is what makes RETRY-6's `{GET, HEAD, OPTIONS, PUT, DELETE}` one + # constant and not two -- and a body-bearing request iff its body says it is replayable + # (phase 3b's Body#replayable?, false by default). So a bare non-idempotent POST is NOT + # re-sendable even though there is no payload to resend (RETRY-7), and a PUT with a + # non-replayable body is not either (RETRY-8: both axes, neither implying the other). + # + # Phase 6b extends this module in place with the body-only question REDIR-6 asks + # (.replayable_body?) and its NotReplayableError; nothing here anticipates them, and nothing + # here is a "no-op if the file exists" skip (the plan's ownership note). + module Resend + extend self + + # Whether `request` may be sent to the wire a second time. + # + # @param request [Dexpace::Request] + # @return [Boolean] + def eligible?(request) + body = request.body + return request.method.idempotent? if body.nil? + + body.replayable? + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb new file mode 100644 index 0000000..86316f0 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb @@ -0,0 +1,189 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../model" +require_relative "../clock" +require_relative "../config" +require_relative "../configuration" +require_relative "../error/invalid_argument_error" +require_relative "policy" + +module Dexpace + module Resilience + # RECOV-34 and RETRY-12: the ONE retry configuration, a frozen Data validated and defensively + # copied at construction, that both stacks build from identical defaults -- RECOV-30's "the + # SAME default schedule so the two stacks cannot drift apart" made an object rather than a + # promise, because there is no second type to type a base delay into. Every default is + # Policy's constant, read here and re-typed nowhere. + # + # `total_timeout` is carried for both stacks and read by one: RecoveryRetry applies it through + # Policy.budget_remaining and RetryStep/AsyncRetryStep never read the member at all, which is + # RETRY-28's prohibition kept structural (R6). `pacing_header_order` is nil by default and each + # driver substitutes Policy::DEFAULT_PACING_HEADER_ORDER for nil, so RETRY-21's split + # precedence -- fixed on the recovery stack, caller-configurable on the stage stack -- reads + # from one member. `random` defaults to the ::Random CLASS, whose .rand is the process + # generator CRuby already makes safe to share, rather than to a fresh instance every settings + # object would then share unsynchronised across threads (RETRY-42's caveat, stated rather + # than hidden); a seeded ::Random.new is what a test passes. `clock` is 5a's seam. + # + # `max_retries` is the STAGE vocabulary (excluding the initial send) and the recovery stack's + # max_attempts is always max_retries + 1 (RETRY-14, P6-6). When a caller passes none, .build + # reads 5a's Configuration::Keys::MAX_RETRY_ATTEMPTS off the process-wide slot -- this class is + # that key's first and only reader -- and falls back to Policy::DEFAULT_MAX_RETRIES; the read + # happens ONCE, at build, so a settings object is a snapshot and never re-consults the slot. + class RetrySettings < Data.define( + :initial_delay, :multiplier, :max_delay, :jitter, :max_retries, :total_timeout, + :retryable_statuses, :pacing_header_order, :random, :clock, + ) + include Dexpace::Model + + private_class_method :new + + # The absent-argument sentinel for max_retries: what lets "the caller passed nothing" be told + # apart from "the caller passed the default", so the configured key is consulted only in the + # first case and an explicit argument -- including one #with routes back through .build -- + # always wins. Private, and never a member's value. + UNSET = ::Object.new.freeze + private_constant :UNSET + + # The validating factory. Every keyword defaults to the shared policy constant. + # + # @param initial_delay [Numeric] seconds, non-negative + # @param multiplier [Numeric] >= 1.0 + # @param max_delay [Numeric] seconds, non-negative + # @param jitter [Numeric] in [0.0, 1.0] + # @param max_retries [Integer] retries after the initial send, >= 0; the configured + # MAX_RETRY_ATTEMPTS, else Policy::DEFAULT_MAX_RETRIES, when omitted + # @param total_timeout [Numeric] seconds; 0 disables the recovery stack's budget + # @param retryable_statuses [Enumerable] XCUT-7's configured set, copied + # @param pacing_header_order [Array, nil] RETRY-21's order, copied; nil for the + # shared default + # @param random [#rand] the generator the jitter draws from + # @param clock [Dexpace::_Clock] the seam every wait and every elapsed reading goes through + # @return [RetrySettings] frozen + # @raise [Dexpace::InvalidArgumentError] naming the member, on any invalid value + def self.build(initial_delay: Policy::DEFAULT_INITIAL_DELAY, + multiplier: Policy::DEFAULT_MULTIPLIER, max_delay: Policy::DEFAULT_MAX_DELAY, + jitter: Policy::DEFAULT_JITTER, max_retries: UNSET, total_timeout: 0, + retryable_statuses: Policy::DEFAULT_RETRYABLE_STATUSES, + pacing_header_order: nil, random: ::Random, clock: Clock::SYSTEM) + new( + initial_delay: initial_delay, multiplier: multiplier, max_delay: max_delay, + jitter: jitter, max_retries: resolve_max_retries(max_retries), + total_timeout: total_timeout, retryable_statuses: retryable_statuses, + pacing_header_order: pacing_header_order, random: random, clock: clock, + ) + end + + # RETRY-12 / P6-6: the configured key, read once, only when nothing was passed. + def self.resolve_max_retries(max_retries) + return max_retries unless UNSET.equal?(max_retries) + + Dexpace.configuration.integer(Configuration::Keys::MAX_RETRY_ATTEMPTS, + default: Policy::DEFAULT_MAX_RETRIES,) || + Policy::DEFAULT_MAX_RETRIES + end + private_class_method :resolve_max_retries + + # RECOV-34, clause by clause, each failure naming its member (SEAM-29); the collections + # are copied and deep-frozen here, once, through Model.own. + def initialize(initial_delay:, multiplier:, max_delay:, jitter:, max_retries:, + total_timeout:, retryable_statuses:, pacing_header_order:, random:, clock:) + Model.required!("random", random) + Model.required!("clock", clock) + super( + initial_delay: duration!("initial_delay", initial_delay), + multiplier: multiplier!(multiplier), max_delay: duration!("max_delay", max_delay), + jitter: jitter!(jitter), max_retries: max_retries!(max_retries), + total_timeout: duration!("total_timeout", total_timeout), + retryable_statuses: statuses!(retryable_statuses), + pacing_header_order: order!(pacing_header_order), random: random, clock: clock, + ) + end + + # The keyword hash Policy.backoff_delay takes, so neither driver spells the five members + # out by hand (RETRY-13). + # + # @return [Hash] + def backoff_arguments + { initial_delay: initial_delay, multiplier: multiplier, max_delay: max_delay, + jitter: jitter, random: random, } + end + + # RETRY-21: the order the drivers walk -- the caller's, or the shared default. + # + # @return [Array] frozen + def header_order + pacing_header_order || Policy::DEFAULT_PACING_HEADER_ORDER + end + + private + + # Non-negative, finite, and representable within the ~292-year nanosecond ceiling + # (design §10.18); returned as a Float so the arithmetic downstream is uniform. + def duration!(name, value) + Model.required!(name, value) + seconds = numeric!(name, value) + if seconds.negative? || !seconds.finite? + raise InvalidArgumentError, "#{name} must be a finite non-negative number of seconds" + end + if seconds * 1_000_000_000 > Policy::MAX_DURATION_NANOSECONDS + raise InvalidArgumentError, "#{name} exceeds the representable ceiling (RECOV-34)" + end + + seconds + end + + def multiplier!(value) + Model.required!("multiplier", value) + factor = numeric!("multiplier", value) + raise InvalidArgumentError, "multiplier must be >= 1.0 (RECOV-34)" if factor < 1.0 + raise InvalidArgumentError, "multiplier must be finite" unless factor.finite? + + factor + end + + def jitter!(value) + Model.required!("jitter", value) + fraction = numeric!("jitter", value) + return fraction if fraction.between?(0.0, 1.0) + + raise InvalidArgumentError, "jitter must lie within [0.0, 1.0] (RECOV-34)" + end + + def max_retries!(value) + Model.required!("max_retries", value) + return value if value.is_a?(::Integer) && !value.negative? + + raise InvalidArgumentError, "max_retries must be a non-negative Integer (RECOV-34)" + end + + def statuses!(value) + Model.required!("retryable_statuses", value) + unless value.respond_to?(:each) && value.all?(::Integer) + raise InvalidArgumentError, "retryable_statuses must be a collection of Integers" + end + + Model.own(::Set.new(value)) + end + + def order!(value) + return nil if value.nil? + unless value.is_a?(::Array) && value.all?(::String) + raise InvalidArgumentError, "pacing_header_order must be an Array of header names" + end + + Model.own(value.dup) + end + + def numeric!(name, value) + unless value.is_a?(::Numeric) + raise InvalidArgumentError, "#{name} must be a number, got #{value.class}" + end + + number = value #: untyped + number.to_f + end + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs b/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs index b6172cb..33b512f 100644 --- a/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs +++ b/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs @@ -2,6 +2,8 @@ module Dexpace class ProtocolError < ::StandardError include Dexpace::Error + @retryable_by_status: bool + attr_reader response: Dexpace::Response attr_reader status: Dexpace::Status @@ -9,6 +11,9 @@ module Dexpace def self.for_or_nil: (Dexpace::Response response) -> Dexpace::ProtocolError? def initialize: (Dexpace::Response response) -> void + # XCUT-5's baked flag (RETRY-3), added by phase 6a: a widening, the constructor unchanged. + def retryable_by_status?: () -> bool + private def describe: (Dexpace::Status status) -> String diff --git a/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs b/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs new file mode 100644 index 0000000..1e33088 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs @@ -0,0 +1,5 @@ +module Dexpace + class RetryPredicateError < ::StandardError + include Dexpace::Error + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs new file mode 100644 index 0000000..279084b --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs @@ -0,0 +1,18 @@ +# Dexpace::Resilience::PacingParsers is a private_constant and not public API (P6-11): this +# declaration exists because the strict `core` Steep target types Policy.pacing_delay's calls into +# it. RBS has no visibility for a constant, so the privacy lives in +# lib/dexpace/resilience/pacing_parsers.rb alone. `random` is untyped: ::Random is outside NFR-11's +# stdlib allowlist, and the contract is #rand over a Range (trace_id_flavour.rbs's precedent). +module Dexpace + module Resilience + module PacingParsers + DECIMAL_GRAMMAR: Regexp + INTEGER_GRAMMAR: Regexp + RESET_JITTER_CEILING: Float + + def self?.parse_retry_after: (untyped value, now: ::Time) -> Float? + def self?.parse_millis: (untyped value) -> Float? + def self?.parse_epoch_reset: (untyped value, now: ::Time, random: untyped) -> Float? + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/policy.rbs b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs new file mode 100644 index 0000000..5a3f69d --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs @@ -0,0 +1,36 @@ +module Dexpace + module Resilience + # The one shared retry policy core (RETRY-13, RECOV-30, R5). `random` is untyped on the two + # methods that take one: ::Random is outside NFR-11's stdlib allowlist and the contract is + # #rand over a Range, the shape trace_id_flavour.rbs already uses for its generator. + module Policy + DEFAULT_INITIAL_DELAY: Float + DEFAULT_MULTIPLIER: Float + DEFAULT_MAX_DELAY: Float + DEFAULT_JITTER: Float + DEFAULT_MAX_RETRIES: Integer + DEFAULT_RETRYABLE_STATUSES: ::Set[Integer] + DEFAULT_PACING_HEADER_ORDER: Array[String] + MAX_PACING_DELAY_SECONDS: Integer + MAX_DURATION_NANOSECONDS: Integer + NANOSECOND: Float + + def self?.retry_eligible?: (Integer status, set: ::Set[Integer]) -> bool + def self?.throwable_retryable?: (::Exception error) -> bool + def self?.retryable?: (::Exception error, retryable_statuses: ::Set[Integer]) -> bool + def self?.backoff_delay: (Integer attempt, initial_delay: Numeric, multiplier: Numeric, + max_delay: Numeric, jitter: Numeric, random: untyped) -> Float + def self?.pacing_delay: (Dexpace::Headers headers, header_order: Array[String], + ?now: ::Time, ?random: untyped) -> Float? + def self?.effective_max_retries: (override: Integer?, configured: Integer, + ?logger: Instrumentation::Logger) -> Integer + def self?.budget_remaining: (elapsed: Numeric, total_timeout: Numeric) -> Float + + private + + def seconds: (Numeric value) -> Float + def jittered: (Float unjittered, Numeric jitter, untyped random) -> Float + def parse_form: (String name, String value, ::Time now, untyped random) -> Float? + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/resend.rbs b/gems/dexpace-core/sig/dexpace/resilience/resend.rbs new file mode 100644 index 0000000..0382395 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/resend.rbs @@ -0,0 +1,9 @@ +module Dexpace + module Resilience + # RETRY-5 through RETRY-8, RECOV-18: the re-sendability gate (R5). Phase 6b widens this + # module in place with REDIR-6's body-only predicate and its error. + module Resend + def self?.eligible?: (Dexpace::Request request) -> bool + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs new file mode 100644 index 0000000..81d62a0 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs @@ -0,0 +1,56 @@ +module Dexpace + module Resilience + # RECOV-34, RETRY-12: the one retry configuration both stacks build from. `random` is untyped + # (::Random is outside NFR-11's allowlist; the contract is #rand over a Range) and the + # arguments are `untyped` where the constructor validates rather than trusts. + class RetrySettings < Data + include Model + + UNSET: Object + + attr_reader initial_delay: Float + attr_reader multiplier: Float + attr_reader max_delay: Float + attr_reader jitter: Float + attr_reader max_retries: Integer + attr_reader total_timeout: Float + attr_reader retryable_statuses: ::Set[Integer] + attr_reader pacing_header_order: Array[String]? + attr_reader random: untyped + attr_reader clock: _Clock + + private def self.new: ( + initial_delay: untyped, multiplier: untyped, max_delay: untyped, jitter: untyped, + max_retries: untyped, total_timeout: untyped, retryable_statuses: untyped, + pacing_header_order: untyped, random: untyped, clock: _Clock, + ) -> instance + def initialize: ( + initial_delay: untyped, multiplier: untyped, max_delay: untyped, jitter: untyped, + max_retries: untyped, total_timeout: untyped, retryable_statuses: untyped, + pacing_header_order: untyped, random: untyped, clock: _Clock, + ) -> void + + def self.build: ( + ?initial_delay: Numeric, ?multiplier: Numeric, ?max_delay: Numeric, ?jitter: Numeric, + ?max_retries: untyped, ?total_timeout: Numeric, + ?retryable_statuses: Enumerable[Integer], + ?pacing_header_order: Array[String]?, ?random: untyped, ?clock: _Clock, + ) -> RetrySettings + + def backoff_arguments: () -> { initial_delay: Float, multiplier: Float, max_delay: Float, + jitter: Float, random: untyped } + def header_order: () -> Array[String] + + private + + def self.resolve_max_retries: (untyped max_retries) -> Integer + def duration!: (String name, untyped value) -> Float + def multiplier!: (untyped value) -> Float + def jitter!: (untyped value) -> Float + def max_retries!: (untyped value) -> Integer + def statuses!: (untyped value) -> ::Set[Integer] + def order!: (untyped value) -> Array[String]? + def numeric!: (String name, untyped value) -> Float + end + end +end diff --git a/gems/dexpace-core/test/dexpace/http_date_test.rb b/gems/dexpace-core/test/dexpace/http_date_test.rb index ea00d81..b57b451 100644 --- a/gems/dexpace-core/test/dexpace/http_date_test.rb +++ b/gems/dexpace-core/test/dexpace/http_date_test.rb @@ -103,7 +103,7 @@ class ParseTest < DexpaceTestCase "Sun,\n06 Nov 1994 08:49:37 GMT", # \s+ would accept this; a literal space does not "Sun, 06 Nov 1994 08:49:37 GMT\n", # \Z would accept this; \z does not "Sun, 06 Nov 1994 08:49:37 GMT junk", - "Sun, 6 Nov 1994 08:49:37 GMT", # single-digit day + "06 Nov 1994 08:49:37 GMT", # no weekday at all: CFG-31's prefix is required "Sun, 06 Nov 1994 08:49:37", # no zone "Sun, 06 Nov 1994 08:49:37 +0100", # a non-zero offset is not one of the four tokens "Sun, 06 Nov 1994 08:49:37 EST", diff --git a/gems/dexpace-core/test/dexpace_test.rb b/gems/dexpace-core/test/dexpace_test.rb index 845dd69..d759701 100644 --- a/gems/dexpace-core/test/dexpace_test.rb +++ b/gems/dexpace-core/test/dexpace_test.rb @@ -37,11 +37,13 @@ class DexpaceTest < DexpaceTestCase # Every public constant the surface manifest records, and the check that catches a file added # to lib/ and forgotten in the entry point. Phase 1's domain model, then phase 2's seam layer, # then phase 3a's byte-streaming layer, then phase 3b's body layer, then phase 4a's execution - # context, phase 4b's recovery layer, phase 4c's pipeline and phase 5a's configuration layer; + # context, phase 4b's recovery layer, phase 4c's pipeline, phase 5a's configuration layer and + # phase 6a's retry layer (the flat RetryPredicateError beside the Resilience namespace); # Dexpace::Hooks, Dexpace::BoundedMap, Dexpace::CallKey, Dexpace::Recovery::Ownership, the two - # pipeline drivers, Dexpace::ConfigParsers, Dexpace::DeepValue and Dexpace::ProxyResolution are - # private_constants and appear in no constants(false) list. Phase 5c's tracing and metrics - # layer and phase 5b's logging layer add no flat constant: everything either ships is under + # pipeline drivers, Dexpace::ConfigParsers, Dexpace::DeepValue, Dexpace::ProxyResolution and + # 6a's Resilience::PacingParsers are private_constants and appear in no constants(false) list. + # Phase 5c's tracing and metrics layer and phase 5b's logging layer add no flat constant: + # everything either ships is under # Dexpace::Instrumentation, which the Layers case below pins. DOMAIN_MODEL = %i[ Error InvalidArgumentError Model Builder HeaderSyntax HeaderName Headers Status Method @@ -65,9 +67,10 @@ class DexpaceTest < DexpaceTestCase CONFIGURATION_LAYER = %i[ BuildInfo UUID Retryability HTTPDate Clock Configuration Proxy ].freeze + RESILIENCE_LAYER = %i[RetryPredicateError Resilience].freeze LAYERS = [ DOMAIN_MODEL, SEAM_LAYER, IO_LAYER, BODY_LAYER, CONTEXT_LAYER, RECOVERY_LAYER, PIPELINE_LAYER, - CONFIGURATION_LAYER, + CONFIGURATION_LAYER, RESILIENCE_LAYER, ].flatten.freeze test "defines nothing outside the Dexpace namespace" do From dea1240924978f670ef0735e5dc105246b1a030c Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:12:52 +0300 Subject: [PATCH 02/13] feat: carry the per-call instrumentation bundle on the pipeline cursor Widen Pipeline::Cursor with one read-only member, #bundle -- CTX-14's per-call correlation bundle -- seeded by a new optional `bundle:` keyword on Pipeline#call and AsyncPipeline#call (Bundle::NONE when omitted), validated in Cursor.build, threaded through both private drivers' #advance and carried across #fork exactly as #options is. No existing signature moves, and Transport.conforms? still accepts both runtimes because the keyword is optional beside the SPI's three positionals (P6-51). The one consumer is 5b's instrumentation step, on both runtimes: #open_span now takes the cursor's bundle and prefers its tracer factory when the bundle is not NONE, else the step's own keyword, and Tracing.correlate is handed the same bundle, so a populated one pushes its trace and span ids onto the diagnostic context. That is the reconciled precedence's first clause, which phase 5 recorded as having no implementation path; a call that seeds nothing resolves exactly as phase 5 did. The bundle carries no operation name, so the span is still named by the method token (P6-52), and it is not the retry step's HTTP-tracer source (P6-7). Existing tests only widened where the code required it: the cursor's public-method pin gains #bundle, and the two step suites' hand-rolled cursor stand-ins answer #bundle with NONE. --- .../lib/dexpace/async_pipeline.rb | 10 ++++- .../lib/dexpace/instrumentation/async_step.rb | 5 ++- .../lib/dexpace/instrumentation/bundle.rb | 4 +- .../lib/dexpace/instrumentation/step.rb | 39 +++++++++++-------- .../lib/dexpace/instrumentation/tracing.rb | 11 +++--- gems/dexpace-core/lib/dexpace/pipeline.rb | 12 +++++- .../lib/dexpace/pipeline/async_driver.rb | 4 +- .../lib/dexpace/pipeline/cursor.rb | 37 +++++++++++++++--- .../lib/dexpace/pipeline/sync_driver.rb | 4 +- .../sig/dexpace/async_pipeline.rbs | 2 +- .../sig/dexpace/instrumentation/step.rbs | 2 +- gems/dexpace-core/sig/dexpace/pipeline.rbs | 2 +- .../sig/dexpace/pipeline/async_driver.rbs | 2 +- .../sig/dexpace/pipeline/cursor.rbs | 9 +++-- .../sig/dexpace/pipeline/sync_driver.rbs | 2 +- .../instrumentation/async_step_test.rb | 2 + .../test/dexpace/instrumentation/step_test.rb | 1 + .../test/dexpace/pipeline/cursor_test.rb | 4 +- 18 files changed, 106 insertions(+), 46 deletions(-) diff --git a/gems/dexpace-core/lib/dexpace/async_pipeline.rb b/gems/dexpace-core/lib/dexpace/async_pipeline.rb index 7e02f71..50d5955 100644 --- a/gems/dexpace-core/lib/dexpace/async_pipeline.rb +++ b/gems/dexpace-core/lib/dexpace/async_pipeline.rb @@ -4,6 +4,7 @@ require_relative "closeable" require_relative "http/request_options" require_relative "cancellation" +require_relative "instrumentation/bundle" require_relative "async/future" require_relative "pipeline" require_relative "pipeline/cursor" @@ -117,16 +118,21 @@ def initialize(entries:, transport:, driver_class:) # is reached qualified because this class's cref does not include Dexpace::Pipeline; there is # exactly one Cursor class and both runtimes use it (P4-30). # + # `bundle:` is phase 6a's widening, as on Pipeline#call: the empty branch dispatches with + # no cursor and therefore carries no bundle, which is PIPE-9's own shape. + # # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call correlation bundle # @return [Dexpace::Async::Future] - def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none) + def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none, + bundle: Instrumentation::Bundle::NONE) driver = @driver_class.new(self) return driver.dispatch(request, options, cancellation) if @entries.empty? Pipeline::Cursor.build(drive: driver, request: request, options: options, - cancellation: cancellation,).call(request) + cancellation: cancellation, bundle: bundle,).call(request) end end end diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb b/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb index 04993df..77e0081 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb @@ -80,8 +80,9 @@ def call(request, cursor) # @return [Array] the future and the Pending the settlement side needs def head(request, cursor) started = @clock.monotonic - span = open_span(request) - scope = Tracing.correlate(span, Bundle::NONE) + bundle = cursor.bundle + span = open_span(request, bundle) + scope = Tracing.correlate(span, bundle) pending = nil #: Pending? begin request = prepare(request) diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb b/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb index cbdf890..c1620a6 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb @@ -21,7 +21,9 @@ module Instrumentation # NONE.with (both validate), may add METHODS here and to TraceIdFlavour, may give the two # no-op classes their protocols, and may not rename or remove a member, add one, store # validity, change the sentinels, replace the flavour with a Symbol or hand this class a - # second NONE. + # second NONE. Phase 6a honoured the same obligation: it added no member and no method, and + # made a bundle reachable by a pipeline step through Pipeline::Cursor#bundle, seeded per + # call by the `bundle:` keyword on Pipeline#call and AsyncPipeline#call (its Task 8). # # trace_state is a list of pairs, not a Hash: OBS-26 says "a vendor trace-state list" and W3C # tracestate is ordered, most recent vendor first, which a Hash would lose. trace_flags stays diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb index e689b64..5510ad1 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb @@ -38,16 +38,20 @@ module Instrumentation # # The two slots are keywords defaulting to the constants the phase ships -- phase 4a's # NO_TRACER_FACTORY and phase 5c's NO_METER (P5-33) -- and the two instruments are created - # once here, in .build, never per request (OBS-31). The reconciled precedence's first clause, - # "the request context's bundle when it is not Bundle::NONE", is not implementable in phase - # 5: no reader exists by which a pipeline step reaches a RequestContext or a Bundle (4c does - # not consume 4a; Request's members are method, url, headers and body; PIPE-11 forbids - # ambient carriage), so the step resolves to its keyword and to Bundle::NONE, which is - # OBS-34's and XCUT-19(e)'s own default configuration. Phase 6a's Task 8 owns the Cursor - # widening that closes it, at which point #open_span is the one method that changes. Until - # then the span is named by the request's method token, which is also what the tracer - # factory is asked for: the operation identifier is 4a's RequestContext#operation_name, out - # of reach for the same reason. + # once here, in .build, never per request (OBS-31). The reconciled precedence's three + # clauses -- "the request context's bundle when it is not Bundle::NONE, else the step's + # constructor keyword, else the constant" -- are honoured for the TRACER FACTORY since phase + # 6a's Task 8 widened Cursor with #bundle: #open_span reads the cursor's bundle and takes its + # tracer factory when the bundle is not Bundle::NONE, else this step's keyword (which is the + # constant when nothing was passed), and Tracing.correlate is handed the same bundle, so a + # populated one pushes its trace and span ids onto the diagnostic context (OBS-23). The + # meter has no bundle source: CTX-14's bundle carries no meter and the two instruments are + # created once at .build (OBS-31), so the meter's precedence is the keyword, then the + # constant, and that is the whole of it. A call that seeds no bundle resolves exactly as + # phase 5 did -- the keyword and Bundle::NONE, OBS-34's and XCUT-19(e)'s own default + # configuration. The span is still named by the request's method token, which is also what + # the tracer factory is asked for: the operation identifier is 4a's + # RequestContext#operation_name and the cursor's bundle carries no name (P6-52). # # The level guard is read once at the top of #call and consulted at three sites -- the two # emissions and the body wrapping -- and a step that re-read it inside a helper would be one @@ -124,8 +128,9 @@ def stage # @return [Dexpace::Response] the response, its body wrapped at the body level def call(request, cursor) started = @clock.monotonic - span = open_span(request) - scope = Tracing.correlate(span, Bundle::NONE) + bundle = cursor.bundle + span = open_span(request, bundle) + scope = Tracing.correlate(span, bundle) begin request = prepare(request) response = cursor.call(request) @@ -144,11 +149,13 @@ def call(request, cursor) private - # 4a's #tracer, then 5c's #start_span, both named by the method token until phase 6a's - # Task 8 makes the operation name reachable (see the class comment). - def open_span(request) + # 4a's #tracer, then 5c's #start_span, both named by the method token (see the class + # comment). The factory is the cursor's bundle's when the call seeded one, else this + # step's own: the reconciled precedence's first clause, live since phase 6a's Task 8. + def open_span(request, bundle) name = request.method.to_s - @tracer_factory.tracer(name: name).start_span(name) + factory = bundle.equal?(Bundle::NONE) ? @tracer_factory : bundle.tracer_factory + factory.tracer(name: name).start_span(name) end # 5c's #finish and the two instruments, unwrapped. `attributes:` is optional on both diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb b/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb index cf3e1fd..2fc2ae2 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb @@ -80,11 +80,12 @@ def with_span(span) # the push is skipped and activation delegates to plain current-span activation, in the # requirement's own words. And for a bundle that is not #valid? it likewise delegates: an # all-zero id "MUST be treated as invalid/no-trace" (OBS-26), and a recording span paired - # with Bundle::NONE is the only state 5b's step can reach in phase 5 (R11) -- without the - # guard every log event of a tracing-enabled client would carry trace.id=<32 zeros>, a - # fake trace and worse than an absent key. Bundle#valid? is 4a's derived predicate and is - # exactly this test; once phase 6a's Task 8 makes a populated bundle reachable, the guard - # stops firing. NO_SCOPE when the span and both key values are already in place. + # with Bundle::NONE was the only state 5b's step could reach in phase 5 (R11) -- without + # the guard every log event of a tracing-enabled client would carry trace.id=<32 zeros>, + # a fake trace and worse than an absent key. Bundle#valid? is 4a's derived predicate and + # is exactly this test; since phase 6a's Task 8 made a populated bundle reachable through + # Cursor#bundle, the guard fires only for a call that seeded none. NO_SCOPE when the span + # and both key values are already in place. # # @param span [Object] the _Span to make current # @param bundle [Bundle] the correlation bundle whose ids are pushed diff --git a/gems/dexpace-core/lib/dexpace/pipeline.rb b/gems/dexpace-core/lib/dexpace/pipeline.rb index 9b473c8..6aee352 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline.rb @@ -4,6 +4,7 @@ require_relative "closeable" require_relative "http/request_options" require_relative "cancellation" +require_relative "instrumentation/bundle" module Dexpace # The synchronous stage-based execution pipeline (design §5.1): sixteen totally ordered stages @@ -82,15 +83,22 @@ def initialize(entries:, transport:, driver_class:) # holds with no cursor allocated at all. There is no reading under which a NON-empty pipeline # may skip the cursor (R12). # + # `bundle:` is phase 6a's widening (Task 8): the per-call instrumentation bundle the cursor + # carries to every step (Cursor#bundle), Bundle::NONE when omitted. An optional keyword + # beside the transport SPI's three positionals, which Transport.conforms? still accepts, so + # a Pipeline stays a Dexpace::Transport by the duck type (P4-38's reasoning, widened, P6-51). + # # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] the caller's per-call options, threaded unchanged # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call correlation bundle # @return [Dexpace::Response] - def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none) + def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none, + bundle: Instrumentation::Bundle::NONE) return @transport.call(request, options, cancellation) if @entries.empty? Cursor.build(drive: @driver_class.new(self), request: request, options: options, - cancellation: cancellation,).call(request) + cancellation: cancellation, bundle: bundle,).call(request) end end end diff --git a/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb b/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb index a807aa2..baee4cf 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb @@ -44,14 +44,14 @@ def entry_at(index) = @pipeline.entries.fetch(index) # PIPE-13 on the async side: the step's future, or the transport's, or a future failed with # whatever the invocation raised (PIPE-30). - def advance(position:, request:, options:, cancellation:, state:) + def advance(position:, request:, options:, cancellation:, bundle:, state:) entries = @pipeline.entries return dispatch(request, options, cancellation) if position >= entries.size normalise(terminal: false) do cursor = Cursor.send(:new, drive: self, owner_index: position, position: position + 1, request: request, options: options, - cancellation: cancellation, state: state,) + cancellation: cancellation, bundle: bundle, state: state,) entries.fetch(position).step.call(request, cursor) end end diff --git a/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb b/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb index 290da69..7813661 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb @@ -4,6 +4,7 @@ require_relative "stage" require_relative "../error/pipeline_error" require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/bundle" module Dexpace class Pipeline @@ -35,6 +36,18 @@ class Pipeline # Per-call state lives HERE and is passed as an argument, never read from ambient storage # (PIPE-11): not Fiber[], not Thread.current[]. A step that spawns a thread or a fiber hands # it the cursor explicitly; nothing is inherited. + # + # #bundle is phase 6a's widening (its Task 8): the per-call instrumentation bundle CTX-14 + # names, seeded by the one optional `bundle:` keyword on Pipeline#call and AsyncPipeline#call, + # carried across #fork exactly as #options is, and Bundle::NONE when nothing seeded it. It is + # what lets 5b's instrumentation step honour the reconciled precedence's first clause -- the + # request's own bundle before the step's constructor keyword -- which had no implementation + # path in phase 5 (4c consumed nothing of 4a; Request carries no bundle; PIPE-11 forbids + # ambient carriage). A read-only member and a widening under NFR-4: no existing signature + # moved, and a caller who passes nothing gets exactly what phase 5 gave it. It is NOT the + # retry step's HTTP-tracer source: OBS-29's per-operation tracer is a different kind of + # object from Bundle#tracer_factory (P6-7), and the retry drivers take their own factory, + # called with the cursor itself. class Cursor # One shared frozen hash for a stage nothing wrote, and one for the root cursor's map, so # neither the empty read nor the root allocates (verified fact 4). @@ -54,6 +67,10 @@ class Cursor # @return [Dexpace::Cancellation] the token a step checks at every resume point and the # terminal dispatch is handed attr_reader :cancellation + # @return [Dexpace::Instrumentation::Bundle] the per-call correlation bundle, the same + # frozen object at every position and across every fork; Bundle::NONE when the call + # seeded none (CTX-14, phase 6a's Task 8) + attr_reader :bundle # The root cursor for one send: bound to no entry, at position 0, with empty state. There is # no owner_index:, position: or state: keyword here and a caller cannot name one -- those @@ -63,19 +80,29 @@ class Cursor # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call bundle; NONE by default # @return [Dexpace::Pipeline::Cursor] - def self.build(drive:, request:, options:, cancellation:) + # @raise [Dexpace::InvalidArgumentError] for a bundle: that is not a Bundle + def self.build(drive:, request:, options:, cancellation:, + bundle: Instrumentation::Bundle::NONE) + unless bundle.is_a?(Instrumentation::Bundle) + raise InvalidArgumentError, + "bundle: takes an Instrumentation::Bundle, got #{bundle.class}" + end + new(drive: drive, owner_index: -1, position: 0, request: request, options: options, - cancellation: cancellation, state: EMPTY_STATE,) + cancellation: cancellation, bundle: bundle, state: EMPTY_STATE,) end - def initialize(drive:, owner_index:, position:, request:, options:, cancellation:, state:) + def initialize(drive:, owner_index:, position:, request:, options:, cancellation:, bundle:, + state:) @drive = drive @owner_index = owner_index @position = position @request = request @options = options @cancellation = cancellation + @bundle = bundle @state = state @spent = false end @@ -125,7 +152,7 @@ def call(request = @request) @spent = true @drive.advance(position: @position, request: request, options: @options, - cancellation: @cancellation, state: @state,) + cancellation: @cancellation, bundle: @bundle, state: @state,) end # PIPE-15, PIPE-16. A fresh cursor at the SAME position as this one, carrying the current @@ -172,7 +199,7 @@ def fork(state: nil) Cursor.send(:new, drive: @drive, owner_index: @owner_index, position: @position, request: @request, options: @options, cancellation: @cancellation, - state: merged_state(stage, state),) + bundle: @bundle, state: merged_state(stage, state),) end private diff --git a/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb b/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb index 54e6228..4ea2b2e 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb @@ -25,7 +25,7 @@ def entry_at(index) = @pipeline.entries.fetch(index) # PIPE-13: invoke the entry at `position` with a cursor bound to it, or -- past the last # entry -- dispatch to the terminal transport with the in-flight request, the caller's # options and the token (PIPE-17). - def advance(position:, request:, options:, cancellation:, state:) + def advance(position:, request:, options:, cancellation:, bundle:, state:) entries = @pipeline.entries return @pipeline.transport.call(request, options, cancellation) if position >= entries.size @@ -33,7 +33,7 @@ def advance(position:, request:, options:, cancellation:, state:) # entry it is invoking, at the position after it). cursor = Cursor.send(:new, drive: self, owner_index: position, position: position + 1, request: request, options: options, cancellation: cancellation, - state: state,) + bundle: bundle, state: state,) entries.fetch(position).step.call(request, cursor) end end diff --git a/gems/dexpace-core/sig/dexpace/async_pipeline.rbs b/gems/dexpace-core/sig/dexpace/async_pipeline.rbs index 1e76d98..54b17f6 100644 --- a/gems/dexpace-core/sig/dexpace/async_pipeline.rbs +++ b/gems/dexpace-core/sig/dexpace/async_pipeline.rbs @@ -19,6 +19,6 @@ module Dexpace def self.direct: (untyped transport) -> AsyncPipeline def self.map_response: (Dexpace::Async::Future source) { (Dexpace::Response response) -> untyped } -> Dexpace::Async::Future - def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation) -> Dexpace::Async::Future + def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation, ?bundle: Instrumentation::Bundle) -> Dexpace::Async::Future end end diff --git a/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs b/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs index 5b12586..35bbda3 100644 --- a/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs +++ b/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs @@ -33,7 +33,7 @@ module Dexpace def self.level!: (untyped level, untyped preview_bytes) -> void # untyped: a _Span is what the tracer returns and `_Span & Object` is what Tracing takes # (5c's identity device); the step passes one straight to the other. - def open_span: (Dexpace::Request request) -> untyped + def open_span: (Dexpace::Request request, Bundle bundle) -> untyped def finish: (untyped span, Float started) -> void def elapsed_ms: (Float started) -> Float def logged?: () -> bool diff --git a/gems/dexpace-core/sig/dexpace/pipeline.rbs b/gems/dexpace-core/sig/dexpace/pipeline.rbs index c04d474..b6d930b 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline.rbs @@ -19,6 +19,6 @@ module Dexpace def self.builder: (transport: untyped) -> Builder def self.direct: (untyped transport) -> Pipeline - def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation) -> Dexpace::Response + def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation, ?bundle: Instrumentation::Bundle) -> Dexpace::Response end end diff --git a/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs b/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs index 621202f..0c224f6 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs @@ -10,7 +10,7 @@ module Dexpace def initialize: (AsyncPipeline pipeline) -> void def entry_at: (Integer index) -> Entry - def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> Dexpace::Async::Future + def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> Dexpace::Async::Future def dispatch: (Dexpace::Request request, Dexpace::RequestOptions options, Dexpace::Cancellation cancellation) -> Dexpace::Async::Future private diff --git a/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs b/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs index 55a6355..ece37dd 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs @@ -10,20 +10,23 @@ module Dexpace @request: Dexpace::Request @options: Dexpace::RequestOptions @cancellation: Dexpace::Cancellation + @bundle: Instrumentation::Bundle @state: Hash[Stage, Hash[untyped, untyped]] @spent: bool attr_reader request: Dexpace::Request attr_reader options: Dexpace::RequestOptions attr_reader cancellation: Dexpace::Cancellation + # Phase 6a's Task 8: the per-call instrumentation bundle, a widening. + attr_reader bundle: Instrumentation::Bundle # .new is private and takes owner_index:/position:/state: a caller cannot name; only .build # is public, and it produces the cursor bound to no entry (owner_index -1): it advances INTO # entry 0 and cannot fork (R10). Every cursor a step is handed is made by the driver. - private def self.new: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> instance - def initialize: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> void + private def self.new: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> instance + def initialize: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> void - def self.build: (drive: untyped, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation) -> Cursor + def self.build: (drive: untyped, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, ?bundle: Instrumentation::Bundle) -> Cursor def spent?: () -> bool def may_fork?: () -> bool diff --git a/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs b/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs index f7ed353..56e44de 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs @@ -10,7 +10,7 @@ module Dexpace def initialize: (Pipeline pipeline) -> void def entry_at: (Integer index) -> Entry - def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> untyped + def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> untyped end end end diff --git a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb index aa05e83..06a667e 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb @@ -223,6 +223,7 @@ class SettlementTest < DexpaceTestCase meter: meter,) raising_cursor = ::Object.new def raising_cursor.call(_request) = raise(::IOError, "synchronous head failure") + def raising_cursor.bundle = Dexpace::Instrumentation::Bundle::NONE assert_raises(::IOError) { step.call(build_request, raising_cursor) } assert_equal(1, factory.tracers.first.spans.first.finished_at.size) @@ -289,6 +290,7 @@ class SharedShapeTest < DexpaceTestCase completer.future end cursor.define_singleton_method(:fork) { |state: nil| raise "forked with #{state.inspect}" } + cursor.define_singleton_method(:bundle) { Dexpace::Instrumentation::Bundle::NONE } request = build_request assert_same(completer.future, step.call(request, cursor)) diff --git a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb index a15d6f4..bd4cc48 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb @@ -142,6 +142,7 @@ def call(request, _options, _cancellation) response end cursor.define_singleton_method(:fork) { |state: nil| raise "forked with #{state.inspect}" } + cursor.define_singleton_method(:bundle) { Dexpace::Instrumentation::Bundle::NONE } assert_same(response, step.call(request, cursor)) assert_equal([request], cursor.instance_variable_get(:@calls)) diff --git a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb index d3d85c3..516e406 100644 --- a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb +++ b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb @@ -310,7 +310,9 @@ class StateTest < DexpaceTestCase methods = CURSOR.public_instance_methods(false) assert_empty(methods.grep(/=\z/), "no writer of any name may appear on Cursor") - assert_equal(%i[call cancellation fork may_fork? options request spent? state], methods.sort) + # #bundle is phase 6a's Task 8 reader; the pin grew by one reader and no writer. + assert_equal(%i[bundle call cancellation fork may_fork? options request spent? state], + methods.sort,) end test "R11 assertion 2: the reachable state map is frozen, and unwritten stages share one" do From 1d2e12aebfe58b3bf307da490be48be96a247774 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:13:59 +0300 Subject: [PATCH 03/13] feat: the stage-based retry step on both runtimes Add Resilience::RetryStep and Resilience::AsyncRetryStep at Stages::RETRY, over the private RetryStepHelpers mixin that holds the decision, the delay resolution, the two override hooks and the trail handling both share. Each drives the downstream chain once per attempt through a fresh Cursor#fork -- the first drive included -- so hop 1 and hop n are one object and each attempt re-executes the chain with fresh per-attempt state (RETRY-44); neither writes cursor state of its own. The synchronous step waits through 5a's cancellable Clock#sleep and checks the token at the top of every attempt. The async step is a trampoline, not a recursion: one Pump per call resumes under a re-arm flag held by a Thread::Mutex across the flip only, so an inline settlement re-enters #resume without growing the stack (the design's recursive sketch overflows at about 1,500 attempts on every supported Ruby); a positive delay goes through Dexpace::Async.delay, which without a Fiber.scheduler raises SeamError synchronously and fails the future with the trail attached (R2's third route), and a zero-length delay continues inline. A settled fatal answering #retryable? is delivered unclassified rather than retried (P6-55). OBS-29's per-attempt group is emitted through the tracer the `http_tracer_factory:` keyword produces once per operation, called with the cursor (R3, P6-7); retries_exhausted fires only when a retryable failure met a spent budget. The interface _HTTPTracer phase 5c declined to declare arrives in the http_tracer sig with the wiring, `context` untyped because the recovery stack hands a Request where the stage stack hands a Cursor. The error-status response is closed before the wait and before any throwable propagates (RETRY-35); no total-timeout is named on this stack (RETRY-28, P6-5). --- gems/dexpace-core/lib/dexpace.rb | 3 + .../dexpace/instrumentation/http_tracer.rb | 15 +- .../dexpace/resilience/async_retry_step.rb | 288 ++++++++++++++++++ .../lib/dexpace/resilience/retry_step.rb | 187 ++++++++++++ .../dexpace/resilience/retry_step_helpers.rb | 162 ++++++++++ .../dexpace/instrumentation/http_tracer.rbs | 30 +- .../dexpace/resilience/async_retry_step.rbs | 63 ++++ .../sig/dexpace/resilience/retry_step.rbs | 47 +++ .../dexpace/resilience/retry_step_helpers.rbs | 35 +++ 9 files changed, 822 insertions(+), 8 deletions(-) create mode 100644 gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/retry_step.rb create mode 100644 gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb create mode 100644 gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs create mode 100644 gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs diff --git a/gems/dexpace-core/lib/dexpace.rb b/gems/dexpace-core/lib/dexpace.rb index 16119fd..2046aaf 100644 --- a/gems/dexpace-core/lib/dexpace.rb +++ b/gems/dexpace-core/lib/dexpace.rb @@ -205,6 +205,9 @@ require_relative "dexpace/resilience/policy" require_relative "dexpace/resilience/resend" require_relative "dexpace/resilience/retry_settings" +require_relative "dexpace/resilience/retry_step_helpers" +require_relative "dexpace/resilience/retry_step" +require_relative "dexpace/resilience/async_retry_step" # The dexpace Ruby SDK: an HTTP-client toolkit, not an HTTP client. # diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb b/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb index 99559b9..2dd41c3 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb @@ -20,10 +20,17 @@ module Instrumentation # SAME error object; and one tracer instance corresponds 1:1 to one logical operation # lifecycle, created by its factory per operation. ordering_test.rb drives a conformant # emitter through both a succeeding and a retry-exhausted operation, which is the - # requirement's own conformance clause. Nothing in phase 5 emits any of it -- the per-attempt - # group is phase 6a's retry step (its Task 9), the transport milestones are phase 8's, and - # the operation triple is on phase 10's inbound list -- which OBS-29's own last sentence - # anticipates: "pipeline/transport wiring to emit it is a follow-up" (R14). + # requirement's own conformance clause. The per-attempt group is emitted by phase 6a's three + # retry drivers (Resilience::RetryStep, ::AsyncRetryStep and ::RecoveryRetry), through the + # tracer their `http_tracer_factory:` produces once per operation -- called with the + # Pipeline::Cursor on the stage stack and with the Dexpace::Request on the recovery stack -- + # and retries_exhausted fires there only when a RETRYABLE failure met a spent budget, never + # for a failure that was never retryable (ordering_test.rb's third case). The transport + # milestones and the operation-lifecycle triple are emitted by nothing in v1 (phase 10's + # inbound list; docs/first-release.md's behavioural-asymmetries entry), which OBS-29's own + # last sentence anticipates: "pipeline/transport wiring to emit it is a follow-up" (R14). + # The interface _HTTPTracer, which 5c deliberately did not declare, arrived with the wiring + # (6a's R3) in this file's sig/ mirror. # # OBS-30 binds every implementation: each callback is safe to invoke concurrently and from # a transport thread other than the caller's, and never throws -- the runtime does not diff --git a/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb new file mode 100644 index 0000000..6638af0 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb @@ -0,0 +1,288 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "retry_step_helpers" +require_relative "../async/completer" +require_relative "../async/future" +require_relative "../async/delay" +require_relative "../error/cancelled_error" +require_relative "../error/seam_error" +require_relative "../pipeline/stages" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The stage-based retry step, asynchronous half: the same policy, the same decision and the + # same delay resolution as RetryStep (one RetryStepHelpers, so the two cannot drift, design + # §11.12) over 4c's _AsyncStep contract -- #call returns a Dexpace::Async::Future and every + # downstream drive is a fork's future. Declares Stages::RETRY, forks for every drive + # (pipeline/86343352) and writes no cursor state. + # + # RETRY-30: an ITERATIVE trampoline, never a recursive one. Each #call allocates one private + # Pump holding the per-call state (RETRY-42, RECOV-28: nothing on the step). The pump runs + # attempts in a `loop` on whatever frame is driving it; a downstream future or a delay + # future that settles INLINE -- which every scripted transport does, and which Async.delay + # does for a zero-length delay (RETRY-31) -- does not call back into a nested attempt but + # flips a RE-ARM flag under the pump's mutex, and the loop already running picks the next + # attempt up when the current one returns. Only a future that settles LATER, on another + # thread or a scheduler fiber, starts the loop again from its callback, on a fresh stack. + # Measured: the plan's recursive shape overflowed at ~1500 attempts on every interpreter; + # this one is flat at 2000 (async_retry_step_test.rb asserts the depth, not just the count). + # + # R2's route, as built: a positive delay goes through 5a's Async.delay, which unmounts the + # fiber under a registered Fiber.scheduler and RAISES Dexpace::SeamError synchronously with + # none -- the raise is caught by RETRY-33's fence and fails the returned future, carrying the + # prior trail; a zero-length delay completes inline and re-arms the pump without a frame; and + # nothing here installs, reads or shuts down a scheduler (RETRY-45). Never Clock#sleep: a + # blocking wait on the async path is what RETRY-31 and RETRY-26 forbid. + # + # RETRY-33: every callback body runs inside one fence that closes any open retryable + # response and fails the completer on a StandardError -- a throwing predicate, delay + # computation, log call, tracer callback or scheduler rejection each completes the future + # exceptionally rather than escaping into whoever settled the downstream future and leaving + # the returned future hanging. The fatal family is re-raised after the completer is failed + # (RETRY-25), because a hung future is not an acceptable price for propagating it. + # + # RETRY-32: once the returned future is settled or cancelled -- by the caller, by a deadline + # -- no further attempt is launched, a pending delay is cancelled, and a response arriving + # from an attempt still in flight is closed rather than leaked. RETRY-23 / RECOV-27: the + # cursor's token is checked at the top of every attempt and, on cancellation during a wait, + # cancels the pending delay and fails the future with the CancelledError, the trail attached. + class AsyncRetryStep + include RetryStepHelpers + + private_class_method :new + + # Builds a frozen step; the keywords are RetryStep.build's, with the same meanings. + # + # @param settings [RetrySettings] + # @param http_tracer_factory [#call] called once per operation with the cursor + # @param delay_override [#call, nil] RETRY-39's first tier + # @param should_retry [#call, nil] the caller's condition predicate + # @param logger [Instrumentation::Logger] + # @return [AsyncRetryStep] frozen + # @raise [Dexpace::InvalidArgumentError] as RetryStep.build + def self.build(settings: RetrySettings.build, http_tracer_factory: NULL_TRACER_FACTORY, + delay_override: nil, should_retry: nil, + logger: Instrumentation::Logger::NULL) + new(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, logger: logger,).freeze + end + + NULL_TRACER_FACTORY = ->(_cursor) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + end + + # 4c's optional declaration: the Stages::RETRY pillar. + # + # @return [Dexpace::Pipeline::Stage] + def stage + Pipeline::Stages::RETRY + end + + # One operation, as a future that settles with the terminal response or fails with the + # terminal throwable (its trail attached), or with the CancelledError of a cancelled token. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt (RETRY-44) + # @param cursor [Dexpace::Pipeline::Cursor] forked for every drive + # @return [Dexpace::Async::Future] + def call(request, cursor) + Pump.new(settings: @settings, http_tracer_factory: @http_tracer_factory, + delay_override: @delay_override, should_retry: @should_retry, logger: @logger, + request: request, cursor: cursor,).start + end + + # The per-call state and the trampoline (RETRY-30). One instance per #call, carrying the + # step's five configuration values and the same helpers, reachable from nothing but the + # callbacks it registers; a private_constant with a sig/ mirror. Over the class-length + # default by fifteen lines, and deliberately one class: the re-arm protocol, the fence and + # the terminal paths share the completer and the flags, and splitting them across objects + # would scatter the one invariant the trampoline rests on (the recorded exception + # .rubocop.yml's metric note asks for). + class Pump # rubocop:disable Metrics/ClassLength -- one per-call state machine, see above + include RetryStepHelpers + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:, + request:, cursor:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + @request = request + @cursor = cursor + @completer = Dexpace::Async::Completer.new + @tracer = Instrumentation::NULL + @max_retries = 0 + @attempt = 1 + @trail = [] + @mutex = ::Thread::Mutex.new + @running = false + @rearm = false + @pending_delay = nil + end + + # Arms the cancellation bridges, runs the first attempt, and returns the future. + def start + guarded(nil) do + @tracer = @http_tracer_factory.call(@cursor) + @max_retries = effective_max_retries(@cursor) + @completer.on_cancel { @pending_delay&.cancel } + subscription = @cursor.cancellation.on_cancel { |reason| abort(reason) } + @completer.future.on_settle { subscription.detach } + resume + end + @completer.future + end + + private + + # The re-arm protocol (RETRY-30), called from the frame that wants the next attempt to + # run. When a pump loop is already running on some frame, set the flag and return -- + # that loop picks the attempt up when its current one returns; otherwise become that + # loop, and keep launching until nothing re-armed it. The flags flip under the mutex + # only, never across an attempt, so a settlement from another thread cannot lose its + # re-arm between the loop's last check and its exit. + def resume + return unless claim(:start) + + loop do + launch + break unless claim(:continue) + end + end + + # :start claims the loop when none runs (else re-arms and answers false); :continue + # consumes a re-arm (else releases the loop and answers false). + def claim(step) + @mutex.synchronize do + if step == :start && @running + @rearm = true + false + elsif step == :start + @running = true + elsif @rearm + @rearm = false + true + else + @running = false + end + end + end + + # One attempt: the token check, the tracer, a fresh fork, and the settlement callback -- + # which runs inline when the downstream settled synchronously. + def launch + return if @completer.settled? + + guarded(nil) do + @cursor.cancellation.check! + @tracer.attempt_started(@cursor, @attempt) + @cursor.fork.call(@request).on_settle { |settlement| settled(settlement) } + end + end + + # A downstream settlement: RETRY-32's abandoned-attempt close first, then a terminal + # success or a terminal fatal, then the decision. + def settled(settlement) + response = settlement.response + guarded(response) do + next Dexpace.close_quietly(response, logger: @logger) if @completer.settled? + next if delivered?(response, settlement.error) + + failure = failure_of(response, settlement.error) + verdict = decision(@request, failure, @attempt, @max_retries) + next finish(response, failure, verdict) unless verdict == RETRY + + retry_after(response, settlement.error, failure) + end + end + + # The two settlements that need no decision: a non-error response is delivered, and a + # fatal-family error -- which on this path arrives as a settlement and meets no rescue + # arm -- is delivered unclassified, unretried and with no trail attached (RETRY-25), + # rather than reaching the capability query a lying subclass could answer. + def delivered?(response, error) + if error.nil? + return false if response.nil? || response.status.error? + + @completer.fulfil(response) + else + return false if error.is_a?(::StandardError) + + @completer.fail(error) + end + true + end + + # The retry path: the delay from the still-open response, the tracer, the trail, the + # close (RETRY-35), then the wait. + def retry_after(response, error, failure) + delay = resolve_delay(@attempt, response, error) + @tracer.attempt_failed(@cursor, failure, delay) + @trail << failure + Dexpace.close_quietly(response, onto: failure) + wait(delay) + end + + # The terminal path, mirroring RetryStep#settle: a response is delivered as it is, a + # throwable with the trail attached, retries_exhausted first when the budget stopped it. + def finish(response, failure, verdict) + return @completer.fulfil(response) if response && verdict == STOP + + attach_trail(failure, @trail) + @tracer.retries_exhausted(@cursor, failure) if verdict == EXHAUSTED + response ? @completer.fulfil(response) : @completer.fail(failure) + end + + # RETRY-31 / R2: the wait is Async.delay's future. Zero-length settles inline, so the + # callback runs now and re-arms the running loop; positive under a scheduler settles on + # the scheduler's fiber, whose callback starts a fresh loop; positive with no scheduler + # raises SeamError here, synchronously, into the fence around this method's caller. + def wait(delay) + @attempt += 1 + future = Dexpace::Async.delay(delay) + @pending_delay = future + future.on_settle do |elapsed| + @pending_delay = nil + guarded(nil) do + if elapsed.error + @completer.fail(attach_trail(elapsed.error, @trail)) + elsif !@completer.settled? # RETRY-32 + resume + end + end + end + end + + # RETRY-23 / RECOV-27 during a wait: cancel the pending timer and surface the + # cancellation; a no-op once the future is settled. + def abort(reason) + return if @completer.settled? + + @pending_delay&.cancel(reason) + @completer.fail(attach_trail(Dexpace::CancelledError.new(reason), @trail)) + end + + # RETRY-33's fence, and RETRY-25's: close the open response, fail the future, and let a + # fatal-family error propagate after the future is settled. The trail travels with the + # failure the fence reports. + def guarded(response) + yield + rescue ::StandardError => error + Dexpace.close_quietly(response, onto: error) + @completer.fail(attach_trail(error, @trail)) + rescue ::Exception => error # rubocop:disable Lint/RescueException -- RETRY-25 with RETRY-33: fail, then re-raise unchanged + Dexpace.close_quietly(response, logger: @logger) + @completer.fail(error) + raise + end + end + private_constant :Pump + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb new file mode 100644 index 0000000..815afa2 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "retry_step_helpers" +require_relative "../error/cancelled_error" +require_relative "../pipeline/stages" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The stage-based retry step, synchronous half: chapter 9's second stack at Stages::RETRY + # (order 500, between REDIRECT and AUTH, PIPE-2), driving the downstream chain once per + # attempt through a FRESH fork of its cursor -- for every drive, the first included, and + # never through its own cursor's #call (pipeline/86343352, P4-39) -- so hop 1 and hop n are + # the same object and each attempt re-executes the chain with fresh per-attempt state + # (RETRY-44). It writes no cursor state of its own (boundary 2: the cross-origin marker is + # 6b's and 6c's), so a RETRY fork's slot stays empty under every stage's key. + # + # RETRY-42: constructed once, frozen, shared by every call a pipeline serves, and stateless + # -- the attempt counter, the trail and the tracer are locals of one #call. The wait is 5a's + # cancellable queue wait, Clock#sleep(duration, cancellation:) on the settings' clock, never + # Kernel#sleep and never an interrupt (RETRY-26, CFG-15, boundary 5), and the cancellation + # token is checked at the top of EVERY attempt as well as by the wait (RETRY-23, RECOV-27). + # No total-timeout: nothing in this file or in RetryStepHelpers names + # RetrySettings#total_timeout or Policy.budget_remaining, which is RETRY-28 as a property of + # the source (R6, P6-5). + # + # OBS-29's per-attempt group is emitted through the tracer `http_tracer_factory:` produces + # -- called ONCE per #call, with the cursor as its argument and as the `context` every + # callback receives (R3, P6-7) -- and never through Cursor#bundle, whose tracer factory + # produces span tracers (a different kind of object). attempt_started before every drive, + # attempt_failed with the resolved delay before every wait, and retries_exhausted only when + # a RETRYABLE failure met a spent budget; the operation-lifecycle triple is emitted by + # nothing in v1 (docs/first-release.md's behavioural-asymmetries entry). A throwing tracer + # propagates and fails the request: OBS-30 puts that on the implementer, and core wraps no + # tracer callback in a rescue (OBS-20's carve-out). + # + # RETRY-35's three orderings, in #call: the delay is resolved from the still-open response + # FIRST, the response is closed BEFORE the wait, and both the retry decision and the delay + # resolution are fenced so the response is closed before any throwable -- a + # RetryPredicateError above all -- propagates. RETRY-34 on the terminal path: the whole + # prior trail is attached to the instance actually surfaced, through + # Dexpace.attach_suppressed, and is discarded on success or on a returned error-status + # response, which is a Response and not a throwable to attach to (stated, not dropped). + class RetryStep + include RetryStepHelpers + + private_class_method :new + + # Builds a frozen step. + # + # @param settings [RetrySettings] the shared configuration; the defaults when omitted + # @param http_tracer_factory [#call] called once per operation with the cursor, answering + # an OBS-28 HTTP tracer; the frozen no-op NULL by default (R3, P6-7) + # @param delay_override [#call, nil] RETRY-39's first tier: called with + # (attempt, response, error) and answering a delay in seconds or nil to decline; a + # raise or a bad answer is logged and falls through (RETRY-40) + # @param should_retry [#call, nil] the caller's condition predicate, called with + # (failure, request) in place of the shared classifier; a raise aborts the call as + # RetryPredicateError (RETRY-40); it can never override the re-sendability gate (RETRY-8) + # @param logger [Instrumentation::Logger] where RETRY-40's fallback and RETRY-41's clamp + # are reported, contained; Logger::NULL by default + # @return [RetryStep] frozen + # @raise [Dexpace::InvalidArgumentError] for a settings that is not a RetrySettings, a + # non-callable factory or hook, or a logger that is not a Logger + def self.build(settings: RetrySettings.build, http_tracer_factory: NULL_TRACER_FACTORY, + delay_override: nil, should_retry: nil, + logger: Instrumentation::Logger::NULL) + new(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, logger: logger,).freeze + end + + # The default factory: the shared no-op, whatever the cursor. + NULL_TRACER_FACTORY = ->(_cursor) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + end + + # 4c's optional declaration, read once at install: this is the Stages::RETRY pillar. + # + # @return [Dexpace::Pipeline::Stage] + def stage + Pipeline::Stages::RETRY + end + + # One operation: attempts until a success, a non-retryable failure or a spent budget. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt (RETRY-44) + # @param cursor [Dexpace::Pipeline::Cursor] forked for every drive + # @return [Dexpace::Response] the terminal response, error status or not, unclosed + # @raise [Exception] the terminal throwable, carrying the prior trail as suppressed + # @raise [Dexpace::CancelledError] when the token is cancelled at an attempt boundary or + # during a wait + # @raise [Dexpace::RetryPredicateError] when a caller's should_retry raised + def call(request, cursor) + run = Run.new(tracer: @http_tracer_factory.call(cursor), cursor: cursor, + max_retries: effective_max_retries(cursor), trail: [],) + attempt = 1 + loop do + terminal = attempt_once(run, request, attempt) + return terminal unless terminal.nil? + + attempt += 1 + end + end + + # The per-call locals one operation threads through its helpers (RETRY-42: on the stack, + # never on the step), a private value so the helpers take one argument for the four. + class Run < Data.define(:tracer, :cursor, :max_retries, :trail) + end + private_constant :Run + + private + + # One attempt, start to finish: the token check, the tracer, the drive, the decision and + # -- on a retry -- the wait. Answers the terminal response, or nil for "again". + def attempt_once(run, request, attempt) + response, error = drive(run, request, attempt) + return response if !response.nil? && !response.status.error? + + failure = failure_of(response, error) + verdict = fenced(response) { decision(request, failure, attempt, run.max_retries) } + return settle(run, response, failure, verdict) unless verdict == RETRY + + wait(run, attempt, response, failure) + end + + # The retry path: the delay from the still-open response (fenced, RETRY-35), the tracer, + # the trail, the close BEFORE the wait, then the wait on the settings' clock. On the + # exception path the failure IS the error the override sees; on the response path the + # override sees the response and no error (RETRY-39). + # + # @return [nil] always: the loop's "again" + def wait(run, attempt, response, failure) + delay = fenced(response) { resolve_delay(attempt, response, response ? nil : failure) } + run.tracer.attempt_failed(run.cursor, failure, delay) + run.trail << failure + Dexpace.close_quietly(response, onto: failure) + @settings.clock.sleep(delay, cancellation: run.cursor.cancellation) + end + + # One drive: the token check at the attempt boundary (RETRY-23), the tracer, then the + # downstream through a fresh fork. Only the drive's StandardError is rescued: the fatal + # family propagates through the absent arm, unclassified and unretried (RETRY-25), and a + # CancelledError from the check is not a drive failure and propagates as itself. + # + # @return [Array] the response and nil, or nil and the error + def drive(run, request, attempt) + run.cursor.cancellation.check! + run.tracer.attempt_started(run.cursor, attempt) + begin + [run.cursor.fork.call(request), nil] + rescue ::StandardError => error + [nil, error] + end + end + + # RETRY-35's third ordering: whatever the block raises -- fatal family included -- the + # open response is closed first and the raise propagates unchanged. + def fenced(response) + yield + rescue ::Exception # rubocop:disable Lint/RescueException -- close, then re-raise unchanged (RETRY-25, RETRY-35) + Dexpace.close_quietly(response, logger: @logger) + raise + end + + # The terminal path. A throwable is surfaced with the whole trail attached and, when the + # budget is what stopped it, retries_exhausted first; a response is returned as it is, its + # trail discarded, since a Response carries no suppressed trail to attach to (RETRY-34) -- + # the ProtocolError the tracer sees for an exhausted error status carries it instead. + def settle(run, response, failure, verdict) + return response if response && verdict == STOP + + attach_trail(failure, run.trail) + run.tracer.retries_exhausted(run.cursor, failure) if verdict == EXHAUSTED + return response if response + + raise failure, cause: nil # pipeline/7ce4431d: a carried error, never a bare raise + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb new file mode 100644 index 0000000..7ac1158 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "policy" +require_relative "resend" +require_relative "retry_settings" +require_relative "../error/protocol_error" +require_relative "../error/retry_predicate_error" +require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/contain" +require_relative "../instrumentation/keys" +require_relative "../instrumentation/logger" +require_relative "../instrumentation/http_tracer" +require_relative "../pipeline/stages" + +module Dexpace + module Resilience + # What RetryStep and AsyncRetryStep share above Policy itself: the constructor's validation, + # the retry DECISION (both axes, the caller's predicate, the budget) and the delay + # RESOLUTION (RETRY-39's precedence with RETRY-40's fatal/non-fatal split), so RETRY-13's + # "one calculator" extends to the logic sitting on it and the two drivers cannot drift + # (design §11.12's rule for every sync/async pair). A private_constant mixin with a sig/ + # mirror -- the strict `core` target types its two includers -- and no test/ mirror: every + # branch is asserted through the two steps (P6-11, as built). + # + # RETRY-28 in the source: no method here names RetrySettings#total_timeout or + # Policy.budget_remaining, and neither driver does either -- the text scan in + # retry_step_test.rb is the guard. + module RetryStepHelpers + # The three answers a decision can give. Only :exhausted emits retries_exhausted: a failure + # that was never retryable is not an exhausted retry (5c's ordering test, third case). + RETRY = :retry + EXHAUSTED = :exhausted + STOP = :stop + private_constant :RETRY, :EXHAUSTED, :STOP + + private + + # The constructor's validation, shared: the settings are a RetrySettings, the factory and + # the two hooks are callables, the logger is a Logger. + def initialize_retry_step(settings:, http_tracer_factory:, delay_override:, should_retry:, + logger:) + unless settings.is_a?(RetrySettings) + raise InvalidArgumentError, "settings: takes a RetrySettings, got #{settings.class}" + end + unless logger.is_a?(Instrumentation::Logger) + raise InvalidArgumentError, "logger: takes an Instrumentation::Logger, got " \ + "#{logger.class}" + end + + @settings = settings + @http_tracer_factory = callable!("http_tracer_factory", http_tracer_factory) + @delay_override = delay_override.nil? ? nil : callable!("delay_override", delay_override) + @should_retry = should_retry.nil? ? nil : callable!("should_retry", should_retry) + @logger = logger + end + + def callable!(name, value) + return value if value.respond_to?(:call) + + raise InvalidArgumentError, "#{name}: must answer #call" + end + + # RETRY-41 at the top of every call: the per-call override wins over the configured value. + def effective_max_retries(cursor) + Policy.effective_max_retries(override: cursor.options.max_retries, + configured: @settings.max_retries, logger: @logger,) + end + + # RETRY-8: BOTH axes, neither implying the other, in this order -- re-sendability first + # and unconditionally (a caller predicate may widen or narrow the CONDITION and may never + # authorise re-sending a bare POST or a consumed body, RETRY-7), then the condition + # (Policy.retryable? or the caller's predicate, RETRY-39/RETRY-40), then the budget. The + # budget comes LAST so that :exhausted means exactly "retryable, and the budget is spent". + # + # @return [Symbol] RETRY, EXHAUSTED or STOP + def decision(request, failure, attempt, max_retries) + return STOP unless Resend.eligible?(request) + return STOP unless retryable_condition?(failure, request) + + attempt <= max_retries ? RETRY : EXHAUSTED + end + + # The condition axis: the shared classifier by default, the caller's predicate when one + # was supplied -- and a predicate that raises a StandardError aborts the call as + # RetryPredicateError with the raise as its cause (RETRY-40). The fatal family passes + # through the absent arm unchanged (RETRY-25). + def retryable_condition?(failure, request) + if @should_retry.nil? + return Policy.retryable?(failure, retryable_statuses: @settings.retryable_statuses) + end + + begin + @should_retry.call(failure, request) ? true : false + rescue ::StandardError => error + raise RetryPredicateError.new("the should_retry predicate raised #{error.class}"), + cause: error + end + end + + # RETRY-39: caller delay-override, then the server's pacing headers on the response path + # only, then exponential backoff. (The fixed-delay tier between the last two is RETRY-43, + # a MAY declined for v1, so the precedence has three live sources.) `now:` is the settings + # clock's, so a fake clock drives the absolute forms. + def resolve_delay(attempt, response, failure) + overridden = overridden_delay(attempt, response, failure) + return overridden unless overridden.nil? + + if response + hinted = Policy.pacing_delay(response.headers, header_order: @settings.header_order, + now: @settings.clock.now, + random: @settings.random,) + return hinted unless hinted.nil? + end + + Policy.backoff_delay(attempt, **@settings.backoff_arguments) + end + + # RETRY-40's non-fatal half: a delay-override that raises a StandardError, or answers + # something that is not a non-negative number, is logged as one WARNING diagnostic through + # 5b's facade (contained, OBS-20) and the precedence falls through. nil is the documented + # "no opinion" answer and is not logged. The fatal family propagates (RETRY-25). + def overridden_delay(attempt, response, failure) + return nil if @delay_override.nil? + + value = @delay_override.call(attempt, response, failure) + return nil if value.nil? + return Float(value) if value.is_a?(::Numeric) && value.finite? && !value.negative? + + report_override(message: "delay_override answered #{value.inspect}; falling back " \ + "(RETRY-40)") + nil + rescue ::StandardError => error + report_override(cause: error, message: "delay_override raised; falling back (RETRY-40)") + nil + end + + def report_override(message:, cause: nil) + Instrumentation.diagnostic(@logger, event: Instrumentation::Events::INSTRUMENTATION_HOOK, + cause: cause, message: message,) + end + + # The Exception the tracer and the trail carry for an attempt: the throwable itself, or + # the error-status response converted ONCE, so the object the tracer reported is the object + # the trail holds (OBS-28: #attempt_failed's second argument is an Exception). + def failure_of(response, error) + return error unless error.nil? + raise InvalidArgumentError, "an attempt yields a response or an error" if response.nil? + + ProtocolError.for(response) + end + + # RETRY-34's terminal attachment: every prior attempt's failure onto the surfaced instance, + # through the one helper whose skip-self guard is 4b's. + def attach_trail(error, trail) + trail.each { |prior| Dexpace.attach_suppressed(error, prior) } + error + end + end + private_constant :RetryStepHelpers + end +end diff --git a/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs b/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs index 43fccd6..7d41b43 100644 --- a/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs +++ b/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs @@ -1,9 +1,31 @@ module Dexpace module Instrumentation - # OBS-28's eleven-method vocabulary, a module an implementer includes. No `interface - # _HTTPTracer` is declared, deliberately: eleven signatures with no core caller would be - # eleven NFR-4-locked shapes nothing checks, and the module's own declaration is the - # contract; if phase 6 wires the vocabulary, the interface arrives with the wiring. + # OBS-28's eleven-method vocabulary: the module an implementer includes, and -- since phase 6a + # wired the per-attempt group (R3) -- the interface the retry drivers' `http_tracer_factory:` + # keyword produces. Phase 5c declared no interface because eleven NFR-4-locked signatures with + # no core caller would have been shapes nothing checks; the caller exists now, so the + # interface arrives with the wiring, exactly as 5c said it would. `context` stays untyped + # deliberately: OBS-28 fixes the vocabulary and not the correlation handle, and 6a passes a + # Pipeline::Cursor from the two stage drivers and a Dexpace::Request from RecoveryRetry -- + # typing it to either would make the other a Steep error and lock a type no requirement + # names. `error` is an Exception: the drivers convert an error-status response with + # ProtocolError.for before reporting it and never hand the vocabulary a Response. + interface _HTTPTracer + def operation_started: (untyped context) -> void + def operation_succeeded: (untyped context, Dexpace::Response response) -> void + def operation_failed: (untyped context, Exception error) -> void + + def attempt_started: (untyped context, Integer attempt) -> void + def attempt_failed: (untyped context, Exception error, Numeric next_delay) -> void + def retries_exhausted: (untyped context, Exception error) -> void + + def request_url_resolved: (untyped context, untyped url) -> void + def connection_acquired: (untyped context, String host, Integer port) -> void + def request_sent: (untyped context, Integer byte_count) -> void + def response_headers_received: (untyped context, Integer status, untyped headers) -> void + def response_received: (untyped context, Integer byte_count) -> void + end + module HTTPTracer def operation_started: (untyped context) -> void def operation_succeeded: (untyped context, untyped response) -> void diff --git a/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs b/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs new file mode 100644 index 0000000..99b6577 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs @@ -0,0 +1,63 @@ +module Dexpace + module Resilience + # The asynchronous stage-based retry step at Stages::RETRY, over 4c's _AsyncStep. The keyword + # types are RetryStep's. + class AsyncRetryStep + include RetryStepHelpers + + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + + private def self.new: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (?settings: RetrySettings, ?http_tracer_factory: untyped, + ?delay_override: untyped, ?should_retry: untyped, + ?logger: Instrumentation::Logger) -> AsyncRetryStep + + def stage: () -> Pipeline::Stage + def call: (Dexpace::Request request, Pipeline::Cursor cursor) -> Dexpace::Async::Future + + # A private_constant with no visibility in RBS: the per-call trampoline (RETRY-30), + # declared because the strict `core` Steep target types its construction and its body. + class Pump + include RetryStepHelpers + + @request: Dexpace::Request + @cursor: Pipeline::Cursor + @completer: Dexpace::Async::Completer + @tracer: untyped + @max_retries: Integer + @attempt: Integer + @trail: Array[::Exception] + @mutex: ::Thread::Mutex + @running: bool + @rearm: bool + @pending_delay: Dexpace::Async::Future? + + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger, request: Dexpace::Request, + cursor: Pipeline::Cursor) -> void + def start: () -> Dexpace::Async::Future + + private + + def resume: () -> void + def claim: (Symbol step) -> bool + def launch: () -> void + def settled: (Dexpace::Async::Settlement settlement) -> void + def delivered?: (Dexpace::Response? response, ::Exception? error) -> bool + def retry_after: (Dexpace::Response? response, ::Exception? error, ::Exception failure) -> void + def finish: (Dexpace::Response? response, ::Exception failure, Symbol verdict) -> void + def wait: (Float delay) -> void + def abort: (untyped reason) -> void + # The block answers whatever its terminal path answered; nothing reads it. + def guarded: (Dexpace::Response? response) { () -> untyped } -> untyped + end + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs new file mode 100644 index 0000000..454fbe1 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs @@ -0,0 +1,47 @@ +module Dexpace + module Resilience + # The synchronous stage-based retry step at Stages::RETRY (RETRY-1..45's stage half). The + # factory and the two hooks are untyped callables: `^(Pipeline::Cursor) -> _HTTPTracer` is + # what the factory answers, and the hooks' argument shapes are in the YARD; typing a caller's + # lambda strictly here would refuse every arity-tolerant Proc a caller writes (P5-14's + # precedent for the generator). + class RetryStep + include RetryStepHelpers + + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + + private def self.new: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (?settings: RetrySettings, ?http_tracer_factory: untyped, + ?delay_override: untyped, ?should_retry: untyped, + ?logger: Instrumentation::Logger) -> RetryStep + + def stage: () -> Pipeline::Stage + def call: (Dexpace::Request request, Pipeline::Cursor cursor) -> Dexpace::Response + + # A private_constant: the per-call locals, declared for the strict target. + class Run < Data + attr_reader tracer: untyped + attr_reader cursor: Pipeline::Cursor + attr_reader max_retries: Integer + attr_reader trail: Array[::Exception] + + def self.new: (tracer: untyped, cursor: Pipeline::Cursor, max_retries: Integer, + trail: Array[::Exception]) -> instance + end + + private + + def drive: (Run run, Dexpace::Request request, Integer attempt) -> [Dexpace::Response?, ::Exception?] + def fenced: [T] (Dexpace::Response? response) { () -> T } -> T + def attempt_once: (Run run, Dexpace::Request request, Integer attempt) -> Dexpace::Response? + def wait: (Run run, Integer attempt, Dexpace::Response? response, ::Exception failure) -> nil + def settle: (Run run, Dexpace::Response? response, ::Exception failure, Symbol verdict) -> Dexpace::Response + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs new file mode 100644 index 0000000..1796647 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs @@ -0,0 +1,35 @@ +# Dexpace::Resilience::RetryStepHelpers is a private_constant and not public API (P6-11): this +# declaration exists because the strict `core` Steep target types its two includers, RetryStep +# and AsyncRetryStep. RBS has no visibility for a constant, so the privacy lives in +# lib/dexpace/resilience/retry_step_helpers.rb alone. +module Dexpace + module Resilience + module RetryStepHelpers + RETRY: Symbol + EXHAUSTED: Symbol + STOP: Symbol + + @settings: RetrySettings + @http_tracer_factory: untyped + @delay_override: untyped + @should_retry: untyped + @logger: Instrumentation::Logger + + private + + def initialize_retry_step: (settings: untyped, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: untyped) -> void + def callable!: (String name, untyped value) -> untyped + def effective_max_retries: (Pipeline::Cursor cursor) -> Integer + def decision: (Dexpace::Request request, ::Exception failure, Integer attempt, + Integer max_retries) -> Symbol + def retryable_condition?: (::Exception failure, Dexpace::Request request) -> bool + def resolve_delay: (Integer attempt, Dexpace::Response? response, ::Exception? failure) -> Float + def overridden_delay: (Integer attempt, Dexpace::Response? response, ::Exception? failure) -> Float? + def report_override: (message: String, ?cause: ::Exception?) -> void + def failure_of: (Dexpace::Response? response, ::Exception? error) -> ::Exception + def attach_trail: (::Exception error, Array[::Exception] trail) -> ::Exception + end + end +end From 420014e98b73850c52c82fcf8eaf0af400aca34a Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:14:18 +0300 Subject: [PATCH 04/13] feat: the recovery-chain retry engine and the widened core surface Add Resilience::RecoveryRetry, chapter 9's first stack (RECOV-17 to RECOV-30 and RECOV-34, which phase 4 postponed), installed as Recovery::Orchestrator's `transport:` decorator rather than as a ResponseChain step (P6-3): Transform#apply carries no request to re-send, and only a transport-decorator position can dispatch its own re-sends while the request chain's stamping stays once per exchange. Sitting below the orchestrator's rescue, a raising transport is retried here as an exception (P6-8), each attempt's response is classified through phase 4b's own Recovery.buffer_error_body then ProtocolError.for_or_nil -- the buffering is what frees the connection before the wait -- and the two terminal shapes are a returned response when the failure was never retryable and a raised throwable otherwise, with every prior failure attached as suppressed (P6-9). This is the only stack with a total-timeout: the budget is a maximum-attempts cap counting the first send as attempt 1 (max_retries plus one, an identity, P6-6) and the time remaining through Policy.budget_remaining, so a delay that would overshoot is suppressed and the last failure surfaced. The HTTP tracer is produced once per operation with the Request, there being no cursor on this stack. The entry file now requires the whole retry layer, the entry test pins the six Resilience constants and the flat error with the private helpers unreachable, and the runtime surface manifest gains the forty-eight rows the phase's public surface adds. --- gems/dexpace-core/lib/dexpace.rb | 1 + .../lib/dexpace/resilience/recovery_retry.rb | 236 ++++++++++++++++++ .../sig/dexpace/resilience/recovery_retry.rbs | 60 +++++ gems/dexpace-core/test/dexpace_test.rb | 25 +- test/fixtures/surface/dexpace-core.txt | 48 ++++ 5 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb create mode 100644 gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs diff --git a/gems/dexpace-core/lib/dexpace.rb b/gems/dexpace-core/lib/dexpace.rb index 2046aaf..2a36e94 100644 --- a/gems/dexpace-core/lib/dexpace.rb +++ b/gems/dexpace-core/lib/dexpace.rb @@ -208,6 +208,7 @@ require_relative "dexpace/resilience/retry_step_helpers" require_relative "dexpace/resilience/retry_step" require_relative "dexpace/resilience/async_retry_step" +require_relative "dexpace/resilience/recovery_retry" # The dexpace Ruby SDK: an HTTP-client toolkit, not an HTTP client. # diff --git a/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb b/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb new file mode 100644 index 0000000..b9a4d14 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb @@ -0,0 +1,236 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "policy" +require_relative "resend" +require_relative "retry_settings" +require_relative "../recovery" +require_relative "../registry" +require_relative "../error/protocol_error" +require_relative "../error/cancelled_error" +require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The recovery-chain retry: chapter 9's FIRST stack, RECOV-17 through RECOV-30 and RECOV-34, + # which phase 4 postponed here. Installed as Recovery::Orchestrator's own `transport:` + # argument, decorating the raw transport, and NOT as a Recovery::ResponseChain recovery + # step (P6-3): Recovery::Transform#apply(outcome) carries no request to re-send, while + # RECOV-19's "each RE-SENT attempt's response MUST be re-classified" describes an engine + # that dispatches its own re-sends and inspects each -- which only a transport-decorator + # position can do while keeping RequestChain's stamping (the idempotency key, the client + # identity) applied exactly once per exchange rather than once per attempt. It is itself a + # Dexpace::Transport by phase 2's duck type, so it composes anywhere a transport does. + # + # Sitting BELOW the orchestrator's rescue region, a transport that raises reaches this + # engine as an exception (P6-8), so a connection reset or a socket timeout -- RECOV-17's and + # RETRY-4's "always retryable" case -- is retried here rather than converted first and + # never retried; only StandardError is rescued and the fatal family propagates at the throw + # site (RETRY-25). Each attempt's response is classified through phase 4b's own two + # primitives, Recovery.buffer_error_body then ProtocolError.for_or_nil, purely to decide + # whether to keep going (RECOV-19, RETRY-36): the buffering is what releases the connection + # before the wait (RETRY-35, RECOV-16), and the BUFFERED response is what every later line + # sees -- the pacing parse, the RECOV-19 pass-through, and whatever ErrorMappingStep the + # caller configured, which runs exactly once, on the terminal response, with the caller's + # own factory (the plan's open question 4: this engine has no factory: keyword). + # + # Two terminal shapes (P6-9): a RESPONSE whose failure was never retryable -- a status + # outside the configured set, or a request that was not re-sendable -- is RETURNED, "passes + # through as Success" (RECOV-19), for the outer chain to map; a THROWABLE is raised whether + # it was retryable or not, and so is a retryable status that met a spent attempt cap or a + # spent total-timeout -- "the terminal failure's throwable MUST be surfaced" (RECOV-20) -- + # with every prior attempt's failure attached as suppressed first (RETRY-34) and, when the + # budget is what stopped a retryable failure, retries_exhausted emitted immediately before. + # + # The budget (RECOV-20, RETRY-27): a maximum-attempts cap that counts the initial send as + # attempt 1 -- RetrySettings#max_retries plus one, an identity and never a second number + # (RETRY-14, P6-6) -- AND a total-timeout applied as the time REMAINING through + # Policy.budget_remaining, zero meaning unbounded; a delay that would push the elapsed time + # past the budget is suppressed and the last failure surfaced, so no wait can overshoot + # (RECOV-21's clamp, RECOV-22's on a hint). The elapsed figure is the settings clock's + # monotonic reading, never Time.now (CFG-16). The wait is Clock#sleep with the caller's + # token (RECOV-27, CFG-15), the token is checked at every attempt boundary too, and a + # cancellation surfaces as the CancelledError it raised, which aborts the loop and is never + # itself retryable (RETRY-23). RECOV-28: the attempt count, the start instant and the trail + # are locals of one #call; nothing is written to this frozen object. + # + # OBS-29's per-attempt group goes through the tracer `http_tracer_factory:` produces once + # per operation, called with the REQUEST -- there is no cursor on this stack -- which is + # also the `context` every callback receives. retries_exhausted fires immediately before the + # terminal raise and only there (a returned response was never retryable, and a spent + # budget on a retryable failure is always a raise here); nothing emits operation_failed, + # which docs/first-release.md's behavioural-asymmetries entry records as conforming. + class RecoveryRetry + private_class_method :new + + # Builds a frozen engine over `transport`. + # + # @param transport [#call] the raw Dexpace::Transport this engine decorates + # @param settings [RetrySettings] the shared configuration; total_timeout is read here + # @param http_tracer_factory [#call] called once per operation with the request; the + # frozen no-op NULL by default + # @param logger [Instrumentation::Logger] where RETRY-41's clamp is reported, contained + # @return [RecoveryRetry] frozen + # @raise [Dexpace::InvalidArgumentError] for a transport that is not a three-positional + # callable, a settings that is not a RetrySettings, or a logger that is not a Logger + def self.build(transport:, settings: RetrySettings.build, + http_tracer_factory: NULL_TRACER_FACTORY, + logger: Instrumentation::Logger::NULL) + new(transport: transport, settings: settings, http_tracer_factory: http_tracer_factory, + logger: logger,).freeze + end + + NULL_TRACER_FACTORY = ->(_request) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(transport:, settings:, http_tracer_factory:, logger:) + unless Registry.callable?(transport, arity: 3) + raise InvalidArgumentError, + "transport: must be a Dexpace::Transport, a callable of three positionals" + end + unless settings.is_a?(RetrySettings) + raise InvalidArgumentError, "settings: takes a RetrySettings, got #{settings.class}" + end + unless http_tracer_factory.respond_to?(:call) + raise InvalidArgumentError, "http_tracer_factory: must answer #call(request)" + end + unless logger.is_a?(Instrumentation::Logger) + raise InvalidArgumentError, "logger: takes an Instrumentation::Logger" + end + + @transport = transport + @settings = settings + @http_tracer_factory = http_tracer_factory + @logger = logger + end + + # The transport SPI: one exchange, retried within the budget. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt + # @param options [Dexpace::RequestOptions] threaded to the transport unchanged + # @param cancellation [Dexpace::Cancellation] the token the wait and every boundary honour + # @return [Dexpace::Response] the terminal response -- a success, or an error status that + # was never retryable, buffered + # @raise [Exception] the terminal throwable, carrying the prior trail as suppressed + # @raise [Dexpace::CancelledError] when the token is cancelled + def call(request, options, cancellation) + run = Run.new(tracer: @http_tracer_factory.call(request), request: request, + options: options, cancellation: cancellation, max_attempts: max_attempts, + started: @settings.clock.monotonic, trail: [],) + attempt = 1 + loop do + terminal = attempt_once(run, attempt) + return terminal unless terminal.nil? + + attempt += 1 + end + end + + # The per-call locals (RECOV-28): on the stack, never on the engine. + class Run < Data.define(:tracer, :request, :options, :cancellation, :max_attempts, :started, + :trail,) + end + private_constant :Run + + RETRY = :retry + EXHAUSTED = :exhausted + STOP = :stop + private_constant :RETRY, :EXHAUSTED, :STOP + + private + + # One attempt: the boundary check, the tracer, the send, the decision, and -- on a retry + # -- the budgeted delay and the wait. Answers the terminal response, or nil for "again". + def attempt_once(run, attempt) + run.cancellation.check! + run.tracer.attempt_started(run.request, attempt) + response, error = exchange(run.request, run.options, run.cancellation) + return response if error.nil? + + verdict = decision(run.request, error, attempt, run.max_attempts) + return response if verdict == STOP && response + + surface(run, error, exhausted: verdict == EXHAUSTED) unless verdict == RETRY + wait(run, attempt, response, error) + end + + # The retry path: the delay against the budget (nil surfaces the failure, RECOV-20), the + # tracer, the trail, then the wait on the settings' clock with the caller's token. + # + # @return [nil] always: the loop's "again" + def wait(run, attempt, response, error) + delay = delay_within_budget(run, attempt, response) + surface(run, error, exhausted: true) if delay.nil? + run.tracer.attempt_failed(run.request, error, delay) + run.trail << error + @settings.clock.sleep(delay, cancellation: run.cancellation) + end + + # RETRY-14 / P6-6: the attempts cap is the stage vocabulary plus the initial send, with + # RETRY-41's clamp applied to the configured value on this stack too. + def max_attempts + Policy.effective_max_retries(override: nil, configured: @settings.max_retries, + logger: @logger,) + 1 + end + + # One send, classified: a success or a non-error status is [response, nil]; an error + # status is buffered (RECOV-16, the release before the wait) and paired with its + # ProtocolError; a raise is [nil, error]. StandardError only (RETRY-25). + def exchange(request, options, cancellation) + response = @transport.call(request, options, cancellation) + return [response, nil] unless response.status.error? + + buffered = Recovery.buffer_error_body(response) + [buffered, ProtocolError.for(buffered)] + rescue ::StandardError => error + [nil, error] + end + + # RECOV-17 and RECOV-18: the configured set for a status, the capability for a throwable + # (Policy.retryable?), AND the re-sendability gate -- then the cap. :stop is "never + # retryable", :exhausted "retryable, cap spent". + def decision(request, error, attempt, max_attempts) + return STOP unless Resend.eligible?(request) + return STOP unless Policy.retryable?(error, + retryable_statuses: @settings.retryable_statuses,) + + attempt < max_attempts ? RETRY : EXHAUSTED + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27: the delay -- the pacing hint from the buffered + # response through the fixed precedence, else the shared backoff -- against the budget + # REMAINING. nil when the budget is spent or the delay would overshoot it: RECOV-20's + # "suppressed, and the last failure surfaced unchanged". A zero budget is unbounded. + def delay_within_budget(run, attempt, response) + delay = hinted_delay(response) || Policy.backoff_delay(attempt, + **@settings.backoff_arguments,) + elapsed = @settings.clock.monotonic - run.started + remaining = Policy.budget_remaining(elapsed: elapsed, + total_timeout: @settings.total_timeout,) + return nil if remaining <= 0.0 || delay > remaining + + delay + end + + # RECOV-22 through RECOV-24, RECOV-29: the recovery stack's FIXED precedence, from the + # still-open (buffered) response; nil when there is no response or no usable hint, which is + # what makes a malformed header fall back to backoff without masking the failure. + def hinted_delay(response) + return nil if response.nil? + + Policy.pacing_delay(response.headers, header_order: Policy::DEFAULT_PACING_HEADER_ORDER, + now: @settings.clock.now, random: @settings.random,) + end + + # RECOV-20's surfacing and RETRY-34's trail: retries_exhausted when the budget stopped a + # retryable failure (never for one that was never retryable), then the carried raise. + def surface(run, error, exhausted:) + run.trail.each { |prior| Dexpace.attach_suppressed(error, prior) } + run.tracer.retries_exhausted(run.request, error) if exhausted + raise error, cause: nil # pipeline/7ce4431d: a carried error, never a bare raise + end + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs b/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs new file mode 100644 index 0000000..706b830 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs @@ -0,0 +1,60 @@ +module Dexpace + module Resilience + # The recovery-chain retry engine (RECOV-17..30, RECOV-34), a Dexpace::Transport decorator + # installed as Recovery::Orchestrator's `transport:` (P6-3). The transport and the factory are + # untyped callables, as on the two stage drivers. + class RecoveryRetry + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + RETRY: Symbol + EXHAUSTED: Symbol + STOP: Symbol + + @transport: untyped + @settings: RetrySettings + @http_tracer_factory: untyped + @logger: Instrumentation::Logger + + # A private_constant: the per-call locals (RECOV-28), declared for the strict target. + class Run < Data + attr_reader tracer: untyped + attr_reader request: Dexpace::Request + attr_reader options: Dexpace::RequestOptions + attr_reader cancellation: Dexpace::Cancellation + attr_reader max_attempts: Integer + attr_reader started: Float + attr_reader trail: Array[::Exception] + + def self.new: (tracer: untyped, request: Dexpace::Request, + options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, + max_attempts: Integer, started: Float, + trail: Array[::Exception]) -> instance + end + + private def self.new: (transport: untyped, settings: RetrySettings, + http_tracer_factory: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (transport: untyped, settings: RetrySettings, http_tracer_factory: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (transport: untyped, ?settings: RetrySettings, + ?http_tracer_factory: untyped, + ?logger: Instrumentation::Logger) -> RecoveryRetry + + def call: (Dexpace::Request request, Dexpace::RequestOptions options, + Dexpace::Cancellation cancellation) -> Dexpace::Response + + private + + def attempt_once: (Run run, Integer attempt) -> Dexpace::Response? + def wait: (Run run, Integer attempt, Dexpace::Response? response, ::Exception error) -> nil + def max_attempts: () -> Integer + def exchange: (Dexpace::Request request, Dexpace::RequestOptions options, + Dexpace::Cancellation cancellation) -> [Dexpace::Response?, ::Exception?] + def decision: (Dexpace::Request request, ::Exception error, Integer attempt, + Integer max_attempts) -> Symbol + def delay_within_budget: (Run run, Integer attempt, Dexpace::Response? response) -> Float? + def hinted_delay: (Dexpace::Response? response) -> Float? + def surface: (Run run, ::Exception error, exhausted: bool) -> bot + end + end +end diff --git a/gems/dexpace-core/test/dexpace_test.rb b/gems/dexpace-core/test/dexpace_test.rb index d759701..43ff031 100644 --- a/gems/dexpace-core/test/dexpace_test.rb +++ b/gems/dexpace-core/test/dexpace_test.rb @@ -41,9 +41,9 @@ class DexpaceTest < DexpaceTestCase # phase 6a's retry layer (the flat RetryPredicateError beside the Resilience namespace); # Dexpace::Hooks, Dexpace::BoundedMap, Dexpace::CallKey, Dexpace::Recovery::Ownership, the two # pipeline drivers, Dexpace::ConfigParsers, Dexpace::DeepValue, Dexpace::ProxyResolution and - # 6a's Resilience::PacingParsers are private_constants and appear in no constants(false) list. - # Phase 5c's tracing and metrics layer and phase 5b's logging layer add no flat constant: - # everything either ships is under + # 6a's Resilience::PacingParsers and Resilience::RetryStepHelpers are private_constants and + # appear in no constants(false) list. Phase 5c's tracing and metrics layer and phase 5b's + # logging layer add no flat constant: everything either ships is under # Dexpace::Instrumentation, which the Layers case below pins. DOMAIN_MODEL = %i[ Error InvalidArgumentError Model Builder HeaderSyntax HeaderName Headers Status Method @@ -172,6 +172,25 @@ class Layers < DexpaceTestCase assert_raises(::NameError) { Dexpace::CallKey } end + # A consumer requires "dexpace" and nothing else: the retry layer resolves too (phase 6a) -- + # the five public Resilience constants, the flat error, and the two private helpers and the + # two private per-call classes as unreachable as Dexpace::Hooks. Phase 6b and 6c add their + # own constants under Resilience beside these. + test "requiring dexpace alone makes the whole retry layer resolve, its helpers private" do + resilience = Dexpace::Resilience + + assert_equal(%i[AsyncRetryStep Policy RecoveryRetry Resend RetrySettings RetryStep], + resilience.constants(false).sort,) + assert_equal(Dexpace::RetryPredicateError, Dexpace.const_get(:RetryPredicateError)) + assert_equal(2, resilience::Policy::DEFAULT_MAX_RETRIES) + assert_raises(::NameError) { Dexpace::Resilience::PacingParsers } + assert_raises(::NameError) { Dexpace::Resilience::RetryStepHelpers } + assert_raises(::NameError) { Dexpace::Resilience::AsyncRetryStep::Pump } + assert_raises(::NameError) { Dexpace::Resilience::RetryStep::Run } + assert_raises(::NameError) { Dexpace::Resilience::RecoveryRetry::Run } + assert_raises(::NameError) { Dexpace::Resilience::RetrySettings::UNSET } + end + # A consumer requires "dexpace" and nothing else: the seam layer resolves too (phase 2). test "requiring dexpace alone makes the whole seam layer resolve" do assert_equal(Dexpace::Transport, Dexpace.const_get(:Transport)) diff --git a/test/fixtures/surface/dexpace-core.txt b/test/fixtures/surface/dexpace-core.txt index 0b4f5f3..fdc5b39 100644 --- a/test/fixtures/surface/dexpace-core.txt +++ b/test/fixtures/surface/dexpace-core.txt @@ -622,6 +622,7 @@ Dexpace::Pipeline::Builder#replace Dexpace::Pipeline::Builder.flattening Dexpace::Pipeline::Builder.nesting Dexpace::Pipeline::Cursor +Dexpace::Pipeline::Cursor#bundle Dexpace::Pipeline::Cursor#call Dexpace::Pipeline::Cursor#cancellation Dexpace::Pipeline::Cursor#fork @@ -681,6 +682,7 @@ Dexpace::Protocol::HTTP_2 : Dexpace::Protocol Dexpace::Protocol::WIRE_FORMS : Array Dexpace::ProtocolError Dexpace::ProtocolError#response +Dexpace::ProtocolError#retryable_by_status? Dexpace::ProtocolError#status Dexpace::ProtocolError.for Dexpace::ProtocolError.for_or_nil @@ -832,6 +834,51 @@ Dexpace::RequestOptions::Builder#max_retries= Dexpace::RequestOptions::Builder#tag Dexpace::RequestOptions::Builder#timeout= Dexpace::RequestOptions::EMPTY : Dexpace::RequestOptions +Dexpace::Resilience +Dexpace::Resilience::AsyncRetryStep +Dexpace::Resilience::AsyncRetryStep#call +Dexpace::Resilience::AsyncRetryStep#stage +Dexpace::Resilience::AsyncRetryStep.build +Dexpace::Resilience::Policy +Dexpace::Resilience::Policy#backoff_delay +Dexpace::Resilience::Policy#budget_remaining +Dexpace::Resilience::Policy#effective_max_retries +Dexpace::Resilience::Policy#pacing_delay +Dexpace::Resilience::Policy#retry_eligible? +Dexpace::Resilience::Policy#retryable? +Dexpace::Resilience::Policy#throwable_retryable? +Dexpace::Resilience::Policy::DEFAULT_INITIAL_DELAY : Float +Dexpace::Resilience::Policy::DEFAULT_JITTER : Float +Dexpace::Resilience::Policy::DEFAULT_MAX_DELAY : Float +Dexpace::Resilience::Policy::DEFAULT_MAX_RETRIES : Integer +Dexpace::Resilience::Policy::DEFAULT_MULTIPLIER : Float +Dexpace::Resilience::Policy::DEFAULT_PACING_HEADER_ORDER : Array +Dexpace::Resilience::Policy::DEFAULT_RETRYABLE_STATUSES : Set +Dexpace::Resilience::Policy::MAX_DURATION_NANOSECONDS : Integer +Dexpace::Resilience::Policy::MAX_PACING_DELAY_SECONDS : Integer +Dexpace::Resilience::RecoveryRetry +Dexpace::Resilience::RecoveryRetry#call +Dexpace::Resilience::RecoveryRetry.build +Dexpace::Resilience::Resend +Dexpace::Resilience::Resend#eligible? +Dexpace::Resilience::RetrySettings +Dexpace::Resilience::RetrySettings#backoff_arguments +Dexpace::Resilience::RetrySettings#clock +Dexpace::Resilience::RetrySettings#header_order +Dexpace::Resilience::RetrySettings#initial_delay +Dexpace::Resilience::RetrySettings#jitter +Dexpace::Resilience::RetrySettings#max_delay +Dexpace::Resilience::RetrySettings#max_retries +Dexpace::Resilience::RetrySettings#multiplier +Dexpace::Resilience::RetrySettings#pacing_header_order +Dexpace::Resilience::RetrySettings#random +Dexpace::Resilience::RetrySettings#retryable_statuses +Dexpace::Resilience::RetrySettings#total_timeout +Dexpace::Resilience::RetrySettings.build +Dexpace::Resilience::RetryStep +Dexpace::Resilience::RetryStep#call +Dexpace::Resilience::RetryStep#stage +Dexpace::Resilience::RetryStep.build Dexpace::Response Dexpace::Response#body Dexpace::Response#body_bytes @@ -876,6 +923,7 @@ Dexpace::ResponseLoggingBody#preview_bytes Dexpace::ResponseLoggingBody#snapshot Dexpace::ResponseLoggingBody#source Dexpace::ResponseLoggingBody#write_to +Dexpace::RetryPredicateError Dexpace::Retryability Dexpace::Retryability#retryable_status? Dexpace::SeamError From 3515b6f698c6d972a35906a4b669cf72f8f6f6f7 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 16:26:28 +0300 Subject: [PATCH 05/13] fix: bound the pacing parser, guard cancellation, fence the tracer Review round 0 of the phase-6a stack found four behaviours the design states and the code did not fully honour, and one comment nit. The pacing parsers converted an unbounded digit run: a 10 MB header value stalled the retry decision for seconds in String#to_f / #to_i and, past ~309 digits, emitted Ruby's out-of-range warning that the suite's NFR-6 raiser turned into an error Policy#parse_form's rescue swallowed. Both grammars now bound their runs at fifteen digits -- the largest run a Float carries exactly, and 10**15 seconds is thirty million years, so a longer run is RETRY-16's out-of-range value and answers nil without a conversion -- and a 64-byte ceiling in front of every parser keeps the HTTP-date attempt from reading a hostile value either (the longest well-formed form is the 29-byte RFC 1123 date). RetryStep#wait emitted the tracer's attempt_failed outside the RETRY-35 fence, so a tracer that raised left the superseded response open on the sync driver while the async driver closed it. The emission now runs inside the same fence as the delay resolution, so the response is closed before the raise propagates on both drivers. A caller's should_retry answering true retried a downstream CancelledError. Policy.cancellation? walks the cause chain for a CancelledError; Policy.retryable? answers false for one before either branch, and the stage drivers' decision answers :stop for one before the re-sendability gate and before the predicate, so RETRY-23's "never" holds on all three drivers whatever a capability or a predicate says. A negative configured MAX_RETRY_ATTEMPTS raised at RetrySettings.build, so RETRY-41's clamp-and-log was unreachable from any driver. RetrySettings.build takes a logger: keyword, read once at build, and resolves the configured value through Policy.effective_max_retries, so a negative configured value is clamped to the default and the clamp logged as one contained config diagnostic; an explicit negative max_retries: is still refused by RECOV-34. Also: step.rb's class comment cited P6-52 for the no-operation-name statement, which is P6-51's; the RBS mirrors follow each change and the core manifest gains Policy#cancellation? (1004 -> 1005 rows). --- .../lib/dexpace/instrumentation/step.rb | 3 +- .../lib/dexpace/resilience/pacing_parsers.rb | 54 +++++++++++++------ .../lib/dexpace/resilience/policy.rb | 26 +++++++-- .../lib/dexpace/resilience/retry_settings.rb | 28 +++++++--- .../lib/dexpace/resilience/retry_step.rb | 26 +++++---- .../dexpace/resilience/retry_step_helpers.rb | 14 +++-- .../sig/dexpace/resilience/pacing_parsers.rbs | 5 ++ .../sig/dexpace/resilience/policy.rbs | 1 + .../sig/dexpace/resilience/retry_settings.rbs | 3 +- test/fixtures/surface/dexpace-core.txt | 1 + 10 files changed, 116 insertions(+), 45 deletions(-) diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb index 5510ad1..f27cce9 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb @@ -51,7 +51,8 @@ module Instrumentation # phase 5 did -- the keyword and Bundle::NONE, OBS-34's and XCUT-19(e)'s own default # configuration. The span is still named by the request's method token, which is also what # the tracer factory is asked for: the operation identifier is 4a's - # RequestContext#operation_name and the cursor's bundle carries no name (P6-52). + # RequestContext#operation_name and the cursor's bundle carries no name (P6-51; + # docs/first-release.md's behavioural-asymmetries entry). # # The level guard is read once at the top of #call and consulted at three sites -- the two # emissions and the body wrapping -- and a step that re-read it inside a helper would be one diff --git a/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb index 00a0df8..ef22eb1 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb @@ -15,19 +15,33 @@ module Resilience # no NFR-4 lock. It has a sig/ mirror because the strict `core` Steep target types its call # sites, and no test/ mirror -- every branch is asserted through Policy.pacing_delay. # - # Three grammars, each anchored \A..\z, each compiled once with its own timeout (never - # Regexp.timeout), and none can backtrack: digit runs and one optional fraction. RETRY-19's - # screen is the DECIMAL grammar, applied BEFORE String#to_f -- "30d", "0x10", "1e3", "5_0", - # "Infinity", "NaN" and " 5" all fail it and fall through to the HTTP-date attempt, which - # rejects them too, so the answer is nil. Nothing here calls Float() or Integer() on an - # unscreened value: Integer("5_0", 10) is 50 on every supported Ruby (verified 3.2.11, - # 3.4.10, 4.0.6), which is exactly the mis-parse the screen exists to prevent. + # Two grammars, each anchored \A..\z, each compiled once with its own timeout (never + # Regexp.timeout), and none can backtrack: bounded digit runs and one optional bounded + # fraction. RETRY-19's screen is the DECIMAL grammar, applied BEFORE String#to_f -- "30d", + # "0x10", "1e3", "5_0", "Infinity", "NaN" and " 5" all fail it and fall through to the + # HTTP-date attempt, which rejects them too, so the answer is nil. Nothing here calls Float() + # or Integer() on an unscreened value: Integer("5_0", 10) is 50 on every supported Ruby + # (verified 3.2.11, 3.4.10, 4.0.6), which is exactly the mis-parse the screen exists to + # prevent. + # + # The runs are BOUNDED (P6-61): fifteen integer digits is the largest run a Float carries + # exactly and 10**15 seconds is thirty million years, so a longer run is RETRY-16's + # out-of-range value and answers nil before anything converts it -- String#to_f and #to_i + # over an unbounded run cost seconds on a multi-megabyte value (measured: 16-20 s for a 10 MB + # retry-after-ms) and, past ~309 digits, emit Ruby's "Float ... out of range" warning, which + # NFR-6's fatal-warnings suite turns into a raise Policy#parse_form's fence was quietly + # swallowing. MAX_VALUE_BYTES bounds the HTTP-date attempt the same way, in front of every + # parser: the longest well-formed pacing value is the 29-byte RFC 1123 date, and a value past + # 64 bytes is no hint without a grammar or the date parser ever reading it. module PacingParsers extend self - DECIMAL_GRAMMAR = ::Regexp.new('\A\d+(\.\d+)?\z', timeout: 1.0).freeze - INTEGER_GRAMMAR = ::Regexp.new('\A\d+\z', timeout: 1.0).freeze - private_constant :DECIMAL_GRAMMAR, :INTEGER_GRAMMAR + # The longest well-formed pacing value is an RFC 1123 date, 29 bytes; twice that, rounded + # up, is the ceiling every parser applies before any grammar runs. + MAX_VALUE_BYTES = 64 + DECIMAL_GRAMMAR = ::Regexp.new('\A\d{1,15}(\.\d{1,15})?\z', timeout: 1.0).freeze + INTEGER_GRAMMAR = ::Regexp.new('\A\d{1,15}\z', timeout: 1.0).freeze + private_constant :MAX_VALUE_BYTES, :DECIMAL_GRAMMAR, :INTEGER_GRAMMAR # The upper bound of RECOV-25's positive jitter: [100%, 120%] of the computed delta. RESET_JITTER_CEILING = 1.2 @@ -37,7 +51,7 @@ module PacingParsers # RFC 1123 HTTP-date through the ONE parser, whose past instants floor to zero (RETRY-17). # An unparseable value is nil, distinct from a past date's 0.0. def parse_retry_after(value, now:) - return nil unless value.is_a?(::String) + return nil unless readable?(value) return value.to_f if DECIMAL_GRAMMAR.match?(value) instant = Dexpace::HTTPDate.parse(value) @@ -48,7 +62,7 @@ def parse_retry_after(value, now:) # retry-after-ms / x-ms-retry-after-ms: a non-negative integer count of milliseconds. def parse_millis(value) - return nil unless value.is_a?(::String) && INTEGER_GRAMMAR.match?(value) + return nil unless readable?(value) && INTEGER_GRAMMAR.match?(value) value.to_i / 1000.0 end @@ -56,13 +70,12 @@ def parse_millis(value) # X-RateLimit-Reset: Unix epoch seconds. A reset already past floors to zero (RETRY-17); a # future one is jittered UPWARD to [100%, 120%] of the delta (RECOV-25), inside the parser, # so the caller applies no second jitter on top (RETRY-20, RECOV-22). The jitter is drawn - # only over a finite range: a digit run long enough to convert to Infinity is out of range - # and answers nil (RETRY-16), and a finite delta whose 120% would overflow is returned - # unjittered for the dispatcher's 365-day clamp to bound -- Random#rand raises EDOM on an - # infinite bound (verified on 4.0.6), and RECOV-26's "never surface an arithmetic overflow + # only over a finite range: the bounded run keeps every delta finite, and the `finite?` + # guards stay as the belt behind the grammar -- Random#rand raises EDOM on an infinite + # bound (verified on 4.0.6), and RECOV-26's "never surface an arithmetic overflow # mid-retry" covers a server's hint before any clamp reaches it. def parse_epoch_reset(value, now:, random:) - return nil unless value.is_a?(::String) && INTEGER_GRAMMAR.match?(value) + return nil unless readable?(value) && INTEGER_GRAMMAR.match?(value) delta = [(value.to_i - now.to_i).to_f, 0.0].max return nil unless delta.finite? @@ -71,6 +84,13 @@ def parse_epoch_reset(value, now:, random:) ceiling = delta * RESET_JITTER_CEILING ceiling.finite? ? random.rand(delta..ceiling) : delta end + + private + + # A String within the byte ceiling; anything else is no hint before a grammar runs. + def readable?(value) + value.is_a?(::String) && value.bytesize <= MAX_VALUE_BYTES + end end private_constant :PacingParsers end diff --git a/gems/dexpace-core/lib/dexpace/resilience/policy.rb b/gems/dexpace-core/lib/dexpace/resilience/policy.rb index e7c3993..5e28f98 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/policy.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/policy.rb @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MIT require_relative "../each_cause" +require_relative "../error/cancelled_error" require_relative "../error/invalid_argument_error" require_relative "../error/protocol_error" require_relative "../instrumentation/contain" @@ -114,15 +115,32 @@ def throwable_retryable?(error) end end - # The one dispatch point both stacks call: a ProtocolError is classified by its status - # against the configured set (RETRY-37), anything else by the capability query (RETRY-2). - # A ProtocolError deliberately does not answer #retryable? (P6-10), so a wrapped one buried - # in a cause chain can never smuggle the baked set past the configured one. + # RETRY-23 / RECOV-27: whether `error` is a cancellation -- a Dexpace::CancelledError + # itself, or one anywhere in its cause chain, so a transport that wrapped the token's raise + # in an error of its own (one that may well answer #retryable?) is still read as the + # cancellation it carries. Consulted BEFORE either branch of .retryable? and before a + # caller's should_retry predicate on the stage drivers, so "cancellation MUST never be + # treated as a retryable failure" holds whatever a capability or a predicate answers + # (P6-60). The walk is Dexpace.each_cause, cycle-safe by identity (XCUT-9). + # + # @param error [Exception] + # @return [Boolean] + def cancellation?(error) + Dexpace.each_cause(error).any?(Dexpace::CancelledError) + end + + # The one dispatch point both stacks call: a cancellation is never retryable (RETRY-23), + # then a ProtocolError is classified by its status against the configured set (RETRY-37), + # anything else by the capability query (RETRY-2). A ProtocolError deliberately does not + # answer #retryable? (P6-10), so a wrapped one buried in a cause chain can never smuggle + # the baked set past the configured one. # # @param error [Exception] # @param retryable_statuses [Set] the configured set # @return [Boolean] def retryable?(error, retryable_statuses:) + return false if cancellation?(error) + if error.is_a?(Dexpace::ProtocolError) retry_eligible?(error.status.code, set: retryable_statuses) else diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb index 86316f0..d695143 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb @@ -6,6 +6,7 @@ require_relative "../config" require_relative "../configuration" require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/logger" require_relative "policy" module Dexpace @@ -31,6 +32,11 @@ module Resilience # reads 5a's Configuration::Keys::MAX_RETRY_ATTEMPTS off the process-wide slot -- this class is # that key's first and only reader -- and falls back to Policy::DEFAULT_MAX_RETRIES; the read # happens ONCE, at build, so a settings object is a snapshot and never re-consults the slot. + # A NEGATIVE configured value meets RETRY-41 here, at that one read: it is clamped to + # Policy::DEFAULT_MAX_RETRIES through the same Policy.effective_max_retries the drivers + # resolve with, and the clamp is logged once through `logger:`, contained (P6-59). An explicit + # `max_retries:` argument is a caller's construction input and RECOV-34 refuses a negative one + # instead -- the two rules meet at the one place the configured value enters. class RetrySettings < Data.define( :initial_delay, :multiplier, :max_delay, :jitter, :max_retries, :total_timeout, :retryable_statuses, :pacing_header_order, :random, :clock, @@ -60,28 +66,36 @@ class RetrySettings < Data.define( # shared default # @param random [#rand] the generator the jitter draws from # @param clock [Dexpace::_Clock] the seam every wait and every elapsed reading goes through + # @param logger [Dexpace::Instrumentation::Logger] where RETRY-41's clamp of a negative + # configured MAX_RETRY_ATTEMPTS is reported, contained; read at build only and never + # held, so it is taken on trust as Proxy.resolve's and Hooks.notify's are # @return [RetrySettings] frozen # @raise [Dexpace::InvalidArgumentError] naming the member, on any invalid value def self.build(initial_delay: Policy::DEFAULT_INITIAL_DELAY, multiplier: Policy::DEFAULT_MULTIPLIER, max_delay: Policy::DEFAULT_MAX_DELAY, jitter: Policy::DEFAULT_JITTER, max_retries: UNSET, total_timeout: 0, retryable_statuses: Policy::DEFAULT_RETRYABLE_STATUSES, - pacing_header_order: nil, random: ::Random, clock: Clock::SYSTEM) + pacing_header_order: nil, random: ::Random, clock: Clock::SYSTEM, + logger: Instrumentation::Logger::NULL) new( initial_delay: initial_delay, multiplier: multiplier, max_delay: max_delay, - jitter: jitter, max_retries: resolve_max_retries(max_retries), + jitter: jitter, max_retries: resolve_max_retries(max_retries, logger), total_timeout: total_timeout, retryable_statuses: retryable_statuses, pacing_header_order: pacing_header_order, random: random, clock: clock, ) end - # RETRY-12 / P6-6: the configured key, read once, only when nothing was passed. - def self.resolve_max_retries(max_retries) + # RETRY-12 / P6-6: the configured key, read once, only when nothing was passed -- and + # RETRY-41 on what it reads: a negative configured value is clamped to the default and the + # clamp logged, by the one resolver (P6-59). An explicit argument passes through untouched + # to RECOV-34's validation. + def self.resolve_max_retries(max_retries, logger) return max_retries unless UNSET.equal?(max_retries) - Dexpace.configuration.integer(Configuration::Keys::MAX_RETRY_ATTEMPTS, - default: Policy::DEFAULT_MAX_RETRIES,) || - Policy::DEFAULT_MAX_RETRIES + configured = Dexpace.configuration.integer(Configuration::Keys::MAX_RETRY_ATTEMPTS, + default: Policy::DEFAULT_MAX_RETRIES,) || + Policy::DEFAULT_MAX_RETRIES + Policy.effective_max_retries(override: nil, configured: configured, logger: logger) end private_class_method :resolve_max_retries diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb index 815afa2..9154295 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb @@ -37,10 +37,11 @@ module Resilience # tracer callback in a rescue (OBS-20's carve-out). # # RETRY-35's three orderings, in #call: the delay is resolved from the still-open response - # FIRST, the response is closed BEFORE the wait, and both the retry decision and the delay - # resolution are fenced so the response is closed before any throwable -- a - # RetryPredicateError above all -- propagates. RETRY-34 on the terminal path: the whole - # prior trail is attached to the instance actually surfaced, through + # FIRST, the response is closed BEFORE the wait, and the retry decision, the delay + # resolution and the tracer's attempt_failed are all fenced so the response is closed before + # any throwable -- a RetryPredicateError above all, a throwing tracer too -- propagates, + # exactly as the async driver's guarded block already did. RETRY-34 on the terminal path: + # the whole prior trail is attached to the instance actually surfaced, through # Dexpace.attach_suppressed, and is discarded on success or on a returned error-status # response, which is a Response and not a throwable to attach to (stated, not dropped). class RetryStep @@ -130,15 +131,20 @@ def attempt_once(run, request, attempt) wait(run, attempt, response, failure) end - # The retry path: the delay from the still-open response (fenced, RETRY-35), the tracer, - # the trail, the close BEFORE the wait, then the wait on the settings' clock. On the - # exception path the failure IS the error the override sees; on the response path the - # override sees the response and no error (RETRY-39). + # The retry path: the delay from the still-open response and the tracer's attempt_failed, + # both inside ONE fence (RETRY-35: a tracer that raises propagates, OBS-30, but the + # superseded response is closed first, as on the async driver), the trail, the close BEFORE + # the wait, then the wait on the settings' clock. On the exception path the failure IS the + # error the override sees; on the response path the override sees the response and no error + # (RETRY-39). # # @return [nil] always: the loop's "again" def wait(run, attempt, response, failure) - delay = fenced(response) { resolve_delay(attempt, response, response ? nil : failure) } - run.tracer.attempt_failed(run.cursor, failure, delay) + delay = fenced(response) do + resolved = resolve_delay(attempt, response, response ? nil : failure) + run.tracer.attempt_failed(run.cursor, failure, resolved) + resolved + end run.trail << failure Dexpace.close_quietly(response, onto: failure) @settings.clock.sleep(delay, cancellation: run.cursor.cancellation) diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb index 7ac1158..b6895eb 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb @@ -67,14 +67,18 @@ def effective_max_retries(cursor) configured: @settings.max_retries, logger: @logger,) end - # RETRY-8: BOTH axes, neither implying the other, in this order -- re-sendability first - # and unconditionally (a caller predicate may widen or narrow the CONDITION and may never - # authorise re-sending a bare POST or a consumed body, RETRY-7), then the condition - # (Policy.retryable? or the caller's predicate, RETRY-39/RETRY-40), then the budget. The - # budget comes LAST so that :exhausted means exactly "retryable, and the budget is spent". + # RETRY-23 first: a cancellation -- the token's own raise or one a downstream wrapped -- + # is terminal before any axis is consulted, so a caller's should_retry never sees it and + # can never answer true for it (P6-60). Then RETRY-8's BOTH axes, neither implying the + # other, in this order -- re-sendability first and unconditionally (a caller predicate may + # widen or narrow the CONDITION and may never authorise re-sending a bare POST or a consumed + # body, RETRY-7), then the condition (Policy.retryable? or the caller's predicate, + # RETRY-39/RETRY-40), then the budget. The budget comes LAST so that :exhausted means + # exactly "retryable, and the budget is spent". # # @return [Symbol] RETRY, EXHAUSTED or STOP def decision(request, failure, attempt, max_retries) + return STOP if Policy.cancellation?(failure) return STOP unless Resend.eligible?(request) return STOP unless retryable_condition?(failure, request) diff --git a/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs index 279084b..4c4cfe3 100644 --- a/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs +++ b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs @@ -6,6 +6,7 @@ module Dexpace module Resilience module PacingParsers + MAX_VALUE_BYTES: Integer DECIMAL_GRAMMAR: Regexp INTEGER_GRAMMAR: Regexp RESET_JITTER_CEILING: Float @@ -13,6 +14,10 @@ module Dexpace def self?.parse_retry_after: (untyped value, now: ::Time) -> Float? def self?.parse_millis: (untyped value) -> Float? def self?.parse_epoch_reset: (untyped value, now: ::Time, random: untyped) -> Float? + + private + + def readable?: (untyped value) -> bool end end end diff --git a/gems/dexpace-core/sig/dexpace/resilience/policy.rbs b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs index 5a3f69d..ee04eb9 100644 --- a/gems/dexpace-core/sig/dexpace/resilience/policy.rbs +++ b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs @@ -17,6 +17,7 @@ module Dexpace def self?.retry_eligible?: (Integer status, set: ::Set[Integer]) -> bool def self?.throwable_retryable?: (::Exception error) -> bool + def self?.cancellation?: (::Exception error) -> bool def self?.retryable?: (::Exception error, retryable_statuses: ::Set[Integer]) -> bool def self?.backoff_delay: (Integer attempt, initial_delay: Numeric, multiplier: Numeric, max_delay: Numeric, jitter: Numeric, random: untyped) -> Float diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs index 81d62a0..a571920 100644 --- a/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs @@ -35,6 +35,7 @@ module Dexpace ?max_retries: untyped, ?total_timeout: Numeric, ?retryable_statuses: Enumerable[Integer], ?pacing_header_order: Array[String]?, ?random: untyped, ?clock: _Clock, + ?logger: Instrumentation::Logger, ) -> RetrySettings def backoff_arguments: () -> { initial_delay: Float, multiplier: Float, max_delay: Float, @@ -43,7 +44,7 @@ module Dexpace private - def self.resolve_max_retries: (untyped max_retries) -> Integer + def self.resolve_max_retries: (untyped max_retries, Instrumentation::Logger logger) -> Integer def duration!: (String name, untyped value) -> Float def multiplier!: (untyped value) -> Float def jitter!: (untyped value) -> Float diff --git a/test/fixtures/surface/dexpace-core.txt b/test/fixtures/surface/dexpace-core.txt index fdc5b39..a5d8b0f 100644 --- a/test/fixtures/surface/dexpace-core.txt +++ b/test/fixtures/surface/dexpace-core.txt @@ -842,6 +842,7 @@ Dexpace::Resilience::AsyncRetryStep.build Dexpace::Resilience::Policy Dexpace::Resilience::Policy#backoff_delay Dexpace::Resilience::Policy#budget_remaining +Dexpace::Resilience::Policy#cancellation? Dexpace::Resilience::Policy#effective_max_retries Dexpace::Resilience::Policy#pacing_delay Dexpace::Resilience::Policy#retry_eligible? From a7882183778cce105011bbd26576b12883852c72 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 17:38:38 +0300 Subject: [PATCH 06/13] fix: fence the terminal tracer emission on the sync retry step Review round 1 of the phase-6a stack found the sync/async drift round 0 closed for attempt_failed still open on the terminal path: RetryStep's settle emitted the tracer's retries_exhausted outside any fence, so a tracer that raised there propagated while the terminal error-status response it had just been handed stayed open, whereas the async pump's finish already ran inside its guarded block and closed it. The trail attachment and the retries_exhausted emission now run inside the same fence as the decision and the delay resolution, so a throwing tracer propagates (OBS-30) with the terminal response closed first on both drivers; a tracer that does not raise leaves the returned response open exactly as before (RETRY-34), and the exception path is untouched. The class comment and the method's comment state the terminal path too. --- .../lib/dexpace/resilience/retry_step.rb | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb index 9154295..1da6aaf 100644 --- a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb @@ -38,12 +38,14 @@ module Resilience # # RETRY-35's three orderings, in #call: the delay is resolved from the still-open response # FIRST, the response is closed BEFORE the wait, and the retry decision, the delay - # resolution and the tracer's attempt_failed are all fenced so the response is closed before + # resolution and BOTH tracer emissions that follow a failure -- attempt_failed on the retry + # path, retries_exhausted on the terminal path -- are fenced so the response is closed before # any throwable -- a RetryPredicateError above all, a throwing tracer too -- propagates, - # exactly as the async driver's guarded block already did. RETRY-34 on the terminal path: - # the whole prior trail is attached to the instance actually surfaced, through - # Dexpace.attach_suppressed, and is discarded on success or on a returned error-status - # response, which is a Response and not a throwable to attach to (stated, not dropped). + # exactly as the async driver's guarded block does on both of its paths (R0-4, R1-1). + # RETRY-34 on the terminal path: the whole prior trail is attached to the instance actually + # surfaced, through Dexpace.attach_suppressed, and is discarded on success or on a returned + # error-status response, which is a Response and not a throwable to attach to (stated, not + # dropped). class RetryStep include RetryStepHelpers @@ -178,12 +180,17 @@ def fenced(response) # The terminal path. A throwable is surfaced with the whole trail attached and, when the # budget is what stopped it, retries_exhausted first; a response is returned as it is, its # trail discarded, since a Response carries no suppressed trail to attach to (RETRY-34) -- - # the ProtocolError the tracer sees for an exhausted error status carries it instead. + # the ProtocolError the tracer sees for an exhausted error status carries it instead. The + # emission runs inside the fence, as attempt_failed does on the retry path: a tracer that + # raises in retries_exhausted propagates (OBS-30) but the terminal error-status response + # is closed first, exactly as Pump#finish is closed by its guarded block (R1-1). def settle(run, response, failure, verdict) return response if response && verdict == STOP - attach_trail(failure, run.trail) - run.tracer.retries_exhausted(run.cursor, failure) if verdict == EXHAUSTED + fenced(response) do + attach_trail(failure, run.trail) + run.tracer.retries_exhausted(run.cursor, failure) if verdict == EXHAUSTED + end return response if response raise failure, cause: nil # pipeline/7ce4431d: a carried error, never a bare raise From 9d898fc951f4e14a165895149cb51d40160b0c22 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:14:40 +0300 Subject: [PATCH 07/13] test: the scripted transports and the retry fixtures Add the three doubles phase 6a's retry suites fold over, all under test/support/ and top level like every double since phase 2: ScriptedTransport, which consumes a per-call script of responses, Exceptions and callables so a suite can stage 503, 503, 200 or a run of throwables and assert with assert_same that one request object was re-sent every time; ScriptedAsyncTransport, its SEAM-16 twin over a fresh Completer per call, settling inline by default (the shape that overflowed the plan's recursive pump) or held pending under settle_later: with settle_next! for the cancel-in-flight cases; and the RetryFixtures mixin over RecoveryFixtures -- a request per method, a response over a real ResponseBody so body.closed? is the RETRY-35 assertion, RetryableError and UnretryableError answering XCUT-6's capability, RetryableFatal for RETRY-25, jitter-free settings over a FakeClock, and the RecordingCursor that forwards to the real cursor the driver minted so the fork-for-every-drive rule is asserted against the real runtime. Phase 2's FakeTransport is untouched: seven suites and three doubles require it by name and shape. --- .../test/support/retry_fixtures.rb | 115 ++++++++++++++++++ .../test/support/scripted_async_transport.rb | 58 +++++++++ .../test/support/scripted_transport.rb | 45 +++++++ 3 files changed, 218 insertions(+) create mode 100644 gems/dexpace-core/test/support/retry_fixtures.rb create mode 100644 gems/dexpace-core/test/support/scripted_async_transport.rb create mode 100644 gems/dexpace-core/test/support/scripted_transport.rb diff --git a/gems/dexpace-core/test/support/retry_fixtures.rb b/gems/dexpace-core/test/support/retry_fixtures.rb new file mode 100644 index 0000000..b63a266 --- /dev/null +++ b/gems/dexpace-core/test/support/retry_fixtures.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require "dexpace" +require_relative "recovery_fixtures" +require_relative "fake_clock" +require_relative "fake_body" + +# The fixtures phase 6a's retry suites fold over: a request per method, a response over a REAL +# ResponseBody (so `response.body.closed?` is the RETRY-35 assertion -- Dexpace::Response has no +# #closed? of its own), a retryable throwable that answers XCUT-6's capability and nothing else, +# and a step that hands a RetryStep a RECORDING cursor over the real one the driver minted, so +# the fork-for-every-drive rule is asserted against the real runtime rather than a stand-in +# (5b's checklist item 12's route). Built on RecoveryFixtures rather than beside it. +module RetryFixtures + include RecoveryFixtures + + # A throwable that is retryable by capability -- an IOError, as a socket read timeout would + # be (RETRY-24's shape) -- and one that answers the capability with false. + class RetryableError < ::IOError + def retryable? = true + end + + # The other answer: an IOError that says no, which a type match would call retryable. + class UnretryableError < ::IOError + def retryable? = false + end + + # RETRY-25's discriminating fixture: a fatal-family error that ANSWERS the capability. A driver + # that classified it would retry it; the correct driver never asks, because the fatal family + # passes through an absent rescue arm before any classification runs. + class RetryableFatal < ::NoMemoryError + def retryable? = true + end + + # Forwards everything to the real cursor and counts what the step did with it. #call raises + # rather than counting: a pillar step that calls its own cursor is the defect the count exists + # to find, and a raise names it at the site. + class RecordingCursor + attr_reader :forks, :calls, :real + + def initialize(real) + @real = real + @forks = 0 + @calls = 0 + end + + def fork(state: nil) + @forks += 1 + @real.fork(state: state) + end + + def call(request = @real.request) + @calls += 1 + @real.call(request) + end + + def request = @real.request + def options = @real.options + def cancellation = @real.cancellation + def bundle = @real.bundle + def spent? = @real.spent? + def may_fork? = @real.may_fork? + def state(stage) = @real.state(stage) + end + + # A step that wraps the cursor before handing it on, for #forks and #calls. + class RecordingWrapper + attr_reader :cursors + + def initialize(step) + @step = step + @cursors = [] + end + + def stage = @step.stage + + def call(request, cursor) + recording = RecordingCursor.new(cursor) + @cursors << recording + @step.call(request, recording) + end + end + + def retry_request(method: "GET", body: nil, url: "https://example.test/api") + Dexpace::Request.build(method: Dexpace::Method.of(method), url: url, + headers: Dexpace::Headers::EMPTY, body: body,) + end + + # A response over a closable single-use body, with optional inbound headers. + def retry_response(code, headers: {}, request: retry_request) + builder = Dexpace::Response.builder + builder.request = request + builder.protocol = Dexpace::Protocol::HTTP_1_1 + builder.status = code + builder.body = response_body("body #{code}") + unless headers.empty? + inbound = Dexpace::Headers.inbound_builder + headers.each { |name, value| inbound.add(name, value) } + builder.headers = inbound.build + end + builder.build + end + + def replayable_body = FakeBody.new("payload", replayable: true) + def consumed_body = FakeBody.new("payload", replayable: false) + + # Settings over a FakeClock with a flat, jitter-free schedule unless told otherwise. + def retry_settings(clock: FakeClock.new, **overrides) + Dexpace::Resilience::RetrySettings.build( + initial_delay: 0.1, multiplier: 2.0, max_delay: 8.0, jitter: 0.0, + clock: clock, **overrides, + ) + end +end diff --git a/gems/dexpace-core/test/support/scripted_async_transport.rb b/gems/dexpace-core/test/support/scripted_async_transport.rb new file mode 100644 index 0000000..25c0a0d --- /dev/null +++ b/gems/dexpace-core/test/support/scripted_async_transport.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "scripted_transport" + +# The async twin of ScriptedTransport (SEAM-16): the same script -- a callable, an Exception or a +# response per call, in order -- delivered through a fresh Completer per call. By default each +# future settles INLINE, inside #call, which is the shape a scripted downstream has and the shape +# that made the plan's recursive pump overflow (RETRY-30); with `settle_later: true` every future +# is left pending and #settle_next! settles the oldest unsettled one, so a test can hold an +# attempt in flight, cancel the returned future, and then let the attempt land (RETRY-32). +# +# A Proc entry is invoked with (request, options, cancellation) and its answer -- a Response or an +# Exception -- is what the future settles with, so a script can cancel a token from inside the +# attempt it is serving. #calls and the three-positional #call are ScriptedTransport's. +class ScriptedAsyncTransport + attr_reader :pending + + def initialize(script, settle_later: false) + @inner = ScriptedTransport.new(script) + @settle_later = settle_later + @pending = [] + @mutex = ::Thread::Mutex.new + end + + def calls = @inner.calls + + def call(request, options, cancellation) + completer = Dexpace::Async::Completer.new + outcome = begin + [@inner.call(request, options, cancellation), nil] + rescue ::StandardError => error + [nil, error] + end + if @settle_later + @mutex.synchronize { @pending << [completer, outcome] } + else + deliver(completer, outcome) + end + completer.future + end + + # Settles the oldest pending future; nil when none is pending. + def settle_next! + entry = @mutex.synchronize { @pending.shift } + return nil if entry.nil? + + deliver(*entry) + true + end + + private + + def deliver(completer, outcome) + response, error = outcome + error.nil? ? completer.fulfil(response) : completer.fail(error) + end +end diff --git a/gems/dexpace-core/test/support/scripted_transport.rb b/gems/dexpace-core/test/support/scripted_transport.rb new file mode 100644 index 0000000..2a72d8e --- /dev/null +++ b/gems/dexpace-core/test/support/scripted_transport.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +# A scriptable Dexpace::Transport double (phase 2's duck type, SEAM-11), phase 6a's: where +# FakeTransport answers ONE response or raise for every call, this one consumes a script -- one +# entry per call, in order -- so a retry suite can stage a 503, 503, 200 sequence (RECOV-19, +# RETRY-36) or a run of throwables (RETRY-34). Each entry is consumed once: a callable is +# invoked with (request, options, cancellation) and its answer used; an Exception instance is +# raised; anything else is returned as it is. A call past the end of the script raises, so a +# driver that sends once too often fails loudly rather than reading nil. +# +# #calls records every call's [request, options, cancellation] triple, frozen, so a test can +# assert the SAME request object was re-sent every time (RETRY-44, RECOV-19) with assert_same. +# `def call(request, options, cancellation)` with three positionals is what keeps +# Registry.callable?(transport, arity: 3) true, which Recovery::Orchestrator.build checks. +# +# Not phase 2's FakeTransport, which seven suites and three other doubles require by name and +# shape and which stays exactly as it is; a new double under a new name (phase 6a's checklist, +# "Deviations from the plan"). Top level, like every double under test/support/ since phase 2. +class ScriptedTransport + attr_reader :calls + + def initialize(script) + @script = script.dup + @calls = [] + @mutex = ::Thread::Mutex.new + end + + def call(request, options, cancellation) + entry = @mutex.synchronize do + @calls << [request, options, cancellation].freeze + raise "ScriptedTransport: script exhausted after #{@calls.size - 1} calls" if @script.empty? + + @script.shift + end + case entry + when ::Exception then raise entry + when ::Proc, ::Method then entry.call(request, options, cancellation) + else entry + end + end + + # The script entries not yet consumed. + def remaining = @mutex.synchronize { @script.size } +end From 29375c5dcb2a963598a1e09e59c84c124fcbe9af Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:14:54 +0300 Subject: [PATCH 08/13] test: the retry policy, re-sendability and settings suites Add policy_test.rb (the two-axis classifier consult, the backoff calculator including the 2,000-attempt run that found the NaN at attempt 1,025, the pacing-header parse in every form RETRY-15 names, the retry-count resolver's clamp and its logged diagnostic, and the recovery-only budget), resend_test.rb (the re-sendability gate over phase 1's idempotent set and 3b's replayability, with the streaming and chunked bodies that can never be re-sent) and retry_settings_test.rb (the frozen settings, its validators, the UNSET sentinel and the first read of 5a's Keys::MAX_RETRY_ATTEMPTS under a fake config source). Extend two earlier-phase suites exactly as their postponements said: protocol_error_test.rb gains the nested RetryableTest over the baked retryable_by_status? flag, agreeing with Retryability across 400..599 and deliberately not answering #retryable? (P6-10); http_date_test.rb gains RETRY-15's two single-digit-day tests, one for what the widening admits and one for what it still refuses. --- .../test/dexpace/error/protocol_error_test.rb | 53 +- .../test/dexpace/http_date_test.rb | 17 + .../test/dexpace/resilience/policy_test.rb | 510 ++++++++++++++++++ .../test/dexpace/resilience/resend_test.rb | 65 +++ .../dexpace/resilience/retry_settings_test.rb | 229 ++++++++ 5 files changed, 871 insertions(+), 3 deletions(-) create mode 100644 gems/dexpace-core/test/dexpace/resilience/policy_test.rb create mode 100644 gems/dexpace-core/test/dexpace/resilience/resend_test.rb create mode 100644 gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb diff --git a/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb b/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb index 8e7a3db..c96f4b5 100644 --- a/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb +++ b/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb @@ -72,7 +72,54 @@ class DexpaceProtocolErrorTest < DexpaceTestCase assert_raises(Dexpace::InvalidArgumentError) { Dexpace::ProtocolError.for_or_nil(nil) } end - # No test pins the ABSENCE of #retryable_by_status?: XCUT-5's baked flag is postponed to phase - # 6a, Task 6, which adds the predicate to THIS class and extends THIS suite. A postponement is - # recorded in a checklist row, not pinned by an assertion a later phase has to delete. + # Phase 6a's Task 6 added the predicate 4b postponed; the nested class below is its suite, + # extending this file exactly as the postponement said it would. + + # XCUT-5 / RETRY-3: the baked flag, computed at construction from the SINGLE shared status + # classifier (5a's Dexpace::Retryability), and deliberately not named #retryable? (P6-10). + class RetryableTest < DexpaceTestCase + include RecoveryFixtures + + test "RETRY-3 / XCUT-5: retryable_by_status? is computed at construction from Retryability" do + error = Dexpace::ProtocolError.for(build_response(503)) + + assert_predicate(error, :retryable_by_status?) + assert_equal(Dexpace::Retryability.retryable_status?(503), error.retryable_by_status?) + assert_same(error.retryable_by_status?, error.retryable_by_status?, "baked, not recomputed") + end + + test "RETRY-3 / XCUT-5: the flag agrees with the classifier across the whole error range" do + (400..599).each do |code| + error = Dexpace::ProtocolError.for(build_response(code)) + expected = Dexpace::Retryability.retryable_status?(code) + + assert_equal(expected, error.retryable_by_status?, code.to_s) + end + refute_predicate(Dexpace::ProtocolError.for(build_response(501)), :retryable_by_status?) + refute_predicate(Dexpace::ProtocolError.for(build_response(505)), :retryable_by_status?) + refute_predicate(Dexpace::ProtocolError.for(build_response(404)), :retryable_by_status?) + assert_predicate(Dexpace::ProtocolError.for(build_response(408)), :retryable_by_status?) + assert_predicate(Dexpace::ProtocolError.for(build_response(429)), :retryable_by_status?) + end + + test "RETRY-3: no per-subclass constant -- a subclass inherits the computed flag" do + subclass = Class.new(Dexpace::ProtocolError) + + assert_predicate(subclass.new(build_response(502)), :retryable_by_status?) + refute_predicate(subclass.new(build_response(501)), :retryable_by_status?) + end + + test "P6-10: a ProtocolError does NOT answer XCUT-6's generic #retryable? capability" do + error = Dexpace::ProtocolError.for(build_response(503)) + + refute_respond_to(error, :retryable?) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error)) + end + + test "NFR-4: the constructor keeps 4b's single positional parameter and the two factories" do + assert_equal(1, Dexpace::ProtocolError.instance_method(:initialize).arity) + assert_instance_of(Dexpace::ProtocolError, Dexpace::ProtocolError.for(build_response(503))) + assert_nil(Dexpace::ProtocolError.for_or_nil(build_response(200))) + end + end end diff --git a/gems/dexpace-core/test/dexpace/http_date_test.rb b/gems/dexpace-core/test/dexpace/http_date_test.rb index b57b451..a2691be 100644 --- a/gems/dexpace-core/test/dexpace/http_date_test.rb +++ b/gems/dexpace-core/test/dexpace/http_date_test.rb @@ -113,6 +113,23 @@ class ParseTest < DexpaceTestCase end end + # Phase 6a's R1: RETRY-15's HTTP-date tolerances, asserted against the ONE parser. The day + # group was widened from (\d{2}) to (\d{1,2}); the weekday was already informational (CFG-30) + # and stays required (CFG-31) -- the row above asserts the bare form still fails. + test "RETRY-15: a single-digit day parses as that day, the weekday still informational" do + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("Sun, 6 Nov 1994 08:49:37 GMT")) + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("Mon, 6 Nov 1994 08:49:37 GMT")) + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("xyz, 6 nov 1994 08:49:37 utc")) + assert_equal(Time.utc(2024, 1, 1), Dexpace::HTTPDate.parse("Mon, 1 Jan 2024 00:00:00 GMT")) + end + + test "RETRY-15: the widening admits digit COUNT, not calendar nonsense or a third digit" do + ["Sun, 0 Nov 1994 08:49:37 GMT", "Sun, 006 Nov 1994 08:49:37 GMT", + "Sun, 6 Nov 1994 08:49:37 GMT", "Sun, 6 Nov 1994 08:49:37 GMT",].each do |text| + assert_raises(Dexpace::InvalidArgumentError, text.inspect) { Dexpace::HTTPDate.parse(text) } + end + end + test "CFG-31: a well-formed but impossible date fails as a parse error, not an ArgumentError" do ["Sun, 32 Nov 1994 08:49:37 GMT", "Sun, 31 Nov 1994 08:49:37 GMT", "Sun, 06 Nov 1994 24:00:00 GMT", "Sun, 06 Nov 1994 08:60:37 GMT", diff --git a/gems/dexpace-core/test/dexpace/resilience/policy_test.rb b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb new file mode 100644 index 0000000..62c793b --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb @@ -0,0 +1,510 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/recovery_fixtures" +require_relative "../../support/cyclic_errors" +require_relative "../../support/recording_sink" + +# Exercises: RETRY-1, RETRY-2, RETRY-9, RETRY-10, RETRY-11, RETRY-12, RETRY-13, RETRY-15, +# RETRY-16, RETRY-17, RETRY-18, RETRY-19, RETRY-20, RETRY-21, RETRY-22, RETRY-37, RETRY-41, +# RETRY-42, RECOV-17, RECOV-20, RECOV-21, RECOV-22, RECOV-23, RECOV-24, RECOV-25, RECOV-26, +# RECOV-29, RECOV-30, CFG-35, XCUT-6, XCUT-7, XCUT-9, P6-5, P6-10, P6-12 +# +# The shared policy core: the two-branch classifier consult, the calculator, the resolver, the +# recovery-only budget and the pacing dispatcher. Split into nested classes under +# Metrics/ClassLength: classification, backoff and budgets, then the pacing parser's forms and +# its totality (RETRY-16 is a NEGATIVE suite -- every malformed input asserts nil, never +# assert_nothing_raised). +class DexpaceResiliencePolicyTest < DexpaceTestCase + Policy = Dexpace::Resilience::Policy + + # Shared by the nested classes. + module Fixtures + include RecoveryFixtures + + # A retryable throwable by XCUT-6's capability, and nothing else about it: not an IOError, + # so a concrete-type classifier would refuse it and the capability one must not. + class RetryableByCapability < ::StandardError + def retryable? = true + end + + # An IOError that says it is NOT retryable: the case a concrete-type classifier gets wrong + # in the other direction (5a's R1: StreamError is an IOError and not a transport timeout). + class UnretryableIOError < ::IOError + def retryable? = false + end + + # Exception#cause is assigned by the interpreter at the raise site and cannot be set + # through an ivar, so every cause chain here is built by actually raising. + def wrap(inner, outer_class = ::StandardError, message = "wrapper") + begin + raise inner + rescue inner.class + raise outer_class, message + end + rescue outer_class => error + error + end + + def headers(pairs) + builder = Dexpace::Headers.inbound_builder + pairs.each { |name, value| builder.add(name, value) } + builder.build + end + end + include Fixtures + + test "RETRY-12 / RETRY-13: the defaults are 200 ms, x2, 8 s, 0.2 and two retries (3 sends)" do + assert_in_delta(0.2, Policy::DEFAULT_INITIAL_DELAY) + assert_in_delta(2.0, Policy::DEFAULT_MULTIPLIER) + assert_in_delta(8.0, Policy::DEFAULT_MAX_DELAY) + assert_in_delta(0.2, Policy::DEFAULT_JITTER) + assert_equal(2, Policy::DEFAULT_MAX_RETRIES) + assert_equal(3, Policy::DEFAULT_MAX_RETRIES + 1, "RETRY-14: three total wire sends") + end + + test "XCUT-7: the default configurable set is {408, 429, 500, 502, 503, 504}, frozen, a subset" do + assert_equal(::Set[408, 429, 500, 502, 503, 504], Policy::DEFAULT_RETRYABLE_STATUSES) + assert_predicate(Policy::DEFAULT_RETRYABLE_STATUSES, :frozen?) + Policy::DEFAULT_RETRYABLE_STATUSES.each do |code| + assert(Dexpace::Retryability.retryable_status?(code), "#{code} is in the baked set too") + end + refute_includes(Policy::DEFAULT_RETRYABLE_STATUSES, 599, "the baked set is wider") + end + + test "RETRY-21: the fixed precedence is Retry-After, the two ms variants, then the reset" do + assert_equal(%w[Retry-After retry-after-ms x-ms-retry-after-ms X-RateLimit-Reset], + Policy::DEFAULT_PACING_HEADER_ORDER,) + assert_predicate(Policy::DEFAULT_PACING_HEADER_ORDER, :frozen?) + end + + test "RETRY-18 / RECOV-34: the ceilings are 365 days and the signed 64-bit nanosecond count" do + assert_equal(31_536_000, Policy::MAX_PACING_DELAY_SECONDS) + assert_equal(9_223_372_036_854_775_807, Policy::MAX_DURATION_NANOSECONDS) + end + + test "RETRY-42 / P6-5: Policy is a frozen-constant, extend-self module with no state" do + assert_empty(Policy.instance_variables) + assert_kind_of(Policy, Policy, "extend self, never module_function") + assert_equal( + %i[backoff_delay budget_remaining effective_max_retries pacing_delay retry_eligible? + retryable? throwable_retryable?], + Policy.public_instance_methods(false).sort, + ) + end + + # RETRY-1, RETRY-2, RETRY-37, RECOV-17, CFG-35, XCUT-6, XCUT-7, XCUT-9, P6-10. + class ClassificationTest < DexpaceTestCase + include Fixtures + + test "RETRY-37 / XCUT-7: retry_eligible? consults the set alone; a NARROWING set narrows" do + assert(Policy.retry_eligible?(418, set: ::Set[418])) + refute(Policy.retry_eligible?(500, set: ::Set[418])) + # 500 is retryable under 5a's baked classifier and still refused by a set that omits it: + # authoritative-contains, never an intersection with the baked flag. + assert(Dexpace::Retryability.retryable_status?(500)) + refute(Policy.retry_eligible?(500, set: ::Set[503])) + assert(Policy.retry_eligible?(404, set: ::Set[404]), "and a set that WIDENS widens") + end + + test "RETRY-2 / CFG-35: throwable_retryable? is the capability query, not a type match" do + assert(Policy.throwable_retryable?(RetryableByCapability.new("reset"))) + refute(Policy.throwable_retryable?(UnretryableIOError.new("an IOError saying no"))) + refute(Policy.throwable_retryable?(Dexpace::StreamError.new("an IOError with no capability"))) + refute(Policy.throwable_retryable?(::StandardError.new("no capability"))) + end + + test "RETRY-2: the capability is found anywhere in the cause chain, nearest first" do + wrapped = wrap(RetryableByCapability.new("inner")) + + assert_kind_of(RetryableByCapability, wrapped.cause, "the fixture is load-bearing") + assert(Policy.throwable_retryable?(wrapped)) + assert(Policy.throwable_retryable?(wrap(wrapped, ::RuntimeError, "outer"))) + refute(Policy.throwable_retryable?(wrap(::StandardError.new("inner")))) + end + + test "RETRY-2 / XCUT-9: a cyclic cause chain terminates, through 4b's identity-tracking walk" do + first = CyclicErrorFixtures::CyclicPair.new("one") + second = CyclicErrorFixtures::CyclicPair.new("two") + first.other_cause = second + second.other_cause = first + + refute(Policy.throwable_retryable?(first)) + refute(Policy.throwable_retryable?(CyclicErrorFixtures::SelfCause.new("self"))) + end + + test "RETRY-2: a non-Exception is refused rather than classified" do + assert_raises(Dexpace::InvalidArgumentError) { Policy.throwable_retryable?("boom") } + end + + test "RETRY-37: retryable? classifies a ProtocolError by the configured set alone" do + error = Dexpace::ProtocolError.for(build_response(418)) + + assert(Policy.retryable?(error, retryable_statuses: ::Set[418])) + refute(Policy.retryable?(error, retryable_statuses: ::Set[500])) + refute(Policy.retryable?(Dexpace::ProtocolError.for(build_response(503)), + retryable_statuses: ::Set[418],), "503 narrowed away",) + end + + test "RETRY-2: retryable? classifies anything else by the capability" do + assert(Policy.retryable?(RetryableByCapability.new("x"), retryable_statuses: ::Set[])) + refute(Policy.retryable?(::RuntimeError.new("x"), retryable_statuses: ::Set[500])) + end + + test "P6-10: a ProtocolError buried in a cause chain cannot answer the capability query" do + protocol = Dexpace::ProtocolError.for(build_response(503)) + + assert_predicate(protocol, :retryable_by_status?) + refute_respond_to(protocol, :retryable?, "the baked flag is not the open capability") + wrapped = wrap(protocol, Dexpace::PipelineError, "downstream") + + assert_same(protocol, wrapped.cause) + refute(Policy.throwable_retryable?(wrapped)) + refute(Policy.retryable?(wrapped, retryable_statuses: ::Set[503]), + "a configured set that ADMITS 503 still does not reach a wrapped ProtocolError " \ + "through the capability branch; the status branch needs the error itself",) + end + end + + # RETRY-9, RETRY-10, RETRY-11, RETRY-41, RECOV-20, RECOV-21, P6-12. + class BackoffAndBudgetTest < DexpaceTestCase + include Fixtures + + def backoff(attempt, **overrides) + Policy.backoff_delay( + attempt, initial_delay: 0.2, multiplier: 2.0, max_delay: 8.0, jitter: 0.0, + random: ::Random.new(1), **overrides, + ) + end + + test "RETRY-9 / RECOV-21: the unjittered delay is initial * multiplier**(n - 1), capped" do + assert_in_delta(0.2, backoff(1)) + assert_in_delta(0.4, backoff(2)) + assert_in_delta(0.8, backoff(3)) + assert_in_delta(6.4, backoff(6)) + assert_in_delta(8.0, backoff(7), 0.001, "12.8 capped to the 8.0 max") + assert_in_delta(8.0, backoff(8)) + end + + test "RETRY-11: an attempt below 1 is refused, and the cap saturates a huge attempt" do + assert_raises(Dexpace::InvalidArgumentError) { backoff(0) } + assert_raises(Dexpace::InvalidArgumentError) { backoff(-1) } + assert_raises(Dexpace::InvalidArgumentError) { backoff(1.5) } + assert_in_delta(8.0, backoff(1_000), 0.001, "2.0**999 overflows to Infinity, never raises") + assert_in_delta(8.0, backoff(100_000)) + end + + test "RETRY-11 / P6-53: a zero initial delay is zero at any attempt, never 0 * Infinity" do + assert_in_delta(0.0, backoff(1, initial_delay: 0.0)) + assert_in_delta(0.0, backoff(1_025, initial_delay: 0.0), 0.001, + "2.0**1024 is Infinity; 0 * Inf is NaN",) + assert_in_delta(0.0, backoff(5_000, initial_delay: 0, jitter: 1.0)) + end + + test "RETRY-9: Integer inputs are accepted and the result is always a Float" do + assert_in_delta(0.5, backoff(2, initial_delay: 1, multiplier: 2, max_delay: 8, jitter: 0) + .then { |d| d / 4 },) + assert_instance_of(::Float, backoff(1, initial_delay: 1, multiplier: 1, max_delay: 1)) + end + + test "RETRY-10: jitter samples uniformly within [d(1 - j/2), d(1 + j/2)] with midpoint d" do + random = ::Random.new(20_260_918) + samples = Array.new(2_000) { backoff(1, initial_delay: 1.0, jitter: 0.5, random: random) } + + samples.each do |d| + assert_operator(d, :>=, 0.75) + assert_operator(d, :<=, 1.25) + end + assert_in_delta(1.0, samples.sum / samples.size, 0.02, "midpoint stays at d") + assert_operator(samples.uniq.size, :>, 1_000, "a real draw, not a constant") + end + + test "RETRY-10: jitter 0 returns d exactly; a degenerate sub-nanosecond spread returns d" do + # Exact, so the deltas are tighter than the spread a draw would produce: an autocorrected + # default delta of 0.001 would let a drawn 9.6e-13 pass for 1e-12 (the guard ran green). + assert_in_delta(0.2, backoff(1, jitter: 0.0), 0.0) + assert_in_delta(1e-12, backoff(1, initial_delay: 1e-12, jitter: 0.5), 0.0) + assert_in_delta(1e-9, backoff(1, initial_delay: 1e-9, jitter: 0.5), 0.0, + "spread 5e-10 < 1e-9",) + refute_in_delta(3e-9, backoff(1, initial_delay: 3e-9, jitter: 0.5), 1e-12, + "spread 1.5e-9 draws",) + end + + test "RETRY-10: at the cap the jitter band is drawn around the cap (base == cap)" do + random = ::Random.new(7) + 20.times do + d = backoff(50, jitter: 1.0, random: random) + + assert_operator(d, :>=, 4.0) + assert_operator(d, :<=, 12.0, "d(1 + j/2) with d = 8.0 and j = 1.0") + end + end + + test "RETRY-10: a negative sample is floored at zero" do + below_zero = ::Object.new + def below_zero.rand(_range) = -0.5 + + assert_in_delta(0.0, backoff(1, jitter: 1.0, random: below_zero)) + end + end + + # RETRY-41, RECOV-20, P6-12: the resolver and the recovery-only budget. + class ResolverAndBudgetTest < DexpaceTestCase + include Fixtures + + test "RETRY-41: present-override-wins, else configured; negative clamped; zero allowed" do + assert_equal(5, Policy.effective_max_retries(override: 5, configured: 2)) + assert_equal(0, Policy.effective_max_retries(override: 0, configured: 2), "zero: no retries") + assert_equal(2, Policy.effective_max_retries(override: nil, configured: 2)) + assert_equal(0, Policy.effective_max_retries(override: nil, configured: 0)) + assert_equal(Policy::DEFAULT_MAX_RETRIES, + Policy.effective_max_retries(override: nil, configured: -1),) + end + + test "RETRY-41: a negative or non-Integer override is validated, not silently ignored" do + assert_raises(Dexpace::InvalidArgumentError) do + Policy.effective_max_retries(override: -1, configured: 2) + end + assert_raises(Dexpace::InvalidArgumentError) do + Policy.effective_max_retries(override: 1.5, configured: 2) + end + end + + test "RETRY-41 / P6-12: the clamp is logged as one WARNING config diagnostic, nothing else" do + sink = RecordingSink.new + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(2, Policy.effective_max_retries(override: nil, configured: 2, logger: logger)) + assert_equal(2, Policy.effective_max_retries(override: 9, configured: -3, logger: logger) + .then { 2 },) + assert_empty(sink.entries, "neither a configured nor an overridden value logs") + + Policy.effective_max_retries(override: nil, configured: -3, logger: logger) + + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_CONFIG, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/clamped/, payload[Dexpace::Instrumentation::Keys::MESSAGE]) + assert_match(/-3/, payload[Dexpace::Instrumentation::Keys::MESSAGE]) + end + + test "RETRY-41 / OBS-20: a raising sink cannot fail the resolution" do + sink = RecordingSink.new + def sink.warn(*) = raise("sink down") + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, + Policy.effective_max_retries(override: nil, configured: -1, logger: logger),) + end + + test "RECOV-20: budget_remaining is the time REMAINING, Infinity for a zero budget" do + assert_equal(::Float::INFINITY, Policy.budget_remaining(elapsed: 100.0, total_timeout: 0)) + assert_equal(::Float::INFINITY, Policy.budget_remaining(elapsed: 0.0, total_timeout: 0.0)) + assert_in_delta(4.0, Policy.budget_remaining(elapsed: 6.0, total_timeout: 10.0)) + assert_in_delta(-2.0, Policy.budget_remaining(elapsed: 12.0, total_timeout: 10), 0.001, + "negative once spent; a caller compares against remaining, never elapsed",) + assert_instance_of(::Float, Policy.budget_remaining(elapsed: 1, total_timeout: 10)) + end + end + + # RETRY-15, RETRY-17, RETRY-19, RETRY-21, RECOV-24, RECOV-25: the forms and the precedence. + class PacingFormsTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + + # No keyword parameter here on purpose: a braceless `delay("Retry-After" => "5")` would + # otherwise be read as keywords under Ruby 3's separation. + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + def delay_with(pairs, random) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW, random: random) + end + + test "RETRY-15 / RECOV-24: Retry-After as delta-seconds, integer and fractional" do + assert_in_delta(5.0, delay("Retry-After" => "5")) + assert_in_delta(2.5, delay("Retry-After" => "2.5")) + assert_in_delta(0.001, delay("Retry-After" => "0.001"), 0.00001, "sub-second honoured") + assert_in_delta(0.0, delay("Retry-After" => "0")) + end + + test "RETRY-15: Retry-After as an RFC 1123 HTTP-date, measured from `now`" do + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:01:30 GMT")) + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:01:30 UTC")) + end + + test "RETRY-15 / R1: the HTTP-date form tolerates a wrong weekday and a single-digit day" do + assert_in_delta(90.0, delay("Retry-After" => "Mon, 18 Sep 2026 12:01:30 GMT"), 0.001, + "wrong weekday",) + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 sep 2026 12:01:30 gmt"), 0.001, "case") + later = ::Time.utc(2026, 9, 1, 12, 0, 0) + + early = headers("Retry-After" => "Tue, 1 Sep 2026 12:01:00 GMT") + + assert_in_delta(60.0, Policy.pacing_delay(early, header_order: ORDER, now: later)) + end + + test "RETRY-17 / RECOV-23: a valid HTTP-date or epoch already past is ZERO, not nil" do + assert_in_delta(0.0, delay("Retry-After" => "Sun, 06 Nov 1994 08:49:37 GMT")) + assert_in_delta(0.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:00:00 GMT"), 0.001, + "exactly now",) + assert_in_delta(0.0, delay("X-RateLimit-Reset" => (NOW.to_i - 100).to_s)) + assert_in_delta(0.0, delay("X-RateLimit-Reset" => NOW.to_i.to_s)) + end + + test "RETRY-15 / RECOV-24: retry-after-ms and x-ms-retry-after-ms as integer milliseconds" do + assert_in_delta(1.5, delay("retry-after-ms" => "1500")) + assert_in_delta(0.25, delay("x-ms-retry-after-ms" => "250")) + assert_in_delta(0.0, delay("retry-after-ms" => "0")) + assert_nil(delay("retry-after-ms" => "1.5"), "milliseconds are an integer count") + end + + test "RETRY-15 / RECOV-25: X-RateLimit-Reset as epoch seconds, jittered up to [100%, 120%]" do + random = ::Random.new(20_260_918) + pairs = { "X-RateLimit-Reset" => (NOW.to_i + 10).to_s } + samples = Array.new(500) { delay_with(pairs, random) } + + samples.each do |d| + assert_operator(d, :>=, 10.0) + assert_operator(d, :<=, 12.0) + end + assert_operator(samples.uniq.size, :>, 100, "a real draw") + assert_operator(samples.min, :<, 10.5) + assert_operator(samples.max, :>, 11.5) + end + + test "RETRY-21 / RECOV-24: the precedence returns the first parseable value" do + assert_in_delta(5.0, delay("Retry-After" => "5", "retry-after-ms" => "9999")) + assert_in_delta(1.0, delay("retry-after-ms" => "1000", "x-ms-retry-after-ms" => "9999")) + assert_in_delta(2.0, delay("x-ms-retry-after-ms" => "2000", + "X-RateLimit-Reset" => (NOW.to_i + 9_999).to_s,),) + end + + test "RETRY-21: the stage stack's caller-configurable order is honoured as given" do + pairs = { "Retry-After" => "5", "retry-after-ms" => "1000" } + + reversed = %w[retry-after-ms Retry-After] + + assert_in_delta(1.0, Policy.pacing_delay(headers(pairs), now: NOW, header_order: reversed)) + assert_nil(Policy.pacing_delay(headers(pairs), now: NOW, header_order: [])) + assert_nil(Policy.pacing_delay(headers(pairs), now: NOW, header_order: %w[X-RateLimit-Reset])) + end + + test "HTTP-13: header names in the order are matched case-insensitively, through Headers#[]" do + lower = headers("retry-after" => "5") + + assert_in_delta(5.0, Policy.pacing_delay(lower, now: NOW, header_order: %w[RETRY-AFTER])) + end + + test "a multi-valued header is read by its first value" do + builder = Dexpace::Headers.inbound_builder + builder.add("Retry-After", "3") + builder.add("Retry-After", "9") + + assert_in_delta(3.0, Policy.pacing_delay(builder.build, header_order: ORDER, now: NOW)) + end + end + + # RETRY-16, RETRY-18, RETRY-19, RETRY-22, RECOV-23, RECOV-26, RECOV-29: the negative suite. + class PacingTotalityTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + + # No keyword parameter here on purpose: a braceless `delay("Retry-After" => "5")` would + # otherwise be read as keywords under Ruby 3's separation. + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + def delay_with(pairs, random) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW, random: random) + end + + test "RETRY-19 / RECOV-24: the decimal screen rejects every non-decimal spelling first" do + %w[0x10 1e3 5_0 5f 5d 30d 5s Infinity NaN 5,0 +5 .5 5. 0b1 1r 1i].each do |bad| + assert_nil(delay("Retry-After" => bad), "expected #{bad.inspect} to be no hint") + end + assert_equal(50, Integer("5_0", 10), "the screen is load-bearing: Integer() reads this as 50") + end + + test "RETRY-16 / RECOV-23: malformed, negative and out-of-range values are no hint, not 0" do + ["-5", "-0.5", "not a date", "", " 5", "5 ", "Sun, 32 Nov 1994 08:49:37 GMT", + "Sunday, 06-Nov-94 08:49:37 GMT", "9" * 400,].each do |bad| + assert_nil(delay("Retry-After" => bad), bad.inspect) + end + ["-1", "1.5", "abc", "", "9" * 400].each do |bad| + assert_nil(delay("X-RateLimit-Reset" => bad), bad.inspect) + assert_nil(delay("retry-after-ms" => bad), bad.inspect) unless bad == "9" * 400 + end + end + + test "RETRY-16: no matching header at all is no hint" do + assert_nil(delay({})) + assert_nil(delay("Content-Type" => "text/plain")) + end + + test "RETRY-18 / RECOV-26: a finite computed delta past 365 days is clamped to the ceiling" do + ceiling = Policy::MAX_PACING_DELAY_SECONDS.to_f + + assert_equal(ceiling, delay("Retry-After" => (400 * 24 * 60 * 60).to_s)) + assert_equal(ceiling, delay("retry-after-ms" => (400 * 24 * 60 * 60 * 1000).to_s)) + assert_equal(ceiling, delay("X-RateLimit-Reset" => (NOW.to_i + (400 * 24 * 60 * 60)).to_s)) + assert_equal(ceiling, delay("Retry-After" => "Fri, 18 Sep 2099 12:00:00 GMT")) + assert_equal(ceiling, delay("Retry-After" => "9" * 300), "1e300 seconds is finite; clamped") + end + + test "RETRY-18: exactly the ceiling and one below pass through unclamped" do + ceiling = Policy::MAX_PACING_DELAY_SECONDS + + assert_equal(ceiling.to_f, delay("Retry-After" => ceiling.to_s)) + assert_equal(ceiling - 1.0, delay("Retry-After" => (ceiling - 1).to_s)) + end + + test "RETRY-22 / RECOV-29: one malformed header does not mask a later, well-formed one" do + assert_in_delta(1.0, delay("Retry-After" => "garbage", "retry-after-ms" => "1000")) + assert_in_delta(7.0, delay("Retry-After" => "30d", "retry-after-ms" => "x", + "x-ms-retry-after-ms" => "7000",),) + end + + test "RETRY-16: a parser that raises inside the dispatch is contained; the scan continues" do + exploding = ::Object.new + def exploding.rand(_range) = raise("generator down") + pairs = { "X-RateLimit-Reset" => (NOW.to_i + 10).to_s, "Retry-After" => "garbage" } + + assert_nil(delay_with(pairs, exploding)) + assert_in_delta(2.0, delay_with(pairs.merge("retry-after-ms" => "2000"), exploding)) + end + + test "RETRY-19 / RECOV-24: the grammars are anchored, frozen, and carry their own timeout" do + parsers = Dexpace::Resilience.const_get(:PacingParsers) + %i[DECIMAL_GRAMMAR INTEGER_GRAMMAR].each do |name| + grammar = parsers.const_get(name) + + assert_kind_of(::Regexp, grammar) + refute_nil(grammar.timeout, "#{name}: per-pattern, never Regexp.timeout") + assert_predicate(grammar, :frozen?) + assert_match(/\A\\A.*\\z\z/, grammar.source, "#{name} is anchored at both ends") + end + refute_includes(Dexpace::Resilience.constants, :PacingParsers, "a private_constant") + end + + test "RETRY-16: a value with no grammar at all -- a non-String -- is no hint" do + forged = ::Object.new + def forged.[](_name) = [nil] + forged2 = ::Object.new + def forged2.[](_name) = [42] + + assert_nil(Policy.pacing_delay(forged, header_order: ORDER, now: NOW)) + assert_nil(Policy.pacing_delay(forged2, header_order: ORDER, now: NOW)) + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/resend_test.rb b/gems/dexpace-core/test/dexpace/resilience/resend_test.rb new file mode 100644 index 0000000..2ed6cd5 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/resend_test.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require "stringio" +require_relative "../../support/fake_body" + +# Exercises: RETRY-5, RETRY-6, RETRY-7, RETRY-8, RECOV-18, HTTP-9, BODY-1 +# +# The re-sendability gate's four-cell truth table -- body-less idempotent and non-idempotent, +# body-bearing replayable and non-replayable -- plus the two single-source assertions: the +# idempotent set is phase 1's and the replayability answer is the body's own. +class DexpaceResilienceResendTest < DexpaceTestCase + Resend = Dexpace::Resilience::Resend + + def request(method, body: nil) + Dexpace::Request.build(method: Dexpace::Method.of(method), url: "https://example.test/x", + headers: Dexpace::Headers::EMPTY, body: body,) + end + + test "RETRY-5 / RETRY-6: a body-less request is eligible iff its method is idempotent" do + %w[GET HEAD OPTIONS PUT DELETE].each do |method| + assert(Resend.eligible?(request(method)), "#{method} is idempotent") + end + %w[POST PATCH CONNECT TRACE].each do |method| + refute(Resend.eligible?(request(method)), "#{method} is not idempotent") + end + end + + test "RETRY-6 / HTTP-9: the idempotent set is phase 1's single source, never a second list" do + assert_equal(%w[GET HEAD OPTIONS PUT DELETE], Dexpace::Method::IDEMPOTENT) + source = File.read(File.expand_path("../../../lib/dexpace/resilience/resend.rb", __dir__)) + code = source.gsub(/^\s*#.*$/, "") + + refute_match(/\b(GET|HEAD|OPTIONS|DELETE)\b/, code, "no second method set") + assert_match(/idempotent\?/, code) + end + + test "RETRY-7 / RECOV-18: a bare non-idempotent POST is NOT eligible, payload or no payload" do + refute(Resend.eligible?(request("POST"))) + refute(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: false)))) + end + + test "RETRY-5 / BODY-1: a body-bearing request is eligible iff its body is replayable" do + assert(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: true)))) + refute(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: false)))) + assert(Resend.eligible?(request("PATCH", body: FakeBody.new("x", replayable: true))), + "POST/PATCH are re-sendable only via the replayable-body path (RETRY-6)",) + end + + test "RETRY-8 / RECOV-18: an idempotent method with a non-replayable body is NOT eligible" do + refute(Resend.eligible?(request("PUT", body: FakeBody.new("x", replayable: false)))) + refute(Resend.eligible?(request("DELETE", body: FakeBody.new("x", replayable: false)))) + assert(Resend.eligible?(request("PUT", body: FakeBody.new("x", replayable: true)))) + end + + test "RETRY-5: the shipped bodies answer for themselves -- bytes yes, an owned stream no" do + bytes = Dexpace::Body.bytes("payload".b) + stream = Dexpace::Body.stream(StringIO.new("payload"), close: true) + + assert(Resend.eligible?(request("POST", body: bytes))) + refute(Resend.eligible?(request("POST", body: stream)), "close: true forces single use") + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb new file mode 100644 index 0000000..1d7509f --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/fake_clock" +require_relative "../../support/fake_config_source" + +# Exercises: RECOV-34, RETRY-12, RETRY-13, RETRY-14, RETRY-21, RETRY-42, RECOV-30, CFG-14, +# SEAM-29, P6-6 +# +# The one retry configuration: its defaults are Policy's constants, its validation is RECOV-34 +# clause by clause, its collections are copied and frozen, and its max_retries is the first +# reader of 5a's Keys::MAX_RETRY_ATTEMPTS. Hermetic: every test that consults the process slot +# installs a FakeConfigSource env seam and resets the slot in teardown (5a's rule), so a host +# MAX_RETRY_ATTEMPTS variable cannot change the answer. +class DexpaceResilienceRetrySettingsTest < DexpaceTestCase + Settings = Dexpace::Resilience::RetrySettings + Policy = Dexpace::Resilience::Policy + KEY = Dexpace::Configuration::Keys::MAX_RETRY_ATTEMPTS + + # The hermetic slot, shared by the nested classes. + module Hermetic + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + end + include Hermetic + + test "RECOV-34 / RETRY-12: the defaults are Policy's shared constants; the object is frozen" do + settings = Settings.build + + assert_in_delta(Policy::DEFAULT_INITIAL_DELAY, settings.initial_delay) + assert_in_delta(Policy::DEFAULT_MULTIPLIER, settings.multiplier) + assert_in_delta(Policy::DEFAULT_MAX_DELAY, settings.max_delay) + assert_in_delta(Policy::DEFAULT_JITTER, settings.jitter) + assert_equal(Policy::DEFAULT_MAX_RETRIES, settings.max_retries) + assert_in_delta(0.0, settings.total_timeout, 0.0, "unbounded by default") + assert_equal(Policy::DEFAULT_RETRYABLE_STATUSES, settings.retryable_statuses) + assert_nil(settings.pacing_header_order) + assert_same(::Random, settings.random) + assert_same(Dexpace::Clock::SYSTEM, settings.clock) + assert_predicate(settings, :frozen?) + refute_respond_to(Settings, :new) + end + + test "RETRY-13: backoff_arguments is exactly the keyword set Policy.backoff_delay takes" do + settings = Settings.build(initial_delay: 1, multiplier: 3, max_delay: 9, jitter: 0) + arguments = settings.backoff_arguments + + assert_equal(%i[initial_delay multiplier max_delay jitter random], arguments.keys) + assert_in_delta(3.0, Policy.backoff_delay(2, **arguments), 0.0, "1 * 3**1 = 3, uncapped") + assert_in_delta(9.0, Policy.backoff_delay(3, **arguments), 0.0, "1 * 3**2 = 9, at the cap") + end + + test "RETRY-21: header_order is the caller's list, else the shared default" do + assert_same(Policy::DEFAULT_PACING_HEADER_ORDER, Settings.build.header_order) + custom = Settings.build(pacing_header_order: %w[X-RateLimit-Reset]) + + assert_equal(%w[X-RateLimit-Reset], custom.header_order) + assert_equal(%w[X-RateLimit-Reset], custom.pacing_header_order) + end + + # RECOV-34's validation clauses. + class ValidationTest < DexpaceTestCase + include Hermetic + + test "RECOV-34: durations are non-negative, finite and within the ~292-year ceiling" do + %i[initial_delay max_delay total_timeout].each do |member| + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => -1.0) } + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => nil) } + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => "1") } + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => ::Float::INFINITY) + end + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => ::Float::NAN) + end + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => 9_223_372_037.0) # one second past the ceiling + end + assert_in_delta(9_223_372_036.0, + Settings.build(member => 9_223_372_036).public_send(member),) + assert_in_delta(0.0, Settings.build(member => 0).public_send(member)) + end + end + + test "RECOV-34: the multiplier is >= 1.0 and finite" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: 0.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: 0) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: ::Float::INFINITY) } + assert_in_delta(1.0, Settings.build(multiplier: 1).multiplier) + end + + test "RECOV-34: the jitter lies within [0.0, 1.0]" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: 1.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: -0.1) } + assert_in_delta(0.0, Settings.build(jitter: 0).jitter) + assert_in_delta(1.0, Settings.build(jitter: 1).jitter) + end + + test "RECOV-34: max_retries is a non-negative Integer; zero disables retries" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: -1) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: 1.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: nil) } + assert_equal(0, Settings.build(max_retries: 0).max_retries) + end + + test "SEAM-29: every refusal names its member" do + error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_delay: -1) } + + assert_includes(error.message, "max_delay") + error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: 2) } + + assert_includes(error.message, "jitter") + end + + test "RECOV-34: retryable_statuses and pacing_header_order are copied and deep-frozen" do + statuses = ::Set[500] + order = [+"Retry-After"] + settings = Settings.build(retryable_statuses: statuses, pacing_header_order: order) + statuses << 599 + order << "retry-after-ms" + order.first << "-mutated" + + assert_equal(::Set[500], settings.retryable_statuses) + assert_equal(%w[Retry-After], settings.pacing_header_order) + assert_predicate(settings.retryable_statuses, :frozen?) + assert_predicate(settings.pacing_header_order, :frozen?) + assert_predicate(settings.pacing_header_order.first, :frozen?) + refute_predicate(statuses, :frozen?, "the caller's own collection is untouched") + end + + test "RECOV-34: retryable_statuses accepts any Integer collection and refuses anything else" do + assert_equal(::Set[1, 2], Settings.build(retryable_statuses: [1, 2, 2]).retryable_statuses) + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(retryable_statuses: %w[500]) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(retryable_statuses: 500) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(pacing_header_order: "x") } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(pacing_header_order: [:x]) } + end + end + + # RETRY-12 / P6-6: the configured key, and #with. + class ConfiguredTest < DexpaceTestCase + include Hermetic + + test "RETRY-12 / P6-6: max_retries defaults through Keys::MAX_RETRY_ATTEMPTS when not passed" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + + assert_equal(7, Settings.build.max_retries) + assert_equal("MAX_RETRY_ATTEMPTS", KEY, "5a's name, its first reader here") + end + + test "RETRY-12 / P6-6: an explicit max_retries: wins over the configured key, zero included" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + + assert_equal(1, Settings.build(max_retries: 1).max_retries) + assert_equal(0, Settings.build(max_retries: 0).max_retries) + end + + test "RETRY-12: the key is read ONCE at build; a later configure does not move the snapshot" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + settings = Settings.build + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "9") } + + assert_equal(7, settings.max_retries) + assert_equal(9, Settings.build.max_retries) + end + + test "RETRY-12: with the key unset the default is Policy::DEFAULT_MAX_RETRIES" do + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries) + end + + test "RETRY-12 / RECOV-34: a negative configured value is refused at build, not clamped here" do + # RETRY-41's clamp is the DRIVER's, at resolution time against the per-call override; the + # settings object itself holds only a valid value (RECOV-34). + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-1") } + + error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build } + + assert_includes(error.message, "max_retries") + end + + test "RETRY-12 / CFG-5: an unparseable configured value falls to the default (5a's parser)" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "seven") } + + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries) + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "010") } + + assert_equal(10, Settings.build.max_retries, "base 10 explicit (CFG-5)") + end + + test "P6-6 / RETRY-14: the configured number is the STAGE vocabulary; attempts are one more" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "4") } + + assert_equal(4, Settings.build.max_retries) + assert_equal(5, Settings.build.max_retries + 1, "the recovery stack's max_attempts") + end + + test "#with re-validates through .build on every Ruby, never through Data#with" do + settings = Settings.build(max_retries: 3) + clock = FakeClock.new + + assert_same(clock, settings.with(clock: clock).clock) + assert_equal(3, settings.with(clock: clock).max_retries, + "an explicit member survives #with",) + assert_raises(Dexpace::InvalidArgumentError) { settings.with(jitter: 5.0) } + assert_raises(Dexpace::InvalidArgumentError) { settings.with(initial_delay: -1) } + end + + test "#with of max_retries never re-hits the configured key" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + settings = Settings.build(max_retries: 1) + + assert_equal(1, settings.with(jitter: 0.0).max_retries) + end + + test "RETRY-42: value equality is over the members; two default settings are ==" do + assert_equal(Settings.build, Settings.build) + refute_equal(Settings.build, Settings.build(max_retries: 9)) + end + end +end From 99b3e10924a15aee61ca7551f70f9f1ad94ae1a8 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:15:09 +0300 Subject: [PATCH 09/13] test: the cursor-bundle widening on both runtimes Extend five phase-4c and phase-5b suites with the tests of the one widening Task 8 made. cursor_test.rb's BundleTest: Cursor#bundle is NONE by default, the seeded object at every position and across every fork, refused when not a Bundle, and no writer of any name appears. pipeline_test.rb's and async_pipeline_test.rb's BundleSeedingTest: the `bundle:` keyword seeds the cursor for one call, the empty pipeline still dispatches with no cursor, both runtimes remain a Transport by the duck type, and a non-Bundle is refused before any step runs. step_test.rb's and async_step_test.rb's BundlePrecedenceTest: the reconciled precedence's first clause -- the seeded bundle's tracer factory wins over the step's keyword, the keyword over the constant, a call that seeds nothing resolves exactly as phase 5 did, and the seeded bundle's ids reach the diagnostic context through Tracing.correlate. --- .../test/dexpace/async_pipeline_test.rb | 46 +++++++++++ .../instrumentation/async_step_test.rb | 23 ++++++ .../test/dexpace/instrumentation/step_test.rb | 68 +++++++++++++++ .../test/dexpace/pipeline/cursor_test.rb | 82 +++++++++++++++++++ .../test/dexpace/pipeline_test.rb | 70 ++++++++++++++++ 5 files changed, 289 insertions(+) diff --git a/gems/dexpace-core/test/dexpace/async_pipeline_test.rb b/gems/dexpace-core/test/dexpace/async_pipeline_test.rb index ff8cc24..6651942 100644 --- a/gems/dexpace-core/test/dexpace/async_pipeline_test.rb +++ b/gems/dexpace-core/test/dexpace/async_pipeline_test.rb @@ -256,4 +256,50 @@ def closed? = @closes.positive? assert_predicate(transport.completer.future.value, :closed?) end end + + # Phase 6a's Task 8: the `bundle:` keyword on the async runtime. + class BundleSeedingTest < DexpaceTestCase + BUNDLE = Dexpace::Instrumentation::Bundle + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C,) + end + + def request + Dexpace::Request.build(method: Dexpace::Method::GET, url: "https://example.test/", + headers: Dexpace::Headers::EMPTY,) + end + + def settled(value) + completer = Dexpace::Async::Completer.new + completer.fulfil(value) + completer.future + end + + test "cursor bundle: AsyncPipeline#call seeds the cursor's bundle, NONE when omitted" do + seen = [] + step = lambda { |req, cur| + seen << cur.bundle + cur.call(req) + } + transport = ->(_req, _opts, _canc) { settled(:response) } + pipeline = Dexpace::Pipeline::Builder.new(transport: transport) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build_async + bundle = seeded + + assert_equal(:response, pipeline.call(request).value) + assert_equal(:response, pipeline.call(request, bundle: bundle).value) + assert_same(BUNDLE::NONE, seen[0]) + assert_same(bundle, seen[1]) + end + + test "cursor bundle: the empty async pipeline accepts the keyword and allocates no cursor" do + pipeline = Dexpace::AsyncPipeline.direct(->(_r, _o, _c) { settled(:response) }) + + assert_equal(:response, pipeline.call(request, bundle: seeded).value) + assert(Dexpace::AsyncTransport.conforms?(pipeline)) + end + end end diff --git a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb index 06a667e..d619d62 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb @@ -489,4 +489,27 @@ class ThreadBoundaryTest < DexpaceTestCase assert_equal(:error, sink.entries[1].severity) end end + + # Phase 6a's Task 8 on the async runtime: the seeded bundle's factory wins in the head. + class BundlePrecedenceTest < DexpaceTestCase + include Fixtures + + test "cursor bundle: a seeded bundle's tracer factory wins over the async step's keyword" do + own = Dexpace::RecordingTracerFactory.new + from_bundle = Dexpace::RecordingTracerFactory.new + bundle = Dexpace::Instrumentation::Bundle.build( + trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C, tracer_factory: from_bundle, + ) + step = async_step(RecordingSink.new, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + transport = FakeAsyncTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle).value + + assert_equal(1, from_bundle.tracers.size) + assert_equal(1, from_bundle.tracers.first.spans.first.finished_at.size) + assert_empty(own.tracers) + end + end end diff --git a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb index bd4cc48..e3cb509 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb @@ -711,4 +711,72 @@ def factory.tracer(*, **) = raise(::StandardError, "tracer failure") assert_same(Dexpace::Instrumentation::NO_SPAN, Dexpace::Instrumentation::Tracing.current_span) end end + + # Phase 6a's Task 8: the reconciled precedence's first clause, live through Cursor#bundle. + class BundlePrecedenceTest < DexpaceTestCase + include Fixtures + + W3C = Dexpace::Instrumentation::TraceIdFlavour::W3C + + def bundle_with(factory, span: Dexpace::Instrumentation::NO_SPAN) + Dexpace::Instrumentation::Bundle.build(trace_id: "a" * 32, span_id: "b" * 16, flavour: W3C, + tracer_factory: factory, span: span,) + end + + test "cursor bundle: a seeded bundle's tracer factory wins over the step's own keyword" do + own = Dexpace::RecordingTracerFactory.new + from_bundle = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + transport = FakeTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle_with(from_bundle)) + + assert_equal(1, from_bundle.tracers.size) + assert_equal("GET", from_bundle.tracers.first.name) + assert_equal(1, from_bundle.tracers.first.spans.first.finished_at.size) + assert_empty(own.tracers, "the keyword is the SECOND clause") + end + + test "cursor bundle: with no seeded bundle the step's keyword resolves, as in phase 5" do + own = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + + drive(step, request, response: build_response(request)) + + assert_equal(1, own.tracers.size) + end + + test "cursor bundle / OBS-23: a valid seeded bundle pushes its ids onto the diagnostics" do + seen = {} + factory = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE) + request = build_request + transport = FakeTransport.new(response: build_response(request), before_return: lambda { + seen[:trace] = ::Fiber[Dexpace::Instrumentation::Diagnostics::TRACE_ID] + seen[:span] = ::Fiber[Dexpace::Instrumentation::Diagnostics::SPAN_ID] + seen[:current] = Dexpace::Instrumentation::Tracing.current_span + },) + + pipeline(step, transport).call(request, bundle: bundle_with(factory)) + + assert_equal("a" * 32, seen[:trace]) + assert_equal("b" * 16, seen[:span]) + assert_same(factory.tracers.first.spans.first, seen[:current], "a recording span was current") + assert_nil(::Fiber[Dexpace::Instrumentation::Diagnostics::TRACE_ID], "restored after") + assert_same(Dexpace::Instrumentation::NO_SPAN, Dexpace::Instrumentation::Tracing.current_span) + end + + test "cursor bundle: the meter has no bundle source -- the keyword resolves regardless" do + meter = Dexpace::RecordingMeter.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, meter: meter) + request = build_request + transport = FakeTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle_with(Dexpace::RecordingTracerFactory.new)) + + assert_equal(1, meter.counters.first.records.size) + end + end end diff --git a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb index 516e406..893be2b 100644 --- a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb +++ b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb @@ -413,4 +413,86 @@ def call(req, cur) assert_same(parent_state, fork_state) end end + + # Phase 6a's Task 8: the per-call instrumentation bundle, the one reader the widening added. + class BundleTest < DexpaceTestCase + include Cursors + + BUNDLE = Dexpace::Instrumentation::Bundle + W3C = Dexpace::Instrumentation::TraceIdFlavour::W3C + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, flavour: W3C) + end + + test "cursor bundle: #bundle defaults to Bundle::NONE on the root cursor" do + assert_same(BUNDLE::NONE, cursor_over([]).bundle) + end + + test "cursor bundle: a seeded bundle is readable at every position and survives #fork" do + bundle = seeded + seen = [] + probe = lambda do |req, cur| + seen << cur.bundle + fork = cur.fork + seen << fork.bundle + fork.call(req) + end + reader = lambda do |req, cur| + seen << cur.bundle + cur.call(req) + end + root = CURSOR.build( + drive: SYNC_DRIVER.new(DummySyncPipeline.new( + [entry(STAGES::RETRY, probe), + entry(STAGES::AUTH, reader),], @transport, + )), + request: @request, options: @options, cancellation: @cancellation, bundle: bundle, + ) + + root.call(@request) + + assert_same(bundle, root.bundle) + assert_equal(3, seen.size) + seen.each { |carried| assert_same(bundle, carried) } + end + + test "cursor bundle: a fork of a fork carries the same object; state: does not touch it" do + bundle = seeded + forks = [] + probe = lambda do |req, cur| + first = cur.fork(state: { cross_origin: true }) + forks << first + forks << cur.fork + first.call(req) + end + CURSOR.build( + drive: SYNC_DRIVER.new(DummySyncPipeline.new([entry(STAGES::REDIRECT, probe)], @transport)), + request: @request, options: @options, cancellation: @cancellation, bundle: bundle, + ).call(@request) + + forks.each { |fork| assert_same(bundle, fork.bundle) } + assert_equal({ cross_origin: true }, forks.first.state(STAGES::REDIRECT)) + end + + test "cursor bundle: .build refuses anything that is not a Bundle" do + drive = SYNC_DRIVER.new(DummySyncPipeline.new([], @transport)) + + assert_raises(Dexpace::InvalidArgumentError) do + CURSOR.build(drive: drive, request: @request, options: @options, + cancellation: @cancellation, bundle: nil,) + end + assert_raises(Dexpace::InvalidArgumentError) do + CURSOR.build(drive: drive, request: @request, options: @options, + cancellation: @cancellation, bundle: { trace_id: "x" },) + end + end + + test "cursor bundle: there is still no writer, and #bundle is the only new reader" do + writers = CURSOR.public_instance_methods(false).grep(/=\z|bundle./) + + assert_empty(writers) + refute_respond_to(cursor_over([]), :bundle=) + end + end end diff --git a/gems/dexpace-core/test/dexpace/pipeline_test.rb b/gems/dexpace-core/test/dexpace/pipeline_test.rb index 984b5de..a502144 100644 --- a/gems/dexpace-core/test/dexpace/pipeline_test.rb +++ b/gems/dexpace-core/test/dexpace/pipeline_test.rb @@ -348,4 +348,74 @@ class CompositionTest < DexpaceTestCase assert_same(base, nest.build.transport) end end + + # Phase 6a's Task 8: the `bundle:` keyword seeds Cursor#bundle for one call. + class BundleSeedingTest < DexpaceTestCase + BUNDLE = Dexpace::Instrumentation::Bundle + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C,) + end + + def request + Dexpace::Request.build(method: Dexpace::Method::GET, url: "https://example.test/", + headers: Dexpace::Headers::EMPTY,) + end + + test "cursor bundle: Pipeline#call seeds the cursor's bundle, NONE when omitted" do + seen = [] + step = lambda { |req, cur| + seen << cur.bundle + cur.call(req) + } + transport = ->(_req, _opts, _canc) { :response } + pipeline = Dexpace::Pipeline::Builder.new(transport: transport) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build + bundle = seeded + + pipeline.call(request) + pipeline.call(request, bundle: bundle) + pipeline.call(request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none, + bundle: bundle,) + + assert_same(BUNDLE::NONE, seen[0]) + assert_same(bundle, seen[1]) + assert_same(bundle, seen[2]) + end + + test "cursor bundle: the empty pipeline dispatches with no cursor and accepts the keyword" do + calls = [] + transport = lambda { |req, opts, canc| + calls << [req, opts, canc] + :response + } + pipeline = Dexpace::Pipeline.direct(transport) + + assert_equal(:response, pipeline.call(request, bundle: seeded)) + assert_equal(1, calls.size) + end + + test "cursor bundle / SEAM-11: a Pipeline is still a Transport by the duck type" do + pipeline = Dexpace::Pipeline.direct(->(_r, _o, _c) { :response }) + + assert(Dexpace::Transport.conforms?(pipeline)) + assert(Dexpace::Registry.callable?(pipeline, arity: 3)) + end + + test "cursor bundle: a non-Bundle is refused before any step runs" do + ran = false + step = lambda { |req, cur| + ran = true + cur.call(req) + } + pipeline = Dexpace::Pipeline::Builder.new(transport: ->(*) { :r }) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build + + assert_raises(Dexpace::InvalidArgumentError) { pipeline.call(request, bundle: :none) } + refute(ran) + end + end end From b09aa213d892f35ea583ba37d0f445ec36fa019c Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:15:24 +0300 Subject: [PATCH 10/13] test: the stage-based retry step suites, sync and async Add retry_step_test.rb and async_retry_step_test.rb, both driven through a REAL Pipeline or AsyncPipeline over the scripted transports and a RecordingCursor, so the fork-for-every-drive rule is asserted against the runtime that mints the cursor. The synchronous suite covers eligibility (the two axes, the re-sendability gate no predicate can override), the three terminal paths with the trail attached to the surfaced instance, the delay ladder in RETRY-39's order with the override hook's fall-through logged, the predicate's abort as RetryPredicateError, the budget, cancellation at the top of every attempt and inside the wait, the HTTP-tracer group in OBS-29's order with retries_exhausted only after a retryable failure, and the source guards (no Kernel#sleep, no total-timeout named on this stack). The asynchronous suite adds the trampoline: 2,000 inline-settling attempts complete without growing the stack, a positive delay without a Fiber.scheduler fails the future synchronously with the trail attached and a zero-length delay continues inline (R2), a positive delay under 5a's ParkingScheduler parks a fiber and resumes, a cancelled future aborts at the next boundary while an in-flight attempt is allowed to land, and a settled fatal answering #retryable? is delivered unclassified (P6-55). --- .../resilience/async_retry_step_test.rb | 482 ++++++++++++ .../dexpace/resilience/retry_step_test.rb | 686 ++++++++++++++++++ 2 files changed, 1168 insertions(+) create mode 100644 gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb create mode 100644 gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb diff --git a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb new file mode 100644 index 0000000..42cd533 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb @@ -0,0 +1,482 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_async_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" +require_relative "../../support/parking_scheduler" + +# Exercises: RETRY-2, RETRY-5, RETRY-7, RETRY-8, RETRY-13, RETRY-20, RETRY-23, RETRY-24, +# RETRY-25, RETRY-26, RETRY-30, RETRY-31, RETRY-32, RETRY-33, RETRY-34, RETRY-35, RETRY-39, +# RETRY-40, RETRY-41, RETRY-42, RETRY-44, RETRY-45, OBS-29, OBS-30, PIPE-29, PIPE-30, XCUT-3, +# CFG-18, P6-5, P6-7, R2 +# +# The asynchronous stage-based retry step, driven through a REAL Dexpace::AsyncPipeline over a +# ScriptedAsyncTransport whose futures settle INLINE -- the shape that overflowed the plan's +# recursive pump -- and, for R2's scheduler route, through 5a's ParkingScheduler, the one double +# that can drive a timed wait (its loop runs in #close, so every case joins the scheduler's +# thread). No test here sleeps a thread: a zero-length delay completes inline, a positive one +# either parks a fiber or fails the future. Split into nested classes under Metrics/ClassLength. +class DexpaceResilienceAsyncRetryStepTest < DexpaceTestCase + AsyncRetryStep = Dexpace::Resilience::AsyncRetryStep + Stages = Dexpace::Pipeline::Stages + + # The async pipeline shape every nested class drives. + module Fixtures + include RetryFixtures + + def pipeline(step, script, settle_later: false) + transport = ScriptedAsyncTransport.new(script, settle_later: settle_later) + wrapper = RecordingWrapper.new(step) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(wrapper).build_async + [built, transport, wrapper] + end + + def drive(step, script, request: retry_request, options: Dexpace::RequestOptions::EMPTY, + cancellation: Dexpace::Cancellation.none) + built, transport, wrapper = pipeline(step, script) + future = built.call(request, options, cancellation) + [future, transport, wrapper] + end + + # Zero-length waits, so no scheduler is needed and nothing sleeps (RETRY-31). + def inline_settings(**overrides) + retry_settings(initial_delay: 0.0, **overrides) + end + end + include Fixtures + + test "declares Stages::RETRY, is frozen, hides .new, and returns a Future" do + step = AsyncRetryStep.build + + assert_same(Stages::RETRY, step.stage) + assert_predicate(step, :frozen?) + refute_respond_to(AsyncRetryStep, :new) + future, = drive(step, [retry_response(200)]) + + assert_kind_of(Dexpace::Async::Future, future) + assert_equal(200, future.value.status.code) + end + + test "pipeline/86343352: forks for EVERY drive including the first, never calls its cursor" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + future, transport, wrapper = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, future.value.status.code) + assert_equal(3, transport.calls.size) + assert_equal(3, wrapper.cursors.first.forks) + assert_equal(0, wrapper.cursors.first.calls) + end + + test "RETRY-44: the SAME request object is re-sent on every attempt" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + request = retry_request + future, transport, = drive(step, [retry_response(503), retry_response(200)], request: request) + + assert_equal(200, future.value.status.code) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-42: eight threads through one frozen step keep their attempt counts apart" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + results = Array.new(8) do + ::Thread.new do + future, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + [future.value.status.code, transport.calls.size] + end + end.map(&:value) + + assert_equal([[200, 3]] * 8, results) + end + + # RETRY-30, RETRY-31, R2's three routes, RETRY-45. + class TrampolineTest < DexpaceTestCase + include Fixtures + + # The depth probe: a transport that records the call-stack depth at every attempt. A pump + # that recursed would show a depth growing with the attempt number; a flat one shows the + # same depth at attempt 1 and attempt N. + def depth_probe(depths, response) + lambda do |_request, _options, _cancellation| + depths << caller.size + response + end + end + + test "RETRY-30 / RETRY-31: 2000 zero-length retries build no N-deep stack, measured" do + depths = [] + failures = Array.new(2_000) { depth_probe(depths, retry_response(503)) } + script = failures + [depth_probe(depths, retry_response(200))] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2_000)) + + assert_nil(::Fiber.scheduler, "no scheduler anywhere in this test") + future, transport, = drive(step, script) + + assert_equal(200, future.value.status.code) + assert_equal(2_001, transport.calls.size) + assert_equal(2_001, depths.size) + assert_equal(depths.first, depths.last, "attempt 1 and attempt 2001 at the same depth") + assert_equal(1, depths.uniq.size, "every attempt at one depth: the pump is flat") + end + + test "R2 / RETRY-31: a zero-length delay completes inline and re-arms the pump, no scheduler" do + assert_nil(::Fiber.scheduler) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1)) + future, transport, = drive(step, [retry_response(503), retry_response(200)]) + + assert_predicate(future, :settled?, "settled before #call returned: no wait happened") + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + end + + test "R2 / CFG-18: a positive delay under a scheduler parks the fiber, blocks no thread" do + scheduler = ParkingScheduler.new + settings = retry_settings(initial_delay: 0.01, max_retries: 2) + step = AsyncRetryStep.build(settings: settings) + future = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, = drive(step, [retry_response(503), retry_response(503), retry_response(200)]) + end + end + thread.join + + assert_predicate(future, :settled?) + assert_equal(200, future.value.status.code) + assert_operator(scheduler.block_count, :>=, 2, "one park per positive wait") + assert_equal(0, scheduler.kernel_sleep_count) + end + + test "R2 / RETRY-33: a positive delay with NO scheduler fails the future with SeamError" do + assert_nil(::Fiber.scheduler) + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.01, max_retries: 2)) + first = RetryableError.new("first") + future, transport, = drive(step, [first, retry_response(200)]) + + assert_predicate(future, :settled?, "not a hang, not a blocking fallback") + error = assert_raises(Dexpace::SeamError) { future.value } + + assert_match(/Fiber\.set_scheduler/, error.message) + assert_equal([first], Dexpace.suppressed(error), "the trail travels with the SeamError") + assert_equal(1, transport.calls.size, "no second attempt was launched") + end + + test "RETRY-45: no scheduler is installed, read or shut down by the driver" do + scheduler = ParkingScheduler.new + closed = false + scheduler.define_singleton_method(:close) do + closed = true + super() + end + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.001, max_retries: 1)) + future = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule { future, = drive(step, [retry_response(503), retry_response(200)]) } + + refute(closed, "the driver did not close the scheduler while it was running") + end + thread.join + + assert_equal(200, future.value.status.code) + assert(closed, "the interpreter closed it at thread end, as CFG-18's own suite relies on") + end + end + + # RETRY-32, RETRY-33, RETRY-23: the async terminal paths. + class TerminalPathsTest < DexpaceTestCase + include Fixtures + + test "RETRY-32: a cancelled future launches no further attempt and closes the in-flight one" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + in_flight = retry_response(503) + transport = ScriptedAsyncTransport.new([in_flight, retry_response(503)], settle_later: true) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + future = built.call(retry_request) + + refute_predicate(future, :settled?) + future.cancel(:caller) + + assert_predicate(future, :cancelled?) + assert(transport.settle_next!, "the abandoned attempt now lands") + assert_predicate(in_flight.body, :closed?, "closed rather than leaked") + assert_equal(1, transport.calls.size, "no second attempt after the cancellation") + assert_nil(transport.settle_next!) + end + + test "RETRY-32: a deadline that cancels the future during a wait stops the loop" do + scheduler = ParkingScheduler.new + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.05, max_retries: 5)) + transport = nil + outcome = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, transport, = drive(step, Array.new(6) { retry_response(503) }) + future.cancel(:deadline) + outcome = future.cancelled? + end + end + thread.join + + assert(outcome) + assert_equal(1, transport.calls.size) + end + + test "RETRY-33: a throwing should_retry completes the future exceptionally, response closed" do + open_response = retry_response(503) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + should_retry: ->(*) { raise "boom" },) + future, = drive(step, [open_response]) + + assert_predicate(future, :settled?, "the one outcome RETRY-33 names is a hang; not this") + error = assert_raises(Dexpace::RetryPredicateError) { future.value } + + assert_equal("boom", error.cause.message) + assert_predicate(open_response.body, :closed?) + end + + test "RETRY-33: a throwing tracer callback completes the future exceptionally, no hang" do + exploding = ::Object.new + def exploding.attempt_started(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + future, = drive(step, [retry_response(200)]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + end + + test "RETRY-33: a throwing factory completes the future exceptionally" do + step = AsyncRetryStep.build(settings: inline_settings, + http_tracer_factory: ->(_cursor) { raise "no tracer" },) + future, transport, = drive(step, [retry_response(200)]) + + assert_raises(::RuntimeError) { future.value } + assert_equal(0, transport.calls.size) + end + + test "RETRY-33 / RETRY-35: a throwing delay computation closes the response and fails" do + open_response = retry_response(503) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + delay_override: ->(*) { raise ::NotImplementedError, "fatal" },) + transport = ScriptedAsyncTransport.new([open_response]) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + # The fatal family is re-raised AFTER the future is failed (RETRY-25 with RETRY-33), and + # the async driver's normalisation lets a ScriptError through, so it reaches this frame. + assert_raises(::NotImplementedError) { built.call(retry_request) } + assert_predicate(open_response.body, :closed?) + end + + test "RETRY-25: a fatal-family error from the downstream is surfaced unchanged, unretried" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + fatal = ::NoMemoryError.new("oom") + completer = Dexpace::Async::Completer.new + completer.fail(fatal) + transport = ->(_request, _options, _cancellation) { completer.future } + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + future = built.call(retry_request) + + surfaced = assert_raises(::NoMemoryError) { future.value } + + assert_same(fatal, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + end + + # RETRY-23 / RECOV-27 on the async path: the token during a wait and at the boundary. + class CancellationTest < DexpaceTestCase + include Fixtures + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + fatal = RetryableFatal.new("oom, and lying about it") + completer = Dexpace::Async::Completer.new + completer.fail(fatal) + sends = 0 + transport = lambda do |_request, _options, _cancellation| + sends += 1 + completer.future + end + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + surfaced = assert_raises(RetryableFatal) { built.call(retry_request).value } + + assert_same(fatal, surfaced) + assert_equal(1, sends) + end + + test "RETRY-23 / XCUT-3: a token cancelled during a wait cancels the timer, fails the future" do + scheduler = ParkingScheduler.new + source = Dexpace::Cancellation.source + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.05, max_retries: 5)) + first = RetryableError.new("first") + result = nil + transport = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, transport, = drive(step, [first, retry_response(200)], + cancellation: source.token,) + source.cancel(:token) + result = begin + future.value + rescue Dexpace::CancelledError => error + error + end + end + end + thread.join + + assert_kind_of(Dexpace::CancelledError, result) + assert_equal(:token, result.reason) + assert_equal([first], Dexpace.suppressed(result)) + assert_equal(1, transport.calls.size) + end + + test "RETRY-23: a cancelled token is checked at the top of every attempt" do + source = Dexpace::Cancellation.source + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + script = [lambda { |*| + source.cancel(:mid) + retry_response(503) + }, retry_response(200),] + future, transport, = drive(step, script, cancellation: source.token) + + error = assert_raises(Dexpace::CancelledError) { future.value } + + assert_equal(:mid, error.reason) + assert_equal(1, transport.calls.size) + end + end + + # The shared decision and delay logic, asserted on the async path too (design §11.12). + class SharedShapeTest < DexpaceTestCase + include Fixtures + + test "RETRY-5 / RETRY-7 / RETRY-8: the re-sendability gate, predicate or no predicate" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5), + should_retry: ->(*) { true },) + future, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, future.value.status.code) + assert_equal(1, transport.calls.size) + future, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST", body: replayable_body),) + + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + end + + test "RETRY-34: the terminal failure carries the whole prior trail, skipping itself" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + errors = Array.new(3) { |i| RetryableError.new("attempt #{i + 1}") } + future, = drive(step, errors) + + surfaced = assert_raises(RetryableError) { future.value } + + assert_same(errors[2], surfaced) + assert_equal(errors[0..1], Dexpace.suppressed(surfaced)) + same = RetryableError.new("same") + future, = drive(step, [same, same, same]) + + assert_same(same, assert_raises(RetryableError) { future.value }) + assert_empty(Dexpace.suppressed(same)) + end + + test "RETRY-34: a terminal error status is DELIVERED, never failed" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1)) + future, transport, = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, future.value.status.code) + assert_equal(2, transport.calls.size) + refute_predicate(future.value.body, :closed?) + end + + test "RETRY-2 / RETRY-24: the capability query decides the throwable branch" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + future, transport, = drive(step, [RetryableError.new("timeout"), retry_response(200)]) + + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + future, transport, = drive(step, [UnretryableError.new("no"), retry_response(200)]) + + assert_raises(UnretryableError) { future.value } + assert_equal(1, transport.calls.size) + end + + test "RETRY-20 / RETRY-39: the override sees the response on one path, the error on another" do + seen = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2, initial_delay: 0.0), + delay_override: lambda { |attempt, response, error| + seen << [attempt, response&.status&.code, error.class] + 0.0 + },) + future, = drive(step, [retry_response(503, headers: { "Retry-After" => "0" }), + RetryableError.new("x"), retry_response(200),],) + + assert_equal(200, future.value.status.code) + assert_equal([[1, 503, NilClass], [2, nil, RetryableError]], seen) + end + + test "RETRY-40: a throwing delay-override is logged and falls back on the async path too" do + sink = RecordingSink.new + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + delay_override: ->(*) { raise "override down" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + future, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(200, future.value.status.code) + assert_equal([:warn], sink.entries.map(&:severity)) + end + + test "RETRY-41: the per-call override wins, on the async path too" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 1, tags: {}) + future, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(503, future.value.status.code) + assert_equal(2, transport.calls.size) + end + + test "OBS-29 / P6-7: the per-attempt group, from one tracer per operation, with the cursor" do + tracer = Dexpace::RecordingHTTPTracer.new + cursors = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: lambda { |cursor| + cursors << cursor + tracer + },) + future, _transport, wrapper = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, future.value.status.code) + assert_equal([wrapper.cursors.first], cursors) + assert_equal(%i[attempt_started attempt_failed attempt_started retries_exhausted], + tracer.events.map(&:first),) + tracer.events.each { |event| assert_same(wrapper.cursors.first, event[1]) } + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2]) + assert_in_delta(0.0, tracer.events[1][3]) + end + + test "OBS-29: a failure that was never retryable ends with no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + future, = drive(step, [retry_response(404)]) + + assert_equal(404, future.value.status.code) + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb new file mode 100644 index 0000000..0dd953a --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb @@ -0,0 +1,686 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" +require_relative "../../support/state_probe" + +# Exercises: RETRY-2, RETRY-5, RETRY-6, RETRY-7, RETRY-8, RETRY-9, RETRY-10, RETRY-11, RETRY-12, +# RETRY-13, RETRY-15, RETRY-20, RETRY-21, RETRY-22, RETRY-23, RETRY-24, RETRY-25, RETRY-26, +# RETRY-28, RETRY-34, RETRY-35, RETRY-36, RETRY-39, RETRY-40, RETRY-41, RETRY-42, RETRY-44, +# RETRY-45, OBS-29, OBS-30, PIPE-15, PIPE-16, PIPE-40, XCUT-3, P6-5, P6-7, P6-11 +# +# The synchronous stage-based retry step, driven through a REAL Dexpace::Pipeline over a +# ScriptedTransport: Builder#append reads #stage, the driver mints the step's cursor, and the +# fork-for-every-drive rule is asserted through a recording cursor over that real one. Every +# wait is a FakeClock's -- a real Clock::SYSTEM wait in this suite is a finding -- and the one +# SYSTEM-clock test is the RETRY-23 case whose zero-length wait cannot observe the token. +# Split into nested classes under Metrics/ClassLength. +class DexpaceResilienceRetryStepTest < DexpaceTestCase + RetryStep = Dexpace::Resilience::RetryStep + Stages = Dexpace::Pipeline::Stages + + # The pipeline shape every nested class drives, and the counters it reads off. + module Fixtures + include RetryFixtures + + def pipeline(step, script, wrap: true, extra: []) + transport = ScriptedTransport.new(script) + installed = wrap ? RecordingWrapper.new(step) : step + builder = Dexpace::Pipeline::Builder.new(transport: transport).append(installed) + extra.each { |(probe, stage)| builder.append(probe, stage: stage) } + [builder.build, transport, installed] + end + + def drive(step, script, request: retry_request, options: Dexpace::RequestOptions::EMPTY, + cancellation: Dexpace::Cancellation.none) + built, transport, wrapper = pipeline(step, script) + response = built.call(request, options, cancellation) + [response, transport, wrapper] + end + end + include Fixtures + + test "declares Stages::RETRY, is frozen, hides .new, and is installed by its own stage" do + step = RetryStep.build + + assert_same(Stages::RETRY, step.stage) + assert_predicate(step, :frozen?) + refute_respond_to(RetryStep, :new) + assert(Dexpace::Pipeline::Step.conforms?(step)) + entries = Dexpace::Pipeline::Builder.new(transport: ScriptedTransport.new([])).append(step) + .entries + + assert_equal([Stages::RETRY], entries.map(&:stage)) + end + + test "the constructor validates every keyword and names it" do + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(settings: :defaults) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(http_tracer_factory: :none) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(delay_override: 1.0) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(should_retry: true) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(logger: RecordingSink.new) } + end + + test "pipeline/86343352 / PIPE-15 / PIPE-16: forks for EVERY drive, first included, no call" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + response, transport, wrapper = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + assert_equal(1, wrapper.cursors.size, "the step's own #call ran once for the operation") + assert_equal(3, wrapper.cursors.first.forks, "one fork per drive, attempt 1 included") + assert_equal(0, wrapper.cursors.first.calls, "the owning cursor's #call was never invoked") + refute_predicate(wrapper.cursors.first.real, :spent?) + end + + test "RETRY-44: every attempt re-sends the SAME request object through a fresh fork" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + request = retry_request + _response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: request,) + + assert_equal(2, transport.calls.size) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-36: a 503, 503, 200 sequence terminates on the 200 after three sends" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + refute_predicate(response.body, :closed?, "the response handed back is never closed") + end + + test "RETRY-42: eight threads through one frozen step keep their attempt counts apart" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + results = Array.new(8) do + ::Thread.new do + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + [response.status.code, transport.calls.size] + end + end.map(&:value) + + assert_equal([[200, 3]] * 8, results) + end + + test "boundary 2: a RETRY fork writes no cursor state under any stage's key" do + probe = StateProbe.new(stage_to_read: Stages::RETRY) + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + built, = pipeline(step, [retry_response(503), retry_response(200)], + extra: [[probe, Stages::AUTH]],) + + built.call(retry_request) + + assert_equal(2, probe.reads.size) + probe.reads.each { |read| assert_empty(read) } + end + + # RETRY-5..8, RETRY-2, RETRY-24, RETRY-25, RETRY-34: the two axes and the terminal paths. + class EligibilityTest < DexpaceTestCase + include Fixtures + + test "RETRY-5 / RETRY-7: a bare non-idempotent POST gets exactly one attempt, any status" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-5 / RETRY-8: a replayable body is re-sent; a consumed one is not, PUT or POST" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST", body: replayable_body),) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "PUT", body: consumed_body),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-8: a should_retry answering true cannot override the re-sendability gate" do + step = RetryStep.build(settings: retry_settings(max_retries: 5), should_retry: ->(*) { true }) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-2 / RETRY-24: a throwable answering the capability is retried; one without, not" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + response, transport, = drive(step, [RetryableError.new("read timeout"), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + error = assert_raises(UnretryableError) do + drive(step, [UnretryableError.new("no"), retry_response(200)]) + end + + assert_empty(Dexpace.suppressed(error)) + error = assert_raises(::ArgumentError) { drive(step, [::ArgumentError.new("bug")]) } + + assert_equal("bug", error.message) + end + + test "RETRY-37 / XCUT-7: the configured set is authoritative, wider and narrower" do + wider = RetryStep.build(settings: retry_settings(max_retries: 2, retryable_statuses: [418])) + response, transport, = drive(wider, [retry_response(418), retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + narrower = RetryStep.build(settings: retry_settings(max_retries: 2, + retryable_statuses: [429],)) + response, transport, = drive(narrower, [retry_response(503), retry_response(200)]) + + assert_equal(503, response.status.code, "503 is baked-retryable and still refused") + assert_equal(1, transport.calls.size) + end + end + + # RETRY-34 and RETRY-25: the terminal paths. + class TerminalPathTest < DexpaceTestCase + include Fixtures + + test "RETRY-34: the terminal raise is the LAST error, carrying the prior trail, never itself" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + errors = Array.new(3) { |i| RetryableError.new("attempt #{i + 1}") } + + surfaced = assert_raises(RetryableError) { drive(step, errors) } + + assert_same(errors[2], surfaced) + assert_equal(errors[0..1], Dexpace.suppressed(surfaced)) + refute_includes(Dexpace.suppressed(surfaced), surfaced) + assert_nil(surfaced.cause, "pipeline/7ce4431d: the carried re-raise assigns no cause") + end + + test "RETRY-34 / pipeline/7ce4431d: the carried re-raise adds nothing to a cause already set" do + # $! is non-nil inside anything called from a CALLER's rescue block. On this stack every + # surfaced throwable was RAISED by the downstream inside that same extent, so Ruby assigned + # the caller's exception as its cause at the transport's own raise, and a cause once set + # survives every later raise (verified on 4.0.6) -- which is why this test can assert only + # that the step's `raise failure, cause: nil` leaves that cause exactly as it found it. The + # DISCRIMINATING fixture, an error constructed and never raised, is RecoveryRetry's + # ProtocolError; its suite is where a bare `raise error` runs red. + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + errors = [RetryableError.new("one"), RetryableError.new("two")] + surfaced = nil + in_flight = nil + begin + raise ::IOError, "the caller's own in-flight exception" + rescue ::IOError => error + in_flight = error + surfaced = assert_raises(RetryableError) { drive(step, errors) } + end + + assert_same(errors[1], surfaced) + assert_same(in_flight, surfaced.cause, "set by the transport's raise, untouched since") + assert_equal([errors[0]], Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: a reused error instance trips no self-suppression -- the skip-self guard" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + same = RetryableError.new("same instance every time") + + surfaced = assert_raises(RetryableError) { drive(step, [same, same, same]) } + + assert_same(same, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: on eventual success the prior trail is discarded" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + first = RetryableError.new("one") + response, = drive(step, [first, retry_response(200)]) + + assert_equal(200, response.status.code) + assert_empty(Dexpace.suppressed(first), "nothing was ever attached to a non-terminal error") + end + + test "RETRY-34: a terminal error status is RETURNED, never raised, its trail discarded" do + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + response, transport, = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, response.status.code) + assert_equal(2, transport.calls.size) + refute_predicate(response.body, :closed?) + end + + test "RETRY-25: a fatal-family error propagates unchanged, unretried, unattached, unreported" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 3), + http_tracer_factory: ->(_cursor) { tracer },) + fatal = ::NoMemoryError.new("oom") + + surfaced = assert_raises(::NoMemoryError) { drive(step, [fatal, retry_response(200)]) } + + assert_same(fatal, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + assert_equal(%i[attempt_started], tracer.events.map(&:first), "no attempt_failed for it") + end + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + fatal = RetryableFatal.new("oom, and lying about it") + built, transport, = pipeline(step, [fatal, retry_response(200)]) + + surfaced = assert_raises(RetryableFatal) { built.call(retry_request) } + + assert_same(fatal, surfaced) + assert_equal(1, transport.calls.size, "never a second send") + end + + test "RETRY-25: a fatal-family error from inside the retry decision is not classified either" do + step = RetryStep.build(settings: retry_settings(max_retries: 3), + should_retry: ->(*) { raise ::NoMemoryError, "oom in predicate" },) + open_response = retry_response(503) + + assert_raises(::NoMemoryError) { drive(step, [open_response]) } + assert_predicate(open_response.body, :closed?, "RETRY-35: closed before propagating") + end + end + + # RETRY-9..12, RETRY-15, RETRY-20..22, RETRY-35, RETRY-39, RETRY-40: delays and the close. + class DelayTest < DexpaceTestCase + include Fixtures + + test "RETRY-9 / RETRY-12 / RETRY-13: the schedule is Policy's, on the settings' clock" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 3) + step = RetryStep.build(settings: settings) + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(503), retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(4, transport.calls.size) + assert_equal([0.1, 0.2, 0.4], clock.sleeps.map { _1[:duration] }) + expected = (1..3).map do |attempt| + Dexpace::Resilience::Policy.backoff_delay(attempt, **settings.backoff_arguments) + end + + assert_equal(expected, clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-26 / XCUT-3: every wait is the clock's cancellable sleep with the cursor's token" do + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + source = Dexpace::Cancellation.source + drive(step, [retry_response(503), retry_response(200)], cancellation: source.token) + + assert_equal(1, clock.sleeps.size) + assert_same(source.token, clock.sleeps.first[:cancellation]) + end + + test "RETRY-35: the response is released BEFORE the wait; the delay is read from it first" do + closed_at_sleep = [] + hinted = retry_response(503, headers: { "Retry-After" => "3" }) + clock = FakeClock.new + clock.define_singleton_method(:sleep) do |duration, cancellation: nil| + closed_at_sleep << hinted.body.closed? + super(duration, cancellation: cancellation) + end + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + response, = drive(step, [hinted, retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal([true], closed_at_sleep, "closed before the wait") + assert_equal([3.0], clock.sleeps.map { _1[:duration] }, "the hint was read while open") + end + + test "RETRY-20 / RETRY-22: a pacing hint REPLACES the schedule verbatim; a bad one falls" do + clock = FakeClock.new + settings = retry_settings(clock: clock, initial_delay: 30.0, max_delay: 30.0, jitter: 0.5, + max_retries: 2,) + step = RetryStep.build(settings: settings) + drive(step, [retry_response(503, headers: { "Retry-After" => "2" }), + retry_response(429, headers: { "Retry-After" => "soon" }), + retry_response(200),],) + first, second = clock.sleeps.map { _1[:duration] } + + assert_in_delta(2.0, first, 0.001, "verbatim: not 30.0 and not jittered into [1.5, 2.5]") + assert_operator(second, :>=, 30.0 * 0.75) + assert_operator(second, :<=, 30.0 * 1.25) + end + + test "RETRY-21: the stage stack walks the caller's header order" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 1, + pacing_header_order: %w[retry-after-ms Retry-After],) + step = RetryStep.build(settings: settings) + both = { "Retry-After" => "9", "retry-after-ms" => "500" } + drive(step, [retry_response(503, headers: both), retry_response(200)]) + + assert_equal([0.5], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-15: an HTTP-date hint is measured from the settings clock's own now" do + clock = FakeClock.new(now: ::Time.utc(2026, 9, 18, 12, 0, 0)) + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + hinted = retry_response(503, headers: { "Retry-After" => "Fri, 18 Sep 2026 12:00:45 GMT" }) + drive(step, [hinted, retry_response(200)]) + + assert_equal([45.0], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-39: the exception path skips the header step and reaches backoff" do + clock = FakeClock.new + seen = [] + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: lambda { |attempt, response, error| + seen << [attempt, response, error] + nil + },) + drive(step, [RetryableError.new("boom"), retry_response(200)]) + + assert_equal(1, seen.size) + assert_equal(1, seen.first[0]) + assert_nil(seen.first[1], "no response on the exception path") + assert_kind_of(RetryableError, seen.first[2]) + assert_equal([0.1], clock.sleeps.map { _1[:duration] }) + end + end + + # RETRY-39's first tier and RETRY-40's non-fatal half. + class DelayOverrideTest < DexpaceTestCase + include Fixtures + + test "RETRY-39: a present delay-override wins over a present pacing hint" do + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: ->(_attempt, _response, _error) { 7 },) + drive(step, [retry_response(503, headers: { "Retry-After" => "2" }), retry_response(200)]) + + assert_equal([7.0], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-40: a throwing delay-override is non-fatal -- logged, and backoff applies" do + clock = FakeClock.new + sink = RecordingSink.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: ->(*) { raise "override down" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + response, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal([0.1], clock.sleeps.map { _1[:duration] }) + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_HOOK, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/override down/, payload[Dexpace::Instrumentation::Keys::CAUSE]) + end + + test "RETRY-40: a delay-override answering a negative or non-numeric value is logged too" do + clock = FakeClock.new + sink = RecordingSink.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 2), + delay_override: ->(attempt, *) { attempt == 1 ? -1 : "soon" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + drive(step, [retry_response(503), retry_response(503), retry_response(200)]) + + assert_equal([0.1, 0.2], clock.sleeps.map { _1[:duration] }) + assert_equal(2, sink.entries.size) + end + + test "RETRY-35 / RETRY-40: a throwing delay computation still closes the response first" do + open_response = retry_response(503) + step = RetryStep.build(settings: retry_settings(max_retries: 1), + delay_override: ->(*) { raise ::NotImplementedError, "fatal-shaped" },) + + assert_raises(::NotImplementedError) { drive(step, [open_response]) } + assert_predicate(open_response.body, :closed?) + end + end + + # RETRY-40's predicate, RETRY-41's count, RETRY-23's cancellation. + class PredicateAndBudgetTest < DexpaceTestCase + include Fixtures + + test "RETRY-40: a throwing should_retry aborts as RetryPredicateError, the raise its cause" do + open_response = retry_response(503) + step = RetryStep.build(settings: retry_settings(max_retries: 1), + should_retry: ->(*) { raise "boom" },) + + error = assert_raises(Dexpace::RetryPredicateError) { drive(step, [open_response]) } + + assert_kind_of(::RuntimeError, error.cause) + assert_equal("boom", error.cause.message) + assert_kind_of(Dexpace::Error, error) + assert_predicate(open_response.body, :closed?, "RETRY-35's third ordering") + end + + test "RETRY-40: should_retry receives the failure and the request and decides the condition" do + seen = [] + step = RetryStep.build(settings: retry_settings(max_retries: 5), + should_retry: lambda { |failure, request| + seen << [failure, request] + failure.is_a?(Dexpace::ProtocolError) && failure.status.code == 404 + },) + request = retry_request + response, transport, = drive(step, [retry_response(404), retry_response(503)], + request: request,) + + assert_equal(503, response.status.code, "404 retried by the predicate, 503 refused by it") + assert_equal(2, transport.calls.size) + assert_equal(2, seen.size) + assert_same(request, seen.first[1]) + assert_equal(404, seen.first[0].status.code) + end + + test "RETRY-41: the per-call max_retries override wins over the configured value" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 1, tags: {}) + response, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(503, response.status.code) + assert_equal(2, transport.calls.size, "one retry: the override") + + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 0, tags: {}) + _response, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(1, transport.calls.size, "zero means no retries") + end + + test "RETRY-41: with no override the configured value applies, and zero disables retries" do + step = RetryStep.build(settings: retry_settings(max_retries: 0)) + _response, transport, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(1, transport.calls.size) + end + + test "RETRY-23 / RECOV-27: cancellation during the wait aborts the loop as CancelledError" do + source = Dexpace::Cancellation.source + step = RetryStep.build(settings: retry_settings(max_retries: 5, initial_delay: 5.0)) + script = [lambda { |*| + source.cancel(:test) + retry_response(503) + }, retry_response(200),] + + error = assert_raises(Dexpace::CancelledError) do + drive(step, script, cancellation: source.token) + end + + assert_equal(:test, error.reason) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error), "never a retryable condition") + end + + test "RETRY-23: a cancelled token is checked at the top of every attempt, wait or no wait" do + # Under Clock::SYSTEM a zero-length sleep returns before the token check (5a, verified), so + # this attempt-boundary check is the only thing standing between the cancellation and a + # second send. The one SYSTEM-clock wait in this suite is this zero-length one. + source = Dexpace::Cancellation.source + settings = Dexpace::Resilience::RetrySettings.build(max_retries: 5, initial_delay: 0.0, + jitter: 0.0,) + step = RetryStep.build(settings: settings) + script = [lambda { |*| + source.cancel(:mid_attempt) + retry_response(503) + }, retry_response(200),] + + error = assert_raises(Dexpace::CancelledError) do + drive(step, script, cancellation: source.token) + end + + assert_equal(:mid_attempt, error.reason) + end + + test "RETRY-23: a downstream cancellation surfaces as terminal, is never retried" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + error = assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + + assert_equal(:downstream, error.reason) + end + + test "RETRY-23: no attempt is launched after a cancellation observed mid-loop" do + source = Dexpace::Cancellation.source + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + script = [lambda { |*| + source.cancel(:stop) + retry_response(503) + }, retry_response(200),] + + assert_raises(Dexpace::CancelledError) { drive(step, script, cancellation: source.token) } + # the script's second entry was never consumed + _built, transport, = pipeline(step, script) + + assert_equal(0, transport.calls.size) + end + end + + # OBS-29's per-attempt group, and the three text-scan guards (RETRY-13, RETRY-28, RETRY-45). + class TracerAndGuardsTest < DexpaceTestCase + include Fixtures + + RESILIENCE = File.expand_path("../../../lib/dexpace/resilience", __dir__) + + def source(name) = File.read(File.join(RESILIENCE, "#{name}.rb")).gsub(/^\s*#.*$/, "") + + test "OBS-29 / P6-7: the factory is called once per operation, with the cursor as context" do + tracers = [] + cursors = [] + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: lambda { |cursor| + cursors << cursor + tracers << Dexpace::RecordingHTTPTracer.new + tracers.last + },) + _response, _transport, wrapper = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(1, tracers.size) + assert_equal([wrapper.cursors.first], cursors, "the step's own cursor, not a fork") + tracers.first.events.each { |event| assert_same(wrapper.cursors.first, event[1]) } + end + + test "OBS-29: attempt_started, attempt_failed with the delay, attempt_started, then success" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(503), retry_response(200)]) + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started], names) + assert_equal(1, tracer.events[0][2]) + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2], "an Exception, never a Response") + assert_equal(503, tracer.events[1][2].status.code) + assert_in_delta(0.1, tracer.events[1][3]) + assert_equal(2, tracer.events[2][2]) + end + + test "OBS-29: retries_exhausted fires once, after the last attempt, with the surfaced error" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { tracer },) + errors = [RetryableError.new("one"), RetryableError.new("two")] + + surfaced = assert_raises(RetryableError) { drive(step, errors) } + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started retries_exhausted], names) + assert_same(surfaced, tracer.events.last[2]) + assert_same(errors[0], tracer.events[1][2]) + end + + test "OBS-29: an exhausted error STATUS reports a ProtocolError carrying the trail" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(503), retry_response(503)]) + exhausted = tracer.events.last + + assert_equal(:retries_exhausted, exhausted.first) + assert_equal(503, exhausted[2].status.code) + assert_equal([tracer.events[1][2]], Dexpace.suppressed(exhausted[2])) + end + + test "OBS-29: a failure that was never retryable ends with no retries_exhausted at all" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(404)]) + + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + assert_raises(::ArgumentError) { drive(step, [::ArgumentError.new("bug")]) } + assert_equal(%i[attempt_started attempt_started], tracer.events.map(&:first)) + end + + test "OBS-30: a throwing tracer propagates and fails the request; core wraps no callback" do + exploding = ::Object.new + def exploding.attempt_started(*) = raise("tracer blew up") + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [retry_response(200)]) } + + assert_equal("tracer blew up", error.message) + end + + test "RETRY-28 / P6-5: neither stage driver names the budget, by text" do + %w[retry_step async_retry_step retry_step_helpers].each do |name| + code = source(name) + + refute_match(/budget_remaining|total_timeout/, code, name) + end + assert_match(/budget_remaining/, source("recovery_retry"), "the recovery driver alone") + end + + test "RETRY-13: no delay literal outside Policy, in any resilience file" do + %w[retry_step async_retry_step retry_step_helpers recovery_retry resend retry_settings] + .each do |name| + code = source(name) + + refute_match(/\b(0\.2|2\.0|8\.0)\b/, code, "#{name} re-types a Policy default") + refute_match(/DEFAULT_(INITIAL_DELAY|MULTIPLIER|MAX_DELAY|JITTER)\s*=/, code, name) + end + end + + test "RETRY-45: the engine never shuts down or installs a scheduler, by text" do + Dir[File.join(RESILIENCE, "*.rb")].each do |path| + code = File.read(path).gsub(/^\s*#.*$/, "") + + touches = code.scan(/set_scheduler|scheduler&?\.close|shutdown/) + + assert_empty(touches, "#{File.basename(path)} touches a scheduler") + end + end + end +end From 03b42a37a3f96b57dcfbee6984fa71631e7392b9 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 15:15:36 +0300 Subject: [PATCH 11/13] test: the recovery-chain retry suite and the budget equivalence Add recovery_retry_test.rb: the classification of each attempt through 4b's own buffer-and-classify primitives with the buffered response what every later line sees, the two terminal shapes (a returned response when the failure was never retryable, a raised throwable otherwise, with the trail attached to the constructed ProtocolError so RETRY-34's cause discrimination holds on this stack), the attempts cap counting the first send as attempt 1, the total-timeout applied as time remaining so no wait can overshoot and a hint past the budget suppresses the wait, the cancellation token at every boundary and inside the wait, the HTTP-tracer group produced once per operation with the Request, and the engine installed as Recovery::Orchestrator's transport: with the request chain's stamping applied once per exchange rather than once per attempt. Add budget_equivalence_test.rb, the sub-phase's convergence point: the three drivers built from ONE RetrySettings and driven against an identical failure sequence exhaust after the same number of wire sends, three under the shared defaults -- max-retries plus one is max-attempts, an identity and not a coincidence (P6-6). --- .../resilience/budget_equivalence_test.rb | 123 +++++ .../dexpace/resilience/recovery_retry_test.rb | 499 ++++++++++++++++++ 2 files changed, 622 insertions(+) create mode 100644 gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb create mode 100644 gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb diff --git a/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb b/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb new file mode 100644 index 0000000..49838bf --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/scripted_async_transport" +require_relative "../../support/fake_config_source" + +# Exercises: RETRY-14, RETRY-13, RECOV-30, RETRY-12, P6-6 +# +# The convergence point inside 6a (the charter's second): the three drivers -- the recovery +# engine and the two stage steps -- built from ONE RetrySettings, driven against an identical +# failure sequence, exhaust after the SAME number of wire sends. Under the shared defaults that +# number is three: the stage stack's max-retries (2) plus one initial send, which is the +# recovery stack's max-attempts (3), an identity and not a coincidence (P6-6). No lib/ mirror: +# this file asserts an equivalence between three files rather than one file's contract. +class DexpaceResilienceBudgetEquivalenceTest < DexpaceTestCase + include RetryFixtures + + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + + def recovery_sends(settings, failures) + transport = ScriptedTransport.new(failures) + engine = Dexpace::Resilience::RecoveryRetry.build(transport: transport, settings: settings) + assert_raises(Dexpace::ProtocolError) do + engine.call(retry_request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + transport.calls.size + end + + def stage_sends(settings, failures) + transport = ScriptedTransport.new(failures) + step = Dexpace::Resilience::RetryStep.build(settings: settings) + response = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build + .call(retry_request) + + assert_equal(503, response.status.code) + transport.calls.size + end + + def async_stage_sends(settings, failures) + transport = ScriptedAsyncTransport.new(failures) + step = Dexpace::Resilience::AsyncRetryStep.build(settings: settings) + future = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + .call(retry_request) + + assert_equal(503, future.value.status.code) + transport.calls.size + end + + def unavailable(count) = Array.new(count) { retry_response(503) } + + test "RETRY-14: under the shared defaults all three drivers exhaust after exactly three sends" do + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + clock: FakeClock.new,) + + assert_equal(2, settings.max_retries, "RETRY-12's default, the stage vocabulary") + recovery = recovery_sends(settings, unavailable(6)) + stage = stage_sends(settings, unavailable(6)) + async = async_stage_sends(settings, unavailable(6)) + + assert_equal(3, recovery, "max-attempts 3: max_retries + the initial send") + assert_equal(3, stage, "max-retries 2 + the initial send") + assert_equal(3, async) + assert_equal(recovery, stage) + assert_equal(stage, async) + end + + test "RETRY-14 / P6-6: the equivalence holds for any configured value, as an identity" do + [0, 1, 4].each do |retries| + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + max_retries: retries, + clock: FakeClock.new,) + sends = retries + 1 + + assert_equal(sends, recovery_sends(settings, unavailable(sends + 2)), "recovery, #{retries}") + assert_equal(sends, stage_sends(settings, unavailable(sends + 2)), "stage, #{retries}") + assert_equal(sends, async_stage_sends(settings, unavailable(sends + 2)), "async, #{retries}") + end + end + + test "RETRY-14 / RETRY-12: the configured MAX_RETRY_ATTEMPTS reaches all three stacks alike" do + Dexpace.configure do |c| + c.env_source = FakeConfigSource.new(Dexpace::Configuration::Keys::MAX_RETRY_ATTEMPTS => "3") + end + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + clock: FakeClock.new,) + + assert_equal(3, settings.max_retries) + assert_equal(4, recovery_sends(settings, unavailable(8))) + assert_equal(4, stage_sends(settings, unavailable(8))) + assert_equal(4, async_stage_sends(settings, unavailable(8))) + end + + test "RETRY-13 / RECOV-30: the two clock-driven stacks wait on ONE schedule from one settings" do + # The async driver waits through Async.delay and never the clock: with no scheduler a + # positive delay fails its future (R2), so its row in this file is the send count above and + # its schedule is the same RetryStepHelpers#resolve_delay the sync step's suite pins. + clocks = Array.new(2) { FakeClock.new } + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.3, multiplier: 3.0, + max_delay: 5.0, jitter: 0.0, + max_retries: 3, clock: clocks[0],) + recovery_sends(settings, unavailable(6)) + stage_sends(settings.with(clock: clocks[1]), unavailable(6)) + schedules = clocks.map { |clock| clock.sleeps.map { _1[:duration] } } + + assert_equal(3, schedules[0].size) + [0.3, 0.9, 2.7].zip(schedules[0]).each do |(expected, actual)| + assert_in_delta(expected, actual) + end + assert_equal(schedules[0], schedules[1], "one calculator, two stacks, one schedule") + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb new file mode 100644 index 0000000..e4a2b4e --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb @@ -0,0 +1,499 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" + +# Exercises: RECOV-16, RECOV-17, RECOV-18, RECOV-19, RECOV-20, RECOV-21, RECOV-22, RECOV-23, +# RECOV-24, RECOV-26, RECOV-27, RECOV-28, RECOV-29, RECOV-30, RECOV-34, RETRY-4, RETRY-13, +# RETRY-14, RETRY-23, RETRY-25, RETRY-27, RETRY-34, RETRY-35, RETRY-36, RETRY-37, RETRY-41, +# RETRY-42, RETRY-44, OBS-29, CFG-16, SEAM-11, P6-3, P6-6, P6-8, P6-9 +# +# The recovery-stack engine, driven directly as the Dexpace::Transport it is and, where the +# integration is the point, beneath a real Recovery::Orchestrator with a real +# ErrorMappingStep. Every wait is a FakeClock's, whose monotonic reading advances by exactly the +# slept duration, so the total-timeout arithmetic is exact rather than wall-clock-dependent. +# Split into nested classes under Metrics/ClassLength. +class DexpaceResilienceRecoveryRetryTest < DexpaceTestCase + RecoveryRetry = Dexpace::Resilience::RecoveryRetry + + # The engine over a script, and the one call every test makes. + module Fixtures + include RetryFixtures + + def engine(script, settings: retry_settings, **keywords) + transport = ScriptedTransport.new(script) + [RecoveryRetry.build(transport: transport, settings: settings, **keywords), transport] + end + + def send_through(built, request: retry_request, cancellation: Dexpace::Cancellation.none) + built.call(request, Dexpace::RequestOptions::EMPTY, cancellation) + end + end + include Fixtures + + test "SEAM-11 / P6-3: the engine is a Dexpace::Transport by the duck type, frozen, .new hidden" do + built, = engine([retry_response(200)]) + + assert(Dexpace::Transport.conforms?(built)) + assert(Dexpace::Registry.callable?(built, arity: 3)) + assert_predicate(built, :frozen?) + refute_respond_to(RecoveryRetry, :new) + end + + test "the constructor validates every keyword" do + assert_raises(Dexpace::InvalidArgumentError) { RecoveryRetry.build(transport: :none) } + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ->(_r, _o) {}) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), settings: :defaults) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), http_tracer_factory: :none) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), logger: RecordingSink.new) + end + end + + test "RECOV-28: a success on the first attempt touches no clock and returns the response as is" do + clock = FakeClock.new + built, transport = engine([retry_response(200)], settings: retry_settings(clock: clock)) + response = send_through(built) + + assert_equal(200, response.status.code) + assert_empty(clock.sleeps) + assert_equal(1, transport.calls.size) + refute_predicate(response.body, :closed?) + end + + test "RECOV-19 / RETRY-36 / RETRY-44: 503, 503, 200 ends on the 200, re-sending ONE request" do + built, transport = engine([retry_response(503), retry_response(503), retry_response(200)]) + request = retry_request + response = send_through(built, request: request) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-42 / RECOV-28: eight concurrent calls through one engine keep their budgets apart" do + built, transport = engine(Array.new(16) { |i| retry_response(i.even? ? 503 : 200) }, + settings: retry_settings(max_retries: 1),) + results = Array.new(8) { ::Thread.new { send_through(built).status.code } }.map(&:value) + + assert_equal(16, transport.calls.size) + assert_equal(8, results.count(200), results.inspect) + end + + # RECOV-17, RECOV-18, RETRY-4, RETRY-25, RETRY-37: the two axes, on this stack. + class ClassificationTest < DexpaceTestCase + include Fixtures + + test "RECOV-17 / RETRY-4 / P6-8: a THROWN retryable failure is retried, not propagated" do + built, transport = engine([RetryableError.new("reset"), RetryableError.new("reset"), + retry_response(200),]) + response = send_through(built) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size, "three sends, not one raise") + end + + test "RECOV-17: a thrown failure with no capability is surfaced on the first attempt" do + built, transport = engine([::ArgumentError.new("bug"), retry_response(200)]) + + error = assert_raises(::ArgumentError) { send_through(built) } + + assert_equal("bug", error.message) + assert_empty(Dexpace.suppressed(error)) + assert_equal(1, transport.calls.size) + end + + test "RECOV-17 / RETRY-37: the configured set is authoritative -- wider, and narrower" do + built, transport = engine([retry_response(418), retry_response(200)], + settings: retry_settings(retryable_statuses: [418]),) + + assert_equal(200, send_through(built).status.code) + assert_equal(2, transport.calls.size, "418 is outside the baked set and still retried") + built, transport = engine([retry_response(503), retry_response(200)], + settings: retry_settings(retryable_statuses: [429]),) + + assert_equal(503, send_through(built).status.code, "503 is baked-retryable and refused") + assert_equal(1, transport.calls.size) + end + + test "RECOV-19: a NON-retryable error status passes through as Success -- returned, buffered" do + built, transport = engine([retry_response(404)]) + response = send_through(built) + + assert_equal(404, response.status.code) + assert_equal(1, transport.calls.size) + assert_kind_of(Dexpace::BufferBody, response.body, "buffered per RECOV-16") + assert_equal("body 404", response.body_string) + end + + test "RECOV-18 / RETRY-5 / RETRY-7: a bare POST is sent exactly once and its 503 returned" do + built, transport = engine([retry_response(503), retry_response(200)], + settings: retry_settings(max_retries: 5),) + response = send_through(built, request: retry_request(method: "POST")) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RECOV-18: a replayable body is re-sent and a consumed body is not" do + built, transport = engine([retry_response(503), retry_response(200)]) + response = send_through(built, request: retry_request(method: "POST", body: replayable_body)) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + built, transport = engine([RetryableError.new("reset"), retry_response(200)]) + + assert_raises(RetryableError) do + send_through(built, request: retry_request(method: "PUT", body: consumed_body)) + end + assert_equal(1, transport.calls.size) + end + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + built, transport = engine([RetryableFatal.new("oom, lying"), retry_response(200)], + settings: retry_settings(max_retries: 3),) + + assert_raises(RetryableFatal) { send_through(built) } + assert_equal(1, transport.calls.size) + end + + test "RETRY-25: a fatal-family error propagates at the throw site, unclassified, unattached" do + tracer = Dexpace::RecordingHTTPTracer.new + built, transport = engine([::NoMemoryError.new("oom"), retry_response(200)], + http_tracer_factory: ->(_request) { tracer },) + + error = assert_raises(::NoMemoryError) { send_through(built) } + + assert_empty(Dexpace.suppressed(error)) + assert_equal(1, transport.calls.size) + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27, RETRY-34, P6-9: the budget and the terminal paths. + class BudgetTest < DexpaceTestCase + include Fixtures + + test "RECOV-20 / RETRY-34 / P6-9: exhausting the attempt cap RAISES, with the whole trail" do + built, transport = engine(Array.new(3) { retry_response(503) }, + settings: retry_settings(max_retries: 2),) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(3, transport.calls.size, "max_retries 2 + 1 initial send (RETRY-14, P6-6)") + assert_equal(503, error.status.code) + assert_equal(2, Dexpace.suppressed(error).size) + assert_equal([503, 503], Dexpace.suppressed(error).map { |prior| prior.status.code }) + assert_nil(error.cause, "pipeline/7ce4431d: the carried raise assigns no cause") + end + + test "RETRY-34 / pipeline/7ce4431d: the carried raise assigns no cause in a caller's rescue" do + built, = engine(Array.new(2) do + retry_response(503) + end, settings: retry_settings(max_retries: 1),) + surfaced = nil + begin + raise ::IOError, "the caller's own in-flight exception" + rescue ::IOError + surfaced = assert_raises(Dexpace::ProtocolError) { send_through(built) } + end + + assert_nil(surfaced.cause) + assert_equal(1, Dexpace.suppressed(surfaced).size) + end + + test "RETRY-34: a reused error instance trips no self-suppression on this stack either" do + same = RetryableError.new("same") + built, = engine([same, same, same], settings: retry_settings(max_retries: 2)) + + surfaced = assert_raises(RetryableError) { send_through(built) } + + assert_same(same, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: on eventual success the trail is discarded" do + first = RetryableError.new("one") + built, = engine([first, retry_response(200)]) + + assert_equal(200, send_through(built).status.code) + assert_empty(Dexpace.suppressed(first)) + end + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27: the total-timeout budget on a fake clock. + class TotalTimeoutTest < DexpaceTestCase + include Fixtures + + test "RECOV-20: a positive total_timeout aborts on `remaining`, at the attempt it names" do + # remaining = total_timeout - elapsed, on a clock that advances by exactly each sleep: + # 1.0 -> sleep 0.4 -> 0.6 -> sleep 0.4 -> 0.2, and 0.4 > 0.2 suppresses the third delay. + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 1.0, initial_delay: 0.4, + max_delay: 0.4, max_retries: 10,) + built, transport = engine(Array.new(11) { retry_response(503) }, settings: settings) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(3, transport.calls.size, "exact, not merely under 11") + assert_equal([0.4, 0.4], clock.sleeps.map { _1[:duration] }, "never over budget") + assert_equal(2, Dexpace.suppressed(error).size) + end + + test "RECOV-20: elapsed at or past the budget suppresses the next delay, delay or no delay" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 1.0, initial_delay: 0.0, + max_retries: 10,) + slow = lambda { |*| + clock.advance(0.5) + retry_response(503) + } + built, transport = engine([slow, slow, slow], settings: settings) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(2, transport.calls.size, "1.0 elapsed after the second send: remaining 0.0") + end + + test "RECOV-20 / RETRY-27: a zero total_timeout is unbounded -- only the cap stops the loop" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 0, initial_delay: 100.0, + max_delay: 100.0, max_retries: 4,) + built, transport = engine(Array.new(5) { retry_response(503) }, settings: settings) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(5, transport.calls.size) + assert_equal([100.0] * 4, clock.sleeps.map { _1[:duration] }) + end + + test "RECOV-22 / RETRY-20: a pacing hint replaces the schedule and is still budget-clamped" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 5.0, initial_delay: 30.0, + max_delay: 30.0, max_retries: 5,) + built, transport = engine([retry_response(503, headers: { "Retry-After" => "2" }), + retry_response(503, headers: { "Retry-After" => "9" }), + retry_response(200),], settings: settings,) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal([2.0], clock.sleeps.map { _1[:duration] }, "2 honoured; 9 > 3 remaining") + assert_equal(2, transport.calls.size) + end + + test "RECOV-24 / RECOV-29: the fixed precedence, and a malformed hint falls back to backoff" do + clock = FakeClock.new + built, = engine([retry_response(503, headers: { "Retry-After" => "garbage", + "retry-after-ms" => "1500", },), + retry_response(503, headers: { "Retry-After" => "junk" }), + retry_response(200),], settings: retry_settings(clock: clock),) + + send_through(built) + + durations = clock.sleeps.map { |sleep| sleep[:duration] } + + assert_equal([1.5, 0.2], durations, "the hint, then attempt 2's backoff") + end + + test "RECOV-21 / RETRY-13: the schedule is Policy's own, on the settings clock" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 3) + built, = engine(Array.new(3) { retry_response(503) } << retry_response(200), + settings: settings,) + + send_through(built) + expected = (1..3).map do |attempt| + Dexpace::Resilience::Policy.backoff_delay(attempt, **settings.backoff_arguments) + end + + assert_equal(expected, clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-41 / RECOV-34: zero retries is one send, and a valid settings never clamps" do + # A negative max_retries cannot be built (RECOV-34 refuses it at construction), so + # RETRY-41's clamp is unreachable on this stack through the public constructor; the + # resolver is still the shared one and logs nothing for a valid value. + sink = RecordingSink.new + built, transport = engine(Array.new(4) { retry_response(503) }, + settings: retry_settings(max_retries: 0), + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(1, transport.calls.size, "zero retries: one send") + assert_empty(sink.entries) + end + end + + # RECOV-27, RETRY-23, RETRY-35, RECOV-16, OBS-29, the orchestrator integration. + class IntegrationTest < DexpaceTestCase + include Fixtures + + test "RECOV-27 / RETRY-23: cancellation during the wait aborts the loop as CancelledError" do + source = Dexpace::Cancellation.source + script = [lambda { |*| + source.cancel(:test) + retry_response(503) + }, retry_response(200),] + built, transport = engine(script, settings: retry_settings(initial_delay: 5.0)) + + error = assert_raises(Dexpace::CancelledError) do + send_through(built, cancellation: source.token) + end + + assert_equal(1, transport.calls.size, "the second send never happened") + assert_equal(:test, error.reason) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error)) + end + + test "RECOV-27: the wait is the clock's cancellable sleep with the caller's token" do + clock = FakeClock.new + source = Dexpace::Cancellation.source + built, = engine([retry_response(503), retry_response(200)], + settings: retry_settings(clock: clock),) + + send_through(built, cancellation: source.token) + + assert_equal(1, clock.sleeps.size) + assert_same(source.token, clock.sleeps.first[:cancellation]) + end + + test "RETRY-23: a cancelled token is checked at every attempt boundary" do + source = Dexpace::Cancellation.source + source.cancel(:already) + built, transport = engine([retry_response(200)]) + + assert_raises(Dexpace::CancelledError) { send_through(built, cancellation: source.token) } + assert_equal(0, transport.calls.size) + end + + test "RETRY-35 / RECOV-16: the error body is buffered (connection released) before the wait" do + original = retry_response(503) + released_at_sleep = [] + clock = FakeClock.new + clock.define_singleton_method(:sleep) do |duration, cancellation: nil| + released_at_sleep << original.body.closed? + super(duration, cancellation: cancellation) + end + built, = engine([original, retry_response(200)], settings: retry_settings(clock: clock)) + + send_through(built) + + assert_equal([true], released_at_sleep) + end + + test "RETRY-35: the pacing delay is read from the buffered response, headers intact" do + clock = FakeClock.new + built, = engine([retry_response(503, headers: { "Retry-After" => "3" }), retry_response(200)], + settings: retry_settings(clock: clock),) + + send_through(built) + + assert_equal([3.0], clock.sleeps.map { _1[:duration] }) + end + + test "OBS-29 / P6-7: the tracer is made once per operation with the REQUEST as its context" do + tracer = Dexpace::RecordingHTTPTracer.new + contexts = [] + built, = engine([retry_response(503), retry_response(503), retry_response(200)], + http_tracer_factory: lambda { |request| + contexts << request + tracer + },) + request = retry_request + + send_through(built, request: request) + + assert_equal([request], contexts) + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started attempt_failed + attempt_started], names,) + tracer.events.each { |event| assert_same(request, event[1]) } + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2]) + assert_in_delta(0.1, tracer.events[1][3]) + end + + test "OBS-29: retries_exhausted immediately precedes the terminal raise, with its error" do + tracer = Dexpace::RecordingHTTPTracer.new + built, = engine([retry_response(503), retry_response(503)], + settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_request) { tracer },) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(:retries_exhausted, tracer.events.last.first) + assert_same(error, tracer.events.last[2]) + end + + test "OBS-29: a never-retryable failure ends with no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + built, = engine([retry_response(404)], http_tracer_factory: ->(_request) { tracer }) + send_through(built) + + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end + + # P6-3: the engine beneath a real Recovery::Orchestrator. + class OrchestratorTest < DexpaceTestCase + include Fixtures + + test "P6-3: beneath a real Orchestrator, RequestChain stamps once and the factory maps once" do + calls = [] + factory = lambda do |response| + calls << response.status.code + Dexpace::ProtocolError.for(response) + end + built, transport = engine([retry_response(503), retry_response(404)], + settings: retry_settings(max_retries: 3),) + stamp = Dexpace::Recovery::IdempotencyKeyStep.build( + header: "Idempotency-Key", strategy: ->(_request) { Dexpace::UUID.generate }, + methods: [Dexpace::Method::PUT], + ) + request_chain = Dexpace::Recovery::RequestChain.build(steps: [stamp]) + orchestrator = Dexpace::Recovery::Orchestrator.build( + transport: built, request_chain: request_chain, + response_chain: Dexpace::Recovery::ResponseChain.build( + response_steps: [Dexpace::Recovery::ErrorMappingStep.build(factory: factory)], + ), + ) + + request = retry_request(method: "PUT", body: replayable_body) + error = assert_raises(Dexpace::ProtocolError) do + orchestrator.call(request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + + assert_equal(404, error.status.code) + assert_equal([404], calls, "once, on the terminal response, never on the retried 503") + keys = transport.calls.map { |(sent, _o, _c)| sent.headers["Idempotency-Key"] } + + assert_equal(2, keys.size) + assert_equal(keys.first, keys.last, "one key, stamped once, re-sent as is") + refute_nil(keys.first) + end + + test "P6-3: beneath the Orchestrator an exhausted retryable status surfaces as the raise" do + built, = engine([retry_response(503), retry_response(503)], + settings: retry_settings(max_retries: 1),) + orchestrator = Dexpace::Recovery::Orchestrator.build( + transport: built, request_chain: Dexpace::Recovery::RequestChain.build, + response_chain: Dexpace::Recovery::ResponseChain.build, + ) + + error = assert_raises(Dexpace::ProtocolError) do + orchestrator.call(retry_request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + + assert_equal(503, error.status.code) + assert_equal(1, Dexpace.suppressed(error).size) + end + end +end From a1d07bf851e05ac32dd231668250f59a2f3d9489 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 16:34:29 +0300 Subject: [PATCH 12/13] test: seam the retry suites, mirror the error, prove round 1's guards Review round 0's two blocking findings and the test halves of its five should-fix findings, on the tests layer. The three driver suites built their default settings off the live configuration slot, so a host MAX_RETRY_ATTEMPTS changed their answers (=0 broke the recovery suite, =-1 all three). Each suite's Fixtures module now installs a FakeConfigSource env seam in setup and resets the slot in teardown, the shape retry_settings_test.rb and budget_equivalence_test.rb already had; the seven suites run identically with the variable set to 0 and to -1. error/retry_predicate_error.rb, a public lib file, had no test mirror; retry_predicate_error_test.rb pins its shape (StandardError, Dexpace:: Error, flat under Dexpace::, the predicate's raise as #cause). For the code fix on the branch below: policy_test.rb's PacingBoundsTest proves a 10 MB value in every form answers nil in under 50 ms on the parser alone, that a 400-digit run emits no Ruby warning (recorded with WarningCapture, since the suite's raiser and Policy#parse_form's fence had hidden it), the fifteen-digit boundary and the 64-byte ceiling; the '9' * 300 clamp case moves to the out-of-range set and the unexplained retry-after-ms exemption is gone. The at-the-cap jitter test asserts samples on BOTH sides of the cap, which the jitter-then-clip order cannot pass (the reviewer's surviving mutation m02). A tracer raising in attempt_failed is asserted to close the superseded response on both stage drivers. A should_retry answering true is asserted not to retry a downstream CancelledError on both stage drivers, a cancellation wrapped by a retryable error is terminal on all three, and Policy.cancellation? has its own cases. A negative configured MAX_RETRY_ATTEMPTS is asserted clamped and logged at RetrySettings.build, contained, with an explicit negative max_retries: still refused; the Policy method pin gains cancellation?. --- .../error/retry_predicate_error_test.rb | 58 ++++++++ .../resilience/async_retry_step_test.rb | 72 +++++++++- .../test/dexpace/resilience/policy_test.rb | 129 ++++++++++++++++-- .../dexpace/resilience/recovery_retry_test.rb | 43 +++++- .../dexpace/resilience/retry_settings_test.rb | 56 +++++++- .../dexpace/resilience/retry_step_test.rb | 102 +++++++++++++- 6 files changed, 433 insertions(+), 27 deletions(-) create mode 100644 gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb diff --git a/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb b/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb new file mode 100644 index 0000000..4ab8beb --- /dev/null +++ b/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" + +# RETRY-40, RETRY-25, P6-11, P6-56: the well-typed abort the two stage-based retry drivers raise +# when a caller's should_retry predicate itself raises. The raise sites and the wrapping are +# asserted where they happen (retry_step_test.rb, async_retry_step_test.rb); this mirror pins the +# class's own shape. +class DexpaceRetryPredicateErrorTest < DexpaceTestCase + test "is caught by rescue Dexpace::Error through Module#===" do + caught = begin + raise Dexpace::RetryPredicateError, "the should_retry predicate raised RuntimeError" + rescue Dexpace::Error => error + error + end + + assert_instance_of(Dexpace::RetryPredicateError, caught) + assert_kind_of(Dexpace::Suppressible, caught) + end + + test "subclasses StandardError and includes Dexpace::Error" do + assert_operator(Dexpace::RetryPredicateError, :<, ::StandardError) + assert_includes(Dexpace::RetryPredicateError.ancestors, Dexpace::Error) + end + + # P6-56: flat under Dexpace::, on the tree's one-error-per-file convention, and not under + # Dexpace::Resilience, where the plan named it half the time. + test "P6-56: lives flat under Dexpace::, not under Dexpace::Resilience" do + assert_equal("Dexpace::RetryPredicateError", Dexpace::RetryPredicateError.name) + refute(Dexpace::Resilience.const_defined?(:RetryPredicateError, false)) + end + + # RETRY-40: the predicate's own failure travels as #cause, which a `cause:` at the raise site + # sets -- a genuine wrap of a fresh failure, so pipeline/7ce4431d's `cause: nil` rule for a + # CARRIED error does not apply here. + test "RETRY-40: carries the predicate's raise as #cause when raised with cause:" do + inner = ::RuntimeError.new("boom") + wrapped = begin + raise Dexpace::RetryPredicateError.new("the should_retry predicate raised RuntimeError"), + cause: inner + rescue Dexpace::RetryPredicateError => error + error + end + + assert_same(inner, wrapped.cause) + assert_equal("the should_retry predicate raised RuntimeError", wrapped.message) + end + + test "carries no fields of its own" do + assert_empty(Dexpace::RetryPredicateError.instance_methods(false)) + end + + test "is not Dexpace::InvalidArgumentError -- a misbehaving hook is not a bad argument" do + refute_operator(Dexpace::RetryPredicateError, :<, Dexpace::InvalidArgumentError) + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb index 42cd533..e283cad 100644 --- a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb @@ -8,26 +8,41 @@ require_relative "../../support/recording_http_tracer" require_relative "../../support/recording_sink" require_relative "../../support/parking_scheduler" +require_relative "../../support/fake_config_source" # Exercises: RETRY-2, RETRY-5, RETRY-7, RETRY-8, RETRY-13, RETRY-20, RETRY-23, RETRY-24, # RETRY-25, RETRY-26, RETRY-30, RETRY-31, RETRY-32, RETRY-33, RETRY-34, RETRY-35, RETRY-39, # RETRY-40, RETRY-41, RETRY-42, RETRY-44, RETRY-45, OBS-29, OBS-30, PIPE-29, PIPE-30, XCUT-3, -# CFG-18, P6-5, P6-7, R2 +# CFG-18, P6-5, P6-7, P6-60, R2 # # The asynchronous stage-based retry step, driven through a REAL Dexpace::AsyncPipeline over a # ScriptedAsyncTransport whose futures settle INLINE -- the shape that overflowed the plan's # recursive pump -- and, for R2's scheduler route, through 5a's ParkingScheduler, the one double # that can drive a timed wait (its loop runs in #close, so every case joins the scheduler's # thread). No test here sleeps a thread: a zero-length delay completes inline, a positive one -# either parks a fiber or fails the future. Split into nested classes under Metrics/ClassLength. +# either parks a fiber or fails the future. Hermetic: a build with no settings reads the +# configured MAX_RETRY_ATTEMPTS, so every nested class runs behind a FakeConfigSource env seam. +# Split into nested classes under Metrics/ClassLength. class DexpaceResilienceAsyncRetryStepTest < DexpaceTestCase AsyncRetryStep = Dexpace::Resilience::AsyncRetryStep Stages = Dexpace::Pipeline::Stages - # The async pipeline shape every nested class drives. + # The async pipeline shape every nested class drives, and the seam. module Fixtures include RetryFixtures + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + def pipeline(step, script, settle_later: false) transport = ScriptedAsyncTransport.new(script, settle_later: settle_later) wrapper = RecordingWrapper.new(step) @@ -256,6 +271,23 @@ def exploding.attempt_started(*) = raise("tracer blew up") assert_equal("tracer blew up", error.message) end + test "RETRY-33 / RETRY-35: a tracer raising in attempt_failed closes the response, no hang" do + open_response = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { exploding },) + future, transport, = drive(step, [open_response, retry_response(200)]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + assert_predicate(open_response.body, :closed?, "the sync driver closes it the same way") + assert_equal(1, transport.calls.size) + end + test "RETRY-33: a throwing factory completes the future exceptionally" do step = AsyncRetryStep.build(settings: inline_settings, http_tracer_factory: ->(_cursor) { raise "no tracer" },) @@ -358,6 +390,40 @@ class CancellationTest < DexpaceTestCase assert_equal(:mid, error.reason) assert_equal(1, transport.calls.size) end + + test "RETRY-23 / P6-60: a should_retry answering true cannot retry a downstream cancellation" do + seen = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5), + should_retry: lambda { |failure, _request| + seen << failure + true + },) + future, transport, = drive(step, [Dexpace::CancelledError.new(:downstream), + retry_response(200),],) + + error = assert_raises(Dexpace::CancelledError) { future.value } + + assert_equal(:downstream, error.reason) + assert_equal(1, transport.calls.size) + assert_empty(seen, "the predicate is never consulted for a cancellation") + end + + test "RETRY-23 / P6-60: a cancellation wrapped by a retryable error is terminal on this path" do + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + future, transport, = drive(step, [wrapped, retry_response(200)]) + + assert_same(wrapped, assert_raises(RetryableError) { future.value }) + assert_equal(1, transport.calls.size) + end end # The shared decision and delay logic, asserted on the async path too (design §11.12). diff --git a/gems/dexpace-core/test/dexpace/resilience/policy_test.rb b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb index 62c793b..aad3819 100644 --- a/gems/dexpace-core/test/dexpace/resilience/policy_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb @@ -6,11 +6,13 @@ require_relative "../../support/recovery_fixtures" require_relative "../../support/cyclic_errors" require_relative "../../support/recording_sink" +require_relative "../../support/warning_capture" # Exercises: RETRY-1, RETRY-2, RETRY-9, RETRY-10, RETRY-11, RETRY-12, RETRY-13, RETRY-15, -# RETRY-16, RETRY-17, RETRY-18, RETRY-19, RETRY-20, RETRY-21, RETRY-22, RETRY-37, RETRY-41, -# RETRY-42, RECOV-17, RECOV-20, RECOV-21, RECOV-22, RECOV-23, RECOV-24, RECOV-25, RECOV-26, -# RECOV-29, RECOV-30, CFG-35, XCUT-6, XCUT-7, XCUT-9, P6-5, P6-10, P6-12 +# RETRY-16, RETRY-17, RETRY-18, RETRY-19, RETRY-20, RETRY-21, RETRY-22, RETRY-23, RETRY-37, +# RETRY-41, RETRY-42, RECOV-17, RECOV-20, RECOV-21, RECOV-22, RECOV-23, RECOV-24, RECOV-25, +# RECOV-26, RECOV-27, RECOV-29, RECOV-30, CFG-35, NFR-6, XCUT-6, XCUT-7, XCUT-9, P6-5, P6-10, +# P6-12, P6-60, P6-61 # # The shared policy core: the two-branch classifier consult, the calculator, the resolver, the # recovery-only budget and the pacing dispatcher. Split into nested classes under @@ -89,8 +91,8 @@ def headers(pairs) assert_empty(Policy.instance_variables) assert_kind_of(Policy, Policy, "extend self, never module_function") assert_equal( - %i[backoff_delay budget_remaining effective_max_retries pacing_delay retry_eligible? - retryable? throwable_retryable?], + %i[backoff_delay budget_remaining cancellation? effective_max_retries pacing_delay + retry_eligible? retryable? throwable_retryable?], Policy.public_instance_methods(false).sort, ) end @@ -166,6 +168,29 @@ class ClassificationTest < DexpaceTestCase "a configured set that ADMITS 503 still does not reach a wrapped ProtocolError " \ "through the capability branch; the status branch needs the error itself",) end + + test "RETRY-23 / P6-60: cancellation? finds a CancelledError itself or anywhere in the chain" do + cancelled = Dexpace::CancelledError.new(:token) + + assert(Policy.cancellation?(cancelled)) + assert(Policy.cancellation?(wrap(cancelled, ::IOError, "wrapped by a transport"))) + assert(Policy.cancellation?(wrap(wrap(cancelled), ::RuntimeError, "two levels up"))) + refute(Policy.cancellation?(::IOError.new("no cancellation anywhere"))) + refute(Policy.cancellation?(wrap(::StandardError.new("inner")))) + assert_raises(Dexpace::InvalidArgumentError) { Policy.cancellation?("boom") } + end + + test "RETRY-23 / RECOV-27: a cancellation is never retryable, whatever wraps it answers" do + cancelled = Dexpace::CancelledError.new(:token) + # A transport error that answers the capability and CARRIES the cancellation: the + # capability branch alone would call it retryable; the cancellation guard runs first. + carrier = wrap(cancelled, RetryableByCapability, "read interrupted") + + assert(Policy.throwable_retryable?(carrier), "the fixture is load-bearing: capability yes") + refute(Policy.retryable?(carrier, retryable_statuses: ::Set[503])) + refute(Policy.retryable?(cancelled, retryable_statuses: ::Set[503])) + refute(Policy.throwable_retryable?(cancelled), "CancelledError answers no capability") + end end # RETRY-9, RETRY-10, RETRY-11, RETRY-41, RECOV-20, RECOV-21, P6-12. @@ -232,14 +257,19 @@ def backoff(attempt, **overrides) "spread 1.5e-9 draws",) end - test "RETRY-10: at the cap the jitter band is drawn around the cap (base == cap)" do + test "RETRY-10 / RETRY-9: at the cap the band is drawn AROUND the cap: capped, then jittered" do random = ::Random.new(7) - 20.times do - d = backoff(50, jitter: 1.0, random: random) + samples = Array.new(200) { backoff(50, jitter: 1.0, random: random) } + samples.each do |d| assert_operator(d, :>=, 4.0) assert_operator(d, :<=, 12.0, "d(1 + j/2) with d = 8.0 and j = 1.0") end + # The order RECOV-21 fixes: cap FIRST, then perturb. Jitter applied to the raw + # 0.2 * 2**49 and clipped to the cap afterwards would answer exactly 8.0 every time and + # never a sample above it, so the band's two halves are asserted, not only its bounds. + assert_operator(samples.max, :>, 8.0, "a sample above the cap: jittered after capping") + assert_operator(samples.min, :<, 8.0) end test "RETRY-10: a negative sample is floored at zero" do @@ -438,12 +468,12 @@ def delay_with(pairs, random) test "RETRY-16 / RECOV-23: malformed, negative and out-of-range values are no hint, not 0" do ["-5", "-0.5", "not a date", "", " 5", "5 ", "Sun, 32 Nov 1994 08:49:37 GMT", - "Sunday, 06-Nov-94 08:49:37 GMT", "9" * 400,].each do |bad| + "Sunday, 06-Nov-94 08:49:37 GMT", "9" * 16, "9" * 300, "9" * 400,].each do |bad| assert_nil(delay("Retry-After" => bad), bad.inspect) end - ["-1", "1.5", "abc", "", "9" * 400].each do |bad| + ["-1", "1.5", "abc", "", "9" * 16, "9" * 400].each do |bad| assert_nil(delay("X-RateLimit-Reset" => bad), bad.inspect) - assert_nil(delay("retry-after-ms" => bad), bad.inspect) unless bad == "9" * 400 + assert_nil(delay("retry-after-ms" => bad), bad.inspect) end end @@ -459,7 +489,9 @@ def delay_with(pairs, random) assert_equal(ceiling, delay("retry-after-ms" => (400 * 24 * 60 * 60 * 1000).to_s)) assert_equal(ceiling, delay("X-RateLimit-Reset" => (NOW.to_i + (400 * 24 * 60 * 60)).to_s)) assert_equal(ceiling, delay("Retry-After" => "Fri, 18 Sep 2099 12:00:00 GMT")) - assert_equal(ceiling, delay("Retry-After" => "9" * 300), "1e300 seconds is finite; clamped") + assert_equal(ceiling, delay("Retry-After" => "9" * 15), "the longest run the grammar admits") + assert_equal(ceiling, delay("retry-after-ms" => "9" * 15)) + assert_equal(ceiling, delay("X-RateLimit-Reset" => "9" * 15)) end test "RETRY-18: exactly the ceiling and one below pass through unclamped" do @@ -507,4 +539,77 @@ def forged2.[](_name) = [42] assert_nil(Policy.pacing_delay(forged2, header_order: ORDER, now: NOW)) end end + + # RETRY-16, RETRY-19, NFR-6, P6-61: the parser's bounds -- a run past fifteen digits or a + # value past 64 bytes is no hint before any conversion, so a hostile value costs microseconds + # and emits no warning. + class PacingBoundsTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + FORMS = %w[Retry-After retry-after-ms X-RateLimit-Reset].freeze + + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + # A headers stand-in answering one huge value: phase 1's inbound builder validates a value + # byte by byte (seconds for 10 MB) and is not what is under test here; the parser is. + def forged(name, value) + stand_in = ::Object.new + stand_in.define_singleton_method(:[]) { |asked| asked == name ? [value] : nil } + stand_in + end + + test "P6-61: a 10 MB value in every form is no hint, in microseconds, on the parser alone" do + # The digit run exercises the grammars' bound in every form; the letter run exercises the + # byte ceiling in front of the HTTP-date attempt, which 5a's anchored grammar would + # otherwise scan (measured at 0.1 s for 10 MB). Both answer in well under a millisecond. + cases = FORMS.map { |name| [name, "9" * 10_000_000] } << ["Retry-After", "x" * 10_000_000] + cases.each do |name, huge| + started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + answer = Policy.pacing_delay(forged(name, huge), header_order: [name], now: NOW) + elapsed = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started + + assert_nil(answer, name) + assert_operator(elapsed, :<, 0.05, "#{name} #{huge[0]}: #{elapsed} s; the value was read") + end + end + + test "NFR-6 / RETRY-16: a run long enough to overflow a Float emits no Ruby warning" do + # The suite's raiser turns a warning into an error that Policy#parse_form's fence would + # swallow, so the totality cases above cannot see one; recording the warnings can. + FORMS.each do |name| + warnings = WarningCapture.record { assert_nil(delay(name => "9" * 400), name) } + + assert_empty(warnings, "#{name}: String#to_f/#to_i ran on the run") + end + end + + test "P6-61: fifteen digits is the boundary, and the fraction is bounded the same way" do + assert_equal(Policy::MAX_PACING_DELAY_SECONDS.to_f, delay("Retry-After" => "1" * 15)) + assert_nil(delay("Retry-After" => "1" * 16)) + assert_in_delta(1.5, delay("Retry-After" => "1.5")) + assert_in_delta("1.#{"5" * 15}".to_f, delay("Retry-After" => "1.#{"5" * 15}"), 0.0) + assert_nil(delay("Retry-After" => "1.#{"5" * 16}")) + assert_in_delta(1.5, delay("retry-after-ms" => "1500")) + assert_nil(delay("retry-after-ms" => "1500.0"), "millis take no fraction") + end + + test "P6-61: the 64-byte ceiling sits above every well-formed form" do + parsers = Dexpace::Resilience.const_get(:PacingParsers) + + assert_equal(64, parsers.const_get(:MAX_VALUE_BYTES)) + # The two longest well-formed values -- the 29-byte RFC 1123 date and the 31-byte decimal + # the bounded grammar admits (fifteen digits, a point, fifteen digits) -- both parse. + longest = "#{"1" * 15}.#{"5" * 15}" + + assert_equal(31, longest.bytesize) + assert_in_delta(45.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:00:45 GMT")) + assert_equal(Policy::MAX_PACING_DELAY_SECONDS.to_f, delay("Retry-After" => longest)) + assert_nil(delay("Retry-After" => "x" * 65)) + assert_nil(delay("Retry-After" => "x" * 64), "the date grammar refuses; the ceiling admits") + end + end end diff --git a/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb index e4a2b4e..9518cd9 100644 --- a/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb @@ -7,24 +7,39 @@ require_relative "../../support/scripted_transport" require_relative "../../support/recording_http_tracer" require_relative "../../support/recording_sink" +require_relative "../../support/fake_config_source" # Exercises: RECOV-16, RECOV-17, RECOV-18, RECOV-19, RECOV-20, RECOV-21, RECOV-22, RECOV-23, # RECOV-24, RECOV-26, RECOV-27, RECOV-28, RECOV-29, RECOV-30, RECOV-34, RETRY-4, RETRY-13, # RETRY-14, RETRY-23, RETRY-25, RETRY-27, RETRY-34, RETRY-35, RETRY-36, RETRY-37, RETRY-41, -# RETRY-42, RETRY-44, OBS-29, CFG-16, SEAM-11, P6-3, P6-6, P6-8, P6-9 +# RETRY-42, RETRY-44, OBS-29, CFG-16, SEAM-11, P6-3, P6-6, P6-8, P6-9, P6-60 # # The recovery-stack engine, driven directly as the Dexpace::Transport it is and, where the # integration is the point, beneath a real Recovery::Orchestrator with a real # ErrorMappingStep. Every wait is a FakeClock's, whose monotonic reading advances by exactly the # slept duration, so the total-timeout arithmetic is exact rather than wall-clock-dependent. -# Split into nested classes under Metrics/ClassLength. +# Hermetic: the engine's default settings read the configured MAX_RETRY_ATTEMPTS, so every +# nested class runs behind a FakeConfigSource env seam. Split into nested classes under +# Metrics/ClassLength. class DexpaceResilienceRecoveryRetryTest < DexpaceTestCase RecoveryRetry = Dexpace::Resilience::RecoveryRetry - # The engine over a script, and the one call every test makes. + # The engine over a script, the one call every test makes, and the seam. module Fixtures include RetryFixtures + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + def engine(script, settings: retry_settings, **keywords) transport = ScriptedTransport.new(script) [RecoveryRetry.build(transport: transport, settings: settings, **keywords), transport] @@ -374,6 +389,28 @@ class IntegrationTest < DexpaceTestCase assert_equal(0, transport.calls.size) end + test "RETRY-23 / P6-60: a cancellation the transport WRAPPED in a retryable error is final" do + # RecoveryRetry has no caller predicate; the discriminator on this stack is the wrapped + # form, which the capability branch alone would have retried. + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + built, transport = engine([wrapped, retry_response(200)], + settings: retry_settings(max_retries: 5),) + + surfaced = assert_raises(RetryableError) { send_through(built) } + + assert_same(wrapped, surfaced) + assert_equal(1, transport.calls.size, "never re-sent") + assert_empty(Dexpace.suppressed(surfaced)) + end + test "RETRY-35 / RECOV-16: the error body is buffered (connection released) before the wait" do original = retry_response(503) released_at_sleep = [] diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb index 1d7509f..60d3dd8 100644 --- a/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb @@ -5,9 +5,10 @@ require "dexpace" require_relative "../../support/fake_clock" require_relative "../../support/fake_config_source" +require_relative "../../support/recording_sink" -# Exercises: RECOV-34, RETRY-12, RETRY-13, RETRY-14, RETRY-21, RETRY-42, RECOV-30, CFG-14, -# SEAM-29, P6-6 +# Exercises: RECOV-34, RETRY-12, RETRY-13, RETRY-14, RETRY-21, RETRY-41, RETRY-42, RECOV-30, +# CFG-14, OBS-20, SEAM-29, P6-6, P6-57, P6-59 # # The one retry configuration: its defaults are Policy's constants, its validation is RECOV-34 # clause by clause, its collections are copied and frozen, and its max_retries is the first @@ -177,14 +178,57 @@ class ConfiguredTest < DexpaceTestCase assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries) end - test "RETRY-12 / RECOV-34: a negative configured value is refused at build, not clamped here" do - # RETRY-41's clamp is the DRIVER's, at resolution time against the per-call override; the - # settings object itself holds only a valid value (RECOV-34). + test "RETRY-41 / P6-59: a negative CONFIGURED value is clamped to the default and logged" do + # The one read of the configured key is where RETRY-41's "a negative configured value MUST + # be clamped to the default (and the clamp logged)" is met, through the same resolver the + # drivers use; the settings object then holds a valid value (RECOV-34) and every driver + # built on it sees the default without a second clamp or a second log line. Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-1") } + sink = RecordingSink.new + settings = Settings.build(logger: Dexpace::Instrumentation::Logger.build(sink: sink)) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, settings.max_retries) + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_CONFIG, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/-1.*clamped to #{Policy::DEFAULT_MAX_RETRIES}/o, + payload[Dexpace::Instrumentation::Keys::MESSAGE],) + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries, + "Logger::NULL by default: clamped, reported nowhere",) + end + + test "RETRY-41 / OBS-20: the clamp's log is contained: a raising sink cannot fail the build" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-3") } + sink = RecordingSink.new + sink.define_singleton_method(:warn) { |*| raise ::IOError, "sink write failure" } + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build(logger: logger).max_retries) + end + + test "RECOV-34: an EXPLICIT negative max_retries: is still refused, never clamped" do + # The configured key is the environment's value; an explicit argument is the caller's + # construction input, and RECOV-34 refuses that at initialize -- with the key negative too. + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-1") } + sink = RecordingSink.new + logger = Dexpace::Instrumentation::Logger.build(sink: sink) - error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build } + error = assert_raises(Dexpace::InvalidArgumentError) do + Settings.build(max_retries: -1, logger: logger) + end assert_includes(error.message, "max_retries") + assert_empty(sink.entries, "no clamp, so no clamp diagnostic") + end + + test "P6-59: logger: is read at build and never held -- not a member, absent from #to_h" do + settings = Settings.build(logger: Dexpace::Instrumentation::Logger::NULL) + + refute_includes(settings.to_h.keys, :logger) + refute_respond_to(settings, :logger) + assert_equal(settings, Settings.build, "value equality is over the ten members alone") end test "RETRY-12 / CFG-5: an unparseable configured value falls to the default (5a's parser)" do diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb index 0dd953a..33b0ea5 100644 --- a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb @@ -8,26 +8,41 @@ require_relative "../../support/recording_http_tracer" require_relative "../../support/recording_sink" require_relative "../../support/state_probe" +require_relative "../../support/fake_config_source" # Exercises: RETRY-2, RETRY-5, RETRY-6, RETRY-7, RETRY-8, RETRY-9, RETRY-10, RETRY-11, RETRY-12, # RETRY-13, RETRY-15, RETRY-20, RETRY-21, RETRY-22, RETRY-23, RETRY-24, RETRY-25, RETRY-26, # RETRY-28, RETRY-34, RETRY-35, RETRY-36, RETRY-39, RETRY-40, RETRY-41, RETRY-42, RETRY-44, -# RETRY-45, OBS-29, OBS-30, PIPE-15, PIPE-16, PIPE-40, XCUT-3, P6-5, P6-7, P6-11 +# RETRY-45, OBS-29, OBS-30, PIPE-15, PIPE-16, PIPE-40, XCUT-3, P6-5, P6-7, P6-11, P6-60 # # The synchronous stage-based retry step, driven through a REAL Dexpace::Pipeline over a # ScriptedTransport: Builder#append reads #stage, the driver mints the step's cursor, and the # fork-for-every-drive rule is asserted through a recording cursor over that real one. Every # wait is a FakeClock's -- a real Clock::SYSTEM wait in this suite is a finding -- and the one # SYSTEM-clock test is the RETRY-23 case whose zero-length wait cannot observe the token. -# Split into nested classes under Metrics/ClassLength. +# Hermetic: a RetryStep.build with no settings reads the configured MAX_RETRY_ATTEMPTS, so every +# nested class runs behind a FakeConfigSource env seam. Split into nested classes under +# Metrics/ClassLength. class DexpaceResilienceRetryStepTest < DexpaceTestCase RetryStep = Dexpace::Resilience::RetryStep Stages = Dexpace::Pipeline::Stages - # The pipeline shape every nested class drives, and the counters it reads off. + # The pipeline shape every nested class drives, the counters it reads off, and the seam. module Fixtures include RetryFixtures + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + def pipeline(step, script, wrap: true, extra: []) transport = ScriptedTransport.new(script) installed = wrap ? RecordingWrapper.new(step) : step @@ -344,6 +359,25 @@ class DelayTest < DexpaceTestCase assert_equal([3.0], clock.sleeps.map { _1[:duration] }, "the hint was read while open") end + test "RETRY-35 / OBS-30: a tracer raising in attempt_failed propagates, the response closed" do + # The emission sits inside the same fence as the delay resolution: a throwing tracer fails + # the request (OBS-30, never contained) but never leaves the superseded response open + # across the propagation -- the async driver's guarded block already closed it (R0-4). + open_response = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = raise("tracer blew up") + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 2), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [open_response, retry_response(200)]) } + + assert_equal("tracer blew up", error.message) + assert_predicate(open_response.body, :closed?, "closed before the raise propagated") + assert_empty(clock.sleeps, "the wait was never reached") + end + test "RETRY-20 / RETRY-22: a pacing hint REPLACES the schedule verbatim; a bad one falls" do clock = FakeClock.new settings = retry_settings(clock: clock, initial_delay: 30.0, max_delay: 30.0, jitter: 0.5, @@ -567,6 +601,68 @@ class PredicateAndBudgetTest < DexpaceTestCase end end + # RETRY-23's guard ahead of the caller's predicate and the capability (P6-60). + class CancellationGuardTest < DexpaceTestCase + include Fixtures + + test "RETRY-23 / P6-60: a should_retry answering true cannot retry a downstream cancellation" do + step = RetryStep.build(settings: retry_settings(max_retries: 5), should_retry: ->(*) { true }) + seen = [] + spy = RetryStep.build(settings: retry_settings(max_retries: 5), + should_retry: lambda { |failure, _request| + seen << failure + true + },) + + error = assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + + assert_equal(:downstream, error.reason) + assert_raises(Dexpace::CancelledError) do + drive(spy, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + assert_empty(seen, "the predicate is never consulted for a cancellation") + end + + test "RETRY-23 / P6-60: a cancellation WRAPPED by a retryable error is terminal too" do + # A transport that wraps the token's raise in its own retryable error: the capability + # branch alone would retry it; the guard reads the cancellation it carries. + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + + surfaced = assert_raises(RetryableError) { drive(step, [wrapped, retry_response(200)]) } + + assert_same(wrapped, surfaced) + assert_kind_of(Dexpace::CancelledError, surfaced.cause) + assert_empty(Dexpace.suppressed(surfaced), "one attempt: no trail") + _built, transport, = pipeline(step, [wrapped, retry_response(200)]) + + assert_equal(0, transport.calls.size) + end + + test "RETRY-23: the guard runs before the re-sendability gate and emits no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 0), + http_tracer_factory: ->(_cursor) { tracer },) + + assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream)], + request: retry_request(method: "POST", body: replayable_body),) + end + assert_equal(%i[attempt_started], tracer.events.map(&:first), + "a cancellation is :stop, never :exhausted, at any budget",) + end + end + # OBS-29's per-attempt group, and the three text-scan guards (RETRY-13, RETRY-28, RETRY-45). class TracerAndGuardsTest < DexpaceTestCase include Fixtures From 86762676096d7c520416d385cd3b6411bfc5a2b0 Mon Sep 17 00:00:00 2001 From: Mohammad Wahbeh Date: Fri, 18 Sep 2026 17:48:17 +0300 Subject: [PATCH 13/13] test: prove the terminal fence and the async no-blocking-sleep clause Review round 2 of the phase-6a stack: the two guards behind round 1's findings, each seen red on 4.0.6 and 3.2.11. A tracer raising in retries_exhausted now closes the terminal error-status response on both stage drivers: the sync case is the one the fenced settle makes true (round 1's tree leaves the body open) and the async case pins Pump#finish's guarded block, which closed it all along. The async suite's tracer tests move into a TracerFencesTest of their own, because TerminalPathsTest and SharedShapeTest were both at the class-length ceiling with the new case. RETRY-31's "never a blocking sleep" clause was stated and not mechanised: a blocking Clock#sleep inserted beside Async.delay in the pump's wait left the whole async suite green, because every async case runs on a FakeClock whose #sleep records and returns and nothing read it. The inline, parked and no-scheduler cases now assert clock.sleeps empty, and a text scan refuses the token sleep in async_retry_step.rb, whose one wait is Async.delay's. --- .../resilience/async_retry_step_test.rb | 105 +++++++++++++----- .../dexpace/resilience/retry_step_test.rb | 19 ++++ 2 files changed, 94 insertions(+), 30 deletions(-) diff --git a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb index e283cad..1b4ae88 100644 --- a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb @@ -141,12 +141,14 @@ def depth_probe(depths, response) test "R2 / RETRY-31: a zero-length delay completes inline and re-arms the pump, no scheduler" do assert_nil(::Fiber.scheduler) - step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1)) + clock = FakeClock.new + step = AsyncRetryStep.build(settings: inline_settings(clock: clock, max_retries: 1)) future, transport, = drive(step, [retry_response(503), retry_response(200)]) assert_predicate(future, :settled?, "settled before #call returned: no wait happened") assert_equal(200, future.value.status.code) assert_equal(2, transport.calls.size) + assert_empty(clock.sleeps, "RETRY-31: the wait is Async.delay's, never the clock's sleep") end test "R2 / CFG-18: a positive delay under a scheduler parks the fiber, blocks no thread" do @@ -166,11 +168,14 @@ def depth_probe(depths, response) assert_equal(200, future.value.status.code) assert_operator(scheduler.block_count, :>=, 2, "one park per positive wait") assert_equal(0, scheduler.kernel_sleep_count) + assert_empty(settings.clock.sleeps, "RETRY-31 / RETRY-26: no Clock#sleep beside the park") end test "R2 / RETRY-33: a positive delay with NO scheduler fails the future with SeamError" do assert_nil(::Fiber.scheduler) - step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.01, max_retries: 2)) + clock = FakeClock.new + settings = retry_settings(clock: clock, initial_delay: 0.01, max_retries: 2) + step = AsyncRetryStep.build(settings: settings) first = RetryableError.new("first") future, transport, = drive(step, [first, retry_response(200)]) @@ -180,6 +185,19 @@ def depth_probe(depths, response) assert_match(/Fiber\.set_scheduler/, error.message) assert_equal([first], Dexpace.suppressed(error), "the trail travels with the SeamError") assert_equal(1, transport.calls.size, "no second attempt was launched") + assert_empty(clock.sleeps, "the clock was never asked to sleep: no blocking fallback") + end + + test "RETRY-31 / RETRY-26: the async driver names no blocking sleep, by text" do + # Every behavioural case in this suite runs on a FakeClock, whose #sleep records and + # returns, so a blocking Clock#sleep slipped in beside Async.delay kept the whole suite green + # on its own (R1-2); the three clock.sleeps assertions above and this scan are what turn it + # red. The one wait the driver may name is Async.delay's. + path = File.expand_path("../../../lib/dexpace/resilience/async_retry_step.rb", __dir__) + code = File.read(path).gsub(/^\s*#.*$/, "") + + assert_empty(code.scan(/\bsleep\b/), "async_retry_step.rb names a sleep") + assert_equal(1, code.scan("Dexpace::Async.delay(").size, "the one wait is Async.delay's") end test "RETRY-45: no scheduler is installed, read or shut down by the driver" do @@ -258,6 +276,40 @@ class TerminalPathsTest < DexpaceTestCase assert_predicate(open_response.body, :closed?) end + test "RETRY-33 / RETRY-35: a throwing delay computation closes the response and fails" do + open_response = retry_response(503) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + delay_override: ->(*) { raise ::NotImplementedError, "fatal" },) + transport = ScriptedAsyncTransport.new([open_response]) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + # The fatal family is re-raised AFTER the future is failed (RETRY-25 with RETRY-33), and + # the async driver's normalisation lets a ScriptError through, so it reaches this frame. + assert_raises(::NotImplementedError) { built.call(retry_request) } + assert_predicate(open_response.body, :closed?) + end + + test "RETRY-25: a fatal-family error from the downstream is surfaced unchanged, unretried" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + fatal = ::NoMemoryError.new("oom") + completer = Dexpace::Async::Completer.new + completer.fail(fatal) + transport = ->(_request, _options, _cancellation) { completer.future } + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + future = built.call(retry_request) + + surfaced = assert_raises(::NoMemoryError) { future.value } + + assert_same(fatal, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + end + + # RETRY-33, RETRY-35, OBS-30: the tracer and its factory inside the pump's fence -- a raise in any + # callback fails the future with the response it was handed closed first, on both terminal paths. + class TracerFencesTest < DexpaceTestCase + include Fixtures + test "RETRY-33: a throwing tracer callback completes the future exceptionally, no hang" do exploding = ::Object.new def exploding.attempt_started(*) = raise("tracer blew up") @@ -288,6 +340,27 @@ def exploding.attempt_failed(*) = raise("tracer blew up") assert_equal(1, transport.calls.size) end + test "RETRY-33 / RETRY-35: a tracer raising in retries_exhausted closes the terminal one" do + # R1-1's twin: Pump#finish runs inside guarded(response), so the terminal error-status + # response is closed and the future fails with the tracer's raise -- the sync driver's + # fenced settle now does the same. + terminal = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = nil + def exploding.retries_exhausted(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + future, transport, = drive(step, [retry_response(503), terminal]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + assert_predicate(terminal.body, :closed?, "the sync driver closes it the same way") + assert_equal(2, transport.calls.size) + end + test "RETRY-33: a throwing factory completes the future exceptionally" do step = AsyncRetryStep.build(settings: inline_settings, http_tracer_factory: ->(_cursor) { raise "no tracer" },) @@ -296,34 +369,6 @@ def exploding.attempt_failed(*) = raise("tracer blew up") assert_raises(::RuntimeError) { future.value } assert_equal(0, transport.calls.size) end - - test "RETRY-33 / RETRY-35: a throwing delay computation closes the response and fails" do - open_response = retry_response(503) - step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), - delay_override: ->(*) { raise ::NotImplementedError, "fatal" },) - transport = ScriptedAsyncTransport.new([open_response]) - built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async - - # The fatal family is re-raised AFTER the future is failed (RETRY-25 with RETRY-33), and - # the async driver's normalisation lets a ScriptError through, so it reaches this frame. - assert_raises(::NotImplementedError) { built.call(retry_request) } - assert_predicate(open_response.body, :closed?) - end - - test "RETRY-25: a fatal-family error from the downstream is surfaced unchanged, unretried" do - step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) - fatal = ::NoMemoryError.new("oom") - completer = Dexpace::Async::Completer.new - completer.fail(fatal) - transport = ->(_request, _options, _cancellation) { completer.future } - built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async - future = built.call(retry_request) - - surfaced = assert_raises(::NoMemoryError) { future.value } - - assert_same(fatal, surfaced) - assert_empty(Dexpace.suppressed(surfaced)) - end end # RETRY-23 / RECOV-27 on the async path: the token during a wait and at the boundary. diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb index 33b0ea5..961240d 100644 --- a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb +++ b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb @@ -278,6 +278,25 @@ class TerminalPathTest < DexpaceTestCase refute_predicate(response.body, :closed?) end + test "RETRY-35 / OBS-30: a tracer raising in retries_exhausted propagates, terminal closed" do + # R1-1: the terminal emission sits inside the same fence as the decision -- a throwing + # tracer fails the request (OBS-30, never contained) but the terminal error-status response + # it would otherwise have returned is closed first, as the async pump's guarded block + # already did; round 0 had closed the same drift for attempt_failed only (R0-4). + terminal = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = nil + def exploding.retries_exhausted(*) = raise("tracer blew up") + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [retry_response(503), terminal]) } + + assert_equal("tracer blew up", error.message) + assert_predicate(terminal.body, :closed?, "closed before the raise propagated") + end + test "RETRY-25: a fatal-family error propagates unchanged, unretried, unattached, unreported" do tracer = Dexpace::RecordingHTTPTracer.new step = RetryStep.build(settings: retry_settings(max_retries: 3),