diff --git a/gems/dexpace-core/lib/dexpace.rb b/gems/dexpace-core/lib/dexpace.rb index b8b0cde..2a36e94 100644 --- a/gems/dexpace-core/lib/dexpace.rb +++ b/gems/dexpace-core/lib/dexpace.rb @@ -192,6 +192,24 @@ require_relative "dexpace/instrumentation/step" require_relative "dexpace/instrumentation/async_step" +# Phase 6a: the retry layer, in dependency order -- after 5b's block, because the settings read +# the clock and the configuration slot (5a) and the steps emit through the logging facade (5b) +# and the HTTP-tracer vocabulary (5c). The one flat error first, then the shared policy core +# (the private parsers, the policy over them, the re-sendability gate, the settings), then the +# private helpers the two stage drivers share, the two drivers, and the recovery-stack engine. +# Three earlier files gained a method or a keyword in place and appear above: http_date.rb (the +# day group), error/protocol_error.rb (the baked flag) and pipeline/cursor.rb with the two +# runtimes and the two instrumentation steps (the context-bundle widening). +require_relative "dexpace/error/retry_predicate_error" +require_relative "dexpace/resilience/pacing_parsers" +require_relative "dexpace/resilience/policy" +require_relative "dexpace/resilience/resend" +require_relative "dexpace/resilience/retry_settings" +require_relative "dexpace/resilience/retry_step_helpers" +require_relative "dexpace/resilience/retry_step" +require_relative "dexpace/resilience/async_retry_step" +require_relative "dexpace/resilience/recovery_retry" + # The dexpace Ruby SDK: an HTTP-client toolkit, not an HTTP client. # # This file issues explicit `require_relative`s for the whole tree rather than using an diff --git a/gems/dexpace-core/lib/dexpace/async_pipeline.rb b/gems/dexpace-core/lib/dexpace/async_pipeline.rb index 7e02f71..50d5955 100644 --- a/gems/dexpace-core/lib/dexpace/async_pipeline.rb +++ b/gems/dexpace-core/lib/dexpace/async_pipeline.rb @@ -4,6 +4,7 @@ require_relative "closeable" require_relative "http/request_options" require_relative "cancellation" +require_relative "instrumentation/bundle" require_relative "async/future" require_relative "pipeline" require_relative "pipeline/cursor" @@ -117,16 +118,21 @@ def initialize(entries:, transport:, driver_class:) # is reached qualified because this class's cref does not include Dexpace::Pipeline; there is # exactly one Cursor class and both runtimes use it (P4-30). # + # `bundle:` is phase 6a's widening, as on Pipeline#call: the empty branch dispatches with + # no cursor and therefore carries no bundle, which is PIPE-9's own shape. + # # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call correlation bundle # @return [Dexpace::Async::Future] - def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none) + def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none, + bundle: Instrumentation::Bundle::NONE) driver = @driver_class.new(self) return driver.dispatch(request, options, cancellation) if @entries.empty? Pipeline::Cursor.build(drive: driver, request: request, options: options, - cancellation: cancellation,).call(request) + cancellation: cancellation, bundle: bundle,).call(request) end end end diff --git a/gems/dexpace-core/lib/dexpace/error/protocol_error.rb b/gems/dexpace-core/lib/dexpace/error/protocol_error.rb index d75cea9..dc53546 100644 --- a/gems/dexpace-core/lib/dexpace/error/protocol_error.rb +++ b/gems/dexpace-core/lib/dexpace/error/protocol_error.rb @@ -4,6 +4,7 @@ require_relative "../error" require_relative "invalid_argument_error" require_relative "../http/response" +require_relative "../retryability" module Dexpace # XCUT-4's branch (a): a protocol error carries a fully-received response -- status, headers and @@ -24,9 +25,14 @@ module Dexpace # a customer identifier, and OBS-11..OBS-19's redaction is phase 5's. #response is how a caller # who wants the body reads it (the plan's open question 5). # - # No #retryable_by_status?. XCUT-5 requires the baked flag to be computed once at construction - # from a SINGLE shared status classifier, and that classifier is phase 5a's; phase 6a (Task 6) - # adds the predicate to this class. Adding a method widens a signature, which NFR-4 permits. + # #retryable_by_status? is XCUT-5's baked flag (RETRY-3), computed ONCE at construction from + # the SINGLE shared status classifier -- phase 5a's Dexpace::Retryability, which phase 4b + # could not build and phase 6a reads rather than rebuilds (its Task 6). It is deliberately + # NOT named #retryable?, the open capability XCUT-6's throwable query looks for (P6-10): a + # ProtocolError answering that query would let the baked set override the CONFIGURED set + # RETRY-37 makes authoritative whenever the error is wrapped in another -- including a set + # that deliberately narrows -- so the two questions carry two names, design §6.1's own, and + # the retry drivers consult Resilience::Policy.retry_eligible?(status, set:) and never this. class ProtocolError < ::StandardError include Dexpace::Error @@ -73,9 +79,19 @@ def initialize(response) @response = response @status = response.status + @retryable_by_status = Retryability.retryable_status?(@status) super(describe(@status)) end + # XCUT-5 / RETRY-3: whether the status is one the built-in classifier calls retryable, + # baked at construction and never per-subclass -- a queryable property of the error, and not + # what the retry step consults (XCUT-5's closing NOTE; see the class comment). + # + # @return [Boolean] + def retryable_by_status? + @retryable_by_status + end + private def describe(status) diff --git a/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb b/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb new file mode 100644 index 0000000..fa288bb --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/error/retry_predicate_error.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../error" + +module Dexpace + # RETRY-40's well-typed abort: raised by the two stage-based retry drivers when a + # caller-supplied `should_retry:` predicate itself raises. The predicate's own failure is + # carried as #cause -- a genuine wrap of a fresh failure, raised with `cause:` at the site, + # and not a re-raise of a carried error, so pipeline/7ce4431d's `cause: nil` rule does not + # apply to it. A StandardError including Dexpace::Error, flat under Dexpace:: on the tree's + # one-error-per-file convention (PipelineError, ContextConflictError, ProtocolError), and + # numbered P6-11: RETRY-40 requires the well-typed error and names none. + # + # Only a predicate that raises a StandardError is wrapped. The fatal family -- NoMemoryError, + # SystemStackError -- propagates through an absent rescue arm, unchanged and unwrapped + # (RETRY-25, RETRY-40's "fatal errors rethrown unchanged"). + class RetryPredicateError < ::StandardError + include Dexpace::Error + end +end diff --git a/gems/dexpace-core/lib/dexpace/http_date.rb b/gems/dexpace-core/lib/dexpace/http_date.rb index 57b8b9c..43b95d6 100644 --- a/gems/dexpace-core/lib/dexpace/http_date.rb +++ b/gems/dexpace-core/lib/dexpace/http_date.rb @@ -35,10 +35,17 @@ module HTTPDate # Time.httpdate rejects a bare "06 Nov 1994 ..." too, so the port and the stdlib agree. No # normalisation runs before the match, which is how CFG-31's blank-input failure is a # property of the pattern rather than of whatever a normaliser did to the input. The timeout - # is per-pattern (never Regexp.timeout), on a fixed-width grammar that cannot backtrack, by - # the same rule phase 1 applied to a two-character hex pattern. + # is per-pattern (never Regexp.timeout), on a grammar whose one variable-width group is a + # bounded digit run that cannot backtrack, by the same rule phase 1 applied to a + # two-character hex pattern. The day group is (\d{1,2}), not (\d{2}): RETRY-15 requires the + # Retry-After HTTP-date form to be parsed "tolerant of an informational weekday and + # single-digit day", design §6.1 forbids a second parser, and phase 6a widened this group in + # place (its R1) -- "Sun, 6 Nov 1994 08:49:37 GMT" parses as 6 November. Nothing else moved: + # the weekday is still REQUIRED and still not validated (CFG-30's "informational only" and + # RETRY-15's "informational weekday" are one tolerance), and RFC 850, asctime, a leading + # space, a two-digit year and a trailing token all still fail (CFG-31). GRAMMAR = ::Regexp.new( - '\A[A-Za-z]{3}, (\d{2}) ([A-Za-z]{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) ' \ + '\A[A-Za-z]{3}, (\d{1,2}) ([A-Za-z]{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) ' \ '(GMT|UTC|\+0000|\+00:00)\z', ::Regexp::IGNORECASE, timeout: 1.0, @@ -46,8 +53,8 @@ module HTTPDate # Neither is public surface -- a month table and one pattern with a single reader each, and # P5-1 enumerates HTTPDate alone. A bare reference from inside `module Dexpace; module - # HTTPDate` resolves whatever the visibility, so phase 6a widening the day group edits this - # file at no cost. + # HTTPDate` resolves whatever the visibility, which is what let phase 6a widen the day group + # above in place at no cost to the public surface. private_constant :MONTHS, :GRAMMAR # The canonical HTTP-date form, rendered in UTC with a literal GMT and a zero-padded day @@ -67,10 +74,11 @@ def format(time) time.getutc.httpdate end - # The UTC instant an RFC 1123 date denotes, with CFG-30's four tolerances and CFG-31's - # strictness: the four zone tokens all mean the zero offset, month and zone case do not - # matter, the weekday is stripped and never compared against the date; blank input, a missing - # comma, RFC 850, asctime, a leading space, a single-digit day and a trailing token all fail. + # The UTC instant an RFC 1123 date denotes, with CFG-30's four tolerances, RETRY-15's + # single-digit day (phase 6a's widening) and CFG-31's strictness: the four zone tokens all + # mean the zero offset, month and zone case do not matter, the weekday is stripped and never + # compared against the date, a one-digit day is read as that day; blank input, a missing + # comma, an absent weekday, RFC 850, asctime, a leading space and a trailing token all fail. # So does a syntactically well-formed but impossible date -- Time.utc silently normalises # 31 November, 29 February 1995, hour 24 and second 60 into the next day, month, day or # minute, so every component is checked against what Time.utc built rather than trusted. diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb b/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb index 04993df..77e0081 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/async_step.rb @@ -80,8 +80,9 @@ def call(request, cursor) # @return [Array] the future and the Pending the settlement side needs def head(request, cursor) started = @clock.monotonic - span = open_span(request) - scope = Tracing.correlate(span, Bundle::NONE) + bundle = cursor.bundle + span = open_span(request, bundle) + scope = Tracing.correlate(span, bundle) pending = nil #: Pending? begin request = prepare(request) diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb b/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb index cbdf890..c1620a6 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/bundle.rb @@ -21,7 +21,9 @@ module Instrumentation # NONE.with (both validate), may add METHODS here and to TraceIdFlavour, may give the two # no-op classes their protocols, and may not rename or remove a member, add one, store # validity, change the sentinels, replace the flavour with a Symbol or hand this class a - # second NONE. + # second NONE. Phase 6a honoured the same obligation: it added no member and no method, and + # made a bundle reachable by a pipeline step through Pipeline::Cursor#bundle, seeded per + # call by the `bundle:` keyword on Pipeline#call and AsyncPipeline#call (its Task 8). # # trace_state is a list of pairs, not a Hash: OBS-26 says "a vendor trace-state list" and W3C # tracestate is ordered, most recent vendor first, which a Hash would lose. trace_flags stays diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb b/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb index 99559b9..2dd41c3 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/http_tracer.rb @@ -20,10 +20,17 @@ module Instrumentation # SAME error object; and one tracer instance corresponds 1:1 to one logical operation # lifecycle, created by its factory per operation. ordering_test.rb drives a conformant # emitter through both a succeeding and a retry-exhausted operation, which is the - # requirement's own conformance clause. Nothing in phase 5 emits any of it -- the per-attempt - # group is phase 6a's retry step (its Task 9), the transport milestones are phase 8's, and - # the operation triple is on phase 10's inbound list -- which OBS-29's own last sentence - # anticipates: "pipeline/transport wiring to emit it is a follow-up" (R14). + # requirement's own conformance clause. The per-attempt group is emitted by phase 6a's three + # retry drivers (Resilience::RetryStep, ::AsyncRetryStep and ::RecoveryRetry), through the + # tracer their `http_tracer_factory:` produces once per operation -- called with the + # Pipeline::Cursor on the stage stack and with the Dexpace::Request on the recovery stack -- + # and retries_exhausted fires there only when a RETRYABLE failure met a spent budget, never + # for a failure that was never retryable (ordering_test.rb's third case). The transport + # milestones and the operation-lifecycle triple are emitted by nothing in v1 (phase 10's + # inbound list; docs/first-release.md's behavioural-asymmetries entry), which OBS-29's own + # last sentence anticipates: "pipeline/transport wiring to emit it is a follow-up" (R14). + # The interface _HTTPTracer, which 5c deliberately did not declare, arrived with the wiring + # (6a's R3) in this file's sig/ mirror. # # OBS-30 binds every implementation: each callback is safe to invoke concurrently and from # a transport thread other than the caller's, and never throws -- the runtime does not diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb index e689b64..f27cce9 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/step.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/step.rb @@ -38,16 +38,21 @@ module Instrumentation # # The two slots are keywords defaulting to the constants the phase ships -- phase 4a's # NO_TRACER_FACTORY and phase 5c's NO_METER (P5-33) -- and the two instruments are created - # once here, in .build, never per request (OBS-31). The reconciled precedence's first clause, - # "the request context's bundle when it is not Bundle::NONE", is not implementable in phase - # 5: no reader exists by which a pipeline step reaches a RequestContext or a Bundle (4c does - # not consume 4a; Request's members are method, url, headers and body; PIPE-11 forbids - # ambient carriage), so the step resolves to its keyword and to Bundle::NONE, which is - # OBS-34's and XCUT-19(e)'s own default configuration. Phase 6a's Task 8 owns the Cursor - # widening that closes it, at which point #open_span is the one method that changes. Until - # then the span is named by the request's method token, which is also what the tracer - # factory is asked for: the operation identifier is 4a's RequestContext#operation_name, out - # of reach for the same reason. + # once here, in .build, never per request (OBS-31). The reconciled precedence's three + # clauses -- "the request context's bundle when it is not Bundle::NONE, else the step's + # constructor keyword, else the constant" -- are honoured for the TRACER FACTORY since phase + # 6a's Task 8 widened Cursor with #bundle: #open_span reads the cursor's bundle and takes its + # tracer factory when the bundle is not Bundle::NONE, else this step's keyword (which is the + # constant when nothing was passed), and Tracing.correlate is handed the same bundle, so a + # populated one pushes its trace and span ids onto the diagnostic context (OBS-23). The + # meter has no bundle source: CTX-14's bundle carries no meter and the two instruments are + # created once at .build (OBS-31), so the meter's precedence is the keyword, then the + # constant, and that is the whole of it. A call that seeds no bundle resolves exactly as + # phase 5 did -- the keyword and Bundle::NONE, OBS-34's and XCUT-19(e)'s own default + # configuration. The span is still named by the request's method token, which is also what + # the tracer factory is asked for: the operation identifier is 4a's + # RequestContext#operation_name and the cursor's bundle carries no name (P6-51; + # docs/first-release.md's behavioural-asymmetries entry). # # The level guard is read once at the top of #call and consulted at three sites -- the two # emissions and the body wrapping -- and a step that re-read it inside a helper would be one @@ -124,8 +129,9 @@ def stage # @return [Dexpace::Response] the response, its body wrapped at the body level def call(request, cursor) started = @clock.monotonic - span = open_span(request) - scope = Tracing.correlate(span, Bundle::NONE) + bundle = cursor.bundle + span = open_span(request, bundle) + scope = Tracing.correlate(span, bundle) begin request = prepare(request) response = cursor.call(request) @@ -144,11 +150,13 @@ def call(request, cursor) private - # 4a's #tracer, then 5c's #start_span, both named by the method token until phase 6a's - # Task 8 makes the operation name reachable (see the class comment). - def open_span(request) + # 4a's #tracer, then 5c's #start_span, both named by the method token (see the class + # comment). The factory is the cursor's bundle's when the call seeded one, else this + # step's own: the reconciled precedence's first clause, live since phase 6a's Task 8. + def open_span(request, bundle) name = request.method.to_s - @tracer_factory.tracer(name: name).start_span(name) + factory = bundle.equal?(Bundle::NONE) ? @tracer_factory : bundle.tracer_factory + factory.tracer(name: name).start_span(name) end # 5c's #finish and the two instruments, unwrapped. `attributes:` is optional on both diff --git a/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb b/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb index cf3e1fd..2fc2ae2 100644 --- a/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb +++ b/gems/dexpace-core/lib/dexpace/instrumentation/tracing.rb @@ -80,11 +80,12 @@ def with_span(span) # the push is skipped and activation delegates to plain current-span activation, in the # requirement's own words. And for a bundle that is not #valid? it likewise delegates: an # all-zero id "MUST be treated as invalid/no-trace" (OBS-26), and a recording span paired - # with Bundle::NONE is the only state 5b's step can reach in phase 5 (R11) -- without the - # guard every log event of a tracing-enabled client would carry trace.id=<32 zeros>, a - # fake trace and worse than an absent key. Bundle#valid? is 4a's derived predicate and is - # exactly this test; once phase 6a's Task 8 makes a populated bundle reachable, the guard - # stops firing. NO_SCOPE when the span and both key values are already in place. + # with Bundle::NONE was the only state 5b's step could reach in phase 5 (R11) -- without + # the guard every log event of a tracing-enabled client would carry trace.id=<32 zeros>, + # a fake trace and worse than an absent key. Bundle#valid? is 4a's derived predicate and + # is exactly this test; since phase 6a's Task 8 made a populated bundle reachable through + # Cursor#bundle, the guard fires only for a call that seeded none. NO_SCOPE when the span + # and both key values are already in place. # # @param span [Object] the _Span to make current # @param bundle [Bundle] the correlation bundle whose ids are pushed diff --git a/gems/dexpace-core/lib/dexpace/pipeline.rb b/gems/dexpace-core/lib/dexpace/pipeline.rb index 9b473c8..6aee352 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline.rb @@ -4,6 +4,7 @@ require_relative "closeable" require_relative "http/request_options" require_relative "cancellation" +require_relative "instrumentation/bundle" module Dexpace # The synchronous stage-based execution pipeline (design §5.1): sixteen totally ordered stages @@ -82,15 +83,22 @@ def initialize(entries:, transport:, driver_class:) # holds with no cursor allocated at all. There is no reading under which a NON-empty pipeline # may skip the cursor (R12). # + # `bundle:` is phase 6a's widening (Task 8): the per-call instrumentation bundle the cursor + # carries to every step (Cursor#bundle), Bundle::NONE when omitted. An optional keyword + # beside the transport SPI's three positionals, which Transport.conforms? still accepts, so + # a Pipeline stays a Dexpace::Transport by the duck type (P4-38's reasoning, widened, P6-51). + # # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] the caller's per-call options, threaded unchanged # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call correlation bundle # @return [Dexpace::Response] - def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none) + def call(request, options = RequestOptions::EMPTY, cancellation = Cancellation.none, + bundle: Instrumentation::Bundle::NONE) return @transport.call(request, options, cancellation) if @entries.empty? Cursor.build(drive: @driver_class.new(self), request: request, options: options, - cancellation: cancellation,).call(request) + cancellation: cancellation, bundle: bundle,).call(request) end end end diff --git a/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb b/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb index a807aa2..baee4cf 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/async_driver.rb @@ -44,14 +44,14 @@ def entry_at(index) = @pipeline.entries.fetch(index) # PIPE-13 on the async side: the step's future, or the transport's, or a future failed with # whatever the invocation raised (PIPE-30). - def advance(position:, request:, options:, cancellation:, state:) + def advance(position:, request:, options:, cancellation:, bundle:, state:) entries = @pipeline.entries return dispatch(request, options, cancellation) if position >= entries.size normalise(terminal: false) do cursor = Cursor.send(:new, drive: self, owner_index: position, position: position + 1, request: request, options: options, - cancellation: cancellation, state: state,) + cancellation: cancellation, bundle: bundle, state: state,) entries.fetch(position).step.call(request, cursor) end end diff --git a/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb b/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb index 290da69..7813661 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/cursor.rb @@ -4,6 +4,7 @@ require_relative "stage" require_relative "../error/pipeline_error" require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/bundle" module Dexpace class Pipeline @@ -35,6 +36,18 @@ class Pipeline # Per-call state lives HERE and is passed as an argument, never read from ambient storage # (PIPE-11): not Fiber[], not Thread.current[]. A step that spawns a thread or a fiber hands # it the cursor explicitly; nothing is inherited. + # + # #bundle is phase 6a's widening (its Task 8): the per-call instrumentation bundle CTX-14 + # names, seeded by the one optional `bundle:` keyword on Pipeline#call and AsyncPipeline#call, + # carried across #fork exactly as #options is, and Bundle::NONE when nothing seeded it. It is + # what lets 5b's instrumentation step honour the reconciled precedence's first clause -- the + # request's own bundle before the step's constructor keyword -- which had no implementation + # path in phase 5 (4c consumed nothing of 4a; Request carries no bundle; PIPE-11 forbids + # ambient carriage). A read-only member and a widening under NFR-4: no existing signature + # moved, and a caller who passes nothing gets exactly what phase 5 gave it. It is NOT the + # retry step's HTTP-tracer source: OBS-29's per-operation tracer is a different kind of + # object from Bundle#tracer_factory (P6-7), and the retry drivers take their own factory, + # called with the cursor itself. class Cursor # One shared frozen hash for a stage nothing wrote, and one for the root cursor's map, so # neither the empty read nor the root allocates (verified fact 4). @@ -54,6 +67,10 @@ class Cursor # @return [Dexpace::Cancellation] the token a step checks at every resume point and the # terminal dispatch is handed attr_reader :cancellation + # @return [Dexpace::Instrumentation::Bundle] the per-call correlation bundle, the same + # frozen object at every position and across every fork; Bundle::NONE when the call + # seeded none (CTX-14, phase 6a's Task 8) + attr_reader :bundle # The root cursor for one send: bound to no entry, at position 0, with empty state. There is # no owner_index:, position: or state: keyword here and a caller cannot name one -- those @@ -63,19 +80,29 @@ class Cursor # @param request [Dexpace::Request] # @param options [Dexpace::RequestOptions] # @param cancellation [Dexpace::Cancellation] + # @param bundle [Dexpace::Instrumentation::Bundle] the per-call bundle; NONE by default # @return [Dexpace::Pipeline::Cursor] - def self.build(drive:, request:, options:, cancellation:) + # @raise [Dexpace::InvalidArgumentError] for a bundle: that is not a Bundle + def self.build(drive:, request:, options:, cancellation:, + bundle: Instrumentation::Bundle::NONE) + unless bundle.is_a?(Instrumentation::Bundle) + raise InvalidArgumentError, + "bundle: takes an Instrumentation::Bundle, got #{bundle.class}" + end + new(drive: drive, owner_index: -1, position: 0, request: request, options: options, - cancellation: cancellation, state: EMPTY_STATE,) + cancellation: cancellation, bundle: bundle, state: EMPTY_STATE,) end - def initialize(drive:, owner_index:, position:, request:, options:, cancellation:, state:) + def initialize(drive:, owner_index:, position:, request:, options:, cancellation:, bundle:, + state:) @drive = drive @owner_index = owner_index @position = position @request = request @options = options @cancellation = cancellation + @bundle = bundle @state = state @spent = false end @@ -125,7 +152,7 @@ def call(request = @request) @spent = true @drive.advance(position: @position, request: request, options: @options, - cancellation: @cancellation, state: @state,) + cancellation: @cancellation, bundle: @bundle, state: @state,) end # PIPE-15, PIPE-16. A fresh cursor at the SAME position as this one, carrying the current @@ -172,7 +199,7 @@ def fork(state: nil) Cursor.send(:new, drive: @drive, owner_index: @owner_index, position: @position, request: @request, options: @options, cancellation: @cancellation, - state: merged_state(stage, state),) + bundle: @bundle, state: merged_state(stage, state),) end private diff --git a/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb b/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb index 54e6228..4ea2b2e 100644 --- a/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb +++ b/gems/dexpace-core/lib/dexpace/pipeline/sync_driver.rb @@ -25,7 +25,7 @@ def entry_at(index) = @pipeline.entries.fetch(index) # PIPE-13: invoke the entry at `position` with a cursor bound to it, or -- past the last # entry -- dispatch to the terminal transport with the in-flight request, the caller's # options and the token (PIPE-17). - def advance(position:, request:, options:, cancellation:, state:) + def advance(position:, request:, options:, cancellation:, bundle:, state:) entries = @pipeline.entries return @pipeline.transport.call(request, options, cancellation) if position >= entries.size @@ -33,7 +33,7 @@ def advance(position:, request:, options:, cancellation:, state:) # entry it is invoking, at the position after it). cursor = Cursor.send(:new, drive: self, owner_index: position, position: position + 1, request: request, options: options, cancellation: cancellation, - state: state,) + bundle: bundle, state: state,) entries.fetch(position).step.call(request, cursor) end end diff --git a/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb new file mode 100644 index 0000000..6638af0 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/async_retry_step.rb @@ -0,0 +1,288 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "retry_step_helpers" +require_relative "../async/completer" +require_relative "../async/future" +require_relative "../async/delay" +require_relative "../error/cancelled_error" +require_relative "../error/seam_error" +require_relative "../pipeline/stages" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The stage-based retry step, asynchronous half: the same policy, the same decision and the + # same delay resolution as RetryStep (one RetryStepHelpers, so the two cannot drift, design + # §11.12) over 4c's _AsyncStep contract -- #call returns a Dexpace::Async::Future and every + # downstream drive is a fork's future. Declares Stages::RETRY, forks for every drive + # (pipeline/86343352) and writes no cursor state. + # + # RETRY-30: an ITERATIVE trampoline, never a recursive one. Each #call allocates one private + # Pump holding the per-call state (RETRY-42, RECOV-28: nothing on the step). The pump runs + # attempts in a `loop` on whatever frame is driving it; a downstream future or a delay + # future that settles INLINE -- which every scripted transport does, and which Async.delay + # does for a zero-length delay (RETRY-31) -- does not call back into a nested attempt but + # flips a RE-ARM flag under the pump's mutex, and the loop already running picks the next + # attempt up when the current one returns. Only a future that settles LATER, on another + # thread or a scheduler fiber, starts the loop again from its callback, on a fresh stack. + # Measured: the plan's recursive shape overflowed at ~1500 attempts on every interpreter; + # this one is flat at 2000 (async_retry_step_test.rb asserts the depth, not just the count). + # + # R2's route, as built: a positive delay goes through 5a's Async.delay, which unmounts the + # fiber under a registered Fiber.scheduler and RAISES Dexpace::SeamError synchronously with + # none -- the raise is caught by RETRY-33's fence and fails the returned future, carrying the + # prior trail; a zero-length delay completes inline and re-arms the pump without a frame; and + # nothing here installs, reads or shuts down a scheduler (RETRY-45). Never Clock#sleep: a + # blocking wait on the async path is what RETRY-31 and RETRY-26 forbid. + # + # RETRY-33: every callback body runs inside one fence that closes any open retryable + # response and fails the completer on a StandardError -- a throwing predicate, delay + # computation, log call, tracer callback or scheduler rejection each completes the future + # exceptionally rather than escaping into whoever settled the downstream future and leaving + # the returned future hanging. The fatal family is re-raised after the completer is failed + # (RETRY-25), because a hung future is not an acceptable price for propagating it. + # + # RETRY-32: once the returned future is settled or cancelled -- by the caller, by a deadline + # -- no further attempt is launched, a pending delay is cancelled, and a response arriving + # from an attempt still in flight is closed rather than leaked. RETRY-23 / RECOV-27: the + # cursor's token is checked at the top of every attempt and, on cancellation during a wait, + # cancels the pending delay and fails the future with the CancelledError, the trail attached. + class AsyncRetryStep + include RetryStepHelpers + + private_class_method :new + + # Builds a frozen step; the keywords are RetryStep.build's, with the same meanings. + # + # @param settings [RetrySettings] + # @param http_tracer_factory [#call] called once per operation with the cursor + # @param delay_override [#call, nil] RETRY-39's first tier + # @param should_retry [#call, nil] the caller's condition predicate + # @param logger [Instrumentation::Logger] + # @return [AsyncRetryStep] frozen + # @raise [Dexpace::InvalidArgumentError] as RetryStep.build + def self.build(settings: RetrySettings.build, http_tracer_factory: NULL_TRACER_FACTORY, + delay_override: nil, should_retry: nil, + logger: Instrumentation::Logger::NULL) + new(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, logger: logger,).freeze + end + + NULL_TRACER_FACTORY = ->(_cursor) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + end + + # 4c's optional declaration: the Stages::RETRY pillar. + # + # @return [Dexpace::Pipeline::Stage] + def stage + Pipeline::Stages::RETRY + end + + # One operation, as a future that settles with the terminal response or fails with the + # terminal throwable (its trail attached), or with the CancelledError of a cancelled token. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt (RETRY-44) + # @param cursor [Dexpace::Pipeline::Cursor] forked for every drive + # @return [Dexpace::Async::Future] + def call(request, cursor) + Pump.new(settings: @settings, http_tracer_factory: @http_tracer_factory, + delay_override: @delay_override, should_retry: @should_retry, logger: @logger, + request: request, cursor: cursor,).start + end + + # The per-call state and the trampoline (RETRY-30). One instance per #call, carrying the + # step's five configuration values and the same helpers, reachable from nothing but the + # callbacks it registers; a private_constant with a sig/ mirror. Over the class-length + # default by fifteen lines, and deliberately one class: the re-arm protocol, the fence and + # the terminal paths share the completer and the flags, and splitting them across objects + # would scatter the one invariant the trampoline rests on (the recorded exception + # .rubocop.yml's metric note asks for). + class Pump # rubocop:disable Metrics/ClassLength -- one per-call state machine, see above + include RetryStepHelpers + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:, + request:, cursor:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + @request = request + @cursor = cursor + @completer = Dexpace::Async::Completer.new + @tracer = Instrumentation::NULL + @max_retries = 0 + @attempt = 1 + @trail = [] + @mutex = ::Thread::Mutex.new + @running = false + @rearm = false + @pending_delay = nil + end + + # Arms the cancellation bridges, runs the first attempt, and returns the future. + def start + guarded(nil) do + @tracer = @http_tracer_factory.call(@cursor) + @max_retries = effective_max_retries(@cursor) + @completer.on_cancel { @pending_delay&.cancel } + subscription = @cursor.cancellation.on_cancel { |reason| abort(reason) } + @completer.future.on_settle { subscription.detach } + resume + end + @completer.future + end + + private + + # The re-arm protocol (RETRY-30), called from the frame that wants the next attempt to + # run. When a pump loop is already running on some frame, set the flag and return -- + # that loop picks the attempt up when its current one returns; otherwise become that + # loop, and keep launching until nothing re-armed it. The flags flip under the mutex + # only, never across an attempt, so a settlement from another thread cannot lose its + # re-arm between the loop's last check and its exit. + def resume + return unless claim(:start) + + loop do + launch + break unless claim(:continue) + end + end + + # :start claims the loop when none runs (else re-arms and answers false); :continue + # consumes a re-arm (else releases the loop and answers false). + def claim(step) + @mutex.synchronize do + if step == :start && @running + @rearm = true + false + elsif step == :start + @running = true + elsif @rearm + @rearm = false + true + else + @running = false + end + end + end + + # One attempt: the token check, the tracer, a fresh fork, and the settlement callback -- + # which runs inline when the downstream settled synchronously. + def launch + return if @completer.settled? + + guarded(nil) do + @cursor.cancellation.check! + @tracer.attempt_started(@cursor, @attempt) + @cursor.fork.call(@request).on_settle { |settlement| settled(settlement) } + end + end + + # A downstream settlement: RETRY-32's abandoned-attempt close first, then a terminal + # success or a terminal fatal, then the decision. + def settled(settlement) + response = settlement.response + guarded(response) do + next Dexpace.close_quietly(response, logger: @logger) if @completer.settled? + next if delivered?(response, settlement.error) + + failure = failure_of(response, settlement.error) + verdict = decision(@request, failure, @attempt, @max_retries) + next finish(response, failure, verdict) unless verdict == RETRY + + retry_after(response, settlement.error, failure) + end + end + + # The two settlements that need no decision: a non-error response is delivered, and a + # fatal-family error -- which on this path arrives as a settlement and meets no rescue + # arm -- is delivered unclassified, unretried and with no trail attached (RETRY-25), + # rather than reaching the capability query a lying subclass could answer. + def delivered?(response, error) + if error.nil? + return false if response.nil? || response.status.error? + + @completer.fulfil(response) + else + return false if error.is_a?(::StandardError) + + @completer.fail(error) + end + true + end + + # The retry path: the delay from the still-open response, the tracer, the trail, the + # close (RETRY-35), then the wait. + def retry_after(response, error, failure) + delay = resolve_delay(@attempt, response, error) + @tracer.attempt_failed(@cursor, failure, delay) + @trail << failure + Dexpace.close_quietly(response, onto: failure) + wait(delay) + end + + # The terminal path, mirroring RetryStep#settle: a response is delivered as it is, a + # throwable with the trail attached, retries_exhausted first when the budget stopped it. + def finish(response, failure, verdict) + return @completer.fulfil(response) if response && verdict == STOP + + attach_trail(failure, @trail) + @tracer.retries_exhausted(@cursor, failure) if verdict == EXHAUSTED + response ? @completer.fulfil(response) : @completer.fail(failure) + end + + # RETRY-31 / R2: the wait is Async.delay's future. Zero-length settles inline, so the + # callback runs now and re-arms the running loop; positive under a scheduler settles on + # the scheduler's fiber, whose callback starts a fresh loop; positive with no scheduler + # raises SeamError here, synchronously, into the fence around this method's caller. + def wait(delay) + @attempt += 1 + future = Dexpace::Async.delay(delay) + @pending_delay = future + future.on_settle do |elapsed| + @pending_delay = nil + guarded(nil) do + if elapsed.error + @completer.fail(attach_trail(elapsed.error, @trail)) + elsif !@completer.settled? # RETRY-32 + resume + end + end + end + end + + # RETRY-23 / RECOV-27 during a wait: cancel the pending timer and surface the + # cancellation; a no-op once the future is settled. + def abort(reason) + return if @completer.settled? + + @pending_delay&.cancel(reason) + @completer.fail(attach_trail(Dexpace::CancelledError.new(reason), @trail)) + end + + # RETRY-33's fence, and RETRY-25's: close the open response, fail the future, and let a + # fatal-family error propagate after the future is settled. The trail travels with the + # failure the fence reports. + def guarded(response) + yield + rescue ::StandardError => error + Dexpace.close_quietly(response, onto: error) + @completer.fail(attach_trail(error, @trail)) + rescue ::Exception => error # rubocop:disable Lint/RescueException -- RETRY-25 with RETRY-33: fail, then re-raise unchanged + Dexpace.close_quietly(response, logger: @logger) + @completer.fail(error) + raise + end + end + private_constant :Pump + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb new file mode 100644 index 0000000..ef22eb1 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/pacing_parsers.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../http_date" +require_relative "../error/invalid_argument_error" + +module Dexpace + module Resilience + # RETRY-15, RETRY-17, RETRY-19, RECOV-24, RECOV-25: the per-form pacing-header value parsers, + # one method per wire form, each TOTAL over a String -- nil for anything it cannot read, never + # a raise and never a zero delay it did not earn (RETRY-16, RECOV-23). Policy.pacing_delay is + # the public dispatch over them and nothing outside this file names a form directly, which is + # why this is a private_constant on 5a's ConfigParsers precedent (the plan's open question 2): + # Policy's public surface stays the methods the design names and the parsing internals carry + # no NFR-4 lock. It has a sig/ mirror because the strict `core` Steep target types its call + # sites, and no test/ mirror -- every branch is asserted through Policy.pacing_delay. + # + # Two grammars, each anchored \A..\z, each compiled once with its own timeout (never + # Regexp.timeout), and none can backtrack: bounded digit runs and one optional bounded + # fraction. RETRY-19's screen is the DECIMAL grammar, applied BEFORE String#to_f -- "30d", + # "0x10", "1e3", "5_0", "Infinity", "NaN" and " 5" all fail it and fall through to the + # HTTP-date attempt, which rejects them too, so the answer is nil. Nothing here calls Float() + # or Integer() on an unscreened value: Integer("5_0", 10) is 50 on every supported Ruby + # (verified 3.2.11, 3.4.10, 4.0.6), which is exactly the mis-parse the screen exists to + # prevent. + # + # The runs are BOUNDED (P6-61): fifteen integer digits is the largest run a Float carries + # exactly and 10**15 seconds is thirty million years, so a longer run is RETRY-16's + # out-of-range value and answers nil before anything converts it -- String#to_f and #to_i + # over an unbounded run cost seconds on a multi-megabyte value (measured: 16-20 s for a 10 MB + # retry-after-ms) and, past ~309 digits, emit Ruby's "Float ... out of range" warning, which + # NFR-6's fatal-warnings suite turns into a raise Policy#parse_form's fence was quietly + # swallowing. MAX_VALUE_BYTES bounds the HTTP-date attempt the same way, in front of every + # parser: the longest well-formed pacing value is the 29-byte RFC 1123 date, and a value past + # 64 bytes is no hint without a grammar or the date parser ever reading it. + module PacingParsers + extend self + + # The longest well-formed pacing value is an RFC 1123 date, 29 bytes; twice that, rounded + # up, is the ceiling every parser applies before any grammar runs. + MAX_VALUE_BYTES = 64 + DECIMAL_GRAMMAR = ::Regexp.new('\A\d{1,15}(\.\d{1,15})?\z', timeout: 1.0).freeze + INTEGER_GRAMMAR = ::Regexp.new('\A\d{1,15}\z', timeout: 1.0).freeze + private_constant :MAX_VALUE_BYTES, :DECIMAL_GRAMMAR, :INTEGER_GRAMMAR + + # The upper bound of RECOV-25's positive jitter: [100%, 120%] of the computed delta. + RESET_JITTER_CEILING = 1.2 + private_constant :RESET_JITTER_CEILING + + # Retry-After: delta-seconds first (integer and fractional, sub-second honoured), then an + # RFC 1123 HTTP-date through the ONE parser, whose past instants floor to zero (RETRY-17). + # An unparseable value is nil, distinct from a past date's 0.0. + def parse_retry_after(value, now:) + return nil unless readable?(value) + return value.to_f if DECIMAL_GRAMMAR.match?(value) + + instant = Dexpace::HTTPDate.parse(value) + [(instant - now).to_f, 0.0].max + rescue Dexpace::InvalidArgumentError + nil + end + + # retry-after-ms / x-ms-retry-after-ms: a non-negative integer count of milliseconds. + def parse_millis(value) + return nil unless readable?(value) && INTEGER_GRAMMAR.match?(value) + + value.to_i / 1000.0 + end + + # X-RateLimit-Reset: Unix epoch seconds. A reset already past floors to zero (RETRY-17); a + # future one is jittered UPWARD to [100%, 120%] of the delta (RECOV-25), inside the parser, + # so the caller applies no second jitter on top (RETRY-20, RECOV-22). The jitter is drawn + # only over a finite range: the bounded run keeps every delta finite, and the `finite?` + # guards stay as the belt behind the grammar -- Random#rand raises EDOM on an infinite + # bound (verified on 4.0.6), and RECOV-26's "never surface an arithmetic overflow + # mid-retry" covers a server's hint before any clamp reaches it. + def parse_epoch_reset(value, now:, random:) + return nil unless readable?(value) && INTEGER_GRAMMAR.match?(value) + + delta = [(value.to_i - now.to_i).to_f, 0.0].max + return nil unless delta.finite? + return 0.0 if delta.zero? + + ceiling = delta * RESET_JITTER_CEILING + ceiling.finite? ? random.rand(delta..ceiling) : delta + end + + private + + # A String within the byte ceiling; anything else is no hint before a grammar runs. + def readable?(value) + value.is_a?(::String) && value.bytesize <= MAX_VALUE_BYTES + end + end + private_constant :PacingParsers + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/policy.rb b/gems/dexpace-core/lib/dexpace/resilience/policy.rb new file mode 100644 index 0000000..5e28f98 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/policy.rb @@ -0,0 +1,298 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../each_cause" +require_relative "../error/cancelled_error" +require_relative "../error/invalid_argument_error" +require_relative "../error/protocol_error" +require_relative "../instrumentation/contain" +require_relative "../instrumentation/keys" +require_relative "../instrumentation/logger" +require_relative "pacing_parsers" + +module Dexpace + # The resilience layer: phase 6's namespace, design §6.1's. Retry is 6a's; redirect (6b) and + # authentication (6c) add their own constants beside these. + module Resilience + # The ONE shared retry policy core (RETRY-13, RECOV-30): the tuning constants, the backoff + # calculator, the pacing-header dispatcher, the two-branch retryability consult and the + # effective-retry-count resolver. Both retry stacks -- RecoveryRetry beneath phase 4b's + # orchestrator and RetryStep/AsyncRetryStep at Stages::RETRY -- call these exact methods and + # carry no formula or constant of their own; a text scan over lib/dexpace/resilience/ for a + # literal delay constant outside this file is the mechanised form of that claim. + # + # A module with no instance side, frozen constants and pure functions, so RETRY-42's + # "immutable and stateless after construction and safe for concurrent invocation" is + # structural: nothing here has state to share. `extend self` and never module_function + # (Style/ModuleFunction), on 5a's HTTPDate and Retryability precedent. + # + # Public (R5): a generated SDK assembling its own retry driver needs to call the SAME + # calculator, and XCUT-6's capability query is the extension point a third-party transport + # uses by defining #retryable? on its own error class -- both only work if the consulting + # code is public and stable. It coexists with 5a's Dexpace::Retryability without touching + # it: that module is XCUT-5's fixed baked classifier, read once by ProtocolError at + # construction; .retry_eligible? below is XCUT-7's CONFIGURABLE set, consulted by the retry + # drivers, and neither reads the other (XCUT-5's closing NOTE, P6-10). + # + # Two budget policies, one calculator (R6, P6-5): .backoff_delay takes no total-timeout + # parameter at all, and RECOV-20's budget is the separate .budget_remaining that only + # RecoveryRetry calls. RETRY-28's prohibition on the stage stack is then a fact about which + # files name which method, checkable by a text scan, rather than a keyword every future edit + # must remember not to pass. + module Policy + extend self + + # RETRY-12's first default: the wait before the first retry, 200 ms in seconds. The five + # defaults are one schedule -- 200 ms, x2, 8 s, 0.2 and three sends -- and every one is + # read from here by RetrySettings and re-typed nowhere (RETRY-13). + DEFAULT_INITIAL_DELAY = 0.2 + # RETRY-12: the exponential factor between successive waits. + DEFAULT_MULTIPLIER = 2.0 + # RETRY-12: the cap on any one wait, 8 s in seconds. + DEFAULT_MAX_DELAY = 8.0 + # RETRY-12: the symmetric jitter fraction, a fifth of the delay either way. + DEFAULT_JITTER = 0.2 + # RETRY-12's budget of three sends, spelled in the STAGE vocabulary as two retries after + # the initial send; the recovery stack's max-attempts is this plus one (RETRY-14, P6-6). + DEFAULT_MAX_RETRIES = 2 + + # XCUT-7's configurable retryable-status set, at its default: a SUBSET of Retryability's + # baked classification (which also admits 500-599 less 501 and 505 in full), enumerable + # because it is data and not a rule. + DEFAULT_RETRYABLE_STATUSES = ::Set[408, 429, 500, 502, 503, 504].freeze + + # RETRY-21's fixed recovery-stack precedence, and the stage stack's default when a caller + # configures no order of its own: Retry-After (numeric, then date), the two millisecond + # variants, then the epoch reset. + DEFAULT_PACING_HEADER_ORDER = %w[ + Retry-After retry-after-ms x-ms-retry-after-ms X-RateLimit-Reset + ].freeze + + # RETRY-18 / RECOV-26: the finite ceiling every pacing delta is clamped to, 365 days in + # seconds, applied before any nanosecond conversion could overflow. + MAX_PACING_DELAY_SECONDS = 365 * 24 * 60 * 60 + + # RECOV-34's representability ceiling: the largest signed 64-bit nanosecond count, ~292 + # years, which design §10.18 substitutes for the reference's Duration bound. + MAX_DURATION_NANOSECONDS = (1 << 63) - 1 + + # Sub-nanosecond: RETRY-10's degenerate jitter range, below which the base delay is + # returned unperturbed. + NANOSECOND = 1e-9 + private_constant :NANOSECOND + + # RETRY-37 / XCUT-7 / RECOV-17: whether `status` is in the CONFIGURED retryable-status + # set, consulted alone -- authoritative-contains, never AND-ed with XCUT-5's baked flag. + # There is no baked-flag parameter on this method by design, so the intersection RETRY-37 + # forbids is a method that does not exist. A set that NARROWS narrows: 500 is retryable + # under Retryability and refused here when the set omits it. + # + # @param status [Integer] the response status code + # @param set [Set] the configured retryable-status set + # @return [Boolean] + def retry_eligible?(status, set:) + set.include?(status) + end + + # RETRY-2 / XCUT-6 / CFG-35's throwable half: the retryable-throwable set, defined here and + # nowhere else, as the CAPABILITY query -- `respond_to?(:retryable?) && retryable?` -- + # walked over the error and its whole cause chain through Dexpace.each_cause, which is + # already cycle-safe by reference identity (XCUT-9), so RETRY-2's termination clause needs + # no second walk. Never a concrete-type match: `is_a?(::IOError)` is wrong in both + # directions (5a's R1) -- it would call phase 3a's StreamError retryable and a bare + # Errno::ETIMEDOUT not. The residual that comes with the capability form is P6-4: a bare, + # unwrapped stdlib I/O or timeout error answers no capability and classifies not + # retryable, which is why phase 8's adapters must wrap what they let escape. + # + # @param error [Exception] + # @return [Boolean] + def throwable_retryable?(error) + Dexpace.each_cause(error).any? do |cause| + # The capability is a duck type the signature cannot name; probed untyped, as 5a's + # Retryability probes #code. + probe = cause #: untyped + probe.respond_to?(:retryable?) && probe.retryable? + end + end + + # RETRY-23 / RECOV-27: whether `error` is a cancellation -- a Dexpace::CancelledError + # itself, or one anywhere in its cause chain, so a transport that wrapped the token's raise + # in an error of its own (one that may well answer #retryable?) is still read as the + # cancellation it carries. Consulted BEFORE either branch of .retryable? and before a + # caller's should_retry predicate on the stage drivers, so "cancellation MUST never be + # treated as a retryable failure" holds whatever a capability or a predicate answers + # (P6-60). The walk is Dexpace.each_cause, cycle-safe by identity (XCUT-9). + # + # @param error [Exception] + # @return [Boolean] + def cancellation?(error) + Dexpace.each_cause(error).any?(Dexpace::CancelledError) + end + + # The one dispatch point both stacks call: a cancellation is never retryable (RETRY-23), + # then a ProtocolError is classified by its status against the configured set (RETRY-37), + # anything else by the capability query (RETRY-2). A ProtocolError deliberately does not + # answer #retryable? (P6-10), so a wrapped one buried in a cause chain can never smuggle + # the baked set past the configured one. + # + # @param error [Exception] + # @param retryable_statuses [Set] the configured set + # @return [Boolean] + def retryable?(error, retryable_statuses:) + return false if cancellation?(error) + + if error.is_a?(Dexpace::ProtocolError) + retry_eligible?(error.status.code, set: retryable_statuses) + else + throwable_retryable?(error) + end + end + + # RETRY-9, RETRY-10, RETRY-11, RECOV-21: `initial_delay * multiplier**(attempt - 1)`, + # capped at `max_delay`, then symmetric jitter drawn uniformly from + # [d * (1 - j/2), d * (1 + j/2)] -- midpoint d, j = 0 returning d exactly, a degenerate + # sub-nanosecond spread returning d, a negative sample floored to zero. `attempt` is + # 1-based, 1 being the wait before the first retry, and anything below is refused. + # + # Overflow-safe by Ruby's own float arithmetic plus ONE guard: `2.0**9999` is Infinity + # (never a raise, verified on 3.2.11, 3.4.10 and 4.0.6) and `[Infinity, max_delay].min` is + # `max_delay`, so a large attempt saturates to the cap -- but `0.0 * Infinity` is NaN, and + # `[NaN, 8.0].min` RAISES ("comparison of Float with NaN failed"), so a zero initial delay + # answers zero without touching the power (found by the 2000-attempt trampoline test; + # P6-53). No budget parameter: the total-timeout is RecoveryRetry's alone, applied through + # .budget_remaining (R6). + # + # @param attempt [Integer] 1-based + # @param initial_delay [Float] seconds + # @param multiplier [Float] >= 1.0 + # @param max_delay [Float] seconds, the cap + # @param jitter [Float] in [0.0, 1.0] + # @param random [#rand] the generator the jitter sample is drawn from + # @return [Float] seconds + # @raise [Dexpace::InvalidArgumentError] for an attempt below 1 + def backoff_delay(attempt, initial_delay:, multiplier:, max_delay:, jitter:, random:) + unless attempt.is_a?(::Integer) && attempt >= 1 + raise InvalidArgumentError, "attempt must be an Integer >= 1, got #{attempt.inspect}" + end + + initial = seconds(initial_delay) + return 0.0 if initial.zero? + + raw = initial * (seconds(multiplier)**(attempt - 1)) + jittered([raw, seconds(max_delay)].min, jitter, random) + end + + # RETRY-15 through RETRY-22, RECOV-22 through RECOV-26, RECOV-29: the total pacing + # dispatch. Walks `header_order` -- the recovery stack's fixed precedence or the stage + # stack's caller-configurable list (RETRY-21) -- and returns the first usable value, + # clamped to MAX_PACING_DELAY_SECONDS, or nil for no hint. Each form's parser is total and + # every parser call is fenced besides, so a malformed value in one header falls through to + # the next rather than aborting the scan, and nothing here ever raises: RETRY-22's "a + # failure while parsing a pacing header MUST NOT mask the real upstream failure" is honoured + # one level up, by the caller falling back to .backoff_delay on nil, and one level down, + # by no single value poisoning the scan. A result that is not finite is no hint (RETRY-16's + # out-of-range clause); a finite one past the ceiling is the ceiling (RETRY-18). + # + # @param headers [Dexpace::Headers] the response's headers + # @param header_order [Array] the names to consult, in precedence order + # @param now [Time] the instant absolute forms are measured from; the driver passes its + # clock's so a fake clock drives the arithmetic + # @param random [#rand] the generator RECOV-25's reset jitter draws from + # @return [Float, nil] seconds, or nil for no hint + def pacing_delay(headers, header_order:, now: ::Time.now, random: ::Random) + header_order.each do |name| + value = headers[name]&.first + next if value.nil? + + delay = parse_form(name, value, now, random) + next unless delay.is_a?(::Float) && delay.finite? && delay >= 0.0 + + return [delay, MAX_PACING_DELAY_SECONDS.to_f].min + end + nil + end + + # RETRY-41: the stage stack's effective retry count -- a present per-call override wins + # (validated non-negative), else the configured value, with a negative configured value + # clamped to DEFAULT_MAX_RETRIES and the clamp logged; zero means no retries. The log is a + # WARNING `http.instrumentation.config` diagnostic through 5b's facade, contained (OBS-20) + # so a raising sink cannot fail the request, and Logger::NULL by default so the + # non-clamping path allocates nothing and touches no sink (P6-12). + # + # @param override [Integer, nil] the per-call RequestOptions#max_retries + # @param configured [Integer] the settings' max_retries + # @param logger [Dexpace::Instrumentation::Logger] where the clamp is reported + # @return [Integer] >= 0 + # @raise [Dexpace::InvalidArgumentError] for a negative override + def effective_max_retries(override:, configured:, logger: Instrumentation::Logger::NULL) + unless override.nil? + unless override.is_a?(::Integer) && override >= 0 + raise InvalidArgumentError, "max_retries override must be a non-negative Integer" + end + + return override + end + return configured if configured >= 0 + + Instrumentation.diagnostic( + logger, event: Instrumentation::Events::INSTRUMENTATION_CONFIG, + message: "max_retries #{configured} is negative and was clamped to " \ + "#{DEFAULT_MAX_RETRIES} (RETRY-41)", + ) + DEFAULT_MAX_RETRIES + end + + # RECOV-20 / RETRY-27: the recovery stack's budget, as the time REMAINING -- never the + # elapsed figure -- so every caller compares one number against a delay: Float::INFINITY + # when the budget is zero ("unbounded, deadline disabled"), else `total_timeout - elapsed`, + # which is negative once the budget is spent. Recovery-driver only: RetryStep and + # AsyncRetryStep never call this, which is what makes RETRY-28 a property of the source + # (R6, P6-5). + # + # @param elapsed [Float] seconds since the engine's own start instant + # @param total_timeout [Numeric] seconds; zero disables the budget + # @return [Float] seconds remaining, possibly negative, or Float::INFINITY + def budget_remaining(elapsed:, total_timeout:) + return ::Float::INFINITY if total_timeout.zero? + + seconds(total_timeout) - seconds(elapsed) + end + + private + + # A Numeric as a Float. rbs's Numeric declares no #to_f (its subclasses do), and Kernel#Float + # is typed as answering nil, so the conversion reads the value untyped -- the same spelling + # 5a's Clock::Guard uses. + def seconds(value) + number = value #: untyped + number.to_f + end + + # RETRY-10's symmetric draw over [d(1 - j/2), d(1 + j/2)]: d itself for j = 0 or a + # sub-nanosecond spread, and a negative sample floored at zero. + def jittered(unjittered, jitter, random) + return unjittered if jitter.zero? + + spread = unjittered * jitter + return unjittered if spread < NANOSECOND + + half = spread / 2.0 + [random.rand((unjittered - half)..(unjittered + half)), 0.0].max + end + + # The per-form dispatch, fenced: a parser is total by contract and this rescue is the + # belt behind it. Header names are matched case-insensitively with a bare #downcase + # (HTTP-13); Headers#[] already folded the lookup. + def parse_form(name, value, now, random) + case name.downcase + when "retry-after" then PacingParsers.parse_retry_after(value, now: now) + when "retry-after-ms", "x-ms-retry-after-ms" then PacingParsers.parse_millis(value) + when "x-ratelimit-reset" + PacingParsers.parse_epoch_reset(value, now: now, random: random) + end + rescue ::StandardError + nil + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb b/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb new file mode 100644 index 0000000..b9a4d14 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/recovery_retry.rb @@ -0,0 +1,236 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "policy" +require_relative "resend" +require_relative "retry_settings" +require_relative "../recovery" +require_relative "../registry" +require_relative "../error/protocol_error" +require_relative "../error/cancelled_error" +require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The recovery-chain retry: chapter 9's FIRST stack, RECOV-17 through RECOV-30 and RECOV-34, + # which phase 4 postponed here. Installed as Recovery::Orchestrator's own `transport:` + # argument, decorating the raw transport, and NOT as a Recovery::ResponseChain recovery + # step (P6-3): Recovery::Transform#apply(outcome) carries no request to re-send, while + # RECOV-19's "each RE-SENT attempt's response MUST be re-classified" describes an engine + # that dispatches its own re-sends and inspects each -- which only a transport-decorator + # position can do while keeping RequestChain's stamping (the idempotency key, the client + # identity) applied exactly once per exchange rather than once per attempt. It is itself a + # Dexpace::Transport by phase 2's duck type, so it composes anywhere a transport does. + # + # Sitting BELOW the orchestrator's rescue region, a transport that raises reaches this + # engine as an exception (P6-8), so a connection reset or a socket timeout -- RECOV-17's and + # RETRY-4's "always retryable" case -- is retried here rather than converted first and + # never retried; only StandardError is rescued and the fatal family propagates at the throw + # site (RETRY-25). Each attempt's response is classified through phase 4b's own two + # primitives, Recovery.buffer_error_body then ProtocolError.for_or_nil, purely to decide + # whether to keep going (RECOV-19, RETRY-36): the buffering is what releases the connection + # before the wait (RETRY-35, RECOV-16), and the BUFFERED response is what every later line + # sees -- the pacing parse, the RECOV-19 pass-through, and whatever ErrorMappingStep the + # caller configured, which runs exactly once, on the terminal response, with the caller's + # own factory (the plan's open question 4: this engine has no factory: keyword). + # + # Two terminal shapes (P6-9): a RESPONSE whose failure was never retryable -- a status + # outside the configured set, or a request that was not re-sendable -- is RETURNED, "passes + # through as Success" (RECOV-19), for the outer chain to map; a THROWABLE is raised whether + # it was retryable or not, and so is a retryable status that met a spent attempt cap or a + # spent total-timeout -- "the terminal failure's throwable MUST be surfaced" (RECOV-20) -- + # with every prior attempt's failure attached as suppressed first (RETRY-34) and, when the + # budget is what stopped a retryable failure, retries_exhausted emitted immediately before. + # + # The budget (RECOV-20, RETRY-27): a maximum-attempts cap that counts the initial send as + # attempt 1 -- RetrySettings#max_retries plus one, an identity and never a second number + # (RETRY-14, P6-6) -- AND a total-timeout applied as the time REMAINING through + # Policy.budget_remaining, zero meaning unbounded; a delay that would push the elapsed time + # past the budget is suppressed and the last failure surfaced, so no wait can overshoot + # (RECOV-21's clamp, RECOV-22's on a hint). The elapsed figure is the settings clock's + # monotonic reading, never Time.now (CFG-16). The wait is Clock#sleep with the caller's + # token (RECOV-27, CFG-15), the token is checked at every attempt boundary too, and a + # cancellation surfaces as the CancelledError it raised, which aborts the loop and is never + # itself retryable (RETRY-23). RECOV-28: the attempt count, the start instant and the trail + # are locals of one #call; nothing is written to this frozen object. + # + # OBS-29's per-attempt group goes through the tracer `http_tracer_factory:` produces once + # per operation, called with the REQUEST -- there is no cursor on this stack -- which is + # also the `context` every callback receives. retries_exhausted fires immediately before the + # terminal raise and only there (a returned response was never retryable, and a spent + # budget on a retryable failure is always a raise here); nothing emits operation_failed, + # which docs/first-release.md's behavioural-asymmetries entry records as conforming. + class RecoveryRetry + private_class_method :new + + # Builds a frozen engine over `transport`. + # + # @param transport [#call] the raw Dexpace::Transport this engine decorates + # @param settings [RetrySettings] the shared configuration; total_timeout is read here + # @param http_tracer_factory [#call] called once per operation with the request; the + # frozen no-op NULL by default + # @param logger [Instrumentation::Logger] where RETRY-41's clamp is reported, contained + # @return [RecoveryRetry] frozen + # @raise [Dexpace::InvalidArgumentError] for a transport that is not a three-positional + # callable, a settings that is not a RetrySettings, or a logger that is not a Logger + def self.build(transport:, settings: RetrySettings.build, + http_tracer_factory: NULL_TRACER_FACTORY, + logger: Instrumentation::Logger::NULL) + new(transport: transport, settings: settings, http_tracer_factory: http_tracer_factory, + logger: logger,).freeze + end + + NULL_TRACER_FACTORY = ->(_request) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(transport:, settings:, http_tracer_factory:, logger:) + unless Registry.callable?(transport, arity: 3) + raise InvalidArgumentError, + "transport: must be a Dexpace::Transport, a callable of three positionals" + end + unless settings.is_a?(RetrySettings) + raise InvalidArgumentError, "settings: takes a RetrySettings, got #{settings.class}" + end + unless http_tracer_factory.respond_to?(:call) + raise InvalidArgumentError, "http_tracer_factory: must answer #call(request)" + end + unless logger.is_a?(Instrumentation::Logger) + raise InvalidArgumentError, "logger: takes an Instrumentation::Logger" + end + + @transport = transport + @settings = settings + @http_tracer_factory = http_tracer_factory + @logger = logger + end + + # The transport SPI: one exchange, retried within the budget. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt + # @param options [Dexpace::RequestOptions] threaded to the transport unchanged + # @param cancellation [Dexpace::Cancellation] the token the wait and every boundary honour + # @return [Dexpace::Response] the terminal response -- a success, or an error status that + # was never retryable, buffered + # @raise [Exception] the terminal throwable, carrying the prior trail as suppressed + # @raise [Dexpace::CancelledError] when the token is cancelled + def call(request, options, cancellation) + run = Run.new(tracer: @http_tracer_factory.call(request), request: request, + options: options, cancellation: cancellation, max_attempts: max_attempts, + started: @settings.clock.monotonic, trail: [],) + attempt = 1 + loop do + terminal = attempt_once(run, attempt) + return terminal unless terminal.nil? + + attempt += 1 + end + end + + # The per-call locals (RECOV-28): on the stack, never on the engine. + class Run < Data.define(:tracer, :request, :options, :cancellation, :max_attempts, :started, + :trail,) + end + private_constant :Run + + RETRY = :retry + EXHAUSTED = :exhausted + STOP = :stop + private_constant :RETRY, :EXHAUSTED, :STOP + + private + + # One attempt: the boundary check, the tracer, the send, the decision, and -- on a retry + # -- the budgeted delay and the wait. Answers the terminal response, or nil for "again". + def attempt_once(run, attempt) + run.cancellation.check! + run.tracer.attempt_started(run.request, attempt) + response, error = exchange(run.request, run.options, run.cancellation) + return response if error.nil? + + verdict = decision(run.request, error, attempt, run.max_attempts) + return response if verdict == STOP && response + + surface(run, error, exhausted: verdict == EXHAUSTED) unless verdict == RETRY + wait(run, attempt, response, error) + end + + # The retry path: the delay against the budget (nil surfaces the failure, RECOV-20), the + # tracer, the trail, then the wait on the settings' clock with the caller's token. + # + # @return [nil] always: the loop's "again" + def wait(run, attempt, response, error) + delay = delay_within_budget(run, attempt, response) + surface(run, error, exhausted: true) if delay.nil? + run.tracer.attempt_failed(run.request, error, delay) + run.trail << error + @settings.clock.sleep(delay, cancellation: run.cancellation) + end + + # RETRY-14 / P6-6: the attempts cap is the stage vocabulary plus the initial send, with + # RETRY-41's clamp applied to the configured value on this stack too. + def max_attempts + Policy.effective_max_retries(override: nil, configured: @settings.max_retries, + logger: @logger,) + 1 + end + + # One send, classified: a success or a non-error status is [response, nil]; an error + # status is buffered (RECOV-16, the release before the wait) and paired with its + # ProtocolError; a raise is [nil, error]. StandardError only (RETRY-25). + def exchange(request, options, cancellation) + response = @transport.call(request, options, cancellation) + return [response, nil] unless response.status.error? + + buffered = Recovery.buffer_error_body(response) + [buffered, ProtocolError.for(buffered)] + rescue ::StandardError => error + [nil, error] + end + + # RECOV-17 and RECOV-18: the configured set for a status, the capability for a throwable + # (Policy.retryable?), AND the re-sendability gate -- then the cap. :stop is "never + # retryable", :exhausted "retryable, cap spent". + def decision(request, error, attempt, max_attempts) + return STOP unless Resend.eligible?(request) + return STOP unless Policy.retryable?(error, + retryable_statuses: @settings.retryable_statuses,) + + attempt < max_attempts ? RETRY : EXHAUSTED + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27: the delay -- the pacing hint from the buffered + # response through the fixed precedence, else the shared backoff -- against the budget + # REMAINING. nil when the budget is spent or the delay would overshoot it: RECOV-20's + # "suppressed, and the last failure surfaced unchanged". A zero budget is unbounded. + def delay_within_budget(run, attempt, response) + delay = hinted_delay(response) || Policy.backoff_delay(attempt, + **@settings.backoff_arguments,) + elapsed = @settings.clock.monotonic - run.started + remaining = Policy.budget_remaining(elapsed: elapsed, + total_timeout: @settings.total_timeout,) + return nil if remaining <= 0.0 || delay > remaining + + delay + end + + # RECOV-22 through RECOV-24, RECOV-29: the recovery stack's FIXED precedence, from the + # still-open (buffered) response; nil when there is no response or no usable hint, which is + # what makes a malformed header fall back to backoff without masking the failure. + def hinted_delay(response) + return nil if response.nil? + + Policy.pacing_delay(response.headers, header_order: Policy::DEFAULT_PACING_HEADER_ORDER, + now: @settings.clock.now, random: @settings.random,) + end + + # RECOV-20's surfacing and RETRY-34's trail: retries_exhausted when the budget stopped a + # retryable failure (never for one that was never retryable), then the carried raise. + def surface(run, error, exhausted:) + run.trail.each { |prior| Dexpace.attach_suppressed(error, prior) } + run.tracer.retries_exhausted(run.request, error) if exhausted + raise error, cause: nil # pipeline/7ce4431d: a carried error, never a bare raise + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/resend.rb b/gems/dexpace-core/lib/dexpace/resilience/resend.rb new file mode 100644 index 0000000..4e1068d --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/resend.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../http/method" +require_relative "../http/body" + +module Dexpace + module Resilience + # RETRY-5, RETRY-6, RETRY-7, RETRY-8 and RECOV-18: the re-sendability gate, the second of + # the two independent axes retry needs, single-sourced. Both retry stacks call this exact + # method and neither restates the rule; 6b's REDIR-6 and 6c's AUTH-31 are its other two + # callers, as phase 3b's forward table anticipated (R5, confirmed unchanged). + # + # The rule in one line: a body-less request is re-sendable iff its method is idempotent -- + # HTTP-9's single set, phase 1's Method::IDEMPOTENT read through #idempotent? and never + # re-listed here, which is what makes RETRY-6's `{GET, HEAD, OPTIONS, PUT, DELETE}` one + # constant and not two -- and a body-bearing request iff its body says it is replayable + # (phase 3b's Body#replayable?, false by default). So a bare non-idempotent POST is NOT + # re-sendable even though there is no payload to resend (RETRY-7), and a PUT with a + # non-replayable body is not either (RETRY-8: both axes, neither implying the other). + # + # Phase 6b extends this module in place with the body-only question REDIR-6 asks + # (.replayable_body?) and its NotReplayableError; nothing here anticipates them, and nothing + # here is a "no-op if the file exists" skip (the plan's ownership note). + module Resend + extend self + + # Whether `request` may be sent to the wire a second time. + # + # @param request [Dexpace::Request] + # @return [Boolean] + def eligible?(request) + body = request.body + return request.method.idempotent? if body.nil? + + body.replayable? + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb new file mode 100644 index 0000000..d695143 --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_settings.rb @@ -0,0 +1,203 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../model" +require_relative "../clock" +require_relative "../config" +require_relative "../configuration" +require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/logger" +require_relative "policy" + +module Dexpace + module Resilience + # RECOV-34 and RETRY-12: the ONE retry configuration, a frozen Data validated and defensively + # copied at construction, that both stacks build from identical defaults -- RECOV-30's "the + # SAME default schedule so the two stacks cannot drift apart" made an object rather than a + # promise, because there is no second type to type a base delay into. Every default is + # Policy's constant, read here and re-typed nowhere. + # + # `total_timeout` is carried for both stacks and read by one: RecoveryRetry applies it through + # Policy.budget_remaining and RetryStep/AsyncRetryStep never read the member at all, which is + # RETRY-28's prohibition kept structural (R6). `pacing_header_order` is nil by default and each + # driver substitutes Policy::DEFAULT_PACING_HEADER_ORDER for nil, so RETRY-21's split + # precedence -- fixed on the recovery stack, caller-configurable on the stage stack -- reads + # from one member. `random` defaults to the ::Random CLASS, whose .rand is the process + # generator CRuby already makes safe to share, rather than to a fresh instance every settings + # object would then share unsynchronised across threads (RETRY-42's caveat, stated rather + # than hidden); a seeded ::Random.new is what a test passes. `clock` is 5a's seam. + # + # `max_retries` is the STAGE vocabulary (excluding the initial send) and the recovery stack's + # max_attempts is always max_retries + 1 (RETRY-14, P6-6). When a caller passes none, .build + # reads 5a's Configuration::Keys::MAX_RETRY_ATTEMPTS off the process-wide slot -- this class is + # that key's first and only reader -- and falls back to Policy::DEFAULT_MAX_RETRIES; the read + # happens ONCE, at build, so a settings object is a snapshot and never re-consults the slot. + # A NEGATIVE configured value meets RETRY-41 here, at that one read: it is clamped to + # Policy::DEFAULT_MAX_RETRIES through the same Policy.effective_max_retries the drivers + # resolve with, and the clamp is logged once through `logger:`, contained (P6-59). An explicit + # `max_retries:` argument is a caller's construction input and RECOV-34 refuses a negative one + # instead -- the two rules meet at the one place the configured value enters. + class RetrySettings < Data.define( + :initial_delay, :multiplier, :max_delay, :jitter, :max_retries, :total_timeout, + :retryable_statuses, :pacing_header_order, :random, :clock, + ) + include Dexpace::Model + + private_class_method :new + + # The absent-argument sentinel for max_retries: what lets "the caller passed nothing" be told + # apart from "the caller passed the default", so the configured key is consulted only in the + # first case and an explicit argument -- including one #with routes back through .build -- + # always wins. Private, and never a member's value. + UNSET = ::Object.new.freeze + private_constant :UNSET + + # The validating factory. Every keyword defaults to the shared policy constant. + # + # @param initial_delay [Numeric] seconds, non-negative + # @param multiplier [Numeric] >= 1.0 + # @param max_delay [Numeric] seconds, non-negative + # @param jitter [Numeric] in [0.0, 1.0] + # @param max_retries [Integer] retries after the initial send, >= 0; the configured + # MAX_RETRY_ATTEMPTS, else Policy::DEFAULT_MAX_RETRIES, when omitted + # @param total_timeout [Numeric] seconds; 0 disables the recovery stack's budget + # @param retryable_statuses [Enumerable] XCUT-7's configured set, copied + # @param pacing_header_order [Array, nil] RETRY-21's order, copied; nil for the + # shared default + # @param random [#rand] the generator the jitter draws from + # @param clock [Dexpace::_Clock] the seam every wait and every elapsed reading goes through + # @param logger [Dexpace::Instrumentation::Logger] where RETRY-41's clamp of a negative + # configured MAX_RETRY_ATTEMPTS is reported, contained; read at build only and never + # held, so it is taken on trust as Proxy.resolve's and Hooks.notify's are + # @return [RetrySettings] frozen + # @raise [Dexpace::InvalidArgumentError] naming the member, on any invalid value + def self.build(initial_delay: Policy::DEFAULT_INITIAL_DELAY, + multiplier: Policy::DEFAULT_MULTIPLIER, max_delay: Policy::DEFAULT_MAX_DELAY, + jitter: Policy::DEFAULT_JITTER, max_retries: UNSET, total_timeout: 0, + retryable_statuses: Policy::DEFAULT_RETRYABLE_STATUSES, + pacing_header_order: nil, random: ::Random, clock: Clock::SYSTEM, + logger: Instrumentation::Logger::NULL) + new( + initial_delay: initial_delay, multiplier: multiplier, max_delay: max_delay, + jitter: jitter, max_retries: resolve_max_retries(max_retries, logger), + total_timeout: total_timeout, retryable_statuses: retryable_statuses, + pacing_header_order: pacing_header_order, random: random, clock: clock, + ) + end + + # RETRY-12 / P6-6: the configured key, read once, only when nothing was passed -- and + # RETRY-41 on what it reads: a negative configured value is clamped to the default and the + # clamp logged, by the one resolver (P6-59). An explicit argument passes through untouched + # to RECOV-34's validation. + def self.resolve_max_retries(max_retries, logger) + return max_retries unless UNSET.equal?(max_retries) + + configured = Dexpace.configuration.integer(Configuration::Keys::MAX_RETRY_ATTEMPTS, + default: Policy::DEFAULT_MAX_RETRIES,) || + Policy::DEFAULT_MAX_RETRIES + Policy.effective_max_retries(override: nil, configured: configured, logger: logger) + end + private_class_method :resolve_max_retries + + # RECOV-34, clause by clause, each failure naming its member (SEAM-29); the collections + # are copied and deep-frozen here, once, through Model.own. + def initialize(initial_delay:, multiplier:, max_delay:, jitter:, max_retries:, + total_timeout:, retryable_statuses:, pacing_header_order:, random:, clock:) + Model.required!("random", random) + Model.required!("clock", clock) + super( + initial_delay: duration!("initial_delay", initial_delay), + multiplier: multiplier!(multiplier), max_delay: duration!("max_delay", max_delay), + jitter: jitter!(jitter), max_retries: max_retries!(max_retries), + total_timeout: duration!("total_timeout", total_timeout), + retryable_statuses: statuses!(retryable_statuses), + pacing_header_order: order!(pacing_header_order), random: random, clock: clock, + ) + end + + # The keyword hash Policy.backoff_delay takes, so neither driver spells the five members + # out by hand (RETRY-13). + # + # @return [Hash] + def backoff_arguments + { initial_delay: initial_delay, multiplier: multiplier, max_delay: max_delay, + jitter: jitter, random: random, } + end + + # RETRY-21: the order the drivers walk -- the caller's, or the shared default. + # + # @return [Array] frozen + def header_order + pacing_header_order || Policy::DEFAULT_PACING_HEADER_ORDER + end + + private + + # Non-negative, finite, and representable within the ~292-year nanosecond ceiling + # (design §10.18); returned as a Float so the arithmetic downstream is uniform. + def duration!(name, value) + Model.required!(name, value) + seconds = numeric!(name, value) + if seconds.negative? || !seconds.finite? + raise InvalidArgumentError, "#{name} must be a finite non-negative number of seconds" + end + if seconds * 1_000_000_000 > Policy::MAX_DURATION_NANOSECONDS + raise InvalidArgumentError, "#{name} exceeds the representable ceiling (RECOV-34)" + end + + seconds + end + + def multiplier!(value) + Model.required!("multiplier", value) + factor = numeric!("multiplier", value) + raise InvalidArgumentError, "multiplier must be >= 1.0 (RECOV-34)" if factor < 1.0 + raise InvalidArgumentError, "multiplier must be finite" unless factor.finite? + + factor + end + + def jitter!(value) + Model.required!("jitter", value) + fraction = numeric!("jitter", value) + return fraction if fraction.between?(0.0, 1.0) + + raise InvalidArgumentError, "jitter must lie within [0.0, 1.0] (RECOV-34)" + end + + def max_retries!(value) + Model.required!("max_retries", value) + return value if value.is_a?(::Integer) && !value.negative? + + raise InvalidArgumentError, "max_retries must be a non-negative Integer (RECOV-34)" + end + + def statuses!(value) + Model.required!("retryable_statuses", value) + unless value.respond_to?(:each) && value.all?(::Integer) + raise InvalidArgumentError, "retryable_statuses must be a collection of Integers" + end + + Model.own(::Set.new(value)) + end + + def order!(value) + return nil if value.nil? + unless value.is_a?(::Array) && value.all?(::String) + raise InvalidArgumentError, "pacing_header_order must be an Array of header names" + end + + Model.own(value.dup) + end + + def numeric!(name, value) + unless value.is_a?(::Numeric) + raise InvalidArgumentError, "#{name} must be a number, got #{value.class}" + end + + number = value #: untyped + number.to_f + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb new file mode 100644 index 0000000..1da6aaf --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "retry_step_helpers" +require_relative "../error/cancelled_error" +require_relative "../pipeline/stages" +require_relative "../instrumentation/http_tracer" +require_relative "../instrumentation/logger" + +module Dexpace + module Resilience + # The stage-based retry step, synchronous half: chapter 9's second stack at Stages::RETRY + # (order 500, between REDIRECT and AUTH, PIPE-2), driving the downstream chain once per + # attempt through a FRESH fork of its cursor -- for every drive, the first included, and + # never through its own cursor's #call (pipeline/86343352, P4-39) -- so hop 1 and hop n are + # the same object and each attempt re-executes the chain with fresh per-attempt state + # (RETRY-44). It writes no cursor state of its own (boundary 2: the cross-origin marker is + # 6b's and 6c's), so a RETRY fork's slot stays empty under every stage's key. + # + # RETRY-42: constructed once, frozen, shared by every call a pipeline serves, and stateless + # -- the attempt counter, the trail and the tracer are locals of one #call. The wait is 5a's + # cancellable queue wait, Clock#sleep(duration, cancellation:) on the settings' clock, never + # Kernel#sleep and never an interrupt (RETRY-26, CFG-15, boundary 5), and the cancellation + # token is checked at the top of EVERY attempt as well as by the wait (RETRY-23, RECOV-27). + # No total-timeout: nothing in this file or in RetryStepHelpers names + # RetrySettings#total_timeout or Policy.budget_remaining, which is RETRY-28 as a property of + # the source (R6, P6-5). + # + # OBS-29's per-attempt group is emitted through the tracer `http_tracer_factory:` produces + # -- called ONCE per #call, with the cursor as its argument and as the `context` every + # callback receives (R3, P6-7) -- and never through Cursor#bundle, whose tracer factory + # produces span tracers (a different kind of object). attempt_started before every drive, + # attempt_failed with the resolved delay before every wait, and retries_exhausted only when + # a RETRYABLE failure met a spent budget; the operation-lifecycle triple is emitted by + # nothing in v1 (docs/first-release.md's behavioural-asymmetries entry). A throwing tracer + # propagates and fails the request: OBS-30 puts that on the implementer, and core wraps no + # tracer callback in a rescue (OBS-20's carve-out). + # + # RETRY-35's three orderings, in #call: the delay is resolved from the still-open response + # FIRST, the response is closed BEFORE the wait, and the retry decision, the delay + # resolution and BOTH tracer emissions that follow a failure -- attempt_failed on the retry + # path, retries_exhausted on the terminal path -- are fenced so the response is closed before + # any throwable -- a RetryPredicateError above all, a throwing tracer too -- propagates, + # exactly as the async driver's guarded block does on both of its paths (R0-4, R1-1). + # RETRY-34 on the terminal path: the whole prior trail is attached to the instance actually + # surfaced, through Dexpace.attach_suppressed, and is discarded on success or on a returned + # error-status response, which is a Response and not a throwable to attach to (stated, not + # dropped). + class RetryStep + include RetryStepHelpers + + private_class_method :new + + # Builds a frozen step. + # + # @param settings [RetrySettings] the shared configuration; the defaults when omitted + # @param http_tracer_factory [#call] called once per operation with the cursor, answering + # an OBS-28 HTTP tracer; the frozen no-op NULL by default (R3, P6-7) + # @param delay_override [#call, nil] RETRY-39's first tier: called with + # (attempt, response, error) and answering a delay in seconds or nil to decline; a + # raise or a bad answer is logged and falls through (RETRY-40) + # @param should_retry [#call, nil] the caller's condition predicate, called with + # (failure, request) in place of the shared classifier; a raise aborts the call as + # RetryPredicateError (RETRY-40); it can never override the re-sendability gate (RETRY-8) + # @param logger [Instrumentation::Logger] where RETRY-40's fallback and RETRY-41's clamp + # are reported, contained; Logger::NULL by default + # @return [RetryStep] frozen + # @raise [Dexpace::InvalidArgumentError] for a settings that is not a RetrySettings, a + # non-callable factory or hook, or a logger that is not a Logger + def self.build(settings: RetrySettings.build, http_tracer_factory: NULL_TRACER_FACTORY, + delay_override: nil, should_retry: nil, + logger: Instrumentation::Logger::NULL) + new(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, logger: logger,).freeze + end + + # The default factory: the shared no-op, whatever the cursor. + NULL_TRACER_FACTORY = ->(_cursor) { Instrumentation::NULL } + private_constant :NULL_TRACER_FACTORY + + def initialize(settings:, http_tracer_factory:, delay_override:, should_retry:, logger:) + initialize_retry_step(settings: settings, http_tracer_factory: http_tracer_factory, + delay_override: delay_override, should_retry: should_retry, + logger: logger,) + end + + # 4c's optional declaration, read once at install: this is the Stages::RETRY pillar. + # + # @return [Dexpace::Pipeline::Stage] + def stage + Pipeline::Stages::RETRY + end + + # One operation: attempts until a success, a non-retryable failure or a spent budget. + # + # @param request [Dexpace::Request] re-sent as the same object on every attempt (RETRY-44) + # @param cursor [Dexpace::Pipeline::Cursor] forked for every drive + # @return [Dexpace::Response] the terminal response, error status or not, unclosed + # @raise [Exception] the terminal throwable, carrying the prior trail as suppressed + # @raise [Dexpace::CancelledError] when the token is cancelled at an attempt boundary or + # during a wait + # @raise [Dexpace::RetryPredicateError] when a caller's should_retry raised + def call(request, cursor) + run = Run.new(tracer: @http_tracer_factory.call(cursor), cursor: cursor, + max_retries: effective_max_retries(cursor), trail: [],) + attempt = 1 + loop do + terminal = attempt_once(run, request, attempt) + return terminal unless terminal.nil? + + attempt += 1 + end + end + + # The per-call locals one operation threads through its helpers (RETRY-42: on the stack, + # never on the step), a private value so the helpers take one argument for the four. + class Run < Data.define(:tracer, :cursor, :max_retries, :trail) + end + private_constant :Run + + private + + # One attempt, start to finish: the token check, the tracer, the drive, the decision and + # -- on a retry -- the wait. Answers the terminal response, or nil for "again". + def attempt_once(run, request, attempt) + response, error = drive(run, request, attempt) + return response if !response.nil? && !response.status.error? + + failure = failure_of(response, error) + verdict = fenced(response) { decision(request, failure, attempt, run.max_retries) } + return settle(run, response, failure, verdict) unless verdict == RETRY + + wait(run, attempt, response, failure) + end + + # The retry path: the delay from the still-open response and the tracer's attempt_failed, + # both inside ONE fence (RETRY-35: a tracer that raises propagates, OBS-30, but the + # superseded response is closed first, as on the async driver), the trail, the close BEFORE + # the wait, then the wait on the settings' clock. On the exception path the failure IS the + # error the override sees; on the response path the override sees the response and no error + # (RETRY-39). + # + # @return [nil] always: the loop's "again" + def wait(run, attempt, response, failure) + delay = fenced(response) do + resolved = resolve_delay(attempt, response, response ? nil : failure) + run.tracer.attempt_failed(run.cursor, failure, resolved) + resolved + end + run.trail << failure + Dexpace.close_quietly(response, onto: failure) + @settings.clock.sleep(delay, cancellation: run.cursor.cancellation) + end + + # One drive: the token check at the attempt boundary (RETRY-23), the tracer, then the + # downstream through a fresh fork. Only the drive's StandardError is rescued: the fatal + # family propagates through the absent arm, unclassified and unretried (RETRY-25), and a + # CancelledError from the check is not a drive failure and propagates as itself. + # + # @return [Array] the response and nil, or nil and the error + def drive(run, request, attempt) + run.cursor.cancellation.check! + run.tracer.attempt_started(run.cursor, attempt) + begin + [run.cursor.fork.call(request), nil] + rescue ::StandardError => error + [nil, error] + end + end + + # RETRY-35's third ordering: whatever the block raises -- fatal family included -- the + # open response is closed first and the raise propagates unchanged. + def fenced(response) + yield + rescue ::Exception # rubocop:disable Lint/RescueException -- close, then re-raise unchanged (RETRY-25, RETRY-35) + Dexpace.close_quietly(response, logger: @logger) + raise + end + + # The terminal path. A throwable is surfaced with the whole trail attached and, when the + # budget is what stopped it, retries_exhausted first; a response is returned as it is, its + # trail discarded, since a Response carries no suppressed trail to attach to (RETRY-34) -- + # the ProtocolError the tracer sees for an exhausted error status carries it instead. The + # emission runs inside the fence, as attempt_failed does on the retry path: a tracer that + # raises in retries_exhausted propagates (OBS-30) but the terminal error-status response + # is closed first, exactly as Pump#finish is closed by its guarded block (R1-1). + def settle(run, response, failure, verdict) + return response if response && verdict == STOP + + fenced(response) do + attach_trail(failure, run.trail) + run.tracer.retries_exhausted(run.cursor, failure) if verdict == EXHAUSTED + end + return response if response + + raise failure, cause: nil # pipeline/7ce4431d: a carried error, never a bare raise + end + end + end +end diff --git a/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb new file mode 100644 index 0000000..b6895eb --- /dev/null +++ b/gems/dexpace-core/lib/dexpace/resilience/retry_step_helpers.rb @@ -0,0 +1,166 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "policy" +require_relative "resend" +require_relative "retry_settings" +require_relative "../error/protocol_error" +require_relative "../error/retry_predicate_error" +require_relative "../error/invalid_argument_error" +require_relative "../instrumentation/contain" +require_relative "../instrumentation/keys" +require_relative "../instrumentation/logger" +require_relative "../instrumentation/http_tracer" +require_relative "../pipeline/stages" + +module Dexpace + module Resilience + # What RetryStep and AsyncRetryStep share above Policy itself: the constructor's validation, + # the retry DECISION (both axes, the caller's predicate, the budget) and the delay + # RESOLUTION (RETRY-39's precedence with RETRY-40's fatal/non-fatal split), so RETRY-13's + # "one calculator" extends to the logic sitting on it and the two drivers cannot drift + # (design §11.12's rule for every sync/async pair). A private_constant mixin with a sig/ + # mirror -- the strict `core` target types its two includers -- and no test/ mirror: every + # branch is asserted through the two steps (P6-11, as built). + # + # RETRY-28 in the source: no method here names RetrySettings#total_timeout or + # Policy.budget_remaining, and neither driver does either -- the text scan in + # retry_step_test.rb is the guard. + module RetryStepHelpers + # The three answers a decision can give. Only :exhausted emits retries_exhausted: a failure + # that was never retryable is not an exhausted retry (5c's ordering test, third case). + RETRY = :retry + EXHAUSTED = :exhausted + STOP = :stop + private_constant :RETRY, :EXHAUSTED, :STOP + + private + + # The constructor's validation, shared: the settings are a RetrySettings, the factory and + # the two hooks are callables, the logger is a Logger. + def initialize_retry_step(settings:, http_tracer_factory:, delay_override:, should_retry:, + logger:) + unless settings.is_a?(RetrySettings) + raise InvalidArgumentError, "settings: takes a RetrySettings, got #{settings.class}" + end + unless logger.is_a?(Instrumentation::Logger) + raise InvalidArgumentError, "logger: takes an Instrumentation::Logger, got " \ + "#{logger.class}" + end + + @settings = settings + @http_tracer_factory = callable!("http_tracer_factory", http_tracer_factory) + @delay_override = delay_override.nil? ? nil : callable!("delay_override", delay_override) + @should_retry = should_retry.nil? ? nil : callable!("should_retry", should_retry) + @logger = logger + end + + def callable!(name, value) + return value if value.respond_to?(:call) + + raise InvalidArgumentError, "#{name}: must answer #call" + end + + # RETRY-41 at the top of every call: the per-call override wins over the configured value. + def effective_max_retries(cursor) + Policy.effective_max_retries(override: cursor.options.max_retries, + configured: @settings.max_retries, logger: @logger,) + end + + # RETRY-23 first: a cancellation -- the token's own raise or one a downstream wrapped -- + # is terminal before any axis is consulted, so a caller's should_retry never sees it and + # can never answer true for it (P6-60). Then RETRY-8's BOTH axes, neither implying the + # other, in this order -- re-sendability first and unconditionally (a caller predicate may + # widen or narrow the CONDITION and may never authorise re-sending a bare POST or a consumed + # body, RETRY-7), then the condition (Policy.retryable? or the caller's predicate, + # RETRY-39/RETRY-40), then the budget. The budget comes LAST so that :exhausted means + # exactly "retryable, and the budget is spent". + # + # @return [Symbol] RETRY, EXHAUSTED or STOP + def decision(request, failure, attempt, max_retries) + return STOP if Policy.cancellation?(failure) + return STOP unless Resend.eligible?(request) + return STOP unless retryable_condition?(failure, request) + + attempt <= max_retries ? RETRY : EXHAUSTED + end + + # The condition axis: the shared classifier by default, the caller's predicate when one + # was supplied -- and a predicate that raises a StandardError aborts the call as + # RetryPredicateError with the raise as its cause (RETRY-40). The fatal family passes + # through the absent arm unchanged (RETRY-25). + def retryable_condition?(failure, request) + if @should_retry.nil? + return Policy.retryable?(failure, retryable_statuses: @settings.retryable_statuses) + end + + begin + @should_retry.call(failure, request) ? true : false + rescue ::StandardError => error + raise RetryPredicateError.new("the should_retry predicate raised #{error.class}"), + cause: error + end + end + + # RETRY-39: caller delay-override, then the server's pacing headers on the response path + # only, then exponential backoff. (The fixed-delay tier between the last two is RETRY-43, + # a MAY declined for v1, so the precedence has three live sources.) `now:` is the settings + # clock's, so a fake clock drives the absolute forms. + def resolve_delay(attempt, response, failure) + overridden = overridden_delay(attempt, response, failure) + return overridden unless overridden.nil? + + if response + hinted = Policy.pacing_delay(response.headers, header_order: @settings.header_order, + now: @settings.clock.now, + random: @settings.random,) + return hinted unless hinted.nil? + end + + Policy.backoff_delay(attempt, **@settings.backoff_arguments) + end + + # RETRY-40's non-fatal half: a delay-override that raises a StandardError, or answers + # something that is not a non-negative number, is logged as one WARNING diagnostic through + # 5b's facade (contained, OBS-20) and the precedence falls through. nil is the documented + # "no opinion" answer and is not logged. The fatal family propagates (RETRY-25). + def overridden_delay(attempt, response, failure) + return nil if @delay_override.nil? + + value = @delay_override.call(attempt, response, failure) + return nil if value.nil? + return Float(value) if value.is_a?(::Numeric) && value.finite? && !value.negative? + + report_override(message: "delay_override answered #{value.inspect}; falling back " \ + "(RETRY-40)") + nil + rescue ::StandardError => error + report_override(cause: error, message: "delay_override raised; falling back (RETRY-40)") + nil + end + + def report_override(message:, cause: nil) + Instrumentation.diagnostic(@logger, event: Instrumentation::Events::INSTRUMENTATION_HOOK, + cause: cause, message: message,) + end + + # The Exception the tracer and the trail carry for an attempt: the throwable itself, or + # the error-status response converted ONCE, so the object the tracer reported is the object + # the trail holds (OBS-28: #attempt_failed's second argument is an Exception). + def failure_of(response, error) + return error unless error.nil? + raise InvalidArgumentError, "an attempt yields a response or an error" if response.nil? + + ProtocolError.for(response) + end + + # RETRY-34's terminal attachment: every prior attempt's failure onto the surfaced instance, + # through the one helper whose skip-self guard is 4b's. + def attach_trail(error, trail) + trail.each { |prior| Dexpace.attach_suppressed(error, prior) } + error + end + end + private_constant :RetryStepHelpers + end +end diff --git a/gems/dexpace-core/sig/dexpace/async_pipeline.rbs b/gems/dexpace-core/sig/dexpace/async_pipeline.rbs index 1e76d98..54b17f6 100644 --- a/gems/dexpace-core/sig/dexpace/async_pipeline.rbs +++ b/gems/dexpace-core/sig/dexpace/async_pipeline.rbs @@ -19,6 +19,6 @@ module Dexpace def self.direct: (untyped transport) -> AsyncPipeline def self.map_response: (Dexpace::Async::Future source) { (Dexpace::Response response) -> untyped } -> Dexpace::Async::Future - def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation) -> Dexpace::Async::Future + def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation, ?bundle: Instrumentation::Bundle) -> Dexpace::Async::Future end end diff --git a/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs b/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs index b6172cb..33b512f 100644 --- a/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs +++ b/gems/dexpace-core/sig/dexpace/error/protocol_error.rbs @@ -2,6 +2,8 @@ module Dexpace class ProtocolError < ::StandardError include Dexpace::Error + @retryable_by_status: bool + attr_reader response: Dexpace::Response attr_reader status: Dexpace::Status @@ -9,6 +11,9 @@ module Dexpace def self.for_or_nil: (Dexpace::Response response) -> Dexpace::ProtocolError? def initialize: (Dexpace::Response response) -> void + # XCUT-5's baked flag (RETRY-3), added by phase 6a: a widening, the constructor unchanged. + def retryable_by_status?: () -> bool + private def describe: (Dexpace::Status status) -> String diff --git a/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs b/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs new file mode 100644 index 0000000..1e33088 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/error/retry_predicate_error.rbs @@ -0,0 +1,5 @@ +module Dexpace + class RetryPredicateError < ::StandardError + include Dexpace::Error + end +end diff --git a/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs b/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs index 43fccd6..7d41b43 100644 --- a/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs +++ b/gems/dexpace-core/sig/dexpace/instrumentation/http_tracer.rbs @@ -1,9 +1,31 @@ module Dexpace module Instrumentation - # OBS-28's eleven-method vocabulary, a module an implementer includes. No `interface - # _HTTPTracer` is declared, deliberately: eleven signatures with no core caller would be - # eleven NFR-4-locked shapes nothing checks, and the module's own declaration is the - # contract; if phase 6 wires the vocabulary, the interface arrives with the wiring. + # OBS-28's eleven-method vocabulary: the module an implementer includes, and -- since phase 6a + # wired the per-attempt group (R3) -- the interface the retry drivers' `http_tracer_factory:` + # keyword produces. Phase 5c declared no interface because eleven NFR-4-locked signatures with + # no core caller would have been shapes nothing checks; the caller exists now, so the + # interface arrives with the wiring, exactly as 5c said it would. `context` stays untyped + # deliberately: OBS-28 fixes the vocabulary and not the correlation handle, and 6a passes a + # Pipeline::Cursor from the two stage drivers and a Dexpace::Request from RecoveryRetry -- + # typing it to either would make the other a Steep error and lock a type no requirement + # names. `error` is an Exception: the drivers convert an error-status response with + # ProtocolError.for before reporting it and never hand the vocabulary a Response. + interface _HTTPTracer + def operation_started: (untyped context) -> void + def operation_succeeded: (untyped context, Dexpace::Response response) -> void + def operation_failed: (untyped context, Exception error) -> void + + def attempt_started: (untyped context, Integer attempt) -> void + def attempt_failed: (untyped context, Exception error, Numeric next_delay) -> void + def retries_exhausted: (untyped context, Exception error) -> void + + def request_url_resolved: (untyped context, untyped url) -> void + def connection_acquired: (untyped context, String host, Integer port) -> void + def request_sent: (untyped context, Integer byte_count) -> void + def response_headers_received: (untyped context, Integer status, untyped headers) -> void + def response_received: (untyped context, Integer byte_count) -> void + end + module HTTPTracer def operation_started: (untyped context) -> void def operation_succeeded: (untyped context, untyped response) -> void diff --git a/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs b/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs index 5b12586..35bbda3 100644 --- a/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs +++ b/gems/dexpace-core/sig/dexpace/instrumentation/step.rbs @@ -33,7 +33,7 @@ module Dexpace def self.level!: (untyped level, untyped preview_bytes) -> void # untyped: a _Span is what the tracer returns and `_Span & Object` is what Tracing takes # (5c's identity device); the step passes one straight to the other. - def open_span: (Dexpace::Request request) -> untyped + def open_span: (Dexpace::Request request, Bundle bundle) -> untyped def finish: (untyped span, Float started) -> void def elapsed_ms: (Float started) -> Float def logged?: () -> bool diff --git a/gems/dexpace-core/sig/dexpace/pipeline.rbs b/gems/dexpace-core/sig/dexpace/pipeline.rbs index c04d474..b6d930b 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline.rbs @@ -19,6 +19,6 @@ module Dexpace def self.builder: (transport: untyped) -> Builder def self.direct: (untyped transport) -> Pipeline - def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation) -> Dexpace::Response + def call: (Dexpace::Request request, ?Dexpace::RequestOptions options, ?Dexpace::Cancellation cancellation, ?bundle: Instrumentation::Bundle) -> Dexpace::Response end end diff --git a/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs b/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs index 621202f..0c224f6 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/async_driver.rbs @@ -10,7 +10,7 @@ module Dexpace def initialize: (AsyncPipeline pipeline) -> void def entry_at: (Integer index) -> Entry - def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> Dexpace::Async::Future + def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> Dexpace::Async::Future def dispatch: (Dexpace::Request request, Dexpace::RequestOptions options, Dexpace::Cancellation cancellation) -> Dexpace::Async::Future private diff --git a/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs b/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs index 55a6355..ece37dd 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/cursor.rbs @@ -10,20 +10,23 @@ module Dexpace @request: Dexpace::Request @options: Dexpace::RequestOptions @cancellation: Dexpace::Cancellation + @bundle: Instrumentation::Bundle @state: Hash[Stage, Hash[untyped, untyped]] @spent: bool attr_reader request: Dexpace::Request attr_reader options: Dexpace::RequestOptions attr_reader cancellation: Dexpace::Cancellation + # Phase 6a's Task 8: the per-call instrumentation bundle, a widening. + attr_reader bundle: Instrumentation::Bundle # .new is private and takes owner_index:/position:/state: a caller cannot name; only .build # is public, and it produces the cursor bound to no entry (owner_index -1): it advances INTO # entry 0 and cannot fork (R10). Every cursor a step is handed is made by the driver. - private def self.new: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> instance - def initialize: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> void + private def self.new: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> instance + def initialize: (drive: untyped, owner_index: Integer, position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> void - def self.build: (drive: untyped, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation) -> Cursor + def self.build: (drive: untyped, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, ?bundle: Instrumentation::Bundle) -> Cursor def spent?: () -> bool def may_fork?: () -> bool diff --git a/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs b/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs index f7ed353..56e44de 100644 --- a/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs +++ b/gems/dexpace-core/sig/dexpace/pipeline/sync_driver.rbs @@ -10,7 +10,7 @@ module Dexpace def initialize: (Pipeline pipeline) -> void def entry_at: (Integer index) -> Entry - def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, state: Hash[Stage, Hash[untyped, untyped]]) -> untyped + def advance: (position: Integer, request: Dexpace::Request, options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, bundle: Instrumentation::Bundle, state: Hash[Stage, Hash[untyped, untyped]]) -> untyped end end end diff --git a/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs b/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs new file mode 100644 index 0000000..99b6577 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/async_retry_step.rbs @@ -0,0 +1,63 @@ +module Dexpace + module Resilience + # The asynchronous stage-based retry step at Stages::RETRY, over 4c's _AsyncStep. The keyword + # types are RetryStep's. + class AsyncRetryStep + include RetryStepHelpers + + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + + private def self.new: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (?settings: RetrySettings, ?http_tracer_factory: untyped, + ?delay_override: untyped, ?should_retry: untyped, + ?logger: Instrumentation::Logger) -> AsyncRetryStep + + def stage: () -> Pipeline::Stage + def call: (Dexpace::Request request, Pipeline::Cursor cursor) -> Dexpace::Async::Future + + # A private_constant with no visibility in RBS: the per-call trampoline (RETRY-30), + # declared because the strict `core` Steep target types its construction and its body. + class Pump + include RetryStepHelpers + + @request: Dexpace::Request + @cursor: Pipeline::Cursor + @completer: Dexpace::Async::Completer + @tracer: untyped + @max_retries: Integer + @attempt: Integer + @trail: Array[::Exception] + @mutex: ::Thread::Mutex + @running: bool + @rearm: bool + @pending_delay: Dexpace::Async::Future? + + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger, request: Dexpace::Request, + cursor: Pipeline::Cursor) -> void + def start: () -> Dexpace::Async::Future + + private + + def resume: () -> void + def claim: (Symbol step) -> bool + def launch: () -> void + def settled: (Dexpace::Async::Settlement settlement) -> void + def delivered?: (Dexpace::Response? response, ::Exception? error) -> bool + def retry_after: (Dexpace::Response? response, ::Exception? error, ::Exception failure) -> void + def finish: (Dexpace::Response? response, ::Exception failure, Symbol verdict) -> void + def wait: (Float delay) -> void + def abort: (untyped reason) -> void + # The block answers whatever its terminal path answered; nothing reads it. + def guarded: (Dexpace::Response? response) { () -> untyped } -> untyped + end + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs new file mode 100644 index 0000000..4c4cfe3 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/pacing_parsers.rbs @@ -0,0 +1,23 @@ +# Dexpace::Resilience::PacingParsers is a private_constant and not public API (P6-11): this +# declaration exists because the strict `core` Steep target types Policy.pacing_delay's calls into +# it. RBS has no visibility for a constant, so the privacy lives in +# lib/dexpace/resilience/pacing_parsers.rb alone. `random` is untyped: ::Random is outside NFR-11's +# stdlib allowlist, and the contract is #rand over a Range (trace_id_flavour.rbs's precedent). +module Dexpace + module Resilience + module PacingParsers + MAX_VALUE_BYTES: Integer + DECIMAL_GRAMMAR: Regexp + INTEGER_GRAMMAR: Regexp + RESET_JITTER_CEILING: Float + + def self?.parse_retry_after: (untyped value, now: ::Time) -> Float? + def self?.parse_millis: (untyped value) -> Float? + def self?.parse_epoch_reset: (untyped value, now: ::Time, random: untyped) -> Float? + + private + + def readable?: (untyped value) -> bool + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/policy.rbs b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs new file mode 100644 index 0000000..ee04eb9 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/policy.rbs @@ -0,0 +1,37 @@ +module Dexpace + module Resilience + # The one shared retry policy core (RETRY-13, RECOV-30, R5). `random` is untyped on the two + # methods that take one: ::Random is outside NFR-11's stdlib allowlist and the contract is + # #rand over a Range, the shape trace_id_flavour.rbs already uses for its generator. + module Policy + DEFAULT_INITIAL_DELAY: Float + DEFAULT_MULTIPLIER: Float + DEFAULT_MAX_DELAY: Float + DEFAULT_JITTER: Float + DEFAULT_MAX_RETRIES: Integer + DEFAULT_RETRYABLE_STATUSES: ::Set[Integer] + DEFAULT_PACING_HEADER_ORDER: Array[String] + MAX_PACING_DELAY_SECONDS: Integer + MAX_DURATION_NANOSECONDS: Integer + NANOSECOND: Float + + def self?.retry_eligible?: (Integer status, set: ::Set[Integer]) -> bool + def self?.throwable_retryable?: (::Exception error) -> bool + def self?.cancellation?: (::Exception error) -> bool + def self?.retryable?: (::Exception error, retryable_statuses: ::Set[Integer]) -> bool + def self?.backoff_delay: (Integer attempt, initial_delay: Numeric, multiplier: Numeric, + max_delay: Numeric, jitter: Numeric, random: untyped) -> Float + def self?.pacing_delay: (Dexpace::Headers headers, header_order: Array[String], + ?now: ::Time, ?random: untyped) -> Float? + def self?.effective_max_retries: (override: Integer?, configured: Integer, + ?logger: Instrumentation::Logger) -> Integer + def self?.budget_remaining: (elapsed: Numeric, total_timeout: Numeric) -> Float + + private + + def seconds: (Numeric value) -> Float + def jittered: (Float unjittered, Numeric jitter, untyped random) -> Float + def parse_form: (String name, String value, ::Time now, untyped random) -> Float? + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs b/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs new file mode 100644 index 0000000..706b830 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/recovery_retry.rbs @@ -0,0 +1,60 @@ +module Dexpace + module Resilience + # The recovery-chain retry engine (RECOV-17..30, RECOV-34), a Dexpace::Transport decorator + # installed as Recovery::Orchestrator's `transport:` (P6-3). The transport and the factory are + # untyped callables, as on the two stage drivers. + class RecoveryRetry + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + RETRY: Symbol + EXHAUSTED: Symbol + STOP: Symbol + + @transport: untyped + @settings: RetrySettings + @http_tracer_factory: untyped + @logger: Instrumentation::Logger + + # A private_constant: the per-call locals (RECOV-28), declared for the strict target. + class Run < Data + attr_reader tracer: untyped + attr_reader request: Dexpace::Request + attr_reader options: Dexpace::RequestOptions + attr_reader cancellation: Dexpace::Cancellation + attr_reader max_attempts: Integer + attr_reader started: Float + attr_reader trail: Array[::Exception] + + def self.new: (tracer: untyped, request: Dexpace::Request, + options: Dexpace::RequestOptions, cancellation: Dexpace::Cancellation, + max_attempts: Integer, started: Float, + trail: Array[::Exception]) -> instance + end + + private def self.new: (transport: untyped, settings: RetrySettings, + http_tracer_factory: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (transport: untyped, settings: RetrySettings, http_tracer_factory: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (transport: untyped, ?settings: RetrySettings, + ?http_tracer_factory: untyped, + ?logger: Instrumentation::Logger) -> RecoveryRetry + + def call: (Dexpace::Request request, Dexpace::RequestOptions options, + Dexpace::Cancellation cancellation) -> Dexpace::Response + + private + + def attempt_once: (Run run, Integer attempt) -> Dexpace::Response? + def wait: (Run run, Integer attempt, Dexpace::Response? response, ::Exception error) -> nil + def max_attempts: () -> Integer + def exchange: (Dexpace::Request request, Dexpace::RequestOptions options, + Dexpace::Cancellation cancellation) -> [Dexpace::Response?, ::Exception?] + def decision: (Dexpace::Request request, ::Exception error, Integer attempt, + Integer max_attempts) -> Symbol + def delay_within_budget: (Run run, Integer attempt, Dexpace::Response? response) -> Float? + def hinted_delay: (Dexpace::Response? response) -> Float? + def surface: (Run run, ::Exception error, exhausted: bool) -> bot + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/resend.rbs b/gems/dexpace-core/sig/dexpace/resilience/resend.rbs new file mode 100644 index 0000000..0382395 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/resend.rbs @@ -0,0 +1,9 @@ +module Dexpace + module Resilience + # RETRY-5 through RETRY-8, RECOV-18: the re-sendability gate (R5). Phase 6b widens this + # module in place with REDIR-6's body-only predicate and its error. + module Resend + def self?.eligible?: (Dexpace::Request request) -> bool + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs new file mode 100644 index 0000000..a571920 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_settings.rbs @@ -0,0 +1,57 @@ +module Dexpace + module Resilience + # RECOV-34, RETRY-12: the one retry configuration both stacks build from. `random` is untyped + # (::Random is outside NFR-11's allowlist; the contract is #rand over a Range) and the + # arguments are `untyped` where the constructor validates rather than trusts. + class RetrySettings < Data + include Model + + UNSET: Object + + attr_reader initial_delay: Float + attr_reader multiplier: Float + attr_reader max_delay: Float + attr_reader jitter: Float + attr_reader max_retries: Integer + attr_reader total_timeout: Float + attr_reader retryable_statuses: ::Set[Integer] + attr_reader pacing_header_order: Array[String]? + attr_reader random: untyped + attr_reader clock: _Clock + + private def self.new: ( + initial_delay: untyped, multiplier: untyped, max_delay: untyped, jitter: untyped, + max_retries: untyped, total_timeout: untyped, retryable_statuses: untyped, + pacing_header_order: untyped, random: untyped, clock: _Clock, + ) -> instance + def initialize: ( + initial_delay: untyped, multiplier: untyped, max_delay: untyped, jitter: untyped, + max_retries: untyped, total_timeout: untyped, retryable_statuses: untyped, + pacing_header_order: untyped, random: untyped, clock: _Clock, + ) -> void + + def self.build: ( + ?initial_delay: Numeric, ?multiplier: Numeric, ?max_delay: Numeric, ?jitter: Numeric, + ?max_retries: untyped, ?total_timeout: Numeric, + ?retryable_statuses: Enumerable[Integer], + ?pacing_header_order: Array[String]?, ?random: untyped, ?clock: _Clock, + ?logger: Instrumentation::Logger, + ) -> RetrySettings + + def backoff_arguments: () -> { initial_delay: Float, multiplier: Float, max_delay: Float, + jitter: Float, random: untyped } + def header_order: () -> Array[String] + + private + + def self.resolve_max_retries: (untyped max_retries, Instrumentation::Logger logger) -> Integer + def duration!: (String name, untyped value) -> Float + def multiplier!: (untyped value) -> Float + def jitter!: (untyped value) -> Float + def max_retries!: (untyped value) -> Integer + def statuses!: (untyped value) -> ::Set[Integer] + def order!: (untyped value) -> Array[String]? + def numeric!: (String name, untyped value) -> Float + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs new file mode 100644 index 0000000..454fbe1 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_step.rbs @@ -0,0 +1,47 @@ +module Dexpace + module Resilience + # The synchronous stage-based retry step at Stages::RETRY (RETRY-1..45's stage half). The + # factory and the two hooks are untyped callables: `^(Pipeline::Cursor) -> _HTTPTracer` is + # what the factory answers, and the hooks' argument shapes are in the YARD; typing a caller's + # lambda strictly here would refuse every arity-tolerant Proc a caller writes (P5-14's + # precedent for the generator). + class RetryStep + include RetryStepHelpers + + NULL_TRACER_FACTORY: ^(untyped) -> Instrumentation::_HTTPTracer + + private def self.new: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> instance + def initialize: (settings: RetrySettings, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: Instrumentation::Logger) -> void + + def self.build: (?settings: RetrySettings, ?http_tracer_factory: untyped, + ?delay_override: untyped, ?should_retry: untyped, + ?logger: Instrumentation::Logger) -> RetryStep + + def stage: () -> Pipeline::Stage + def call: (Dexpace::Request request, Pipeline::Cursor cursor) -> Dexpace::Response + + # A private_constant: the per-call locals, declared for the strict target. + class Run < Data + attr_reader tracer: untyped + attr_reader cursor: Pipeline::Cursor + attr_reader max_retries: Integer + attr_reader trail: Array[::Exception] + + def self.new: (tracer: untyped, cursor: Pipeline::Cursor, max_retries: Integer, + trail: Array[::Exception]) -> instance + end + + private + + def drive: (Run run, Dexpace::Request request, Integer attempt) -> [Dexpace::Response?, ::Exception?] + def fenced: [T] (Dexpace::Response? response) { () -> T } -> T + def attempt_once: (Run run, Dexpace::Request request, Integer attempt) -> Dexpace::Response? + def wait: (Run run, Integer attempt, Dexpace::Response? response, ::Exception failure) -> nil + def settle: (Run run, Dexpace::Response? response, ::Exception failure, Symbol verdict) -> Dexpace::Response + end + end +end diff --git a/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs b/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs new file mode 100644 index 0000000..1796647 --- /dev/null +++ b/gems/dexpace-core/sig/dexpace/resilience/retry_step_helpers.rbs @@ -0,0 +1,35 @@ +# Dexpace::Resilience::RetryStepHelpers is a private_constant and not public API (P6-11): this +# declaration exists because the strict `core` Steep target types its two includers, RetryStep +# and AsyncRetryStep. RBS has no visibility for a constant, so the privacy lives in +# lib/dexpace/resilience/retry_step_helpers.rb alone. +module Dexpace + module Resilience + module RetryStepHelpers + RETRY: Symbol + EXHAUSTED: Symbol + STOP: Symbol + + @settings: RetrySettings + @http_tracer_factory: untyped + @delay_override: untyped + @should_retry: untyped + @logger: Instrumentation::Logger + + private + + def initialize_retry_step: (settings: untyped, http_tracer_factory: untyped, + delay_override: untyped, should_retry: untyped, + logger: untyped) -> void + def callable!: (String name, untyped value) -> untyped + def effective_max_retries: (Pipeline::Cursor cursor) -> Integer + def decision: (Dexpace::Request request, ::Exception failure, Integer attempt, + Integer max_retries) -> Symbol + def retryable_condition?: (::Exception failure, Dexpace::Request request) -> bool + def resolve_delay: (Integer attempt, Dexpace::Response? response, ::Exception? failure) -> Float + def overridden_delay: (Integer attempt, Dexpace::Response? response, ::Exception? failure) -> Float? + def report_override: (message: String, ?cause: ::Exception?) -> void + def failure_of: (Dexpace::Response? response, ::Exception? error) -> ::Exception + def attach_trail: (::Exception error, Array[::Exception] trail) -> ::Exception + end + end +end diff --git a/gems/dexpace-core/test/dexpace/async_pipeline_test.rb b/gems/dexpace-core/test/dexpace/async_pipeline_test.rb index ff8cc24..6651942 100644 --- a/gems/dexpace-core/test/dexpace/async_pipeline_test.rb +++ b/gems/dexpace-core/test/dexpace/async_pipeline_test.rb @@ -256,4 +256,50 @@ def closed? = @closes.positive? assert_predicate(transport.completer.future.value, :closed?) end end + + # Phase 6a's Task 8: the `bundle:` keyword on the async runtime. + class BundleSeedingTest < DexpaceTestCase + BUNDLE = Dexpace::Instrumentation::Bundle + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C,) + end + + def request + Dexpace::Request.build(method: Dexpace::Method::GET, url: "https://example.test/", + headers: Dexpace::Headers::EMPTY,) + end + + def settled(value) + completer = Dexpace::Async::Completer.new + completer.fulfil(value) + completer.future + end + + test "cursor bundle: AsyncPipeline#call seeds the cursor's bundle, NONE when omitted" do + seen = [] + step = lambda { |req, cur| + seen << cur.bundle + cur.call(req) + } + transport = ->(_req, _opts, _canc) { settled(:response) } + pipeline = Dexpace::Pipeline::Builder.new(transport: transport) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build_async + bundle = seeded + + assert_equal(:response, pipeline.call(request).value) + assert_equal(:response, pipeline.call(request, bundle: bundle).value) + assert_same(BUNDLE::NONE, seen[0]) + assert_same(bundle, seen[1]) + end + + test "cursor bundle: the empty async pipeline accepts the keyword and allocates no cursor" do + pipeline = Dexpace::AsyncPipeline.direct(->(_r, _o, _c) { settled(:response) }) + + assert_equal(:response, pipeline.call(request, bundle: seeded).value) + assert(Dexpace::AsyncTransport.conforms?(pipeline)) + end + end end diff --git a/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb b/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb index 8e7a3db..c96f4b5 100644 --- a/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb +++ b/gems/dexpace-core/test/dexpace/error/protocol_error_test.rb @@ -72,7 +72,54 @@ class DexpaceProtocolErrorTest < DexpaceTestCase assert_raises(Dexpace::InvalidArgumentError) { Dexpace::ProtocolError.for_or_nil(nil) } end - # No test pins the ABSENCE of #retryable_by_status?: XCUT-5's baked flag is postponed to phase - # 6a, Task 6, which adds the predicate to THIS class and extends THIS suite. A postponement is - # recorded in a checklist row, not pinned by an assertion a later phase has to delete. + # Phase 6a's Task 6 added the predicate 4b postponed; the nested class below is its suite, + # extending this file exactly as the postponement said it would. + + # XCUT-5 / RETRY-3: the baked flag, computed at construction from the SINGLE shared status + # classifier (5a's Dexpace::Retryability), and deliberately not named #retryable? (P6-10). + class RetryableTest < DexpaceTestCase + include RecoveryFixtures + + test "RETRY-3 / XCUT-5: retryable_by_status? is computed at construction from Retryability" do + error = Dexpace::ProtocolError.for(build_response(503)) + + assert_predicate(error, :retryable_by_status?) + assert_equal(Dexpace::Retryability.retryable_status?(503), error.retryable_by_status?) + assert_same(error.retryable_by_status?, error.retryable_by_status?, "baked, not recomputed") + end + + test "RETRY-3 / XCUT-5: the flag agrees with the classifier across the whole error range" do + (400..599).each do |code| + error = Dexpace::ProtocolError.for(build_response(code)) + expected = Dexpace::Retryability.retryable_status?(code) + + assert_equal(expected, error.retryable_by_status?, code.to_s) + end + refute_predicate(Dexpace::ProtocolError.for(build_response(501)), :retryable_by_status?) + refute_predicate(Dexpace::ProtocolError.for(build_response(505)), :retryable_by_status?) + refute_predicate(Dexpace::ProtocolError.for(build_response(404)), :retryable_by_status?) + assert_predicate(Dexpace::ProtocolError.for(build_response(408)), :retryable_by_status?) + assert_predicate(Dexpace::ProtocolError.for(build_response(429)), :retryable_by_status?) + end + + test "RETRY-3: no per-subclass constant -- a subclass inherits the computed flag" do + subclass = Class.new(Dexpace::ProtocolError) + + assert_predicate(subclass.new(build_response(502)), :retryable_by_status?) + refute_predicate(subclass.new(build_response(501)), :retryable_by_status?) + end + + test "P6-10: a ProtocolError does NOT answer XCUT-6's generic #retryable? capability" do + error = Dexpace::ProtocolError.for(build_response(503)) + + refute_respond_to(error, :retryable?) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error)) + end + + test "NFR-4: the constructor keeps 4b's single positional parameter and the two factories" do + assert_equal(1, Dexpace::ProtocolError.instance_method(:initialize).arity) + assert_instance_of(Dexpace::ProtocolError, Dexpace::ProtocolError.for(build_response(503))) + assert_nil(Dexpace::ProtocolError.for_or_nil(build_response(200))) + end + end end diff --git a/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb b/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb new file mode 100644 index 0000000..4ab8beb --- /dev/null +++ b/gems/dexpace-core/test/dexpace/error/retry_predicate_error_test.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" + +# RETRY-40, RETRY-25, P6-11, P6-56: the well-typed abort the two stage-based retry drivers raise +# when a caller's should_retry predicate itself raises. The raise sites and the wrapping are +# asserted where they happen (retry_step_test.rb, async_retry_step_test.rb); this mirror pins the +# class's own shape. +class DexpaceRetryPredicateErrorTest < DexpaceTestCase + test "is caught by rescue Dexpace::Error through Module#===" do + caught = begin + raise Dexpace::RetryPredicateError, "the should_retry predicate raised RuntimeError" + rescue Dexpace::Error => error + error + end + + assert_instance_of(Dexpace::RetryPredicateError, caught) + assert_kind_of(Dexpace::Suppressible, caught) + end + + test "subclasses StandardError and includes Dexpace::Error" do + assert_operator(Dexpace::RetryPredicateError, :<, ::StandardError) + assert_includes(Dexpace::RetryPredicateError.ancestors, Dexpace::Error) + end + + # P6-56: flat under Dexpace::, on the tree's one-error-per-file convention, and not under + # Dexpace::Resilience, where the plan named it half the time. + test "P6-56: lives flat under Dexpace::, not under Dexpace::Resilience" do + assert_equal("Dexpace::RetryPredicateError", Dexpace::RetryPredicateError.name) + refute(Dexpace::Resilience.const_defined?(:RetryPredicateError, false)) + end + + # RETRY-40: the predicate's own failure travels as #cause, which a `cause:` at the raise site + # sets -- a genuine wrap of a fresh failure, so pipeline/7ce4431d's `cause: nil` rule for a + # CARRIED error does not apply here. + test "RETRY-40: carries the predicate's raise as #cause when raised with cause:" do + inner = ::RuntimeError.new("boom") + wrapped = begin + raise Dexpace::RetryPredicateError.new("the should_retry predicate raised RuntimeError"), + cause: inner + rescue Dexpace::RetryPredicateError => error + error + end + + assert_same(inner, wrapped.cause) + assert_equal("the should_retry predicate raised RuntimeError", wrapped.message) + end + + test "carries no fields of its own" do + assert_empty(Dexpace::RetryPredicateError.instance_methods(false)) + end + + test "is not Dexpace::InvalidArgumentError -- a misbehaving hook is not a bad argument" do + refute_operator(Dexpace::RetryPredicateError, :<, Dexpace::InvalidArgumentError) + end +end diff --git a/gems/dexpace-core/test/dexpace/http_date_test.rb b/gems/dexpace-core/test/dexpace/http_date_test.rb index ea00d81..a2691be 100644 --- a/gems/dexpace-core/test/dexpace/http_date_test.rb +++ b/gems/dexpace-core/test/dexpace/http_date_test.rb @@ -103,7 +103,7 @@ class ParseTest < DexpaceTestCase "Sun,\n06 Nov 1994 08:49:37 GMT", # \s+ would accept this; a literal space does not "Sun, 06 Nov 1994 08:49:37 GMT\n", # \Z would accept this; \z does not "Sun, 06 Nov 1994 08:49:37 GMT junk", - "Sun, 6 Nov 1994 08:49:37 GMT", # single-digit day + "06 Nov 1994 08:49:37 GMT", # no weekday at all: CFG-31's prefix is required "Sun, 06 Nov 1994 08:49:37", # no zone "Sun, 06 Nov 1994 08:49:37 +0100", # a non-zero offset is not one of the four tokens "Sun, 06 Nov 1994 08:49:37 EST", @@ -113,6 +113,23 @@ class ParseTest < DexpaceTestCase end end + # Phase 6a's R1: RETRY-15's HTTP-date tolerances, asserted against the ONE parser. The day + # group was widened from (\d{2}) to (\d{1,2}); the weekday was already informational (CFG-30) + # and stays required (CFG-31) -- the row above asserts the bare form still fails. + test "RETRY-15: a single-digit day parses as that day, the weekday still informational" do + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("Sun, 6 Nov 1994 08:49:37 GMT")) + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("Mon, 6 Nov 1994 08:49:37 GMT")) + assert_equal(EXAMPLE, Dexpace::HTTPDate.parse("xyz, 6 nov 1994 08:49:37 utc")) + assert_equal(Time.utc(2024, 1, 1), Dexpace::HTTPDate.parse("Mon, 1 Jan 2024 00:00:00 GMT")) + end + + test "RETRY-15: the widening admits digit COUNT, not calendar nonsense or a third digit" do + ["Sun, 0 Nov 1994 08:49:37 GMT", "Sun, 006 Nov 1994 08:49:37 GMT", + "Sun, 6 Nov 1994 08:49:37 GMT", "Sun, 6 Nov 1994 08:49:37 GMT",].each do |text| + assert_raises(Dexpace::InvalidArgumentError, text.inspect) { Dexpace::HTTPDate.parse(text) } + end + end + test "CFG-31: a well-formed but impossible date fails as a parse error, not an ArgumentError" do ["Sun, 32 Nov 1994 08:49:37 GMT", "Sun, 31 Nov 1994 08:49:37 GMT", "Sun, 06 Nov 1994 24:00:00 GMT", "Sun, 06 Nov 1994 08:60:37 GMT", diff --git a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb index aa05e83..d619d62 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/async_step_test.rb @@ -223,6 +223,7 @@ class SettlementTest < DexpaceTestCase meter: meter,) raising_cursor = ::Object.new def raising_cursor.call(_request) = raise(::IOError, "synchronous head failure") + def raising_cursor.bundle = Dexpace::Instrumentation::Bundle::NONE assert_raises(::IOError) { step.call(build_request, raising_cursor) } assert_equal(1, factory.tracers.first.spans.first.finished_at.size) @@ -289,6 +290,7 @@ class SharedShapeTest < DexpaceTestCase completer.future end cursor.define_singleton_method(:fork) { |state: nil| raise "forked with #{state.inspect}" } + cursor.define_singleton_method(:bundle) { Dexpace::Instrumentation::Bundle::NONE } request = build_request assert_same(completer.future, step.call(request, cursor)) @@ -487,4 +489,27 @@ class ThreadBoundaryTest < DexpaceTestCase assert_equal(:error, sink.entries[1].severity) end end + + # Phase 6a's Task 8 on the async runtime: the seeded bundle's factory wins in the head. + class BundlePrecedenceTest < DexpaceTestCase + include Fixtures + + test "cursor bundle: a seeded bundle's tracer factory wins over the async step's keyword" do + own = Dexpace::RecordingTracerFactory.new + from_bundle = Dexpace::RecordingTracerFactory.new + bundle = Dexpace::Instrumentation::Bundle.build( + trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C, tracer_factory: from_bundle, + ) + step = async_step(RecordingSink.new, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + transport = FakeAsyncTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle).value + + assert_equal(1, from_bundle.tracers.size) + assert_equal(1, from_bundle.tracers.first.spans.first.finished_at.size) + assert_empty(own.tracers) + end + end end diff --git a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb index a15d6f4..e3cb509 100644 --- a/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb +++ b/gems/dexpace-core/test/dexpace/instrumentation/step_test.rb @@ -142,6 +142,7 @@ def call(request, _options, _cancellation) response end cursor.define_singleton_method(:fork) { |state: nil| raise "forked with #{state.inspect}" } + cursor.define_singleton_method(:bundle) { Dexpace::Instrumentation::Bundle::NONE } assert_same(response, step.call(request, cursor)) assert_equal([request], cursor.instance_variable_get(:@calls)) @@ -710,4 +711,72 @@ def factory.tracer(*, **) = raise(::StandardError, "tracer failure") assert_same(Dexpace::Instrumentation::NO_SPAN, Dexpace::Instrumentation::Tracing.current_span) end end + + # Phase 6a's Task 8: the reconciled precedence's first clause, live through Cursor#bundle. + class BundlePrecedenceTest < DexpaceTestCase + include Fixtures + + W3C = Dexpace::Instrumentation::TraceIdFlavour::W3C + + def bundle_with(factory, span: Dexpace::Instrumentation::NO_SPAN) + Dexpace::Instrumentation::Bundle.build(trace_id: "a" * 32, span_id: "b" * 16, flavour: W3C, + tracer_factory: factory, span: span,) + end + + test "cursor bundle: a seeded bundle's tracer factory wins over the step's own keyword" do + own = Dexpace::RecordingTracerFactory.new + from_bundle = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + transport = FakeTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle_with(from_bundle)) + + assert_equal(1, from_bundle.tracers.size) + assert_equal("GET", from_bundle.tracers.first.name) + assert_equal(1, from_bundle.tracers.first.spans.first.finished_at.size) + assert_empty(own.tracers, "the keyword is the SECOND clause") + end + + test "cursor bundle: with no seeded bundle the step's keyword resolves, as in phase 5" do + own = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, tracer_factory: own) + request = build_request + + drive(step, request, response: build_response(request)) + + assert_equal(1, own.tracers.size) + end + + test "cursor bundle / OBS-23: a valid seeded bundle pushes its ids onto the diagnostics" do + seen = {} + factory = Dexpace::RecordingTracerFactory.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE) + request = build_request + transport = FakeTransport.new(response: build_response(request), before_return: lambda { + seen[:trace] = ::Fiber[Dexpace::Instrumentation::Diagnostics::TRACE_ID] + seen[:span] = ::Fiber[Dexpace::Instrumentation::Diagnostics::SPAN_ID] + seen[:current] = Dexpace::Instrumentation::Tracing.current_span + },) + + pipeline(step, transport).call(request, bundle: bundle_with(factory)) + + assert_equal("a" * 32, seen[:trace]) + assert_equal("b" * 16, seen[:span]) + assert_same(factory.tracers.first.spans.first, seen[:current], "a recording span was current") + assert_nil(::Fiber[Dexpace::Instrumentation::Diagnostics::TRACE_ID], "restored after") + assert_same(Dexpace::Instrumentation::NO_SPAN, Dexpace::Instrumentation::Tracing.current_span) + end + + test "cursor bundle: the meter has no bundle source -- the keyword resolves regardless" do + meter = Dexpace::RecordingMeter.new + step = Step.build(logger: Logger::NULL, level: HTTPLogging::NONE, meter: meter) + request = build_request + transport = FakeTransport.new(response: build_response(request)) + + pipeline(step, transport).call(request, bundle: bundle_with(Dexpace::RecordingTracerFactory.new)) + + assert_equal(1, meter.counters.first.records.size) + end + end end diff --git a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb index d3d85c3..893be2b 100644 --- a/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb +++ b/gems/dexpace-core/test/dexpace/pipeline/cursor_test.rb @@ -310,7 +310,9 @@ class StateTest < DexpaceTestCase methods = CURSOR.public_instance_methods(false) assert_empty(methods.grep(/=\z/), "no writer of any name may appear on Cursor") - assert_equal(%i[call cancellation fork may_fork? options request spent? state], methods.sort) + # #bundle is phase 6a's Task 8 reader; the pin grew by one reader and no writer. + assert_equal(%i[bundle call cancellation fork may_fork? options request spent? state], + methods.sort,) end test "R11 assertion 2: the reachable state map is frozen, and unwritten stages share one" do @@ -411,4 +413,86 @@ def call(req, cur) assert_same(parent_state, fork_state) end end + + # Phase 6a's Task 8: the per-call instrumentation bundle, the one reader the widening added. + class BundleTest < DexpaceTestCase + include Cursors + + BUNDLE = Dexpace::Instrumentation::Bundle + W3C = Dexpace::Instrumentation::TraceIdFlavour::W3C + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, flavour: W3C) + end + + test "cursor bundle: #bundle defaults to Bundle::NONE on the root cursor" do + assert_same(BUNDLE::NONE, cursor_over([]).bundle) + end + + test "cursor bundle: a seeded bundle is readable at every position and survives #fork" do + bundle = seeded + seen = [] + probe = lambda do |req, cur| + seen << cur.bundle + fork = cur.fork + seen << fork.bundle + fork.call(req) + end + reader = lambda do |req, cur| + seen << cur.bundle + cur.call(req) + end + root = CURSOR.build( + drive: SYNC_DRIVER.new(DummySyncPipeline.new( + [entry(STAGES::RETRY, probe), + entry(STAGES::AUTH, reader),], @transport, + )), + request: @request, options: @options, cancellation: @cancellation, bundle: bundle, + ) + + root.call(@request) + + assert_same(bundle, root.bundle) + assert_equal(3, seen.size) + seen.each { |carried| assert_same(bundle, carried) } + end + + test "cursor bundle: a fork of a fork carries the same object; state: does not touch it" do + bundle = seeded + forks = [] + probe = lambda do |req, cur| + first = cur.fork(state: { cross_origin: true }) + forks << first + forks << cur.fork + first.call(req) + end + CURSOR.build( + drive: SYNC_DRIVER.new(DummySyncPipeline.new([entry(STAGES::REDIRECT, probe)], @transport)), + request: @request, options: @options, cancellation: @cancellation, bundle: bundle, + ).call(@request) + + forks.each { |fork| assert_same(bundle, fork.bundle) } + assert_equal({ cross_origin: true }, forks.first.state(STAGES::REDIRECT)) + end + + test "cursor bundle: .build refuses anything that is not a Bundle" do + drive = SYNC_DRIVER.new(DummySyncPipeline.new([], @transport)) + + assert_raises(Dexpace::InvalidArgumentError) do + CURSOR.build(drive: drive, request: @request, options: @options, + cancellation: @cancellation, bundle: nil,) + end + assert_raises(Dexpace::InvalidArgumentError) do + CURSOR.build(drive: drive, request: @request, options: @options, + cancellation: @cancellation, bundle: { trace_id: "x" },) + end + end + + test "cursor bundle: there is still no writer, and #bundle is the only new reader" do + writers = CURSOR.public_instance_methods(false).grep(/=\z|bundle./) + + assert_empty(writers) + refute_respond_to(cursor_over([]), :bundle=) + end + end end diff --git a/gems/dexpace-core/test/dexpace/pipeline_test.rb b/gems/dexpace-core/test/dexpace/pipeline_test.rb index 984b5de..a502144 100644 --- a/gems/dexpace-core/test/dexpace/pipeline_test.rb +++ b/gems/dexpace-core/test/dexpace/pipeline_test.rb @@ -348,4 +348,74 @@ class CompositionTest < DexpaceTestCase assert_same(base, nest.build.transport) end end + + # Phase 6a's Task 8: the `bundle:` keyword seeds Cursor#bundle for one call. + class BundleSeedingTest < DexpaceTestCase + BUNDLE = Dexpace::Instrumentation::Bundle + + def seeded + BUNDLE.build(trace_id: "a" * 32, span_id: "b" * 16, + flavour: Dexpace::Instrumentation::TraceIdFlavour::W3C,) + end + + def request + Dexpace::Request.build(method: Dexpace::Method::GET, url: "https://example.test/", + headers: Dexpace::Headers::EMPTY,) + end + + test "cursor bundle: Pipeline#call seeds the cursor's bundle, NONE when omitted" do + seen = [] + step = lambda { |req, cur| + seen << cur.bundle + cur.call(req) + } + transport = ->(_req, _opts, _canc) { :response } + pipeline = Dexpace::Pipeline::Builder.new(transport: transport) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build + bundle = seeded + + pipeline.call(request) + pipeline.call(request, bundle: bundle) + pipeline.call(request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none, + bundle: bundle,) + + assert_same(BUNDLE::NONE, seen[0]) + assert_same(bundle, seen[1]) + assert_same(bundle, seen[2]) + end + + test "cursor bundle: the empty pipeline dispatches with no cursor and accepts the keyword" do + calls = [] + transport = lambda { |req, opts, canc| + calls << [req, opts, canc] + :response + } + pipeline = Dexpace::Pipeline.direct(transport) + + assert_equal(:response, pipeline.call(request, bundle: seeded)) + assert_equal(1, calls.size) + end + + test "cursor bundle / SEAM-11: a Pipeline is still a Transport by the duck type" do + pipeline = Dexpace::Pipeline.direct(->(_r, _o, _c) { :response }) + + assert(Dexpace::Transport.conforms?(pipeline)) + assert(Dexpace::Registry.callable?(pipeline, arity: 3)) + end + + test "cursor bundle: a non-Bundle is refused before any step runs" do + ran = false + step = lambda { |req, cur| + ran = true + cur.call(req) + } + pipeline = Dexpace::Pipeline::Builder.new(transport: ->(*) { :r }) + .append(step, stage: Dexpace::Pipeline::Stages::PRE_RETRY) + .build + + assert_raises(Dexpace::InvalidArgumentError) { pipeline.call(request, bundle: :none) } + refute(ran) + end + end end diff --git a/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb new file mode 100644 index 0000000..1b4ae88 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/async_retry_step_test.rb @@ -0,0 +1,593 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_async_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" +require_relative "../../support/parking_scheduler" +require_relative "../../support/fake_config_source" + +# Exercises: RETRY-2, RETRY-5, RETRY-7, RETRY-8, RETRY-13, RETRY-20, RETRY-23, RETRY-24, +# RETRY-25, RETRY-26, RETRY-30, RETRY-31, RETRY-32, RETRY-33, RETRY-34, RETRY-35, RETRY-39, +# RETRY-40, RETRY-41, RETRY-42, RETRY-44, RETRY-45, OBS-29, OBS-30, PIPE-29, PIPE-30, XCUT-3, +# CFG-18, P6-5, P6-7, P6-60, R2 +# +# The asynchronous stage-based retry step, driven through a REAL Dexpace::AsyncPipeline over a +# ScriptedAsyncTransport whose futures settle INLINE -- the shape that overflowed the plan's +# recursive pump -- and, for R2's scheduler route, through 5a's ParkingScheduler, the one double +# that can drive a timed wait (its loop runs in #close, so every case joins the scheduler's +# thread). No test here sleeps a thread: a zero-length delay completes inline, a positive one +# either parks a fiber or fails the future. Hermetic: a build with no settings reads the +# configured MAX_RETRY_ATTEMPTS, so every nested class runs behind a FakeConfigSource env seam. +# Split into nested classes under Metrics/ClassLength. +class DexpaceResilienceAsyncRetryStepTest < DexpaceTestCase + AsyncRetryStep = Dexpace::Resilience::AsyncRetryStep + Stages = Dexpace::Pipeline::Stages + + # The async pipeline shape every nested class drives, and the seam. + module Fixtures + include RetryFixtures + + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + + def pipeline(step, script, settle_later: false) + transport = ScriptedAsyncTransport.new(script, settle_later: settle_later) + wrapper = RecordingWrapper.new(step) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(wrapper).build_async + [built, transport, wrapper] + end + + def drive(step, script, request: retry_request, options: Dexpace::RequestOptions::EMPTY, + cancellation: Dexpace::Cancellation.none) + built, transport, wrapper = pipeline(step, script) + future = built.call(request, options, cancellation) + [future, transport, wrapper] + end + + # Zero-length waits, so no scheduler is needed and nothing sleeps (RETRY-31). + def inline_settings(**overrides) + retry_settings(initial_delay: 0.0, **overrides) + end + end + include Fixtures + + test "declares Stages::RETRY, is frozen, hides .new, and returns a Future" do + step = AsyncRetryStep.build + + assert_same(Stages::RETRY, step.stage) + assert_predicate(step, :frozen?) + refute_respond_to(AsyncRetryStep, :new) + future, = drive(step, [retry_response(200)]) + + assert_kind_of(Dexpace::Async::Future, future) + assert_equal(200, future.value.status.code) + end + + test "pipeline/86343352: forks for EVERY drive including the first, never calls its cursor" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + future, transport, wrapper = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, future.value.status.code) + assert_equal(3, transport.calls.size) + assert_equal(3, wrapper.cursors.first.forks) + assert_equal(0, wrapper.cursors.first.calls) + end + + test "RETRY-44: the SAME request object is re-sent on every attempt" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + request = retry_request + future, transport, = drive(step, [retry_response(503), retry_response(200)], request: request) + + assert_equal(200, future.value.status.code) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-42: eight threads through one frozen step keep their attempt counts apart" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + results = Array.new(8) do + ::Thread.new do + future, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + [future.value.status.code, transport.calls.size] + end + end.map(&:value) + + assert_equal([[200, 3]] * 8, results) + end + + # RETRY-30, RETRY-31, R2's three routes, RETRY-45. + class TrampolineTest < DexpaceTestCase + include Fixtures + + # The depth probe: a transport that records the call-stack depth at every attempt. A pump + # that recursed would show a depth growing with the attempt number; a flat one shows the + # same depth at attempt 1 and attempt N. + def depth_probe(depths, response) + lambda do |_request, _options, _cancellation| + depths << caller.size + response + end + end + + test "RETRY-30 / RETRY-31: 2000 zero-length retries build no N-deep stack, measured" do + depths = [] + failures = Array.new(2_000) { depth_probe(depths, retry_response(503)) } + script = failures + [depth_probe(depths, retry_response(200))] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2_000)) + + assert_nil(::Fiber.scheduler, "no scheduler anywhere in this test") + future, transport, = drive(step, script) + + assert_equal(200, future.value.status.code) + assert_equal(2_001, transport.calls.size) + assert_equal(2_001, depths.size) + assert_equal(depths.first, depths.last, "attempt 1 and attempt 2001 at the same depth") + assert_equal(1, depths.uniq.size, "every attempt at one depth: the pump is flat") + end + + test "R2 / RETRY-31: a zero-length delay completes inline and re-arms the pump, no scheduler" do + assert_nil(::Fiber.scheduler) + clock = FakeClock.new + step = AsyncRetryStep.build(settings: inline_settings(clock: clock, max_retries: 1)) + future, transport, = drive(step, [retry_response(503), retry_response(200)]) + + assert_predicate(future, :settled?, "settled before #call returned: no wait happened") + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + assert_empty(clock.sleeps, "RETRY-31: the wait is Async.delay's, never the clock's sleep") + end + + test "R2 / CFG-18: a positive delay under a scheduler parks the fiber, blocks no thread" do + scheduler = ParkingScheduler.new + settings = retry_settings(initial_delay: 0.01, max_retries: 2) + step = AsyncRetryStep.build(settings: settings) + future = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, = drive(step, [retry_response(503), retry_response(503), retry_response(200)]) + end + end + thread.join + + assert_predicate(future, :settled?) + assert_equal(200, future.value.status.code) + assert_operator(scheduler.block_count, :>=, 2, "one park per positive wait") + assert_equal(0, scheduler.kernel_sleep_count) + assert_empty(settings.clock.sleeps, "RETRY-31 / RETRY-26: no Clock#sleep beside the park") + end + + test "R2 / RETRY-33: a positive delay with NO scheduler fails the future with SeamError" do + assert_nil(::Fiber.scheduler) + clock = FakeClock.new + settings = retry_settings(clock: clock, initial_delay: 0.01, max_retries: 2) + step = AsyncRetryStep.build(settings: settings) + first = RetryableError.new("first") + future, transport, = drive(step, [first, retry_response(200)]) + + assert_predicate(future, :settled?, "not a hang, not a blocking fallback") + error = assert_raises(Dexpace::SeamError) { future.value } + + assert_match(/Fiber\.set_scheduler/, error.message) + assert_equal([first], Dexpace.suppressed(error), "the trail travels with the SeamError") + assert_equal(1, transport.calls.size, "no second attempt was launched") + assert_empty(clock.sleeps, "the clock was never asked to sleep: no blocking fallback") + end + + test "RETRY-31 / RETRY-26: the async driver names no blocking sleep, by text" do + # Every behavioural case in this suite runs on a FakeClock, whose #sleep records and + # returns, so a blocking Clock#sleep slipped in beside Async.delay kept the whole suite green + # on its own (R1-2); the three clock.sleeps assertions above and this scan are what turn it + # red. The one wait the driver may name is Async.delay's. + path = File.expand_path("../../../lib/dexpace/resilience/async_retry_step.rb", __dir__) + code = File.read(path).gsub(/^\s*#.*$/, "") + + assert_empty(code.scan(/\bsleep\b/), "async_retry_step.rb names a sleep") + assert_equal(1, code.scan("Dexpace::Async.delay(").size, "the one wait is Async.delay's") + end + + test "RETRY-45: no scheduler is installed, read or shut down by the driver" do + scheduler = ParkingScheduler.new + closed = false + scheduler.define_singleton_method(:close) do + closed = true + super() + end + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.001, max_retries: 1)) + future = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule { future, = drive(step, [retry_response(503), retry_response(200)]) } + + refute(closed, "the driver did not close the scheduler while it was running") + end + thread.join + + assert_equal(200, future.value.status.code) + assert(closed, "the interpreter closed it at thread end, as CFG-18's own suite relies on") + end + end + + # RETRY-32, RETRY-33, RETRY-23: the async terminal paths. + class TerminalPathsTest < DexpaceTestCase + include Fixtures + + test "RETRY-32: a cancelled future launches no further attempt and closes the in-flight one" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + in_flight = retry_response(503) + transport = ScriptedAsyncTransport.new([in_flight, retry_response(503)], settle_later: true) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + future = built.call(retry_request) + + refute_predicate(future, :settled?) + future.cancel(:caller) + + assert_predicate(future, :cancelled?) + assert(transport.settle_next!, "the abandoned attempt now lands") + assert_predicate(in_flight.body, :closed?, "closed rather than leaked") + assert_equal(1, transport.calls.size, "no second attempt after the cancellation") + assert_nil(transport.settle_next!) + end + + test "RETRY-32: a deadline that cancels the future during a wait stops the loop" do + scheduler = ParkingScheduler.new + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.05, max_retries: 5)) + transport = nil + outcome = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, transport, = drive(step, Array.new(6) { retry_response(503) }) + future.cancel(:deadline) + outcome = future.cancelled? + end + end + thread.join + + assert(outcome) + assert_equal(1, transport.calls.size) + end + + test "RETRY-33: a throwing should_retry completes the future exceptionally, response closed" do + open_response = retry_response(503) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + should_retry: ->(*) { raise "boom" },) + future, = drive(step, [open_response]) + + assert_predicate(future, :settled?, "the one outcome RETRY-33 names is a hang; not this") + error = assert_raises(Dexpace::RetryPredicateError) { future.value } + + assert_equal("boom", error.cause.message) + assert_predicate(open_response.body, :closed?) + end + + test "RETRY-33 / RETRY-35: a throwing delay computation closes the response and fails" do + open_response = retry_response(503) + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + delay_override: ->(*) { raise ::NotImplementedError, "fatal" },) + transport = ScriptedAsyncTransport.new([open_response]) + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + # The fatal family is re-raised AFTER the future is failed (RETRY-25 with RETRY-33), and + # the async driver's normalisation lets a ScriptError through, so it reaches this frame. + assert_raises(::NotImplementedError) { built.call(retry_request) } + assert_predicate(open_response.body, :closed?) + end + + test "RETRY-25: a fatal-family error from the downstream is surfaced unchanged, unretried" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + fatal = ::NoMemoryError.new("oom") + completer = Dexpace::Async::Completer.new + completer.fail(fatal) + transport = ->(_request, _options, _cancellation) { completer.future } + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + future = built.call(retry_request) + + surfaced = assert_raises(::NoMemoryError) { future.value } + + assert_same(fatal, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + end + + # RETRY-33, RETRY-35, OBS-30: the tracer and its factory inside the pump's fence -- a raise in any + # callback fails the future with the response it was handed closed first, on both terminal paths. + class TracerFencesTest < DexpaceTestCase + include Fixtures + + test "RETRY-33: a throwing tracer callback completes the future exceptionally, no hang" do + exploding = ::Object.new + def exploding.attempt_started(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + future, = drive(step, [retry_response(200)]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + end + + test "RETRY-33 / RETRY-35: a tracer raising in attempt_failed closes the response, no hang" do + open_response = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { exploding },) + future, transport, = drive(step, [open_response, retry_response(200)]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + assert_predicate(open_response.body, :closed?, "the sync driver closes it the same way") + assert_equal(1, transport.calls.size) + end + + test "RETRY-33 / RETRY-35: a tracer raising in retries_exhausted closes the terminal one" do + # R1-1's twin: Pump#finish runs inside guarded(response), so the terminal error-status + # response is closed and the future fails with the tracer's raise -- the sync driver's + # fenced settle now does the same. + terminal = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = nil + def exploding.retries_exhausted(*) = raise("tracer blew up") + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + future, transport, = drive(step, [retry_response(503), terminal]) + + assert_predicate(future, :settled?) + error = assert_raises(::RuntimeError) { future.value } + + assert_equal("tracer blew up", error.message) + assert_predicate(terminal.body, :closed?, "the sync driver closes it the same way") + assert_equal(2, transport.calls.size) + end + + test "RETRY-33: a throwing factory completes the future exceptionally" do + step = AsyncRetryStep.build(settings: inline_settings, + http_tracer_factory: ->(_cursor) { raise "no tracer" },) + future, transport, = drive(step, [retry_response(200)]) + + assert_raises(::RuntimeError) { future.value } + assert_equal(0, transport.calls.size) + end + end + + # RETRY-23 / RECOV-27 on the async path: the token during a wait and at the boundary. + class CancellationTest < DexpaceTestCase + include Fixtures + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 3)) + fatal = RetryableFatal.new("oom, and lying about it") + completer = Dexpace::Async::Completer.new + completer.fail(fatal) + sends = 0 + transport = lambda do |_request, _options, _cancellation| + sends += 1 + completer.future + end + built = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + + surfaced = assert_raises(RetryableFatal) { built.call(retry_request).value } + + assert_same(fatal, surfaced) + assert_equal(1, sends) + end + + test "RETRY-23 / XCUT-3: a token cancelled during a wait cancels the timer, fails the future" do + scheduler = ParkingScheduler.new + source = Dexpace::Cancellation.source + step = AsyncRetryStep.build(settings: retry_settings(initial_delay: 0.05, max_retries: 5)) + first = RetryableError.new("first") + result = nil + transport = nil + thread = ::Thread.new do + ::Fiber.set_scheduler(scheduler) + ::Fiber.schedule do + future, transport, = drive(step, [first, retry_response(200)], + cancellation: source.token,) + source.cancel(:token) + result = begin + future.value + rescue Dexpace::CancelledError => error + error + end + end + end + thread.join + + assert_kind_of(Dexpace::CancelledError, result) + assert_equal(:token, result.reason) + assert_equal([first], Dexpace.suppressed(result)) + assert_equal(1, transport.calls.size) + end + + test "RETRY-23: a cancelled token is checked at the top of every attempt" do + source = Dexpace::Cancellation.source + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + script = [lambda { |*| + source.cancel(:mid) + retry_response(503) + }, retry_response(200),] + future, transport, = drive(step, script, cancellation: source.token) + + error = assert_raises(Dexpace::CancelledError) { future.value } + + assert_equal(:mid, error.reason) + assert_equal(1, transport.calls.size) + end + + test "RETRY-23 / P6-60: a should_retry answering true cannot retry a downstream cancellation" do + seen = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5), + should_retry: lambda { |failure, _request| + seen << failure + true + },) + future, transport, = drive(step, [Dexpace::CancelledError.new(:downstream), + retry_response(200),],) + + error = assert_raises(Dexpace::CancelledError) { future.value } + + assert_equal(:downstream, error.reason) + assert_equal(1, transport.calls.size) + assert_empty(seen, "the predicate is never consulted for a cancellation") + end + + test "RETRY-23 / P6-60: a cancellation wrapped by a retryable error is terminal on this path" do + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + future, transport, = drive(step, [wrapped, retry_response(200)]) + + assert_same(wrapped, assert_raises(RetryableError) { future.value }) + assert_equal(1, transport.calls.size) + end + end + + # The shared decision and delay logic, asserted on the async path too (design §11.12). + class SharedShapeTest < DexpaceTestCase + include Fixtures + + test "RETRY-5 / RETRY-7 / RETRY-8: the re-sendability gate, predicate or no predicate" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5), + should_retry: ->(*) { true },) + future, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, future.value.status.code) + assert_equal(1, transport.calls.size) + future, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST", body: replayable_body),) + + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + end + + test "RETRY-34: the terminal failure carries the whole prior trail, skipping itself" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + errors = Array.new(3) { |i| RetryableError.new("attempt #{i + 1}") } + future, = drive(step, errors) + + surfaced = assert_raises(RetryableError) { future.value } + + assert_same(errors[2], surfaced) + assert_equal(errors[0..1], Dexpace.suppressed(surfaced)) + same = RetryableError.new("same") + future, = drive(step, [same, same, same]) + + assert_same(same, assert_raises(RetryableError) { future.value }) + assert_empty(Dexpace.suppressed(same)) + end + + test "RETRY-34: a terminal error status is DELIVERED, never failed" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1)) + future, transport, = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, future.value.status.code) + assert_equal(2, transport.calls.size) + refute_predicate(future.value.body, :closed?) + end + + test "RETRY-2 / RETRY-24: the capability query decides the throwable branch" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2)) + future, transport, = drive(step, [RetryableError.new("timeout"), retry_response(200)]) + + assert_equal(200, future.value.status.code) + assert_equal(2, transport.calls.size) + future, transport, = drive(step, [UnretryableError.new("no"), retry_response(200)]) + + assert_raises(UnretryableError) { future.value } + assert_equal(1, transport.calls.size) + end + + test "RETRY-20 / RETRY-39: the override sees the response on one path, the error on another" do + seen = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2, initial_delay: 0.0), + delay_override: lambda { |attempt, response, error| + seen << [attempt, response&.status&.code, error.class] + 0.0 + },) + future, = drive(step, [retry_response(503, headers: { "Retry-After" => "0" }), + RetryableError.new("x"), retry_response(200),],) + + assert_equal(200, future.value.status.code) + assert_equal([[1, 503, NilClass], [2, nil, RetryableError]], seen) + end + + test "RETRY-40: a throwing delay-override is logged and falls back on the async path too" do + sink = RecordingSink.new + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + delay_override: ->(*) { raise "override down" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + future, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(200, future.value.status.code) + assert_equal([:warn], sink.entries.map(&:severity)) + end + + test "RETRY-41: the per-call override wins, on the async path too" do + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 5)) + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 1, tags: {}) + future, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(503, future.value.status.code) + assert_equal(2, transport.calls.size) + end + + test "OBS-29 / P6-7: the per-attempt group, from one tracer per operation, with the cursor" do + tracer = Dexpace::RecordingHTTPTracer.new + cursors = [] + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 1), + http_tracer_factory: lambda { |cursor| + cursors << cursor + tracer + },) + future, _transport, wrapper = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, future.value.status.code) + assert_equal([wrapper.cursors.first], cursors) + assert_equal(%i[attempt_started attempt_failed attempt_started retries_exhausted], + tracer.events.map(&:first),) + tracer.events.each { |event| assert_same(wrapper.cursors.first, event[1]) } + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2]) + assert_in_delta(0.0, tracer.events[1][3]) + end + + test "OBS-29: a failure that was never retryable ends with no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + step = AsyncRetryStep.build(settings: inline_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + future, = drive(step, [retry_response(404)]) + + assert_equal(404, future.value.status.code) + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb b/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb new file mode 100644 index 0000000..49838bf --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/budget_equivalence_test.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/scripted_async_transport" +require_relative "../../support/fake_config_source" + +# Exercises: RETRY-14, RETRY-13, RECOV-30, RETRY-12, P6-6 +# +# The convergence point inside 6a (the charter's second): the three drivers -- the recovery +# engine and the two stage steps -- built from ONE RetrySettings, driven against an identical +# failure sequence, exhaust after the SAME number of wire sends. Under the shared defaults that +# number is three: the stage stack's max-retries (2) plus one initial send, which is the +# recovery stack's max-attempts (3), an identity and not a coincidence (P6-6). No lib/ mirror: +# this file asserts an equivalence between three files rather than one file's contract. +class DexpaceResilienceBudgetEquivalenceTest < DexpaceTestCase + include RetryFixtures + + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + + def recovery_sends(settings, failures) + transport = ScriptedTransport.new(failures) + engine = Dexpace::Resilience::RecoveryRetry.build(transport: transport, settings: settings) + assert_raises(Dexpace::ProtocolError) do + engine.call(retry_request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + transport.calls.size + end + + def stage_sends(settings, failures) + transport = ScriptedTransport.new(failures) + step = Dexpace::Resilience::RetryStep.build(settings: settings) + response = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build + .call(retry_request) + + assert_equal(503, response.status.code) + transport.calls.size + end + + def async_stage_sends(settings, failures) + transport = ScriptedAsyncTransport.new(failures) + step = Dexpace::Resilience::AsyncRetryStep.build(settings: settings) + future = Dexpace::Pipeline::Builder.new(transport: transport).append(step).build_async + .call(retry_request) + + assert_equal(503, future.value.status.code) + transport.calls.size + end + + def unavailable(count) = Array.new(count) { retry_response(503) } + + test "RETRY-14: under the shared defaults all three drivers exhaust after exactly three sends" do + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + clock: FakeClock.new,) + + assert_equal(2, settings.max_retries, "RETRY-12's default, the stage vocabulary") + recovery = recovery_sends(settings, unavailable(6)) + stage = stage_sends(settings, unavailable(6)) + async = async_stage_sends(settings, unavailable(6)) + + assert_equal(3, recovery, "max-attempts 3: max_retries + the initial send") + assert_equal(3, stage, "max-retries 2 + the initial send") + assert_equal(3, async) + assert_equal(recovery, stage) + assert_equal(stage, async) + end + + test "RETRY-14 / P6-6: the equivalence holds for any configured value, as an identity" do + [0, 1, 4].each do |retries| + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + max_retries: retries, + clock: FakeClock.new,) + sends = retries + 1 + + assert_equal(sends, recovery_sends(settings, unavailable(sends + 2)), "recovery, #{retries}") + assert_equal(sends, stage_sends(settings, unavailable(sends + 2)), "stage, #{retries}") + assert_equal(sends, async_stage_sends(settings, unavailable(sends + 2)), "async, #{retries}") + end + end + + test "RETRY-14 / RETRY-12: the configured MAX_RETRY_ATTEMPTS reaches all three stacks alike" do + Dexpace.configure do |c| + c.env_source = FakeConfigSource.new(Dexpace::Configuration::Keys::MAX_RETRY_ATTEMPTS => "3") + end + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.0, jitter: 0.0, + clock: FakeClock.new,) + + assert_equal(3, settings.max_retries) + assert_equal(4, recovery_sends(settings, unavailable(8))) + assert_equal(4, stage_sends(settings, unavailable(8))) + assert_equal(4, async_stage_sends(settings, unavailable(8))) + end + + test "RETRY-13 / RECOV-30: the two clock-driven stacks wait on ONE schedule from one settings" do + # The async driver waits through Async.delay and never the clock: with no scheduler a + # positive delay fails its future (R2), so its row in this file is the send count above and + # its schedule is the same RetryStepHelpers#resolve_delay the sync step's suite pins. + clocks = Array.new(2) { FakeClock.new } + settings = Dexpace::Resilience::RetrySettings.build(initial_delay: 0.3, multiplier: 3.0, + max_delay: 5.0, jitter: 0.0, + max_retries: 3, clock: clocks[0],) + recovery_sends(settings, unavailable(6)) + stage_sends(settings.with(clock: clocks[1]), unavailable(6)) + schedules = clocks.map { |clock| clock.sleeps.map { _1[:duration] } } + + assert_equal(3, schedules[0].size) + [0.3, 0.9, 2.7].zip(schedules[0]).each do |(expected, actual)| + assert_in_delta(expected, actual) + end + assert_equal(schedules[0], schedules[1], "one calculator, two stacks, one schedule") + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/policy_test.rb b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb new file mode 100644 index 0000000..aad3819 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/policy_test.rb @@ -0,0 +1,615 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/recovery_fixtures" +require_relative "../../support/cyclic_errors" +require_relative "../../support/recording_sink" +require_relative "../../support/warning_capture" + +# Exercises: RETRY-1, RETRY-2, RETRY-9, RETRY-10, RETRY-11, RETRY-12, RETRY-13, RETRY-15, +# RETRY-16, RETRY-17, RETRY-18, RETRY-19, RETRY-20, RETRY-21, RETRY-22, RETRY-23, RETRY-37, +# RETRY-41, RETRY-42, RECOV-17, RECOV-20, RECOV-21, RECOV-22, RECOV-23, RECOV-24, RECOV-25, +# RECOV-26, RECOV-27, RECOV-29, RECOV-30, CFG-35, NFR-6, XCUT-6, XCUT-7, XCUT-9, P6-5, P6-10, +# P6-12, P6-60, P6-61 +# +# The shared policy core: the two-branch classifier consult, the calculator, the resolver, the +# recovery-only budget and the pacing dispatcher. Split into nested classes under +# Metrics/ClassLength: classification, backoff and budgets, then the pacing parser's forms and +# its totality (RETRY-16 is a NEGATIVE suite -- every malformed input asserts nil, never +# assert_nothing_raised). +class DexpaceResiliencePolicyTest < DexpaceTestCase + Policy = Dexpace::Resilience::Policy + + # Shared by the nested classes. + module Fixtures + include RecoveryFixtures + + # A retryable throwable by XCUT-6's capability, and nothing else about it: not an IOError, + # so a concrete-type classifier would refuse it and the capability one must not. + class RetryableByCapability < ::StandardError + def retryable? = true + end + + # An IOError that says it is NOT retryable: the case a concrete-type classifier gets wrong + # in the other direction (5a's R1: StreamError is an IOError and not a transport timeout). + class UnretryableIOError < ::IOError + def retryable? = false + end + + # Exception#cause is assigned by the interpreter at the raise site and cannot be set + # through an ivar, so every cause chain here is built by actually raising. + def wrap(inner, outer_class = ::StandardError, message = "wrapper") + begin + raise inner + rescue inner.class + raise outer_class, message + end + rescue outer_class => error + error + end + + def headers(pairs) + builder = Dexpace::Headers.inbound_builder + pairs.each { |name, value| builder.add(name, value) } + builder.build + end + end + include Fixtures + + test "RETRY-12 / RETRY-13: the defaults are 200 ms, x2, 8 s, 0.2 and two retries (3 sends)" do + assert_in_delta(0.2, Policy::DEFAULT_INITIAL_DELAY) + assert_in_delta(2.0, Policy::DEFAULT_MULTIPLIER) + assert_in_delta(8.0, Policy::DEFAULT_MAX_DELAY) + assert_in_delta(0.2, Policy::DEFAULT_JITTER) + assert_equal(2, Policy::DEFAULT_MAX_RETRIES) + assert_equal(3, Policy::DEFAULT_MAX_RETRIES + 1, "RETRY-14: three total wire sends") + end + + test "XCUT-7: the default configurable set is {408, 429, 500, 502, 503, 504}, frozen, a subset" do + assert_equal(::Set[408, 429, 500, 502, 503, 504], Policy::DEFAULT_RETRYABLE_STATUSES) + assert_predicate(Policy::DEFAULT_RETRYABLE_STATUSES, :frozen?) + Policy::DEFAULT_RETRYABLE_STATUSES.each do |code| + assert(Dexpace::Retryability.retryable_status?(code), "#{code} is in the baked set too") + end + refute_includes(Policy::DEFAULT_RETRYABLE_STATUSES, 599, "the baked set is wider") + end + + test "RETRY-21: the fixed precedence is Retry-After, the two ms variants, then the reset" do + assert_equal(%w[Retry-After retry-after-ms x-ms-retry-after-ms X-RateLimit-Reset], + Policy::DEFAULT_PACING_HEADER_ORDER,) + assert_predicate(Policy::DEFAULT_PACING_HEADER_ORDER, :frozen?) + end + + test "RETRY-18 / RECOV-34: the ceilings are 365 days and the signed 64-bit nanosecond count" do + assert_equal(31_536_000, Policy::MAX_PACING_DELAY_SECONDS) + assert_equal(9_223_372_036_854_775_807, Policy::MAX_DURATION_NANOSECONDS) + end + + test "RETRY-42 / P6-5: Policy is a frozen-constant, extend-self module with no state" do + assert_empty(Policy.instance_variables) + assert_kind_of(Policy, Policy, "extend self, never module_function") + assert_equal( + %i[backoff_delay budget_remaining cancellation? effective_max_retries pacing_delay + retry_eligible? retryable? throwable_retryable?], + Policy.public_instance_methods(false).sort, + ) + end + + # RETRY-1, RETRY-2, RETRY-37, RECOV-17, CFG-35, XCUT-6, XCUT-7, XCUT-9, P6-10. + class ClassificationTest < DexpaceTestCase + include Fixtures + + test "RETRY-37 / XCUT-7: retry_eligible? consults the set alone; a NARROWING set narrows" do + assert(Policy.retry_eligible?(418, set: ::Set[418])) + refute(Policy.retry_eligible?(500, set: ::Set[418])) + # 500 is retryable under 5a's baked classifier and still refused by a set that omits it: + # authoritative-contains, never an intersection with the baked flag. + assert(Dexpace::Retryability.retryable_status?(500)) + refute(Policy.retry_eligible?(500, set: ::Set[503])) + assert(Policy.retry_eligible?(404, set: ::Set[404]), "and a set that WIDENS widens") + end + + test "RETRY-2 / CFG-35: throwable_retryable? is the capability query, not a type match" do + assert(Policy.throwable_retryable?(RetryableByCapability.new("reset"))) + refute(Policy.throwable_retryable?(UnretryableIOError.new("an IOError saying no"))) + refute(Policy.throwable_retryable?(Dexpace::StreamError.new("an IOError with no capability"))) + refute(Policy.throwable_retryable?(::StandardError.new("no capability"))) + end + + test "RETRY-2: the capability is found anywhere in the cause chain, nearest first" do + wrapped = wrap(RetryableByCapability.new("inner")) + + assert_kind_of(RetryableByCapability, wrapped.cause, "the fixture is load-bearing") + assert(Policy.throwable_retryable?(wrapped)) + assert(Policy.throwable_retryable?(wrap(wrapped, ::RuntimeError, "outer"))) + refute(Policy.throwable_retryable?(wrap(::StandardError.new("inner")))) + end + + test "RETRY-2 / XCUT-9: a cyclic cause chain terminates, through 4b's identity-tracking walk" do + first = CyclicErrorFixtures::CyclicPair.new("one") + second = CyclicErrorFixtures::CyclicPair.new("two") + first.other_cause = second + second.other_cause = first + + refute(Policy.throwable_retryable?(first)) + refute(Policy.throwable_retryable?(CyclicErrorFixtures::SelfCause.new("self"))) + end + + test "RETRY-2: a non-Exception is refused rather than classified" do + assert_raises(Dexpace::InvalidArgumentError) { Policy.throwable_retryable?("boom") } + end + + test "RETRY-37: retryable? classifies a ProtocolError by the configured set alone" do + error = Dexpace::ProtocolError.for(build_response(418)) + + assert(Policy.retryable?(error, retryable_statuses: ::Set[418])) + refute(Policy.retryable?(error, retryable_statuses: ::Set[500])) + refute(Policy.retryable?(Dexpace::ProtocolError.for(build_response(503)), + retryable_statuses: ::Set[418],), "503 narrowed away",) + end + + test "RETRY-2: retryable? classifies anything else by the capability" do + assert(Policy.retryable?(RetryableByCapability.new("x"), retryable_statuses: ::Set[])) + refute(Policy.retryable?(::RuntimeError.new("x"), retryable_statuses: ::Set[500])) + end + + test "P6-10: a ProtocolError buried in a cause chain cannot answer the capability query" do + protocol = Dexpace::ProtocolError.for(build_response(503)) + + assert_predicate(protocol, :retryable_by_status?) + refute_respond_to(protocol, :retryable?, "the baked flag is not the open capability") + wrapped = wrap(protocol, Dexpace::PipelineError, "downstream") + + assert_same(protocol, wrapped.cause) + refute(Policy.throwable_retryable?(wrapped)) + refute(Policy.retryable?(wrapped, retryable_statuses: ::Set[503]), + "a configured set that ADMITS 503 still does not reach a wrapped ProtocolError " \ + "through the capability branch; the status branch needs the error itself",) + end + + test "RETRY-23 / P6-60: cancellation? finds a CancelledError itself or anywhere in the chain" do + cancelled = Dexpace::CancelledError.new(:token) + + assert(Policy.cancellation?(cancelled)) + assert(Policy.cancellation?(wrap(cancelled, ::IOError, "wrapped by a transport"))) + assert(Policy.cancellation?(wrap(wrap(cancelled), ::RuntimeError, "two levels up"))) + refute(Policy.cancellation?(::IOError.new("no cancellation anywhere"))) + refute(Policy.cancellation?(wrap(::StandardError.new("inner")))) + assert_raises(Dexpace::InvalidArgumentError) { Policy.cancellation?("boom") } + end + + test "RETRY-23 / RECOV-27: a cancellation is never retryable, whatever wraps it answers" do + cancelled = Dexpace::CancelledError.new(:token) + # A transport error that answers the capability and CARRIES the cancellation: the + # capability branch alone would call it retryable; the cancellation guard runs first. + carrier = wrap(cancelled, RetryableByCapability, "read interrupted") + + assert(Policy.throwable_retryable?(carrier), "the fixture is load-bearing: capability yes") + refute(Policy.retryable?(carrier, retryable_statuses: ::Set[503])) + refute(Policy.retryable?(cancelled, retryable_statuses: ::Set[503])) + refute(Policy.throwable_retryable?(cancelled), "CancelledError answers no capability") + end + end + + # RETRY-9, RETRY-10, RETRY-11, RETRY-41, RECOV-20, RECOV-21, P6-12. + class BackoffAndBudgetTest < DexpaceTestCase + include Fixtures + + def backoff(attempt, **overrides) + Policy.backoff_delay( + attempt, initial_delay: 0.2, multiplier: 2.0, max_delay: 8.0, jitter: 0.0, + random: ::Random.new(1), **overrides, + ) + end + + test "RETRY-9 / RECOV-21: the unjittered delay is initial * multiplier**(n - 1), capped" do + assert_in_delta(0.2, backoff(1)) + assert_in_delta(0.4, backoff(2)) + assert_in_delta(0.8, backoff(3)) + assert_in_delta(6.4, backoff(6)) + assert_in_delta(8.0, backoff(7), 0.001, "12.8 capped to the 8.0 max") + assert_in_delta(8.0, backoff(8)) + end + + test "RETRY-11: an attempt below 1 is refused, and the cap saturates a huge attempt" do + assert_raises(Dexpace::InvalidArgumentError) { backoff(0) } + assert_raises(Dexpace::InvalidArgumentError) { backoff(-1) } + assert_raises(Dexpace::InvalidArgumentError) { backoff(1.5) } + assert_in_delta(8.0, backoff(1_000), 0.001, "2.0**999 overflows to Infinity, never raises") + assert_in_delta(8.0, backoff(100_000)) + end + + test "RETRY-11 / P6-53: a zero initial delay is zero at any attempt, never 0 * Infinity" do + assert_in_delta(0.0, backoff(1, initial_delay: 0.0)) + assert_in_delta(0.0, backoff(1_025, initial_delay: 0.0), 0.001, + "2.0**1024 is Infinity; 0 * Inf is NaN",) + assert_in_delta(0.0, backoff(5_000, initial_delay: 0, jitter: 1.0)) + end + + test "RETRY-9: Integer inputs are accepted and the result is always a Float" do + assert_in_delta(0.5, backoff(2, initial_delay: 1, multiplier: 2, max_delay: 8, jitter: 0) + .then { |d| d / 4 },) + assert_instance_of(::Float, backoff(1, initial_delay: 1, multiplier: 1, max_delay: 1)) + end + + test "RETRY-10: jitter samples uniformly within [d(1 - j/2), d(1 + j/2)] with midpoint d" do + random = ::Random.new(20_260_918) + samples = Array.new(2_000) { backoff(1, initial_delay: 1.0, jitter: 0.5, random: random) } + + samples.each do |d| + assert_operator(d, :>=, 0.75) + assert_operator(d, :<=, 1.25) + end + assert_in_delta(1.0, samples.sum / samples.size, 0.02, "midpoint stays at d") + assert_operator(samples.uniq.size, :>, 1_000, "a real draw, not a constant") + end + + test "RETRY-10: jitter 0 returns d exactly; a degenerate sub-nanosecond spread returns d" do + # Exact, so the deltas are tighter than the spread a draw would produce: an autocorrected + # default delta of 0.001 would let a drawn 9.6e-13 pass for 1e-12 (the guard ran green). + assert_in_delta(0.2, backoff(1, jitter: 0.0), 0.0) + assert_in_delta(1e-12, backoff(1, initial_delay: 1e-12, jitter: 0.5), 0.0) + assert_in_delta(1e-9, backoff(1, initial_delay: 1e-9, jitter: 0.5), 0.0, + "spread 5e-10 < 1e-9",) + refute_in_delta(3e-9, backoff(1, initial_delay: 3e-9, jitter: 0.5), 1e-12, + "spread 1.5e-9 draws",) + end + + test "RETRY-10 / RETRY-9: at the cap the band is drawn AROUND the cap: capped, then jittered" do + random = ::Random.new(7) + samples = Array.new(200) { backoff(50, jitter: 1.0, random: random) } + + samples.each do |d| + assert_operator(d, :>=, 4.0) + assert_operator(d, :<=, 12.0, "d(1 + j/2) with d = 8.0 and j = 1.0") + end + # The order RECOV-21 fixes: cap FIRST, then perturb. Jitter applied to the raw + # 0.2 * 2**49 and clipped to the cap afterwards would answer exactly 8.0 every time and + # never a sample above it, so the band's two halves are asserted, not only its bounds. + assert_operator(samples.max, :>, 8.0, "a sample above the cap: jittered after capping") + assert_operator(samples.min, :<, 8.0) + end + + test "RETRY-10: a negative sample is floored at zero" do + below_zero = ::Object.new + def below_zero.rand(_range) = -0.5 + + assert_in_delta(0.0, backoff(1, jitter: 1.0, random: below_zero)) + end + end + + # RETRY-41, RECOV-20, P6-12: the resolver and the recovery-only budget. + class ResolverAndBudgetTest < DexpaceTestCase + include Fixtures + + test "RETRY-41: present-override-wins, else configured; negative clamped; zero allowed" do + assert_equal(5, Policy.effective_max_retries(override: 5, configured: 2)) + assert_equal(0, Policy.effective_max_retries(override: 0, configured: 2), "zero: no retries") + assert_equal(2, Policy.effective_max_retries(override: nil, configured: 2)) + assert_equal(0, Policy.effective_max_retries(override: nil, configured: 0)) + assert_equal(Policy::DEFAULT_MAX_RETRIES, + Policy.effective_max_retries(override: nil, configured: -1),) + end + + test "RETRY-41: a negative or non-Integer override is validated, not silently ignored" do + assert_raises(Dexpace::InvalidArgumentError) do + Policy.effective_max_retries(override: -1, configured: 2) + end + assert_raises(Dexpace::InvalidArgumentError) do + Policy.effective_max_retries(override: 1.5, configured: 2) + end + end + + test "RETRY-41 / P6-12: the clamp is logged as one WARNING config diagnostic, nothing else" do + sink = RecordingSink.new + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(2, Policy.effective_max_retries(override: nil, configured: 2, logger: logger)) + assert_equal(2, Policy.effective_max_retries(override: 9, configured: -3, logger: logger) + .then { 2 },) + assert_empty(sink.entries, "neither a configured nor an overridden value logs") + + Policy.effective_max_retries(override: nil, configured: -3, logger: logger) + + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_CONFIG, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/clamped/, payload[Dexpace::Instrumentation::Keys::MESSAGE]) + assert_match(/-3/, payload[Dexpace::Instrumentation::Keys::MESSAGE]) + end + + test "RETRY-41 / OBS-20: a raising sink cannot fail the resolution" do + sink = RecordingSink.new + def sink.warn(*) = raise("sink down") + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, + Policy.effective_max_retries(override: nil, configured: -1, logger: logger),) + end + + test "RECOV-20: budget_remaining is the time REMAINING, Infinity for a zero budget" do + assert_equal(::Float::INFINITY, Policy.budget_remaining(elapsed: 100.0, total_timeout: 0)) + assert_equal(::Float::INFINITY, Policy.budget_remaining(elapsed: 0.0, total_timeout: 0.0)) + assert_in_delta(4.0, Policy.budget_remaining(elapsed: 6.0, total_timeout: 10.0)) + assert_in_delta(-2.0, Policy.budget_remaining(elapsed: 12.0, total_timeout: 10), 0.001, + "negative once spent; a caller compares against remaining, never elapsed",) + assert_instance_of(::Float, Policy.budget_remaining(elapsed: 1, total_timeout: 10)) + end + end + + # RETRY-15, RETRY-17, RETRY-19, RETRY-21, RECOV-24, RECOV-25: the forms and the precedence. + class PacingFormsTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + + # No keyword parameter here on purpose: a braceless `delay("Retry-After" => "5")` would + # otherwise be read as keywords under Ruby 3's separation. + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + def delay_with(pairs, random) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW, random: random) + end + + test "RETRY-15 / RECOV-24: Retry-After as delta-seconds, integer and fractional" do + assert_in_delta(5.0, delay("Retry-After" => "5")) + assert_in_delta(2.5, delay("Retry-After" => "2.5")) + assert_in_delta(0.001, delay("Retry-After" => "0.001"), 0.00001, "sub-second honoured") + assert_in_delta(0.0, delay("Retry-After" => "0")) + end + + test "RETRY-15: Retry-After as an RFC 1123 HTTP-date, measured from `now`" do + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:01:30 GMT")) + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:01:30 UTC")) + end + + test "RETRY-15 / R1: the HTTP-date form tolerates a wrong weekday and a single-digit day" do + assert_in_delta(90.0, delay("Retry-After" => "Mon, 18 Sep 2026 12:01:30 GMT"), 0.001, + "wrong weekday",) + assert_in_delta(90.0, delay("Retry-After" => "Fri, 18 sep 2026 12:01:30 gmt"), 0.001, "case") + later = ::Time.utc(2026, 9, 1, 12, 0, 0) + + early = headers("Retry-After" => "Tue, 1 Sep 2026 12:01:00 GMT") + + assert_in_delta(60.0, Policy.pacing_delay(early, header_order: ORDER, now: later)) + end + + test "RETRY-17 / RECOV-23: a valid HTTP-date or epoch already past is ZERO, not nil" do + assert_in_delta(0.0, delay("Retry-After" => "Sun, 06 Nov 1994 08:49:37 GMT")) + assert_in_delta(0.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:00:00 GMT"), 0.001, + "exactly now",) + assert_in_delta(0.0, delay("X-RateLimit-Reset" => (NOW.to_i - 100).to_s)) + assert_in_delta(0.0, delay("X-RateLimit-Reset" => NOW.to_i.to_s)) + end + + test "RETRY-15 / RECOV-24: retry-after-ms and x-ms-retry-after-ms as integer milliseconds" do + assert_in_delta(1.5, delay("retry-after-ms" => "1500")) + assert_in_delta(0.25, delay("x-ms-retry-after-ms" => "250")) + assert_in_delta(0.0, delay("retry-after-ms" => "0")) + assert_nil(delay("retry-after-ms" => "1.5"), "milliseconds are an integer count") + end + + test "RETRY-15 / RECOV-25: X-RateLimit-Reset as epoch seconds, jittered up to [100%, 120%]" do + random = ::Random.new(20_260_918) + pairs = { "X-RateLimit-Reset" => (NOW.to_i + 10).to_s } + samples = Array.new(500) { delay_with(pairs, random) } + + samples.each do |d| + assert_operator(d, :>=, 10.0) + assert_operator(d, :<=, 12.0) + end + assert_operator(samples.uniq.size, :>, 100, "a real draw") + assert_operator(samples.min, :<, 10.5) + assert_operator(samples.max, :>, 11.5) + end + + test "RETRY-21 / RECOV-24: the precedence returns the first parseable value" do + assert_in_delta(5.0, delay("Retry-After" => "5", "retry-after-ms" => "9999")) + assert_in_delta(1.0, delay("retry-after-ms" => "1000", "x-ms-retry-after-ms" => "9999")) + assert_in_delta(2.0, delay("x-ms-retry-after-ms" => "2000", + "X-RateLimit-Reset" => (NOW.to_i + 9_999).to_s,),) + end + + test "RETRY-21: the stage stack's caller-configurable order is honoured as given" do + pairs = { "Retry-After" => "5", "retry-after-ms" => "1000" } + + reversed = %w[retry-after-ms Retry-After] + + assert_in_delta(1.0, Policy.pacing_delay(headers(pairs), now: NOW, header_order: reversed)) + assert_nil(Policy.pacing_delay(headers(pairs), now: NOW, header_order: [])) + assert_nil(Policy.pacing_delay(headers(pairs), now: NOW, header_order: %w[X-RateLimit-Reset])) + end + + test "HTTP-13: header names in the order are matched case-insensitively, through Headers#[]" do + lower = headers("retry-after" => "5") + + assert_in_delta(5.0, Policy.pacing_delay(lower, now: NOW, header_order: %w[RETRY-AFTER])) + end + + test "a multi-valued header is read by its first value" do + builder = Dexpace::Headers.inbound_builder + builder.add("Retry-After", "3") + builder.add("Retry-After", "9") + + assert_in_delta(3.0, Policy.pacing_delay(builder.build, header_order: ORDER, now: NOW)) + end + end + + # RETRY-16, RETRY-18, RETRY-19, RETRY-22, RECOV-23, RECOV-26, RECOV-29: the negative suite. + class PacingTotalityTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + + # No keyword parameter here on purpose: a braceless `delay("Retry-After" => "5")` would + # otherwise be read as keywords under Ruby 3's separation. + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + def delay_with(pairs, random) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW, random: random) + end + + test "RETRY-19 / RECOV-24: the decimal screen rejects every non-decimal spelling first" do + %w[0x10 1e3 5_0 5f 5d 30d 5s Infinity NaN 5,0 +5 .5 5. 0b1 1r 1i].each do |bad| + assert_nil(delay("Retry-After" => bad), "expected #{bad.inspect} to be no hint") + end + assert_equal(50, Integer("5_0", 10), "the screen is load-bearing: Integer() reads this as 50") + end + + test "RETRY-16 / RECOV-23: malformed, negative and out-of-range values are no hint, not 0" do + ["-5", "-0.5", "not a date", "", " 5", "5 ", "Sun, 32 Nov 1994 08:49:37 GMT", + "Sunday, 06-Nov-94 08:49:37 GMT", "9" * 16, "9" * 300, "9" * 400,].each do |bad| + assert_nil(delay("Retry-After" => bad), bad.inspect) + end + ["-1", "1.5", "abc", "", "9" * 16, "9" * 400].each do |bad| + assert_nil(delay("X-RateLimit-Reset" => bad), bad.inspect) + assert_nil(delay("retry-after-ms" => bad), bad.inspect) + end + end + + test "RETRY-16: no matching header at all is no hint" do + assert_nil(delay({})) + assert_nil(delay("Content-Type" => "text/plain")) + end + + test "RETRY-18 / RECOV-26: a finite computed delta past 365 days is clamped to the ceiling" do + ceiling = Policy::MAX_PACING_DELAY_SECONDS.to_f + + assert_equal(ceiling, delay("Retry-After" => (400 * 24 * 60 * 60).to_s)) + assert_equal(ceiling, delay("retry-after-ms" => (400 * 24 * 60 * 60 * 1000).to_s)) + assert_equal(ceiling, delay("X-RateLimit-Reset" => (NOW.to_i + (400 * 24 * 60 * 60)).to_s)) + assert_equal(ceiling, delay("Retry-After" => "Fri, 18 Sep 2099 12:00:00 GMT")) + assert_equal(ceiling, delay("Retry-After" => "9" * 15), "the longest run the grammar admits") + assert_equal(ceiling, delay("retry-after-ms" => "9" * 15)) + assert_equal(ceiling, delay("X-RateLimit-Reset" => "9" * 15)) + end + + test "RETRY-18: exactly the ceiling and one below pass through unclamped" do + ceiling = Policy::MAX_PACING_DELAY_SECONDS + + assert_equal(ceiling.to_f, delay("Retry-After" => ceiling.to_s)) + assert_equal(ceiling - 1.0, delay("Retry-After" => (ceiling - 1).to_s)) + end + + test "RETRY-22 / RECOV-29: one malformed header does not mask a later, well-formed one" do + assert_in_delta(1.0, delay("Retry-After" => "garbage", "retry-after-ms" => "1000")) + assert_in_delta(7.0, delay("Retry-After" => "30d", "retry-after-ms" => "x", + "x-ms-retry-after-ms" => "7000",),) + end + + test "RETRY-16: a parser that raises inside the dispatch is contained; the scan continues" do + exploding = ::Object.new + def exploding.rand(_range) = raise("generator down") + pairs = { "X-RateLimit-Reset" => (NOW.to_i + 10).to_s, "Retry-After" => "garbage" } + + assert_nil(delay_with(pairs, exploding)) + assert_in_delta(2.0, delay_with(pairs.merge("retry-after-ms" => "2000"), exploding)) + end + + test "RETRY-19 / RECOV-24: the grammars are anchored, frozen, and carry their own timeout" do + parsers = Dexpace::Resilience.const_get(:PacingParsers) + %i[DECIMAL_GRAMMAR INTEGER_GRAMMAR].each do |name| + grammar = parsers.const_get(name) + + assert_kind_of(::Regexp, grammar) + refute_nil(grammar.timeout, "#{name}: per-pattern, never Regexp.timeout") + assert_predicate(grammar, :frozen?) + assert_match(/\A\\A.*\\z\z/, grammar.source, "#{name} is anchored at both ends") + end + refute_includes(Dexpace::Resilience.constants, :PacingParsers, "a private_constant") + end + + test "RETRY-16: a value with no grammar at all -- a non-String -- is no hint" do + forged = ::Object.new + def forged.[](_name) = [nil] + forged2 = ::Object.new + def forged2.[](_name) = [42] + + assert_nil(Policy.pacing_delay(forged, header_order: ORDER, now: NOW)) + assert_nil(Policy.pacing_delay(forged2, header_order: ORDER, now: NOW)) + end + end + + # RETRY-16, RETRY-19, NFR-6, P6-61: the parser's bounds -- a run past fifteen digits or a + # value past 64 bytes is no hint before any conversion, so a hostile value costs microseconds + # and emits no warning. + class PacingBoundsTest < DexpaceTestCase + include Fixtures + + ORDER = Policy::DEFAULT_PACING_HEADER_ORDER + NOW = ::Time.utc(2026, 9, 18, 12, 0, 0) + FORMS = %w[Retry-After retry-after-ms X-RateLimit-Reset].freeze + + def delay(pairs) + Policy.pacing_delay(headers(pairs), header_order: ORDER, now: NOW) + end + + # A headers stand-in answering one huge value: phase 1's inbound builder validates a value + # byte by byte (seconds for 10 MB) and is not what is under test here; the parser is. + def forged(name, value) + stand_in = ::Object.new + stand_in.define_singleton_method(:[]) { |asked| asked == name ? [value] : nil } + stand_in + end + + test "P6-61: a 10 MB value in every form is no hint, in microseconds, on the parser alone" do + # The digit run exercises the grammars' bound in every form; the letter run exercises the + # byte ceiling in front of the HTTP-date attempt, which 5a's anchored grammar would + # otherwise scan (measured at 0.1 s for 10 MB). Both answer in well under a millisecond. + cases = FORMS.map { |name| [name, "9" * 10_000_000] } << ["Retry-After", "x" * 10_000_000] + cases.each do |name, huge| + started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + answer = Policy.pacing_delay(forged(name, huge), header_order: [name], now: NOW) + elapsed = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started + + assert_nil(answer, name) + assert_operator(elapsed, :<, 0.05, "#{name} #{huge[0]}: #{elapsed} s; the value was read") + end + end + + test "NFR-6 / RETRY-16: a run long enough to overflow a Float emits no Ruby warning" do + # The suite's raiser turns a warning into an error that Policy#parse_form's fence would + # swallow, so the totality cases above cannot see one; recording the warnings can. + FORMS.each do |name| + warnings = WarningCapture.record { assert_nil(delay(name => "9" * 400), name) } + + assert_empty(warnings, "#{name}: String#to_f/#to_i ran on the run") + end + end + + test "P6-61: fifteen digits is the boundary, and the fraction is bounded the same way" do + assert_equal(Policy::MAX_PACING_DELAY_SECONDS.to_f, delay("Retry-After" => "1" * 15)) + assert_nil(delay("Retry-After" => "1" * 16)) + assert_in_delta(1.5, delay("Retry-After" => "1.5")) + assert_in_delta("1.#{"5" * 15}".to_f, delay("Retry-After" => "1.#{"5" * 15}"), 0.0) + assert_nil(delay("Retry-After" => "1.#{"5" * 16}")) + assert_in_delta(1.5, delay("retry-after-ms" => "1500")) + assert_nil(delay("retry-after-ms" => "1500.0"), "millis take no fraction") + end + + test "P6-61: the 64-byte ceiling sits above every well-formed form" do + parsers = Dexpace::Resilience.const_get(:PacingParsers) + + assert_equal(64, parsers.const_get(:MAX_VALUE_BYTES)) + # The two longest well-formed values -- the 29-byte RFC 1123 date and the 31-byte decimal + # the bounded grammar admits (fifteen digits, a point, fifteen digits) -- both parse. + longest = "#{"1" * 15}.#{"5" * 15}" + + assert_equal(31, longest.bytesize) + assert_in_delta(45.0, delay("Retry-After" => "Fri, 18 Sep 2026 12:00:45 GMT")) + assert_equal(Policy::MAX_PACING_DELAY_SECONDS.to_f, delay("Retry-After" => longest)) + assert_nil(delay("Retry-After" => "x" * 65)) + assert_nil(delay("Retry-After" => "x" * 64), "the date grammar refuses; the ceiling admits") + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb new file mode 100644 index 0000000..9518cd9 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/recovery_retry_test.rb @@ -0,0 +1,536 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" +require_relative "../../support/fake_config_source" + +# Exercises: RECOV-16, RECOV-17, RECOV-18, RECOV-19, RECOV-20, RECOV-21, RECOV-22, RECOV-23, +# RECOV-24, RECOV-26, RECOV-27, RECOV-28, RECOV-29, RECOV-30, RECOV-34, RETRY-4, RETRY-13, +# RETRY-14, RETRY-23, RETRY-25, RETRY-27, RETRY-34, RETRY-35, RETRY-36, RETRY-37, RETRY-41, +# RETRY-42, RETRY-44, OBS-29, CFG-16, SEAM-11, P6-3, P6-6, P6-8, P6-9, P6-60 +# +# The recovery-stack engine, driven directly as the Dexpace::Transport it is and, where the +# integration is the point, beneath a real Recovery::Orchestrator with a real +# ErrorMappingStep. Every wait is a FakeClock's, whose monotonic reading advances by exactly the +# slept duration, so the total-timeout arithmetic is exact rather than wall-clock-dependent. +# Hermetic: the engine's default settings read the configured MAX_RETRY_ATTEMPTS, so every +# nested class runs behind a FakeConfigSource env seam. Split into nested classes under +# Metrics/ClassLength. +class DexpaceResilienceRecoveryRetryTest < DexpaceTestCase + RecoveryRetry = Dexpace::Resilience::RecoveryRetry + + # The engine over a script, the one call every test makes, and the seam. + module Fixtures + include RetryFixtures + + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + + def engine(script, settings: retry_settings, **keywords) + transport = ScriptedTransport.new(script) + [RecoveryRetry.build(transport: transport, settings: settings, **keywords), transport] + end + + def send_through(built, request: retry_request, cancellation: Dexpace::Cancellation.none) + built.call(request, Dexpace::RequestOptions::EMPTY, cancellation) + end + end + include Fixtures + + test "SEAM-11 / P6-3: the engine is a Dexpace::Transport by the duck type, frozen, .new hidden" do + built, = engine([retry_response(200)]) + + assert(Dexpace::Transport.conforms?(built)) + assert(Dexpace::Registry.callable?(built, arity: 3)) + assert_predicate(built, :frozen?) + refute_respond_to(RecoveryRetry, :new) + end + + test "the constructor validates every keyword" do + assert_raises(Dexpace::InvalidArgumentError) { RecoveryRetry.build(transport: :none) } + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ->(_r, _o) {}) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), settings: :defaults) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), http_tracer_factory: :none) + end + assert_raises(Dexpace::InvalidArgumentError) do + RecoveryRetry.build(transport: ScriptedTransport.new([]), logger: RecordingSink.new) + end + end + + test "RECOV-28: a success on the first attempt touches no clock and returns the response as is" do + clock = FakeClock.new + built, transport = engine([retry_response(200)], settings: retry_settings(clock: clock)) + response = send_through(built) + + assert_equal(200, response.status.code) + assert_empty(clock.sleeps) + assert_equal(1, transport.calls.size) + refute_predicate(response.body, :closed?) + end + + test "RECOV-19 / RETRY-36 / RETRY-44: 503, 503, 200 ends on the 200, re-sending ONE request" do + built, transport = engine([retry_response(503), retry_response(503), retry_response(200)]) + request = retry_request + response = send_through(built, request: request) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-42 / RECOV-28: eight concurrent calls through one engine keep their budgets apart" do + built, transport = engine(Array.new(16) { |i| retry_response(i.even? ? 503 : 200) }, + settings: retry_settings(max_retries: 1),) + results = Array.new(8) { ::Thread.new { send_through(built).status.code } }.map(&:value) + + assert_equal(16, transport.calls.size) + assert_equal(8, results.count(200), results.inspect) + end + + # RECOV-17, RECOV-18, RETRY-4, RETRY-25, RETRY-37: the two axes, on this stack. + class ClassificationTest < DexpaceTestCase + include Fixtures + + test "RECOV-17 / RETRY-4 / P6-8: a THROWN retryable failure is retried, not propagated" do + built, transport = engine([RetryableError.new("reset"), RetryableError.new("reset"), + retry_response(200),]) + response = send_through(built) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size, "three sends, not one raise") + end + + test "RECOV-17: a thrown failure with no capability is surfaced on the first attempt" do + built, transport = engine([::ArgumentError.new("bug"), retry_response(200)]) + + error = assert_raises(::ArgumentError) { send_through(built) } + + assert_equal("bug", error.message) + assert_empty(Dexpace.suppressed(error)) + assert_equal(1, transport.calls.size) + end + + test "RECOV-17 / RETRY-37: the configured set is authoritative -- wider, and narrower" do + built, transport = engine([retry_response(418), retry_response(200)], + settings: retry_settings(retryable_statuses: [418]),) + + assert_equal(200, send_through(built).status.code) + assert_equal(2, transport.calls.size, "418 is outside the baked set and still retried") + built, transport = engine([retry_response(503), retry_response(200)], + settings: retry_settings(retryable_statuses: [429]),) + + assert_equal(503, send_through(built).status.code, "503 is baked-retryable and refused") + assert_equal(1, transport.calls.size) + end + + test "RECOV-19: a NON-retryable error status passes through as Success -- returned, buffered" do + built, transport = engine([retry_response(404)]) + response = send_through(built) + + assert_equal(404, response.status.code) + assert_equal(1, transport.calls.size) + assert_kind_of(Dexpace::BufferBody, response.body, "buffered per RECOV-16") + assert_equal("body 404", response.body_string) + end + + test "RECOV-18 / RETRY-5 / RETRY-7: a bare POST is sent exactly once and its 503 returned" do + built, transport = engine([retry_response(503), retry_response(200)], + settings: retry_settings(max_retries: 5),) + response = send_through(built, request: retry_request(method: "POST")) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RECOV-18: a replayable body is re-sent and a consumed body is not" do + built, transport = engine([retry_response(503), retry_response(200)]) + response = send_through(built, request: retry_request(method: "POST", body: replayable_body)) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + built, transport = engine([RetryableError.new("reset"), retry_response(200)]) + + assert_raises(RetryableError) do + send_through(built, request: retry_request(method: "PUT", body: consumed_body)) + end + assert_equal(1, transport.calls.size) + end + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + built, transport = engine([RetryableFatal.new("oom, lying"), retry_response(200)], + settings: retry_settings(max_retries: 3),) + + assert_raises(RetryableFatal) { send_through(built) } + assert_equal(1, transport.calls.size) + end + + test "RETRY-25: a fatal-family error propagates at the throw site, unclassified, unattached" do + tracer = Dexpace::RecordingHTTPTracer.new + built, transport = engine([::NoMemoryError.new("oom"), retry_response(200)], + http_tracer_factory: ->(_request) { tracer },) + + error = assert_raises(::NoMemoryError) { send_through(built) } + + assert_empty(Dexpace.suppressed(error)) + assert_equal(1, transport.calls.size) + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27, RETRY-34, P6-9: the budget and the terminal paths. + class BudgetTest < DexpaceTestCase + include Fixtures + + test "RECOV-20 / RETRY-34 / P6-9: exhausting the attempt cap RAISES, with the whole trail" do + built, transport = engine(Array.new(3) { retry_response(503) }, + settings: retry_settings(max_retries: 2),) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(3, transport.calls.size, "max_retries 2 + 1 initial send (RETRY-14, P6-6)") + assert_equal(503, error.status.code) + assert_equal(2, Dexpace.suppressed(error).size) + assert_equal([503, 503], Dexpace.suppressed(error).map { |prior| prior.status.code }) + assert_nil(error.cause, "pipeline/7ce4431d: the carried raise assigns no cause") + end + + test "RETRY-34 / pipeline/7ce4431d: the carried raise assigns no cause in a caller's rescue" do + built, = engine(Array.new(2) do + retry_response(503) + end, settings: retry_settings(max_retries: 1),) + surfaced = nil + begin + raise ::IOError, "the caller's own in-flight exception" + rescue ::IOError + surfaced = assert_raises(Dexpace::ProtocolError) { send_through(built) } + end + + assert_nil(surfaced.cause) + assert_equal(1, Dexpace.suppressed(surfaced).size) + end + + test "RETRY-34: a reused error instance trips no self-suppression on this stack either" do + same = RetryableError.new("same") + built, = engine([same, same, same], settings: retry_settings(max_retries: 2)) + + surfaced = assert_raises(RetryableError) { send_through(built) } + + assert_same(same, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: on eventual success the trail is discarded" do + first = RetryableError.new("one") + built, = engine([first, retry_response(200)]) + + assert_equal(200, send_through(built).status.code) + assert_empty(Dexpace.suppressed(first)) + end + end + + # RECOV-20, RECOV-21, RECOV-22, RETRY-27: the total-timeout budget on a fake clock. + class TotalTimeoutTest < DexpaceTestCase + include Fixtures + + test "RECOV-20: a positive total_timeout aborts on `remaining`, at the attempt it names" do + # remaining = total_timeout - elapsed, on a clock that advances by exactly each sleep: + # 1.0 -> sleep 0.4 -> 0.6 -> sleep 0.4 -> 0.2, and 0.4 > 0.2 suppresses the third delay. + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 1.0, initial_delay: 0.4, + max_delay: 0.4, max_retries: 10,) + built, transport = engine(Array.new(11) { retry_response(503) }, settings: settings) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(3, transport.calls.size, "exact, not merely under 11") + assert_equal([0.4, 0.4], clock.sleeps.map { _1[:duration] }, "never over budget") + assert_equal(2, Dexpace.suppressed(error).size) + end + + test "RECOV-20: elapsed at or past the budget suppresses the next delay, delay or no delay" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 1.0, initial_delay: 0.0, + max_retries: 10,) + slow = lambda { |*| + clock.advance(0.5) + retry_response(503) + } + built, transport = engine([slow, slow, slow], settings: settings) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(2, transport.calls.size, "1.0 elapsed after the second send: remaining 0.0") + end + + test "RECOV-20 / RETRY-27: a zero total_timeout is unbounded -- only the cap stops the loop" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 0, initial_delay: 100.0, + max_delay: 100.0, max_retries: 4,) + built, transport = engine(Array.new(5) { retry_response(503) }, settings: settings) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(5, transport.calls.size) + assert_equal([100.0] * 4, clock.sleeps.map { _1[:duration] }) + end + + test "RECOV-22 / RETRY-20: a pacing hint replaces the schedule and is still budget-clamped" do + clock = FakeClock.new + settings = retry_settings(clock: clock, total_timeout: 5.0, initial_delay: 30.0, + max_delay: 30.0, max_retries: 5,) + built, transport = engine([retry_response(503, headers: { "Retry-After" => "2" }), + retry_response(503, headers: { "Retry-After" => "9" }), + retry_response(200),], settings: settings,) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal([2.0], clock.sleeps.map { _1[:duration] }, "2 honoured; 9 > 3 remaining") + assert_equal(2, transport.calls.size) + end + + test "RECOV-24 / RECOV-29: the fixed precedence, and a malformed hint falls back to backoff" do + clock = FakeClock.new + built, = engine([retry_response(503, headers: { "Retry-After" => "garbage", + "retry-after-ms" => "1500", },), + retry_response(503, headers: { "Retry-After" => "junk" }), + retry_response(200),], settings: retry_settings(clock: clock),) + + send_through(built) + + durations = clock.sleeps.map { |sleep| sleep[:duration] } + + assert_equal([1.5, 0.2], durations, "the hint, then attempt 2's backoff") + end + + test "RECOV-21 / RETRY-13: the schedule is Policy's own, on the settings clock" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 3) + built, = engine(Array.new(3) { retry_response(503) } << retry_response(200), + settings: settings,) + + send_through(built) + expected = (1..3).map do |attempt| + Dexpace::Resilience::Policy.backoff_delay(attempt, **settings.backoff_arguments) + end + + assert_equal(expected, clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-41 / RECOV-34: zero retries is one send, and a valid settings never clamps" do + # A negative max_retries cannot be built (RECOV-34 refuses it at construction), so + # RETRY-41's clamp is unreachable on this stack through the public constructor; the + # resolver is still the shared one and logs nothing for a valid value. + sink = RecordingSink.new + built, transport = engine(Array.new(4) { retry_response(503) }, + settings: retry_settings(max_retries: 0), + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + + assert_raises(Dexpace::ProtocolError) { send_through(built) } + assert_equal(1, transport.calls.size, "zero retries: one send") + assert_empty(sink.entries) + end + end + + # RECOV-27, RETRY-23, RETRY-35, RECOV-16, OBS-29, the orchestrator integration. + class IntegrationTest < DexpaceTestCase + include Fixtures + + test "RECOV-27 / RETRY-23: cancellation during the wait aborts the loop as CancelledError" do + source = Dexpace::Cancellation.source + script = [lambda { |*| + source.cancel(:test) + retry_response(503) + }, retry_response(200),] + built, transport = engine(script, settings: retry_settings(initial_delay: 5.0)) + + error = assert_raises(Dexpace::CancelledError) do + send_through(built, cancellation: source.token) + end + + assert_equal(1, transport.calls.size, "the second send never happened") + assert_equal(:test, error.reason) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error)) + end + + test "RECOV-27: the wait is the clock's cancellable sleep with the caller's token" do + clock = FakeClock.new + source = Dexpace::Cancellation.source + built, = engine([retry_response(503), retry_response(200)], + settings: retry_settings(clock: clock),) + + send_through(built, cancellation: source.token) + + assert_equal(1, clock.sleeps.size) + assert_same(source.token, clock.sleeps.first[:cancellation]) + end + + test "RETRY-23: a cancelled token is checked at every attempt boundary" do + source = Dexpace::Cancellation.source + source.cancel(:already) + built, transport = engine([retry_response(200)]) + + assert_raises(Dexpace::CancelledError) { send_through(built, cancellation: source.token) } + assert_equal(0, transport.calls.size) + end + + test "RETRY-23 / P6-60: a cancellation the transport WRAPPED in a retryable error is final" do + # RecoveryRetry has no caller predicate; the discriminator on this stack is the wrapped + # form, which the capability branch alone would have retried. + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + built, transport = engine([wrapped, retry_response(200)], + settings: retry_settings(max_retries: 5),) + + surfaced = assert_raises(RetryableError) { send_through(built) } + + assert_same(wrapped, surfaced) + assert_equal(1, transport.calls.size, "never re-sent") + assert_empty(Dexpace.suppressed(surfaced)) + end + + test "RETRY-35 / RECOV-16: the error body is buffered (connection released) before the wait" do + original = retry_response(503) + released_at_sleep = [] + clock = FakeClock.new + clock.define_singleton_method(:sleep) do |duration, cancellation: nil| + released_at_sleep << original.body.closed? + super(duration, cancellation: cancellation) + end + built, = engine([original, retry_response(200)], settings: retry_settings(clock: clock)) + + send_through(built) + + assert_equal([true], released_at_sleep) + end + + test "RETRY-35: the pacing delay is read from the buffered response, headers intact" do + clock = FakeClock.new + built, = engine([retry_response(503, headers: { "Retry-After" => "3" }), retry_response(200)], + settings: retry_settings(clock: clock),) + + send_through(built) + + assert_equal([3.0], clock.sleeps.map { _1[:duration] }) + end + + test "OBS-29 / P6-7: the tracer is made once per operation with the REQUEST as its context" do + tracer = Dexpace::RecordingHTTPTracer.new + contexts = [] + built, = engine([retry_response(503), retry_response(503), retry_response(200)], + http_tracer_factory: lambda { |request| + contexts << request + tracer + },) + request = retry_request + + send_through(built, request: request) + + assert_equal([request], contexts) + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started attempt_failed + attempt_started], names,) + tracer.events.each { |event| assert_same(request, event[1]) } + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2]) + assert_in_delta(0.1, tracer.events[1][3]) + end + + test "OBS-29: retries_exhausted immediately precedes the terminal raise, with its error" do + tracer = Dexpace::RecordingHTTPTracer.new + built, = engine([retry_response(503), retry_response(503)], + settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_request) { tracer },) + + error = assert_raises(Dexpace::ProtocolError) { send_through(built) } + + assert_equal(:retries_exhausted, tracer.events.last.first) + assert_same(error, tracer.events.last[2]) + end + + test "OBS-29: a never-retryable failure ends with no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + built, = engine([retry_response(404)], http_tracer_factory: ->(_request) { tracer }) + send_through(built) + + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + end + end + + # P6-3: the engine beneath a real Recovery::Orchestrator. + class OrchestratorTest < DexpaceTestCase + include Fixtures + + test "P6-3: beneath a real Orchestrator, RequestChain stamps once and the factory maps once" do + calls = [] + factory = lambda do |response| + calls << response.status.code + Dexpace::ProtocolError.for(response) + end + built, transport = engine([retry_response(503), retry_response(404)], + settings: retry_settings(max_retries: 3),) + stamp = Dexpace::Recovery::IdempotencyKeyStep.build( + header: "Idempotency-Key", strategy: ->(_request) { Dexpace::UUID.generate }, + methods: [Dexpace::Method::PUT], + ) + request_chain = Dexpace::Recovery::RequestChain.build(steps: [stamp]) + orchestrator = Dexpace::Recovery::Orchestrator.build( + transport: built, request_chain: request_chain, + response_chain: Dexpace::Recovery::ResponseChain.build( + response_steps: [Dexpace::Recovery::ErrorMappingStep.build(factory: factory)], + ), + ) + + request = retry_request(method: "PUT", body: replayable_body) + error = assert_raises(Dexpace::ProtocolError) do + orchestrator.call(request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + + assert_equal(404, error.status.code) + assert_equal([404], calls, "once, on the terminal response, never on the retried 503") + keys = transport.calls.map { |(sent, _o, _c)| sent.headers["Idempotency-Key"] } + + assert_equal(2, keys.size) + assert_equal(keys.first, keys.last, "one key, stamped once, re-sent as is") + refute_nil(keys.first) + end + + test "P6-3: beneath the Orchestrator an exhausted retryable status surfaces as the raise" do + built, = engine([retry_response(503), retry_response(503)], + settings: retry_settings(max_retries: 1),) + orchestrator = Dexpace::Recovery::Orchestrator.build( + transport: built, request_chain: Dexpace::Recovery::RequestChain.build, + response_chain: Dexpace::Recovery::ResponseChain.build, + ) + + error = assert_raises(Dexpace::ProtocolError) do + orchestrator.call(retry_request, Dexpace::RequestOptions::EMPTY, Dexpace::Cancellation.none) + end + + assert_equal(503, error.status.code) + assert_equal(1, Dexpace.suppressed(error).size) + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/resend_test.rb b/gems/dexpace-core/test/dexpace/resilience/resend_test.rb new file mode 100644 index 0000000..2ed6cd5 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/resend_test.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require "stringio" +require_relative "../../support/fake_body" + +# Exercises: RETRY-5, RETRY-6, RETRY-7, RETRY-8, RECOV-18, HTTP-9, BODY-1 +# +# The re-sendability gate's four-cell truth table -- body-less idempotent and non-idempotent, +# body-bearing replayable and non-replayable -- plus the two single-source assertions: the +# idempotent set is phase 1's and the replayability answer is the body's own. +class DexpaceResilienceResendTest < DexpaceTestCase + Resend = Dexpace::Resilience::Resend + + def request(method, body: nil) + Dexpace::Request.build(method: Dexpace::Method.of(method), url: "https://example.test/x", + headers: Dexpace::Headers::EMPTY, body: body,) + end + + test "RETRY-5 / RETRY-6: a body-less request is eligible iff its method is idempotent" do + %w[GET HEAD OPTIONS PUT DELETE].each do |method| + assert(Resend.eligible?(request(method)), "#{method} is idempotent") + end + %w[POST PATCH CONNECT TRACE].each do |method| + refute(Resend.eligible?(request(method)), "#{method} is not idempotent") + end + end + + test "RETRY-6 / HTTP-9: the idempotent set is phase 1's single source, never a second list" do + assert_equal(%w[GET HEAD OPTIONS PUT DELETE], Dexpace::Method::IDEMPOTENT) + source = File.read(File.expand_path("../../../lib/dexpace/resilience/resend.rb", __dir__)) + code = source.gsub(/^\s*#.*$/, "") + + refute_match(/\b(GET|HEAD|OPTIONS|DELETE)\b/, code, "no second method set") + assert_match(/idempotent\?/, code) + end + + test "RETRY-7 / RECOV-18: a bare non-idempotent POST is NOT eligible, payload or no payload" do + refute(Resend.eligible?(request("POST"))) + refute(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: false)))) + end + + test "RETRY-5 / BODY-1: a body-bearing request is eligible iff its body is replayable" do + assert(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: true)))) + refute(Resend.eligible?(request("POST", body: FakeBody.new("x", replayable: false)))) + assert(Resend.eligible?(request("PATCH", body: FakeBody.new("x", replayable: true))), + "POST/PATCH are re-sendable only via the replayable-body path (RETRY-6)",) + end + + test "RETRY-8 / RECOV-18: an idempotent method with a non-replayable body is NOT eligible" do + refute(Resend.eligible?(request("PUT", body: FakeBody.new("x", replayable: false)))) + refute(Resend.eligible?(request("DELETE", body: FakeBody.new("x", replayable: false)))) + assert(Resend.eligible?(request("PUT", body: FakeBody.new("x", replayable: true)))) + end + + test "RETRY-5: the shipped bodies answer for themselves -- bytes yes, an owned stream no" do + bytes = Dexpace::Body.bytes("payload".b) + stream = Dexpace::Body.stream(StringIO.new("payload"), close: true) + + assert(Resend.eligible?(request("POST", body: bytes))) + refute(Resend.eligible?(request("POST", body: stream)), "close: true forces single use") + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb new file mode 100644 index 0000000..60d3dd8 --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/retry_settings_test.rb @@ -0,0 +1,273 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/fake_clock" +require_relative "../../support/fake_config_source" +require_relative "../../support/recording_sink" + +# Exercises: RECOV-34, RETRY-12, RETRY-13, RETRY-14, RETRY-21, RETRY-41, RETRY-42, RECOV-30, +# CFG-14, OBS-20, SEAM-29, P6-6, P6-57, P6-59 +# +# The one retry configuration: its defaults are Policy's constants, its validation is RECOV-34 +# clause by clause, its collections are copied and frozen, and its max_retries is the first +# reader of 5a's Keys::MAX_RETRY_ATTEMPTS. Hermetic: every test that consults the process slot +# installs a FakeConfigSource env seam and resets the slot in teardown (5a's rule), so a host +# MAX_RETRY_ATTEMPTS variable cannot change the answer. +class DexpaceResilienceRetrySettingsTest < DexpaceTestCase + Settings = Dexpace::Resilience::RetrySettings + Policy = Dexpace::Resilience::Policy + KEY = Dexpace::Configuration::Keys::MAX_RETRY_ATTEMPTS + + # The hermetic slot, shared by the nested classes. + module Hermetic + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + end + include Hermetic + + test "RECOV-34 / RETRY-12: the defaults are Policy's shared constants; the object is frozen" do + settings = Settings.build + + assert_in_delta(Policy::DEFAULT_INITIAL_DELAY, settings.initial_delay) + assert_in_delta(Policy::DEFAULT_MULTIPLIER, settings.multiplier) + assert_in_delta(Policy::DEFAULT_MAX_DELAY, settings.max_delay) + assert_in_delta(Policy::DEFAULT_JITTER, settings.jitter) + assert_equal(Policy::DEFAULT_MAX_RETRIES, settings.max_retries) + assert_in_delta(0.0, settings.total_timeout, 0.0, "unbounded by default") + assert_equal(Policy::DEFAULT_RETRYABLE_STATUSES, settings.retryable_statuses) + assert_nil(settings.pacing_header_order) + assert_same(::Random, settings.random) + assert_same(Dexpace::Clock::SYSTEM, settings.clock) + assert_predicate(settings, :frozen?) + refute_respond_to(Settings, :new) + end + + test "RETRY-13: backoff_arguments is exactly the keyword set Policy.backoff_delay takes" do + settings = Settings.build(initial_delay: 1, multiplier: 3, max_delay: 9, jitter: 0) + arguments = settings.backoff_arguments + + assert_equal(%i[initial_delay multiplier max_delay jitter random], arguments.keys) + assert_in_delta(3.0, Policy.backoff_delay(2, **arguments), 0.0, "1 * 3**1 = 3, uncapped") + assert_in_delta(9.0, Policy.backoff_delay(3, **arguments), 0.0, "1 * 3**2 = 9, at the cap") + end + + test "RETRY-21: header_order is the caller's list, else the shared default" do + assert_same(Policy::DEFAULT_PACING_HEADER_ORDER, Settings.build.header_order) + custom = Settings.build(pacing_header_order: %w[X-RateLimit-Reset]) + + assert_equal(%w[X-RateLimit-Reset], custom.header_order) + assert_equal(%w[X-RateLimit-Reset], custom.pacing_header_order) + end + + # RECOV-34's validation clauses. + class ValidationTest < DexpaceTestCase + include Hermetic + + test "RECOV-34: durations are non-negative, finite and within the ~292-year ceiling" do + %i[initial_delay max_delay total_timeout].each do |member| + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => -1.0) } + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => nil) } + assert_raises(Dexpace::InvalidArgumentError, member) { Settings.build(member => "1") } + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => ::Float::INFINITY) + end + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => ::Float::NAN) + end + assert_raises(Dexpace::InvalidArgumentError, member) do + Settings.build(member => 9_223_372_037.0) # one second past the ceiling + end + assert_in_delta(9_223_372_036.0, + Settings.build(member => 9_223_372_036).public_send(member),) + assert_in_delta(0.0, Settings.build(member => 0).public_send(member)) + end + end + + test "RECOV-34: the multiplier is >= 1.0 and finite" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: 0.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: 0) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(multiplier: ::Float::INFINITY) } + assert_in_delta(1.0, Settings.build(multiplier: 1).multiplier) + end + + test "RECOV-34: the jitter lies within [0.0, 1.0]" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: 1.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: -0.1) } + assert_in_delta(0.0, Settings.build(jitter: 0).jitter) + assert_in_delta(1.0, Settings.build(jitter: 1).jitter) + end + + test "RECOV-34: max_retries is a non-negative Integer; zero disables retries" do + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: -1) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: 1.5) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_retries: nil) } + assert_equal(0, Settings.build(max_retries: 0).max_retries) + end + + test "SEAM-29: every refusal names its member" do + error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build(max_delay: -1) } + + assert_includes(error.message, "max_delay") + error = assert_raises(Dexpace::InvalidArgumentError) { Settings.build(jitter: 2) } + + assert_includes(error.message, "jitter") + end + + test "RECOV-34: retryable_statuses and pacing_header_order are copied and deep-frozen" do + statuses = ::Set[500] + order = [+"Retry-After"] + settings = Settings.build(retryable_statuses: statuses, pacing_header_order: order) + statuses << 599 + order << "retry-after-ms" + order.first << "-mutated" + + assert_equal(::Set[500], settings.retryable_statuses) + assert_equal(%w[Retry-After], settings.pacing_header_order) + assert_predicate(settings.retryable_statuses, :frozen?) + assert_predicate(settings.pacing_header_order, :frozen?) + assert_predicate(settings.pacing_header_order.first, :frozen?) + refute_predicate(statuses, :frozen?, "the caller's own collection is untouched") + end + + test "RECOV-34: retryable_statuses accepts any Integer collection and refuses anything else" do + assert_equal(::Set[1, 2], Settings.build(retryable_statuses: [1, 2, 2]).retryable_statuses) + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(retryable_statuses: %w[500]) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(retryable_statuses: 500) } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(pacing_header_order: "x") } + assert_raises(Dexpace::InvalidArgumentError) { Settings.build(pacing_header_order: [:x]) } + end + end + + # RETRY-12 / P6-6: the configured key, and #with. + class ConfiguredTest < DexpaceTestCase + include Hermetic + + test "RETRY-12 / P6-6: max_retries defaults through Keys::MAX_RETRY_ATTEMPTS when not passed" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + + assert_equal(7, Settings.build.max_retries) + assert_equal("MAX_RETRY_ATTEMPTS", KEY, "5a's name, its first reader here") + end + + test "RETRY-12 / P6-6: an explicit max_retries: wins over the configured key, zero included" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + + assert_equal(1, Settings.build(max_retries: 1).max_retries) + assert_equal(0, Settings.build(max_retries: 0).max_retries) + end + + test "RETRY-12: the key is read ONCE at build; a later configure does not move the snapshot" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + settings = Settings.build + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "9") } + + assert_equal(7, settings.max_retries) + assert_equal(9, Settings.build.max_retries) + end + + test "RETRY-12: with the key unset the default is Policy::DEFAULT_MAX_RETRIES" do + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries) + end + + test "RETRY-41 / P6-59: a negative CONFIGURED value is clamped to the default and logged" do + # The one read of the configured key is where RETRY-41's "a negative configured value MUST + # be clamped to the default (and the clamp logged)" is met, through the same resolver the + # drivers use; the settings object then holds a valid value (RECOV-34) and every driver + # built on it sees the default without a second clamp or a second log line. + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-1") } + sink = RecordingSink.new + settings = Settings.build(logger: Dexpace::Instrumentation::Logger.build(sink: sink)) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, settings.max_retries) + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_CONFIG, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/-1.*clamped to #{Policy::DEFAULT_MAX_RETRIES}/o, + payload[Dexpace::Instrumentation::Keys::MESSAGE],) + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries, + "Logger::NULL by default: clamped, reported nowhere",) + end + + test "RETRY-41 / OBS-20: the clamp's log is contained: a raising sink cannot fail the build" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-3") } + sink = RecordingSink.new + sink.define_singleton_method(:warn) { |*| raise ::IOError, "sink write failure" } + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build(logger: logger).max_retries) + end + + test "RECOV-34: an EXPLICIT negative max_retries: is still refused, never clamped" do + # The configured key is the environment's value; an explicit argument is the caller's + # construction input, and RECOV-34 refuses that at initialize -- with the key negative too. + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "-1") } + sink = RecordingSink.new + logger = Dexpace::Instrumentation::Logger.build(sink: sink) + + error = assert_raises(Dexpace::InvalidArgumentError) do + Settings.build(max_retries: -1, logger: logger) + end + + assert_includes(error.message, "max_retries") + assert_empty(sink.entries, "no clamp, so no clamp diagnostic") + end + + test "P6-59: logger: is read at build and never held -- not a member, absent from #to_h" do + settings = Settings.build(logger: Dexpace::Instrumentation::Logger::NULL) + + refute_includes(settings.to_h.keys, :logger) + refute_respond_to(settings, :logger) + assert_equal(settings, Settings.build, "value equality is over the ten members alone") + end + + test "RETRY-12 / CFG-5: an unparseable configured value falls to the default (5a's parser)" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "seven") } + + assert_equal(Policy::DEFAULT_MAX_RETRIES, Settings.build.max_retries) + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "010") } + + assert_equal(10, Settings.build.max_retries, "base 10 explicit (CFG-5)") + end + + test "P6-6 / RETRY-14: the configured number is the STAGE vocabulary; attempts are one more" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "4") } + + assert_equal(4, Settings.build.max_retries) + assert_equal(5, Settings.build.max_retries + 1, "the recovery stack's max_attempts") + end + + test "#with re-validates through .build on every Ruby, never through Data#with" do + settings = Settings.build(max_retries: 3) + clock = FakeClock.new + + assert_same(clock, settings.with(clock: clock).clock) + assert_equal(3, settings.with(clock: clock).max_retries, + "an explicit member survives #with",) + assert_raises(Dexpace::InvalidArgumentError) { settings.with(jitter: 5.0) } + assert_raises(Dexpace::InvalidArgumentError) { settings.with(initial_delay: -1) } + end + + test "#with of max_retries never re-hits the configured key" do + Dexpace.configure { |c| c.env_source = FakeConfigSource.new(KEY => "7") } + settings = Settings.build(max_retries: 1) + + assert_equal(1, settings.with(jitter: 0.0).max_retries) + end + + test "RETRY-42: value equality is over the members; two default settings are ==" do + assert_equal(Settings.build, Settings.build) + refute_equal(Settings.build, Settings.build(max_retries: 9)) + end + end +end diff --git a/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb new file mode 100644 index 0000000..961240d --- /dev/null +++ b/gems/dexpace-core/test/dexpace/resilience/retry_step_test.rb @@ -0,0 +1,801 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "../../test_helper" +require "dexpace" +require_relative "../../support/retry_fixtures" +require_relative "../../support/scripted_transport" +require_relative "../../support/recording_http_tracer" +require_relative "../../support/recording_sink" +require_relative "../../support/state_probe" +require_relative "../../support/fake_config_source" + +# Exercises: RETRY-2, RETRY-5, RETRY-6, RETRY-7, RETRY-8, RETRY-9, RETRY-10, RETRY-11, RETRY-12, +# RETRY-13, RETRY-15, RETRY-20, RETRY-21, RETRY-22, RETRY-23, RETRY-24, RETRY-25, RETRY-26, +# RETRY-28, RETRY-34, RETRY-35, RETRY-36, RETRY-39, RETRY-40, RETRY-41, RETRY-42, RETRY-44, +# RETRY-45, OBS-29, OBS-30, PIPE-15, PIPE-16, PIPE-40, XCUT-3, P6-5, P6-7, P6-11, P6-60 +# +# The synchronous stage-based retry step, driven through a REAL Dexpace::Pipeline over a +# ScriptedTransport: Builder#append reads #stage, the driver mints the step's cursor, and the +# fork-for-every-drive rule is asserted through a recording cursor over that real one. Every +# wait is a FakeClock's -- a real Clock::SYSTEM wait in this suite is a finding -- and the one +# SYSTEM-clock test is the RETRY-23 case whose zero-length wait cannot observe the token. +# Hermetic: a RetryStep.build with no settings reads the configured MAX_RETRY_ATTEMPTS, so every +# nested class runs behind a FakeConfigSource env seam. Split into nested classes under +# Metrics/ClassLength. +class DexpaceResilienceRetryStepTest < DexpaceTestCase + RetryStep = Dexpace::Resilience::RetryStep + Stages = Dexpace::Pipeline::Stages + + # The pipeline shape every nested class drives, the counters it reads off, and the seam. + module Fixtures + include RetryFixtures + + # Hermetic (5a's rule, R0-1): a default-settings build reads Keys::MAX_RETRY_ATTEMPTS off the + # process slot, so every test runs behind a FakeConfigSource env seam and resets the slot. + def setup + super + Dexpace.configure { |c| c.env_source = FakeConfigSource.new } + end + + def teardown + Dexpace.reset_config! + super + end + + def pipeline(step, script, wrap: true, extra: []) + transport = ScriptedTransport.new(script) + installed = wrap ? RecordingWrapper.new(step) : step + builder = Dexpace::Pipeline::Builder.new(transport: transport).append(installed) + extra.each { |(probe, stage)| builder.append(probe, stage: stage) } + [builder.build, transport, installed] + end + + def drive(step, script, request: retry_request, options: Dexpace::RequestOptions::EMPTY, + cancellation: Dexpace::Cancellation.none) + built, transport, wrapper = pipeline(step, script) + response = built.call(request, options, cancellation) + [response, transport, wrapper] + end + end + include Fixtures + + test "declares Stages::RETRY, is frozen, hides .new, and is installed by its own stage" do + step = RetryStep.build + + assert_same(Stages::RETRY, step.stage) + assert_predicate(step, :frozen?) + refute_respond_to(RetryStep, :new) + assert(Dexpace::Pipeline::Step.conforms?(step)) + entries = Dexpace::Pipeline::Builder.new(transport: ScriptedTransport.new([])).append(step) + .entries + + assert_equal([Stages::RETRY], entries.map(&:stage)) + end + + test "the constructor validates every keyword and names it" do + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(settings: :defaults) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(http_tracer_factory: :none) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(delay_override: 1.0) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(should_retry: true) } + assert_raises(Dexpace::InvalidArgumentError) { RetryStep.build(logger: RecordingSink.new) } + end + + test "pipeline/86343352 / PIPE-15 / PIPE-16: forks for EVERY drive, first included, no call" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + response, transport, wrapper = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + assert_equal(1, wrapper.cursors.size, "the step's own #call ran once for the operation") + assert_equal(3, wrapper.cursors.first.forks, "one fork per drive, attempt 1 included") + assert_equal(0, wrapper.cursors.first.calls, "the owning cursor's #call was never invoked") + refute_predicate(wrapper.cursors.first.real, :spent?) + end + + test "RETRY-44: every attempt re-sends the SAME request object through a fresh fork" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + request = retry_request + _response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: request,) + + assert_equal(2, transport.calls.size) + transport.calls.each { |(sent, _options, _token)| assert_same(request, sent) } + end + + test "RETRY-36: a 503, 503, 200 sequence terminates on the 200 after three sends" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(3, transport.calls.size) + refute_predicate(response.body, :closed?, "the response handed back is never closed") + end + + test "RETRY-42: eight threads through one frozen step keep their attempt counts apart" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + results = Array.new(8) do + ::Thread.new do + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(200),],) + [response.status.code, transport.calls.size] + end + end.map(&:value) + + assert_equal([[200, 3]] * 8, results) + end + + test "boundary 2: a RETRY fork writes no cursor state under any stage's key" do + probe = StateProbe.new(stage_to_read: Stages::RETRY) + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + built, = pipeline(step, [retry_response(503), retry_response(200)], + extra: [[probe, Stages::AUTH]],) + + built.call(retry_request) + + assert_equal(2, probe.reads.size) + probe.reads.each { |read| assert_empty(read) } + end + + # RETRY-5..8, RETRY-2, RETRY-24, RETRY-25, RETRY-34: the two axes and the terminal paths. + class EligibilityTest < DexpaceTestCase + include Fixtures + + test "RETRY-5 / RETRY-7: a bare non-idempotent POST gets exactly one attempt, any status" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-5 / RETRY-8: a replayable body is re-sent; a consumed one is not, PUT or POST" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST", body: replayable_body),) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "PUT", body: consumed_body),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-8: a should_retry answering true cannot override the re-sendability gate" do + step = RetryStep.build(settings: retry_settings(max_retries: 5), should_retry: ->(*) { true }) + response, transport, = drive(step, [retry_response(503), retry_response(200)], + request: retry_request(method: "POST"),) + + assert_equal(503, response.status.code) + assert_equal(1, transport.calls.size) + end + + test "RETRY-2 / RETRY-24: a throwable answering the capability is retried; one without, not" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + response, transport, = drive(step, [RetryableError.new("read timeout"), + retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + error = assert_raises(UnretryableError) do + drive(step, [UnretryableError.new("no"), retry_response(200)]) + end + + assert_empty(Dexpace.suppressed(error)) + error = assert_raises(::ArgumentError) { drive(step, [::ArgumentError.new("bug")]) } + + assert_equal("bug", error.message) + end + + test "RETRY-37 / XCUT-7: the configured set is authoritative, wider and narrower" do + wider = RetryStep.build(settings: retry_settings(max_retries: 2, retryable_statuses: [418])) + response, transport, = drive(wider, [retry_response(418), retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal(2, transport.calls.size) + + narrower = RetryStep.build(settings: retry_settings(max_retries: 2, + retryable_statuses: [429],)) + response, transport, = drive(narrower, [retry_response(503), retry_response(200)]) + + assert_equal(503, response.status.code, "503 is baked-retryable and still refused") + assert_equal(1, transport.calls.size) + end + end + + # RETRY-34 and RETRY-25: the terminal paths. + class TerminalPathTest < DexpaceTestCase + include Fixtures + + test "RETRY-34: the terminal raise is the LAST error, carrying the prior trail, never itself" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + errors = Array.new(3) { |i| RetryableError.new("attempt #{i + 1}") } + + surfaced = assert_raises(RetryableError) { drive(step, errors) } + + assert_same(errors[2], surfaced) + assert_equal(errors[0..1], Dexpace.suppressed(surfaced)) + refute_includes(Dexpace.suppressed(surfaced), surfaced) + assert_nil(surfaced.cause, "pipeline/7ce4431d: the carried re-raise assigns no cause") + end + + test "RETRY-34 / pipeline/7ce4431d: the carried re-raise adds nothing to a cause already set" do + # $! is non-nil inside anything called from a CALLER's rescue block. On this stack every + # surfaced throwable was RAISED by the downstream inside that same extent, so Ruby assigned + # the caller's exception as its cause at the transport's own raise, and a cause once set + # survives every later raise (verified on 4.0.6) -- which is why this test can assert only + # that the step's `raise failure, cause: nil` leaves that cause exactly as it found it. The + # DISCRIMINATING fixture, an error constructed and never raised, is RecoveryRetry's + # ProtocolError; its suite is where a bare `raise error` runs red. + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + errors = [RetryableError.new("one"), RetryableError.new("two")] + surfaced = nil + in_flight = nil + begin + raise ::IOError, "the caller's own in-flight exception" + rescue ::IOError => error + in_flight = error + surfaced = assert_raises(RetryableError) { drive(step, errors) } + end + + assert_same(errors[1], surfaced) + assert_same(in_flight, surfaced.cause, "set by the transport's raise, untouched since") + assert_equal([errors[0]], Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: a reused error instance trips no self-suppression -- the skip-self guard" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + same = RetryableError.new("same instance every time") + + surfaced = assert_raises(RetryableError) { drive(step, [same, same, same]) } + + assert_same(same, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + end + + test "RETRY-34: on eventual success the prior trail is discarded" do + step = RetryStep.build(settings: retry_settings(max_retries: 2)) + first = RetryableError.new("one") + response, = drive(step, [first, retry_response(200)]) + + assert_equal(200, response.status.code) + assert_empty(Dexpace.suppressed(first), "nothing was ever attached to a non-terminal error") + end + + test "RETRY-34: a terminal error status is RETURNED, never raised, its trail discarded" do + step = RetryStep.build(settings: retry_settings(max_retries: 1)) + response, transport, = drive(step, [retry_response(503), retry_response(503)]) + + assert_equal(503, response.status.code) + assert_equal(2, transport.calls.size) + refute_predicate(response.body, :closed?) + end + + test "RETRY-35 / OBS-30: a tracer raising in retries_exhausted propagates, terminal closed" do + # R1-1: the terminal emission sits inside the same fence as the decision -- a throwing + # tracer fails the request (OBS-30, never contained) but the terminal error-status response + # it would otherwise have returned is closed first, as the async pump's guarded block + # already did; round 0 had closed the same drift for attempt_failed only (R0-4). + terminal = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = nil + def exploding.retries_exhausted(*) = raise("tracer blew up") + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [retry_response(503), terminal]) } + + assert_equal("tracer blew up", error.message) + assert_predicate(terminal.body, :closed?, "closed before the raise propagated") + end + + test "RETRY-25: a fatal-family error propagates unchanged, unretried, unattached, unreported" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 3), + http_tracer_factory: ->(_cursor) { tracer },) + fatal = ::NoMemoryError.new("oom") + + surfaced = assert_raises(::NoMemoryError) { drive(step, [fatal, retry_response(200)]) } + + assert_same(fatal, surfaced) + assert_empty(Dexpace.suppressed(surfaced)) + assert_equal(%i[attempt_started], tracer.events.map(&:first), "no attempt_failed for it") + end + + test "RETRY-25: a fatal-family error answering the capability is still never retried" do + step = RetryStep.build(settings: retry_settings(max_retries: 3)) + fatal = RetryableFatal.new("oom, and lying about it") + built, transport, = pipeline(step, [fatal, retry_response(200)]) + + surfaced = assert_raises(RetryableFatal) { built.call(retry_request) } + + assert_same(fatal, surfaced) + assert_equal(1, transport.calls.size, "never a second send") + end + + test "RETRY-25: a fatal-family error from inside the retry decision is not classified either" do + step = RetryStep.build(settings: retry_settings(max_retries: 3), + should_retry: ->(*) { raise ::NoMemoryError, "oom in predicate" },) + open_response = retry_response(503) + + assert_raises(::NoMemoryError) { drive(step, [open_response]) } + assert_predicate(open_response.body, :closed?, "RETRY-35: closed before propagating") + end + end + + # RETRY-9..12, RETRY-15, RETRY-20..22, RETRY-35, RETRY-39, RETRY-40: delays and the close. + class DelayTest < DexpaceTestCase + include Fixtures + + test "RETRY-9 / RETRY-12 / RETRY-13: the schedule is Policy's, on the settings' clock" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 3) + step = RetryStep.build(settings: settings) + response, transport, = drive(step, [retry_response(503), retry_response(503), + retry_response(503), retry_response(200),],) + + assert_equal(200, response.status.code) + assert_equal(4, transport.calls.size) + assert_equal([0.1, 0.2, 0.4], clock.sleeps.map { _1[:duration] }) + expected = (1..3).map do |attempt| + Dexpace::Resilience::Policy.backoff_delay(attempt, **settings.backoff_arguments) + end + + assert_equal(expected, clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-26 / XCUT-3: every wait is the clock's cancellable sleep with the cursor's token" do + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + source = Dexpace::Cancellation.source + drive(step, [retry_response(503), retry_response(200)], cancellation: source.token) + + assert_equal(1, clock.sleeps.size) + assert_same(source.token, clock.sleeps.first[:cancellation]) + end + + test "RETRY-35: the response is released BEFORE the wait; the delay is read from it first" do + closed_at_sleep = [] + hinted = retry_response(503, headers: { "Retry-After" => "3" }) + clock = FakeClock.new + clock.define_singleton_method(:sleep) do |duration, cancellation: nil| + closed_at_sleep << hinted.body.closed? + super(duration, cancellation: cancellation) + end + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + response, = drive(step, [hinted, retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal([true], closed_at_sleep, "closed before the wait") + assert_equal([3.0], clock.sleeps.map { _1[:duration] }, "the hint was read while open") + end + + test "RETRY-35 / OBS-30: a tracer raising in attempt_failed propagates, the response closed" do + # The emission sits inside the same fence as the delay resolution: a throwing tracer fails + # the request (OBS-30, never contained) but never leaves the superseded response open + # across the propagation -- the async driver's guarded block already closed it (R0-4). + open_response = retry_response(503) + exploding = ::Object.new + def exploding.attempt_started(*) = nil + def exploding.attempt_failed(*) = raise("tracer blew up") + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 2), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [open_response, retry_response(200)]) } + + assert_equal("tracer blew up", error.message) + assert_predicate(open_response.body, :closed?, "closed before the raise propagated") + assert_empty(clock.sleeps, "the wait was never reached") + end + + test "RETRY-20 / RETRY-22: a pacing hint REPLACES the schedule verbatim; a bad one falls" do + clock = FakeClock.new + settings = retry_settings(clock: clock, initial_delay: 30.0, max_delay: 30.0, jitter: 0.5, + max_retries: 2,) + step = RetryStep.build(settings: settings) + drive(step, [retry_response(503, headers: { "Retry-After" => "2" }), + retry_response(429, headers: { "Retry-After" => "soon" }), + retry_response(200),],) + first, second = clock.sleeps.map { _1[:duration] } + + assert_in_delta(2.0, first, 0.001, "verbatim: not 30.0 and not jittered into [1.5, 2.5]") + assert_operator(second, :>=, 30.0 * 0.75) + assert_operator(second, :<=, 30.0 * 1.25) + end + + test "RETRY-21: the stage stack walks the caller's header order" do + clock = FakeClock.new + settings = retry_settings(clock: clock, max_retries: 1, + pacing_header_order: %w[retry-after-ms Retry-After],) + step = RetryStep.build(settings: settings) + both = { "Retry-After" => "9", "retry-after-ms" => "500" } + drive(step, [retry_response(503, headers: both), retry_response(200)]) + + assert_equal([0.5], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-15: an HTTP-date hint is measured from the settings clock's own now" do + clock = FakeClock.new(now: ::Time.utc(2026, 9, 18, 12, 0, 0)) + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1)) + hinted = retry_response(503, headers: { "Retry-After" => "Fri, 18 Sep 2026 12:00:45 GMT" }) + drive(step, [hinted, retry_response(200)]) + + assert_equal([45.0], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-39: the exception path skips the header step and reaches backoff" do + clock = FakeClock.new + seen = [] + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: lambda { |attempt, response, error| + seen << [attempt, response, error] + nil + },) + drive(step, [RetryableError.new("boom"), retry_response(200)]) + + assert_equal(1, seen.size) + assert_equal(1, seen.first[0]) + assert_nil(seen.first[1], "no response on the exception path") + assert_kind_of(RetryableError, seen.first[2]) + assert_equal([0.1], clock.sleeps.map { _1[:duration] }) + end + end + + # RETRY-39's first tier and RETRY-40's non-fatal half. + class DelayOverrideTest < DexpaceTestCase + include Fixtures + + test "RETRY-39: a present delay-override wins over a present pacing hint" do + clock = FakeClock.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: ->(_attempt, _response, _error) { 7 },) + drive(step, [retry_response(503, headers: { "Retry-After" => "2" }), retry_response(200)]) + + assert_equal([7.0], clock.sleeps.map { _1[:duration] }) + end + + test "RETRY-40: a throwing delay-override is non-fatal -- logged, and backoff applies" do + clock = FakeClock.new + sink = RecordingSink.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 1), + delay_override: ->(*) { raise "override down" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + response, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(200, response.status.code) + assert_equal([0.1], clock.sleeps.map { _1[:duration] }) + assert_equal([:warn], sink.entries.map(&:severity)) + payload = sink.payloads.first + + assert_equal(Dexpace::Instrumentation::Events::INSTRUMENTATION_HOOK, + payload[Dexpace::Instrumentation::Keys::EVENT],) + assert_match(/override down/, payload[Dexpace::Instrumentation::Keys::CAUSE]) + end + + test "RETRY-40: a delay-override answering a negative or non-numeric value is logged too" do + clock = FakeClock.new + sink = RecordingSink.new + step = RetryStep.build(settings: retry_settings(clock: clock, max_retries: 2), + delay_override: ->(attempt, *) { attempt == 1 ? -1 : "soon" }, + logger: Dexpace::Instrumentation::Logger.build(sink: sink),) + drive(step, [retry_response(503), retry_response(503), retry_response(200)]) + + assert_equal([0.1, 0.2], clock.sleeps.map { _1[:duration] }) + assert_equal(2, sink.entries.size) + end + + test "RETRY-35 / RETRY-40: a throwing delay computation still closes the response first" do + open_response = retry_response(503) + step = RetryStep.build(settings: retry_settings(max_retries: 1), + delay_override: ->(*) { raise ::NotImplementedError, "fatal-shaped" },) + + assert_raises(::NotImplementedError) { drive(step, [open_response]) } + assert_predicate(open_response.body, :closed?) + end + end + + # RETRY-40's predicate, RETRY-41's count, RETRY-23's cancellation. + class PredicateAndBudgetTest < DexpaceTestCase + include Fixtures + + test "RETRY-40: a throwing should_retry aborts as RetryPredicateError, the raise its cause" do + open_response = retry_response(503) + step = RetryStep.build(settings: retry_settings(max_retries: 1), + should_retry: ->(*) { raise "boom" },) + + error = assert_raises(Dexpace::RetryPredicateError) { drive(step, [open_response]) } + + assert_kind_of(::RuntimeError, error.cause) + assert_equal("boom", error.cause.message) + assert_kind_of(Dexpace::Error, error) + assert_predicate(open_response.body, :closed?, "RETRY-35's third ordering") + end + + test "RETRY-40: should_retry receives the failure and the request and decides the condition" do + seen = [] + step = RetryStep.build(settings: retry_settings(max_retries: 5), + should_retry: lambda { |failure, request| + seen << [failure, request] + failure.is_a?(Dexpace::ProtocolError) && failure.status.code == 404 + },) + request = retry_request + response, transport, = drive(step, [retry_response(404), retry_response(503)], + request: request,) + + assert_equal(503, response.status.code, "404 retried by the predicate, 503 refused by it") + assert_equal(2, transport.calls.size) + assert_equal(2, seen.size) + assert_same(request, seen.first[1]) + assert_equal(404, seen.first[0].status.code) + end + + test "RETRY-41: the per-call max_retries override wins over the configured value" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 1, tags: {}) + response, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(503, response.status.code) + assert_equal(2, transport.calls.size, "one retry: the override") + + options = Dexpace::RequestOptions.build(timeout: nil, max_retries: 0, tags: {}) + _response, transport, = drive(step, Array.new(6) { retry_response(503) }, options: options) + + assert_equal(1, transport.calls.size, "zero means no retries") + end + + test "RETRY-41: with no override the configured value applies, and zero disables retries" do + step = RetryStep.build(settings: retry_settings(max_retries: 0)) + _response, transport, = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(1, transport.calls.size) + end + + test "RETRY-23 / RECOV-27: cancellation during the wait aborts the loop as CancelledError" do + source = Dexpace::Cancellation.source + step = RetryStep.build(settings: retry_settings(max_retries: 5, initial_delay: 5.0)) + script = [lambda { |*| + source.cancel(:test) + retry_response(503) + }, retry_response(200),] + + error = assert_raises(Dexpace::CancelledError) do + drive(step, script, cancellation: source.token) + end + + assert_equal(:test, error.reason) + refute(Dexpace::Resilience::Policy.throwable_retryable?(error), "never a retryable condition") + end + + test "RETRY-23: a cancelled token is checked at the top of every attempt, wait or no wait" do + # Under Clock::SYSTEM a zero-length sleep returns before the token check (5a, verified), so + # this attempt-boundary check is the only thing standing between the cancellation and a + # second send. The one SYSTEM-clock wait in this suite is this zero-length one. + source = Dexpace::Cancellation.source + settings = Dexpace::Resilience::RetrySettings.build(max_retries: 5, initial_delay: 0.0, + jitter: 0.0,) + step = RetryStep.build(settings: settings) + script = [lambda { |*| + source.cancel(:mid_attempt) + retry_response(503) + }, retry_response(200),] + + error = assert_raises(Dexpace::CancelledError) do + drive(step, script, cancellation: source.token) + end + + assert_equal(:mid_attempt, error.reason) + end + + test "RETRY-23: a downstream cancellation surfaces as terminal, is never retried" do + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + error = assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + + assert_equal(:downstream, error.reason) + end + + test "RETRY-23: no attempt is launched after a cancellation observed mid-loop" do + source = Dexpace::Cancellation.source + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + script = [lambda { |*| + source.cancel(:stop) + retry_response(503) + }, retry_response(200),] + + assert_raises(Dexpace::CancelledError) { drive(step, script, cancellation: source.token) } + # the script's second entry was never consumed + _built, transport, = pipeline(step, script) + + assert_equal(0, transport.calls.size) + end + end + + # RETRY-23's guard ahead of the caller's predicate and the capability (P6-60). + class CancellationGuardTest < DexpaceTestCase + include Fixtures + + test "RETRY-23 / P6-60: a should_retry answering true cannot retry a downstream cancellation" do + step = RetryStep.build(settings: retry_settings(max_retries: 5), should_retry: ->(*) { true }) + seen = [] + spy = RetryStep.build(settings: retry_settings(max_retries: 5), + should_retry: lambda { |failure, _request| + seen << failure + true + },) + + error = assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + + assert_equal(:downstream, error.reason) + assert_raises(Dexpace::CancelledError) do + drive(spy, [Dexpace::CancelledError.new(:downstream), retry_response(200)]) + end + assert_empty(seen, "the predicate is never consulted for a cancellation") + end + + test "RETRY-23 / P6-60: a cancellation WRAPPED by a retryable error is terminal too" do + # A transport that wraps the token's raise in its own retryable error: the capability + # branch alone would retry it; the guard reads the cancellation it carries. + wrapped = begin + begin + raise Dexpace::CancelledError, :token + rescue Dexpace::CancelledError + raise RetryableError, "read interrupted" + end + rescue RetryableError => error + error + end + step = RetryStep.build(settings: retry_settings(max_retries: 5)) + + surfaced = assert_raises(RetryableError) { drive(step, [wrapped, retry_response(200)]) } + + assert_same(wrapped, surfaced) + assert_kind_of(Dexpace::CancelledError, surfaced.cause) + assert_empty(Dexpace.suppressed(surfaced), "one attempt: no trail") + _built, transport, = pipeline(step, [wrapped, retry_response(200)]) + + assert_equal(0, transport.calls.size) + end + + test "RETRY-23: the guard runs before the re-sendability gate and emits no retries_exhausted" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 0), + http_tracer_factory: ->(_cursor) { tracer },) + + assert_raises(Dexpace::CancelledError) do + drive(step, [Dexpace::CancelledError.new(:downstream)], + request: retry_request(method: "POST", body: replayable_body),) + end + assert_equal(%i[attempt_started], tracer.events.map(&:first), + "a cancellation is :stop, never :exhausted, at any budget",) + end + end + + # OBS-29's per-attempt group, and the three text-scan guards (RETRY-13, RETRY-28, RETRY-45). + class TracerAndGuardsTest < DexpaceTestCase + include Fixtures + + RESILIENCE = File.expand_path("../../../lib/dexpace/resilience", __dir__) + + def source(name) = File.read(File.join(RESILIENCE, "#{name}.rb")).gsub(/^\s*#.*$/, "") + + test "OBS-29 / P6-7: the factory is called once per operation, with the cursor as context" do + tracers = [] + cursors = [] + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: lambda { |cursor| + cursors << cursor + tracers << Dexpace::RecordingHTTPTracer.new + tracers.last + },) + _response, _transport, wrapper = drive(step, [retry_response(503), retry_response(200)]) + + assert_equal(1, tracers.size) + assert_equal([wrapper.cursors.first], cursors, "the step's own cursor, not a fork") + tracers.first.events.each { |event| assert_same(wrapper.cursors.first, event[1]) } + end + + test "OBS-29: attempt_started, attempt_failed with the delay, attempt_started, then success" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(503), retry_response(200)]) + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started], names) + assert_equal(1, tracer.events[0][2]) + assert_kind_of(Dexpace::ProtocolError, tracer.events[1][2], "an Exception, never a Response") + assert_equal(503, tracer.events[1][2].status.code) + assert_in_delta(0.1, tracer.events[1][3]) + assert_equal(2, tracer.events[2][2]) + end + + test "OBS-29: retries_exhausted fires once, after the last attempt, with the surfaced error" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { tracer },) + errors = [RetryableError.new("one"), RetryableError.new("two")] + + surfaced = assert_raises(RetryableError) { drive(step, errors) } + names = tracer.events.map(&:first) + + assert_equal(%i[attempt_started attempt_failed attempt_started retries_exhausted], names) + assert_same(surfaced, tracer.events.last[2]) + assert_same(errors[0], tracer.events[1][2]) + end + + test "OBS-29: an exhausted error STATUS reports a ProtocolError carrying the trail" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(503), retry_response(503)]) + exhausted = tracer.events.last + + assert_equal(:retries_exhausted, exhausted.first) + assert_equal(503, exhausted[2].status.code) + assert_equal([tracer.events[1][2]], Dexpace.suppressed(exhausted[2])) + end + + test "OBS-29: a failure that was never retryable ends with no retries_exhausted at all" do + tracer = Dexpace::RecordingHTTPTracer.new + step = RetryStep.build(settings: retry_settings(max_retries: 2), + http_tracer_factory: ->(_cursor) { tracer },) + drive(step, [retry_response(404)]) + + assert_equal(%i[attempt_started], tracer.events.map(&:first)) + assert_raises(::ArgumentError) { drive(step, [::ArgumentError.new("bug")]) } + assert_equal(%i[attempt_started attempt_started], tracer.events.map(&:first)) + end + + test "OBS-30: a throwing tracer propagates and fails the request; core wraps no callback" do + exploding = ::Object.new + def exploding.attempt_started(*) = raise("tracer blew up") + step = RetryStep.build(settings: retry_settings(max_retries: 1), + http_tracer_factory: ->(_cursor) { exploding },) + + error = assert_raises(::RuntimeError) { drive(step, [retry_response(200)]) } + + assert_equal("tracer blew up", error.message) + end + + test "RETRY-28 / P6-5: neither stage driver names the budget, by text" do + %w[retry_step async_retry_step retry_step_helpers].each do |name| + code = source(name) + + refute_match(/budget_remaining|total_timeout/, code, name) + end + assert_match(/budget_remaining/, source("recovery_retry"), "the recovery driver alone") + end + + test "RETRY-13: no delay literal outside Policy, in any resilience file" do + %w[retry_step async_retry_step retry_step_helpers recovery_retry resend retry_settings] + .each do |name| + code = source(name) + + refute_match(/\b(0\.2|2\.0|8\.0)\b/, code, "#{name} re-types a Policy default") + refute_match(/DEFAULT_(INITIAL_DELAY|MULTIPLIER|MAX_DELAY|JITTER)\s*=/, code, name) + end + end + + test "RETRY-45: the engine never shuts down or installs a scheduler, by text" do + Dir[File.join(RESILIENCE, "*.rb")].each do |path| + code = File.read(path).gsub(/^\s*#.*$/, "") + + touches = code.scan(/set_scheduler|scheduler&?\.close|shutdown/) + + assert_empty(touches, "#{File.basename(path)} touches a scheduler") + end + end + end +end diff --git a/gems/dexpace-core/test/dexpace_test.rb b/gems/dexpace-core/test/dexpace_test.rb index 845dd69..43ff031 100644 --- a/gems/dexpace-core/test/dexpace_test.rb +++ b/gems/dexpace-core/test/dexpace_test.rb @@ -37,11 +37,13 @@ class DexpaceTest < DexpaceTestCase # Every public constant the surface manifest records, and the check that catches a file added # to lib/ and forgotten in the entry point. Phase 1's domain model, then phase 2's seam layer, # then phase 3a's byte-streaming layer, then phase 3b's body layer, then phase 4a's execution - # context, phase 4b's recovery layer, phase 4c's pipeline and phase 5a's configuration layer; + # context, phase 4b's recovery layer, phase 4c's pipeline, phase 5a's configuration layer and + # phase 6a's retry layer (the flat RetryPredicateError beside the Resilience namespace); # Dexpace::Hooks, Dexpace::BoundedMap, Dexpace::CallKey, Dexpace::Recovery::Ownership, the two - # pipeline drivers, Dexpace::ConfigParsers, Dexpace::DeepValue and Dexpace::ProxyResolution are - # private_constants and appear in no constants(false) list. Phase 5c's tracing and metrics - # layer and phase 5b's logging layer add no flat constant: everything either ships is under + # pipeline drivers, Dexpace::ConfigParsers, Dexpace::DeepValue, Dexpace::ProxyResolution and + # 6a's Resilience::PacingParsers and Resilience::RetryStepHelpers are private_constants and + # appear in no constants(false) list. Phase 5c's tracing and metrics layer and phase 5b's + # logging layer add no flat constant: everything either ships is under # Dexpace::Instrumentation, which the Layers case below pins. DOMAIN_MODEL = %i[ Error InvalidArgumentError Model Builder HeaderSyntax HeaderName Headers Status Method @@ -65,9 +67,10 @@ class DexpaceTest < DexpaceTestCase CONFIGURATION_LAYER = %i[ BuildInfo UUID Retryability HTTPDate Clock Configuration Proxy ].freeze + RESILIENCE_LAYER = %i[RetryPredicateError Resilience].freeze LAYERS = [ DOMAIN_MODEL, SEAM_LAYER, IO_LAYER, BODY_LAYER, CONTEXT_LAYER, RECOVERY_LAYER, PIPELINE_LAYER, - CONFIGURATION_LAYER, + CONFIGURATION_LAYER, RESILIENCE_LAYER, ].flatten.freeze test "defines nothing outside the Dexpace namespace" do @@ -169,6 +172,25 @@ class Layers < DexpaceTestCase assert_raises(::NameError) { Dexpace::CallKey } end + # A consumer requires "dexpace" and nothing else: the retry layer resolves too (phase 6a) -- + # the five public Resilience constants, the flat error, and the two private helpers and the + # two private per-call classes as unreachable as Dexpace::Hooks. Phase 6b and 6c add their + # own constants under Resilience beside these. + test "requiring dexpace alone makes the whole retry layer resolve, its helpers private" do + resilience = Dexpace::Resilience + + assert_equal(%i[AsyncRetryStep Policy RecoveryRetry Resend RetrySettings RetryStep], + resilience.constants(false).sort,) + assert_equal(Dexpace::RetryPredicateError, Dexpace.const_get(:RetryPredicateError)) + assert_equal(2, resilience::Policy::DEFAULT_MAX_RETRIES) + assert_raises(::NameError) { Dexpace::Resilience::PacingParsers } + assert_raises(::NameError) { Dexpace::Resilience::RetryStepHelpers } + assert_raises(::NameError) { Dexpace::Resilience::AsyncRetryStep::Pump } + assert_raises(::NameError) { Dexpace::Resilience::RetryStep::Run } + assert_raises(::NameError) { Dexpace::Resilience::RecoveryRetry::Run } + assert_raises(::NameError) { Dexpace::Resilience::RetrySettings::UNSET } + end + # A consumer requires "dexpace" and nothing else: the seam layer resolves too (phase 2). test "requiring dexpace alone makes the whole seam layer resolve" do assert_equal(Dexpace::Transport, Dexpace.const_get(:Transport)) diff --git a/gems/dexpace-core/test/support/retry_fixtures.rb b/gems/dexpace-core/test/support/retry_fixtures.rb new file mode 100644 index 0000000..b63a266 --- /dev/null +++ b/gems/dexpace-core/test/support/retry_fixtures.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require "dexpace" +require_relative "recovery_fixtures" +require_relative "fake_clock" +require_relative "fake_body" + +# The fixtures phase 6a's retry suites fold over: a request per method, a response over a REAL +# ResponseBody (so `response.body.closed?` is the RETRY-35 assertion -- Dexpace::Response has no +# #closed? of its own), a retryable throwable that answers XCUT-6's capability and nothing else, +# and a step that hands a RetryStep a RECORDING cursor over the real one the driver minted, so +# the fork-for-every-drive rule is asserted against the real runtime rather than a stand-in +# (5b's checklist item 12's route). Built on RecoveryFixtures rather than beside it. +module RetryFixtures + include RecoveryFixtures + + # A throwable that is retryable by capability -- an IOError, as a socket read timeout would + # be (RETRY-24's shape) -- and one that answers the capability with false. + class RetryableError < ::IOError + def retryable? = true + end + + # The other answer: an IOError that says no, which a type match would call retryable. + class UnretryableError < ::IOError + def retryable? = false + end + + # RETRY-25's discriminating fixture: a fatal-family error that ANSWERS the capability. A driver + # that classified it would retry it; the correct driver never asks, because the fatal family + # passes through an absent rescue arm before any classification runs. + class RetryableFatal < ::NoMemoryError + def retryable? = true + end + + # Forwards everything to the real cursor and counts what the step did with it. #call raises + # rather than counting: a pillar step that calls its own cursor is the defect the count exists + # to find, and a raise names it at the site. + class RecordingCursor + attr_reader :forks, :calls, :real + + def initialize(real) + @real = real + @forks = 0 + @calls = 0 + end + + def fork(state: nil) + @forks += 1 + @real.fork(state: state) + end + + def call(request = @real.request) + @calls += 1 + @real.call(request) + end + + def request = @real.request + def options = @real.options + def cancellation = @real.cancellation + def bundle = @real.bundle + def spent? = @real.spent? + def may_fork? = @real.may_fork? + def state(stage) = @real.state(stage) + end + + # A step that wraps the cursor before handing it on, for #forks and #calls. + class RecordingWrapper + attr_reader :cursors + + def initialize(step) + @step = step + @cursors = [] + end + + def stage = @step.stage + + def call(request, cursor) + recording = RecordingCursor.new(cursor) + @cursors << recording + @step.call(request, recording) + end + end + + def retry_request(method: "GET", body: nil, url: "https://example.test/api") + Dexpace::Request.build(method: Dexpace::Method.of(method), url: url, + headers: Dexpace::Headers::EMPTY, body: body,) + end + + # A response over a closable single-use body, with optional inbound headers. + def retry_response(code, headers: {}, request: retry_request) + builder = Dexpace::Response.builder + builder.request = request + builder.protocol = Dexpace::Protocol::HTTP_1_1 + builder.status = code + builder.body = response_body("body #{code}") + unless headers.empty? + inbound = Dexpace::Headers.inbound_builder + headers.each { |name, value| inbound.add(name, value) } + builder.headers = inbound.build + end + builder.build + end + + def replayable_body = FakeBody.new("payload", replayable: true) + def consumed_body = FakeBody.new("payload", replayable: false) + + # Settings over a FakeClock with a flat, jitter-free schedule unless told otherwise. + def retry_settings(clock: FakeClock.new, **overrides) + Dexpace::Resilience::RetrySettings.build( + initial_delay: 0.1, multiplier: 2.0, max_delay: 8.0, jitter: 0.0, + clock: clock, **overrides, + ) + end +end diff --git a/gems/dexpace-core/test/support/scripted_async_transport.rb b/gems/dexpace-core/test/support/scripted_async_transport.rb new file mode 100644 index 0000000..25c0a0d --- /dev/null +++ b/gems/dexpace-core/test/support/scripted_async_transport.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +require_relative "scripted_transport" + +# The async twin of ScriptedTransport (SEAM-16): the same script -- a callable, an Exception or a +# response per call, in order -- delivered through a fresh Completer per call. By default each +# future settles INLINE, inside #call, which is the shape a scripted downstream has and the shape +# that made the plan's recursive pump overflow (RETRY-30); with `settle_later: true` every future +# is left pending and #settle_next! settles the oldest unsettled one, so a test can hold an +# attempt in flight, cancel the returned future, and then let the attempt land (RETRY-32). +# +# A Proc entry is invoked with (request, options, cancellation) and its answer -- a Response or an +# Exception -- is what the future settles with, so a script can cancel a token from inside the +# attempt it is serving. #calls and the three-positional #call are ScriptedTransport's. +class ScriptedAsyncTransport + attr_reader :pending + + def initialize(script, settle_later: false) + @inner = ScriptedTransport.new(script) + @settle_later = settle_later + @pending = [] + @mutex = ::Thread::Mutex.new + end + + def calls = @inner.calls + + def call(request, options, cancellation) + completer = Dexpace::Async::Completer.new + outcome = begin + [@inner.call(request, options, cancellation), nil] + rescue ::StandardError => error + [nil, error] + end + if @settle_later + @mutex.synchronize { @pending << [completer, outcome] } + else + deliver(completer, outcome) + end + completer.future + end + + # Settles the oldest pending future; nil when none is pending. + def settle_next! + entry = @mutex.synchronize { @pending.shift } + return nil if entry.nil? + + deliver(*entry) + true + end + + private + + def deliver(completer, outcome) + response, error = outcome + error.nil? ? completer.fulfil(response) : completer.fail(error) + end +end diff --git a/gems/dexpace-core/test/support/scripted_transport.rb b/gems/dexpace-core/test/support/scripted_transport.rb new file mode 100644 index 0000000..2a72d8e --- /dev/null +++ b/gems/dexpace-core/test/support/scripted_transport.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +# SPDX-License-Identifier: MIT + +# A scriptable Dexpace::Transport double (phase 2's duck type, SEAM-11), phase 6a's: where +# FakeTransport answers ONE response or raise for every call, this one consumes a script -- one +# entry per call, in order -- so a retry suite can stage a 503, 503, 200 sequence (RECOV-19, +# RETRY-36) or a run of throwables (RETRY-34). Each entry is consumed once: a callable is +# invoked with (request, options, cancellation) and its answer used; an Exception instance is +# raised; anything else is returned as it is. A call past the end of the script raises, so a +# driver that sends once too often fails loudly rather than reading nil. +# +# #calls records every call's [request, options, cancellation] triple, frozen, so a test can +# assert the SAME request object was re-sent every time (RETRY-44, RECOV-19) with assert_same. +# `def call(request, options, cancellation)` with three positionals is what keeps +# Registry.callable?(transport, arity: 3) true, which Recovery::Orchestrator.build checks. +# +# Not phase 2's FakeTransport, which seven suites and three other doubles require by name and +# shape and which stays exactly as it is; a new double under a new name (phase 6a's checklist, +# "Deviations from the plan"). Top level, like every double under test/support/ since phase 2. +class ScriptedTransport + attr_reader :calls + + def initialize(script) + @script = script.dup + @calls = [] + @mutex = ::Thread::Mutex.new + end + + def call(request, options, cancellation) + entry = @mutex.synchronize do + @calls << [request, options, cancellation].freeze + raise "ScriptedTransport: script exhausted after #{@calls.size - 1} calls" if @script.empty? + + @script.shift + end + case entry + when ::Exception then raise entry + when ::Proc, ::Method then entry.call(request, options, cancellation) + else entry + end + end + + # The script entries not yet consumed. + def remaining = @mutex.synchronize { @script.size } +end diff --git a/test/fixtures/surface/dexpace-core.txt b/test/fixtures/surface/dexpace-core.txt index 0b4f5f3..a5d8b0f 100644 --- a/test/fixtures/surface/dexpace-core.txt +++ b/test/fixtures/surface/dexpace-core.txt @@ -622,6 +622,7 @@ Dexpace::Pipeline::Builder#replace Dexpace::Pipeline::Builder.flattening Dexpace::Pipeline::Builder.nesting Dexpace::Pipeline::Cursor +Dexpace::Pipeline::Cursor#bundle Dexpace::Pipeline::Cursor#call Dexpace::Pipeline::Cursor#cancellation Dexpace::Pipeline::Cursor#fork @@ -681,6 +682,7 @@ Dexpace::Protocol::HTTP_2 : Dexpace::Protocol Dexpace::Protocol::WIRE_FORMS : Array Dexpace::ProtocolError Dexpace::ProtocolError#response +Dexpace::ProtocolError#retryable_by_status? Dexpace::ProtocolError#status Dexpace::ProtocolError.for Dexpace::ProtocolError.for_or_nil @@ -832,6 +834,52 @@ Dexpace::RequestOptions::Builder#max_retries= Dexpace::RequestOptions::Builder#tag Dexpace::RequestOptions::Builder#timeout= Dexpace::RequestOptions::EMPTY : Dexpace::RequestOptions +Dexpace::Resilience +Dexpace::Resilience::AsyncRetryStep +Dexpace::Resilience::AsyncRetryStep#call +Dexpace::Resilience::AsyncRetryStep#stage +Dexpace::Resilience::AsyncRetryStep.build +Dexpace::Resilience::Policy +Dexpace::Resilience::Policy#backoff_delay +Dexpace::Resilience::Policy#budget_remaining +Dexpace::Resilience::Policy#cancellation? +Dexpace::Resilience::Policy#effective_max_retries +Dexpace::Resilience::Policy#pacing_delay +Dexpace::Resilience::Policy#retry_eligible? +Dexpace::Resilience::Policy#retryable? +Dexpace::Resilience::Policy#throwable_retryable? +Dexpace::Resilience::Policy::DEFAULT_INITIAL_DELAY : Float +Dexpace::Resilience::Policy::DEFAULT_JITTER : Float +Dexpace::Resilience::Policy::DEFAULT_MAX_DELAY : Float +Dexpace::Resilience::Policy::DEFAULT_MAX_RETRIES : Integer +Dexpace::Resilience::Policy::DEFAULT_MULTIPLIER : Float +Dexpace::Resilience::Policy::DEFAULT_PACING_HEADER_ORDER : Array +Dexpace::Resilience::Policy::DEFAULT_RETRYABLE_STATUSES : Set +Dexpace::Resilience::Policy::MAX_DURATION_NANOSECONDS : Integer +Dexpace::Resilience::Policy::MAX_PACING_DELAY_SECONDS : Integer +Dexpace::Resilience::RecoveryRetry +Dexpace::Resilience::RecoveryRetry#call +Dexpace::Resilience::RecoveryRetry.build +Dexpace::Resilience::Resend +Dexpace::Resilience::Resend#eligible? +Dexpace::Resilience::RetrySettings +Dexpace::Resilience::RetrySettings#backoff_arguments +Dexpace::Resilience::RetrySettings#clock +Dexpace::Resilience::RetrySettings#header_order +Dexpace::Resilience::RetrySettings#initial_delay +Dexpace::Resilience::RetrySettings#jitter +Dexpace::Resilience::RetrySettings#max_delay +Dexpace::Resilience::RetrySettings#max_retries +Dexpace::Resilience::RetrySettings#multiplier +Dexpace::Resilience::RetrySettings#pacing_header_order +Dexpace::Resilience::RetrySettings#random +Dexpace::Resilience::RetrySettings#retryable_statuses +Dexpace::Resilience::RetrySettings#total_timeout +Dexpace::Resilience::RetrySettings.build +Dexpace::Resilience::RetryStep +Dexpace::Resilience::RetryStep#call +Dexpace::Resilience::RetryStep#stage +Dexpace::Resilience::RetryStep.build Dexpace::Response Dexpace::Response#body Dexpace::Response#body_bytes @@ -876,6 +924,7 @@ Dexpace::ResponseLoggingBody#preview_bytes Dexpace::ResponseLoggingBody#snapshot Dexpace::ResponseLoggingBody#source Dexpace::ResponseLoggingBody#write_to +Dexpace::RetryPredicateError Dexpace::Retryability Dexpace::Retryability#retryable_status? Dexpace::SeamError