Skip to content

Commit 3634612

Browse files
authored
Merge pull request #291 from durable-workflow/fix/215-redrive-reuse-proof
fix: require redrive reuse authority for replay time
2 parents 6950eda + 0fff23d commit 3634612

4 files changed

Lines changed: 278 additions & 4 deletions

File tree

‎src/durable_workflow/replay_clock.py‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
_CHILD_IDENTITIES = ("child_call_id", "child_workflow_instance_id", "child_workflow_run_id", "child_workflow_type")
2727
_CHILD_EVENTS = frozenset({"ChildWorkflowScheduled", "ChildRunStarted", *_CHILD_TERMINALS})
2828
_CHILD_FAMILY = "child workflow"
29+
_REUSE_FIELDS = ("reused_from_run_id", "reused_recorded_at", "reused_activity_execution_id")
2930

3031

3132
def _task() -> object | None:
@@ -123,6 +124,21 @@ def _identity(value: Any) -> TypeGuard[str]:
123124
return isinstance(value, str) and bool(value.strip())
124125

125126

127+
def _declares_redrive(event: Mapping[str, Any]) -> bool:
128+
payload = event.get("payload")
129+
if not isinstance(payload, Mapping):
130+
return False
131+
# A conflicting kind alias invalidates authority; it must not erase a
132+
# recognizable redrive envelope and turn it into ordinary history.
133+
aliases = (event.get("event_type"), event.get("type"))
134+
if "StartAccepted" in aliases and payload.get("outcome") == "redriven":
135+
return True
136+
return "WorkflowStarted" in aliases and (
137+
payload.get("recovery_kind") == "redrive"
138+
or "replayed_started_at" in payload or "resume_step_sequence" in payload
139+
)
140+
141+
126142
class BusinessReplayClock:
127143
"""Index identities once, then read time only at actual scalar consumption."""
128144

@@ -133,6 +149,7 @@ def __init__(self, events: Sequence[Mapping[str, Any]], workflow_id: str, run_id
133149
self._source_run: str | None = None
134150
self._resume_step: int | None = None
135151
self._redrive_inserted: datetime | None = None
152+
self._redrive_declared = any(_declares_redrive(event) for event in events)
136153
starts = [index for index, event in enumerate(events) if _kind(event) == "WorkflowStarted"]
137154
self.initial: tuple[int | None, datetime | None] = (None, None)
138155
if len(starts) == 1:
@@ -218,12 +235,24 @@ def observe(self, index: int | None, family: str, sequence: int) -> None:
218235
return
219236
event = self.events[index]
220237
payload = event["payload"]
221-
if _kind(event) in {"ActivityCompleted", "TimerFired"} and "reused_recorded_at" in payload:
222-
time = self._reused_time(payload, family, sequence)
223-
else:
224-
time = _time(_recorded_value(event))
238+
time = (_time(_recorded_value(event)) if family == _CHILD_FAMILY
239+
else self._scalar_time(event, payload, family, sequence))
225240
self._pending = (index, time)
226241

242+
def _scalar_time(
243+
self, event: Mapping[str, Any], payload: Mapping[str, Any], family: str, sequence: int,
244+
) -> datetime | None:
245+
if self._redrive_declared and self._source_run is None:
246+
return None
247+
before_resume = self._resume_step is not None and sequence < self._resume_step
248+
if _kind(event) in {"ActivityCompleted", "TimerFired"} and (
249+
before_resume or any(key in payload for key in _REUSE_FIELDS)
250+
):
251+
# The validated resume boundary implies reuse even when both
252+
# markers are missing. Insertion time cannot replace its source.
253+
return self._reused_time(payload, family, sequence)
254+
return _time(_recorded_value(event))
255+
227256
def _child_boundary(self, index: int, sequence: int) -> bool:
228257
indices = self._by_sequence.get(sequence, [])
229258
if len(indices) not in (2, 3) or indices[-1] != index:
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json",
3+
"fixture_schema": "durable-workflow.replay-regression/v1",
4+
"id": "business-clock-redrive-timer-missing-reuse",
5+
"protocol_version": "1.20",
6+
"bindings": [
7+
"python"
8+
],
9+
"workflow": {
10+
"type": "tests.replay.business-clock-core-prefix-timer",
11+
"input": [
12+
"Taylor",
13+
5
14+
],
15+
"payload_codec": "avro",
16+
"workflow_id": "regression-corpus-redrive-successor-timer",
17+
"run_id": "01JBREDRIVESUCCESSORRUN0000"
18+
},
19+
"history": [
20+
{
21+
"sequence": 1,
22+
"event_type": "WorkflowStarted",
23+
"recorded_at": "2026-11-03T00:05:00+00:00",
24+
"payload": {
25+
"workflow_type": "tests.replay.business-clock-core-prefix-timer",
26+
"workflow_instance_id": "regression-corpus-redrive-successor-timer",
27+
"workflow_run_id": "01JBREDRIVESUCCESSORRUN0000",
28+
"workflow_command_id": "01JBREDRIVECOMMAND00000000",
29+
"continued_from_run_id": "01JBREDRIVESOURCERUN000000",
30+
"resume_step_sequence": 3,
31+
"recovery_kind": "redrive",
32+
"execution_deadline_at": null,
33+
"run_deadline_at": null,
34+
"replayed_started_at": "2026-11-03T00:00:00+00:00",
35+
"workflow_link_id": "01JBREDRIVELINK00000000000"
36+
}
37+
},
38+
{
39+
"sequence": 2,
40+
"event_type": "ActivityCompleted",
41+
"recorded_at": "2026-11-03T00:05:00+00:00",
42+
"payload": {
43+
"activity_execution_id": "01JBREDRIVESUCCESSORACT000",
44+
"activity_type": "tests.redrive.first",
45+
"sequence": 1,
46+
"result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=",
47+
"payload_codec": "avro",
48+
"reused_from_run_id": "01JBREDRIVESOURCERUN000000",
49+
"reused_activity_execution_id": "01JBREDRIVESOURCEACTIVITY0",
50+
"reused_recorded_at": "2026-11-03T00:00:00+00:00"
51+
}
52+
},
53+
{
54+
"sequence": 3,
55+
"event_type": "TimerScheduled",
56+
"recorded_at": "2026-11-03T00:05:00+00:00",
57+
"payload": {
58+
"timer_id": "01JBREDRIVESUCCESSORTIMER0",
59+
"sequence": 2,
60+
"delay_seconds": 5,
61+
"fire_at": "2026-11-03T00:00:05.000000Z"
62+
}
63+
},
64+
{
65+
"sequence": 4,
66+
"event_type": "TimerFired",
67+
"recorded_at": "2026-11-03T00:05:00+00:00",
68+
"payload": {
69+
"timer_id": "01JBREDRIVESUCCESSORTIMER0",
70+
"sequence": 2,
71+
"delay_seconds": 5,
72+
"fire_at": "2026-11-03T00:00:05.000000Z",
73+
"fired_at": "2026-11-03T00:00:06.000000Z"
74+
}
75+
}
76+
],
77+
"expected": {
78+
"command_sequence": [
79+
{
80+
"type": "complete_workflow",
81+
"result": {
82+
"clock": [
83+
"2026-11-03T00:00:00.000000+00:00",
84+
"2026-11-03T00:00:00.000000+00:00",
85+
null
86+
],
87+
"legacy": "2026-11-03T00:00:00.000000+00:00"
88+
}
89+
}
90+
]
91+
}
92+
}

‎tests/redrive_fixture_registry.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,17 @@ class NonQualifying:
5858
}
5959

6060
NON_QUALIFYING: dict[str, NonQualifying] = {
61+
"business-clock-redrive-timer-missing-reuse": NonQualifying(
62+
sha256="641be3fcc1a639335c4f2b2c9158ee5972946140b8f1244caec9a3e50c65c7ba",
63+
error="resume_step_sequence must follow the last reused step",
64+
replacement="redrive-successor-reused-timer",
65+
release="unreleased (#215 business replay clock)",
66+
reason=(
67+
"The timer precedes the declared resume step but lacks both reuse fields. Its legacy value still "
68+
"replays, while replay_now must remain unavailable instead of borrowing successor insertion time. "
69+
"This is malformed-input refusal evidence, not a history the Core writer can produce."
70+
),
71+
),
6172
"redrive-start-replayed-clock": NonQualifying(
6273
sha256="70653171faf5ddba5ed264d0a7ce8bb1af97112dbd6e3bb0e39c6f155fee0fd2",
6374
error="history order",
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
from __future__ import annotations
2+
3+
from typing import Any
4+
5+
import pytest
6+
7+
from durable_workflow.errors import NonDeterministicReplayError
8+
from durable_workflow.workflow import CompleteWorkflow, replay
9+
from tests.business_clock_workflows import PrefixedCoreTimerBusinessClockWorkflow
10+
from tests.test_replay_clock import redrive_fixture
11+
12+
ORIGINAL = "2026-11-03T00:00:06.000000+00:00"
13+
INSERTED = "2026-11-03T00:05:00.000000+00:00"
14+
REUSE_FIELDS = ("reused_from_run_id", "reused_recorded_at")
15+
16+
17+
def fixture(prefixed: bool) -> dict[str, Any]:
18+
document = redrive_fixture("business-clock-core-redrive-acceptance-timer")
19+
if not prefixed:
20+
document["history"].pop(0)
21+
for index, event in enumerate(document["history"], start=1):
22+
event["sequence"] = index
23+
return document
24+
25+
26+
def result(document: dict[str, Any]) -> dict[str, Any]:
27+
task = document["workflow"]
28+
outcome = replay(PrefixedCoreTimerBusinessClockWorkflow, document["history"], task["input"],
29+
workflow_id=task["workflow_id"], run_id=task["run_id"], payload_codec="avro")
30+
try:
31+
assert len(outcome.commands) == 1
32+
command = outcome.commands[0]
33+
assert isinstance(command, CompleteWorkflow), command
34+
return dict(command.result)
35+
finally:
36+
outcome.close()
37+
38+
39+
@pytest.mark.parametrize("prefixed", [False, True])
40+
@pytest.mark.parametrize("missing", [REUSE_FIELDS, REUSE_FIELDS[:1], REUSE_FIELDS[1:]])
41+
def test_prefix_before_resume_requires_both_reuse_fields_without_insertion_time_fallback(
42+
prefixed: bool, missing: tuple[str, ...],
43+
) -> None:
44+
document = fixture(prefixed)
45+
expected = result(document)
46+
assert expected["clock"][-1] == ORIGINAL
47+
for key in missing:
48+
document["history"][-1]["payload"].pop(key)
49+
expected["clock"][-1] = None
50+
assert result(document) == expected
51+
assert result(document) == expected # Cold reconstruction repeats the refusal.
52+
53+
54+
@pytest.mark.parametrize("prefixed", [False, True])
55+
@pytest.mark.parametrize("missing", ["workflow_command_id", "continued_from_run_id", "workflow_link_id",
56+
"workflow_run_id", "resume_step_sequence", "recovery_kind", "replayed_started_at"])
57+
def test_unbound_redrive_header_cannot_turn_an_unmarked_timer_into_an_ordinary_result(
58+
prefixed: bool, missing: str,
59+
) -> None:
60+
document = fixture(prefixed)
61+
started = next(event for event in document["history"] if event["event_type"] == "WorkflowStarted")
62+
started["payload"].pop(missing)
63+
for key in REUSE_FIELDS:
64+
document["history"][-1]["payload"].pop(key)
65+
assert result(document)["clock"] == [None] * 3
66+
67+
68+
@pytest.mark.parametrize("prefixed", [False, True])
69+
def test_bound_post_resume_timer_without_reuse_fields_uses_its_own_consumed_time(prefixed: bool) -> None:
70+
document = fixture(prefixed)
71+
started = next(event for event in document["history"] if event["event_type"] == "WorkflowStarted")
72+
started["payload"]["resume_step_sequence"] = 2
73+
for key in REUSE_FIELDS:
74+
document["history"][-1]["payload"].pop(key)
75+
assert result(document)["clock"][-1] == INSERTED
76+
77+
78+
@pytest.mark.parametrize("prefix_fault", ["outcome", "identity"])
79+
def test_unbound_acceptance_prefix_does_not_lend_timer_insertion_time(prefix_fault: str) -> None:
80+
document = fixture(True)
81+
for key in REUSE_FIELDS:
82+
document["history"][-1]["payload"].pop(key)
83+
if prefix_fault == "outcome":
84+
document["history"][0]["payload"]["outcome"] = "started"
85+
else:
86+
document["history"][0]["payload"]["workflow_run_id"] = "foreign"
87+
assert result(document)["clock"] == [None] * 3
88+
89+
90+
def test_missing_start_retains_the_original_legacy_clock_error() -> None:
91+
document = fixture(True)
92+
document["history"].pop(1)
93+
with pytest.raises(NonDeterministicReplayError, match="missing WorkflowStarted persisted start time"):
94+
result(document)
95+
96+
97+
@pytest.mark.parametrize("prefixed", [False, True])
98+
@pytest.mark.parametrize("alias", ["WorkflowTaskCompleted", "StartAccepted"])
99+
def test_conflicting_start_kind_alias_cannot_hide_an_unbound_redrive_header(prefixed: bool, alias: str) -> None:
100+
document = fixture(prefixed)
101+
started = next(event for event in document["history"] if event["event_type"] == "WorkflowStarted")
102+
started["type"] = alias
103+
for key in REUSE_FIELDS:
104+
document["history"][-1]["payload"].pop(key)
105+
assert result(document) == {"clock": [None] * 3, "legacy": "2026-11-03T00:00:00.000000+00:00"}
106+
107+
108+
def test_pre_resume_activity_with_a_schedule_cannot_use_insertion_time_when_reuse_fields_are_missing() -> None:
109+
document = fixture(False)
110+
terminal = document["history"][1]
111+
for key in (*REUSE_FIELDS, "reused_activity_execution_id"):
112+
terminal["payload"].pop(key)
113+
document["history"].insert(1, {
114+
"event_type": "ActivityScheduled", "recorded_at": terminal["recorded_at"],
115+
"payload": {key: terminal["payload"][key] for key in ("sequence", "activity_type", "activity_execution_id")},
116+
})
117+
for index, event in enumerate(document["history"], start=1):
118+
event["sequence"] = index
119+
assert result(document)["clock"] == ["2026-11-03T00:00:00.000000+00:00", None, ORIGINAL]
120+
121+
122+
@pytest.mark.parametrize("continued", [False, True])
123+
def test_ordinary_and_continued_histories_without_redrive_metadata_keep_their_own_times(continued: bool) -> None:
124+
from tests.test_business_replay_clock import history
125+
from tests.test_business_replay_clock import result as scalar_result
126+
127+
events = history()
128+
if continued:
129+
events[0]["payload"]["continued_from_run_id"] = "previous-run"
130+
assert scalar_result(events)["clock"] == [
131+
None, "2026-11-01T00:00:03.123456+00:00", "2026-11-01T00:00:05.654321+00:00",
132+
]
133+
134+
135+
@pytest.mark.parametrize("marker", ["reused_from_run_id", "reused_recorded_at", "reused_activity_execution_id"])
136+
def test_ordinary_success_with_a_stray_reuse_marker_has_no_proved_source_time(marker: str) -> None:
137+
from tests.test_business_replay_clock import history
138+
from tests.test_business_replay_clock import result as scalar_result
139+
140+
events = history()
141+
events[2]["payload"][marker] = "unbound"
142+
assert scalar_result(events)["clock"][1] is None

0 commit comments

Comments
 (0)