From 3ca37a3de0d0e35122b8ff881e0abb33caae3a20 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:25:02 -0700 Subject: [PATCH 01/16] =?UTF-8?q?refactor:=20extract=20filter=E2=86=92SQL?= =?UTF-8?q?=20WHERE=20builder=20into=20=5Ffilter=5Fsql.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the predicate-only condition builder from _reads._list_jobs into a shared build_filter_conditions() helper so cancel_where can reuse the exact same filter logic. Cursor/limit/order_by stay in _list_jobs and are appended after the shared builder call. All existing test_job_filter.py and test_postgres_reads.py tests verify no regression. --- src/taskq/backend/_filter_sql.py | 91 ++++++++++++++++++++++++++++++++ src/taskq/backend/_reads.py | 50 ++---------------- tests/test_filter_sql.py | 56 ++++++++++++++++++++ 3 files changed, 152 insertions(+), 45 deletions(-) create mode 100644 src/taskq/backend/_filter_sql.py create mode 100644 tests/test_filter_sql.py diff --git a/src/taskq/backend/_filter_sql.py b/src/taskq/backend/_filter_sql.py new file mode 100644 index 00000000..e64661f2 --- /dev/null +++ b/src/taskq/backend/_filter_sql.py @@ -0,0 +1,91 @@ +"""Shared filter→SQL WHERE condition builder. + +Extracted from ``_reads._list_jobs`` so that ``cancel_where`` (bulk +cancel) reuses the exact same filter logic. Only predicate fields +(queue, status, actor, identity_key, batch_id, tags, active) are +translated to conditions. The ``cursor``, ``limit``, and ``order_by`` +fields are NOT handled here — callers apply them separately. +""" + +from dataclasses import dataclass, field + +from taskq._json import dumps_str +from taskq.backend._protocol import JobFilter +from taskq.backend.statemachine import ACTIVE_STATUSES, TERMINAL_STATUSES + +__all__ = ["FilterSQL", "build_filter_conditions"] + + +@dataclass(frozen=True, slots=True) +class FilterSQL: + """Built SQL fragments and parameters from a JobFilter.""" + + conditions: list[str] = field(default_factory=list[str]) + params: list[object] = field(default_factory=list[object]) + + +def build_filter_conditions(filter: JobFilter) -> FilterSQL: + """Build WHERE clause conditions and parameters from a JobFilter. + + Shared between ``_list_jobs`` (reads) and ``cancel_where`` (writes) + so the filter semantics are identical for query and mutation. + + Only predicate fields (queue, status, actor, identity_key, batch_id, + tags, active) are translated to conditions. The ``cursor``, ``limit``, + and ``order_by`` fields are NOT handled here — callers apply them + separately: + + - ``_list_jobs`` appends the cursor keyset condition and LIMIT/OFFSET + after calling this helper (preserving the existing behavior). + - ``cancel_where`` ignores cursor/limit/order_by entirely (bulk writes + are not paginated). + """ + conditions: list[str] = [] + params: list[object] = [] + n = 0 + + def _next_param(expr: str) -> str: + nonlocal n + n += 1 + return f"{expr} = ${n}" + + def _next_any_param(expr: str) -> str: + nonlocal n + n += 1 + return f"{expr} = ANY(${n})" + + if filter.queue is not None: + conditions.append(_next_param("queue")) + params.append(filter.queue) + + if filter.status is not None: + if isinstance(filter.status, str): + conditions.append(_next_param("status")) + params.append(filter.status) + else: + conditions.append(_next_any_param("status")) + params.append(list(filter.status)) + elif filter.active is not None: + statuses = list(ACTIVE_STATUSES) if filter.active else list(TERMINAL_STATUSES) + conditions.append(_next_any_param("status")) + params.append(statuses) + + if filter.actor is not None: + conditions.append(_next_param("actor")) + params.append(filter.actor) + + if filter.identity_key is not None: + conditions.append(_next_param("identity_key")) + params.append(filter.identity_key) + + if filter.batch_id is not None: + n += 1 + conditions.append(f"metadata @> ${n}::jsonb") + params.append(dumps_str({"batch_id": str(filter.batch_id)})) + + if filter.tags is not None and len(filter.tags) > 0: + n += 1 + conditions.append(f"tags && ${n}::text[]") + params.append(list(filter.tags)) + + return FilterSQL(conditions=conditions, params=params) diff --git a/src/taskq/backend/_reads.py b/src/taskq/backend/_reads.py index f613de69..868561dc 100644 --- a/src/taskq/backend/_reads.py +++ b/src/taskq/backend/_reads.py @@ -8,8 +8,8 @@ from datetime import timedelta from typing import TYPE_CHECKING -from taskq._json import dumps_str from taskq.backend._cursor import decode_cursor +from taskq.backend._filter_sql import build_filter_conditions from taskq.backend._protocol import ( AttemptRow, EventRow, @@ -24,7 +24,6 @@ jsonb_to_dict, ) from taskq.backend._sql_templates import SqlTemplates -from taskq.backend.statemachine import ACTIVE_STATUSES, TERMINAL_STATUSES from taskq.constants import RECLAIM_EVENT_VISIBILITY_DELAY if TYPE_CHECKING: @@ -55,49 +54,10 @@ async def _list_jobs( schema: str, filters: JobFilter, ) -> list[JobRow]: - conditions: list[str] = [] - params: list[object] = [] - n = 0 - - def _next_param(expr: str) -> str: - nonlocal n - n += 1 - return f"{expr} = ${n}" - - def _next_any_param(expr: str) -> str: - nonlocal n - n += 1 - return f"{expr} = ANY(${n})" - - if filters.queue is not None: - conditions.append(_next_param("queue")) - params.append(filters.queue) - if filters.status is not None: - if isinstance(filters.status, str): - conditions.append(_next_param("status")) - params.append(filters.status) - else: - conditions.append(_next_any_param("status")) - params.append(list(filters.status)) - elif filters.active is not None: - statuses = list(ACTIVE_STATUSES) if filters.active else list(TERMINAL_STATUSES) - conditions.append(_next_any_param("status")) - params.append(statuses) - if filters.actor is not None: - conditions.append(_next_param("actor")) - params.append(filters.actor) - if filters.identity_key is not None: - conditions.append(_next_param("identity_key")) - params.append(filters.identity_key) - if filters.batch_id is not None: - n += 1 - conditions.append(f"metadata @> ${n}::jsonb") - params.append(dumps_str({"batch_id": str(filters.batch_id)})) - - if filters.tags is not None and len(filters.tags) > 0: - n += 1 - conditions.append(f"tags && ${n}::text[]") - params.append(list(filters.tags)) + filter_sql = build_filter_conditions(filters) + conditions = filter_sql.conditions + params: list[object] = list(filter_sql.params) + n = len(params) if filters.cursor is not None: cursor_priority, cursor_scheduled_at, cursor_id = decode_cursor(filters.cursor) diff --git a/tests/test_filter_sql.py b/tests/test_filter_sql.py new file mode 100644 index 00000000..57a8bba7 --- /dev/null +++ b/tests/test_filter_sql.py @@ -0,0 +1,56 @@ +"""Unit tests for the shared filter→SQL WHERE condition builder. + +Verifies that ``build_filter_conditions`` translates only the predicate +fields of ``JobFilter`` (queue, status, actor, identity_key, batch_id, +tags, active) into SQL fragments and parameters, and that cursor / +order_by are ignored (they are handled by the caller). +""" + +from uuid import uuid4 + +from taskq.backend._filter_sql import build_filter_conditions +from taskq.backend._protocol import JobFilter + + +class TestBuildFilterConditions: + def test_empty_filter_produces_no_conditions(self) -> None: + result = build_filter_conditions(JobFilter()) + assert result.conditions == [] + assert result.params == [] + + def test_queue_filter(self) -> None: + result = build_filter_conditions(JobFilter(queue="default")) + assert len(result.conditions) == 1 + assert "queue" in result.conditions[0] + assert result.params == ["default"] + + def test_tags_filter(self) -> None: + result = build_filter_conditions(JobFilter(tags=("alpha", "beta"))) + assert len(result.conditions) == 1 + assert "tags" in result.conditions[0] + assert result.params == [["alpha", "beta"]] + + def test_batch_id_filter(self) -> None: + bid = uuid4() + result = build_filter_conditions(JobFilter(batch_id=bid)) + assert len(result.conditions) == 1 + assert "metadata" in result.conditions[0] + + def test_active_true_filter(self) -> None: + result = build_filter_conditions(JobFilter(active=True)) + assert len(result.conditions) == 1 + assert "status" in result.conditions[0] + + def test_combined_filters(self) -> None: + result = build_filter_conditions( + JobFilter(queue="e2e", actor="my_actor", tags=("run-123",)), + ) + assert len(result.conditions) == 3 + + def test_cursor_and_order_by_ignored(self) -> None: + """cancel_where doesn't use cursor/order_by — the builder should + not include them in conditions.""" + result = build_filter_conditions( + JobFilter(cursor="some-cursor", order_by=None), + ) + assert result.conditions == [] From 4262eeb859d406287089b502853c2733de92bbee Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:27:15 -0700 Subject: [PATCH 02/16] =?UTF-8?q?refactor:=20extract=20filter=E2=86=92SQL?= =?UTF-8?q?=20WHERE=20builder=20into=20=5Ffilter=5Fsql.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the predicate-only condition builder from _reads._list_jobs into a shared build_filter_conditions() helper so cancel_where can reuse the exact same filter logic. Cursor/limit/order_by stay in _list_jobs and are appended after the shared builder call. FilterSQL uses immutable tuples for conditions/params (2P6 review feedback — mutable lists in a frozen dataclass only prevent reassignment, not in-place mutation). All existing test_job_filter.py tests verify no regression. --- src/taskq/backend/_filter_sql.py | 21 ++++++++++++++++----- src/taskq/backend/_reads.py | 2 +- tests/test_filter_sql.py | 10 +++++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/taskq/backend/_filter_sql.py b/src/taskq/backend/_filter_sql.py index e64661f2..371cef52 100644 --- a/src/taskq/backend/_filter_sql.py +++ b/src/taskq/backend/_filter_sql.py @@ -5,9 +5,15 @@ (queue, status, actor, identity_key, batch_id, tags, active) are translated to conditions. The ``cursor``, ``limit``, and ``order_by`` fields are NOT handled here — callers apply them separately. + +This module is SQL-only; the in-memory backend filters via its own +implementation in ``testing/_reads.py``. Backend equivalence is enforced +by the shared test suite (``test_backend_equivalence.py``), not shared +code — the two filter-matching strategies (SQL WHERE vs Python +predicates) do not share a trivial interface. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from taskq._json import dumps_str from taskq.backend._protocol import JobFilter @@ -18,10 +24,15 @@ @dataclass(frozen=True, slots=True) class FilterSQL: - """Built SQL fragments and parameters from a JobFilter.""" + """Built SQL fragments and parameters from a JobFilter. + + ``conditions`` and ``params`` are stored as tuples so the frozen + contract is meaningful (mutable list fields in a frozen dataclass + only prevent reassignment, not in-place mutation). + """ - conditions: list[str] = field(default_factory=list[str]) - params: list[object] = field(default_factory=list[object]) + conditions: tuple[str, ...] = () + params: tuple[object, ...] = () def build_filter_conditions(filter: JobFilter) -> FilterSQL: @@ -88,4 +99,4 @@ def _next_any_param(expr: str) -> str: conditions.append(f"tags && ${n}::text[]") params.append(list(filter.tags)) - return FilterSQL(conditions=conditions, params=params) + return FilterSQL(conditions=tuple(conditions), params=tuple(params)) diff --git a/src/taskq/backend/_reads.py b/src/taskq/backend/_reads.py index 868561dc..24029fbe 100644 --- a/src/taskq/backend/_reads.py +++ b/src/taskq/backend/_reads.py @@ -55,7 +55,7 @@ async def _list_jobs( filters: JobFilter, ) -> list[JobRow]: filter_sql = build_filter_conditions(filters) - conditions = filter_sql.conditions + conditions: list[str] = list(filter_sql.conditions) params: list[object] = list(filter_sql.params) n = len(params) diff --git a/tests/test_filter_sql.py b/tests/test_filter_sql.py index 57a8bba7..cc8634d1 100644 --- a/tests/test_filter_sql.py +++ b/tests/test_filter_sql.py @@ -15,20 +15,20 @@ class TestBuildFilterConditions: def test_empty_filter_produces_no_conditions(self) -> None: result = build_filter_conditions(JobFilter()) - assert result.conditions == [] - assert result.params == [] + assert result.conditions == () + assert result.params == () def test_queue_filter(self) -> None: result = build_filter_conditions(JobFilter(queue="default")) assert len(result.conditions) == 1 assert "queue" in result.conditions[0] - assert result.params == ["default"] + assert result.params == ("default",) def test_tags_filter(self) -> None: result = build_filter_conditions(JobFilter(tags=("alpha", "beta"))) assert len(result.conditions) == 1 assert "tags" in result.conditions[0] - assert result.params == [["alpha", "beta"]] + assert result.params == (["alpha", "beta"],) def test_batch_id_filter(self) -> None: bid = uuid4() @@ -53,4 +53,4 @@ def test_cursor_and_order_by_ignored(self) -> None: result = build_filter_conditions( JobFilter(cursor="some-cursor", order_by=None), ) - assert result.conditions == [] + assert result.conditions == () From 6d6726798e0c9628355a98ea027ff78bfaf16040 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:28:13 -0700 Subject: [PATCH 03/16] test: add SQL injection safety tests for build_filter_conditions Pin the security property that user-supplied values are always parameterized via positional binding, never interpolated into the SQL condition string. Tests verify: - SQL metacharacter payloads in queue/actor/tags/identity_key appear in params, not in conditions - Conditions only contain known column names, operators, and placeholders --- tests/test_filter_sql.py | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test_filter_sql.py b/tests/test_filter_sql.py index cc8634d1..9afcc33d 100644 --- a/tests/test_filter_sql.py +++ b/tests/test_filter_sql.py @@ -54,3 +54,83 @@ def test_cursor_and_order_by_ignored(self) -> None: JobFilter(cursor="some-cursor", order_by=None), ) assert result.conditions == () + + +class TestSQLInjectionSafety: + """Verify user-supplied values are always parameterized, never + interpolated into the SQL condition string. + + Every condition must use ``$N`` positional binding — no raw user + value may appear in ``conditions``. Column names are hardcoded + literals, not derived from input. + """ + + def test_queue_with_sql_metacharacters_is_parameterized(self) -> None: + """A queue name containing SQL injection payload must end up in + params, not in the condition string.""" + payload = "'; DROP TABLE jobs; --" + result = build_filter_conditions(JobFilter(queue=payload)) + assert result.params == (payload,) + for cond in result.conditions: + assert payload not in cond + assert "$" in cond + + def test_actor_with_sql_metacharacters_is_parameterized(self) -> None: + payload = "admin' OR '1'='1" + result = build_filter_conditions(JobFilter(actor=payload)) + assert result.params == (payload,) + for cond in result.conditions: + assert payload not in cond + + def test_tags_with_sql_metacharacters_are_parameterized(self) -> None: + payload = ("'; DELETE FROM jobs WHERE '1'='1",) + result = build_filter_conditions(JobFilter(tags=payload)) + assert result.params == (list(payload),) + for cond in result.conditions: + assert payload[0] not in cond + + def test_identity_key_with_sql_metacharacters_is_parameterized(self) -> None: + from taskq.backend._protocol import IdentityKey + + payload = IdentityKey("x'; DROP TABLE jobs; --") + result = build_filter_conditions(JobFilter(identity_key=payload)) + assert result.params == (payload,) + for cond in result.conditions: + assert str(payload) not in cond + + def test_conditions_only_contain_placeholders_and_column_names(self) -> None: + """Every condition string must be composed solely of known column + names, operators, and $N placeholders — never raw user input.""" + result = build_filter_conditions( + JobFilter( + queue="myqueue", + actor="myactor", + tags=("tag1", "tag2"), + batch_id=uuid4(), + ) + ) + allowed_substrings = { + "queue", + "status", + "actor", + "identity_key", + "metadata", + "tags", + " = $", + " = ANY($", + " @> $", + " && $", + "::jsonb", + "::text[]", + } + for cond in result.conditions: + stripped = cond + for s in allowed_substrings: + stripped = stripped.replace(s, "") + stripped = stripped.replace("$", "").replace("0", "").replace("1", "") + stripped = stripped.replace("2", "").replace("3", "").replace("4", "") + stripped = stripped.replace("5", "").replace("6", "").replace("7", "") + stripped = stripped.replace("8", "").replace("9", "") + assert stripped == "", ( + f"Unexpected content in condition: {cond!r} (residue: {stripped!r})" + ) From b0c8dca50c217eab7358e6c6652f7b466bf9a66b Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:31:21 -0700 Subject: [PATCH 04/16] feat: add BulkCancelResult type and EmptyFilterError exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BulkCancelResult is defined in _protocol.py (next to ScheduleRecord) to avoid a circular import — types.py already imports from _protocol, so a back-edge would fail at import time. Re-exported through the same chain as CancelResult: _protocol → types → client → __init__. EmptyFilterError is the guardrail exception for cancel_where with no filter predicates. Also fixes stale 'pydantic-free' docstring in types.py (ScheduleRecord already broke that claim). --- docs/specs/2026-07-29-cancel-and-filter.md | 2574 ++++++++++++++++++++ src/taskq/__init__.py | 5 +- src/taskq/backend/__init__.py | 2 + src/taskq/backend/_protocol.py | 30 + src/taskq/client/__init__.py | 3 +- src/taskq/exceptions.py | 18 + src/taskq/types.py | 17 +- tests/test_bulk_cancel_types.py | 52 + 8 files changed, 2692 insertions(+), 9 deletions(-) create mode 100644 docs/specs/2026-07-29-cancel-and-filter.md create mode 100644 tests/test_bulk_cancel_types.py diff --git a/docs/specs/2026-07-29-cancel-and-filter.md b/docs/specs/2026-07-29-cancel-and-filter.md new file mode 100644 index 00000000..8367b68a --- /dev/null +++ b/docs/specs/2026-07-29-cancel-and-filter.md @@ -0,0 +1,2574 @@ +# Spec: Sub-job Tags (#57) & Bulk Cancel by Filter (#54) + +**Date:** 2026-07-29 +**Status:** Draft, revised post-review (2026-07-29) +**Issues:** [#57](https://github.com/rich/taskq/issues/57), [#54](https://github.com/rich/taskq/issues/54) + +--- + +## Goal + +Make sub-jobs enqueued from inside actor bodies visible to tag-based filters by adding `tags` (and other missing fields) to `SubJobEnqueuer.enqueue()`, with parent-tag inheritance by default; and add a set-based `cancel_where(filter)` operation that cancels all jobs matching a `JobFilter` in a single SQL round-trip, with guardrails against accidental full-table cancel. + +## Non-goals + +- Metadata-based filtering (`JobFilter` metadata predicates) — out of scope; tags solve the discovery problem. +- Bulk cancel via the admin web UI — the API method is added here; UI wiring is a follow-up. +- Bulk *retry* or bulk *delete* by filter — same pattern but separate spec. +- Changing the cooperative cancellation state machine phases — `cancel_where` reuses the existing phase-1 cooperative path for running jobs and the existing direct-to-terminal path for pending/scheduled. +- Sub-job `queue` override — sub-jobs use the actor's declared queue (documented design choice, unchanged). +- Widening the tag charset — issues #54/#57 use colon-form tags in prose (`tenant:acme`, `run:{run_id}`), but TaskQ's validator (`_args.py:35`, `^[\w][\w\-]+[\w]$`) accepts word chars and hyphens only, and downstream (cennan) already ships hyphenated tags after a production incident with the colon form. All examples in this spec use the valid hyphenated form; a charset widening would be a separate, separately-motivated change. + +--- + +## Architecture Overview + +### Current state + +``` +JobsClient.enqueue(tags=...) ──► build_enqueue_args(tags=...) ──► EnqueueArgs.tags +EnqueueItem(tags=...) ──► build_batch_args ──► EnqueueArgs.tags +SubJobEnqueuer.enqueue() ──► build_enqueue_args(tags=) ──► EnqueueArgs.tags = () + ^^^^^^^^^^ + Issue #57: tags always empty + +JobsClient.cancel(job_id) ──► Backend.write_cancel_request(job_id, reason) + ├─ pending/scheduled → UPDATE to 'cancelled' (terminal) + └─ running → UPDATE cancel_phase=1 (cooperative) + NOTIFY + ^^^^^^^^^^ + Issue #54: one job at a time only +``` + +### Target state + +``` +SubJobEnqueuer.enqueue(tags=..., inherit_tags=True) + │ + ├─ inherit_tags=True, tags=None → use parent job's tags (from ContextVar) + ├─ inherit_tags=True, tags=[...] → merge parent tags + explicit tags (union, parent first) + └─ inherit_tags=False, tags=None → empty tags (current behavior) + +JobsClient.cancel_where(JobFilter(tags=("tenant-acme",), active=True), reason="offboard") + │ + └─► Backend.cancel_where(filter, reason) + ├─ pending/scheduled rows → UPDATE to 'cancelled' (terminal) + state_change events + └─ running rows → UPDATE cancel_phase=1 (cooperative) + cancel_request events + NOTIFY + └─► BulkCancelResult(cancelled_directly=N, cancel_requested=M, ...) +``` + +### File structure — files to create or modify + +``` +src/taskq/ +├── backend/ +│ ├── _protocol.py MODIFY: add cancel_where to Backend protocol; define BulkCancelResult +│ ├── _reads.py MODIFY: extract filter→SQL WHERE builder for reuse (refactor) +│ ├── _filter_sql.py CREATE: shared filter→SQL WHERE builder (extracted from _reads) +│ ├── _cancel_bulk.py CREATE: bulk cancel implementation for PostgresBackend +│ └── postgres.py MODIFY: wire cancel_where to _cancel_bulk +├── client/ +│ ├── __init__.py MODIFY: re-export BulkCancelResult alongside CancelResult +│ ├── _jobs.py MODIFY: add cancel_where method to JobsClient +│ ├── _enqueuer.py MODIFY: add tags, inherit_tags, schedule_to_close, start_to_close, heartbeat_timeout to enqueue() +│ ├── _taskq.py MODIFY: add cancel_where delegate to TaskQ +│ └── _args.py MODIFY: (no change needed — build_enqueue_args already accepts tags) +├── worker/ +│ ├── _consumer.py MODIFY: set parent tags ContextVar before actor invocation (gated by setting) +│ └── run.py MODIFY: set parent tags in stub consumer (unconditional — test harness) +├── settings.py MODIFY: add sub_job_inherit_tags field to WorkerSettings (fleet kill switch) +├── testing/ +│ ├── in_memory.py MODIFY: add cancel_where to InMemoryBackend +│ └── _cancel_bulk.py CREATE: in-memory bulk cancel implementation +├── types.py MODIFY: re-export BulkCancelResult from _protocol +├── exceptions.py MODIFY: add EmptyFilterError (guardrail) +└── __init__.py MODIFY: export BulkCancelResult, EmptyFilterError + +tests/ +├── test_sub_job_tags.py CREATE: unit tests for sub-job tags + inheritance +├── test_cancel_where.py CREATE: unit tests for cancel_where (in-memory) +├── test_cancel_where_pg.py CREATE: integration tests for cancel_where (postgres) +├── test_cancel_where_client.py CREATE: client-level cancel_where tests (guardrail, counter, schema errors) +├── test_filter_sql.py CREATE: filter→SQL builder extraction tests +├── test_bulk_cancel_types.py CREATE: BulkCancelResult, EmptyFilterError type tests +├── test_sub_job_enqueuer.py MODIFY: add tags parameter tests + backward compat +├── test_backend_protocol.py MODIFY: add cancel_where protocol conformance test; update member count +└── e2e/ + ├── actors.py MODIFY: add tagged pipeline actors + ├── test_sub_job_tags.py CREATE: e2e tests for sub-job tags in a real pipeline + └── test_cancel_where.py CREATE: e2e tests for bulk cancel + +docs/ +├── guides/jobs-clients.md MODIFY: document cancel_where and sub-job tags +└── architecture.md MODIFY: document bulk cancel in cancel protocol section +``` + +> **Note:** `context.py` is NOT modified — parent tags are propagated via a `contextvars.ContextVar` defined in `_enqueuer.py`, not via `JobContext`. The SQL is inlined in `_cancel_bulk.py` (matching the dynamic-SQL precedent in `_reads.py`); no `SqlTemplates.cancel_where` field or `_sql.py` change is needed. + +--- + +## API Surface + +### Issue #57: SubJobEnqueuer.enqueue() — tags and missing fields + +#### Modified signature + +```python +# src/taskq/client/_enqueuer.py + +class SubJobEnqueuer: + async def enqueue[P: BaseModel, R: BaseModel | None]( + self, + actor_ref: ActorRef[P, R], + payload: P, + *, + connection: asyncpg.Connection | None = None, + scheduled_at: datetime | None = None, + priority: int | None = None, + fairness_key: str | None = None, + metadata: dict[str, object] | None = None, + identity_key: IdentityKey | None = None, + idempotency_key: IdempotencyKey | str | None = None, + idempotency_scope: str | None = None, + unique_for: timedelta | None = None, + unique_states: tuple[JobStatus, ...] | None = None, + max_pending: int | None = None, + # ── NEW parameters ────────────────────────────────── + tags: list[str] | None = None, + inherit_tags: bool = True, + schedule_to_close: datetime | None = None, + start_to_close: timedelta | None = None, + heartbeat_timeout: timedelta | None = None, + # ── END new parameters ────────────────────────────── + ) -> JobHandle[R]: ... +``` + +#### Tag inheritance semantics + +| `inherit_tags` | `tags` | Resulting job tags | +|---|---|---| +| `True` (default) | `None` | Parent job's tags (or `()` if parent has none) | +| `True` (default) | `["new-tag"]` | Parent tags + explicit tags, merged (union, parent-first order, deduped) | +| `False` | `None` | `()` (current behavior) | +| `False` | `["new-tag"]` | `("new-tag",)` (explicit only, no inheritance) | + +#### Parent tag propagation via `contextvars.ContextVar` + +The `SubJobEnqueuer` is shared across concurrent consumers in the same event loop. A per-instance field would be racy. Instead, use a `contextvars.ContextVar` that the consumer sets before each actor invocation — asyncio Tasks copy the context, so concurrent consumers each see their own value. + +```python +# src/taskq/client/_enqueuer.py + +import contextvars + +_parent_tags_var: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar( + "taskq_parent_tags", + default=(), +) + +def set_parent_tags(tags: tuple[str, ...]) -> contextvars.Token[tuple[str, ...]]: + """Set the parent job's tags for sub-job tag inheritance. + + Called by the consumer before actor invocation. The returned token + can be used to reset the context after the actor completes. + """ + return _parent_tags_var.set(tags) +``` + +Inside `enqueue()`: + +```python +async def enqueue[P: BaseModel, R: BaseModel | None](self, ...) -> JobHandle[R]: + # Resolve tags with inheritance + resolved_tags = self._resolve_tags(tags, inherit_tags) + args = build_enqueue_args( + actor_ref, + payload, + # ... existing params ... + tags=resolved_tags, # NEW + schedule_to_close=schedule_to_close, # NEW + start_to_close=start_to_close, # NEW + heartbeat_timeout=heartbeat_timeout, # NEW + clock=self._clock, + ) + # ... rest unchanged ... + +def _resolve_tags( + self, + tags: list[str] | None, + inherit_tags: bool, +) -> list[str] | None: + """Resolve tags with parent inheritance. + + Returns a list suitable for build_enqueue_args, or None for empty. + """ + parent_tags = _parent_tags_var.get() if inherit_tags else () + + if tags is None: + if parent_tags: + return list(parent_tags) + return None + + if not inherit_tags or not parent_tags: + return tags + + # Merge: parent tags first, then explicit tags, deduped + seen: set[str] = set(parent_tags) + merged = list(parent_tags) + for tag in tags: + if tag not in seen: + seen.add(tag) + merged.append(tag) + return merged +``` + +#### Consumer integration + +```python +# src/taskq/worker/_consumer.py + +from taskq.client._enqueuer import _parent_tags_var, set_parent_tags + +# Inside consume_job(), before constructing JobContext. consume_job is a +# module-level function; the setting is read from _effective_settings +# (WorkerSettings | None, resolved from deps.settings or the settings +# parameter at _consumer.py:347). Settings absent (tests) → inherit. +_inherit = ( + _effective_settings is None or _effective_settings.sub_job_inherit_tags +) +token = set_parent_tags(tuple(job.tags)) if _inherit else None +try: + ctx: JobContext[BaseModel] = JobContext( + # ... existing fields ... + jobs=live_enqueuer, + # ... + ) + # ... run actor ... +finally: + if token is not None: + _parent_tags_var.reset(token) +``` + +The setting itself is added to `WorkerSettings` in `src/taskq/settings.py:338` (NOT `worker/_bootstrap.py` — that module has no settings class): + +```python +# src/taskq/settings.py — in WorkerSettings + +sub_job_inherit_tags: bool = Field( + default=True, + description=( + "When false, sub-jobs enqueued via ctx.jobs.enqueue() do not inherit " + "the parent job's tags (pre-1.0 behavior). Fleet-level kill switch " + "for the inherit_tags=True default; env var TASKQ_SUB_JOB_INHERIT_TAGS." + ), +) +``` + +Because `TaskQSettings` is a pydantic-settings class with `env_prefix = "TASKQ_"`, the field is automatically settable via the `TASKQ_SUB_JOB_INHERIT_TAGS` environment variable — no extra wiring for operators. + +The stub consumer in `worker/run.py` (a test harness with no settings object) follows the same set/reset pattern but calls `set_parent_tags(tuple(job.tags))` unconditionally. + +### Issue #54: Bulk cancel by filter + +#### New type: `BulkCancelResult` + +`BulkCancelResult` is defined in `taskq.backend._protocol` (next to `ScheduleRecord`, which is already a Pydantic `BaseModel` in that module) and re-exported through the same chain as `CancelResult`: `taskq.types` → `taskq.client` → `taskq`. This avoids the circular import that would arise from defining it in `types.py`: `types.py:18` already imports `from taskq.backend._protocol import JobId, JobStatus`, so a back-edge `_protocol → types` would fail at import time before `JobId` is defined. The `types.py` docstring claim that the protocol stays "pydantic-free" is already stale (`ScheduleRecord` at `_protocol.py:594` is a Pydantic model) — the implementation updates that docstring as part of Task 2. + +```python +# src/taskq/backend/_protocol.py — define next to ScheduleRecord + +class BulkCancelResult(BaseModel): + """Structured outcome of a bulk cancellation request. + + Returned by ``JobsClient.cancel_where()`` so callers can inspect + how many jobs were cancelled directly (pending/scheduled → terminal + cancelled) vs how many had cooperative cancel requested (running → + cancel_phase=1). + """ + + model_config = ConfigDict(frozen=True) + + cancelled_directly: int + """Count of pending/scheduled jobs moved straight to terminal 'cancelled'.""" + + cancel_requested: int + """Count of running jobs with cancel_phase=1 set (cooperative cancel).""" + + cancelled_ids: list[UUID] + """IDs of jobs cancelled directly (pending/scheduled → cancelled).""" + + cancel_requested_ids: list[UUID] + """IDs of running jobs with cancel requested.""" + + @property + def total_affected(self) -> int: + """Total jobs affected by the bulk cancel.""" + return self.cancelled_directly + self.cancel_requested +``` + +```python +# src/taskq/types.py — re-export (add to import and __all__) + +from taskq.backend._protocol import BulkCancelResult # noqa: F401 — re-export +__all__ = ["BulkCancelResult", "CancelResult", "StateChangeEvent"] + +# src/taskq/client/__init__.py — re-export alongside CancelResult + +from taskq.types import BulkCancelResult, CancelResult +__all__ = ["BulkCancelResult", "CancelResult", "JobEvent", "JobHandle", "JobsClient", "SubJobEnqueuer", "TaskQ"] + +# src/taskq/__init__.py — re-export at top level via the client surface +# (same import line pattern as CancelResult at __init__.py:40) +from taskq.client import BulkCancelResult, CancelResult, JobEvent, JobHandle, JobsClient, TaskQ +``` + +#### New exception: `EmptyFilterError` + +```python +# src/taskq/exceptions.py + +class EmptyFilterError(TaskQError): + """Raised when cancel_where is called with a filter that has no predicates. + + A filter with no queue, status, actor, identity_key, batch_id, tags, or + active predicate would match every job in the table — almost certainly + a bug. The guardrail is intentionally loud: the caller must add at least + one predicate or explicitly bypass with ``allow_empty_filter=True``. + """ + + def __init__(self) -> None: + super().__init__( + "cancel_where requires at least one filter predicate " + "(queue, status, actor, identity_key, batch_id, tags, or active); " + "an empty filter would cancel the entire table. " + "Pass allow_empty_filter=True to override this guardrail." + ) +``` + +#### Client API: `JobsClient.cancel_where()` + +```python +# src/taskq/client/_jobs.py + +class JobsClient: + async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter* in a single set-based operation. + + Pending/scheduled jobs are moved straight to terminal 'cancelled' + (no running actor to cooperate with). Running jobs get + ``cancel_phase=1`` set (cooperative cancel) — the worker's + heartbeat-driven cancel controller observes the phase change and + sets the in-process ``cancel_event``. + + **Guardrail:** a filter with no predicates (no queue, status, + actor, identity_key, batch_id, tags, or active) is rejected with + :class:`EmptyFilterError` unless ``allow_empty_filter=True`` is + passed. This prevents accidental full-table cancels. + + **Filter fields used:** ``queue``, ``status``, ``actor``, + ``identity_key``, ``batch_id``, ``tags``, ``active``. The + ``limit``, ``cursor``, and ``order_by`` fields are ignored — a + bulk cancel is not paginated. + + **Snapshot boundary:** jobs matching *filter* that are enqueued + *after* the statement's snapshot escape this call. Stop producers + first, or issue a follow-up call; the returned counts make + non-convergence detectable. + + Returns a :class:`BulkCancelResult` with counts and affected IDs + for observability. + + Increments ``taskq.cancellation.requested`` once per call + (regardless of the number of jobs affected). + """ + ... +``` + +#### `TaskQ` delegate + +```python +# src/taskq/client/_taskq.py + +class TaskQ: + async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter*. See :meth:`JobsClient.cancel_where`.""" + return await self._require_open().cancel_where( + filter, reason, allow_empty_filter=allow_empty_filter + ) +``` + +#### Backend protocol addition + +`BulkCancelResult` is already defined in `_protocol.py` (see above), so the protocol method's return annotation has no import dependency issue. + +```python +# src/taskq/backend/_protocol.py + +class Backend(Protocol): + # ── Cancel signals ────────────────────────────────────────── + async def write_cancel_request( + self, + job_id: JobId, + reason: str | None, + ) -> bool: ... + + async def cancel_where( + self, + filter: JobFilter, + reason: str | None, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter* in a set-based operation. + + Pending/scheduled jobs → terminal 'cancelled'. + Running jobs → cancel_phase=1 (cooperative cancel + NOTIFY). + + The filter's ``limit``, ``cursor``, and ``order_by`` fields are + ignored — this is a bulk write, not a paginated read. + + Returns a :class:`BulkCancelResult` with counts and affected IDs. + """ + ... +``` + +**Protocol version:** No bump required. `cancel_where` is purely additive — a v3 implementation lacks the method, and the client raises `AttributeError` loudly (not a silent misbehavior). See the bump rule in `_protocol.py` lines 79-84. + +#### PostgresBackend SQL + +The SQL uses two CTEs in a single statement: one for pending/scheduled (→ terminal cancelled), one for running (→ cooperative cancel_phase=1). Events are inserted within the same transaction via `executemany`. + +**EPQ safety (Critical design requirement):** Under READ COMMITTED, when an UPDATE reaches a row that a concurrent transaction has locked/updated, Postgres waits, then re-evaluates the UPDATE's **own WHERE clause** (EvalPlanQual) against the new row version. Predicates inside a materialized CTE are **not** re-evaluated. Therefore, the status/cancel_phase predicates must appear **both** in the `matching` CTE (for snapshot-time row selection) **and** in each UPDATE's own WHERE clause (for EPQ re-evaluation). This mirrors the existing single-job path: `cancel_pending_scheduled` (`_sql_templates.py:407-415`) locks `FOR UPDATE` and repeats `AND status IN ('pending','scheduled')` in the UPDATE's WHERE; `cancel_running` (`_sql_templates.py:416-420`) puts `AND status = 'running' AND cancel_phase = 0` directly in the UPDATE. + +**Residual window:** A job claimed by a worker (pending→running) between the statement snapshot and the row lock will be skipped by this call (the EPQ re-check sees `running` and rejects it from the pending/scheduled UPDATE). This is correct and safe — the job escapes this cancel call and requires a subsequent `cancel_where` (or the caller stops producers first). This is strictly preferable to overwriting a running job to terminal `cancelled` while a worker executes it. + +**Lock ordering / deadlock handling:** The `matching` CTE scans with `ORDER BY id` so the UPDATEs acquire row locks in ascending `id` order, reducing deadlock probability against dispatch (`FOR UPDATE SKIP LOCKED`, `_dispatch_sql.py:125`) and heartbeat lock ordering. Eliminating deadlocks outright is not claimed — when Postgres detects one it aborts this statement with `asyncpg.DeadlockDetectedError`, and the **backend** retries the whole transaction (max 3 attempts, jittered backoff; safe because the single transaction rolls back atomically and the EPQ predicates re-filter on every attempt). The retry is backend-owned, not client-owned: the exception type is asyncpg-specific and the backend owns the transaction boundary. + +**Large result sets:** A single `cancel_where` call is bounded by transaction size. For tenant-scale cancels (10⁵+ matching rows), the operator should partition via filter (e.g., `JobFilter(queue=..., tags=...)` to split by queue). The implementation does not chunk internally — a single transaction covering 10⁶ rows would hold locks too long. The `BulkCancelResult` counts let the caller verify completeness and issue follow-up calls for remaining partitions. Document this guidance in `jobs-clients.md`. + +**Event parity:** Both backends insert the same event kinds as the existing single-job `write_cancel_request` path: for pending/scheduled jobs, both `state_change` (with actual `from_state`) and `cancel_request`; for running jobs, only `cancel_request`. This matches `postgres.py:555-561` and `in_memory.py:595-601`. + +**Post-snapshot enqueue boundary:** Jobs matching the filter that are enqueued *after* the statement's snapshot escape the cancel. Convergence is the caller's responsibility — stop producers before calling `cancel_where`, or issue a second call to catch stragglers. The `BulkCancelResult` counts let the caller detect non-convergence. + +```sql +-- src/taskq/backend/_cancel_bulk.py — cancel_where SQL (inlined, dynamic) + +-- $1..$N: filter parameters (same positional binding as list_jobs). +-- The reason is never interpolated into SQL or JSON text — it is bound +-- per-row as a jsonb parameter at event-insert time (see below). + +WITH matching AS ( + SELECT id, status, locked_by_worker + FROM "{schema}".jobs + WHERE {filter_conditions} + ORDER BY id -- deterministic lock ordering to reduce deadlocks +), +cancelled AS ( + UPDATE "{schema}".jobs AS j + SET status = 'cancelled', + finished_at = clock_timestamp() + FROM ( + SELECT id, status AS prev_status + FROM matching + WHERE status IN ('pending', 'scheduled') + ) AS prev + WHERE j.id = prev.id + AND j.status IN ('pending', 'scheduled') -- EPQ re-check (Critical) + RETURNING j.id, prev.prev_status +), +cancel_requested AS ( + UPDATE "{schema}".jobs AS j + SET cancel_requested_at = now(), + cancel_phase = 1 + WHERE j.id IN ( + SELECT id FROM matching + WHERE status = 'running' AND cancel_phase = 0 + ) + AND j.status = 'running' AND j.cancel_phase = 0 -- EPQ re-check (Critical) + RETURNING j.id, j.locked_by_worker +) +SELECT + (SELECT count(*)::int FROM cancelled) AS cancelled_directly, + (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, + (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, + (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, + (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, + (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers +``` + +Three points of correctness in this shape: + +- **`FROM prev` captures the previous status** in the same round-trip (mirroring the single-job `cancel_pending_scheduled` template at `_sql_templates.py:407-415`). `UPDATE ... RETURNING` alone can only return the *new* row; the `prev` subquery carries the snapshot status through so `state_change` events record the actual `from_state` (`'pending'` or `'scheduled'`), not a synthetic placeholder. The target table is aliased (`AS j`) so `RETURNING j.id` is unambiguous against `prev.id`. +- **The EPQ-re-checked predicates are the ones on the target table** (`j.status ...`, `j.cancel_phase ...`). Under READ COMMITTED, when the UPDATE blocks on a concurrently-locked row, Postgres re-evaluates the UPDATE's own WHERE clause against the newest row version; `prev.*` values come from the statement snapshot and are not re-evaluated — which is exactly why the status predicates must live on `j`, not only inside `matching`/`prev`. If a job was claimed (→ `running`) or finished (→ terminal) mid-statement, the re-check rejects it and the row is skipped. +- **Aggregate arrays are `ORDER BY id`-aligned**, so Python can zip `cancelled_ids` with `cancelled_prev_statuses` (per-job `from_state`) and `cancel_requested_ids` with `cancel_requested_workers` (NOTIFY targets) — no second query. + +Events are inserted in separate statements within the same transaction (after the main CTE query returns counts/IDs), reusing the existing `sql.insert_event` template with `executemany`; the `detail` JSON is serialized in Python via `jsonb_param` (never f-string interpolation — see the H2 design note in Task 5). + +**NOTIFY for running jobs:** After the transaction commits, `PostgresBackend.cancel_where` sends `pg_notify` to the fleet channel and each affected job's per-worker channel, reusing the channel helpers and payload shape from `write_cancel_request` (`events_channel`/`worker_channel` in `constants.py:221,236`; pattern at `postgres.py:585-603`). For bulk cancel, the NOTIFY calls are batched into a single statement to avoid N round-trips: + +```sql +SELECT pg_notify(channel, payload) +FROM unnest($1::text[], $2::text[]) AS t(channel, payload) +``` + +The send lives in `postgres.py` (not `_cancel_bulk.py`) because the `taskq.cancel.notify_sent` counter is module-level there (`postgres.py:146-149,603`) — importing it from `_cancel_bulk` would create a module cycle. The counter is incremented once per job notified (batch `.add(len(notify_targets))`), matching the single-job path's per-job semantics. + +**Event insertion:** Events are inserted via `executemany` within the same transaction: +- For cancelled (pending/scheduled) jobs: one `state_change` event with `from_state` set to the actual previous status (from `cancelled_prev_statuses`) and one `cancel_request` event — matching the single-job path (`postgres.py:555-561`). Details: `jsonb_param({"from_state": prev_status, "to_state": "cancelled"})` and `jsonb_param({"reason": reason} if reason is not None else {})`. +- For cancel_requested (running) jobs: one `cancel_request` event with `jsonb_param({"reason": reason} if reason is not None else {})`. + +#### In-memory backend implementation + +The in-memory backend must **not** call `_list_jobs` directly with the caller's filter, because `_list_jobs` applies `filters.limit` (default 100) and cursor slicing (`testing/_reads.py:87-98`) — silently capping the cancel to 100 rows and contradicting the contract that `limit`, `cursor`, and `order_by` are ignored. Instead, call `_list_jobs` with a **sanitized filter** that unsets `limit`, `cursor`, and `order_by`: + +```python +# src/taskq/testing/_cancel_bulk.py + +from dataclasses import replace as dc_replace +from typing import TYPE_CHECKING +from uuid import UUID + +from taskq.backend._protocol import BulkCancelResult, CancelPhase, JobFilter +from taskq.testing._reads import _list_jobs + +if TYPE_CHECKING: + from taskq.testing.in_memory import InMemoryBackend + +async def _cancel_where( + self: "InMemoryBackend", # module-fn style, like testing/_reads.py + filter: JobFilter, + reason: str | None, +) -> BulkCancelResult: + # Sanitize the filter: cancel_where ignores limit, cursor, and order_by. + # Use a very large limit (2**31) instead of None because JobFilter.limit + # is typed as int (not int | None) with a __post_init__ guard against + # negatives. cursor=None disables keyset slicing. order_by=None selects + # the default priority/scheduled_at/id sort, which is harmless for cancel. + sanitized = dc_replace(filter, limit=2**31, cursor=None, order_by=None) + rows = await _list_jobs(self, sanitized) + + cancelled_ids: list[UUID] = [] + cancel_requested_ids: list[UUID] = [] + + for row in rows: + if row.status in ("pending", "scheduled"): + now = self._clock.now() + self._jobs[row.id] = dc_replace( + row, + status="cancelled", + finished_at=now, + ) + self._append_state_change_event( + job_id=row.id, + from_state=row.status, # actual previous status + to_state="cancelled", + now=now, + ) + self._append_cancel_request_event(row.id, now, reason) + cancelled_ids.append(row.id) + + elif row.status == "running" and row.cancel_phase == CancelPhase.NONE: + now = self._clock.now() + self._jobs[row.id] = dc_replace( + row, + cancel_requested_at=now, + cancel_phase=CancelPhase.COOPERATIVE, + ) + self._append_cancel_request_event(row.id, now, reason) + for event in self._cancel_wake_subscribers: + event.set() + cancel_requested_ids.append(row.id) + + return BulkCancelResult( + cancelled_directly=len(cancelled_ids), + cancel_requested=len(cancel_requested_ids), + cancelled_ids=cancelled_ids, + cancel_requested_ids=cancel_requested_ids, + ) +``` + +**Event parity:** The in-memory implementation inserts the same event kinds as the existing single-job `write_cancel_request` path and the Postgres bulk path: for pending/scheduled jobs, both `state_change` (with actual `from_state`) and `cancel_request`; for running jobs, only `cancel_request`. This matches `in_memory.py:595-601`. + +#### Filter→WHERE reuse + +The filter-to-SQL-WHERE builder in `_reads._list_jobs` (lines 53-141) builds conditions dynamically. For `cancel_where`, we need the same WHERE clause. Extract the **predicate-only** condition builder (queue, status, actor, identity_key, batch_id, tags, active) into a shared helper. The `schema` parameter is **not** needed — the existing builder in `_reads.py:58-115` produces schema-less fragments (the schema is applied by the caller in the surrounding SQL string). + +```python +# src/taskq/backend/_filter_sql.py (NEW) + +@dataclass(frozen=True, slots=True) +class FilterSQL: + """Built SQL fragments and parameters from a JobFilter.""" + conditions: list[str] + params: list[object] + +def build_filter_conditions(filter: JobFilter) -> FilterSQL: + """Build WHERE clause conditions and parameters from a JobFilter. + + Shared between _list_jobs (reads) and cancel_where (writes) so the + filter semantics are identical for query and mutation. + + Only predicate fields (queue, status, actor, identity_key, batch_id, + tags, active) are translated to conditions. The ``cursor``, ``limit``, + and ``order_by`` fields are NOT handled here — callers apply them + separately: + + - ``_list_jobs`` appends the cursor keyset condition and LIMIT/OFFSET + after calling this helper (preserving the existing behavior at + ``_reads.py:102-115``). + - ``cancel_where`` ignores cursor/limit/order_by entirely (bulk writes + are not paginated). + """ + # ... extracted from _reads._list_jobs lines 58-100 (the predicate + # fields). The cursor keyset block (lines 102-115) and LIMIT/ORDER BY + # (lines 117-138) stay in _list_jobs and are appended after this call. +``` + +Both `_list_jobs` and `_cancel_bulk` call this helper, ensuring filter semantics are DRY. After extraction, `_list_jobs` regains cursor/limit/order_by handling by appending the keyset condition (`_reads.py:102-115`) and `LIMIT $N` after the shared `build_filter_conditions` call — the existing `test_job_filter.py` and `test_postgres_reads.py` suites verify no regression. + +--- + +## Implementation Plan + +### Task 1: Extract filter→SQL builder (refactor) + +**Goal:** Extract the WHERE-clause builder from `_reads._list_jobs` into a shared module so `cancel_where` reuses the exact same filter logic. + +**Files:** +- CREATE: `src/taskq/backend/_filter_sql.py` +- MODIFY: `src/taskq/backend/_reads.py` — import and use the shared builder +- CREATE: `tests/test_filter_sql.py` + +#### TDD — Red + +```python +# tests/test_filter_sql.py + +from taskq.backend._filter_sql import build_filter_conditions, FilterSQL +from taskq.backend._protocol import JobFilter + +class TestBuildFilterConditions: + def test_empty_filter_produces_no_conditions(self) -> None: + result = build_filter_conditions(JobFilter()) + assert result.conditions == [] + assert result.params == [] + + def test_queue_filter(self) -> None: + result = build_filter_conditions(JobFilter(queue="default")) + assert len(result.conditions) == 1 + assert "queue" in result.conditions[0] + assert result.params == ["default"] + + def test_tags_filter(self) -> None: + result = build_filter_conditions(JobFilter(tags=("alpha", "beta"))) + assert len(result.conditions) == 1 + assert "tags" in result.conditions[0] + assert result.params == [["alpha", "beta"]] + + def test_batch_id_filter(self) -> None: + from uuid import uuid4 + bid = uuid4() + result = build_filter_conditions(JobFilter(batch_id=bid)) + assert len(result.conditions) == 1 + assert "metadata" in result.conditions[0] + + def test_active_true_filter(self) -> None: + result = build_filter_conditions(JobFilter(active=True)) + assert len(result.conditions) == 1 + assert "status" in result.conditions[0] + + def test_combined_filters(self) -> None: + result = build_filter_conditions( + JobFilter(queue="e2e", actor="my_actor", tags=("run-123",)), + ) + assert len(result.conditions) == 3 + + def test_cursor_and_order_by_ignored(self) -> None: + """cancel_where doesn't use cursor/order_by — the builder should + not include them in conditions.""" + result = build_filter_conditions( + JobFilter(cursor="some-cursor", order_by=None), + ) + # cursor/order_by are not part of filter conditions + assert result.conditions == [] +``` + +#### TDD — Green + +Extract the condition-building logic from `_reads._list_jobs` into `build_filter_conditions()`. Update `_list_jobs` to call it. Run existing `test_job_filter.py` and `test_postgres_reads.py` to verify no regression. + +#### Acceptance criteria +- All existing `test_job_filter.py` tests pass +- All existing `test_postgres_reads.py` tests pass +- New `test_filter_sql.py` tests pass +- `build_filter_conditions` is pure (no I/O, no global state) + +--- + +### Task 2: Add `BulkCancelResult` type and `EmptyFilterError` exception + +**Goal:** Define the result type and guardrail exception before implementing the operation. + +**Files:** +- MODIFY: `src/taskq/backend/_protocol.py` — define `BulkCancelResult` (next to `ScheduleRecord`) +- MODIFY: `src/taskq/types.py` — re-export `BulkCancelResult`; reconcile the stale "pydantic-free" docstring +- MODIFY: `src/taskq/client/__init__.py` — re-export `BulkCancelResult` alongside `CancelResult` +- MODIFY: `src/taskq/exceptions.py` — add `EmptyFilterError` +- MODIFY: `src/taskq/__init__.py` — export both +- CREATE: `tests/test_bulk_cancel_types.py` + +> **Why `_protocol.py`, not `types.py`:** `types.py:18` imports `from taskq.backend._protocol import JobId, JobStatus`. If `_protocol.py` imported `BulkCancelResult` from `types.py`, the cycle `_protocol → types → _protocol` would fail at import time before `JobId` (line 198) is defined. Defining `BulkCancelResult` in `_protocol.py` (where `ScheduleRecord`, another Pydantic `BaseModel`, already lives at line 594) avoids the cycle. The `types.py` docstring claim that the protocol is "pydantic-free" is already stale due to `ScheduleRecord` and should be updated. + +#### TDD — Red + +```python +# tests/test_bulk_cancel_types.py + +from uuid import uuid4 +import pytest +from taskq.types import BulkCancelResult +from taskq.exceptions import EmptyFilterError + +class TestBulkCancelResult: + def test_construction(self) -> None: + ids = [uuid4() for _ in range(3)] + result = BulkCancelResult( + cancelled_directly=2, + cancel_requested=1, + cancelled_ids=ids[:2], + cancel_requested_ids=ids[2:], + ) + assert result.cancelled_directly == 2 + assert result.cancel_requested == 1 + assert result.total_affected == 3 + assert len(result.cancelled_ids) == 2 + assert len(result.cancel_requested_ids) == 1 + + def test_frozen(self) -> None: + result = BulkCancelResult( + cancelled_directly=0, + cancel_requested=0, + cancelled_ids=[], + cancel_requested_ids=[], + ) + with pytest.raises(Exception): + result.cancelled_directly = 1 # type: ignore[misc] + + def test_zero_counts(self) -> None: + result = BulkCancelResult( + cancelled_directly=0, + cancel_requested=0, + cancelled_ids=[], + cancel_requested_ids=[], + ) + assert result.total_affected == 0 + +class TestEmptyFilterError: + def test_is_taskq_error(self) -> None: + from taskq.exceptions import TaskQError + assert issubclass(EmptyFilterError, TaskQError) + + def test_message_mentions_guardrail(self) -> None: + err = EmptyFilterError() + assert "allow_empty_filter" in str(err) + assert "filter predicate" in str(err) +``` + +#### TDD — Green + +Add the types. Run the tests. + +#### Acceptance criteria +- `BulkCancelResult` is a frozen Pydantic model with `total_affected` property +- `EmptyFilterError` is a `TaskQError` subclass with a helpful message +- Both are exported from `taskq` top-level + +--- + +### Task 3: Add `cancel_where` to `Backend` protocol + +**Goal:** Add the method signature to the `Backend` protocol. + +**Files:** +- MODIFY: `src/taskq/backend/_protocol.py` — add `cancel_where` method; update docstring count +- MODIFY: `tests/test_backend_protocol.py` — update member count and expected member set + +#### Implementation + +```python +# In Backend protocol, after write_cancel_request: +async def cancel_where( + self, + filter: JobFilter, + reason: str | None, +) -> BulkCancelResult: + """Cancel all jobs matching *filter* in a set-based operation.""" + ... +``` + +#### Protocol docstring update + +The `Backend` class docstring (`_protocol.py:693-697`) currently says "31 async methods plus two sync methods (33 methods total)". Adding `cancel_where` (async) makes it **32 async methods plus two sync methods (34 methods total)**. Update the docstring accordingly. + +#### Test updates + +`tests/test_backend_protocol.py:218-262` asserts exactly 36 public members and an exact member-name set. Adding `cancel_where` brings the count to **37**. Update: +- `test_exactly_thirty_six_public_members` → `test_exactly_thirty_seven_public_members` with `assert len(public) == 37` +- Add `"cancel_where"` to the `expected` set in `test_all_member_names_present` + +#### TDD — Red + +```python +# tests/test_backend_protocol.py — add to existing test file + +async def test_protocol_has_cancel_where() -> None: + """Backend protocol declares cancel_where.""" + from taskq.backend._protocol import Backend + assert hasattr(Backend, "cancel_where") +``` + +#### Acceptance criteria +- `Backend` protocol includes `cancel_where` method +- Protocol version not bumped (purely additive, loud failure on missing method) +- `test_backend_protocol.py` updated: member count is 37, `cancel_where` in expected set, docstring count updated to 34 methods total +- All `test_backend_protocol.py` tests pass after update + +--- + +### Task 4: Implement `cancel_where` for InMemoryBackend + +**Goal:** Add bulk cancel to the in-memory backend for unit testing. + +**Files:** +- CREATE: `src/taskq/testing/_cancel_bulk.py` +- MODIFY: `src/taskq/testing/in_memory.py` — wire `cancel_where` method +- CREATE: `tests/test_cancel_where.py` + +#### TDD — Red + +```python +# tests/test_cancel_where.py + +import pytest +from uuid import uuid4 +from taskq.backend._protocol import JobFilter +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend +from taskq.testing.jobs import make_enqueue_args + +from datetime import UTC, datetime + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + +async def test_cancel_where_pending_jobs() -> None: + """cancel_where moves pending jobs straight to 'cancelled'.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + # Enqueue 3 jobs with tag "tenant-acme", 2 without + for i in range(3): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme", "run-001"), scheduled_at=_NOW)) + for i in range(2): + await backend.enqueue(make_enqueue_args(tags=("tenant-other",), scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 3 + assert result.cancel_requested == 0 + assert result.total_affected == 3 + assert len(result.cancelled_ids) == 3 + + # Verify the untagged jobs are still pending + remaining = await backend.list_jobs(JobFilter(tags=("tenant-other",))) + assert len(remaining) == 2 + assert all(r.status == "pending" for r in remaining) + +async def test_cancel_where_running_jobs_cooperative() -> None: + """cancel_where sets cancel_phase=1 for running jobs (cooperative).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + # Enqueue and manually dispatch to running + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + from dataclasses import replace + # Simulate dispatch: set to running + backend._jobs[row.id] = replace(backend._jobs[row.id], status="running", locked_by_worker=uuid4()) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 1 + assert len(result.cancel_requested_ids) == 1 + + # Verify the job is still running but has cancel_phase=1 + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + assert updated.cancel_phase == 1 # CancelPhase.COOPERATIVE + +async def test_cancel_where_mixed_statuses() -> None: + """cancel_where handles both pending and running in one call.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + # 2 pending + 1 running, all tagged "tenant-acme" + for _ in range(2): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + args3 = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row3 = await backend.enqueue(args3) + from dataclasses import replace + backend._jobs[row3.id] = replace(backend._jobs[row3.id], status="running", locked_by_worker=uuid4()) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 2 + assert result.cancel_requested == 1 + assert result.total_affected == 3 + +async def test_cancel_where_no_matches_returns_zero() -> None: + """cancel_where with a filter matching nothing returns zero counts.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(tags=("nonexistent",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 0 + assert result.total_affected == 0 + +async def test_cancel_where_already_cancelled_not_affected() -> None: + """Already-terminal jobs are not re-cancelled.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + from dataclasses import replace + backend._jobs[row.id] = replace(backend._jobs[row.id], status="cancelled", finished_at=_NOW) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.total_affected == 0 + +async def test_cancel_where_filter_by_batch_id() -> None: + """cancel_where works with batch_id filter.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + bid = uuid4() + for i in range(3): + args = make_enqueue_args( + tags=("tenant-acme",), + scheduled_at=_NOW, + metadata={"batch_id": str(bid)}, + ) + await backend.enqueue(args) + # Untagged job with different batch + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW, metadata={"batch_id": str(uuid4())})) + + result = await backend.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 3 + +async def test_cancel_where_filter_by_queue_and_actor() -> None: + """cancel_where works with queue and actor filters.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + await backend.enqueue(make_enqueue_args(queue="default", actor="worker-a", scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(queue="default", actor="worker-b", scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(queue="priority", actor="worker-a", scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(queue="default", actor="worker-a"), + reason="abort", + ) + + assert result.cancelled_directly == 1 + +async def test_cancel_where_active_filter() -> None: + """cancel_where with active=True targets only non-terminal jobs.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + # 2 active (pending) + 1 terminal (succeeded) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + args3 = make_enqueue_args(scheduled_at=_NOW) + row3 = await backend.enqueue(args3) + from dataclasses import replace + backend._jobs[row3.id] = replace(backend._jobs[row3.id], status="succeeded", finished_at=_NOW) + + result = await backend.cancel_where( + JobFilter(active=True), + reason="drain", + ) + + assert result.cancelled_directly == 2 # only the 2 pending + +async def test_cancel_where_ignores_filter_limit() -> None: + """cancel_where cancels ALL matching jobs even when filter.limit is small. + + This guards against the H3 bug: _list_jobs applies filters.limit (default + 100). If _cancel_where reuses _list_jobs without sanitizing the filter, + a caller passing JobFilter(limit=5, tags=...) would cancel only 5 jobs. + """ + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + # Enqueue 11 jobs matching the tag + for _ in range(11): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + # Pass a restrictive limit — cancel_where must ignore it + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",), limit=5), + reason="offboard", + ) + + assert result.cancelled_directly == 11 # all of them, not just 5 + assert result.total_affected == 11 +``` + +#### TDD — Green + +Implement `_cancel_where` in `src/taskq/testing/_cancel_bulk.py`, wire it in `in_memory.py`. + +#### Acceptance criteria +- All red tests pass +- `InMemoryBackend.cancel_where` satisfies the `Backend` protocol +- Events are inserted for both directly-cancelled and cooperative-cancel jobs +- Cancel wake subscribers are notified for running jobs +- `cancel_where` ignores `filter.limit` and `filter.cursor` — all matching jobs are cancelled regardless of pagination fields (verified by `test_cancel_where_ignores_filter_limit`) + +--- + +### Task 5: Implement `cancel_where` for PostgresBackend + +**Goal:** Add the set-based SQL bulk cancel to the Postgres backend. + +**Files:** +- CREATE: `src/taskq/backend/_cancel_bulk.py` +- MODIFY: `src/taskq/backend/postgres.py` — wire `cancel_where` method +- CREATE: `tests/test_cancel_where_pg.py` + +> **No `SqlTemplates` field needed:** The SQL is inlined in `_cancel_bulk.py` with dynamic filter conditions baked in via f-string (matching the dynamic-SQL precedent in `_reads.py`). A `SqlTemplates.cancel_where` field would require template-level `{filter_conditions}` placeholder substitution that doesn't fit the static-template rendering model — the filter conditions are built at call time from `build_filter_conditions()`, not at schema-render time. + +#### Implementation + +```python +# src/taskq/backend/_cancel_bulk.py + +import asyncio +import random + +import asyncpg + +from taskq.backend._filter_sql import build_filter_conditions +from taskq.backend._records import jsonb_param + +# Returns (result, notify_targets) where notify_targets is +# [(job_id, worker_id)] for running jobs that got cooperative cancel. +# NOTIFY itself is sent by PostgresBackend.cancel_where (see wiring below) +# because the taskq.cancel.notify_sent counter is module-level in postgres.py. +async def _cancel_where( + pool: asyncpg.Pool, + schema: str, + sql: SqlTemplates, + filter: JobFilter, + reason: str | None, +) -> tuple[BulkCancelResult, list[tuple[UUID, UUID]]]: + filter_sql = build_filter_conditions(filter) + conditions_str = " AND ".join(filter_sql.conditions) if filter_sql.conditions else "TRUE" + params = filter_sql.params + + # Single CTE statement: snapshot matching IDs, then two UPDATEs with + # EPQ-safe predicates duplicated in each UPDATE's own WHERE clause. + # ORDER BY id in the matching CTE ensures deterministic lock ordering + # to reduce deadlock probability. + cancel_sql = f""" + WITH matching AS ( + SELECT id, status, locked_by_worker + FROM "{schema}".jobs + WHERE {conditions_str} + ORDER BY id + ), + cancelled AS ( + UPDATE "{schema}".jobs AS j + SET status = 'cancelled', finished_at = clock_timestamp() + FROM ( + SELECT id, status AS prev_status + FROM matching + WHERE status IN ('pending', 'scheduled') + ) AS prev + WHERE j.id = prev.id + AND j.status IN ('pending', 'scheduled') -- EPQ re-check (Critical) + RETURNING j.id, prev.prev_status + ), + cancel_requested AS ( + UPDATE "{schema}".jobs AS j + SET cancel_requested_at = now(), cancel_phase = 1 + WHERE j.id IN ( + SELECT id FROM matching + WHERE status = 'running' AND cancel_phase = 0 + ) + AND j.status = 'running' AND j.cancel_phase = 0 -- EPQ re-check (Critical) + RETURNING j.id, j.locked_by_worker + ) + SELECT + (SELECT count(*)::int FROM cancelled) AS cancelled_directly, + (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, + (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, + (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, + (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, + (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers + """ + + # Deadlock retry: the bulk UPDATE locks rows in id order while dispatch + # and heartbeat transactions lock rows in their own orders, so Postgres + # may abort this statement with DeadlockDetectedError. The retry lives in + # the BACKEND (not JobsClient) because the exception type is + # asyncpg-specific and the client layer stays backend-agnostic. The whole + # UPDATE+events runs in one transaction, so a deadlocked attempt rolls + # back completely and re-execution from a fresh snapshot is safe (the + # EPQ predicates re-filter on every attempt). NOTIFY is sent by the + # caller only after a successful commit, so no notify can fire for a + # rolled-back attempt. + for attempt in range(3): + try: + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow(cancel_sql, *params) + + assert row is not None # aggregate SELECT always returns one row + cancelled_ids: list[UUID] = list(row["cancelled_ids"] or []) + cancel_requested_ids: list[UUID] = list(row["cancel_requested_ids"] or []) + prev_statuses: dict[UUID, str] = dict( + zip(cancelled_ids, row["cancelled_prev_statuses"] or [], strict=True) + ) + notify_targets = [ + (jid, wid) + for jid, wid in zip( + cancel_requested_ids, + row["cancel_requested_workers"] or [], + strict=True, + ) + if wid is not None + ] + + # Events — same kinds as single-job write_cancel_request + # (postgres.py:555-561): state_change + cancel_request for + # pending/scheduled; cancel_request only for running. + # detail JSON is serialized in Python via jsonb_param — + # never f-string interpolation (a reason containing " + # or \ would otherwise produce malformed jsonb and abort + # the transaction; see H2 design note below). + cr_detail = jsonb_param({"reason": reason} if reason is not None else {}) + if cancelled_ids: + await conn.executemany( + sql.insert_event, # (job_id, kind, detail) — kind is $2 + [ + ( + jid, + "state_change", + jsonb_param( + {"from_state": prev_statuses[jid], "to_state": "cancelled"} + ), + ) + for jid in cancelled_ids + ], + ) + await conn.executemany( + sql.insert_event, + [(jid, "cancel_request", cr_detail) for jid in cancelled_ids], + ) + if cancel_requested_ids: + await conn.executemany( + sql.insert_event, + [(jid, "cancel_request", cr_detail) for jid in cancel_requested_ids], + ) + break # committed successfully + except asyncpg.DeadlockDetectedError: + if attempt == 2: + raise + await asyncio.sleep(0.1 * (2**attempt) + random.random() * 0.05) + + result = BulkCancelResult( + cancelled_directly=len(cancelled_ids), + cancel_requested=len(cancel_requested_ids), + cancelled_ids=cancelled_ids, + cancel_requested_ids=cancel_requested_ids, + ) + return result, notify_targets +``` + +#### Wiring in `postgres.py` + +```python +# src/taskq/backend/postgres.py + +async def cancel_where( + self, + filter: JobFilter, + reason: str | None, +) -> BulkCancelResult: + result, notify_targets = await _cancel_bulk._cancel_where( + self._worker_pool, self._schema_name, self._sql, filter, reason + ) + if notify_targets: + # Post-commit NOTIFY, same pattern as write_cancel_request + # (postgres.py:585-603) but batched into one statement. + channels: list[str] = [] + payloads: list[str] = [] + for job_id, worker_id in notify_targets: + payload = dumps_str( + {"type": "cancel", "job_id": str(job_id), "worker_id": str(worker_id)} + ) + channels.extend( + [ + events_channel(self._schema_name), + worker_channel(self._schema_name, str(worker_id)), + ] + ) + payloads.extend([payload, payload]) + async with self._worker_pool.acquire() as notify_conn: + await notify_conn.execute( + "SELECT pg_notify(channel, payload) " + "FROM unnest($1::text[], $2::text[]) AS t(channel, payload)", + channels, + payloads, + ) + _cancel_notify_sent_counter.add(len(notify_targets), {"schema": self._schema_name}) + return result +``` + +**Design note — capturing `prev_status` (L5):** The `cancelled` CTE uses the `FROM prev` pattern from the existing single-job `cancel_pending_scheduled` template (`_sql_templates.py:407-415`) so the actual previous status (`'pending'` or `'scheduled'`) flows into the `state_change` event detail in a single round-trip — not a synthetic `'pending_or_scheduled'` placeholder, and no second query. Two details matter: the target table must be aliased (`AS j`) because `prev` also exposes an `id` column (`RETURNING id` would be ambiguous), and the EPQ-re-checked predicate must reference the target table (`j.status`), since EPQ re-evaluates only the UPDATE's own WHERE clause against the newest row version — `prev.*` values are snapshot values. + +**Design note — safe JSON serialization (H2 fix):** The `reason` string is serialized in Python via `jsonb_param({"reason": reason})` (which uses `dumps_str`/orjson), then bound as a jsonb parameter. This avoids the f-string injection bug where `reason` containing `"` or `\` would produce invalid JSON → `asyncpg.DataError` mid-transaction (rolling back the entire bulk cancel), or structurally valid but operator-shaped JSON. The existing single-job path does this safely at `_terminal.py:129-144`. The red test `test_pg_cancel_where_reason_with_quotes` pins the fix. + +**Design note — deadlock retry (M6):** The retry loop lives in `_cancel_bulk._cancel_where` (the backend), not in `JobsClient`, for two reasons: the exception type is `asyncpg.DeadlockDetectedError` — catching it in the client would couple the backend-agnostic client layer to asyncpg — and the backend owns the transaction boundary, so it alone can guarantee that a retried attempt starts from a fresh snapshot with no partial effects (the single transaction rolls back UPDATEs and event inserts atomically). Max 3 attempts with jittered exponential backoff (100ms base). The `ORDER BY id` in the `matching` CTE reduces (but does not eliminate) deadlock probability against dispatch/heartbeat lock ordering. NOTIFY is sent by `PostgresBackend.cancel_where` only after a successful commit, so a retried-or-failed attempt never fires a spurious notify. The in-memory backend never deadlocks and needs no retry. + +#### TDD — Red + +```python +# tests/test_cancel_where_pg.py + +import asyncio +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from taskq.backend._protocol import JobFilter +from taskq.testing.fixtures import JobsApp +from taskq.testing.jobs import make_enqueue_args + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + +@pytest.mark.integration +class TestCancelWherePostgres: + async def test_pg_cancel_where_pending(self, backend_pair) -> None: + """PostgresBackend.cancel_where cancels pending jobs by tag.""" + from taskq.types import BulkCancelResult + + for _ in range(3): + await backend_pair.enqueue( + make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + ) + await backend_pair.enqueue( + make_enqueue_args(tags=("other",), scheduled_at=_NOW) + ) + + result = await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 3 + assert result.cancel_requested == 0 + + # Verify via list + remaining = await backend_pair.list_jobs(JobFilter(tags=("other",))) + assert len(remaining) == 1 + assert remaining[0].status == "pending" + + async def test_pg_cancel_where_events_inserted(self, backend_pair) -> None: + """cancel_where inserts job_events for cancelled jobs — both + state_change (with actual from_state) and cancel_request, + matching single-job write_cancel_request semantics.""" + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend_pair.enqueue(args) + + await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + events = await backend_pair.get_events(row.id) + kinds = [e.kind for e in events] + # Both event kinds must be present (event parity with single-job path) + assert "state_change" in kinds + assert "cancel_request" in kinds + # state_change should have actual from_state, not a synthetic placeholder + sc = [e for e in events if e.kind == "state_change"] + assert sc[0].detail.get("from_state") in ("pending", "scheduled") + assert sc[0].detail.get("to_state") == "cancelled" + + async def test_pg_cancel_where_reason_with_quotes(self, backend_pair) -> None: + """Reason containing double-quotes does not cause DataError. + + Guards against H2: f-string JSON interpolation would produce + invalid JSON for reasons containing " or \\. + """ + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend_pair.enqueue(args) + + result = await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason='offboard "tenant-acme" \\ done', + ) + + assert result.cancelled_directly == 1 + events = await backend_pair.get_events(row.id) + cr = [e for e in events if e.kind == "cancel_request"] + assert cr[0].detail.get("reason") == 'offboard "tenant-acme" \\ done' + + async def test_pg_cancel_where_batch_id_filter(self, backend_pair) -> None: + """cancel_where works with batch_id filter on Postgres.""" + bid = uuid4() + for _ in range(3): + await backend_pair.enqueue( + make_enqueue_args( + scheduled_at=_NOW, + metadata={"batch_id": str(bid)}, + ) + ) + + result = await backend_pair.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 3 + + async def test_pg_cancel_where_running_cooperative( + self, clean_jobs_app: JobsApp + ) -> None: + """cancel_where sets cancel_phase=1 for running jobs on Postgres. + + Uses a direct SQL UPDATE to simulate dispatch (status 'running' + with a worker ID). clean_jobs_app provides a PG-only backend plus + WorkerDeps with direct pool access; the in-memory path is covered + by the Task 4 tests and the backend_pair tests above. + """ + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + + # Simulate dispatch via direct SQL ('running' as a SQL literal so the + # schema-scoped job_status enum coerces without a parameter cast). + worker_id = uuid4() + async with deps.worker_pool.acquire() as conn: + await conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 1 + + # Verify the job is still running but has cancel_phase=1 + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + assert updated.cancel_phase == 1 + + # Verify cancel_request event was inserted (no state_change for running) + events = await backend.get_events(row.id) + kinds = [e.kind for e in events] + assert "cancel_request" in kinds + assert "state_change" not in kinds + + async def test_pg_cancel_where_does_not_clobber_concurrent_claim( + self, clean_jobs_app: JobsApp + ) -> None: + """EPQ regression (C1): a job claimed (pending→running) while + cancel_where executes must NOT be overwritten to terminal 'cancelled'. + + The bulk UPDATE blocks on the row lock held by the simulated claim + transaction; after the claim commits, EvalPlanQual re-evaluates the + UPDATE's own WHERE clause against the new row version, sees + status='running', and skips the row. The job escapes this call + entirely (documented residual window) instead of being clobbered. + """ + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + worker_id = uuid4() + + claim_conn = await deps.worker_pool.acquire() + try: + # Simulate a dispatch claim in a held-open transaction: the row is + # locked and updated to 'running' but not yet committed. + claim_tx = claim_conn.transaction() + await claim_tx.start() + await claim_conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + # Run cancel_where concurrently. Its CTE snapshot is taken at + # statement start (before the claim commits, so the snapshot sees + # 'pending'); its UPDATE then blocks on the claim's row lock. + cancel_task = asyncio.create_task( + backend.cancel_where(JobFilter(tags=("tenant-acme",)), reason="offboard") + ) + await asyncio.sleep(0.2) # let cancel_where reach the row lock + await claim_tx.commit() + result = await cancel_task + + # Safety property: the claimed job was NOT clobbered to terminal + # 'cancelled'. It escaped this call (EPQ re-check rejected it for + # the pending/scheduled UPDATE; the snapshot excluded it from the + # running UPDATE), so a follow-up call is needed to cancel it. + assert result.cancelled_directly == 0 + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + finally: + await deps.worker_pool.release(claim_conn) + + async def test_pg_cancel_where_notify_sent_for_running( + self, clean_jobs_app: JobsApp, pg_dsn: str + ) -> None: + """Batched NOTIFY fires on the fleet and per-worker channels for + running jobs — same listener pattern as test_cancel_notify_integration.py.""" + import asyncpg + from taskq.constants import events_channel, worker_channel + + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + worker_id = uuid4() + async with deps.worker_pool.acquire() as conn: + await conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + received: list[str] = [] + listen_conn = await asyncpg.connect(pg_dsn) + try: + await listen_conn.add_listener( + events_channel(schema), + lambda _c, _p, _ch, payload: received.append(payload), + ) + await listen_conn.add_listener( + worker_channel(schema, str(worker_id)), + lambda _c, _p, _ch, payload: received.append(payload), + ) + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), reason="offboard" + ) + assert result.cancel_requested == 1 + await asyncio.sleep(0.3) # allow asyncpg NOTIFY delivery + finally: + await listen_conn.close() + + assert len(received) == 2 # one fleet-channel + one per-worker-channel payload +``` + +#### TDD — Green + +Implement the SQL and wire the method. Run the integration tests: the `backend_pair` tests run against both backends (the `pg` param requires `@pytest.mark.integration`, enforced by the fixture guard); the `clean_jobs_app` tests run PG-only with direct pool access for dispatch simulation. + +#### Acceptance criteria +- All red tests pass against both in-memory and Postgres backends +- `job_events` rows are inserted for cancelled jobs — both `state_change` (with actual `from_state`) and `cancel_request` for pending/scheduled; `cancel_request` only for running (event parity with single-job `write_cancel_request`) +- NOTIFY is sent for running jobs, batched into one statement (verified by `test_pg_cancel_where_notify_sent_for_running`, same listener pattern as `test_cancel_notify_integration.py`); `taskq.cancel.notify_sent` counter incremented per job +- Single SQL statement for the UPDATEs (with EPQ-safe duplicated predicates on the target table, `FROM prev` for `prev_status`); `executemany` for events within the same transaction via the shared `sql.insert_event` template +- `reason` JSON is serialized safely via `jsonb_param` (not f-string interpolation) +- `ORDER BY id` in the matching CTE for deterministic lock ordering +- Backend retries the transaction on `asyncpg.DeadlockDetectedError` (max 3 attempts, jittered backoff); NOTIFY fires only after a successful commit +- No clobbering of concurrently-claimed rows (verified by `test_pg_cancel_where_does_not_clobber_concurrent_claim`: the claimed job stays `running`, not `cancelled`) + +--- + +### Task 6: Add `cancel_where` to `JobsClient` and `TaskQ` + +**Goal:** Add the client-layer method with the empty-filter guardrail. + +**Files:** +- MODIFY: `src/taskq/client/_jobs.py` — add `cancel_where` +- MODIFY: `src/taskq/client/_taskq.py` — add `cancel_where` delegate +- CREATE: `tests/test_cancel_where_client.py` + +#### TDD — Red + +```python +# tests/test_cancel_where_client.py + +import pytest +from taskq.backend._protocol import JobFilter +from taskq.client._jobs import JobsClient +from taskq.exceptions import EmptyFilterError +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend +from taskq.testing.jobs import make_enqueue_args +from taskq.types import BulkCancelResult + +from datetime import UTC, datetime + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + +async def test_client_cancel_where_with_tags() -> None: + """JobsClient.cancel_where cancels jobs by tag filter.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + for _ in range(3): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert isinstance(result, BulkCancelResult) + assert result.cancelled_directly == 3 + +async def test_client_cancel_where_empty_filter_raises() -> None: + """Empty filter (no predicates) raises EmptyFilterError.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + with pytest.raises(EmptyFilterError, match="filter predicate"): + await client.cancel_where(JobFilter(), reason="oops") + +async def test_client_cancel_where_empty_filter_override() -> None: + """allow_empty_filter=True bypasses the guardrail.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(), + reason="drain all", + allow_empty_filter=True, + ) + + assert result.cancelled_directly == 2 + +async def test_client_cancel_where_increments_counter() -> None: + """cancel_where increments taskq.cancellation.requested once.""" + # Same OTel fixture pattern as test_jobs_client_cancel.py + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + await client.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + # Counter should be 1 (one call, regardless of jobs affected) + # ... OTel reader assertion ... + +async def test_client_cancel_where_translates_schema_errors() -> None: + """cancel_where wraps UndefinedTableError in SchemaNotMigratedError.""" + # ... same pattern as enqueue test ... +``` + +#### TDD — Green + +```python +# In JobsClient: + +async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, +) -> BulkCancelResult: + from taskq.exceptions import EmptyFilterError + from taskq.obs import record_cancel_requested + + # Guardrail: reject empty filter + if not allow_empty_filter: + if ( + filter.queue is None + and filter.status is None + and filter.actor is None + and filter.identity_key is None + and filter.batch_id is None + and (filter.tags is None or len(filter.tags) == 0) + and filter.active is None + ): + raise EmptyFilterError() + + record_cancel_requested() + + with self._translate_schema_errors(): + return await self._backend.cancel_where(filter, reason) +``` + +#### Acceptance criteria +- Empty filter raises `EmptyFilterError` by default +- `allow_empty_filter=True` overrides the guardrail +- `taskq.cancellation.requested` counter incremented once per call +- `SchemaNotMigratedError` wrapping works (same pattern as other client methods) +- `TaskQ.cancel_where` delegates correctly (requires open client) +- Client stays thin: no `DeadlockDetectedError` handling here — deadlock retry is backend-owned (see Task 5 design note); the client must not import asyncpg for this + +--- + +### Task 7: Add `tags` to `SubJobEnqueuer.enqueue()` with parent-tag inheritance + +**Goal:** Add the `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, and `heartbeat_timeout` parameters to `SubJobEnqueuer.enqueue()`. + +**Files:** +- MODIFY: `src/taskq/client/_enqueuer.py` — add parameters, ContextVar, tag resolution logic +- MODIFY: `src/taskq/worker/_consumer.py` — set parent tags before actor invocation (gated by `sub_job_inherit_tags` setting) +- MODIFY: `src/taskq/worker/run.py` — set parent tags in stub consumer (unconditional — test harness) +- MODIFY: `src/taskq/settings.py` — add `sub_job_inherit_tags: bool = True` field to `WorkerSettings` (fleet-level kill switch; `TASKQ_SUB_JOB_INHERIT_TAGS` env var via pydantic-settings) +- CREATE: `tests/test_sub_job_tags.py` +- MODIFY: `tests/test_sub_job_enqueuer.py` — add tags tests + +#### Worker-level kill switch (`sub_job_inherit_tags`) + +`inherit_tags=True` as a default is a production behavior change: after upgrade, every existing sub-job enqueued inside an actor whose parent has tags becomes tag-findable — and via #54, tag-cancellable. A shared/utility sub-job enqueued by a tenant-tagged parent will now be swept up in that tenant's `cancel_where`. Per-call `inherit_tags=False` is not a practical rollback for a fleet. + +The `sub_job_inherit_tags` worker setting (default `True`) provides a fleet-level opt-out. When set to `False`, the consumer does **not** call `set_parent_tags()` — the ContextVar remains at its `()` default, so `inherit_tags=True` on `enqueue()` produces `()` (identical to pre-upgrade behavior). This allows operators to disable inheritance across an entire worker fleet without code changes. + +**Rollout guidance:** +1. Deploy with `sub_job_inherit_tags=False` (preserves existing behavior). +2. Verify no regressions in production. +3. Enable `sub_job_inherit_tags=True` per-queue or per-worker-group as confidence grows. +4. Document the blast-radius implication in `jobs-clients.md`: sub-jobs inherit parent tags → they are visible to `cancel_where` filters matching those tags. + +#### Batch enqueue asymmetry (`enqueue_batch`) + +`ctx.jobs.enqueue_batch` (via `EnqueueItem.tags`) does **not** inherit parent tags in this spec. This is a deliberate scoping decision for this iteration: + +- `enqueue_batch` fans out N items, each potentially with its own `tags` field. Applying parent-tag inheritance per-item would require merging parent tags into each `EnqueueItem.tags` — a different code path (`batch.py`) than the single-enqueue path (`_enqueuer.py`). +- The primary use case for batch enqueue (fan-out chunks) already sets tags per `EnqueueItem` at call sites (e.g., cennan's `EnqueueItem(tags=...)` per sync-run/binding). These callers explicitly tag their batch items. +- Extending `inherit_tags` to `enqueue_batch` is a follow-up spec that can add a per-call `inherit_tags: bool` parameter to `enqueue_batch` and merge parent tags into each item's `tags` field. This is noted as a non-goal for this spec to keep the scope bounded. + +**The asymmetry is documented** in Design Decisions (#57, decision 7) and in the updated `jobs-clients.md` guide so callers are aware that single `enqueue()` inherits by default while `enqueue_batch()` does not. + +#### TDD — Red + +```python +# tests/test_sub_job_tags.py + +import pytest +from datetime import UTC, datetime, timedelta +from uuid import uuid4 +from pydantic import BaseModel, TypeAdapter + +from taskq.actor import ActorRef +from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend + +class _Payload(BaseModel): + value: str = "test" + +class _Result(BaseModel): + ok: bool = True + +def _make_actor_ref(name: str = "child") -> ActorRef[_Payload, _Result]: + async def _handler(payload: _Payload) -> _Result: + return _Result() + return ActorRef( + name=name, queue="default", fn=_handler, wants_ctx=False, + dependencies={}, payload_type=_Payload, + result_adapter=TypeAdapter(_Result), + retry=__import__("taskq.retry", fromlist=["RetryPolicy"]).RetryPolicy(), + result_ttl=None, singleton=False, unique_for=None, max_pending=None, + ) + +_NOW = datetime(2025, 1, 1, tzinfo=UTC) + +def _make_enqueuer(backend: InMemoryBackend | None = None) -> SubJobEnqueuer: + if backend is None: + backend = InMemoryBackend(clock=FakeClock(_NOW)) + return SubJobEnqueuer( + loop_scope_resolved=None, + worker_pool=object(), # sentinel + backend=backend, + clock=FakeClock(_NOW), + ) + +class TestSubJobExplicitTags: + async def test_explicit_tags_no_inheritance(self) -> None: + """tags= with inherit_tags=False sets only explicit tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["alpha", "beta"], + inherit_tags=False, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("alpha", "beta") + + async def test_tags_validated(self) -> None: + """Invalid tags raise ValueError.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + with pytest.raises(ValueError, match="invalid tag"): + await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["ab"], # too short + ) + + async def test_tags_deduplicated(self) -> None: + """Duplicate tags are deduplicated.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["alpha", "alpha", "beta"], + inherit_tags=False, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("alpha", "beta") + +class TestSubJobTagInheritance: + async def test_inherit_parent_tags_default(self) -> None: + """With no explicit tags and default inherit_tags=True, sub-job inherits parent tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + # Set parent tags (simulating consumer setting them before actor invocation) + token = set_parent_tags(("run-001", "tenant-acme")) + try: + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("run-001", "tenant-acme") + + async def test_inherit_and_merge_tags(self) -> None: + """Explicit tags merge with parent tags (parent first, deduped).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001", "tenant-acme")) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["stage-2", "tenant-acme"], # tenant-acme is a dup + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("run-001", "tenant-acme", "stage-2") + + async def test_no_parent_tags_no_explicit_tags(self) -> None: + """With no parent tags and no explicit tags, sub-job has empty tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + # No set_parent_tags call — ContextVar default is () + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == () + + async def test_inherit_false_no_parent_tags(self) -> None: + """inherit_tags=False with no explicit tags → empty tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001",)) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + inherit_tags=False, + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == () + + async def test_inherit_false_with_explicit_tags(self) -> None: + """inherit_tags=False with explicit tags → only explicit tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001",)) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["custom-tag"], + inherit_tags=False, + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("custom-tag",) + +class TestSubJobMissingFields: + async def test_schedule_to_close(self) -> None: + """schedule_to_close is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + deadline = _NOW + timedelta(hours=1) + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + schedule_to_close=deadline, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.schedule_to_close == deadline + + async def test_start_to_close(self) -> None: + """start_to_close is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + start_to_close=timedelta(minutes=30), + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.start_to_close == timedelta(minutes=30) + + async def test_heartbeat_timeout(self) -> None: + """heartbeat_timeout is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + heartbeat_timeout=timedelta(seconds=10), + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.heartbeat_timeout == timedelta(seconds=10) + +class TestContextVarIsolation: + async def test_concurrent_jobs_separate_parent_tags(self) -> None: + """ContextVar ensures concurrent consumers don't share parent tags.""" + import asyncio + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + async def enqueue_with_parent(parent_tags: tuple[str, ...]) -> UUID: + token = set_parent_tags(parent_tags) + try: + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + return handle.job_id + finally: + _parent_tags_var.reset(token) + + # Run two "jobs" concurrently with different parent tags + id1, id2 = await asyncio.gather( + enqueue_with_parent(("run-a",)), + enqueue_with_parent(("run-b",)), + ) + + row1 = await backend.get(JobId(id1)) + row2 = await backend.get(JobId(id2)) + assert row1.tags == ("run-a",) + assert row2.tags == ("run-b",) +``` + +#### TDD — Green + +1. Add `contextvars.ContextVar` and `set_parent_tags()` to `_enqueuer.py` +2. Add `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, `heartbeat_timeout` to `enqueue()` +3. Add `_resolve_tags()` method +4. Pass the new parameters to `build_enqueue_args()` +5. In `_consumer.py`, call `set_parent_tags(tuple(job.tags))` before constructing `JobContext` +6. Reset the ContextVar after the actor completes (in a `finally` block) + +#### Acceptance criteria +- All red tests pass +- Default behavior: `inherit_tags=True` — sub-job inherits parent tags when no explicit tags +- Explicit tags merge with parent tags (union, parent-first, deduped) +- `inherit_tags=False` disables inheritance +- `schedule_to_close`, `start_to_close`, `heartbeat_timeout` are passed through to `build_enqueue_args` +- ContextVar isolation works — concurrent consumers don't cross-contaminate parent tags +- All existing `test_sub_job_enqueuer.py` tests still pass (backward compatible — default `tags=None` with no parent tags → `()`) + +--- + +### Task 8: Backward compatibility — default behavior unchanged + +**Goal:** Verify that existing code that doesn't use tags or `inherit_tags` sees no behavior change. + +**Files:** +- MODIFY: `tests/test_sub_job_enqueuer.py` — add backward compat tests + +#### TDD — Red + +```python +# tests/test_sub_job_enqueuer.py — add: + +class TestBackwardCompatibility: + async def test_no_tags_no_parent_tags_empty(self) -> None: + """Existing code with no tags and no parent context → empty tags.""" + # No set_parent_tags call → ContextVar default is () + enqueuer = _make_enqueuer() + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + row = await enqueuer._backend.get(handle.job_id) + assert row.tags == () + + async def test_existing_enqueue_no_tags_param(self) -> None: + """Calling enqueue without tags= still works (backward compat).""" + enqueuer = _make_enqueuer() + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + assert handle is not None + assert handle.job_id is not None +``` + +#### Acceptance criteria +- All existing sub-job enqueuer tests pass without modification +- Existing code that doesn't set parent tags or pass `tags=` gets `tags=()` (same as before) +- No new required parameters — all additions have defaults + +--- + +### Task 9: E2E tests — sub-job tags in a real pipeline + +**Goal:** Verify that sub-jobs enqueued from inside actor bodies are tagged and findable by `JobFilter(tags=...)` in a real worker container. + +**Files:** +- MODIFY: `tests/e2e/actors.py` — add a tagged pipeline actor +- CREATE: `tests/e2e/test_sub_job_tags.py` + +#### E2E actors + +```python +# tests/e2e/actors.py — add: + +class PipelineStagePayload(BaseModel): + run_id: str + stage: int + total_stages: int = 3 + +@actor(name="pipeline_stage", queue="e2e") +async def pipeline_stage( + payload: PipelineStagePayload, + ctx: JobContext[PipelineStagePayload], + *, + pool: asyncpg.Pool, +) -> None: + """Linear pipeline: each stage enqueues the next via ctx.jobs.enqueue().""" + await _record_effect(pool, ctx, "stage", { + "run_id": payload.run_id, + "stage": payload.stage, + }) + await asyncio.sleep(0.05) + + if payload.stage < payload.total_stages: + # Enqueue next stage — inherits parent tags by default + await ctx.jobs.enqueue( + pipeline_stage, + PipelineStagePayload( + run_id=payload.run_id, + stage=payload.stage + 1, + total_stages=payload.total_stages, + ), + ) +``` + +#### E2E test + +```python +# tests/e2e/test_sub_job_tags.py + +from __future__ import annotations +from typing import TYPE_CHECKING +import pytest +from taskq.backend._protocol import JobFilter +from ._assertions import wait_for_effects, poll_until +from .actors import ( + PipelineStagePayload, + TaggedPipelineStagePayload, + pipeline_stage, + tagged_pipeline_stage, +) + +if TYPE_CHECKING: + import asyncpg + from taskq import TaskQ + from .conftest import E2ESchema, E2EWorker + +pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] + +async def test_sub_job_inherits_tags_in_pipeline( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """Sub-jobs enqueued via ctx.jobs.enqueue() inherit parent tags. + + A pipeline of 3 stages is enqueued with tag "run-{run_id}". + Each stage enqueues the next via ctx.jobs.enqueue() with no + explicit tags. All 3 jobs should be findable by the tag filter. + """ + tag = f"run-{run_id[:8]}" + handle = await e2e_client.enqueue( + pipeline_stage, + PipelineStagePayload(run_id=run_id, stage=1, total_stages=3), + tags=[tag], + ) + + # Wait for all 3 stages to complete + await wait_for_effects( + e2e_pg_pool, + e2e_schema.schema_name, + run_id, + kind="stage", + min_count=3, + timeout=30, + ) + + # All 3 jobs should be findable by the tag + page = await e2e_client.list(JobFilter(tags=(tag,))) + assert len(page.jobs) == 3, ( + f"Expected 3 jobs with tag {tag!r}, found {len(page.jobs)}: " + f"{[j.id for j in page.jobs]}" + ) +``` + +#### E2E actors (additional for merge test) + +```python +# tests/e2e/actors.py — add: + +class TaggedPipelineStagePayload(BaseModel): + run_id: str + stage: int + total_stages: int = 3 + +@actor(name="tagged_pipeline_stage", queue="e2e") +async def tagged_pipeline_stage( + payload: TaggedPipelineStagePayload, + ctx: JobContext[TaggedPipelineStagePayload], + *, + pool: asyncpg.Pool, +) -> None: + """Pipeline that passes explicit tags to sub-job enqueue.""" + await _record_effect(pool, ctx, "stage", { + "run_id": payload.run_id, + "stage": payload.stage, + }) + await asyncio.sleep(0.05) + + if payload.stage < payload.total_stages: + # Explicit per-stage tag — merges with the inherited parent tag + await ctx.jobs.enqueue( + tagged_pipeline_stage, + TaggedPipelineStagePayload( + run_id=payload.run_id, + stage=payload.stage + 1, + total_stages=payload.total_stages, + ), + tags=[f"stage-{payload.stage + 1}"], + ) +``` + +#### E2E test (merge) + +```python +async def test_sub_job_explicit_tags_merge_with_parent( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """Explicit tags on sub-job merge with inherited parent tags. + + Stage 1 is enqueued with parent tag "run-{run_id}". Each stage + enqueues the next with an explicit stage tag (e.g. "stage-2"). + The sub-job should carry both the inherited run tag and the + explicit stage tag. + """ + parent_tag = f"run-{run_id[:8]}" + + handle = await e2e_client.enqueue( + tagged_pipeline_stage, + TaggedPipelineStagePayload(run_id=run_id, stage=1, total_stages=3), + tags=[parent_tag], + ) + + # Wait for all 3 stages to complete + await wait_for_effects( + e2e_pg_pool, + e2e_schema.schema_name, + run_id, + kind="stage", + min_count=3, + timeout=30, + ) + + # All 3 jobs should have the parent tag + page = await e2e_client.list(JobFilter(tags=(parent_tag,))) + assert len(page.jobs) == 3, ( + f"Expected 3 jobs with parent tag {parent_tag!r}, found {len(page.jobs)}" + ) + + # Stage 2 should also have the "stage-2" explicit tag + stage2 = await e2e_client.list(JobFilter(tags=("stage-2",))) + assert len(stage2.jobs) == 1, ( + f"Expected 1 job with stage-2 tag, found {len(stage2.jobs)}" + ) + # Verify it also has the parent tag (merged) + assert parent_tag in stage2.jobs[0].tags + assert "stage-2" in stage2.jobs[0].tags +``` + +#### Acceptance criteria +- Sub-jobs enqueued from actor bodies are findable by the parent's tag +- All pipeline stages share the run tag +- E2E test passes against real Postgres + worker container + +--- + +### Task 10: E2E tests — bulk cancel by filter + +**Goal:** Verify `cancel_where` works end-to-end against real Postgres + worker. + +**Files:** +- MODIFY: `tests/e2e/actors.py` — add bulk-cancel test actors if needed +- CREATE: `tests/e2e/test_cancel_where.py` + +#### E2E test + +```python +# tests/e2e/test_cancel_where.py + +from __future__ import annotations +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +import pytest +from taskq.backend._protocol import JobFilter +from taskq.types import BulkCancelResult +from ._assertions import poll_until, wait_for_handle_status +from .actors import GenerateReportPayload, generate_report + +if TYPE_CHECKING: + import asyncpg + from taskq import TaskQ + from .conftest import E2ESchema, E2EWorker + +pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] + +async def test_cancel_where_pending_jobs_by_tag( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where cancels all pending jobs matching a tag filter.""" + tag = f"tenant-{run_id[:8]}" + + # Enqueue 5 jobs with the tag, scheduled far in the future (won't dispatch) + future = datetime.now(UTC) + timedelta(seconds=120) + for i in range(5): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=run_id, report_id=f"r-{i}"), + scheduled_at=future, + tags=[tag], + ) + + # Enqueue 2 jobs without the tag (should not be affected) + for i in range(2): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=f"other-{run_id}", report_id=f"o-{i}"), + scheduled_at=future, + ) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,)), + reason="tenant offboarded", + ) + + assert result.cancelled_directly == 5 + assert result.cancel_requested == 0 + assert result.total_affected == 5 + + # Verify via list: tagged jobs are cancelled, untagged are still scheduled + tagged = await e2e_client.list(JobFilter(tags=(tag,), status="cancelled")) + assert len(tagged.jobs) == 5 + + untagged = await e2e_client.list( + JobFilter(status="scheduled", queue="e2e") + ) + assert len(untagged.jobs) == 2 + +async def test_cancel_where_running_jobs_cooperative( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where sets cooperative cancel for running jobs.""" + tag = f"run-{run_id[:8]}" + + # Enqueue a long-running job + handle = await e2e_client.enqueue( + generate_report, + GenerateReportPayload( + run_id=run_id, + report_id=f"r-{run_id[:8]}", + stages=4, + stage_latency_ms=2000, + ), + tags=[tag], + ) + await wait_for_handle_status(handle, "running", timeout=30) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,), status="running"), + reason="abort run", + ) + + assert result.cancel_requested >= 1 + assert result.cancelled_directly == 0 + + # The running job should eventually reach 'cancelled' + await wait_for_handle_status(handle, "cancelled", timeout=30) + +async def test_cancel_where_empty_filter_raises( + e2e_client: TaskQ, + e2e_worker: E2EWorker, +) -> None: + """Empty filter is rejected even in e2e.""" + from taskq.exceptions import EmptyFilterError + + with pytest.raises(EmptyFilterError): + await e2e_client.cancel_where(JobFilter(), reason="oops") + +async def test_cancel_where_batch_id_filter( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where works with batch_id filter.""" + from uuid import uuid4 + from taskq.batch import EnqueueItem + from .actors import ImportContactsChunkPayload, import_contacts_chunk + + future = datetime.now(UTC) + timedelta(seconds=120) + bid = uuid4() + + items = [ + EnqueueItem( + actor_ref=import_contacts_chunk, + payload=ImportContactsChunkPayload( + run_id=run_id, upload_id=str(bid), chunk_id=i, + start_row=i*100, end_row=(i+1)*100, + ), + scheduled_at=future, + ) + for i in range(5) + ] + batch = await e2e_client.enqueue_batch(items, batch_id=bid) + + result = await e2e_client.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 5 +``` + +#### Acceptance criteria +- Bulk cancel works against real Postgres with a real worker container +- Pending/scheduled jobs go straight to `cancelled` +- Running jobs get cooperative cancel and eventually reach `cancelled` +- Batch_id filter works end-to-end +- Empty filter guardrail fires in e2e context + +--- + +### Task 11: Update documentation + +**Goal:** Update the user-facing docs to reflect the new APIs. + +**Files:** +- MODIFY: `docs/guides/jobs-clients.md` — document `cancel_where`, sub-job `tags`, `inherit_tags` +- MODIFY: `docs/architecture.md` — document bulk cancel in the cancel protocol section + +#### Key doc additions + +1. **SubJobEnqueuer.enqueue()** — update the signature to show `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, `heartbeat_timeout`. Document the inheritance semantics table. Update the existing exclusion list at `jobs-clients.md:789` ("No `schedule_to_close`, `start_to_close`, or `heartbeat_timeout`") to reflect that these are now accepted on the sub-job path — this is a deliberate reversal of the documented exclusion; call it out in the guide, including the snooze/`schedule_to_close` warning (see Design Decisions #57-4). Document the `sub_job_inherit_tags` worker setting and the blast-radius implication (sub-jobs inherit parent tags → visible to `cancel_where`). Document that `enqueue_batch` does **not** inherit parent tags (asymmetry noted). All doc examples must use tags valid under the existing charset (`^[\w][\w\-]+[\w]$` — hyphenated, e.g. `tenant-acme`, not `tenant:acme`). + +2. **New `cancel_where` section** in jobs-clients.md: + ````markdown + ### `cancel_where()` + + ```python + result = await client.cancel_where( + JobFilter(tags=("tenant-acme",), active=True), + reason="tenant offboarded", + ) + ``` + + Cancel all jobs matching a filter in a single set-based operation... + ```` + +3. **Architecture.md** — add `cancel_where` to the Backend protocol listing and the cancel protocol section. + +--- + +## Test Coverage Requirements + +### Unit tests (in-memory backend) +- `test_filter_sql.py` — filter→SQL builder extraction (6+ tests) +- `test_bulk_cancel_types.py` — BulkCancelResult, EmptyFilterError (4+ tests) +- `test_cancel_where.py` — in-memory bulk cancel (9+ tests, including limit-ignoring test) +- `test_cancel_where_client.py` — client-layer guardrail, counter, schema errors (5+ tests) +- `test_sub_job_tags.py` — sub-job tags, inheritance, ContextVar isolation (12+ tests) +- `test_sub_job_enqueuer.py` — backward compat additions (2+ tests) + +### Integration tests (Postgres backend) +- `test_cancel_where_pg.py` — PostgresBackend bulk cancel (7 tests: pending, events-parity, reason-with-quotes, batch_id via `backend_pair`; running-cooperative, concurrent-claim EPQ race, and batched NOTIFY via `clean_jobs_app`) + +### Protocol conformance +- `test_backend_protocol.py` — cancel_where method presence (1 test) + +### E2E tests (real worker + PG) +- `test_sub_job_tags.py` — sub-job tag inheritance in a real pipeline (2+ tests) +- `test_cancel_where.py` — bulk cancel by tag/batch_id, cooperative cancel, guardrail (4+ tests) + +### Coverage targets +- New code paths must achieve ≥95% line coverage +- Tag inheritance logic (`_resolve_tags`) must have 100% branch coverage +- Guardrail validation must have 100% branch coverage +- SQL builder must have 100% branch coverage for all filter combinations + +--- + +## Backward Compatibility Analysis + +### SubJobEnqueuer.enqueue() changes + +| Change | Impact on existing code | Mitigation | +|---|---|---| +| New `tags=None` param | None — default is `None`, no behavior change when parent tags are also empty | None needed | +| New `inherit_tags=True` param | Sub-jobs now inherit parent tags by default. **Behavior change** when parent job has tags and code relies on sub-job tags being empty. Via #54, inherited tags make sub-jobs visible to `cancel_where` — a shared/utility sub-job enqueued by a tenant-tagged parent becomes tag-cancellable. | The ContextVar defaults to `()`, so if the consumer doesn't call `set_parent_tags()`, the behavior is identical to before. Only code that runs inside a real worker with the updated consumer will see inheritance. **Fleet-level kill switch:** `sub_job_inherit_tags` worker setting (default `True`) — when `False`, the consumer skips `set_parent_tags()`, preserving pre-upgrade behavior across the entire fleet. Deploy with `False` first, verify, then enable. | +| New `schedule_to_close=None` param | None — default is `None`, passes through to `build_enqueue_args` which already handles `None` | None needed | +| New `start_to_close=None` param | None — default is `None`, resolved to actor-declared value in `build_enqueue_args` | None needed | +| New `heartbeat_timeout=None` param | None — default is `None` | None needed | + +**Key backward-compat guarantee:** The `contextvars.ContextVar` default is `()`. Existing tests that construct `SubJobEnqueuer` directly (without going through the consumer) will see `()` parent tags, so `inherit_tags=True` with no explicit tags produces `()` — identical to the current behavior. Only code that runs inside a real worker with the updated consumer will see tag inheritance. For production rollouts, the `sub_job_inherit_tags` worker setting (default `True`) can be set to `False` to disable inheritance fleet-wide without code changes. + +### cancel_where addition + +| Change | Impact on existing code | Mitigation | +|---|---|---| +| New `Backend.cancel_where` method | Custom `Backend` implementations lack this method | Method is purely additive; `AttributeError` is loud. Third-party backends must add the method to support bulk cancel. | +| New `JobsClient.cancel_where` method | None — new method, no existing code calls it | None needed | +| New `BulkCancelResult` type | None — new type | None needed | +| New `EmptyFilterError` exception | None — new type | None needed | +| Filter→SQL builder extraction | `_list_jobs` internals change | All existing `test_job_filter.py` and `test_postgres_reads.py` tests verify no regression | + +### Protocol version + +No `BACKEND_PROTOCOL_VERSION` bump required. The `cancel_where` method is purely additive — a v3 implementation lacks the method, and calling it raises `AttributeError` (a loud failure, not a silent misbehavior). See the bump rule in `_protocol.py` lines 79-84: "Purely additive changes an old implementation can ignore without producing incorrect behaviour do not require a bump." + +--- + +## Downstream Consumer Impact Analysis + +> **Methodology and framing:** The contract for this section is the downstream need documented in issues #54/#57 and in the downstream codebases' own comments — not the code those repos happen to run today. Each entry states the documented need, the verified current baseline (local checkouts were grepped; claims that did not hold up were corrected), the end-state this spec enables, and the migration required to get there. + +### warden (~/src/warden) — Hybrid LLM proxy + +**Documented need (from issue #54):** tenant-scoped job grouping at enqueue time and tenant offboarding / run abort as a single operation. #54's motivating example is exactly this shape: jobs tagged per tenant, offboarded via a paginate-and-cancel loop that is slow (one round trip per job) and racy (workers keep dispatching behind the cursor; new matching jobs land behind the cursor, so convergence needs an outer retry loop). The documented need — not warden's current code — is the contract here. + +**How it uses TaskQ today (verified against the local checkout):** +- Enqueues jobs via `tq.enqueue` (e.g., `routes/admin.py:1438`, `routes/inference.py:1867,1996,2327`) +- Does **not** currently pass TaskQ `tags=` on any enqueue call (the `tags=` hits in `src/` are FastAPI route metadata, not TaskQ) +- Does **not** currently use `ctx.jobs` (sub-job enqueue) in `src/` — actor test harnesses construct `JobContext(jobs=None)` with a "never enqueues sub-jobs" comment (`app.py:217,252`) +- Does **not** currently call TaskQ `cancel()` in `src/` (only asyncio task cancels) + +**End-state this spec enables:** +- **#57:** Once warden tags jobs per tenant at enqueue (`tags=["tenant-"]` — hyphenated; the colon form in #54's prose is rejected by the current validator) and actors fan out via `ctx.jobs.enqueue()`, sub-jobs inherit tenant tags automatically and are findable by `JobFilter(tags=("tenant-",))`. +- **#54:** `cancel_where(JobFilter(tags=("tenant-",), active=True), reason="tenant offboarded")` replaces the paginate-and-cancel loop: one set-based write, no cursor race, counts/IDs returned for observability. Convergence for post-snapshot enqueues is the caller's job (stop producers, or repeat the call — see the snapshot-boundary contract). + +**Migration path:** +- No change required to existing enqueue calls (nothing is tagged today; `sub_job_inherit_tags` has no observable effect until sub-jobs exist) +- Adopt tags: pass `tags=["tenant-"]` at the `tq.enqueue` call sites +- Adopt bulk cancel: new offboarding flows call `cancel_where` instead of per-job loops + +### cennan (~/src/cennan) — Enterprise knowledge management + +**How it uses TaskQ today:** +- Ingestion pipeline: list → fetch → extract → chunk → embed → store +- Tags jobs per sync-run and per binding via `EnqueueItem.tags` in batch enqueue (`cennan/api/enqueue.py:299,519,565,574,604`), built by `_job_tags()` (`enqueue.py:240-252`) as hyphenated `sync-{sync_run_id}` / `binding-{binding_id}` — the same function documents a production incident where colon-form tags (`sync:{id}`) raised `ValueError` at enqueue and 500'd every sync trigger +- Pipeline stages use `ctx.jobs` chains (`pipeline/actors.py`); `pipeline/models.py:4-5` carries a comment documenting the exact #57 limitation ("`ctx.jobs` … does not accept `tags` in the installed TaskQ version"), which is why payloads re-carry `sync_run_id`/`binding_id` +- Needs to stop runaway ingestion runs + +**What this spec enables:** +- **#57:** Pipeline stages enqueued via `ctx.jobs.enqueue()` will inherit the `sync-*`/`binding-*` tags. The full pipeline is findable by tag, not just the batch-fan-out chunks. The `pipeline/models.py` workaround comment can be deleted; payload ids stay (they feed counters/liveness, not just discovery). +- **#54:** `cancel_where(JobFilter(tags=("sync-",), active=True))` stops a runaway ingestion run in one call. Pending/scheduled stages go straight to cancelled; running stages get cooperative cancel. + +**Migration path:** +- No code change required for tag inheritance (automatic once workers are upgraded); keep `sub_job_inherit_tags` at its default `True` +- Delete the `pipeline/models.py` limitation comment; drop any secondary lookup paths maintained only because sub-jobs were untaggable +- Replace manual cancel loops with `cancel_where` +- Can now tag sub-jobs with stage-specific tags (e.g., `tags=["stage-embed"]`) that merge with inherited sync/binding tags + +### aacrtool (~/src/aacrtool) — Agentic code review tool + +**Baseline (verified):** TaskQ is present only in `.venv` (dependency declared); **no TaskQ usage exists in `src/` yet**. Everything below is planned usage, not a description of current code. + +**End-state this spec enables (when TaskQ is adopted):** +- Review jobs tagged per repo at enqueue (`tags=["repo--"]` — hyphenated per the existing tag charset). +- **#57:** If review actors fan out sub-jobs (e.g., per-file analysis), those sub-jobs inherit the repo tag automatically. +- **#54:** `cancel_where(JobFilter(tags=("repo--",), active=True))` aborts a review run in one call. + +**Migration path:** +- N/A — TaskQ adoption is future work; adopt `cancel_where` and sub-job tags from day one rather than building paginate-and-cancel loops. + +--- + +## Design Decisions Summary + +### #57: Sub-job tags + +1. **Inheritance default: `True`** — sub-jobs inherit parent tags by default because the primary use case (run/tenant correlation) requires sub-jobs to be findable by the same tags as the parent. Opting out with `inherit_tags=False` is available for cases where sub-jobs should be untagged or only carry explicit tags. A worker-level `sub_job_inherit_tags` setting (default `True`) provides a fleet-level kill switch for operators who need to roll out the behavior change gradually. + +2. **Merge semantics: union, parent-first** — when both parent tags and explicit tags are provided, the union preserves parent tags first, then adds new tags. This lets callers add stage-specific tags while keeping the run/tenant correlation tag. Deduplication follows the same `_validate_and_dedup_tags` logic. + +3. **Propagation via `contextvars.ContextVar`** — the `SubJobEnqueuer` is shared across concurrent consumers in the same event loop, so a per-instance field would be racy. `ContextVar` is the asyncio-native solution: each Task gets its own context copy, so concurrent consumers each see their own parent tags. This is the same mechanism Python uses for `contextvars.copy_context()` in `asyncio.Task`. + +4. **Also add `schedule_to_close`, `start_to_close`, `heartbeat_timeout`** — these are passed through to `build_enqueue_args`, which already handles them. **This reverses a documented deliberate exclusion:** `docs/guides/jobs-clients.md:789` currently lists "No `schedule_to_close`, `start_to_close`, or `heartbeat_timeout` (set on the actor declaration)" as an intentional constraint of the sub-job surface. Issue #57 asks whether this is deliberate or an omission, and floats the hypothesis that "'you can't set it from inside an actor' may be a feature rather than an omission." This spec takes the position that the parameters should be available — the client and sub-job surfaces should not drift without a reason, and the issue's concrete cost ("a sub-job that needs a different timeout than its actor's declared default … has to be enqueued from outside the actor") is real — but acknowledges the trade-off: + + **Snooze/finalizer hazard (analyzed, not ignored):** Issue #57 names the hazard directly — "A finalizer that snoozes on `wait_for_batch` for a long time would be killed by one." The verified mechanics: (i) `mark_snoozed` already guards the deadline at snooze time — a running job whose requested snooze delay would cross `schedule_to_close` is failed immediately with `error_class='DeadlineExceeded'` ("schedule_to_close reached before next dispatch", `_sql_templates.py:254-275`), rather than being parked past its deadline; (ii) `sweep_deadline_exceeded` (`_sweeps.py:325+`) fails pending/scheduled jobs whose `schedule_to_close` has passed, which includes snoozed jobs (a snooze returns the row to `scheduled`). So a sub-job carrying a tight caller-supplied `schedule_to_close` fails **deterministically and loudly** — at the snooze attempt or at the deadline — never silently mid-snooze. That is precisely what a wall-clock deadline means; the documented exclusion (a) prevented actor code from opting into it. Reversing the exclusion means the hazard is opt-in per call. **Mitigations:** (a) the default is `None` — no override, so `build_enqueue_args` falls back to the actor's retry time budget exactly as today, and the hazard only manifests when a caller explicitly passes `schedule_to_close`; (b) the docs update (Task 11) must warn that `schedule_to_close` bounds total wall-clock time *including* time snoozed on `wait_for_batch`, so finalizer-style sub-jobs should set it generously or not at all; (c) a future spec could add a `snooze_extends_deadline` flag to make the interaction explicit — out of scope here. + +5. **No `queue` override** — sub-jobs use the actor's declared queue. This is a documented design choice (`docs/guides/jobs-clients.md:788`) and is not changed by this spec. + +6. **`idempotency_key` type: keep `IdempotencyKey | str | None`** — the sub-job enqueuer's wider type (accepting bare `str`) is more ergonomic for actor code. The `JobsClient` uses `IdempotencyKey | None` (the narrower `NewType`); the sub-job enqueuer keeps its wider type. `build_enqueue_args` already handles both. + +7. **Batch enqueue asymmetry** — `ctx.jobs.enqueue_batch` does **not** inherit parent tags in this spec. This is a deliberate scoping decision: `enqueue_batch` uses a different code path (`batch.py`) with per-item `EnqueueItem.tags`, and extending inheritance to batch would require merging parent tags into each item. The asymmetry is documented in the updated `jobs-clients.md` so callers are aware. A follow-up spec can add `inherit_tags` to `enqueue_batch` if needed. + +### #54: Bulk cancel by filter + +1. **Pending/scheduled → terminal `cancelled`** — these jobs have no running actor to cooperate with. The existing `write_cancel_request` already does this for single jobs; bulk cancel follows the same pattern. The issue asks: "should matching rows in pending/scheduled go straight to terminal cancelled?" — yes, they should, for consistency with the existing single-cancel path. + +2. **Running → cooperative `cancel_phase=1`** — running jobs have an actor executing. The cooperative path sets `cancel_phase=1`, which the worker's heartbeat-driven `CancelController` observes and sets the in-process `cancel_event`. The actor checks `ctx.check_cancelled()` at its next stage boundary. This is the existing phase-1 cooperative cancel, just applied in bulk. + +3. **Guardrail: empty filter rejected** — a `JobFilter` with all defaults matches every job in the table. `EmptyFilterError` is raised unless `allow_empty_filter=True` is explicitly passed. This prevents accidental full-table cancels while allowing intentional "cancel everything" operations. Edge case, documented rather than handled: `JobFilter(status=[])` passes the guardrail (`status` is not `None`) but matches no jobs (an empty status sequence renders `status = ANY('{}')`) — the call is a benign no-op returning zero counts. + +4. **Single SQL statement for the UPDATEs (with EPQ-safe predicates)** — the two UPDATEs (pending/scheduled → cancelled, running → cancel_phase=1) are in a single CTE-based statement within a single transaction. **Status/cancel_phase predicates are duplicated in each UPDATE's own WHERE clause** (not just in the `matching` CTE) so that EvalPlanQual re-evaluates them against concurrently-modified rows. Events are inserted via `executemany` in the same transaction. NOTIFY is sent after commit (same pattern as `write_cancel_request`, batched). `ORDER BY id` in the matching CTE reduces deadlock probability; the **backend** retries the transaction on `asyncpg.DeadlockDetectedError` (max 3 attempts, jittered backoff — backend-owned because the exception type is asyncpg-specific and the backend owns the transaction boundary; the client stays backend-agnostic). + +5. **Filter reuse: `JobFilter`** — the same `JobFilter` used by `list_jobs` is used by `cancel_where`. The `limit`, `cursor`, and `order_by` fields are ignored (bulk cancel is not paginated). The `build_filter_conditions` helper (without `schema` parameter — conditions are schema-less fragments) ensures filter semantics are identical between query and mutation. `_list_jobs` regains cursor/limit/order_by handling by appending them after the shared builder call. + +6. **Returns `BulkCancelResult` with counts and IDs** — the counts let callers verify the operation affected the expected number of jobs. The IDs enable observability and follow-up operations (e.g., waiting for cooperative cancels to complete). + +7. **Counter: `taskq.cancellation.requested` incremented once per call** — not once per job. This matches the existing `cancel()` semantics (one increment per API call) and avoids counter inflation for bulk operations. The `taskq.cancel.notify_sent` counter is incremented per job notified (matching the single-job path). + +8. **Event parity with single-job path** — both backends insert the same event kinds as `write_cancel_request`: for pending/scheduled, both `state_change` (with actual `from_state`) and `cancel_request`; for running, only `cancel_request`. This ensures `job_events` consumers (audit, reclaim tooling) see consistent event streams regardless of backend or bulk-vs-single path. + +9. **No protocol version bump** — `cancel_where` is purely additive. A v3 backend implementation lacks the method, and the client raises `AttributeError` loudly. See the bump rule in `_protocol.py` lines 79-84. + +10. **Post-snapshot enqueue boundary** — jobs matching the filter that are enqueued *after* the statement's snapshot escape the cancel. Convergence is the caller's responsibility: stop producers before calling `cancel_where`, or issue a second call to catch stragglers. The `BulkCancelResult` counts let the caller detect non-convergence. + +--- + +## Revision log + +### 2026-07-29 — Post-review revision (verdict: NEEDS REWORK — 1 Critical / 4 High / 8 Medium / 7 Low) + +Revised against `.review/spec-review.md` under the standing 1.0.0 design directive: breaking changes are allowed when the result is strictly better, but no hacks, shims, or dual-path compat code; downstream sections describe the documented needs (issues #54/#57 and downstream code comments) and the correct end-state, not preservation of current usage. + +Resolved: + +- **C1 (EPQ race):** bulk-cancel SQL redesigned — status/`cancel_phase` predicates duplicated on the target table in each UPDATE's own WHERE clause (EvalPlanQual re-evaluates only the UPDATE's WHERE, not CTE contents); `cancelled` CTE uses the `FROM prev` pattern from `cancel_pending_scheduled` (`_sql_templates.py:407-415`) to carry the real `prev_status`; residual claim-boundary window documented (job escapes the call, never clobbered). New PG race test `test_pg_cancel_where_does_not_clobber_concurrent_claim` pins the safety property. +- **H1 (circular import):** `BulkCancelResult` defined in `_protocol.py` (next to `ScheduleRecord`), re-exported via `types.py` → `client/__init__.py` → `__init__.py` (same chain as `CancelResult`); stale "pydantic-free" docstring flagged for update in Task 2. +- **H2 (JSON injection):** `reason` serialized in Python via `jsonb_param`, bound as jsonb — never f-string interpolation; pinned by `test_pg_cancel_where_reason_with_quotes`. +- **H3 (silent limit cap):** in-memory `_cancel_where` sanitizes the filter (`limit=2**31`, `cursor=None`, `order_by=None`) before reusing `_list_jobs`; pinned by `test_cancel_where_ignores_filter_limit` (11 jobs, `limit=5` → all 11 cancelled). +- **H4 (fabricated justification):** removed the misquoted "confirmed by the issue author"; Design Decision #57-4 now explicitly reverses the documented exclusion (`jobs-clients.md:789`), analyzes the finalizer-snooze hazard against the verified mechanism (snooze-time `DeadlineExceeded` guard at `_sql_templates.py:254-275`; `sweep_deadline_exceeded` at `_sweeps.py:325+`), and scopes the hazard as opt-in per call. +- **Medium:** M1 event parity specified for both backends; M2 protocol member-count/docstring updates planned (36→37, 33→34); M3 fleet kill switch `sub_job_inherit_tags` on `WorkerSettings` (`settings.py`, not `_bootstrap.py`); M4 batch asymmetry justified as deliberate scoping (Decision #57-7); M5 all examples converted to the valid hyphenated tag charset, with the colon-form explicitly declared a non-goal; M6 deadlock retry (backend-owned, max 3 attempts) + `ORDER BY id` lock ordering + tenant-scale partitioning guidance; M7 downstream section rewritten per the directive (warden corrected to verified "today" + documented-need framing; cennan verified incl. the colon-tag incident; aacrtool marked planned); M8 builder contract states `_list_jobs` re-appends cursor/limit after the shared call. +- **Low:** L1 stale CTE rationale removed; L2 file-structure/task-list mismatches fixed (`_filter_sql.py`, new test files listed; `context.py` note; no `SqlTemplates.cancel_where` detour); L3 Task 9 placeholder replaced with a real merge test (`tagged_pipeline_stage`); L4 batched-NOTIFY statement written + per-job counter parity + listener-based PG test (`test_pg_cancel_where_notify_sent_for_running`); L5 actual `from_state` via `FROM prev`; L6 snapshot-boundary contract documented (docstring + Decision #54-10); L7 `build_filter_conditions(filter)` without the unused `schema` param. + +Design changes chosen under the directive (all breaking-or-behavioral by intent, no shims): deadlock retry lives in the backend (not the client) because the exception is asyncpg-specific and the backend owns the transaction boundary; NOTIFY send lives in `postgres.py` where the `notify_sent` counter is defined (avoids a module cycle); `schedule_to_close`/`start_to_close`/`heartbeat_timeout` are added to the sub-job surface as a deliberate, documented reversal of the prior exclusion. + +Left unresolved (deliberately): tag-charset widening (colon-form tags) — separate change, declared a non-goal; `inherit_tags` for `enqueue_batch` — follow-up spec (Decision #57-7); `snooze_extends_deadline` flag — noted as future work in Decision #57-4. diff --git a/src/taskq/__init__.py b/src/taskq/__init__.py index e18c8f1f..35fbdcb8 100644 --- a/src/taskq/__init__.py +++ b/src/taskq/__init__.py @@ -37,7 +37,7 @@ ScheduleRecord, ) from taskq.batch import BatchCompletionStatus, BatchHandle, EnqueueItem, wait_for_batch -from taskq.client import CancelResult, JobEvent, JobHandle, JobsClient, TaskQ +from taskq.client import BulkCancelResult, CancelResult, JobEvent, JobHandle, JobsClient, TaskQ from taskq.client._enqueuer import SubJobEnqueuer from taskq.connections import ConnFactory, PoolFactory, RedisFactory, WorkerConnections from taskq.context import JobContext @@ -48,6 +48,7 @@ BackpressureError, DependencyCycle, DIError, + EmptyFilterError, IllegalStateTransition, JobFailed, MaxPendingExceededError, @@ -93,6 +94,7 @@ "BackpressureError", "BatchCompletionStatus", "BatchHandle", + "BulkCancelResult", "CancelPhase", "CancelResult", "ConnFactory", @@ -100,6 +102,7 @@ "DIError", "DependencyCycle", "DstStrategy", + "EmptyFilterError", "EnqueueItem", "ErrorReporter", "EventRow", diff --git a/src/taskq/backend/__init__.py b/src/taskq/backend/__init__.py index 9598d32f..e436a030 100644 --- a/src/taskq/backend/__init__.py +++ b/src/taskq/backend/__init__.py @@ -20,6 +20,7 @@ AttemptRow, Backend, BackendDeps, + BulkCancelResult, CancelFlag, DstStrategy, EnqueueArgs, @@ -57,6 +58,7 @@ def __getattr__(name: str) -> object: "AttemptRow", "Backend", "BackendDeps", + "BulkCancelResult", "CancelFlag", "DstStrategy", "EnqueueArgs", diff --git a/src/taskq/backend/_protocol.py b/src/taskq/backend/_protocol.py index dc96005f..eb8ef420 100644 --- a/src/taskq/backend/_protocol.py +++ b/src/taskq/backend/_protocol.py @@ -49,6 +49,7 @@ "AttemptRow", "Backend", "BackendDeps", + "BulkCancelResult", "CancelFlag", "CancelPhase", "DstStrategy", @@ -616,6 +617,35 @@ class ScheduleRecord(BaseModel): metadata: dict[str, object] +class BulkCancelResult(BaseModel): + """Structured outcome of a bulk cancellation request. + + Returned by ``JobsClient.cancel_where()`` so callers can inspect + how many jobs were cancelled directly (pending/scheduled → terminal + 'cancelled') vs how many had cooperative cancel requested (running → + cancel_phase=1). + """ + + model_config = ConfigDict(frozen=True) + + cancelled_directly: int + """Count of pending/scheduled jobs moved straight to terminal 'cancelled'.""" + + cancel_requested: int + """Count of running jobs with cancel_phase=1 set (cooperative cancel).""" + + cancelled_ids: list[UUID] + """IDs of jobs cancelled directly (pending/scheduled → cancelled).""" + + cancel_requested_ids: list[UUID] + """IDs of running jobs with cancel requested.""" + + @property + def total_affected(self) -> int: + """Total jobs affected by the bulk cancel.""" + return self.cancelled_directly + self.cancel_requested + + @dataclass(frozen=True, slots=True) class ErrorInfo: """Structured error information for terminal writes.""" diff --git a/src/taskq/client/__init__.py b/src/taskq/client/__init__.py index 8418f1f5..4fac140a 100644 --- a/src/taskq/client/__init__.py +++ b/src/taskq/client/__init__.py @@ -13,9 +13,10 @@ from taskq.client._handle import JobHandle from taskq.client._jobs import JobsClient from taskq.client._taskq import JobEvent, TaskQ -from taskq.types import CancelResult +from taskq.types import BulkCancelResult, CancelResult __all__ = [ + "BulkCancelResult", "CancelResult", "JobEvent", "JobHandle", diff --git a/src/taskq/exceptions.py b/src/taskq/exceptions.py index 6b0b107e..6b470580 100644 --- a/src/taskq/exceptions.py +++ b/src/taskq/exceptions.py @@ -387,6 +387,24 @@ def __init__(self, schema: str) -> None: ) +class EmptyFilterError(TaskQError): + """Raised when cancel_where is called with a filter that has no predicates. + + A filter with no queue, status, actor, identity_key, batch_id, tags, or + active predicate would match every job in the table — almost certainly + a bug. The guardrail is intentionally loud: the caller must add at least + one predicate or explicitly bypass with ``allow_empty_filter=True``. + """ + + def __init__(self) -> None: + super().__init__( + "cancel_where requires at least one filter predicate " + "(queue, status, actor, identity_key, batch_id, tags, or active); " + "an empty filter would cancel the entire table. " + "Pass allow_empty_filter=True to override this guardrail." + ) + + class ScopedIdempotencyMigrationPendingError(TaskQError): """``idempotency_scope`` was used, but the schema has not yet had ``01.00.03_01_post_idempotency_scope_drop_old_index.sql`` applied. diff --git a/src/taskq/types.py b/src/taskq/types.py index 9673db1b..c1378d38 100644 --- a/src/taskq/types.py +++ b/src/taskq/types.py @@ -1,12 +1,15 @@ """Client-facing result and event-detail types. -``CancelResult`` is the structured return value of ``JobsClient.cancel()`` - ``StateChangeEvent`` is the JSON payload stored in +``CancelResult`` is the structured return value of ``JobsClient.cancel()``. +``BulkCancelResult`` is the structured return value of +``JobsClient.cancel_where()`` (defined in ``backend._protocol`` to avoid +a circular import — see its docstring). +``StateChangeEvent`` is the JSON payload stored in ``job_events.detail`` for rows with ``kind='state_change'``. -These types live here — not in ``taskq.backend`` — so the Backend protocol -remains pydantic-free and the layering contract is enforceable by import -inspection. +``BulkCancelResult`` is re-exported here (not defined) because +``types.py`` imports from ``backend._protocol`` — defining it here would +create a circular import (``_protocol → types → _protocol``). """ from dataclasses import dataclass @@ -14,9 +17,9 @@ from pydantic import BaseModel, ConfigDict -from taskq.backend._protocol import JobId, JobStatus +from taskq.backend._protocol import BulkCancelResult, JobId, JobStatus -__all__ = ["CancelResult", "StateChangeEvent"] +__all__ = ["BulkCancelResult", "CancelResult", "StateChangeEvent"] class CancelResult(BaseModel): diff --git a/tests/test_bulk_cancel_types.py b/tests/test_bulk_cancel_types.py new file mode 100644 index 00000000..d472c11d --- /dev/null +++ b/tests/test_bulk_cancel_types.py @@ -0,0 +1,52 @@ +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from taskq.exceptions import EmptyFilterError, TaskQError +from taskq.types import BulkCancelResult + + +class TestBulkCancelResult: + def test_construction(self) -> None: + ids = [uuid4() for _ in range(3)] + result = BulkCancelResult( + cancelled_directly=2, + cancel_requested=1, + cancelled_ids=ids[:2], + cancel_requested_ids=ids[2:], + ) + assert result.cancelled_directly == 2 + assert result.cancel_requested == 1 + assert result.total_affected == 3 + assert len(result.cancelled_ids) == 2 + assert len(result.cancel_requested_ids) == 1 + + def test_frozen(self) -> None: + result = BulkCancelResult( + cancelled_directly=0, + cancel_requested=0, + cancelled_ids=[], + cancel_requested_ids=[], + ) + with pytest.raises(ValidationError): + result.cancelled_directly = 1 # type: ignore[misc] + + def test_zero_counts(self) -> None: + result = BulkCancelResult( + cancelled_directly=0, + cancel_requested=0, + cancelled_ids=[], + cancel_requested_ids=[], + ) + assert result.total_affected == 0 + + +class TestEmptyFilterError: + def test_is_taskq_error(self) -> None: + assert issubclass(EmptyFilterError, TaskQError) + + def test_message_mentions_guardrail(self) -> None: + err = EmptyFilterError() + assert "allow_empty_filter" in str(err) + assert "filter predicate" in str(err) From a7df046e3fa1331ca56d2020f03ffdce1b70c846 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:32:49 -0700 Subject: [PATCH 05/16] feat: add cancel_where to Backend protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Purely additive — no BACKEND_PROTOCOL_VERSION bump required. A v3 implementation lacks the method and the client raises AttributeError loudly (not silent misbehavior). Updates member count test (36→37) and protocol docstring (33→34 methods total). --- src/taskq/backend/_protocol.py | 21 +++++++++++++++++++-- tests/test_backend_protocol.py | 12 ++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/taskq/backend/_protocol.py b/src/taskq/backend/_protocol.py index eb8ef420..53c493b6 100644 --- a/src/taskq/backend/_protocol.py +++ b/src/taskq/backend/_protocol.py @@ -720,8 +720,8 @@ def dispatcher_pool(self) -> "asyncpg.Pool | None": class Backend(Protocol): """Contract that both PostgresBackend and InMemoryBackend satisfy. - 31 async methods plus two sync methods (``subscribe_wake`` and - ``subscribe_cancel_wake``) (33 methods total) covering enqueue, + 32 async methods plus two sync methods (``subscribe_wake`` and + ``subscribe_cancel_wake``) (34 methods total) covering enqueue, dispatch, heartbeat, terminal writes, attempt history, cancel signals, scheduling / sweeps, read, NOTIFY hook, and schedule CRUD. Method order grouped for review-grep ergonomics. @@ -976,6 +976,23 @@ async def write_cancel_request( reason: str | None, ) -> bool: ... + async def cancel_where( + self, + filter: JobFilter, + reason: str | None, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter* in a set-based operation. + + Pending/scheduled jobs → terminal 'cancelled'. + Running jobs → cancel_phase=1 (cooperative cancel + NOTIFY). + + The filter's ``limit``, ``cursor``, and ``order_by`` fields are + ignored — this is a bulk write, not a paginated read. + + Returns a :class:`BulkCancelResult` with counts and affected IDs. + """ + ... + async def poll_cancel_flags( self, worker_id: UUID, diff --git a/tests/test_backend_protocol.py b/tests/test_backend_protocol.py index 3090dbd4..c8d5b188 100644 --- a/tests/test_backend_protocol.py +++ b/tests/test_backend_protocol.py @@ -215,9 +215,9 @@ def test_isinstance_dict_is_false(self) -> None: class TestMethodCount: - def test_exactly_thirty_six_public_members(self) -> None: + def test_exactly_thirty_seven_public_members(self) -> None: public = [m for m in dir(Backend) if not m.startswith("_")] - assert len(public) == 36, f"Expected 36 public members, got {len(public)}: {public}" + assert len(public) == 37, f"Expected 37 public members, got {len(public)}: {public}" def test_all_member_names_present(self) -> None: expected = { @@ -243,6 +243,7 @@ def test_all_member_names_present(self) -> None: "get_events", "poll_reclaim_events", "write_cancel_request", + "cancel_where", "poll_cancel_flags", "scheduled_to_pending", "deadline_sweep", @@ -262,6 +263,13 @@ def test_all_member_names_present(self) -> None: assert actual == expected +async def test_protocol_has_cancel_where() -> None: + """Backend protocol declares cancel_where.""" + from taskq.backend._protocol import Backend + + assert hasattr(Backend, "cancel_where") + + # ── Return-type annotations ──────────────────────────────────────────── From 166b53bf58bc9a575793327af850961314d5c867 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:37:15 -0700 Subject: [PATCH 06/16] feat: implement cancel_where for InMemoryBackend Bulk cancel for the in-memory backend following the same module-level function pattern as _reads.py, _terminal.py, etc. Sanitizes the filter (limit=2**31, cursor=None, order_by=None) before reusing _list_jobs to prevent silent capping (H3 fix). Skips running jobs already in cooperative cancel (cancel_phase >= 1). Events match single-job write_cancel_request: state_change + cancel_request for pending/ scheduled, cancel_request only for running. --- src/taskq/testing/_cancel_bulk.py | 71 ++++++++ src/taskq/testing/in_memory.py | 9 + tests/test_cancel_where.py | 290 ++++++++++++++++++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 src/taskq/testing/_cancel_bulk.py create mode 100644 tests/test_cancel_where.py diff --git a/src/taskq/testing/_cancel_bulk.py b/src/taskq/testing/_cancel_bulk.py new file mode 100644 index 00000000..db3ac05c --- /dev/null +++ b/src/taskq/testing/_cancel_bulk.py @@ -0,0 +1,71 @@ +"""Bulk cancel implementation for InMemoryBackend. + +Module-level function following the same pattern as ``testing/_reads.py``, +``testing/_terminal.py``, etc. +""" + +from dataclasses import replace as dc_replace +from typing import TYPE_CHECKING +from uuid import UUID + +from taskq.backend._protocol import BulkCancelResult, CancelPhase, JobFilter + +if TYPE_CHECKING: + from taskq.testing.in_memory import InMemoryBackend + +__all__ = ["_cancel_where"] + + +async def _cancel_where( + self: "InMemoryBackend", + filter: JobFilter, + reason: str | None, +) -> BulkCancelResult: + from taskq.testing._reads import _list_jobs + + # Sanitize the filter: cancel_where ignores limit, cursor, and order_by. + # Use a very large limit instead of None because JobFilter.limit is typed + # as int (not int | None) with a __post_init__ guard against negatives. + # cursor=None disables keyset slicing. order_by=None selects the default + # priority/scheduled_at/id sort, which is harmless for cancel. + sanitized = dc_replace(filter, limit=2**31, cursor=None, order_by=None) + rows = await _list_jobs(self, sanitized) + + cancelled_ids: list[UUID] = [] + cancel_requested_ids: list[UUID] = [] + + for row in rows: + if row.status in ("pending", "scheduled"): + now = self._clock.now() + self._jobs[row.id] = dc_replace( + row, + status="cancelled", + finished_at=now, + ) + self._append_state_change_event( + job_id=row.id, + from_state=row.status, + to_state="cancelled", + now=now, + ) + self._append_cancel_request_event(row.id, now, reason) + cancelled_ids.append(row.id) + + elif row.status == "running" and row.cancel_phase == CancelPhase.NONE: + now = self._clock.now() + self._jobs[row.id] = dc_replace( + row, + cancel_requested_at=now, + cancel_phase=CancelPhase.COOPERATIVE, + ) + self._append_cancel_request_event(row.id, now, reason) + for event in self._cancel_wake_subscribers: + event.set() + cancel_requested_ids.append(row.id) + + return BulkCancelResult( + cancelled_directly=len(cancelled_ids), + cancel_requested=len(cancel_requested_ids), + cancelled_ids=cancelled_ids, + cancel_requested_ids=cancel_requested_ids, + ) diff --git a/src/taskq/testing/in_memory.py b/src/taskq/testing/in_memory.py index 56d663fc..c5b3d58a 100644 --- a/src/taskq/testing/in_memory.py +++ b/src/taskq/testing/in_memory.py @@ -36,6 +36,7 @@ BACKEND_PROTOCOL_VERSION, AttemptOutcome, AttemptRow, + BulkCancelResult, CancelFlag, CancelPhase, EnqueueArgs, @@ -51,6 +52,7 @@ ) from taskq.backend.clock import Clock from taskq.retry import OnRetryExhausted, OnSuccess, RetryClassifierHook, RetryPolicy +from taskq.testing._cancel_bulk import _cancel_where from taskq.testing._dispatch import _dispatch_batch, _set_queue_mode from taskq.testing._enqueue import ( _enqueue, @@ -622,6 +624,13 @@ async def poll_cancel_flags( and row.locked_by_worker == worker_id ] + async def cancel_where( + self, + filter: JobFilter, + reason: str | None, + ) -> BulkCancelResult: + return await _cancel_where(self, filter, reason) + # ── Admin operations ────────────────────────────────────────────── async def retry_job(self, job_id: JobId) -> bool: diff --git a/tests/test_cancel_where.py b/tests/test_cancel_where.py new file mode 100644 index 00000000..0865579a --- /dev/null +++ b/tests/test_cancel_where.py @@ -0,0 +1,290 @@ +"""Unit tests for InMemoryBackend.cancel_where (bulk cancel).""" + +from dataclasses import replace +from datetime import UTC, datetime +from uuid import uuid4 + +from taskq.backend._protocol import CancelPhase, JobFilter +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend +from taskq.testing.jobs import make_enqueue_args +from taskq.types import BulkCancelResult + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + + +async def test_cancel_where_pending_jobs() -> None: + """cancel_where moves pending jobs straight to 'cancelled'.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + for _ in range(3): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme", "run-001"), scheduled_at=_NOW)) + for _ in range(2): + await backend.enqueue(make_enqueue_args(tags=("tenant-other",), scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 3 + assert result.cancel_requested == 0 + assert result.total_affected == 3 + assert len(result.cancelled_ids) == 3 + + remaining = await backend.list_jobs(JobFilter(tags=("tenant-other",))) + assert len(remaining) == 2 + assert all(r.status == "pending" for r in remaining) + + +async def test_cancel_where_running_jobs_cooperative() -> None: + """cancel_where sets cancel_phase=1 for running jobs (cooperative).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + backend._jobs[row.id] = replace( + backend._jobs[row.id], status="running", locked_by_worker=uuid4() + ) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 1 + assert len(result.cancel_requested_ids) == 1 + + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + assert updated.cancel_phase == CancelPhase.COOPERATIVE + + +async def test_cancel_where_mixed_statuses() -> None: + """cancel_where handles both pending and running in one call.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + for _ in range(2): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + args3 = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row3 = await backend.enqueue(args3) + backend._jobs[row3.id] = replace( + backend._jobs[row3.id], status="running", locked_by_worker=uuid4() + ) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 2 + assert result.cancel_requested == 1 + assert result.total_affected == 3 + + +async def test_cancel_where_no_matches_returns_zero() -> None: + """cancel_where with a filter matching nothing returns zero counts.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(tags=("nonexistent",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 0 + assert result.total_affected == 0 + + +async def test_cancel_where_already_cancelled_not_affected() -> None: + """Already-terminal jobs are not re-cancelled.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + backend._jobs[row.id] = replace(backend._jobs[row.id], status="cancelled", finished_at=_NOW) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.total_affected == 0 + + +async def test_cancel_where_filter_by_batch_id() -> None: + """cancel_where works with batch_id filter.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + bid = uuid4() + for _ in range(3): + args = make_enqueue_args( + tags=("tenant-acme",), + scheduled_at=_NOW, + metadata={"batch_id": str(bid)}, + ) + await backend.enqueue(args) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW, metadata={"batch_id": str(uuid4())})) + + result = await backend.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 3 + + +async def test_cancel_where_filter_by_queue_and_actor() -> None: + """cancel_where works with queue and actor filters.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + await backend.enqueue(make_enqueue_args(queue="default", actor="worker-a", scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(queue="default", actor="worker-b", scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(queue="priority", actor="worker-a", scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(queue="default", actor="worker-a"), + reason="abort", + ) + + assert result.cancelled_directly == 1 + + +async def test_cancel_where_active_filter() -> None: + """cancel_where with active=True targets only non-terminal jobs.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + args3 = make_enqueue_args(scheduled_at=_NOW) + row3 = await backend.enqueue(args3) + backend._jobs[row3.id] = replace(backend._jobs[row3.id], status="succeeded", finished_at=_NOW) + + result = await backend.cancel_where( + JobFilter(active=True), + reason="drain", + ) + + assert result.cancelled_directly == 2 + + +async def test_cancel_where_ignores_filter_limit() -> None: + """cancel_where cancels ALL matching jobs even when filter.limit is small. + + Guards against the H3 bug: _list_jobs applies filters.limit (default + 100). If _cancel_where reuses _list_jobs without sanitizing the filter, + a caller passing JobFilter(limit=5, tags=...) would cancel only 5 jobs. + """ + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + for _ in range(11): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",), limit=5), + reason="offboard", + ) + + assert result.cancelled_directly == 11 + assert result.total_affected == 11 + + +async def test_cancel_where_already_cooperative_cancel_not_recounted() -> None: + """A running job already in cooperative cancel (cancel_phase=1) is + NOT double-counted by a subsequent cancel_where call.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + worker_id = uuid4() + backend._jobs[row.id] = replace( + backend._jobs[row.id], + status="running", + locked_by_worker=worker_id, + cancel_phase=CancelPhase.COOPERATIVE, + cancel_requested_at=_NOW, + ) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancel_requested == 0 + assert result.total_affected == 0 + + +async def test_cancel_where_events_inserted() -> None: + """cancel_where inserts state_change + cancel_request events for + pending jobs, matching single-job write_cancel_request semantics.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + + await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + events = await backend.get_events(row.id) + kinds = [e.kind for e in events] + assert "state_change" in kinds + assert "cancel_request" in kinds + sc = [e for e in events if e.kind == "state_change"] + assert sc[0].detail["from_state"] in ("pending", "scheduled") + assert sc[0].detail["to_state"] == "cancelled" + + +async def test_cancel_where_running_event_only_cancel_request() -> None: + """Running jobs get only cancel_request (no state_change).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + backend._jobs[row.id] = replace( + backend._jobs[row.id], status="running", locked_by_worker=uuid4() + ) + + await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + events = await backend.get_events(row.id) + kinds = [e.kind for e in events] + assert "cancel_request" in kinds + assert "state_change" not in kinds + + +async def test_cancel_where_wakes_cancel_subscribers() -> None: + """cancel_where sets cancel wake events for running jobs.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + backend._jobs[row.id] = replace( + backend._jobs[row.id], status="running", locked_by_worker=uuid4() + ) + + async with backend.subscribe_cancel_wake() as event: + assert not event.is_set() + await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + assert event.is_set() + + +async def test_cancel_where_returns_bulk_cancel_result_type() -> None: + """cancel_where returns a BulkCancelResult instance.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + await backend.enqueue(make_enqueue_args(tags=("x",), scheduled_at=_NOW)) + + result = await backend.cancel_where(JobFilter(tags=("x",)), reason=None) + + assert isinstance(result, BulkCancelResult) From 8c26d77444db33abbe865e0de9e32f950e0af34a Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:53:06 -0700 Subject: [PATCH 07/16] feat: implement cancel_where for PostgresBackend + JobsClient/TaskQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgresBackend: single CTE-based SQL statement with EPQ-safe predicates duplicated in each UPDATE's WHERE clause. Events inserted via executemany in the same transaction. Batched post-commit NOTIFY for running jobs. Deadlock retry (3 attempts, jittered backoff) in the backend, not the client (exception type is asyncpg-specific). JobsClient: cancel_where with EmptyFilterError guardrail for empty filters (allow_empty_filter=True bypasses). Counter incremented once per call. Schema errors translated to SchemaNotMigratedError. TaskQ: thin delegate to JobsClient.cancel_where. NotifyTarget NamedTuple prevents positional-index confusion (2P7). All user values use $N positional binding — reason serialized via jsonb_param, never f-string interpolation (H2 fix). --- pyproject.toml | 2 + src/taskq/backend/_cancel_bulk.py | 152 ++++++++++++++++++++++ src/taskq/backend/postgres.py | 38 ++++++ src/taskq/client/_jobs.py | 48 ++++++- src/taskq/client/_taskq.py | 14 +- tests/test_cancel_where_client.py | 172 +++++++++++++++++++++++++ tests/test_cancel_where_pg.py | 207 ++++++++++++++++++++++++++++++ 7 files changed, 630 insertions(+), 3 deletions(-) create mode 100644 src/taskq/backend/_cancel_bulk.py create mode 100644 tests/test_cancel_where_client.py create mode 100644 tests/test_cancel_where_pg.py diff --git a/pyproject.toml b/pyproject.toml index db397dc6..7a99a3ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,6 +202,7 @@ ignore = [ "src/taskq/backend/_terminal.py" = ["S608", "SIM117"] "src/taskq/backend/_enqueue.py" = ["S608", "SIM117"] "src/taskq/backend/_reads.py" = ["S608"] +"src/taskq/backend/_cancel_bulk.py" = ["S608", "SIM117", "S311"] "src/taskq/backend/_dispatch.py" = ["S608", "SIM117"] "src/taskq/backend/_sql_templates.py" = ["S608"] # SIM117: pool.acquire() and conn.transaction() cannot be flattened into a @@ -257,6 +258,7 @@ ignore = [ # Same rationale — schema name validated via PostgresBackend constructor; # all user-supplied values use $N parameter binding. "tests/test_backend_equivalence.py" = ["S608"] +"tests/test_cancel_where_pg.py" = ["S608"] # Same rationale — schema name validated against _IDENT_RE; all # user-supplied values use $N parameter binding. "tests/test_heartbeat_integration.py" = ["S608"] diff --git a/src/taskq/backend/_cancel_bulk.py b/src/taskq/backend/_cancel_bulk.py new file mode 100644 index 00000000..89eb0951 --- /dev/null +++ b/src/taskq/backend/_cancel_bulk.py @@ -0,0 +1,152 @@ +"""Bulk cancel SQL implementation for PostgresBackend. + +Module-level function following the same pattern as ``_reads.py``, +``_terminal.py``, etc. The SQL uses two CTEs in a single statement +with EPQ-safe predicates duplicated in each UPDATE's WHERE clause. +Events are inserted via ``executemany`` within the same transaction. +NOTIFY is sent by the caller (``PostgresBackend.cancel_where``) after +commit because the ``taskq.cancel.notify_sent`` counter lives in +``postgres.py``. +""" + +import asyncio +import random +from typing import TYPE_CHECKING, NamedTuple +from uuid import UUID + +import asyncpg + +from taskq.backend._filter_sql import build_filter_conditions +from taskq.backend._protocol import BulkCancelResult, JobFilter +from taskq.backend._records import jsonb_param +from taskq.backend._sql_templates import SqlTemplates + +if TYPE_CHECKING: + import asyncpg + +__all__ = ["NotifyTarget", "_cancel_where"] + + +class NotifyTarget(NamedTuple): + """A running job that needs a post-commit NOTIFY.""" + + job_id: UUID + worker_id: UUID + + +async def _cancel_where( + pool: "asyncpg.Pool", + schema: str, + sql: SqlTemplates, + filter: JobFilter, + reason: str | None, +) -> tuple[BulkCancelResult, list[NotifyTarget]]: + filter_sql = build_filter_conditions(filter) + conditions_str = " AND ".join(filter_sql.conditions) if filter_sql.conditions else "TRUE" + params = list(filter_sql.params) + + cancel_sql = f""" + WITH matching AS ( + SELECT id, status, locked_by_worker + FROM "{schema}".jobs + WHERE {conditions_str} + ORDER BY id + ), + cancelled AS ( + UPDATE "{schema}".jobs AS j + SET status = 'cancelled', finished_at = clock_timestamp() + FROM ( + SELECT id, status AS prev_status + FROM matching + WHERE status IN ('pending', 'scheduled') + ) AS prev + WHERE j.id = prev.id + AND j.status IN ('pending', 'scheduled') + RETURNING j.id, prev.prev_status + ), + cancel_requested AS ( + UPDATE "{schema}".jobs AS j + SET cancel_requested_at = now(), cancel_phase = 1 + WHERE j.id IN ( + SELECT id FROM matching + WHERE status = 'running' AND cancel_phase = 0 + ) + AND j.status = 'running' AND j.cancel_phase = 0 + RETURNING j.id, j.locked_by_worker + ) + SELECT + (SELECT count(*)::int FROM cancelled) AS cancelled_directly, + (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, + (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, + (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, + (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, + (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers + """ + + cr_detail = jsonb_param({"reason": reason} if reason is not None else {}) + + cancelled_ids: list[UUID] = [] + cancel_requested_ids: list[UUID] = [] + notify_targets: list[NotifyTarget] = [] + + for attempt in range(3): + try: + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow(cancel_sql, *params) + + assert row is not None + cancelled_ids = list(row["cancelled_ids"] or []) + cancel_requested_ids = list(row["cancel_requested_ids"] or []) + prev_statuses: dict[UUID, str] = dict( + zip(cancelled_ids, row["cancelled_prev_statuses"] or [], strict=True) + ) + notify_targets = [ + NotifyTarget(job_id=jid, worker_id=wid) + for jid, wid in zip( + cancel_requested_ids, + row["cancel_requested_workers"] or [], + strict=True, + ) + if wid is not None + ] + + if cancelled_ids: + await conn.executemany( + sql.insert_event, + [ + ( + jid, + "state_change", + jsonb_param( + { + "from_state": prev_statuses[jid], + "to_state": "cancelled", + } + ), + ) + for jid in cancelled_ids + ], + ) + await conn.executemany( + sql.insert_event, + [(jid, "cancel_request", cr_detail) for jid in cancelled_ids], + ) + if cancel_requested_ids: + await conn.executemany( + sql.insert_event, + [(jid, "cancel_request", cr_detail) for jid in cancel_requested_ids], + ) + break + except asyncpg.DeadlockDetectedError: + if attempt == 2: + raise + await asyncio.sleep(0.1 * (2**attempt) + random.random() * 0.05) + + result = BulkCancelResult( + cancelled_directly=len(cancelled_ids), + cancel_requested=len(cancel_requested_ids), + cancelled_ids=cancelled_ids, + cancel_requested_ids=cancel_requested_ids, + ) + return result, notify_targets diff --git a/src/taskq/backend/postgres.py b/src/taskq/backend/postgres.py index 4d0f85e2..82a5f39c 100644 --- a/src/taskq/backend/postgres.py +++ b/src/taskq/backend/postgres.py @@ -23,6 +23,7 @@ import structlog from taskq._json import dumps_str +from taskq.backend._cancel_bulk import _cancel_where from taskq.backend._dispatch import ( _dispatch_batch as _dispatch, ) @@ -41,6 +42,7 @@ AttemptOutcome, AttemptRow, BackendDeps, + BulkCancelResult, CancelFlag, ConnLike, EnqueueArgs, @@ -619,6 +621,42 @@ async def poll_cancel_flags( for rec in recs ] + async def cancel_where( + self, + filter: JobFilter, + reason: str | None, + ) -> BulkCancelResult: + result, notify_targets = await _cancel_where( + self._worker_pool, self._schema_name, self._sql, filter, reason + ) + if notify_targets: + channels: list[str] = [] + payloads: list[str] = [] + for target in notify_targets: + payload = dumps_str( + { + "type": "cancel", + "job_id": str(target.job_id), + "worker_id": str(target.worker_id), + } + ) + channels.extend( + [ + events_channel(self._schema_name), + worker_channel(self._schema_name, str(target.worker_id)), + ] + ) + payloads.extend([payload, payload]) + async with self._worker_pool.acquire() as notify_conn: + await notify_conn.execute( + "SELECT pg_notify(channel, payload) " + "FROM unnest($1::text[], $2::text[]) AS t(channel, payload)", + channels, + payloads, + ) + _cancel_notify_sent_counter.add(len(notify_targets), {"schema": self._schema_name}) + return result + # ── Admin operations ────────────────────────────────────────────── async def retry_job(self, job_id: JobId) -> bool: diff --git a/src/taskq/client/_jobs.py b/src/taskq/client/_jobs.py index 59661185..09402324 100644 --- a/src/taskq/client/_jobs.py +++ b/src/taskq/client/_jobs.py @@ -45,8 +45,8 @@ from taskq.client._args import build_batch_args, build_enqueue_args, enqueue_span from taskq.client._capacity import DEFAULT_CAPACITY_CACHE_TTL, ActorCapacityCache from taskq.client._handle import JobHandle -from taskq.exceptions import PayloadValidationError, SchemaNotMigratedError -from taskq.types import CancelResult +from taskq.exceptions import EmptyFilterError, PayloadValidationError, SchemaNotMigratedError +from taskq.types import BulkCancelResult, CancelResult if TYPE_CHECKING: import asyncpg @@ -733,6 +733,50 @@ async def cancel( ) return result + async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter* in a single set-based operation. + + Pending/scheduled jobs are moved straight to terminal 'cancelled' + (no running actor to cooperate with). Running jobs get + ``cancel_phase=1`` set (cooperative cancel) — the worker's + heartbeat-driven cancel controller observes the phase change and + sets the in-process ``cancel_event``. + + **Guardrail:** a filter with no predicates (no queue, status, + actor, identity_key, batch_id, tags, or active) is rejected with + :class:`EmptyFilterError` unless ``allow_empty_filter=True`` is + passed. + + **Filter fields used:** ``queue``, ``status``, ``actor``, + ``identity_key``, ``batch_id``, ``tags``, ``active``. The + ``limit``, ``cursor``, and ``order_by`` fields are ignored. + + Returns a :class:`BulkCancelResult` with counts and affected IDs. + """ + from taskq.obs import record_cancel_requested + + if not allow_empty_filter and ( + filter.queue is None + and filter.status is None + and filter.actor is None + and filter.identity_key is None + and filter.batch_id is None + and (filter.tags is None or len(filter.tags) == 0) + and filter.active is None + ): + raise EmptyFilterError() + + record_cancel_requested() + + with self._translate_schema_errors(): + return await self._backend.cancel_where(filter, reason) + # ── Schedule CRUD ──────────────────────────────────────────────────── async def create_schedule[P: BaseModel, R: BaseModel | None]( diff --git a/src/taskq/client/_taskq.py b/src/taskq/client/_taskq.py index fc21d32a..15308820 100644 --- a/src/taskq/client/_taskq.py +++ b/src/taskq/client/_taskq.py @@ -72,7 +72,7 @@ async def create_task(payload: MyPayload): from taskq.constants import RECLAIM_EVENT_VISIBILITY_DELAY, progress_channel, wake_channel from taskq.cron import ScheduleHandle from taskq.progress._events import ProgressEvent -from taskq.types import CancelResult +from taskq.types import BulkCancelResult, CancelResult __all__ = ["EventRow", "JobEvent", "TaskQ"] @@ -413,6 +413,18 @@ async def cancel( """Request cancellation of a job. Raises :class:`KeyError` if not found.""" return await self._require_open().cancel(job_id, reason) + async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, + ) -> BulkCancelResult: + """Cancel all jobs matching *filter*. See :meth:`JobsClient.cancel_where`.""" + return await self._require_open().cancel_where( + filter, reason, allow_empty_filter=allow_empty_filter + ) + # ── Schedule operations ───────────────────────────────────────────────── async def create_schedule[P: BaseModel, R: BaseModel | None]( diff --git a/tests/test_cancel_where_client.py b/tests/test_cancel_where_client.py new file mode 100644 index 00000000..c27ca7bf --- /dev/null +++ b/tests/test_cancel_where_client.py @@ -0,0 +1,172 @@ +"""Client-layer unit tests for JobsClient.cancel_where and TaskQ.cancel_where.""" + +from datetime import UTC, datetime + +import pytest +from opentelemetry.sdk.metrics._internal.point import NumberDataPoint +from opentelemetry.sdk.metrics.export import InMemoryMetricReader, Metric + +from taskq.backend._protocol import JobFilter +from taskq.client._jobs import JobsClient +from taskq.exceptions import EmptyFilterError +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend +from taskq.testing.jobs import make_enqueue_args +from taskq.types import BulkCancelResult + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + + +@pytest.fixture +def otel_requested_reader(monkeypatch: pytest.MonkeyPatch) -> InMemoryMetricReader: + """Per-test OTel meter isolation for the cancel-requested counter.""" + + from opentelemetry.sdk.metrics import MeterProvider + + import taskq.obs as obs_mod + import taskq.obs._otel as otel_mod + + reader = InMemoryMetricReader() + new_provider = MeterProvider(metric_readers=[reader]) + new_meter = new_provider.get_meter(obs_mod.INSTRUMENTATION_NAME, otel_mod._version()) + + monkeypatch.setattr( + otel_mod, + "_cancellation_requested", + new_meter.create_counter("taskq.cancellation.requested"), + ) + + return reader + + +def _collect_metrics(reader: InMemoryMetricReader) -> list[Metric]: + md = reader.get_metrics_data() + assert md is not None + results: list[Metric] = [] + for rm in md.resource_metrics: + for sm in rm.scope_metrics: + results.extend(sm.metrics) + return results + + +def _data_points(reader: InMemoryMetricReader, metric_name: str) -> list[NumberDataPoint]: + for m in _collect_metrics(reader): + if m.name == metric_name: + return list(m.data.data_points) # type: ignore[return-value] # Why: counter metrics always produce NumberDataPoint instances. + return [] + + +async def test_client_cancel_where_with_tags() -> None: + """JobsClient.cancel_where cancels jobs by tag filter.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + for _ in range(3): + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert isinstance(result, BulkCancelResult) + assert result.cancelled_directly == 3 + + +async def test_client_cancel_where_empty_filter_raises() -> None: + """Empty filter (no predicates) raises EmptyFilterError.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + with pytest.raises(EmptyFilterError, match="filter predicate"): + await client.cancel_where(JobFilter(), reason="oops") + + +async def test_client_cancel_where_empty_filter_override() -> None: + """allow_empty_filter=True bypasses the guardrail.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(), + reason="drain all", + allow_empty_filter=True, + ) + + assert result.cancelled_directly == 2 + + +async def test_client_cancel_where_with_status_filter() -> None: + """Status filter alone is a valid predicate (not empty).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(status="pending"), + reason="drain pending", + ) + + assert result.cancelled_directly == 1 + + +async def test_client_cancel_where_with_active_filter() -> None: + """Active filter alone is a valid predicate (not empty).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) + + result = await client.cancel_where( + JobFilter(active=True), + reason="drain active", + ) + + assert result.cancelled_directly == 1 + + +async def test_client_cancel_where_increments_counter( + otel_requested_reader: InMemoryMetricReader, +) -> None: + """cancel_where increments taskq.cancellation.requested once.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + + await client.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + dps = _data_points(otel_requested_reader, "taskq.cancellation.requested") + assert len(dps) >= 1 + assert dps[0].value == 1 + + +async def test_client_cancel_where_translates_schema_errors() -> None: + """cancel_where wraps UndefinedTableError in SchemaNotMigratedError.""" + from taskq.exceptions import SchemaNotMigratedError + + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend, settings=type("S", (), {"schema_name": "test_schema"})()) + + await backend.enqueue(make_enqueue_args(tags=("x",), scheduled_at=_NOW)) + + import asyncpg + + original = backend.cancel_where + + async def raise_undefined(*args: object, **kwargs: object) -> None: + raise asyncpg.exceptions.UndefinedTableError("relation does not exist") + + backend.cancel_where = raise_undefined # type: ignore[method-assign] + try: + with pytest.raises(SchemaNotMigratedError): + await client.cancel_where(JobFilter(tags=("x",)), reason="test") + finally: + backend.cancel_where = original # type: ignore[method-assign] diff --git a/tests/test_cancel_where_pg.py b/tests/test_cancel_where_pg.py new file mode 100644 index 00000000..5bd3ca2a --- /dev/null +++ b/tests/test_cancel_where_pg.py @@ -0,0 +1,207 @@ +"""Integration tests for PostgresBackend.cancel_where.""" + +import asyncio +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from taskq.backend._protocol import Backend, JobFilter +from taskq.testing.fixtures import JobsApp +from taskq.testing.jobs import make_enqueue_args + +_NOW = datetime(2026, 1, 1, tzinfo=UTC) + + +@pytest.mark.integration +class TestCancelWherePostgres: + async def test_pg_cancel_where_pending(self, backend_pair: Backend) -> None: + """PostgresBackend.cancel_where cancels pending jobs by tag.""" + for _ in range(3): + await backend_pair.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) + await backend_pair.enqueue(make_enqueue_args(tags=("other",), scheduled_at=_NOW)) + + result = await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 3 + assert result.cancel_requested == 0 + + remaining = await backend_pair.list_jobs(JobFilter(tags=("other",))) + assert len(remaining) == 1 + assert remaining[0].status in ("pending", "scheduled") + + async def test_pg_cancel_where_events_inserted(self, backend_pair: Backend) -> None: + """cancel_where inserts job_events for cancelled jobs.""" + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend_pair.enqueue(args) + + await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="test", + ) + + events = await backend_pair.get_events(row.id) + kinds = [e.kind for e in events] + assert "state_change" in kinds + assert "cancel_request" in kinds + sc = [e for e in events if e.kind == "state_change"] + assert sc[0].detail.get("from_state") in ("pending", "scheduled") + assert sc[0].detail.get("to_state") == "cancelled" + + async def test_pg_cancel_where_reason_with_quotes(self, backend_pair: Backend) -> None: + """Reason containing double-quotes does not cause DataError.""" + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend_pair.enqueue(args) + + result = await backend_pair.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason='offboard "tenant-acme" \\ done', + ) + + assert result.cancelled_directly == 1 + events = await backend_pair.get_events(row.id) + cr = [e for e in events if e.kind == "cancel_request"] + assert cr[0].detail.get("reason") == 'offboard "tenant-acme" \\ done' + + async def test_pg_cancel_where_batch_id_filter(self, backend_pair: Backend) -> None: + """cancel_where works with batch_id filter on Postgres.""" + bid = uuid4() + for _ in range(3): + await backend_pair.enqueue( + make_enqueue_args( + scheduled_at=_NOW, + metadata={"batch_id": str(bid)}, + ) + ) + + result = await backend_pair.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 3 + + async def test_pg_cancel_where_running_cooperative( + self, + clean_jobs_app: JobsApp, + ) -> None: + """cancel_where sets cancel_phase=1 for running jobs on Postgres.""" + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + + worker_id = uuid4() + async with deps.worker_pool.acquire() as conn: + await conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + result = await backend.cancel_where( + JobFilter(tags=("tenant-acme",)), + reason="offboard", + ) + + assert result.cancelled_directly == 0 + assert result.cancel_requested == 1 + + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + assert updated.cancel_phase == 1 + + events = await backend.get_events(row.id) + kinds = [e.kind for e in events] + assert "cancel_request" in kinds + assert "state_change" not in kinds + + async def test_pg_cancel_where_does_not_clobber_concurrent_claim( + self, + clean_jobs_app: JobsApp, + ) -> None: + """EPQ regression: a job claimed while cancel_where executes must + NOT be overwritten to terminal 'cancelled'.""" + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + worker_id = uuid4() + + claim_conn = await deps.worker_pool.acquire() + try: + claim_tx = claim_conn.transaction() + await claim_tx.start() + await claim_conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + cancel_task = asyncio.create_task( + backend.cancel_where(JobFilter(tags=("tenant-acme",)), reason="offboard") + ) + await asyncio.sleep(0.2) + await claim_tx.commit() + result = await cancel_task + + assert result.cancelled_directly == 0 + updated = await backend.get(row.id) + assert updated is not None + assert updated.status == "running" + finally: + await deps.worker_pool.release(claim_conn) + + async def test_pg_cancel_where_notify_sent_for_running( + self, + clean_jobs_app: JobsApp, + pg_dsn: str, + ) -> None: + """Batched NOTIFY fires on the fleet and per-worker channels.""" + import asyncpg + + from taskq.constants import events_channel, worker_channel + + backend = clean_jobs_app.backend + deps = clean_jobs_app.deps + schema = deps.settings.schema_name + + args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) + row = await backend.enqueue(args) + worker_id = uuid4() + async with deps.worker_pool.acquire() as conn: + await conn.execute( + f'UPDATE "{schema}".jobs ' + f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", + worker_id, + row.id, + ) + + received: list[str] = [] + listen_conn = await asyncpg.connect(pg_dsn) + try: + await listen_conn.add_listener( + events_channel(schema), + lambda _c, _p, _ch, payload: received.append(payload), + ) + await listen_conn.add_listener( + worker_channel(schema, str(worker_id)), + lambda _c, _p, _ch, payload: received.append(payload), + ) + result = await backend.cancel_where(JobFilter(tags=("tenant-acme",)), reason="offboard") + assert result.cancel_requested == 1 + await asyncio.sleep(0.3) + finally: + await listen_conn.close() + + assert len(received) == 2 From dfca461d8008d5cde6e6ec48b5f6207e89dc398c Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 16:59:58 -0700 Subject: [PATCH 08/16] feat: add tags and missing fields to SubJobEnqueuer.enqueue() Add tags, inherit_tags, schedule_to_close, start_to_close, and heartbeat_timeout parameters to SubJobEnqueuer.enqueue(). Parent tags are propagated via a contextvars.ContextVar set by the consumer before actor invocation -- asyncio Tasks copy the context, so concurrent consumers each see their own parent tags. Tag inheritance defaults to True with a fleet-level kill switch (sub_job_inherit_tags worker setting, TASKQ_SUB_JOB_INHERIT_TAGS env var). When disabled, the consumer skips set_parent_tags() and the ContextVar remains at its () default -- identical to pre-upgrade behavior. _resolve_tags handles the tags=[] edge case intuitively: an empty list means 'no additional tags', so parent tags are still inherited (2P1 review feedback). --- src/taskq/client/_enqueuer.py | 57 ++++++- src/taskq/settings.py | 8 + src/taskq/worker/_consumer.py | 7 +- src/taskq/worker/run.py | 5 +- tests/test_sub_job_tags.py | 283 ++++++++++++++++++++++++++++++++++ 5 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 tests/test_sub_job_tags.py diff --git a/src/taskq/client/_enqueuer.py b/src/taskq/client/_enqueuer.py index e1e9c32b..60c35014 100644 --- a/src/taskq/client/_enqueuer.py +++ b/src/taskq/client/_enqueuer.py @@ -9,6 +9,7 @@ from __future__ import annotations +import contextvars from collections.abc import Mapping, Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, cast @@ -39,10 +40,24 @@ from taskq.actor import ActorRef -__all__ = ["SubJobEnqueuer"] +__all__ = ["SubJobEnqueuer", "_parent_tags_var", "set_parent_tags"] _log: structlog.stdlib.BoundLogger = structlog.get_logger(__name__) +_parent_tags_var: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar( + "taskq_parent_tags", + default=(), +) + + +def set_parent_tags(tags: tuple[str, ...]) -> contextvars.Token[tuple[str, ...]]: + """Set the parent job's tags for sub-job tag inheritance. + + Called by the consumer before actor invocation. The returned token + can be used to reset the context after the actor completes. + """ + return _parent_tags_var.set(tags) + class SubJobEnqueuer: """Enqueue sub-jobs from within an actor body. @@ -90,6 +105,11 @@ async def enqueue[P: BaseModel, R: BaseModel | None]( unique_for: timedelta | None = None, unique_states: tuple[JobStatus, ...] | None = None, max_pending: int | None = None, + tags: list[str] | None = None, + inherit_tags: bool = True, + schedule_to_close: datetime | None = None, + start_to_close: timedelta | None = None, + heartbeat_timeout: timedelta | None = None, ) -> JobHandle[R]: """Enqueue a sub-job. ``max_pending`` is a per-call limit resolved against the operator-owned stored cap and the ``@actor(...)`` @@ -113,6 +133,7 @@ async def enqueue[P: BaseModel, R: BaseModel | None]( actor_ref.max_pending, per_call=max_pending, ) + resolved_tags = self._resolve_tags(tags, inherit_tags) args = build_enqueue_args( actor_ref, payload, @@ -125,6 +146,10 @@ async def enqueue[P: BaseModel, R: BaseModel | None]( idempotency_scope=idempotency_scope, trace_id=extracted_trace_id, span_id=extracted_span_id, + tags=resolved_tags, + schedule_to_close=schedule_to_close, + start_to_close=start_to_close, + heartbeat_timeout=heartbeat_timeout, unique_for=unique_for, unique_states=unique_states, max_pending=effective_max_pending, @@ -140,6 +165,36 @@ async def enqueue[P: BaseModel, R: BaseModel | None]( client=None, ) + def _resolve_tags( + self, + tags: list[str] | None, + inherit_tags: bool, + ) -> list[str] | None: + """Resolve tags with parent inheritance. + + Returns a list suitable for build_enqueue_args, or None for empty. + """ + parent_tags = _parent_tags_var.get() if inherit_tags else () + + if tags is None: + if parent_tags: + return list(parent_tags) + return None + + if not inherit_tags or not parent_tags: + return tags + + if not tags: + return list(parent_tags) + + seen: set[str] = set(parent_tags) + merged = list(parent_tags) + for tag in tags: + if tag not in seen: + seen.add(tag) + merged.append(tag) + return merged + def _resolve_connection( self, connection: asyncpg.Connection | None, diff --git a/src/taskq/settings.py b/src/taskq/settings.py index 2fd2187b..600f2485 100644 --- a/src/taskq/settings.py +++ b/src/taskq/settings.py @@ -357,6 +357,14 @@ class WorkerSettings(TaskQSettings): ) # ── Pool sizes ───────────────────────────────────────────────────── + sub_job_inherit_tags: bool = Field( + default=True, + description=( + "When false, sub-jobs enqueued via ctx.jobs.enqueue() do not inherit " + "the parent job's tags (pre-1.0 behavior). Fleet-level kill switch " + "for the inherit_tags=True default; env var TASKQ_SUB_JOB_INHERIT_TAGS." + ), + ) dispatcher_pool_size: int = Field( default=4, ge=1, diff --git a/src/taskq/worker/_consumer.py b/src/taskq/worker/_consumer.py index a851bf87..fe85cd42 100644 --- a/src/taskq/worker/_consumer.py +++ b/src/taskq/worker/_consumer.py @@ -29,7 +29,7 @@ JobRow, ) from taskq.backend.clock import Clock -from taskq.client._enqueuer import SubJobEnqueuer +from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags from taskq.constants import MAX_RESULT_BYTES from taskq.context import JobContext from taskq.exceptions import ( @@ -353,6 +353,9 @@ async def consume_one_job( _buf = _ProgressBuffer(job_id=job.id, base_seq=job.progress_seq) _progress_buffers[job.id] = _buf + _inherit = _effective_settings is None or _effective_settings.sub_job_inherit_tags + _parent_tags_token = set_parent_tags(tuple(job.tags)) if _inherit else None + try: validated_payload = ( validated_payload @@ -567,6 +570,8 @@ async def consume_one_job( ) finally: + if _parent_tags_token is not None: + _parent_tags_var.reset(_parent_tags_token) # Best-effort crash flush: ensures partial progress_state reaches PG # even when the actor raises unexpectedly (). if ( diff --git a/src/taskq/worker/run.py b/src/taskq/worker/run.py index fbfaa1fb..2ce4d438 100644 --- a/src/taskq/worker/run.py +++ b/src/taskq/worker/run.py @@ -46,7 +46,7 @@ from taskq.backend._protocol import Backend, JobRow from taskq.backend._records import jsonb_param from taskq.backend.clock import Clock -from taskq.client._enqueuer import SubJobEnqueuer +from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags from taskq.constants import ( _IDENT_RE, # pyright: ignore[reportPrivateUsage] # Why: canonical identifier regex; copying would drift the validation pattern. ) @@ -346,6 +346,8 @@ async def consumer_loop_stub( if current_task is None: raise RuntimeError("consumer_loop_stub must run inside a TaskGroup") + _parent_tags_token = set_parent_tags(tuple(job.tags)) + ctx: JobContext[_StubPayload] = JobContext( job_id=job.id, actor=job.actor, @@ -398,6 +400,7 @@ async def consumer_loop_stub( _consumer_log.exception("consumer-stub-error", job_id=str(job.id)) finally: + _parent_tags_var.reset(_parent_tags_token) await deps.active_jobs.deregister(job.id) diff --git a/tests/test_sub_job_tags.py b/tests/test_sub_job_tags.py new file mode 100644 index 00000000..446b5840 --- /dev/null +++ b/tests/test_sub_job_tags.py @@ -0,0 +1,283 @@ +"""Unit tests for SubJobEnqueuer.enqueue tags parameter and parent-tag inheritance.""" + +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import pytest +from pydantic import BaseModel, TypeAdapter + +from taskq.actor import ActorRef +from taskq.backend._protocol import JobId +from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags +from taskq.retry import RetryPolicy +from taskq.testing.clock import FakeClock +from taskq.testing.in_memory import InMemoryBackend + +_NOW = datetime(2025, 1, 1, tzinfo=UTC) + + +class _Payload(BaseModel): + value: str = "test" + + +class _Result(BaseModel): + ok: bool = True + + +def _make_actor_ref(name: str = "child") -> ActorRef[_Payload, _Result]: + async def _handler(payload: _Payload) -> _Result: + return _Result() + + return ActorRef( + name=name, + queue="default", + fn=_handler, + wants_ctx=False, + dependencies={}, + payload_type=_Payload, + result_adapter=TypeAdapter(_Result), + retry=RetryPolicy(), + result_ttl=None, + singleton=False, + unique_for=None, + max_pending=None, + ) + + +def _make_enqueuer(backend: InMemoryBackend | None = None) -> SubJobEnqueuer: + if backend is None: + backend = InMemoryBackend(clock=FakeClock(_NOW)) + return SubJobEnqueuer( + loop_scope_resolved=None, + worker_pool=object(), + backend=backend, + clock=FakeClock(_NOW), + ) + + +class TestSubJobExplicitTags: + async def test_explicit_tags_no_inheritance(self) -> None: + """tags= with inherit_tags=False sets only explicit tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["alpha", "beta"], + inherit_tags=False, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("alpha", "beta") + + async def test_tags_validated(self) -> None: + """Invalid tags raise ValueError.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + with pytest.raises(ValueError, match="invalid tag"): + await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["ab"], + ) + + async def test_tags_deduplicated(self) -> None: + """Duplicate tags are deduplicated.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["alpha", "alpha", "beta"], + inherit_tags=False, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("alpha", "beta") + + +class TestSubJobTagInheritance: + async def test_inherit_parent_tags_default(self) -> None: + """With no explicit tags and default inherit_tags=True, sub-job inherits parent tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001", "tenant-acme")) + try: + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("run-001", "tenant-acme") + + async def test_inherit_and_merge_tags(self) -> None: + """Explicit tags merge with parent tags (parent first, deduped).""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001", "tenant-acme")) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["stage-2", "tenant-acme"], + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("run-001", "tenant-acme", "stage-2") + + async def test_no_parent_tags_no_explicit_tags(self) -> None: + """With no parent tags and no explicit tags, sub-job has empty tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == () + + async def test_inherit_false_no_parent_tags(self) -> None: + """inherit_tags=False with no explicit tags -> empty tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001",)) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + inherit_tags=False, + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == () + + async def test_inherit_false_with_explicit_tags(self) -> None: + """inherit_tags=False with explicit tags -> only explicit tags.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001",)) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["custom-tag"], + inherit_tags=False, + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("custom-tag",) + + async def test_empty_list_tags_with_inheritance(self) -> None: + """tags=[] with inherit_tags=True and parent tags -> inherits parent tags only. + + An empty list means "no additional tags to add" -- parent tags + are still inherited. This is the intuitive behavior: the caller + didn't add any new tags, so the sub-job carries what the parent + carried. + """ + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + token = set_parent_tags(("run-001", "tenant-acme")) + try: + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=[], + ) + finally: + _parent_tags_var.reset(token) + + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("run-001", "tenant-acme") + + +class TestSubJobMissingFields: + async def test_schedule_to_close(self) -> None: + """schedule_to_close is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + deadline = _NOW + timedelta(hours=1) + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + schedule_to_close=deadline, + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.schedule_to_close == deadline + + async def test_start_to_close(self) -> None: + """start_to_close is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + start_to_close=timedelta(minutes=30), + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.start_to_close == timedelta(minutes=30) + + async def test_heartbeat_timeout(self) -> None: + """heartbeat_timeout is accepted and stored on the row.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + heartbeat_timeout=timedelta(seconds=10), + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.heartbeat_timeout == timedelta(seconds=10) + + +class TestContextVarIsolation: + async def test_concurrent_jobs_separate_parent_tags(self) -> None: + """ContextVar ensures concurrent consumers don't share parent tags.""" + import asyncio + + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + async def enqueue_with_parent(parent_tags: tuple[str, ...]) -> UUID: + token = set_parent_tags(parent_tags) + try: + handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) + return handle.job_id + finally: + _parent_tags_var.reset(token) + + id1, id2 = await asyncio.gather( + enqueue_with_parent(("run-a",)), + enqueue_with_parent(("run-b",)), + ) + + row1 = await backend.get(JobId(id1)) + row2 = await backend.get(JobId(id2)) + assert row1 is not None + assert row2 is not None + assert row1.tags == ("run-a",) + assert row2.tags == ("run-b",) From 95f649397212294a587809249a0093f67685c246 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 17:04:08 -0700 Subject: [PATCH 09/16] test(e2e): add sub-job tags and bulk cancel E2E tests Task 9: Pipeline actors (pipeline_stage, tagged_pipeline_stage) verify sub-job tag inheritance and tag merging in a real worker container. Tests: inherits parent tags, explicit tags merge with inherited. Task 10: Bulk cancel E2E tests against real Postgres + worker: - pending jobs by tag (5 cancelled, 2 untagged survive) - running jobs cooperative cancel (assert total_affected >= 1 per 2P3) - empty filter guardrail - batch_id filter All E2E tests marked @pytest.mark.e2e (manual-only, requires Docker). --- tests/e2e/actors.py | 70 +++++++++++++++ tests/e2e/test_cancel_where.py | 157 +++++++++++++++++++++++++++++++++ tests/e2e/test_sub_job_tags.py | 108 +++++++++++++++++++++++ 3 files changed, 335 insertions(+) create mode 100644 tests/e2e/test_cancel_where.py create mode 100644 tests/e2e/test_sub_job_tags.py diff --git a/tests/e2e/actors.py b/tests/e2e/actors.py index 7f21e709..6c02ab27 100644 --- a/tests/e2e/actors.py +++ b/tests/e2e/actors.py @@ -705,3 +705,73 @@ async def concurrent_tracked_worker( "ct_finished", {"run_id": payload.run_id, "job_index": payload.job_index}, ) + + +# ── Tagged pipeline actors (sub-job tag inheritance / merge) ────────────── + + +class PipelineStagePayload(BaseModel): + run_id: str + stage: int + total_stages: int = 3 + + +@actor(name="pipeline_stage", queue="e2e") +async def pipeline_stage( + payload: PipelineStagePayload, + ctx: JobContext[PipelineStagePayload], + *, + pool: asyncpg.Pool, +) -> None: + """Linear pipeline: each stage enqueues the next via ctx.jobs.enqueue().""" + await _record_effect( + pool, + ctx, + "stage", + {"run_id": payload.run_id, "stage": payload.stage}, + ) + await asyncio.sleep(0.05) + + if payload.stage < payload.total_stages: + await ctx.jobs.enqueue( + pipeline_stage, + PipelineStagePayload( + run_id=payload.run_id, + stage=payload.stage + 1, + total_stages=payload.total_stages, + ), + ) + + +class TaggedPipelineStagePayload(BaseModel): + run_id: str + stage: int + total_stages: int = 3 + + +@actor(name="tagged_pipeline_stage", queue="e2e") +async def tagged_pipeline_stage( + payload: TaggedPipelineStagePayload, + ctx: JobContext[TaggedPipelineStagePayload], + *, + pool: asyncpg.Pool, +) -> None: + """Pipeline that passes explicit tags to sub-job enqueue.""" + await _record_effect( + pool, + ctx, + "stage", + {"run_id": payload.run_id, "stage": payload.stage}, + ) + await asyncio.sleep(0.05) + + if payload.stage < payload.total_stages: + await ctx.jobs.enqueue( + tagged_pipeline_stage, + TaggedPipelineStagePayload( + run_id=payload.run_id, + stage=payload.stage + 1, + total_stages=payload.total_stages, + ), + tags=[f"stage-{payload.stage + 1}"], + ) diff --git a/tests/e2e/test_cancel_where.py b/tests/e2e/test_cancel_where.py new file mode 100644 index 00000000..e9cf1a8b --- /dev/null +++ b/tests/e2e/test_cancel_where.py @@ -0,0 +1,157 @@ +"""Bulk cancel by filter e2e — cancel_where against real Postgres + worker. + +Scenarios: +- pending/scheduled jobs cancelled directly by tag filter (untagged survive) +- running job cooperative cancel via tag filter (total_affected >= 1) +- empty filter guardrail (EmptyFilterError) +- batch_id filter cancellation +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest + +from taskq.backend._protocol import JobFilter +from taskq.batch import EnqueueItem + +from ._assertions import wait_for_handle_status +from .actors import ( + GenerateReportPayload, + ImportContactsChunkPayload, + generate_report, + import_contacts_chunk, +) + +if TYPE_CHECKING: + import asyncpg + + from taskq import TaskQ + + from .conftest import E2ESchema, E2EWorker + +pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] + + +async def test_cancel_where_pending_jobs_by_tag( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where cancels all pending jobs matching a tag filter.""" + tag = f"tenant-{run_id[:8]}" + future = datetime.now(UTC) + timedelta(seconds=120) + + for i in range(5): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=run_id, report_id=f"r-{i}"), + scheduled_at=future, + tags=[tag], + ) + + for i in range(2): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=f"other-{run_id}", report_id=f"o-{i}"), + scheduled_at=future, + ) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,)), + reason="tenant offboarded", + ) + + assert result.cancelled_directly == 5 + assert result.cancel_requested == 0 + assert result.total_affected == 5 + + tagged = await e2e_client.list(JobFilter(tags=(tag,), status="cancelled")) + assert len(tagged.jobs) == 5 + + untagged = await e2e_client.list(JobFilter(status="scheduled", queue="e2e")) + assert len(untagged.jobs) == 2 + + +async def test_cancel_where_running_jobs_cooperative( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where sets cooperative cancel for running jobs.""" + tag = f"run-{run_id[:8]}" + + handle = await e2e_client.enqueue( + generate_report, + GenerateReportPayload( + run_id=run_id, + report_id=f"r-{run_id[:8]}", + stages=4, + stage_latency_ms=2000, + ), + tags=[tag], + ) + await wait_for_handle_status(handle, "running", timeout=30) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,), status="running"), + reason="abort run", + ) + + assert result.total_affected >= 1, "cancel_where should affect the running job" + assert result.cancelled_directly == 0 + + await wait_for_handle_status(handle, "cancelled", timeout=30) + + +async def test_cancel_where_empty_filter_raises( + e2e_client: TaskQ, + e2e_worker: E2EWorker, +) -> None: + """Empty filter is rejected even in e2e.""" + from taskq.exceptions import EmptyFilterError + + with pytest.raises(EmptyFilterError): + await e2e_client.cancel_where(JobFilter(), reason="oops") + + +async def test_cancel_where_batch_id_filter( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where works with batch_id filter.""" + future = datetime.now(UTC) + timedelta(seconds=120) + bid = uuid4() + + items = [ + EnqueueItem( + actor_ref=import_contacts_chunk, + payload=ImportContactsChunkPayload( + run_id=run_id, + upload_id=str(bid), + chunk_id=i, + start_row=i * 100, + end_row=(i + 1) * 100, + ), + scheduled_at=future, + ) + for i in range(5) + ] + await e2e_client.enqueue_batch(items, batch_id=bid) + + result = await e2e_client.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 5 diff --git a/tests/e2e/test_sub_job_tags.py b/tests/e2e/test_sub_job_tags.py new file mode 100644 index 00000000..036789f5 --- /dev/null +++ b/tests/e2e/test_sub_job_tags.py @@ -0,0 +1,108 @@ +"""Sub-job tag inheritance and merge e2e — real pipeline in a worker container. + +Scenario: +parent job enqueued with a tag enqueues sub-jobs via ``ctx.jobs.enqueue()``. +Sub-jobs with no explicit tags inherit the parent tag; sub-jobs with explicit +tags carry both the inherited parent tag and the explicit tag. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from taskq.backend._protocol import JobFilter + +from ._assertions import wait_for_effects +from .actors import ( + PipelineStagePayload, + TaggedPipelineStagePayload, + pipeline_stage, + tagged_pipeline_stage, +) + +if TYPE_CHECKING: + import asyncpg + + from taskq import TaskQ + + from .conftest import E2ESchema, E2EWorker + +pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] + + +async def test_sub_job_inherits_tags_in_pipeline( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """Sub-jobs enqueued via ctx.jobs.enqueue() inherit parent tags. + + A pipeline of 3 stages is enqueued with tag "run-{run_id}". Each stage + enqueues the next via ctx.jobs.enqueue() with no explicit tags. All 3 + jobs should be findable by the tag filter. + """ + tag = f"run-{run_id[:8]}" + await e2e_client.enqueue( + pipeline_stage, + PipelineStagePayload(run_id=run_id, stage=1, total_stages=3), + tags=[tag], + ) + + await wait_for_effects( + e2e_pg_pool, + e2e_schema.schema_name, + run_id, + kind="stage", + min_count=3, + timeout=30, + ) + + page = await e2e_client.list(JobFilter(tags=(tag,))) + assert len(page.jobs) == 3, ( + f"Expected 3 jobs with tag {tag!r}, found {len(page.jobs)}: {[j.id for j in page.jobs]}" + ) + + +async def test_sub_job_explicit_tags_merge_with_parent( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """Explicit tags on sub-job merge with inherited parent tags. + + Stage 1 is enqueued with parent tag "run-{run_id}". Each stage enqueues + the next with an explicit stage tag (e.g. "stage-2"). The sub-job should + carry both the inherited run tag and the explicit stage tag. + """ + parent_tag = f"run-{run_id[:8]}" + + await e2e_client.enqueue( + tagged_pipeline_stage, + TaggedPipelineStagePayload(run_id=run_id, stage=1, total_stages=3), + tags=[parent_tag], + ) + + await wait_for_effects( + e2e_pg_pool, + e2e_schema.schema_name, + run_id, + kind="stage", + min_count=3, + timeout=30, + ) + + page = await e2e_client.list(JobFilter(tags=(parent_tag,))) + assert len(page.jobs) == 3, ( + f"Expected 3 jobs with parent tag {parent_tag!r}, found {len(page.jobs)}" + ) + + stage2 = await e2e_client.list(JobFilter(tags=("stage-2",))) + assert len(stage2.jobs) == 1, f"Expected 1 job with stage-2 tag, found {len(stage2.jobs)}" + assert parent_tag in stage2.jobs[0].tags + assert "stage-2" in stage2.jobs[0].tags From aa1fdfc714f59c57ae042805eba670b9ea5b6ab9 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 17:06:11 -0700 Subject: [PATCH 10/16] refactor: remove sub_job_inherit_tags kill switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-1.0 library — no installed base to protect. The consumer always sets parent tags; per-call inherit_tags=False is the opt-out for callers who need it. Merges cancel_where E2E tests into test_cancellation.py (same cancel protocol, single file). --- src/taskq/settings.py | 8 -- src/taskq/worker/_consumer.py | 6 +- tests/e2e/test_cancel_where.py | 157 ----------------------------- tests/e2e/test_cancellation.py | 178 ++++++++++++++++++++++++++++----- 4 files changed, 157 insertions(+), 192 deletions(-) delete mode 100644 tests/e2e/test_cancel_where.py diff --git a/src/taskq/settings.py b/src/taskq/settings.py index 600f2485..2fd2187b 100644 --- a/src/taskq/settings.py +++ b/src/taskq/settings.py @@ -357,14 +357,6 @@ class WorkerSettings(TaskQSettings): ) # ── Pool sizes ───────────────────────────────────────────────────── - sub_job_inherit_tags: bool = Field( - default=True, - description=( - "When false, sub-jobs enqueued via ctx.jobs.enqueue() do not inherit " - "the parent job's tags (pre-1.0 behavior). Fleet-level kill switch " - "for the inherit_tags=True default; env var TASKQ_SUB_JOB_INHERIT_TAGS." - ), - ) dispatcher_pool_size: int = Field( default=4, ge=1, diff --git a/src/taskq/worker/_consumer.py b/src/taskq/worker/_consumer.py index fe85cd42..cbf5fb1c 100644 --- a/src/taskq/worker/_consumer.py +++ b/src/taskq/worker/_consumer.py @@ -353,8 +353,7 @@ async def consume_one_job( _buf = _ProgressBuffer(job_id=job.id, base_seq=job.progress_seq) _progress_buffers[job.id] = _buf - _inherit = _effective_settings is None or _effective_settings.sub_job_inherit_tags - _parent_tags_token = set_parent_tags(tuple(job.tags)) if _inherit else None + _parent_tags_token = set_parent_tags(tuple(job.tags)) try: validated_payload = ( @@ -570,8 +569,7 @@ async def consume_one_job( ) finally: - if _parent_tags_token is not None: - _parent_tags_var.reset(_parent_tags_token) + _parent_tags_var.reset(_parent_tags_token) # Best-effort crash flush: ensures partial progress_state reaches PG # even when the actor raises unexpectedly (). if ( diff --git a/tests/e2e/test_cancel_where.py b/tests/e2e/test_cancel_where.py deleted file mode 100644 index e9cf1a8b..00000000 --- a/tests/e2e/test_cancel_where.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Bulk cancel by filter e2e — cancel_where against real Postgres + worker. - -Scenarios: -- pending/scheduled jobs cancelled directly by tag filter (untagged survive) -- running job cooperative cancel via tag filter (total_affected >= 1) -- empty filter guardrail (EmptyFilterError) -- batch_id filter cancellation -""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING -from uuid import uuid4 - -import pytest - -from taskq.backend._protocol import JobFilter -from taskq.batch import EnqueueItem - -from ._assertions import wait_for_handle_status -from .actors import ( - GenerateReportPayload, - ImportContactsChunkPayload, - generate_report, - import_contacts_chunk, -) - -if TYPE_CHECKING: - import asyncpg - - from taskq import TaskQ - - from .conftest import E2ESchema, E2EWorker - -pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] - - -async def test_cancel_where_pending_jobs_by_tag( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where cancels all pending jobs matching a tag filter.""" - tag = f"tenant-{run_id[:8]}" - future = datetime.now(UTC) + timedelta(seconds=120) - - for i in range(5): - await e2e_client.enqueue( - generate_report, - GenerateReportPayload(run_id=run_id, report_id=f"r-{i}"), - scheduled_at=future, - tags=[tag], - ) - - for i in range(2): - await e2e_client.enqueue( - generate_report, - GenerateReportPayload(run_id=f"other-{run_id}", report_id=f"o-{i}"), - scheduled_at=future, - ) - - result = await e2e_client.cancel_where( - JobFilter(tags=(tag,)), - reason="tenant offboarded", - ) - - assert result.cancelled_directly == 5 - assert result.cancel_requested == 0 - assert result.total_affected == 5 - - tagged = await e2e_client.list(JobFilter(tags=(tag,), status="cancelled")) - assert len(tagged.jobs) == 5 - - untagged = await e2e_client.list(JobFilter(status="scheduled", queue="e2e")) - assert len(untagged.jobs) == 2 - - -async def test_cancel_where_running_jobs_cooperative( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where sets cooperative cancel for running jobs.""" - tag = f"run-{run_id[:8]}" - - handle = await e2e_client.enqueue( - generate_report, - GenerateReportPayload( - run_id=run_id, - report_id=f"r-{run_id[:8]}", - stages=4, - stage_latency_ms=2000, - ), - tags=[tag], - ) - await wait_for_handle_status(handle, "running", timeout=30) - - result = await e2e_client.cancel_where( - JobFilter(tags=(tag,), status="running"), - reason="abort run", - ) - - assert result.total_affected >= 1, "cancel_where should affect the running job" - assert result.cancelled_directly == 0 - - await wait_for_handle_status(handle, "cancelled", timeout=30) - - -async def test_cancel_where_empty_filter_raises( - e2e_client: TaskQ, - e2e_worker: E2EWorker, -) -> None: - """Empty filter is rejected even in e2e.""" - from taskq.exceptions import EmptyFilterError - - with pytest.raises(EmptyFilterError): - await e2e_client.cancel_where(JobFilter(), reason="oops") - - -async def test_cancel_where_batch_id_filter( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where works with batch_id filter.""" - future = datetime.now(UTC) + timedelta(seconds=120) - bid = uuid4() - - items = [ - EnqueueItem( - actor_ref=import_contacts_chunk, - payload=ImportContactsChunkPayload( - run_id=run_id, - upload_id=str(bid), - chunk_id=i, - start_row=i * 100, - end_row=(i + 1) * 100, - ), - scheduled_at=future, - ) - for i in range(5) - ] - await e2e_client.enqueue_batch(items, batch_id=bid) - - result = await e2e_client.cancel_where( - JobFilter(batch_id=bid), - reason="batch abort", - ) - - assert result.cancelled_directly == 5 diff --git a/tests/e2e/test_cancellation.py b/tests/e2e/test_cancellation.py index 21ecc593..a48e3c57 100644 --- a/tests/e2e/test_cancellation.py +++ b/tests/e2e/test_cancellation.py @@ -1,27 +1,17 @@ -"""Cancellation e2e — cooperative cancel mid-run + clean pre-dispatch cancel. - -Scenario: -long report → ``wait_for_handle_status(handle, "running")`` → -``handle.cancel()`` → terminal ``cancelled``; effects show only early stages. - -Real cancel semantics, verified against the library (not guessed): +"""Cancellation e2e — single-job cancel + bulk cancel_by_filter. +Single-job path: - ``JobsClient.cancel`` → ``PostgresBackend.write_cancel_request`` - (backend/postgres.py) branches on the row's current status - (backend/_sql_templates.py): - - - ``pending``/``scheduled`` → ``cancel_pending_scheduled`` UPDATEs the row - straight to ``status='cancelled'``. The cancel API does NOT reject - pre-dispatch cancels: ``CancelResult.cancellation_initiated`` is True, - ``new_status`` is ``'cancelled'``, and the worker never dispatches the - job — no actor code runs, so no effects exist. - - ``running`` → ``cancel_running`` sets ``cancel_phase=1``; the worker's - heartbeat-driven CancelController sets the in-process ``cancel_event``, - the actor's ``ctx.check_cancelled()`` raises ``asyncio.CancelledError`` - at the next stage boundary, and the consumer writes ``mark_cancelled``. - -- ``handle.wait()`` on any non-success terminal status raises ``JobFailed`` - carrying the row (``client/_handle.py._extract_result``). + branches on the row's current status: + - ``pending``/``scheduled`` → straight to ``status='cancelled'`` + - ``running`` → ``cancel_phase=1`` (cooperative), actor sees + ``ctx.check_cancelled()`` at next stage boundary + +Bulk path (``cancel_where``): +- pending/scheduled → terminal ``cancelled`` (set-based, single SQL) +- running → ``cancel_phase=1`` (cooperative, batched NOTIFY) +- empty filter rejected with ``EmptyFilterError`` +- batch_id filter cancels all jobs in a batch Every test requests ``e2e_worker`` explicitly: the worker container fixture is not autouse, so no worker (and no dispatch) exists unless a test pulls it in. @@ -31,13 +21,21 @@ from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING +from uuid import uuid4 import pytest from taskq import JobFailed +from taskq.backend._protocol import JobFilter +from taskq.batch import EnqueueItem from ._assertions import fetch_effects, wait_for_effects, wait_for_handle_status -from .actors import GenerateReportPayload, generate_report +from .actors import ( + GenerateReportPayload, + ImportContactsChunkPayload, + generate_report, + import_contacts_chunk, +) if TYPE_CHECKING: import asyncpg @@ -139,3 +137,137 @@ async def test_cancel_before_dispatch_is_clean( # The job never dispatched, so no actor code ran: no effects of any kind. assert await fetch_effects(e2e_pg_pool, e2e_schema.schema_name, run_id) == [] + + +# ── Bulk cancel (cancel_where) ───────────────────────────────────────── + + +async def test_cancel_where_pending_jobs_by_tag( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where cancels all scheduled jobs matching a tag filter in one + set-based write. Untagged jobs are unaffected.""" + tag = f"tenant-{run_id[:8]}" + future = datetime.now(UTC) + timedelta(seconds=120) + + for i in range(5): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=run_id, report_id=f"r-{i}"), + scheduled_at=future, + tags=[tag], + ) + for i in range(2): + await e2e_client.enqueue( + generate_report, + GenerateReportPayload(run_id=f"other-{run_id}", report_id=f"o-{i}"), + scheduled_at=future, + ) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,)), + reason="tenant offboarded", + ) + + assert result.cancelled_directly == 5 + assert result.cancel_requested == 0 + assert result.total_affected == 5 + + tagged = await e2e_client.list(JobFilter(tags=(tag,), status="cancelled")) + assert len(tagged.jobs) == 5 + + untagged = await e2e_client.list(JobFilter(status="scheduled", queue="e2e")) + assert len(untagged.jobs) == 2 + + +async def test_cancel_where_running_cooperative( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where on a running job sets cooperative cancel (cancel_phase=1). + The actor observes the cancel signal at the next stage boundary and + reaches terminal 'cancelled' via the normal cooperative path.""" + tag = f"run-{run_id[:8]}" + + handle = await e2e_client.enqueue( + generate_report, + GenerateReportPayload( + run_id=run_id, + report_id=f"r-{run_id[:8]}", + stages=4, + stage_latency_ms=2000, + ), + tags=[tag], + ) + await wait_for_handle_status(handle, "running", timeout=30) + await wait_for_effects( + e2e_pg_pool, + e2e_schema.schema_name, + run_id, + kind="stage", + min_count=1, + timeout=30, + ) + + result = await e2e_client.cancel_where( + JobFilter(tags=(tag,), status="running"), + reason="abort run", + ) + + assert result.total_affected >= 1, "cancel_where should affect the running job" + assert result.cancelled_directly == 0 + + await wait_for_handle_status(handle, "cancelled", timeout=30) + + +async def test_cancel_where_empty_filter_raises( + e2e_client: TaskQ, + e2e_worker: E2EWorker, +) -> None: + """Empty filter is rejected even in e2e.""" + from taskq.exceptions import EmptyFilterError + + with pytest.raises(EmptyFilterError): + await e2e_client.cancel_where(JobFilter(), reason="oops") + + +async def test_cancel_where_batch_id_filter( + e2e_client: TaskQ, + e2e_worker: E2EWorker, + e2e_pg_pool: asyncpg.Pool, + e2e_schema: E2ESchema, + run_id: str, +) -> None: + """cancel_where with batch_id cancels all jobs in a batch.""" + future = datetime.now(UTC) + timedelta(seconds=120) + bid = uuid4() + + items = [ + EnqueueItem( + actor_ref=import_contacts_chunk, + payload=ImportContactsChunkPayload( + run_id=run_id, + upload_id=str(bid), + chunk_id=i, + start_row=i * 100, + end_row=(i + 1) * 100, + ), + scheduled_at=future, + ) + for i in range(5) + ] + await e2e_client.enqueue_batch(items, batch_id=bid) + + result = await e2e_client.cancel_where( + JobFilter(batch_id=bid), + reason="batch abort", + ) + + assert result.cancelled_directly == 5 From fea6db3df8d7da89332b54836743481b4846cbfd Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 17:07:45 -0700 Subject: [PATCH 11/16] docs: document cancel_where, sub-job tags, and tag inheritance Update jobs-clients.md: new cancel_where section with BulkCancelResult fields, guardrail, snapshot boundary, and tenant-scale guidance. Update sub-job enqueue signature with tags/inherit_tags/schedule_to_close/ start_to_close/heartbeat_timeout. Document tag inheritance semantics table and blast-radius. Remove stale exclusion list. Update architecture.md: add cancel_where to Backend protocol listing. Update cancellation.md: add cancel_where as a cancellation method. --- docs/architecture.md | 1 + docs/guides/cancellation.md | 16 ++++++ docs/guides/jobs-clients.md | 109 +++++++++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 8 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index b3522e50..2e17b4a3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -102,6 +102,7 @@ class Backend(Protocol): # Cancel signals async def write_cancel_request(self, job_id, reason) -> bool: ... + async def cancel_where(self, filter: JobFilter, reason: str | None) -> BulkCancelResult: ... async def poll_cancel_flags(self, worker_id) -> list[CancelFlag]: ... # Admin operations diff --git a/docs/guides/cancellation.md b/docs/guides/cancellation.md index 48ca5557..114ac29c 100644 --- a/docs/guides/cancellation.md +++ b/docs/guides/cancellation.md @@ -30,6 +30,22 @@ result = await handle.cancel(reason="deadline exceeded") `JobHandle.cancel()` delegates directly to `JobsClient.cancel(handle.job_id, reason)`. The handle must have been constructed with a `JobsClient` (i.e. via `client.enqueue()` or `client.get()`); handles obtained from inside an actor body via `ctx.jobs.enqueue()` do not have a client and will raise `RuntimeError`. +### Via `JobsClient.cancel_where()` + +```python +from taskq.backend._protocol import JobFilter + +result = await client.cancel_where( + JobFilter(tags=("tenant-acme",), active=True), + reason="tenant offboarded", +) +``` + +`cancel_where()` cancels all jobs matching a `JobFilter` in a single set-based SQL +operation. Pending/scheduled jobs go straight to terminal `cancelled`; running jobs get +`cancel_phase=1` (cooperative cancel). Returns a `BulkCancelResult` with counts and +affected IDs. See [jobs-clients.md](jobs-clients.md#cancel_where) for the full API. + ### Effect by prior status | Prior status | Effect | diff --git a/docs/guides/jobs-clients.md b/docs/guides/jobs-clients.md index 30ddf59e..a8d762d0 100644 --- a/docs/guides/jobs-clients.md +++ b/docs/guides/jobs-clients.md @@ -52,13 +52,14 @@ Terminal statuses (`succeeded`, `failed`, `cancelled`, `crashed`, `abandoned`) h 6. [`JobHandle[R]`](#jobhandler) 7. [`get()`](#get) 8. [`cancel()`](#cancel) -9. [`list()`](#list) -10. [`SubJobEnqueuer`](#subjobenqueuer) -11. [Error handling](#error-handling) -12. [Full enqueue-and-wait example](#full-enqueue-and-wait-example) -13. [Idempotency example](#idempotency-example) -14. [Batch enqueue example](#batch-enqueue-example) -15. [Tags](#tags) +9. [`cancel_where()`](#cancel_where) +10. [`list()`](#list) +11. [`SubJobEnqueuer`](#subjobenqueuer) +12. [Error handling](#error-handling) +13. [Full enqueue-and-wait example](#full-enqueue-and-wait-example) +14. [Idempotency example](#idempotency-example) +15. [Batch enqueue example](#batch-enqueue-example) +16. [Tags](#tags) --- @@ -636,6 +637,63 @@ else: --- +## `cancel_where()` + +```python +async def cancel_where( + self, + filter: JobFilter, + reason: str | None = None, + *, + allow_empty_filter: bool = False, +) -> BulkCancelResult: ... +``` + +Cancel all jobs matching `filter` in a single set-based operation. Pending/scheduled +jobs go straight to terminal `cancelled`; running jobs get `cancel_phase=1` (cooperative +cancel) — the worker's heartbeat observes the phase change and sets the in-process +`cancel_event` at the next tick. + +**Guardrail:** a filter with no predicates (no `queue`, `status`, `actor`, +`identity_key`, `batch_id`, `tags`, or `active`) is rejected with `EmptyFilterError` +unless `allow_empty_filter=True` is passed. This prevents accidental full-table cancels. + +**Filter fields used:** `queue`, `status`, `actor`, `identity_key`, `batch_id`, `tags`, +`active`. The `limit`, `cursor`, and `order_by` fields are ignored — a bulk cancel is +not paginated. + +**Snapshot boundary:** jobs matching the filter that are enqueued *after* the +statement's snapshot escape this call. Stop producers first, or issue a follow-up call; +the returned counts make non-convergence detectable. + +```python +result = await client.cancel_where( + JobFilter(tags=("tenant-acme",), active=True), + reason="tenant offboarded", +) +print(f"cancelled {result.cancelled_directly} pending, " + f"requested cancel for {result.cancel_requested} running") +``` + +### `BulkCancelResult` + +Frozen Pydantic model returned by `cancel_where()`. + +| Field | Type | Description | +|---|---|---| +| `cancelled_directly` | `int` | Pending/scheduled jobs moved straight to terminal `cancelled`. | +| `cancel_requested` | `int` | Running jobs with `cancel_phase=1` set (cooperative cancel). | +| `cancelled_ids` | `list[UUID]` | IDs of jobs cancelled directly. | +| `cancel_requested_ids` | `list[UUID]` | IDs of running jobs with cancel requested. | +| `total_affected` | `int` (property) | `cancelled_directly + cancel_requested`. | + +For tenant-scale cancels (10^5+ matching rows), partition via filter (e.g. +`JobFilter(queue=..., tags=...)` to split by queue). A single transaction covering 10^6 +rows would hold locks too long. The counts let the caller verify completeness and issue +follow-up calls for remaining partitions. + +--- + ## `list()` ```python @@ -781,17 +839,52 @@ async def enqueue( unique_for: timedelta | None = None, unique_states: tuple[JobStatus, ...] | None = None, max_pending: int | None = None, + tags: list[str] | None = None, + inherit_tags: bool = True, + schedule_to_close: datetime | None = None, + start_to_close: timedelta | None = None, + heartbeat_timeout: timedelta | None = None, ) -> JobHandle[R]: ... ``` Enqueues a single sub-job. Accepts the same options as `JobsClient.enqueue()` except: - No `queue` override (sub-jobs use the actor's declared queue). -- No `schedule_to_close`, `start_to_close`, or `heartbeat_timeout` (set on the actor declaration). - No explicit `trace_id` / `span_id` (extracted from the active OTel span). - `connection` may be passed to use a specific `asyncpg.Connection` rather than the LOOP-scope connection. +#### Tag inheritance + +Sub-jobs inherit the parent job's tags by default (`inherit_tags=True`). When no +explicit `tags` are passed, the sub-job carries the parent's tags. When explicit +`tags` are passed, they merge with the parent's tags (parent first, deduplicated). + +| `inherit_tags` | `tags` | Resulting job tags | +|---|---|---| +| `True` (default) | `None` | Parent job's tags (or `()` if parent has none) | +| `True` (default) | `["new-tag"]` | Parent tags + explicit tags, merged (union, parent-first, deduped) | +| `False` | `None` | `()` (no inheritance) | +| `False` | `["new-tag"]` | `("new-tag",)` (explicit only) | + +Pass `inherit_tags=False` to suppress inheritance for a specific sub-job. + +**Blast radius:** inherited tags make sub-jobs visible to `cancel_where` filters +matching those tags. A shared/utility sub-job enqueued by a tenant-tagged parent +will be swept up in that tenant's `cancel_where`. + +`enqueue_batch()` does **not** inherit parent tags — batch items carry their own +`EnqueueItem.tags`. This asymmetry is deliberate: batch fan-out callers typically +set per-item tags explicitly. + +#### `schedule_to_close` / `start_to_close` / `heartbeat_timeout` + +These parameters are passed through to `build_enqueue_args` and override the +actor's declared defaults for this specific sub-job. Note that +`schedule_to_close` bounds total wall-clock time *including* time snoozed on +`wait_for_batch` — finalizer-style sub-jobs that snooze for long periods should +set it generously or not at all. + The per-call `max_pending=` argument is resolved against the operator-owned stored cap and the `@actor(...)` literal, not in place of them: against a non-NULL stored `actor_config.max_pending` the tighter of the two wins (`min(stored, per_call)`) — explicit From 8223898cdd4cb3a2b4fa045e7bf03f002e7a48f2 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 17:14:53 -0700 Subject: [PATCH 12/16] fix: code review findings L1-L3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L1: Replace assert with explicit RuntimeError in _cancel_bulk.py (assert strips under -O, leaving an opaque TypeError) L2: Remove redundant TYPE_CHECKING import of asyncpg in _cancel_bulk.py (already imported at module level for DeadlockDetectedError) L3: Add deadlock retry unit tests (succeed-on-retry, exhausted-raises, no-retry-on-success) — mocks the pool to avoid PG dependency --- src/taskq/backend/_cancel_bulk.py | 10 ++-- tests/test_cancel_where_pg.py | 94 +++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/src/taskq/backend/_cancel_bulk.py b/src/taskq/backend/_cancel_bulk.py index 89eb0951..2a47c708 100644 --- a/src/taskq/backend/_cancel_bulk.py +++ b/src/taskq/backend/_cancel_bulk.py @@ -11,7 +11,7 @@ import asyncio import random -from typing import TYPE_CHECKING, NamedTuple +from typing import NamedTuple from uuid import UUID import asyncpg @@ -21,9 +21,6 @@ from taskq.backend._records import jsonb_param from taskq.backend._sql_templates import SqlTemplates -if TYPE_CHECKING: - import asyncpg - __all__ = ["NotifyTarget", "_cancel_where"] @@ -35,7 +32,7 @@ class NotifyTarget(NamedTuple): async def _cancel_where( - pool: "asyncpg.Pool", + pool: asyncpg.Pool, schema: str, sql: SqlTemplates, filter: JobFilter, @@ -95,7 +92,8 @@ async def _cancel_where( async with conn.transaction(): row = await conn.fetchrow(cancel_sql, *params) - assert row is not None + if row is None: + raise RuntimeError("cancel_where: aggregate query returned no rows") cancelled_ids = list(row["cancelled_ids"] or []) cancel_requested_ids = list(row["cancel_requested_ids"] or []) prev_statuses: dict[UUID, str] = dict( diff --git a/tests/test_cancel_where_pg.py b/tests/test_cancel_where_pg.py index 5bd3ca2a..2cb66172 100644 --- a/tests/test_cancel_where_pg.py +++ b/tests/test_cancel_where_pg.py @@ -2,10 +2,13 @@ import asyncio from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 +import asyncpg import pytest +from taskq.backend._cancel_bulk import _cancel_where from taskq.backend._protocol import Backend, JobFilter from taskq.testing.fixtures import JobsApp from taskq.testing.jobs import make_enqueue_args @@ -205,3 +208,94 @@ async def test_pg_cancel_where_notify_sent_for_running( await listen_conn.close() assert len(received) == 2 + + +class TestDeadlockRetry: + """Unit tests for the deadlock retry loop in _cancel_where. + + These don't need a real Postgres — they mock the pool to simulate + deadlock-then-success and deadlock-then-exhaustion scenarios. + """ + + @staticmethod + def _mock_pool_and_conn( + fetch_row: dict | None = None, + fetch_side_effects: list[Exception] | None = None, + ) -> tuple[MagicMock, MagicMock]: + conn = MagicMock() + + if fetch_side_effects is not None: + fetch_mock = AsyncMock(side_effect=fetch_side_effects) + if fetch_row is not None: + fetch_mock.side_effect = [ + *fetch_side_effects, + fetch_row, + ] + conn.fetchrow = fetch_mock + else: + conn.fetchrow = AsyncMock(return_value=fetch_row) + + conn.executemany = AsyncMock(return_value=None) + + tx = AsyncMock() + tx.__aenter__ = AsyncMock(return_value=tx) + tx.__aexit__ = AsyncMock(return_value=False) + conn.transaction = MagicMock(return_value=tx) + + conn.__aenter__ = AsyncMock(return_value=conn) + conn.__aexit__ = AsyncMock(return_value=False) + + pool = MagicMock() + pool.acquire = MagicMock(return_value=conn) + return pool, conn + + @staticmethod + def _success_row() -> dict: + return { + "cancelled_directly": 1, + "cancel_requested": 0, + "cancelled_ids": [uuid4()], + "cancelled_prev_statuses": ["pending"], + "cancel_requested_ids": [], + "cancel_requested_workers": [], + } + + async def test_deadlock_retry_succeeds_on_second_attempt(self) -> None: + """_cancel_where retries on DeadlockDetectedError and succeeds.""" + row = self._success_row() + pool, _ = self._mock_pool_and_conn( + fetch_row=row, + fetch_side_effects=[asyncpg.DeadlockDetectedError()], + ) + + sql = MagicMock() + sql.insert_event = "INSERT INTO job_events VALUES ($1, $2, $3, $4)" + + result, _notify = await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") + + assert result.cancelled_directly == 1 + assert result.cancel_requested == 0 + assert len(result.cancelled_ids) == 1 + + async def test_deadlock_retry_exhausted_raises(self) -> None: + """_cancel_where raises after 3 failed attempts.""" + pool, _ = self._mock_pool_and_conn( + fetch_side_effects=[asyncpg.DeadlockDetectedError()] * 3, + ) + + sql = MagicMock() + + with pytest.raises(asyncpg.DeadlockDetectedError): + await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") + + async def test_no_deadlock_no_retry(self) -> None: + """_cancel_where succeeds immediately without retry.""" + row = self._success_row() + pool, conn = self._mock_pool_and_conn(fetch_row=row) + + sql = MagicMock() + sql.insert_event = "INSERT INTO job_events VALUES ($1, $2, $3, $4)" + + await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") + + assert conn.fetchrow.call_count == 1 From 3f2b81ea211043e590dae5af923c29db07f571a5 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 18:43:23 -0700 Subject: [PATCH 13/16] =?UTF-8?q?chore:=20remove=20spec=20from=20branch=20?= =?UTF-8?q?=E2=80=94=20review=20against=20codebase=20and=20issues,=20not?= =?UTF-8?q?=20spec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/specs/2026-07-29-cancel-and-filter.md | 2574 -------------------- 1 file changed, 2574 deletions(-) delete mode 100644 docs/specs/2026-07-29-cancel-and-filter.md diff --git a/docs/specs/2026-07-29-cancel-and-filter.md b/docs/specs/2026-07-29-cancel-and-filter.md deleted file mode 100644 index 8367b68a..00000000 --- a/docs/specs/2026-07-29-cancel-and-filter.md +++ /dev/null @@ -1,2574 +0,0 @@ -# Spec: Sub-job Tags (#57) & Bulk Cancel by Filter (#54) - -**Date:** 2026-07-29 -**Status:** Draft, revised post-review (2026-07-29) -**Issues:** [#57](https://github.com/rich/taskq/issues/57), [#54](https://github.com/rich/taskq/issues/54) - ---- - -## Goal - -Make sub-jobs enqueued from inside actor bodies visible to tag-based filters by adding `tags` (and other missing fields) to `SubJobEnqueuer.enqueue()`, with parent-tag inheritance by default; and add a set-based `cancel_where(filter)` operation that cancels all jobs matching a `JobFilter` in a single SQL round-trip, with guardrails against accidental full-table cancel. - -## Non-goals - -- Metadata-based filtering (`JobFilter` metadata predicates) — out of scope; tags solve the discovery problem. -- Bulk cancel via the admin web UI — the API method is added here; UI wiring is a follow-up. -- Bulk *retry* or bulk *delete* by filter — same pattern but separate spec. -- Changing the cooperative cancellation state machine phases — `cancel_where` reuses the existing phase-1 cooperative path for running jobs and the existing direct-to-terminal path for pending/scheduled. -- Sub-job `queue` override — sub-jobs use the actor's declared queue (documented design choice, unchanged). -- Widening the tag charset — issues #54/#57 use colon-form tags in prose (`tenant:acme`, `run:{run_id}`), but TaskQ's validator (`_args.py:35`, `^[\w][\w\-]+[\w]$`) accepts word chars and hyphens only, and downstream (cennan) already ships hyphenated tags after a production incident with the colon form. All examples in this spec use the valid hyphenated form; a charset widening would be a separate, separately-motivated change. - ---- - -## Architecture Overview - -### Current state - -``` -JobsClient.enqueue(tags=...) ──► build_enqueue_args(tags=...) ──► EnqueueArgs.tags -EnqueueItem(tags=...) ──► build_batch_args ──► EnqueueArgs.tags -SubJobEnqueuer.enqueue() ──► build_enqueue_args(tags=) ──► EnqueueArgs.tags = () - ^^^^^^^^^^ - Issue #57: tags always empty - -JobsClient.cancel(job_id) ──► Backend.write_cancel_request(job_id, reason) - ├─ pending/scheduled → UPDATE to 'cancelled' (terminal) - └─ running → UPDATE cancel_phase=1 (cooperative) + NOTIFY - ^^^^^^^^^^ - Issue #54: one job at a time only -``` - -### Target state - -``` -SubJobEnqueuer.enqueue(tags=..., inherit_tags=True) - │ - ├─ inherit_tags=True, tags=None → use parent job's tags (from ContextVar) - ├─ inherit_tags=True, tags=[...] → merge parent tags + explicit tags (union, parent first) - └─ inherit_tags=False, tags=None → empty tags (current behavior) - -JobsClient.cancel_where(JobFilter(tags=("tenant-acme",), active=True), reason="offboard") - │ - └─► Backend.cancel_where(filter, reason) - ├─ pending/scheduled rows → UPDATE to 'cancelled' (terminal) + state_change events - └─ running rows → UPDATE cancel_phase=1 (cooperative) + cancel_request events + NOTIFY - └─► BulkCancelResult(cancelled_directly=N, cancel_requested=M, ...) -``` - -### File structure — files to create or modify - -``` -src/taskq/ -├── backend/ -│ ├── _protocol.py MODIFY: add cancel_where to Backend protocol; define BulkCancelResult -│ ├── _reads.py MODIFY: extract filter→SQL WHERE builder for reuse (refactor) -│ ├── _filter_sql.py CREATE: shared filter→SQL WHERE builder (extracted from _reads) -│ ├── _cancel_bulk.py CREATE: bulk cancel implementation for PostgresBackend -│ └── postgres.py MODIFY: wire cancel_where to _cancel_bulk -├── client/ -│ ├── __init__.py MODIFY: re-export BulkCancelResult alongside CancelResult -│ ├── _jobs.py MODIFY: add cancel_where method to JobsClient -│ ├── _enqueuer.py MODIFY: add tags, inherit_tags, schedule_to_close, start_to_close, heartbeat_timeout to enqueue() -│ ├── _taskq.py MODIFY: add cancel_where delegate to TaskQ -│ └── _args.py MODIFY: (no change needed — build_enqueue_args already accepts tags) -├── worker/ -│ ├── _consumer.py MODIFY: set parent tags ContextVar before actor invocation (gated by setting) -│ └── run.py MODIFY: set parent tags in stub consumer (unconditional — test harness) -├── settings.py MODIFY: add sub_job_inherit_tags field to WorkerSettings (fleet kill switch) -├── testing/ -│ ├── in_memory.py MODIFY: add cancel_where to InMemoryBackend -│ └── _cancel_bulk.py CREATE: in-memory bulk cancel implementation -├── types.py MODIFY: re-export BulkCancelResult from _protocol -├── exceptions.py MODIFY: add EmptyFilterError (guardrail) -└── __init__.py MODIFY: export BulkCancelResult, EmptyFilterError - -tests/ -├── test_sub_job_tags.py CREATE: unit tests for sub-job tags + inheritance -├── test_cancel_where.py CREATE: unit tests for cancel_where (in-memory) -├── test_cancel_where_pg.py CREATE: integration tests for cancel_where (postgres) -├── test_cancel_where_client.py CREATE: client-level cancel_where tests (guardrail, counter, schema errors) -├── test_filter_sql.py CREATE: filter→SQL builder extraction tests -├── test_bulk_cancel_types.py CREATE: BulkCancelResult, EmptyFilterError type tests -├── test_sub_job_enqueuer.py MODIFY: add tags parameter tests + backward compat -├── test_backend_protocol.py MODIFY: add cancel_where protocol conformance test; update member count -└── e2e/ - ├── actors.py MODIFY: add tagged pipeline actors - ├── test_sub_job_tags.py CREATE: e2e tests for sub-job tags in a real pipeline - └── test_cancel_where.py CREATE: e2e tests for bulk cancel - -docs/ -├── guides/jobs-clients.md MODIFY: document cancel_where and sub-job tags -└── architecture.md MODIFY: document bulk cancel in cancel protocol section -``` - -> **Note:** `context.py` is NOT modified — parent tags are propagated via a `contextvars.ContextVar` defined in `_enqueuer.py`, not via `JobContext`. The SQL is inlined in `_cancel_bulk.py` (matching the dynamic-SQL precedent in `_reads.py`); no `SqlTemplates.cancel_where` field or `_sql.py` change is needed. - ---- - -## API Surface - -### Issue #57: SubJobEnqueuer.enqueue() — tags and missing fields - -#### Modified signature - -```python -# src/taskq/client/_enqueuer.py - -class SubJobEnqueuer: - async def enqueue[P: BaseModel, R: BaseModel | None]( - self, - actor_ref: ActorRef[P, R], - payload: P, - *, - connection: asyncpg.Connection | None = None, - scheduled_at: datetime | None = None, - priority: int | None = None, - fairness_key: str | None = None, - metadata: dict[str, object] | None = None, - identity_key: IdentityKey | None = None, - idempotency_key: IdempotencyKey | str | None = None, - idempotency_scope: str | None = None, - unique_for: timedelta | None = None, - unique_states: tuple[JobStatus, ...] | None = None, - max_pending: int | None = None, - # ── NEW parameters ────────────────────────────────── - tags: list[str] | None = None, - inherit_tags: bool = True, - schedule_to_close: datetime | None = None, - start_to_close: timedelta | None = None, - heartbeat_timeout: timedelta | None = None, - # ── END new parameters ────────────────────────────── - ) -> JobHandle[R]: ... -``` - -#### Tag inheritance semantics - -| `inherit_tags` | `tags` | Resulting job tags | -|---|---|---| -| `True` (default) | `None` | Parent job's tags (or `()` if parent has none) | -| `True` (default) | `["new-tag"]` | Parent tags + explicit tags, merged (union, parent-first order, deduped) | -| `False` | `None` | `()` (current behavior) | -| `False` | `["new-tag"]` | `("new-tag",)` (explicit only, no inheritance) | - -#### Parent tag propagation via `contextvars.ContextVar` - -The `SubJobEnqueuer` is shared across concurrent consumers in the same event loop. A per-instance field would be racy. Instead, use a `contextvars.ContextVar` that the consumer sets before each actor invocation — asyncio Tasks copy the context, so concurrent consumers each see their own value. - -```python -# src/taskq/client/_enqueuer.py - -import contextvars - -_parent_tags_var: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar( - "taskq_parent_tags", - default=(), -) - -def set_parent_tags(tags: tuple[str, ...]) -> contextvars.Token[tuple[str, ...]]: - """Set the parent job's tags for sub-job tag inheritance. - - Called by the consumer before actor invocation. The returned token - can be used to reset the context after the actor completes. - """ - return _parent_tags_var.set(tags) -``` - -Inside `enqueue()`: - -```python -async def enqueue[P: BaseModel, R: BaseModel | None](self, ...) -> JobHandle[R]: - # Resolve tags with inheritance - resolved_tags = self._resolve_tags(tags, inherit_tags) - args = build_enqueue_args( - actor_ref, - payload, - # ... existing params ... - tags=resolved_tags, # NEW - schedule_to_close=schedule_to_close, # NEW - start_to_close=start_to_close, # NEW - heartbeat_timeout=heartbeat_timeout, # NEW - clock=self._clock, - ) - # ... rest unchanged ... - -def _resolve_tags( - self, - tags: list[str] | None, - inherit_tags: bool, -) -> list[str] | None: - """Resolve tags with parent inheritance. - - Returns a list suitable for build_enqueue_args, or None for empty. - """ - parent_tags = _parent_tags_var.get() if inherit_tags else () - - if tags is None: - if parent_tags: - return list(parent_tags) - return None - - if not inherit_tags or not parent_tags: - return tags - - # Merge: parent tags first, then explicit tags, deduped - seen: set[str] = set(parent_tags) - merged = list(parent_tags) - for tag in tags: - if tag not in seen: - seen.add(tag) - merged.append(tag) - return merged -``` - -#### Consumer integration - -```python -# src/taskq/worker/_consumer.py - -from taskq.client._enqueuer import _parent_tags_var, set_parent_tags - -# Inside consume_job(), before constructing JobContext. consume_job is a -# module-level function; the setting is read from _effective_settings -# (WorkerSettings | None, resolved from deps.settings or the settings -# parameter at _consumer.py:347). Settings absent (tests) → inherit. -_inherit = ( - _effective_settings is None or _effective_settings.sub_job_inherit_tags -) -token = set_parent_tags(tuple(job.tags)) if _inherit else None -try: - ctx: JobContext[BaseModel] = JobContext( - # ... existing fields ... - jobs=live_enqueuer, - # ... - ) - # ... run actor ... -finally: - if token is not None: - _parent_tags_var.reset(token) -``` - -The setting itself is added to `WorkerSettings` in `src/taskq/settings.py:338` (NOT `worker/_bootstrap.py` — that module has no settings class): - -```python -# src/taskq/settings.py — in WorkerSettings - -sub_job_inherit_tags: bool = Field( - default=True, - description=( - "When false, sub-jobs enqueued via ctx.jobs.enqueue() do not inherit " - "the parent job's tags (pre-1.0 behavior). Fleet-level kill switch " - "for the inherit_tags=True default; env var TASKQ_SUB_JOB_INHERIT_TAGS." - ), -) -``` - -Because `TaskQSettings` is a pydantic-settings class with `env_prefix = "TASKQ_"`, the field is automatically settable via the `TASKQ_SUB_JOB_INHERIT_TAGS` environment variable — no extra wiring for operators. - -The stub consumer in `worker/run.py` (a test harness with no settings object) follows the same set/reset pattern but calls `set_parent_tags(tuple(job.tags))` unconditionally. - -### Issue #54: Bulk cancel by filter - -#### New type: `BulkCancelResult` - -`BulkCancelResult` is defined in `taskq.backend._protocol` (next to `ScheduleRecord`, which is already a Pydantic `BaseModel` in that module) and re-exported through the same chain as `CancelResult`: `taskq.types` → `taskq.client` → `taskq`. This avoids the circular import that would arise from defining it in `types.py`: `types.py:18` already imports `from taskq.backend._protocol import JobId, JobStatus`, so a back-edge `_protocol → types` would fail at import time before `JobId` is defined. The `types.py` docstring claim that the protocol stays "pydantic-free" is already stale (`ScheduleRecord` at `_protocol.py:594` is a Pydantic model) — the implementation updates that docstring as part of Task 2. - -```python -# src/taskq/backend/_protocol.py — define next to ScheduleRecord - -class BulkCancelResult(BaseModel): - """Structured outcome of a bulk cancellation request. - - Returned by ``JobsClient.cancel_where()`` so callers can inspect - how many jobs were cancelled directly (pending/scheduled → terminal - cancelled) vs how many had cooperative cancel requested (running → - cancel_phase=1). - """ - - model_config = ConfigDict(frozen=True) - - cancelled_directly: int - """Count of pending/scheduled jobs moved straight to terminal 'cancelled'.""" - - cancel_requested: int - """Count of running jobs with cancel_phase=1 set (cooperative cancel).""" - - cancelled_ids: list[UUID] - """IDs of jobs cancelled directly (pending/scheduled → cancelled).""" - - cancel_requested_ids: list[UUID] - """IDs of running jobs with cancel requested.""" - - @property - def total_affected(self) -> int: - """Total jobs affected by the bulk cancel.""" - return self.cancelled_directly + self.cancel_requested -``` - -```python -# src/taskq/types.py — re-export (add to import and __all__) - -from taskq.backend._protocol import BulkCancelResult # noqa: F401 — re-export -__all__ = ["BulkCancelResult", "CancelResult", "StateChangeEvent"] - -# src/taskq/client/__init__.py — re-export alongside CancelResult - -from taskq.types import BulkCancelResult, CancelResult -__all__ = ["BulkCancelResult", "CancelResult", "JobEvent", "JobHandle", "JobsClient", "SubJobEnqueuer", "TaskQ"] - -# src/taskq/__init__.py — re-export at top level via the client surface -# (same import line pattern as CancelResult at __init__.py:40) -from taskq.client import BulkCancelResult, CancelResult, JobEvent, JobHandle, JobsClient, TaskQ -``` - -#### New exception: `EmptyFilterError` - -```python -# src/taskq/exceptions.py - -class EmptyFilterError(TaskQError): - """Raised when cancel_where is called with a filter that has no predicates. - - A filter with no queue, status, actor, identity_key, batch_id, tags, or - active predicate would match every job in the table — almost certainly - a bug. The guardrail is intentionally loud: the caller must add at least - one predicate or explicitly bypass with ``allow_empty_filter=True``. - """ - - def __init__(self) -> None: - super().__init__( - "cancel_where requires at least one filter predicate " - "(queue, status, actor, identity_key, batch_id, tags, or active); " - "an empty filter would cancel the entire table. " - "Pass allow_empty_filter=True to override this guardrail." - ) -``` - -#### Client API: `JobsClient.cancel_where()` - -```python -# src/taskq/client/_jobs.py - -class JobsClient: - async def cancel_where( - self, - filter: JobFilter, - reason: str | None = None, - *, - allow_empty_filter: bool = False, - ) -> BulkCancelResult: - """Cancel all jobs matching *filter* in a single set-based operation. - - Pending/scheduled jobs are moved straight to terminal 'cancelled' - (no running actor to cooperate with). Running jobs get - ``cancel_phase=1`` set (cooperative cancel) — the worker's - heartbeat-driven cancel controller observes the phase change and - sets the in-process ``cancel_event``. - - **Guardrail:** a filter with no predicates (no queue, status, - actor, identity_key, batch_id, tags, or active) is rejected with - :class:`EmptyFilterError` unless ``allow_empty_filter=True`` is - passed. This prevents accidental full-table cancels. - - **Filter fields used:** ``queue``, ``status``, ``actor``, - ``identity_key``, ``batch_id``, ``tags``, ``active``. The - ``limit``, ``cursor``, and ``order_by`` fields are ignored — a - bulk cancel is not paginated. - - **Snapshot boundary:** jobs matching *filter* that are enqueued - *after* the statement's snapshot escape this call. Stop producers - first, or issue a follow-up call; the returned counts make - non-convergence detectable. - - Returns a :class:`BulkCancelResult` with counts and affected IDs - for observability. - - Increments ``taskq.cancellation.requested`` once per call - (regardless of the number of jobs affected). - """ - ... -``` - -#### `TaskQ` delegate - -```python -# src/taskq/client/_taskq.py - -class TaskQ: - async def cancel_where( - self, - filter: JobFilter, - reason: str | None = None, - *, - allow_empty_filter: bool = False, - ) -> BulkCancelResult: - """Cancel all jobs matching *filter*. See :meth:`JobsClient.cancel_where`.""" - return await self._require_open().cancel_where( - filter, reason, allow_empty_filter=allow_empty_filter - ) -``` - -#### Backend protocol addition - -`BulkCancelResult` is already defined in `_protocol.py` (see above), so the protocol method's return annotation has no import dependency issue. - -```python -# src/taskq/backend/_protocol.py - -class Backend(Protocol): - # ── Cancel signals ────────────────────────────────────────── - async def write_cancel_request( - self, - job_id: JobId, - reason: str | None, - ) -> bool: ... - - async def cancel_where( - self, - filter: JobFilter, - reason: str | None, - ) -> BulkCancelResult: - """Cancel all jobs matching *filter* in a set-based operation. - - Pending/scheduled jobs → terminal 'cancelled'. - Running jobs → cancel_phase=1 (cooperative cancel + NOTIFY). - - The filter's ``limit``, ``cursor``, and ``order_by`` fields are - ignored — this is a bulk write, not a paginated read. - - Returns a :class:`BulkCancelResult` with counts and affected IDs. - """ - ... -``` - -**Protocol version:** No bump required. `cancel_where` is purely additive — a v3 implementation lacks the method, and the client raises `AttributeError` loudly (not a silent misbehavior). See the bump rule in `_protocol.py` lines 79-84. - -#### PostgresBackend SQL - -The SQL uses two CTEs in a single statement: one for pending/scheduled (→ terminal cancelled), one for running (→ cooperative cancel_phase=1). Events are inserted within the same transaction via `executemany`. - -**EPQ safety (Critical design requirement):** Under READ COMMITTED, when an UPDATE reaches a row that a concurrent transaction has locked/updated, Postgres waits, then re-evaluates the UPDATE's **own WHERE clause** (EvalPlanQual) against the new row version. Predicates inside a materialized CTE are **not** re-evaluated. Therefore, the status/cancel_phase predicates must appear **both** in the `matching` CTE (for snapshot-time row selection) **and** in each UPDATE's own WHERE clause (for EPQ re-evaluation). This mirrors the existing single-job path: `cancel_pending_scheduled` (`_sql_templates.py:407-415`) locks `FOR UPDATE` and repeats `AND status IN ('pending','scheduled')` in the UPDATE's WHERE; `cancel_running` (`_sql_templates.py:416-420`) puts `AND status = 'running' AND cancel_phase = 0` directly in the UPDATE. - -**Residual window:** A job claimed by a worker (pending→running) between the statement snapshot and the row lock will be skipped by this call (the EPQ re-check sees `running` and rejects it from the pending/scheduled UPDATE). This is correct and safe — the job escapes this cancel call and requires a subsequent `cancel_where` (or the caller stops producers first). This is strictly preferable to overwriting a running job to terminal `cancelled` while a worker executes it. - -**Lock ordering / deadlock handling:** The `matching` CTE scans with `ORDER BY id` so the UPDATEs acquire row locks in ascending `id` order, reducing deadlock probability against dispatch (`FOR UPDATE SKIP LOCKED`, `_dispatch_sql.py:125`) and heartbeat lock ordering. Eliminating deadlocks outright is not claimed — when Postgres detects one it aborts this statement with `asyncpg.DeadlockDetectedError`, and the **backend** retries the whole transaction (max 3 attempts, jittered backoff; safe because the single transaction rolls back atomically and the EPQ predicates re-filter on every attempt). The retry is backend-owned, not client-owned: the exception type is asyncpg-specific and the backend owns the transaction boundary. - -**Large result sets:** A single `cancel_where` call is bounded by transaction size. For tenant-scale cancels (10⁵+ matching rows), the operator should partition via filter (e.g., `JobFilter(queue=..., tags=...)` to split by queue). The implementation does not chunk internally — a single transaction covering 10⁶ rows would hold locks too long. The `BulkCancelResult` counts let the caller verify completeness and issue follow-up calls for remaining partitions. Document this guidance in `jobs-clients.md`. - -**Event parity:** Both backends insert the same event kinds as the existing single-job `write_cancel_request` path: for pending/scheduled jobs, both `state_change` (with actual `from_state`) and `cancel_request`; for running jobs, only `cancel_request`. This matches `postgres.py:555-561` and `in_memory.py:595-601`. - -**Post-snapshot enqueue boundary:** Jobs matching the filter that are enqueued *after* the statement's snapshot escape the cancel. Convergence is the caller's responsibility — stop producers before calling `cancel_where`, or issue a second call to catch stragglers. The `BulkCancelResult` counts let the caller detect non-convergence. - -```sql --- src/taskq/backend/_cancel_bulk.py — cancel_where SQL (inlined, dynamic) - --- $1..$N: filter parameters (same positional binding as list_jobs). --- The reason is never interpolated into SQL or JSON text — it is bound --- per-row as a jsonb parameter at event-insert time (see below). - -WITH matching AS ( - SELECT id, status, locked_by_worker - FROM "{schema}".jobs - WHERE {filter_conditions} - ORDER BY id -- deterministic lock ordering to reduce deadlocks -), -cancelled AS ( - UPDATE "{schema}".jobs AS j - SET status = 'cancelled', - finished_at = clock_timestamp() - FROM ( - SELECT id, status AS prev_status - FROM matching - WHERE status IN ('pending', 'scheduled') - ) AS prev - WHERE j.id = prev.id - AND j.status IN ('pending', 'scheduled') -- EPQ re-check (Critical) - RETURNING j.id, prev.prev_status -), -cancel_requested AS ( - UPDATE "{schema}".jobs AS j - SET cancel_requested_at = now(), - cancel_phase = 1 - WHERE j.id IN ( - SELECT id FROM matching - WHERE status = 'running' AND cancel_phase = 0 - ) - AND j.status = 'running' AND j.cancel_phase = 0 -- EPQ re-check (Critical) - RETURNING j.id, j.locked_by_worker -) -SELECT - (SELECT count(*)::int FROM cancelled) AS cancelled_directly, - (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, - (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, - (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, - (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, - (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers -``` - -Three points of correctness in this shape: - -- **`FROM prev` captures the previous status** in the same round-trip (mirroring the single-job `cancel_pending_scheduled` template at `_sql_templates.py:407-415`). `UPDATE ... RETURNING` alone can only return the *new* row; the `prev` subquery carries the snapshot status through so `state_change` events record the actual `from_state` (`'pending'` or `'scheduled'`), not a synthetic placeholder. The target table is aliased (`AS j`) so `RETURNING j.id` is unambiguous against `prev.id`. -- **The EPQ-re-checked predicates are the ones on the target table** (`j.status ...`, `j.cancel_phase ...`). Under READ COMMITTED, when the UPDATE blocks on a concurrently-locked row, Postgres re-evaluates the UPDATE's own WHERE clause against the newest row version; `prev.*` values come from the statement snapshot and are not re-evaluated — which is exactly why the status predicates must live on `j`, not only inside `matching`/`prev`. If a job was claimed (→ `running`) or finished (→ terminal) mid-statement, the re-check rejects it and the row is skipped. -- **Aggregate arrays are `ORDER BY id`-aligned**, so Python can zip `cancelled_ids` with `cancelled_prev_statuses` (per-job `from_state`) and `cancel_requested_ids` with `cancel_requested_workers` (NOTIFY targets) — no second query. - -Events are inserted in separate statements within the same transaction (after the main CTE query returns counts/IDs), reusing the existing `sql.insert_event` template with `executemany`; the `detail` JSON is serialized in Python via `jsonb_param` (never f-string interpolation — see the H2 design note in Task 5). - -**NOTIFY for running jobs:** After the transaction commits, `PostgresBackend.cancel_where` sends `pg_notify` to the fleet channel and each affected job's per-worker channel, reusing the channel helpers and payload shape from `write_cancel_request` (`events_channel`/`worker_channel` in `constants.py:221,236`; pattern at `postgres.py:585-603`). For bulk cancel, the NOTIFY calls are batched into a single statement to avoid N round-trips: - -```sql -SELECT pg_notify(channel, payload) -FROM unnest($1::text[], $2::text[]) AS t(channel, payload) -``` - -The send lives in `postgres.py` (not `_cancel_bulk.py`) because the `taskq.cancel.notify_sent` counter is module-level there (`postgres.py:146-149,603`) — importing it from `_cancel_bulk` would create a module cycle. The counter is incremented once per job notified (batch `.add(len(notify_targets))`), matching the single-job path's per-job semantics. - -**Event insertion:** Events are inserted via `executemany` within the same transaction: -- For cancelled (pending/scheduled) jobs: one `state_change` event with `from_state` set to the actual previous status (from `cancelled_prev_statuses`) and one `cancel_request` event — matching the single-job path (`postgres.py:555-561`). Details: `jsonb_param({"from_state": prev_status, "to_state": "cancelled"})` and `jsonb_param({"reason": reason} if reason is not None else {})`. -- For cancel_requested (running) jobs: one `cancel_request` event with `jsonb_param({"reason": reason} if reason is not None else {})`. - -#### In-memory backend implementation - -The in-memory backend must **not** call `_list_jobs` directly with the caller's filter, because `_list_jobs` applies `filters.limit` (default 100) and cursor slicing (`testing/_reads.py:87-98`) — silently capping the cancel to 100 rows and contradicting the contract that `limit`, `cursor`, and `order_by` are ignored. Instead, call `_list_jobs` with a **sanitized filter** that unsets `limit`, `cursor`, and `order_by`: - -```python -# src/taskq/testing/_cancel_bulk.py - -from dataclasses import replace as dc_replace -from typing import TYPE_CHECKING -from uuid import UUID - -from taskq.backend._protocol import BulkCancelResult, CancelPhase, JobFilter -from taskq.testing._reads import _list_jobs - -if TYPE_CHECKING: - from taskq.testing.in_memory import InMemoryBackend - -async def _cancel_where( - self: "InMemoryBackend", # module-fn style, like testing/_reads.py - filter: JobFilter, - reason: str | None, -) -> BulkCancelResult: - # Sanitize the filter: cancel_where ignores limit, cursor, and order_by. - # Use a very large limit (2**31) instead of None because JobFilter.limit - # is typed as int (not int | None) with a __post_init__ guard against - # negatives. cursor=None disables keyset slicing. order_by=None selects - # the default priority/scheduled_at/id sort, which is harmless for cancel. - sanitized = dc_replace(filter, limit=2**31, cursor=None, order_by=None) - rows = await _list_jobs(self, sanitized) - - cancelled_ids: list[UUID] = [] - cancel_requested_ids: list[UUID] = [] - - for row in rows: - if row.status in ("pending", "scheduled"): - now = self._clock.now() - self._jobs[row.id] = dc_replace( - row, - status="cancelled", - finished_at=now, - ) - self._append_state_change_event( - job_id=row.id, - from_state=row.status, # actual previous status - to_state="cancelled", - now=now, - ) - self._append_cancel_request_event(row.id, now, reason) - cancelled_ids.append(row.id) - - elif row.status == "running" and row.cancel_phase == CancelPhase.NONE: - now = self._clock.now() - self._jobs[row.id] = dc_replace( - row, - cancel_requested_at=now, - cancel_phase=CancelPhase.COOPERATIVE, - ) - self._append_cancel_request_event(row.id, now, reason) - for event in self._cancel_wake_subscribers: - event.set() - cancel_requested_ids.append(row.id) - - return BulkCancelResult( - cancelled_directly=len(cancelled_ids), - cancel_requested=len(cancel_requested_ids), - cancelled_ids=cancelled_ids, - cancel_requested_ids=cancel_requested_ids, - ) -``` - -**Event parity:** The in-memory implementation inserts the same event kinds as the existing single-job `write_cancel_request` path and the Postgres bulk path: for pending/scheduled jobs, both `state_change` (with actual `from_state`) and `cancel_request`; for running jobs, only `cancel_request`. This matches `in_memory.py:595-601`. - -#### Filter→WHERE reuse - -The filter-to-SQL-WHERE builder in `_reads._list_jobs` (lines 53-141) builds conditions dynamically. For `cancel_where`, we need the same WHERE clause. Extract the **predicate-only** condition builder (queue, status, actor, identity_key, batch_id, tags, active) into a shared helper. The `schema` parameter is **not** needed — the existing builder in `_reads.py:58-115` produces schema-less fragments (the schema is applied by the caller in the surrounding SQL string). - -```python -# src/taskq/backend/_filter_sql.py (NEW) - -@dataclass(frozen=True, slots=True) -class FilterSQL: - """Built SQL fragments and parameters from a JobFilter.""" - conditions: list[str] - params: list[object] - -def build_filter_conditions(filter: JobFilter) -> FilterSQL: - """Build WHERE clause conditions and parameters from a JobFilter. - - Shared between _list_jobs (reads) and cancel_where (writes) so the - filter semantics are identical for query and mutation. - - Only predicate fields (queue, status, actor, identity_key, batch_id, - tags, active) are translated to conditions. The ``cursor``, ``limit``, - and ``order_by`` fields are NOT handled here — callers apply them - separately: - - - ``_list_jobs`` appends the cursor keyset condition and LIMIT/OFFSET - after calling this helper (preserving the existing behavior at - ``_reads.py:102-115``). - - ``cancel_where`` ignores cursor/limit/order_by entirely (bulk writes - are not paginated). - """ - # ... extracted from _reads._list_jobs lines 58-100 (the predicate - # fields). The cursor keyset block (lines 102-115) and LIMIT/ORDER BY - # (lines 117-138) stay in _list_jobs and are appended after this call. -``` - -Both `_list_jobs` and `_cancel_bulk` call this helper, ensuring filter semantics are DRY. After extraction, `_list_jobs` regains cursor/limit/order_by handling by appending the keyset condition (`_reads.py:102-115`) and `LIMIT $N` after the shared `build_filter_conditions` call — the existing `test_job_filter.py` and `test_postgres_reads.py` suites verify no regression. - ---- - -## Implementation Plan - -### Task 1: Extract filter→SQL builder (refactor) - -**Goal:** Extract the WHERE-clause builder from `_reads._list_jobs` into a shared module so `cancel_where` reuses the exact same filter logic. - -**Files:** -- CREATE: `src/taskq/backend/_filter_sql.py` -- MODIFY: `src/taskq/backend/_reads.py` — import and use the shared builder -- CREATE: `tests/test_filter_sql.py` - -#### TDD — Red - -```python -# tests/test_filter_sql.py - -from taskq.backend._filter_sql import build_filter_conditions, FilterSQL -from taskq.backend._protocol import JobFilter - -class TestBuildFilterConditions: - def test_empty_filter_produces_no_conditions(self) -> None: - result = build_filter_conditions(JobFilter()) - assert result.conditions == [] - assert result.params == [] - - def test_queue_filter(self) -> None: - result = build_filter_conditions(JobFilter(queue="default")) - assert len(result.conditions) == 1 - assert "queue" in result.conditions[0] - assert result.params == ["default"] - - def test_tags_filter(self) -> None: - result = build_filter_conditions(JobFilter(tags=("alpha", "beta"))) - assert len(result.conditions) == 1 - assert "tags" in result.conditions[0] - assert result.params == [["alpha", "beta"]] - - def test_batch_id_filter(self) -> None: - from uuid import uuid4 - bid = uuid4() - result = build_filter_conditions(JobFilter(batch_id=bid)) - assert len(result.conditions) == 1 - assert "metadata" in result.conditions[0] - - def test_active_true_filter(self) -> None: - result = build_filter_conditions(JobFilter(active=True)) - assert len(result.conditions) == 1 - assert "status" in result.conditions[0] - - def test_combined_filters(self) -> None: - result = build_filter_conditions( - JobFilter(queue="e2e", actor="my_actor", tags=("run-123",)), - ) - assert len(result.conditions) == 3 - - def test_cursor_and_order_by_ignored(self) -> None: - """cancel_where doesn't use cursor/order_by — the builder should - not include them in conditions.""" - result = build_filter_conditions( - JobFilter(cursor="some-cursor", order_by=None), - ) - # cursor/order_by are not part of filter conditions - assert result.conditions == [] -``` - -#### TDD — Green - -Extract the condition-building logic from `_reads._list_jobs` into `build_filter_conditions()`. Update `_list_jobs` to call it. Run existing `test_job_filter.py` and `test_postgres_reads.py` to verify no regression. - -#### Acceptance criteria -- All existing `test_job_filter.py` tests pass -- All existing `test_postgres_reads.py` tests pass -- New `test_filter_sql.py` tests pass -- `build_filter_conditions` is pure (no I/O, no global state) - ---- - -### Task 2: Add `BulkCancelResult` type and `EmptyFilterError` exception - -**Goal:** Define the result type and guardrail exception before implementing the operation. - -**Files:** -- MODIFY: `src/taskq/backend/_protocol.py` — define `BulkCancelResult` (next to `ScheduleRecord`) -- MODIFY: `src/taskq/types.py` — re-export `BulkCancelResult`; reconcile the stale "pydantic-free" docstring -- MODIFY: `src/taskq/client/__init__.py` — re-export `BulkCancelResult` alongside `CancelResult` -- MODIFY: `src/taskq/exceptions.py` — add `EmptyFilterError` -- MODIFY: `src/taskq/__init__.py` — export both -- CREATE: `tests/test_bulk_cancel_types.py` - -> **Why `_protocol.py`, not `types.py`:** `types.py:18` imports `from taskq.backend._protocol import JobId, JobStatus`. If `_protocol.py` imported `BulkCancelResult` from `types.py`, the cycle `_protocol → types → _protocol` would fail at import time before `JobId` (line 198) is defined. Defining `BulkCancelResult` in `_protocol.py` (where `ScheduleRecord`, another Pydantic `BaseModel`, already lives at line 594) avoids the cycle. The `types.py` docstring claim that the protocol is "pydantic-free" is already stale due to `ScheduleRecord` and should be updated. - -#### TDD — Red - -```python -# tests/test_bulk_cancel_types.py - -from uuid import uuid4 -import pytest -from taskq.types import BulkCancelResult -from taskq.exceptions import EmptyFilterError - -class TestBulkCancelResult: - def test_construction(self) -> None: - ids = [uuid4() for _ in range(3)] - result = BulkCancelResult( - cancelled_directly=2, - cancel_requested=1, - cancelled_ids=ids[:2], - cancel_requested_ids=ids[2:], - ) - assert result.cancelled_directly == 2 - assert result.cancel_requested == 1 - assert result.total_affected == 3 - assert len(result.cancelled_ids) == 2 - assert len(result.cancel_requested_ids) == 1 - - def test_frozen(self) -> None: - result = BulkCancelResult( - cancelled_directly=0, - cancel_requested=0, - cancelled_ids=[], - cancel_requested_ids=[], - ) - with pytest.raises(Exception): - result.cancelled_directly = 1 # type: ignore[misc] - - def test_zero_counts(self) -> None: - result = BulkCancelResult( - cancelled_directly=0, - cancel_requested=0, - cancelled_ids=[], - cancel_requested_ids=[], - ) - assert result.total_affected == 0 - -class TestEmptyFilterError: - def test_is_taskq_error(self) -> None: - from taskq.exceptions import TaskQError - assert issubclass(EmptyFilterError, TaskQError) - - def test_message_mentions_guardrail(self) -> None: - err = EmptyFilterError() - assert "allow_empty_filter" in str(err) - assert "filter predicate" in str(err) -``` - -#### TDD — Green - -Add the types. Run the tests. - -#### Acceptance criteria -- `BulkCancelResult` is a frozen Pydantic model with `total_affected` property -- `EmptyFilterError` is a `TaskQError` subclass with a helpful message -- Both are exported from `taskq` top-level - ---- - -### Task 3: Add `cancel_where` to `Backend` protocol - -**Goal:** Add the method signature to the `Backend` protocol. - -**Files:** -- MODIFY: `src/taskq/backend/_protocol.py` — add `cancel_where` method; update docstring count -- MODIFY: `tests/test_backend_protocol.py` — update member count and expected member set - -#### Implementation - -```python -# In Backend protocol, after write_cancel_request: -async def cancel_where( - self, - filter: JobFilter, - reason: str | None, -) -> BulkCancelResult: - """Cancel all jobs matching *filter* in a set-based operation.""" - ... -``` - -#### Protocol docstring update - -The `Backend` class docstring (`_protocol.py:693-697`) currently says "31 async methods plus two sync methods (33 methods total)". Adding `cancel_where` (async) makes it **32 async methods plus two sync methods (34 methods total)**. Update the docstring accordingly. - -#### Test updates - -`tests/test_backend_protocol.py:218-262` asserts exactly 36 public members and an exact member-name set. Adding `cancel_where` brings the count to **37**. Update: -- `test_exactly_thirty_six_public_members` → `test_exactly_thirty_seven_public_members` with `assert len(public) == 37` -- Add `"cancel_where"` to the `expected` set in `test_all_member_names_present` - -#### TDD — Red - -```python -# tests/test_backend_protocol.py — add to existing test file - -async def test_protocol_has_cancel_where() -> None: - """Backend protocol declares cancel_where.""" - from taskq.backend._protocol import Backend - assert hasattr(Backend, "cancel_where") -``` - -#### Acceptance criteria -- `Backend` protocol includes `cancel_where` method -- Protocol version not bumped (purely additive, loud failure on missing method) -- `test_backend_protocol.py` updated: member count is 37, `cancel_where` in expected set, docstring count updated to 34 methods total -- All `test_backend_protocol.py` tests pass after update - ---- - -### Task 4: Implement `cancel_where` for InMemoryBackend - -**Goal:** Add bulk cancel to the in-memory backend for unit testing. - -**Files:** -- CREATE: `src/taskq/testing/_cancel_bulk.py` -- MODIFY: `src/taskq/testing/in_memory.py` — wire `cancel_where` method -- CREATE: `tests/test_cancel_where.py` - -#### TDD — Red - -```python -# tests/test_cancel_where.py - -import pytest -from uuid import uuid4 -from taskq.backend._protocol import JobFilter -from taskq.testing.clock import FakeClock -from taskq.testing.in_memory import InMemoryBackend -from taskq.testing.jobs import make_enqueue_args - -from datetime import UTC, datetime - -_NOW = datetime(2026, 1, 1, tzinfo=UTC) - -async def test_cancel_where_pending_jobs() -> None: - """cancel_where moves pending jobs straight to 'cancelled'.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - # Enqueue 3 jobs with tag "tenant-acme", 2 without - for i in range(3): - await backend.enqueue(make_enqueue_args(tags=("tenant-acme", "run-001"), scheduled_at=_NOW)) - for i in range(2): - await backend.enqueue(make_enqueue_args(tags=("tenant-other",), scheduled_at=_NOW)) - - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.cancelled_directly == 3 - assert result.cancel_requested == 0 - assert result.total_affected == 3 - assert len(result.cancelled_ids) == 3 - - # Verify the untagged jobs are still pending - remaining = await backend.list_jobs(JobFilter(tags=("tenant-other",))) - assert len(remaining) == 2 - assert all(r.status == "pending" for r in remaining) - -async def test_cancel_where_running_jobs_cooperative() -> None: - """cancel_where sets cancel_phase=1 for running jobs (cooperative).""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - # Enqueue and manually dispatch to running - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend.enqueue(args) - from dataclasses import replace - # Simulate dispatch: set to running - backend._jobs[row.id] = replace(backend._jobs[row.id], status="running", locked_by_worker=uuid4()) - - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.cancelled_directly == 0 - assert result.cancel_requested == 1 - assert len(result.cancel_requested_ids) == 1 - - # Verify the job is still running but has cancel_phase=1 - updated = await backend.get(row.id) - assert updated is not None - assert updated.status == "running" - assert updated.cancel_phase == 1 # CancelPhase.COOPERATIVE - -async def test_cancel_where_mixed_statuses() -> None: - """cancel_where handles both pending and running in one call.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - # 2 pending + 1 running, all tagged "tenant-acme" - for _ in range(2): - await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) - - args3 = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row3 = await backend.enqueue(args3) - from dataclasses import replace - backend._jobs[row3.id] = replace(backend._jobs[row3.id], status="running", locked_by_worker=uuid4()) - - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.cancelled_directly == 2 - assert result.cancel_requested == 1 - assert result.total_affected == 3 - -async def test_cancel_where_no_matches_returns_zero() -> None: - """cancel_where with a filter matching nothing returns zero counts.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) - - result = await backend.cancel_where( - JobFilter(tags=("nonexistent",)), - reason="offboard", - ) - - assert result.cancelled_directly == 0 - assert result.cancel_requested == 0 - assert result.total_affected == 0 - -async def test_cancel_where_already_cancelled_not_affected() -> None: - """Already-terminal jobs are not re-cancelled.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend.enqueue(args) - from dataclasses import replace - backend._jobs[row.id] = replace(backend._jobs[row.id], status="cancelled", finished_at=_NOW) - - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.total_affected == 0 - -async def test_cancel_where_filter_by_batch_id() -> None: - """cancel_where works with batch_id filter.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - bid = uuid4() - for i in range(3): - args = make_enqueue_args( - tags=("tenant-acme",), - scheduled_at=_NOW, - metadata={"batch_id": str(bid)}, - ) - await backend.enqueue(args) - # Untagged job with different batch - await backend.enqueue(make_enqueue_args(scheduled_at=_NOW, metadata={"batch_id": str(uuid4())})) - - result = await backend.cancel_where( - JobFilter(batch_id=bid), - reason="batch abort", - ) - - assert result.cancelled_directly == 3 - -async def test_cancel_where_filter_by_queue_and_actor() -> None: - """cancel_where works with queue and actor filters.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - await backend.enqueue(make_enqueue_args(queue="default", actor="worker-a", scheduled_at=_NOW)) - await backend.enqueue(make_enqueue_args(queue="default", actor="worker-b", scheduled_at=_NOW)) - await backend.enqueue(make_enqueue_args(queue="priority", actor="worker-a", scheduled_at=_NOW)) - - result = await backend.cancel_where( - JobFilter(queue="default", actor="worker-a"), - reason="abort", - ) - - assert result.cancelled_directly == 1 - -async def test_cancel_where_active_filter() -> None: - """cancel_where with active=True targets only non-terminal jobs.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - # 2 active (pending) + 1 terminal (succeeded) - await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) - await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) - args3 = make_enqueue_args(scheduled_at=_NOW) - row3 = await backend.enqueue(args3) - from dataclasses import replace - backend._jobs[row3.id] = replace(backend._jobs[row3.id], status="succeeded", finished_at=_NOW) - - result = await backend.cancel_where( - JobFilter(active=True), - reason="drain", - ) - - assert result.cancelled_directly == 2 # only the 2 pending - -async def test_cancel_where_ignores_filter_limit() -> None: - """cancel_where cancels ALL matching jobs even when filter.limit is small. - - This guards against the H3 bug: _list_jobs applies filters.limit (default - 100). If _cancel_where reuses _list_jobs without sanitizing the filter, - a caller passing JobFilter(limit=5, tags=...) would cancel only 5 jobs. - """ - backend = InMemoryBackend(clock=FakeClock(_NOW)) - - # Enqueue 11 jobs matching the tag - for _ in range(11): - await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) - - # Pass a restrictive limit — cancel_where must ignore it - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",), limit=5), - reason="offboard", - ) - - assert result.cancelled_directly == 11 # all of them, not just 5 - assert result.total_affected == 11 -``` - -#### TDD — Green - -Implement `_cancel_where` in `src/taskq/testing/_cancel_bulk.py`, wire it in `in_memory.py`. - -#### Acceptance criteria -- All red tests pass -- `InMemoryBackend.cancel_where` satisfies the `Backend` protocol -- Events are inserted for both directly-cancelled and cooperative-cancel jobs -- Cancel wake subscribers are notified for running jobs -- `cancel_where` ignores `filter.limit` and `filter.cursor` — all matching jobs are cancelled regardless of pagination fields (verified by `test_cancel_where_ignores_filter_limit`) - ---- - -### Task 5: Implement `cancel_where` for PostgresBackend - -**Goal:** Add the set-based SQL bulk cancel to the Postgres backend. - -**Files:** -- CREATE: `src/taskq/backend/_cancel_bulk.py` -- MODIFY: `src/taskq/backend/postgres.py` — wire `cancel_where` method -- CREATE: `tests/test_cancel_where_pg.py` - -> **No `SqlTemplates` field needed:** The SQL is inlined in `_cancel_bulk.py` with dynamic filter conditions baked in via f-string (matching the dynamic-SQL precedent in `_reads.py`). A `SqlTemplates.cancel_where` field would require template-level `{filter_conditions}` placeholder substitution that doesn't fit the static-template rendering model — the filter conditions are built at call time from `build_filter_conditions()`, not at schema-render time. - -#### Implementation - -```python -# src/taskq/backend/_cancel_bulk.py - -import asyncio -import random - -import asyncpg - -from taskq.backend._filter_sql import build_filter_conditions -from taskq.backend._records import jsonb_param - -# Returns (result, notify_targets) where notify_targets is -# [(job_id, worker_id)] for running jobs that got cooperative cancel. -# NOTIFY itself is sent by PostgresBackend.cancel_where (see wiring below) -# because the taskq.cancel.notify_sent counter is module-level in postgres.py. -async def _cancel_where( - pool: asyncpg.Pool, - schema: str, - sql: SqlTemplates, - filter: JobFilter, - reason: str | None, -) -> tuple[BulkCancelResult, list[tuple[UUID, UUID]]]: - filter_sql = build_filter_conditions(filter) - conditions_str = " AND ".join(filter_sql.conditions) if filter_sql.conditions else "TRUE" - params = filter_sql.params - - # Single CTE statement: snapshot matching IDs, then two UPDATEs with - # EPQ-safe predicates duplicated in each UPDATE's own WHERE clause. - # ORDER BY id in the matching CTE ensures deterministic lock ordering - # to reduce deadlock probability. - cancel_sql = f""" - WITH matching AS ( - SELECT id, status, locked_by_worker - FROM "{schema}".jobs - WHERE {conditions_str} - ORDER BY id - ), - cancelled AS ( - UPDATE "{schema}".jobs AS j - SET status = 'cancelled', finished_at = clock_timestamp() - FROM ( - SELECT id, status AS prev_status - FROM matching - WHERE status IN ('pending', 'scheduled') - ) AS prev - WHERE j.id = prev.id - AND j.status IN ('pending', 'scheduled') -- EPQ re-check (Critical) - RETURNING j.id, prev.prev_status - ), - cancel_requested AS ( - UPDATE "{schema}".jobs AS j - SET cancel_requested_at = now(), cancel_phase = 1 - WHERE j.id IN ( - SELECT id FROM matching - WHERE status = 'running' AND cancel_phase = 0 - ) - AND j.status = 'running' AND j.cancel_phase = 0 -- EPQ re-check (Critical) - RETURNING j.id, j.locked_by_worker - ) - SELECT - (SELECT count(*)::int FROM cancelled) AS cancelled_directly, - (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, - (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, - (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, - (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, - (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers - """ - - # Deadlock retry: the bulk UPDATE locks rows in id order while dispatch - # and heartbeat transactions lock rows in their own orders, so Postgres - # may abort this statement with DeadlockDetectedError. The retry lives in - # the BACKEND (not JobsClient) because the exception type is - # asyncpg-specific and the client layer stays backend-agnostic. The whole - # UPDATE+events runs in one transaction, so a deadlocked attempt rolls - # back completely and re-execution from a fresh snapshot is safe (the - # EPQ predicates re-filter on every attempt). NOTIFY is sent by the - # caller only after a successful commit, so no notify can fire for a - # rolled-back attempt. - for attempt in range(3): - try: - async with pool.acquire() as conn: - async with conn.transaction(): - row = await conn.fetchrow(cancel_sql, *params) - - assert row is not None # aggregate SELECT always returns one row - cancelled_ids: list[UUID] = list(row["cancelled_ids"] or []) - cancel_requested_ids: list[UUID] = list(row["cancel_requested_ids"] or []) - prev_statuses: dict[UUID, str] = dict( - zip(cancelled_ids, row["cancelled_prev_statuses"] or [], strict=True) - ) - notify_targets = [ - (jid, wid) - for jid, wid in zip( - cancel_requested_ids, - row["cancel_requested_workers"] or [], - strict=True, - ) - if wid is not None - ] - - # Events — same kinds as single-job write_cancel_request - # (postgres.py:555-561): state_change + cancel_request for - # pending/scheduled; cancel_request only for running. - # detail JSON is serialized in Python via jsonb_param — - # never f-string interpolation (a reason containing " - # or \ would otherwise produce malformed jsonb and abort - # the transaction; see H2 design note below). - cr_detail = jsonb_param({"reason": reason} if reason is not None else {}) - if cancelled_ids: - await conn.executemany( - sql.insert_event, # (job_id, kind, detail) — kind is $2 - [ - ( - jid, - "state_change", - jsonb_param( - {"from_state": prev_statuses[jid], "to_state": "cancelled"} - ), - ) - for jid in cancelled_ids - ], - ) - await conn.executemany( - sql.insert_event, - [(jid, "cancel_request", cr_detail) for jid in cancelled_ids], - ) - if cancel_requested_ids: - await conn.executemany( - sql.insert_event, - [(jid, "cancel_request", cr_detail) for jid in cancel_requested_ids], - ) - break # committed successfully - except asyncpg.DeadlockDetectedError: - if attempt == 2: - raise - await asyncio.sleep(0.1 * (2**attempt) + random.random() * 0.05) - - result = BulkCancelResult( - cancelled_directly=len(cancelled_ids), - cancel_requested=len(cancel_requested_ids), - cancelled_ids=cancelled_ids, - cancel_requested_ids=cancel_requested_ids, - ) - return result, notify_targets -``` - -#### Wiring in `postgres.py` - -```python -# src/taskq/backend/postgres.py - -async def cancel_where( - self, - filter: JobFilter, - reason: str | None, -) -> BulkCancelResult: - result, notify_targets = await _cancel_bulk._cancel_where( - self._worker_pool, self._schema_name, self._sql, filter, reason - ) - if notify_targets: - # Post-commit NOTIFY, same pattern as write_cancel_request - # (postgres.py:585-603) but batched into one statement. - channels: list[str] = [] - payloads: list[str] = [] - for job_id, worker_id in notify_targets: - payload = dumps_str( - {"type": "cancel", "job_id": str(job_id), "worker_id": str(worker_id)} - ) - channels.extend( - [ - events_channel(self._schema_name), - worker_channel(self._schema_name, str(worker_id)), - ] - ) - payloads.extend([payload, payload]) - async with self._worker_pool.acquire() as notify_conn: - await notify_conn.execute( - "SELECT pg_notify(channel, payload) " - "FROM unnest($1::text[], $2::text[]) AS t(channel, payload)", - channels, - payloads, - ) - _cancel_notify_sent_counter.add(len(notify_targets), {"schema": self._schema_name}) - return result -``` - -**Design note — capturing `prev_status` (L5):** The `cancelled` CTE uses the `FROM prev` pattern from the existing single-job `cancel_pending_scheduled` template (`_sql_templates.py:407-415`) so the actual previous status (`'pending'` or `'scheduled'`) flows into the `state_change` event detail in a single round-trip — not a synthetic `'pending_or_scheduled'` placeholder, and no second query. Two details matter: the target table must be aliased (`AS j`) because `prev` also exposes an `id` column (`RETURNING id` would be ambiguous), and the EPQ-re-checked predicate must reference the target table (`j.status`), since EPQ re-evaluates only the UPDATE's own WHERE clause against the newest row version — `prev.*` values are snapshot values. - -**Design note — safe JSON serialization (H2 fix):** The `reason` string is serialized in Python via `jsonb_param({"reason": reason})` (which uses `dumps_str`/orjson), then bound as a jsonb parameter. This avoids the f-string injection bug where `reason` containing `"` or `\` would produce invalid JSON → `asyncpg.DataError` mid-transaction (rolling back the entire bulk cancel), or structurally valid but operator-shaped JSON. The existing single-job path does this safely at `_terminal.py:129-144`. The red test `test_pg_cancel_where_reason_with_quotes` pins the fix. - -**Design note — deadlock retry (M6):** The retry loop lives in `_cancel_bulk._cancel_where` (the backend), not in `JobsClient`, for two reasons: the exception type is `asyncpg.DeadlockDetectedError` — catching it in the client would couple the backend-agnostic client layer to asyncpg — and the backend owns the transaction boundary, so it alone can guarantee that a retried attempt starts from a fresh snapshot with no partial effects (the single transaction rolls back UPDATEs and event inserts atomically). Max 3 attempts with jittered exponential backoff (100ms base). The `ORDER BY id` in the `matching` CTE reduces (but does not eliminate) deadlock probability against dispatch/heartbeat lock ordering. NOTIFY is sent by `PostgresBackend.cancel_where` only after a successful commit, so a retried-or-failed attempt never fires a spurious notify. The in-memory backend never deadlocks and needs no retry. - -#### TDD — Red - -```python -# tests/test_cancel_where_pg.py - -import asyncio -from datetime import UTC, datetime -from uuid import uuid4 - -import pytest - -from taskq.backend._protocol import JobFilter -from taskq.testing.fixtures import JobsApp -from taskq.testing.jobs import make_enqueue_args - -_NOW = datetime(2026, 1, 1, tzinfo=UTC) - -@pytest.mark.integration -class TestCancelWherePostgres: - async def test_pg_cancel_where_pending(self, backend_pair) -> None: - """PostgresBackend.cancel_where cancels pending jobs by tag.""" - from taskq.types import BulkCancelResult - - for _ in range(3): - await backend_pair.enqueue( - make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - ) - await backend_pair.enqueue( - make_enqueue_args(tags=("other",), scheduled_at=_NOW) - ) - - result = await backend_pair.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.cancelled_directly == 3 - assert result.cancel_requested == 0 - - # Verify via list - remaining = await backend_pair.list_jobs(JobFilter(tags=("other",))) - assert len(remaining) == 1 - assert remaining[0].status == "pending" - - async def test_pg_cancel_where_events_inserted(self, backend_pair) -> None: - """cancel_where inserts job_events for cancelled jobs — both - state_change (with actual from_state) and cancel_request, - matching single-job write_cancel_request semantics.""" - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend_pair.enqueue(args) - - await backend_pair.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="test", - ) - - events = await backend_pair.get_events(row.id) - kinds = [e.kind for e in events] - # Both event kinds must be present (event parity with single-job path) - assert "state_change" in kinds - assert "cancel_request" in kinds - # state_change should have actual from_state, not a synthetic placeholder - sc = [e for e in events if e.kind == "state_change"] - assert sc[0].detail.get("from_state") in ("pending", "scheduled") - assert sc[0].detail.get("to_state") == "cancelled" - - async def test_pg_cancel_where_reason_with_quotes(self, backend_pair) -> None: - """Reason containing double-quotes does not cause DataError. - - Guards against H2: f-string JSON interpolation would produce - invalid JSON for reasons containing " or \\. - """ - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend_pair.enqueue(args) - - result = await backend_pair.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason='offboard "tenant-acme" \\ done', - ) - - assert result.cancelled_directly == 1 - events = await backend_pair.get_events(row.id) - cr = [e for e in events if e.kind == "cancel_request"] - assert cr[0].detail.get("reason") == 'offboard "tenant-acme" \\ done' - - async def test_pg_cancel_where_batch_id_filter(self, backend_pair) -> None: - """cancel_where works with batch_id filter on Postgres.""" - bid = uuid4() - for _ in range(3): - await backend_pair.enqueue( - make_enqueue_args( - scheduled_at=_NOW, - metadata={"batch_id": str(bid)}, - ) - ) - - result = await backend_pair.cancel_where( - JobFilter(batch_id=bid), - reason="batch abort", - ) - - assert result.cancelled_directly == 3 - - async def test_pg_cancel_where_running_cooperative( - self, clean_jobs_app: JobsApp - ) -> None: - """cancel_where sets cancel_phase=1 for running jobs on Postgres. - - Uses a direct SQL UPDATE to simulate dispatch (status 'running' - with a worker ID). clean_jobs_app provides a PG-only backend plus - WorkerDeps with direct pool access; the in-memory path is covered - by the Task 4 tests and the backend_pair tests above. - """ - backend = clean_jobs_app.backend - deps = clean_jobs_app.deps - schema = deps.settings.schema_name - - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend.enqueue(args) - - # Simulate dispatch via direct SQL ('running' as a SQL literal so the - # schema-scoped job_status enum coerces without a parameter cast). - worker_id = uuid4() - async with deps.worker_pool.acquire() as conn: - await conn.execute( - f'UPDATE "{schema}".jobs ' - f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", - worker_id, - row.id, - ) - - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert result.cancelled_directly == 0 - assert result.cancel_requested == 1 - - # Verify the job is still running but has cancel_phase=1 - updated = await backend.get(row.id) - assert updated is not None - assert updated.status == "running" - assert updated.cancel_phase == 1 - - # Verify cancel_request event was inserted (no state_change for running) - events = await backend.get_events(row.id) - kinds = [e.kind for e in events] - assert "cancel_request" in kinds - assert "state_change" not in kinds - - async def test_pg_cancel_where_does_not_clobber_concurrent_claim( - self, clean_jobs_app: JobsApp - ) -> None: - """EPQ regression (C1): a job claimed (pending→running) while - cancel_where executes must NOT be overwritten to terminal 'cancelled'. - - The bulk UPDATE blocks on the row lock held by the simulated claim - transaction; after the claim commits, EvalPlanQual re-evaluates the - UPDATE's own WHERE clause against the new row version, sees - status='running', and skips the row. The job escapes this call - entirely (documented residual window) instead of being clobbered. - """ - backend = clean_jobs_app.backend - deps = clean_jobs_app.deps - schema = deps.settings.schema_name - - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend.enqueue(args) - worker_id = uuid4() - - claim_conn = await deps.worker_pool.acquire() - try: - # Simulate a dispatch claim in a held-open transaction: the row is - # locked and updated to 'running' but not yet committed. - claim_tx = claim_conn.transaction() - await claim_tx.start() - await claim_conn.execute( - f'UPDATE "{schema}".jobs ' - f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", - worker_id, - row.id, - ) - - # Run cancel_where concurrently. Its CTE snapshot is taken at - # statement start (before the claim commits, so the snapshot sees - # 'pending'); its UPDATE then blocks on the claim's row lock. - cancel_task = asyncio.create_task( - backend.cancel_where(JobFilter(tags=("tenant-acme",)), reason="offboard") - ) - await asyncio.sleep(0.2) # let cancel_where reach the row lock - await claim_tx.commit() - result = await cancel_task - - # Safety property: the claimed job was NOT clobbered to terminal - # 'cancelled'. It escaped this call (EPQ re-check rejected it for - # the pending/scheduled UPDATE; the snapshot excluded it from the - # running UPDATE), so a follow-up call is needed to cancel it. - assert result.cancelled_directly == 0 - updated = await backend.get(row.id) - assert updated is not None - assert updated.status == "running" - finally: - await deps.worker_pool.release(claim_conn) - - async def test_pg_cancel_where_notify_sent_for_running( - self, clean_jobs_app: JobsApp, pg_dsn: str - ) -> None: - """Batched NOTIFY fires on the fleet and per-worker channels for - running jobs — same listener pattern as test_cancel_notify_integration.py.""" - import asyncpg - from taskq.constants import events_channel, worker_channel - - backend = clean_jobs_app.backend - deps = clean_jobs_app.deps - schema = deps.settings.schema_name - - args = make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW) - row = await backend.enqueue(args) - worker_id = uuid4() - async with deps.worker_pool.acquire() as conn: - await conn.execute( - f'UPDATE "{schema}".jobs ' - f"SET status = 'running', locked_by_worker = $1 WHERE id = $2", - worker_id, - row.id, - ) - - received: list[str] = [] - listen_conn = await asyncpg.connect(pg_dsn) - try: - await listen_conn.add_listener( - events_channel(schema), - lambda _c, _p, _ch, payload: received.append(payload), - ) - await listen_conn.add_listener( - worker_channel(schema, str(worker_id)), - lambda _c, _p, _ch, payload: received.append(payload), - ) - result = await backend.cancel_where( - JobFilter(tags=("tenant-acme",)), reason="offboard" - ) - assert result.cancel_requested == 1 - await asyncio.sleep(0.3) # allow asyncpg NOTIFY delivery - finally: - await listen_conn.close() - - assert len(received) == 2 # one fleet-channel + one per-worker-channel payload -``` - -#### TDD — Green - -Implement the SQL and wire the method. Run the integration tests: the `backend_pair` tests run against both backends (the `pg` param requires `@pytest.mark.integration`, enforced by the fixture guard); the `clean_jobs_app` tests run PG-only with direct pool access for dispatch simulation. - -#### Acceptance criteria -- All red tests pass against both in-memory and Postgres backends -- `job_events` rows are inserted for cancelled jobs — both `state_change` (with actual `from_state`) and `cancel_request` for pending/scheduled; `cancel_request` only for running (event parity with single-job `write_cancel_request`) -- NOTIFY is sent for running jobs, batched into one statement (verified by `test_pg_cancel_where_notify_sent_for_running`, same listener pattern as `test_cancel_notify_integration.py`); `taskq.cancel.notify_sent` counter incremented per job -- Single SQL statement for the UPDATEs (with EPQ-safe duplicated predicates on the target table, `FROM prev` for `prev_status`); `executemany` for events within the same transaction via the shared `sql.insert_event` template -- `reason` JSON is serialized safely via `jsonb_param` (not f-string interpolation) -- `ORDER BY id` in the matching CTE for deterministic lock ordering -- Backend retries the transaction on `asyncpg.DeadlockDetectedError` (max 3 attempts, jittered backoff); NOTIFY fires only after a successful commit -- No clobbering of concurrently-claimed rows (verified by `test_pg_cancel_where_does_not_clobber_concurrent_claim`: the claimed job stays `running`, not `cancelled`) - ---- - -### Task 6: Add `cancel_where` to `JobsClient` and `TaskQ` - -**Goal:** Add the client-layer method with the empty-filter guardrail. - -**Files:** -- MODIFY: `src/taskq/client/_jobs.py` — add `cancel_where` -- MODIFY: `src/taskq/client/_taskq.py` — add `cancel_where` delegate -- CREATE: `tests/test_cancel_where_client.py` - -#### TDD — Red - -```python -# tests/test_cancel_where_client.py - -import pytest -from taskq.backend._protocol import JobFilter -from taskq.client._jobs import JobsClient -from taskq.exceptions import EmptyFilterError -from taskq.testing.clock import FakeClock -from taskq.testing.in_memory import InMemoryBackend -from taskq.testing.jobs import make_enqueue_args -from taskq.types import BulkCancelResult - -from datetime import UTC, datetime - -_NOW = datetime(2026, 1, 1, tzinfo=UTC) - -async def test_client_cancel_where_with_tags() -> None: - """JobsClient.cancel_where cancels jobs by tag filter.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - client = JobsClient(backend) - - for _ in range(3): - await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) - - result = await client.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="offboard", - ) - - assert isinstance(result, BulkCancelResult) - assert result.cancelled_directly == 3 - -async def test_client_cancel_where_empty_filter_raises() -> None: - """Empty filter (no predicates) raises EmptyFilterError.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - client = JobsClient(backend) - - with pytest.raises(EmptyFilterError, match="filter predicate"): - await client.cancel_where(JobFilter(), reason="oops") - -async def test_client_cancel_where_empty_filter_override() -> None: - """allow_empty_filter=True bypasses the guardrail.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - client = JobsClient(backend) - - await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) - await backend.enqueue(make_enqueue_args(scheduled_at=_NOW)) - - result = await client.cancel_where( - JobFilter(), - reason="drain all", - allow_empty_filter=True, - ) - - assert result.cancelled_directly == 2 - -async def test_client_cancel_where_increments_counter() -> None: - """cancel_where increments taskq.cancellation.requested once.""" - # Same OTel fixture pattern as test_jobs_client_cancel.py - backend = InMemoryBackend(clock=FakeClock(_NOW)) - client = JobsClient(backend) - - await backend.enqueue(make_enqueue_args(tags=("tenant-acme",), scheduled_at=_NOW)) - - await client.cancel_where( - JobFilter(tags=("tenant-acme",)), - reason="test", - ) - - # Counter should be 1 (one call, regardless of jobs affected) - # ... OTel reader assertion ... - -async def test_client_cancel_where_translates_schema_errors() -> None: - """cancel_where wraps UndefinedTableError in SchemaNotMigratedError.""" - # ... same pattern as enqueue test ... -``` - -#### TDD — Green - -```python -# In JobsClient: - -async def cancel_where( - self, - filter: JobFilter, - reason: str | None = None, - *, - allow_empty_filter: bool = False, -) -> BulkCancelResult: - from taskq.exceptions import EmptyFilterError - from taskq.obs import record_cancel_requested - - # Guardrail: reject empty filter - if not allow_empty_filter: - if ( - filter.queue is None - and filter.status is None - and filter.actor is None - and filter.identity_key is None - and filter.batch_id is None - and (filter.tags is None or len(filter.tags) == 0) - and filter.active is None - ): - raise EmptyFilterError() - - record_cancel_requested() - - with self._translate_schema_errors(): - return await self._backend.cancel_where(filter, reason) -``` - -#### Acceptance criteria -- Empty filter raises `EmptyFilterError` by default -- `allow_empty_filter=True` overrides the guardrail -- `taskq.cancellation.requested` counter incremented once per call -- `SchemaNotMigratedError` wrapping works (same pattern as other client methods) -- `TaskQ.cancel_where` delegates correctly (requires open client) -- Client stays thin: no `DeadlockDetectedError` handling here — deadlock retry is backend-owned (see Task 5 design note); the client must not import asyncpg for this - ---- - -### Task 7: Add `tags` to `SubJobEnqueuer.enqueue()` with parent-tag inheritance - -**Goal:** Add the `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, and `heartbeat_timeout` parameters to `SubJobEnqueuer.enqueue()`. - -**Files:** -- MODIFY: `src/taskq/client/_enqueuer.py` — add parameters, ContextVar, tag resolution logic -- MODIFY: `src/taskq/worker/_consumer.py` — set parent tags before actor invocation (gated by `sub_job_inherit_tags` setting) -- MODIFY: `src/taskq/worker/run.py` — set parent tags in stub consumer (unconditional — test harness) -- MODIFY: `src/taskq/settings.py` — add `sub_job_inherit_tags: bool = True` field to `WorkerSettings` (fleet-level kill switch; `TASKQ_SUB_JOB_INHERIT_TAGS` env var via pydantic-settings) -- CREATE: `tests/test_sub_job_tags.py` -- MODIFY: `tests/test_sub_job_enqueuer.py` — add tags tests - -#### Worker-level kill switch (`sub_job_inherit_tags`) - -`inherit_tags=True` as a default is a production behavior change: after upgrade, every existing sub-job enqueued inside an actor whose parent has tags becomes tag-findable — and via #54, tag-cancellable. A shared/utility sub-job enqueued by a tenant-tagged parent will now be swept up in that tenant's `cancel_where`. Per-call `inherit_tags=False` is not a practical rollback for a fleet. - -The `sub_job_inherit_tags` worker setting (default `True`) provides a fleet-level opt-out. When set to `False`, the consumer does **not** call `set_parent_tags()` — the ContextVar remains at its `()` default, so `inherit_tags=True` on `enqueue()` produces `()` (identical to pre-upgrade behavior). This allows operators to disable inheritance across an entire worker fleet without code changes. - -**Rollout guidance:** -1. Deploy with `sub_job_inherit_tags=False` (preserves existing behavior). -2. Verify no regressions in production. -3. Enable `sub_job_inherit_tags=True` per-queue or per-worker-group as confidence grows. -4. Document the blast-radius implication in `jobs-clients.md`: sub-jobs inherit parent tags → they are visible to `cancel_where` filters matching those tags. - -#### Batch enqueue asymmetry (`enqueue_batch`) - -`ctx.jobs.enqueue_batch` (via `EnqueueItem.tags`) does **not** inherit parent tags in this spec. This is a deliberate scoping decision for this iteration: - -- `enqueue_batch` fans out N items, each potentially with its own `tags` field. Applying parent-tag inheritance per-item would require merging parent tags into each `EnqueueItem.tags` — a different code path (`batch.py`) than the single-enqueue path (`_enqueuer.py`). -- The primary use case for batch enqueue (fan-out chunks) already sets tags per `EnqueueItem` at call sites (e.g., cennan's `EnqueueItem(tags=...)` per sync-run/binding). These callers explicitly tag their batch items. -- Extending `inherit_tags` to `enqueue_batch` is a follow-up spec that can add a per-call `inherit_tags: bool` parameter to `enqueue_batch` and merge parent tags into each item's `tags` field. This is noted as a non-goal for this spec to keep the scope bounded. - -**The asymmetry is documented** in Design Decisions (#57, decision 7) and in the updated `jobs-clients.md` guide so callers are aware that single `enqueue()` inherits by default while `enqueue_batch()` does not. - -#### TDD — Red - -```python -# tests/test_sub_job_tags.py - -import pytest -from datetime import UTC, datetime, timedelta -from uuid import uuid4 -from pydantic import BaseModel, TypeAdapter - -from taskq.actor import ActorRef -from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags -from taskq.testing.clock import FakeClock -from taskq.testing.in_memory import InMemoryBackend - -class _Payload(BaseModel): - value: str = "test" - -class _Result(BaseModel): - ok: bool = True - -def _make_actor_ref(name: str = "child") -> ActorRef[_Payload, _Result]: - async def _handler(payload: _Payload) -> _Result: - return _Result() - return ActorRef( - name=name, queue="default", fn=_handler, wants_ctx=False, - dependencies={}, payload_type=_Payload, - result_adapter=TypeAdapter(_Result), - retry=__import__("taskq.retry", fromlist=["RetryPolicy"]).RetryPolicy(), - result_ttl=None, singleton=False, unique_for=None, max_pending=None, - ) - -_NOW = datetime(2025, 1, 1, tzinfo=UTC) - -def _make_enqueuer(backend: InMemoryBackend | None = None) -> SubJobEnqueuer: - if backend is None: - backend = InMemoryBackend(clock=FakeClock(_NOW)) - return SubJobEnqueuer( - loop_scope_resolved=None, - worker_pool=object(), # sentinel - backend=backend, - clock=FakeClock(_NOW), - ) - -class TestSubJobExplicitTags: - async def test_explicit_tags_no_inheritance(self) -> None: - """tags= with inherit_tags=False sets only explicit tags.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - tags=["alpha", "beta"], - inherit_tags=False, - ) - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == ("alpha", "beta") - - async def test_tags_validated(self) -> None: - """Invalid tags raise ValueError.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - with pytest.raises(ValueError, match="invalid tag"): - await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - tags=["ab"], # too short - ) - - async def test_tags_deduplicated(self) -> None: - """Duplicate tags are deduplicated.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - tags=["alpha", "alpha", "beta"], - inherit_tags=False, - ) - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == ("alpha", "beta") - -class TestSubJobTagInheritance: - async def test_inherit_parent_tags_default(self) -> None: - """With no explicit tags and default inherit_tags=True, sub-job inherits parent tags.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - # Set parent tags (simulating consumer setting them before actor invocation) - token = set_parent_tags(("run-001", "tenant-acme")) - try: - handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) - finally: - _parent_tags_var.reset(token) - - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == ("run-001", "tenant-acme") - - async def test_inherit_and_merge_tags(self) -> None: - """Explicit tags merge with parent tags (parent first, deduped).""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - token = set_parent_tags(("run-001", "tenant-acme")) - try: - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - tags=["stage-2", "tenant-acme"], # tenant-acme is a dup - ) - finally: - _parent_tags_var.reset(token) - - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == ("run-001", "tenant-acme", "stage-2") - - async def test_no_parent_tags_no_explicit_tags(self) -> None: - """With no parent tags and no explicit tags, sub-job has empty tags.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - # No set_parent_tags call — ContextVar default is () - handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == () - - async def test_inherit_false_no_parent_tags(self) -> None: - """inherit_tags=False with no explicit tags → empty tags.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - token = set_parent_tags(("run-001",)) - try: - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - inherit_tags=False, - ) - finally: - _parent_tags_var.reset(token) - - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == () - - async def test_inherit_false_with_explicit_tags(self) -> None: - """inherit_tags=False with explicit tags → only explicit tags.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - token = set_parent_tags(("run-001",)) - try: - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - tags=["custom-tag"], - inherit_tags=False, - ) - finally: - _parent_tags_var.reset(token) - - row = await backend.get(handle.job_id) - assert row is not None - assert row.tags == ("custom-tag",) - -class TestSubJobMissingFields: - async def test_schedule_to_close(self) -> None: - """schedule_to_close is accepted and stored on the row.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - deadline = _NOW + timedelta(hours=1) - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - schedule_to_close=deadline, - ) - row = await backend.get(handle.job_id) - assert row is not None - assert row.schedule_to_close == deadline - - async def test_start_to_close(self) -> None: - """start_to_close is accepted and stored on the row.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - start_to_close=timedelta(minutes=30), - ) - row = await backend.get(handle.job_id) - assert row is not None - assert row.start_to_close == timedelta(minutes=30) - - async def test_heartbeat_timeout(self) -> None: - """heartbeat_timeout is accepted and stored on the row.""" - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - handle = await enqueuer.enqueue( - _make_actor_ref(), - _Payload(), - heartbeat_timeout=timedelta(seconds=10), - ) - row = await backend.get(handle.job_id) - assert row is not None - assert row.heartbeat_timeout == timedelta(seconds=10) - -class TestContextVarIsolation: - async def test_concurrent_jobs_separate_parent_tags(self) -> None: - """ContextVar ensures concurrent consumers don't share parent tags.""" - import asyncio - backend = InMemoryBackend(clock=FakeClock(_NOW)) - enqueuer = _make_enqueuer(backend) - - async def enqueue_with_parent(parent_tags: tuple[str, ...]) -> UUID: - token = set_parent_tags(parent_tags) - try: - handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) - return handle.job_id - finally: - _parent_tags_var.reset(token) - - # Run two "jobs" concurrently with different parent tags - id1, id2 = await asyncio.gather( - enqueue_with_parent(("run-a",)), - enqueue_with_parent(("run-b",)), - ) - - row1 = await backend.get(JobId(id1)) - row2 = await backend.get(JobId(id2)) - assert row1.tags == ("run-a",) - assert row2.tags == ("run-b",) -``` - -#### TDD — Green - -1. Add `contextvars.ContextVar` and `set_parent_tags()` to `_enqueuer.py` -2. Add `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, `heartbeat_timeout` to `enqueue()` -3. Add `_resolve_tags()` method -4. Pass the new parameters to `build_enqueue_args()` -5. In `_consumer.py`, call `set_parent_tags(tuple(job.tags))` before constructing `JobContext` -6. Reset the ContextVar after the actor completes (in a `finally` block) - -#### Acceptance criteria -- All red tests pass -- Default behavior: `inherit_tags=True` — sub-job inherits parent tags when no explicit tags -- Explicit tags merge with parent tags (union, parent-first, deduped) -- `inherit_tags=False` disables inheritance -- `schedule_to_close`, `start_to_close`, `heartbeat_timeout` are passed through to `build_enqueue_args` -- ContextVar isolation works — concurrent consumers don't cross-contaminate parent tags -- All existing `test_sub_job_enqueuer.py` tests still pass (backward compatible — default `tags=None` with no parent tags → `()`) - ---- - -### Task 8: Backward compatibility — default behavior unchanged - -**Goal:** Verify that existing code that doesn't use tags or `inherit_tags` sees no behavior change. - -**Files:** -- MODIFY: `tests/test_sub_job_enqueuer.py` — add backward compat tests - -#### TDD — Red - -```python -# tests/test_sub_job_enqueuer.py — add: - -class TestBackwardCompatibility: - async def test_no_tags_no_parent_tags_empty(self) -> None: - """Existing code with no tags and no parent context → empty tags.""" - # No set_parent_tags call → ContextVar default is () - enqueuer = _make_enqueuer() - handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) - row = await enqueuer._backend.get(handle.job_id) - assert row.tags == () - - async def test_existing_enqueue_no_tags_param(self) -> None: - """Calling enqueue without tags= still works (backward compat).""" - enqueuer = _make_enqueuer() - handle = await enqueuer.enqueue(_make_actor_ref(), _Payload()) - assert handle is not None - assert handle.job_id is not None -``` - -#### Acceptance criteria -- All existing sub-job enqueuer tests pass without modification -- Existing code that doesn't set parent tags or pass `tags=` gets `tags=()` (same as before) -- No new required parameters — all additions have defaults - ---- - -### Task 9: E2E tests — sub-job tags in a real pipeline - -**Goal:** Verify that sub-jobs enqueued from inside actor bodies are tagged and findable by `JobFilter(tags=...)` in a real worker container. - -**Files:** -- MODIFY: `tests/e2e/actors.py` — add a tagged pipeline actor -- CREATE: `tests/e2e/test_sub_job_tags.py` - -#### E2E actors - -```python -# tests/e2e/actors.py — add: - -class PipelineStagePayload(BaseModel): - run_id: str - stage: int - total_stages: int = 3 - -@actor(name="pipeline_stage", queue="e2e") -async def pipeline_stage( - payload: PipelineStagePayload, - ctx: JobContext[PipelineStagePayload], - *, - pool: asyncpg.Pool, -) -> None: - """Linear pipeline: each stage enqueues the next via ctx.jobs.enqueue().""" - await _record_effect(pool, ctx, "stage", { - "run_id": payload.run_id, - "stage": payload.stage, - }) - await asyncio.sleep(0.05) - - if payload.stage < payload.total_stages: - # Enqueue next stage — inherits parent tags by default - await ctx.jobs.enqueue( - pipeline_stage, - PipelineStagePayload( - run_id=payload.run_id, - stage=payload.stage + 1, - total_stages=payload.total_stages, - ), - ) -``` - -#### E2E test - -```python -# tests/e2e/test_sub_job_tags.py - -from __future__ import annotations -from typing import TYPE_CHECKING -import pytest -from taskq.backend._protocol import JobFilter -from ._assertions import wait_for_effects, poll_until -from .actors import ( - PipelineStagePayload, - TaggedPipelineStagePayload, - pipeline_stage, - tagged_pipeline_stage, -) - -if TYPE_CHECKING: - import asyncpg - from taskq import TaskQ - from .conftest import E2ESchema, E2EWorker - -pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] - -async def test_sub_job_inherits_tags_in_pipeline( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """Sub-jobs enqueued via ctx.jobs.enqueue() inherit parent tags. - - A pipeline of 3 stages is enqueued with tag "run-{run_id}". - Each stage enqueues the next via ctx.jobs.enqueue() with no - explicit tags. All 3 jobs should be findable by the tag filter. - """ - tag = f"run-{run_id[:8]}" - handle = await e2e_client.enqueue( - pipeline_stage, - PipelineStagePayload(run_id=run_id, stage=1, total_stages=3), - tags=[tag], - ) - - # Wait for all 3 stages to complete - await wait_for_effects( - e2e_pg_pool, - e2e_schema.schema_name, - run_id, - kind="stage", - min_count=3, - timeout=30, - ) - - # All 3 jobs should be findable by the tag - page = await e2e_client.list(JobFilter(tags=(tag,))) - assert len(page.jobs) == 3, ( - f"Expected 3 jobs with tag {tag!r}, found {len(page.jobs)}: " - f"{[j.id for j in page.jobs]}" - ) -``` - -#### E2E actors (additional for merge test) - -```python -# tests/e2e/actors.py — add: - -class TaggedPipelineStagePayload(BaseModel): - run_id: str - stage: int - total_stages: int = 3 - -@actor(name="tagged_pipeline_stage", queue="e2e") -async def tagged_pipeline_stage( - payload: TaggedPipelineStagePayload, - ctx: JobContext[TaggedPipelineStagePayload], - *, - pool: asyncpg.Pool, -) -> None: - """Pipeline that passes explicit tags to sub-job enqueue.""" - await _record_effect(pool, ctx, "stage", { - "run_id": payload.run_id, - "stage": payload.stage, - }) - await asyncio.sleep(0.05) - - if payload.stage < payload.total_stages: - # Explicit per-stage tag — merges with the inherited parent tag - await ctx.jobs.enqueue( - tagged_pipeline_stage, - TaggedPipelineStagePayload( - run_id=payload.run_id, - stage=payload.stage + 1, - total_stages=payload.total_stages, - ), - tags=[f"stage-{payload.stage + 1}"], - ) -``` - -#### E2E test (merge) - -```python -async def test_sub_job_explicit_tags_merge_with_parent( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """Explicit tags on sub-job merge with inherited parent tags. - - Stage 1 is enqueued with parent tag "run-{run_id}". Each stage - enqueues the next with an explicit stage tag (e.g. "stage-2"). - The sub-job should carry both the inherited run tag and the - explicit stage tag. - """ - parent_tag = f"run-{run_id[:8]}" - - handle = await e2e_client.enqueue( - tagged_pipeline_stage, - TaggedPipelineStagePayload(run_id=run_id, stage=1, total_stages=3), - tags=[parent_tag], - ) - - # Wait for all 3 stages to complete - await wait_for_effects( - e2e_pg_pool, - e2e_schema.schema_name, - run_id, - kind="stage", - min_count=3, - timeout=30, - ) - - # All 3 jobs should have the parent tag - page = await e2e_client.list(JobFilter(tags=(parent_tag,))) - assert len(page.jobs) == 3, ( - f"Expected 3 jobs with parent tag {parent_tag!r}, found {len(page.jobs)}" - ) - - # Stage 2 should also have the "stage-2" explicit tag - stage2 = await e2e_client.list(JobFilter(tags=("stage-2",))) - assert len(stage2.jobs) == 1, ( - f"Expected 1 job with stage-2 tag, found {len(stage2.jobs)}" - ) - # Verify it also has the parent tag (merged) - assert parent_tag in stage2.jobs[0].tags - assert "stage-2" in stage2.jobs[0].tags -``` - -#### Acceptance criteria -- Sub-jobs enqueued from actor bodies are findable by the parent's tag -- All pipeline stages share the run tag -- E2E test passes against real Postgres + worker container - ---- - -### Task 10: E2E tests — bulk cancel by filter - -**Goal:** Verify `cancel_where` works end-to-end against real Postgres + worker. - -**Files:** -- MODIFY: `tests/e2e/actors.py` — add bulk-cancel test actors if needed -- CREATE: `tests/e2e/test_cancel_where.py` - -#### E2E test - -```python -# tests/e2e/test_cancel_where.py - -from __future__ import annotations -from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING -import pytest -from taskq.backend._protocol import JobFilter -from taskq.types import BulkCancelResult -from ._assertions import poll_until, wait_for_handle_status -from .actors import GenerateReportPayload, generate_report - -if TYPE_CHECKING: - import asyncpg - from taskq import TaskQ - from .conftest import E2ESchema, E2EWorker - -pytestmark = [pytest.mark.e2e, pytest.mark.timeout(900)] - -async def test_cancel_where_pending_jobs_by_tag( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where cancels all pending jobs matching a tag filter.""" - tag = f"tenant-{run_id[:8]}" - - # Enqueue 5 jobs with the tag, scheduled far in the future (won't dispatch) - future = datetime.now(UTC) + timedelta(seconds=120) - for i in range(5): - await e2e_client.enqueue( - generate_report, - GenerateReportPayload(run_id=run_id, report_id=f"r-{i}"), - scheduled_at=future, - tags=[tag], - ) - - # Enqueue 2 jobs without the tag (should not be affected) - for i in range(2): - await e2e_client.enqueue( - generate_report, - GenerateReportPayload(run_id=f"other-{run_id}", report_id=f"o-{i}"), - scheduled_at=future, - ) - - result = await e2e_client.cancel_where( - JobFilter(tags=(tag,)), - reason="tenant offboarded", - ) - - assert result.cancelled_directly == 5 - assert result.cancel_requested == 0 - assert result.total_affected == 5 - - # Verify via list: tagged jobs are cancelled, untagged are still scheduled - tagged = await e2e_client.list(JobFilter(tags=(tag,), status="cancelled")) - assert len(tagged.jobs) == 5 - - untagged = await e2e_client.list( - JobFilter(status="scheduled", queue="e2e") - ) - assert len(untagged.jobs) == 2 - -async def test_cancel_where_running_jobs_cooperative( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where sets cooperative cancel for running jobs.""" - tag = f"run-{run_id[:8]}" - - # Enqueue a long-running job - handle = await e2e_client.enqueue( - generate_report, - GenerateReportPayload( - run_id=run_id, - report_id=f"r-{run_id[:8]}", - stages=4, - stage_latency_ms=2000, - ), - tags=[tag], - ) - await wait_for_handle_status(handle, "running", timeout=30) - - result = await e2e_client.cancel_where( - JobFilter(tags=(tag,), status="running"), - reason="abort run", - ) - - assert result.cancel_requested >= 1 - assert result.cancelled_directly == 0 - - # The running job should eventually reach 'cancelled' - await wait_for_handle_status(handle, "cancelled", timeout=30) - -async def test_cancel_where_empty_filter_raises( - e2e_client: TaskQ, - e2e_worker: E2EWorker, -) -> None: - """Empty filter is rejected even in e2e.""" - from taskq.exceptions import EmptyFilterError - - with pytest.raises(EmptyFilterError): - await e2e_client.cancel_where(JobFilter(), reason="oops") - -async def test_cancel_where_batch_id_filter( - e2e_client: TaskQ, - e2e_worker: E2EWorker, - e2e_pg_pool: asyncpg.Pool, - e2e_schema: E2ESchema, - run_id: str, -) -> None: - """cancel_where works with batch_id filter.""" - from uuid import uuid4 - from taskq.batch import EnqueueItem - from .actors import ImportContactsChunkPayload, import_contacts_chunk - - future = datetime.now(UTC) + timedelta(seconds=120) - bid = uuid4() - - items = [ - EnqueueItem( - actor_ref=import_contacts_chunk, - payload=ImportContactsChunkPayload( - run_id=run_id, upload_id=str(bid), chunk_id=i, - start_row=i*100, end_row=(i+1)*100, - ), - scheduled_at=future, - ) - for i in range(5) - ] - batch = await e2e_client.enqueue_batch(items, batch_id=bid) - - result = await e2e_client.cancel_where( - JobFilter(batch_id=bid), - reason="batch abort", - ) - - assert result.cancelled_directly == 5 -``` - -#### Acceptance criteria -- Bulk cancel works against real Postgres with a real worker container -- Pending/scheduled jobs go straight to `cancelled` -- Running jobs get cooperative cancel and eventually reach `cancelled` -- Batch_id filter works end-to-end -- Empty filter guardrail fires in e2e context - ---- - -### Task 11: Update documentation - -**Goal:** Update the user-facing docs to reflect the new APIs. - -**Files:** -- MODIFY: `docs/guides/jobs-clients.md` — document `cancel_where`, sub-job `tags`, `inherit_tags` -- MODIFY: `docs/architecture.md` — document bulk cancel in the cancel protocol section - -#### Key doc additions - -1. **SubJobEnqueuer.enqueue()** — update the signature to show `tags`, `inherit_tags`, `schedule_to_close`, `start_to_close`, `heartbeat_timeout`. Document the inheritance semantics table. Update the existing exclusion list at `jobs-clients.md:789` ("No `schedule_to_close`, `start_to_close`, or `heartbeat_timeout`") to reflect that these are now accepted on the sub-job path — this is a deliberate reversal of the documented exclusion; call it out in the guide, including the snooze/`schedule_to_close` warning (see Design Decisions #57-4). Document the `sub_job_inherit_tags` worker setting and the blast-radius implication (sub-jobs inherit parent tags → visible to `cancel_where`). Document that `enqueue_batch` does **not** inherit parent tags (asymmetry noted). All doc examples must use tags valid under the existing charset (`^[\w][\w\-]+[\w]$` — hyphenated, e.g. `tenant-acme`, not `tenant:acme`). - -2. **New `cancel_where` section** in jobs-clients.md: - ````markdown - ### `cancel_where()` - - ```python - result = await client.cancel_where( - JobFilter(tags=("tenant-acme",), active=True), - reason="tenant offboarded", - ) - ``` - - Cancel all jobs matching a filter in a single set-based operation... - ```` - -3. **Architecture.md** — add `cancel_where` to the Backend protocol listing and the cancel protocol section. - ---- - -## Test Coverage Requirements - -### Unit tests (in-memory backend) -- `test_filter_sql.py` — filter→SQL builder extraction (6+ tests) -- `test_bulk_cancel_types.py` — BulkCancelResult, EmptyFilterError (4+ tests) -- `test_cancel_where.py` — in-memory bulk cancel (9+ tests, including limit-ignoring test) -- `test_cancel_where_client.py` — client-layer guardrail, counter, schema errors (5+ tests) -- `test_sub_job_tags.py` — sub-job tags, inheritance, ContextVar isolation (12+ tests) -- `test_sub_job_enqueuer.py` — backward compat additions (2+ tests) - -### Integration tests (Postgres backend) -- `test_cancel_where_pg.py` — PostgresBackend bulk cancel (7 tests: pending, events-parity, reason-with-quotes, batch_id via `backend_pair`; running-cooperative, concurrent-claim EPQ race, and batched NOTIFY via `clean_jobs_app`) - -### Protocol conformance -- `test_backend_protocol.py` — cancel_where method presence (1 test) - -### E2E tests (real worker + PG) -- `test_sub_job_tags.py` — sub-job tag inheritance in a real pipeline (2+ tests) -- `test_cancel_where.py` — bulk cancel by tag/batch_id, cooperative cancel, guardrail (4+ tests) - -### Coverage targets -- New code paths must achieve ≥95% line coverage -- Tag inheritance logic (`_resolve_tags`) must have 100% branch coverage -- Guardrail validation must have 100% branch coverage -- SQL builder must have 100% branch coverage for all filter combinations - ---- - -## Backward Compatibility Analysis - -### SubJobEnqueuer.enqueue() changes - -| Change | Impact on existing code | Mitigation | -|---|---|---| -| New `tags=None` param | None — default is `None`, no behavior change when parent tags are also empty | None needed | -| New `inherit_tags=True` param | Sub-jobs now inherit parent tags by default. **Behavior change** when parent job has tags and code relies on sub-job tags being empty. Via #54, inherited tags make sub-jobs visible to `cancel_where` — a shared/utility sub-job enqueued by a tenant-tagged parent becomes tag-cancellable. | The ContextVar defaults to `()`, so if the consumer doesn't call `set_parent_tags()`, the behavior is identical to before. Only code that runs inside a real worker with the updated consumer will see inheritance. **Fleet-level kill switch:** `sub_job_inherit_tags` worker setting (default `True`) — when `False`, the consumer skips `set_parent_tags()`, preserving pre-upgrade behavior across the entire fleet. Deploy with `False` first, verify, then enable. | -| New `schedule_to_close=None` param | None — default is `None`, passes through to `build_enqueue_args` which already handles `None` | None needed | -| New `start_to_close=None` param | None — default is `None`, resolved to actor-declared value in `build_enqueue_args` | None needed | -| New `heartbeat_timeout=None` param | None — default is `None` | None needed | - -**Key backward-compat guarantee:** The `contextvars.ContextVar` default is `()`. Existing tests that construct `SubJobEnqueuer` directly (without going through the consumer) will see `()` parent tags, so `inherit_tags=True` with no explicit tags produces `()` — identical to the current behavior. Only code that runs inside a real worker with the updated consumer will see tag inheritance. For production rollouts, the `sub_job_inherit_tags` worker setting (default `True`) can be set to `False` to disable inheritance fleet-wide without code changes. - -### cancel_where addition - -| Change | Impact on existing code | Mitigation | -|---|---|---| -| New `Backend.cancel_where` method | Custom `Backend` implementations lack this method | Method is purely additive; `AttributeError` is loud. Third-party backends must add the method to support bulk cancel. | -| New `JobsClient.cancel_where` method | None — new method, no existing code calls it | None needed | -| New `BulkCancelResult` type | None — new type | None needed | -| New `EmptyFilterError` exception | None — new type | None needed | -| Filter→SQL builder extraction | `_list_jobs` internals change | All existing `test_job_filter.py` and `test_postgres_reads.py` tests verify no regression | - -### Protocol version - -No `BACKEND_PROTOCOL_VERSION` bump required. The `cancel_where` method is purely additive — a v3 implementation lacks the method, and calling it raises `AttributeError` (a loud failure, not a silent misbehavior). See the bump rule in `_protocol.py` lines 79-84: "Purely additive changes an old implementation can ignore without producing incorrect behaviour do not require a bump." - ---- - -## Downstream Consumer Impact Analysis - -> **Methodology and framing:** The contract for this section is the downstream need documented in issues #54/#57 and in the downstream codebases' own comments — not the code those repos happen to run today. Each entry states the documented need, the verified current baseline (local checkouts were grepped; claims that did not hold up were corrected), the end-state this spec enables, and the migration required to get there. - -### warden (~/src/warden) — Hybrid LLM proxy - -**Documented need (from issue #54):** tenant-scoped job grouping at enqueue time and tenant offboarding / run abort as a single operation. #54's motivating example is exactly this shape: jobs tagged per tenant, offboarded via a paginate-and-cancel loop that is slow (one round trip per job) and racy (workers keep dispatching behind the cursor; new matching jobs land behind the cursor, so convergence needs an outer retry loop). The documented need — not warden's current code — is the contract here. - -**How it uses TaskQ today (verified against the local checkout):** -- Enqueues jobs via `tq.enqueue` (e.g., `routes/admin.py:1438`, `routes/inference.py:1867,1996,2327`) -- Does **not** currently pass TaskQ `tags=` on any enqueue call (the `tags=` hits in `src/` are FastAPI route metadata, not TaskQ) -- Does **not** currently use `ctx.jobs` (sub-job enqueue) in `src/` — actor test harnesses construct `JobContext(jobs=None)` with a "never enqueues sub-jobs" comment (`app.py:217,252`) -- Does **not** currently call TaskQ `cancel()` in `src/` (only asyncio task cancels) - -**End-state this spec enables:** -- **#57:** Once warden tags jobs per tenant at enqueue (`tags=["tenant-"]` — hyphenated; the colon form in #54's prose is rejected by the current validator) and actors fan out via `ctx.jobs.enqueue()`, sub-jobs inherit tenant tags automatically and are findable by `JobFilter(tags=("tenant-",))`. -- **#54:** `cancel_where(JobFilter(tags=("tenant-",), active=True), reason="tenant offboarded")` replaces the paginate-and-cancel loop: one set-based write, no cursor race, counts/IDs returned for observability. Convergence for post-snapshot enqueues is the caller's job (stop producers, or repeat the call — see the snapshot-boundary contract). - -**Migration path:** -- No change required to existing enqueue calls (nothing is tagged today; `sub_job_inherit_tags` has no observable effect until sub-jobs exist) -- Adopt tags: pass `tags=["tenant-"]` at the `tq.enqueue` call sites -- Adopt bulk cancel: new offboarding flows call `cancel_where` instead of per-job loops - -### cennan (~/src/cennan) — Enterprise knowledge management - -**How it uses TaskQ today:** -- Ingestion pipeline: list → fetch → extract → chunk → embed → store -- Tags jobs per sync-run and per binding via `EnqueueItem.tags` in batch enqueue (`cennan/api/enqueue.py:299,519,565,574,604`), built by `_job_tags()` (`enqueue.py:240-252`) as hyphenated `sync-{sync_run_id}` / `binding-{binding_id}` — the same function documents a production incident where colon-form tags (`sync:{id}`) raised `ValueError` at enqueue and 500'd every sync trigger -- Pipeline stages use `ctx.jobs` chains (`pipeline/actors.py`); `pipeline/models.py:4-5` carries a comment documenting the exact #57 limitation ("`ctx.jobs` … does not accept `tags` in the installed TaskQ version"), which is why payloads re-carry `sync_run_id`/`binding_id` -- Needs to stop runaway ingestion runs - -**What this spec enables:** -- **#57:** Pipeline stages enqueued via `ctx.jobs.enqueue()` will inherit the `sync-*`/`binding-*` tags. The full pipeline is findable by tag, not just the batch-fan-out chunks. The `pipeline/models.py` workaround comment can be deleted; payload ids stay (they feed counters/liveness, not just discovery). -- **#54:** `cancel_where(JobFilter(tags=("sync-",), active=True))` stops a runaway ingestion run in one call. Pending/scheduled stages go straight to cancelled; running stages get cooperative cancel. - -**Migration path:** -- No code change required for tag inheritance (automatic once workers are upgraded); keep `sub_job_inherit_tags` at its default `True` -- Delete the `pipeline/models.py` limitation comment; drop any secondary lookup paths maintained only because sub-jobs were untaggable -- Replace manual cancel loops with `cancel_where` -- Can now tag sub-jobs with stage-specific tags (e.g., `tags=["stage-embed"]`) that merge with inherited sync/binding tags - -### aacrtool (~/src/aacrtool) — Agentic code review tool - -**Baseline (verified):** TaskQ is present only in `.venv` (dependency declared); **no TaskQ usage exists in `src/` yet**. Everything below is planned usage, not a description of current code. - -**End-state this spec enables (when TaskQ is adopted):** -- Review jobs tagged per repo at enqueue (`tags=["repo--"]` — hyphenated per the existing tag charset). -- **#57:** If review actors fan out sub-jobs (e.g., per-file analysis), those sub-jobs inherit the repo tag automatically. -- **#54:** `cancel_where(JobFilter(tags=("repo--",), active=True))` aborts a review run in one call. - -**Migration path:** -- N/A — TaskQ adoption is future work; adopt `cancel_where` and sub-job tags from day one rather than building paginate-and-cancel loops. - ---- - -## Design Decisions Summary - -### #57: Sub-job tags - -1. **Inheritance default: `True`** — sub-jobs inherit parent tags by default because the primary use case (run/tenant correlation) requires sub-jobs to be findable by the same tags as the parent. Opting out with `inherit_tags=False` is available for cases where sub-jobs should be untagged or only carry explicit tags. A worker-level `sub_job_inherit_tags` setting (default `True`) provides a fleet-level kill switch for operators who need to roll out the behavior change gradually. - -2. **Merge semantics: union, parent-first** — when both parent tags and explicit tags are provided, the union preserves parent tags first, then adds new tags. This lets callers add stage-specific tags while keeping the run/tenant correlation tag. Deduplication follows the same `_validate_and_dedup_tags` logic. - -3. **Propagation via `contextvars.ContextVar`** — the `SubJobEnqueuer` is shared across concurrent consumers in the same event loop, so a per-instance field would be racy. `ContextVar` is the asyncio-native solution: each Task gets its own context copy, so concurrent consumers each see their own parent tags. This is the same mechanism Python uses for `contextvars.copy_context()` in `asyncio.Task`. - -4. **Also add `schedule_to_close`, `start_to_close`, `heartbeat_timeout`** — these are passed through to `build_enqueue_args`, which already handles them. **This reverses a documented deliberate exclusion:** `docs/guides/jobs-clients.md:789` currently lists "No `schedule_to_close`, `start_to_close`, or `heartbeat_timeout` (set on the actor declaration)" as an intentional constraint of the sub-job surface. Issue #57 asks whether this is deliberate or an omission, and floats the hypothesis that "'you can't set it from inside an actor' may be a feature rather than an omission." This spec takes the position that the parameters should be available — the client and sub-job surfaces should not drift without a reason, and the issue's concrete cost ("a sub-job that needs a different timeout than its actor's declared default … has to be enqueued from outside the actor") is real — but acknowledges the trade-off: - - **Snooze/finalizer hazard (analyzed, not ignored):** Issue #57 names the hazard directly — "A finalizer that snoozes on `wait_for_batch` for a long time would be killed by one." The verified mechanics: (i) `mark_snoozed` already guards the deadline at snooze time — a running job whose requested snooze delay would cross `schedule_to_close` is failed immediately with `error_class='DeadlineExceeded'` ("schedule_to_close reached before next dispatch", `_sql_templates.py:254-275`), rather than being parked past its deadline; (ii) `sweep_deadline_exceeded` (`_sweeps.py:325+`) fails pending/scheduled jobs whose `schedule_to_close` has passed, which includes snoozed jobs (a snooze returns the row to `scheduled`). So a sub-job carrying a tight caller-supplied `schedule_to_close` fails **deterministically and loudly** — at the snooze attempt or at the deadline — never silently mid-snooze. That is precisely what a wall-clock deadline means; the documented exclusion (a) prevented actor code from opting into it. Reversing the exclusion means the hazard is opt-in per call. **Mitigations:** (a) the default is `None` — no override, so `build_enqueue_args` falls back to the actor's retry time budget exactly as today, and the hazard only manifests when a caller explicitly passes `schedule_to_close`; (b) the docs update (Task 11) must warn that `schedule_to_close` bounds total wall-clock time *including* time snoozed on `wait_for_batch`, so finalizer-style sub-jobs should set it generously or not at all; (c) a future spec could add a `snooze_extends_deadline` flag to make the interaction explicit — out of scope here. - -5. **No `queue` override** — sub-jobs use the actor's declared queue. This is a documented design choice (`docs/guides/jobs-clients.md:788`) and is not changed by this spec. - -6. **`idempotency_key` type: keep `IdempotencyKey | str | None`** — the sub-job enqueuer's wider type (accepting bare `str`) is more ergonomic for actor code. The `JobsClient` uses `IdempotencyKey | None` (the narrower `NewType`); the sub-job enqueuer keeps its wider type. `build_enqueue_args` already handles both. - -7. **Batch enqueue asymmetry** — `ctx.jobs.enqueue_batch` does **not** inherit parent tags in this spec. This is a deliberate scoping decision: `enqueue_batch` uses a different code path (`batch.py`) with per-item `EnqueueItem.tags`, and extending inheritance to batch would require merging parent tags into each item. The asymmetry is documented in the updated `jobs-clients.md` so callers are aware. A follow-up spec can add `inherit_tags` to `enqueue_batch` if needed. - -### #54: Bulk cancel by filter - -1. **Pending/scheduled → terminal `cancelled`** — these jobs have no running actor to cooperate with. The existing `write_cancel_request` already does this for single jobs; bulk cancel follows the same pattern. The issue asks: "should matching rows in pending/scheduled go straight to terminal cancelled?" — yes, they should, for consistency with the existing single-cancel path. - -2. **Running → cooperative `cancel_phase=1`** — running jobs have an actor executing. The cooperative path sets `cancel_phase=1`, which the worker's heartbeat-driven `CancelController` observes and sets the in-process `cancel_event`. The actor checks `ctx.check_cancelled()` at its next stage boundary. This is the existing phase-1 cooperative cancel, just applied in bulk. - -3. **Guardrail: empty filter rejected** — a `JobFilter` with all defaults matches every job in the table. `EmptyFilterError` is raised unless `allow_empty_filter=True` is explicitly passed. This prevents accidental full-table cancels while allowing intentional "cancel everything" operations. Edge case, documented rather than handled: `JobFilter(status=[])` passes the guardrail (`status` is not `None`) but matches no jobs (an empty status sequence renders `status = ANY('{}')`) — the call is a benign no-op returning zero counts. - -4. **Single SQL statement for the UPDATEs (with EPQ-safe predicates)** — the two UPDATEs (pending/scheduled → cancelled, running → cancel_phase=1) are in a single CTE-based statement within a single transaction. **Status/cancel_phase predicates are duplicated in each UPDATE's own WHERE clause** (not just in the `matching` CTE) so that EvalPlanQual re-evaluates them against concurrently-modified rows. Events are inserted via `executemany` in the same transaction. NOTIFY is sent after commit (same pattern as `write_cancel_request`, batched). `ORDER BY id` in the matching CTE reduces deadlock probability; the **backend** retries the transaction on `asyncpg.DeadlockDetectedError` (max 3 attempts, jittered backoff — backend-owned because the exception type is asyncpg-specific and the backend owns the transaction boundary; the client stays backend-agnostic). - -5. **Filter reuse: `JobFilter`** — the same `JobFilter` used by `list_jobs` is used by `cancel_where`. The `limit`, `cursor`, and `order_by` fields are ignored (bulk cancel is not paginated). The `build_filter_conditions` helper (without `schema` parameter — conditions are schema-less fragments) ensures filter semantics are identical between query and mutation. `_list_jobs` regains cursor/limit/order_by handling by appending them after the shared builder call. - -6. **Returns `BulkCancelResult` with counts and IDs** — the counts let callers verify the operation affected the expected number of jobs. The IDs enable observability and follow-up operations (e.g., waiting for cooperative cancels to complete). - -7. **Counter: `taskq.cancellation.requested` incremented once per call** — not once per job. This matches the existing `cancel()` semantics (one increment per API call) and avoids counter inflation for bulk operations. The `taskq.cancel.notify_sent` counter is incremented per job notified (matching the single-job path). - -8. **Event parity with single-job path** — both backends insert the same event kinds as `write_cancel_request`: for pending/scheduled, both `state_change` (with actual `from_state`) and `cancel_request`; for running, only `cancel_request`. This ensures `job_events` consumers (audit, reclaim tooling) see consistent event streams regardless of backend or bulk-vs-single path. - -9. **No protocol version bump** — `cancel_where` is purely additive. A v3 backend implementation lacks the method, and the client raises `AttributeError` loudly. See the bump rule in `_protocol.py` lines 79-84. - -10. **Post-snapshot enqueue boundary** — jobs matching the filter that are enqueued *after* the statement's snapshot escape the cancel. Convergence is the caller's responsibility: stop producers before calling `cancel_where`, or issue a second call to catch stragglers. The `BulkCancelResult` counts let the caller detect non-convergence. - ---- - -## Revision log - -### 2026-07-29 — Post-review revision (verdict: NEEDS REWORK — 1 Critical / 4 High / 8 Medium / 7 Low) - -Revised against `.review/spec-review.md` under the standing 1.0.0 design directive: breaking changes are allowed when the result is strictly better, but no hacks, shims, or dual-path compat code; downstream sections describe the documented needs (issues #54/#57 and downstream code comments) and the correct end-state, not preservation of current usage. - -Resolved: - -- **C1 (EPQ race):** bulk-cancel SQL redesigned — status/`cancel_phase` predicates duplicated on the target table in each UPDATE's own WHERE clause (EvalPlanQual re-evaluates only the UPDATE's WHERE, not CTE contents); `cancelled` CTE uses the `FROM prev` pattern from `cancel_pending_scheduled` (`_sql_templates.py:407-415`) to carry the real `prev_status`; residual claim-boundary window documented (job escapes the call, never clobbered). New PG race test `test_pg_cancel_where_does_not_clobber_concurrent_claim` pins the safety property. -- **H1 (circular import):** `BulkCancelResult` defined in `_protocol.py` (next to `ScheduleRecord`), re-exported via `types.py` → `client/__init__.py` → `__init__.py` (same chain as `CancelResult`); stale "pydantic-free" docstring flagged for update in Task 2. -- **H2 (JSON injection):** `reason` serialized in Python via `jsonb_param`, bound as jsonb — never f-string interpolation; pinned by `test_pg_cancel_where_reason_with_quotes`. -- **H3 (silent limit cap):** in-memory `_cancel_where` sanitizes the filter (`limit=2**31`, `cursor=None`, `order_by=None`) before reusing `_list_jobs`; pinned by `test_cancel_where_ignores_filter_limit` (11 jobs, `limit=5` → all 11 cancelled). -- **H4 (fabricated justification):** removed the misquoted "confirmed by the issue author"; Design Decision #57-4 now explicitly reverses the documented exclusion (`jobs-clients.md:789`), analyzes the finalizer-snooze hazard against the verified mechanism (snooze-time `DeadlineExceeded` guard at `_sql_templates.py:254-275`; `sweep_deadline_exceeded` at `_sweeps.py:325+`), and scopes the hazard as opt-in per call. -- **Medium:** M1 event parity specified for both backends; M2 protocol member-count/docstring updates planned (36→37, 33→34); M3 fleet kill switch `sub_job_inherit_tags` on `WorkerSettings` (`settings.py`, not `_bootstrap.py`); M4 batch asymmetry justified as deliberate scoping (Decision #57-7); M5 all examples converted to the valid hyphenated tag charset, with the colon-form explicitly declared a non-goal; M6 deadlock retry (backend-owned, max 3 attempts) + `ORDER BY id` lock ordering + tenant-scale partitioning guidance; M7 downstream section rewritten per the directive (warden corrected to verified "today" + documented-need framing; cennan verified incl. the colon-tag incident; aacrtool marked planned); M8 builder contract states `_list_jobs` re-appends cursor/limit after the shared call. -- **Low:** L1 stale CTE rationale removed; L2 file-structure/task-list mismatches fixed (`_filter_sql.py`, new test files listed; `context.py` note; no `SqlTemplates.cancel_where` detour); L3 Task 9 placeholder replaced with a real merge test (`tagged_pipeline_stage`); L4 batched-NOTIFY statement written + per-job counter parity + listener-based PG test (`test_pg_cancel_where_notify_sent_for_running`); L5 actual `from_state` via `FROM prev`; L6 snapshot-boundary contract documented (docstring + Decision #54-10); L7 `build_filter_conditions(filter)` without the unused `schema` param. - -Design changes chosen under the directive (all breaking-or-behavioral by intent, no shims): deadlock retry lives in the backend (not the client) because the exception is asyncpg-specific and the backend owns the transaction boundary; NOTIFY send lives in `postgres.py` where the `notify_sent` counter is defined (avoids a module cycle); `schedule_to_close`/`start_to_close`/`heartbeat_timeout` are added to the sub-job surface as a deliberate, documented reversal of the prior exclusion. - -Left unresolved (deliberately): tag-charset widening (colon-form tags) — separate change, declared a non-goal; `inherit_tags` for `enqueue_batch` — follow-up spec (Decision #57-7); `snooze_extends_deadline` flag — noted as future work in Decision #57-4. From a9d6ffe68f2f6a3e2a90f774a051205a1de0342c Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 18:44:15 -0700 Subject: [PATCH 14/16] fix: address adversarial review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concurrency: - F1: Move ContextVar token reset to outermost finally in _consumer.py and run.py — prevents parent-tags leak if exception occurs before inner try block (Critical: tag data corruption across jobs) - F3: Remove status='running' filter from cancel_requested CTE subquery — a pending→running transition between snapshot and UPDATE previously escaped both CTEs; now the EPQ guard on the UPDATE handles it correctly - F4: NOTIFY failure after commit no longer masks BulkCancelResult — best-effort with warning log (workers discover cancel via heartbeat poll regardless) Architecture: - F4: Add JobFilter.has_predicates() method — guardrail uses it instead of fragile field-by-field check (Open-Closed Principle) - F5: enqueue_batch fallback now passes tags and inherit_tags=False from EnqueueItem (was silently dropping them) DRY: - Extract _cancel_notify_payload/_cancel_notify_channels helpers in postgres.py — shared between single-job and bulk cancel paths Docs: - Fix protocol docstring method count (34 async + 2 sync = 36) - Fix pre-existing colon-form tag examples in jobs-clients.md Hygiene: - Remove NotifyTarget from __all__ in _cancel_bulk.py Tests: - Parameter numbering () sequential verification - active=False, status-sequence, identity_key happy-path in filter_sql - _resolve_tags branch: inherit=True, no parent, explicit tags - Empty filter guardrail tags=() sub-branch - Deadlock during executemany retry - NotifyTarget filters None worker_id --- docs/guides/jobs-clients.md | 4 +- src/taskq/backend/_cancel_bulk.py | 4 +- src/taskq/backend/_protocol.py | 21 +++++++- src/taskq/backend/postgres.py | 56 +++++++++++----------- src/taskq/client/_enqueuer.py | 3 ++ src/taskq/client/_jobs.py | 10 +--- src/taskq/worker/_consumer.py | 2 +- src/taskq/worker/run.py | 79 ++++++++++++++++--------------- tests/test_cancel_where_client.py | 9 ++++ tests/test_cancel_where_pg.py | 47 ++++++++++++++++++ tests/test_filter_sql.py | 38 ++++++++++++--- tests/test_sub_job_tags.py | 15 ++++++ 12 files changed, 198 insertions(+), 90 deletions(-) diff --git a/docs/guides/jobs-clients.md b/docs/guides/jobs-clients.md index a8d762d0..f1c0e2d6 100644 --- a/docs/guides/jobs-clients.md +++ b/docs/guides/jobs-clients.md @@ -1107,7 +1107,7 @@ Tags are user-defined keyword labels stored in `jobs.tags text[]`. They have no handle = await client.enqueue( send_email, EmailPayload(to="user@example.com"), - tags=["notification", "priority:high", "tenant:acme"], + tags=["notification", "priority-high", "tenant-acme"], ) ``` @@ -1139,7 +1139,7 @@ Use `JobFilter.tags` with array-overlap semantics (matches jobs that have **any* page = await client.list(JobFilter( actor="send_email", status="failed", - tags=["priority:high", "tenant:acme"], + tags=["priority-high", "tenant-acme"], limit=50, )) ``` diff --git a/src/taskq/backend/_cancel_bulk.py b/src/taskq/backend/_cancel_bulk.py index 2a47c708..0af85a0e 100644 --- a/src/taskq/backend/_cancel_bulk.py +++ b/src/taskq/backend/_cancel_bulk.py @@ -21,7 +21,7 @@ from taskq.backend._records import jsonb_param from taskq.backend._sql_templates import SqlTemplates -__all__ = ["NotifyTarget", "_cancel_where"] +__all__ = ["_cancel_where"] class NotifyTarget(NamedTuple): @@ -66,7 +66,7 @@ async def _cancel_where( SET cancel_requested_at = now(), cancel_phase = 1 WHERE j.id IN ( SELECT id FROM matching - WHERE status = 'running' AND cancel_phase = 0 + WHERE cancel_phase = 0 ) AND j.status = 'running' AND j.cancel_phase = 0 RETURNING j.id, j.locked_by_worker diff --git a/src/taskq/backend/_protocol.py b/src/taskq/backend/_protocol.py index 53c493b6..539f2c0c 100644 --- a/src/taskq/backend/_protocol.py +++ b/src/taskq/backend/_protocol.py @@ -524,6 +524,23 @@ def __post_init__(self) -> None: "terminal/non-terminal meta-filter" ) + def has_predicates(self) -> bool: + """Return True if at least one filter predicate is set. + + Used by ``JobsClient.cancel_where`` to reject empty filters that + would match the entire table. New predicate fields added to + ``JobFilter`` automatically participate in this check. + """ + return ( + self.queue is not None + or self.status is not None + or self.actor is not None + or self.identity_key is not None + or self.batch_id is not None + or (self.tags is not None and len(self.tags) > 0) + or self.active is not None + ) + @dataclass(frozen=True, slots=True) class ScheduleCreateArgs: @@ -720,8 +737,8 @@ def dispatcher_pool(self) -> "asyncpg.Pool | None": class Backend(Protocol): """Contract that both PostgresBackend and InMemoryBackend satisfy. - 32 async methods plus two sync methods (``subscribe_wake`` and - ``subscribe_cancel_wake``) (34 methods total) covering enqueue, + 34 async methods plus two sync methods (``subscribe_wake`` and + ``subscribe_cancel_wake``) (36 methods total) covering enqueue, dispatch, heartbeat, terminal writes, attempt history, cancel signals, scheduling / sweeps, read, NOTIFY hook, and schedule CRUD. Method order grouped for review-grep ergonomics. diff --git a/src/taskq/backend/postgres.py b/src/taskq/backend/postgres.py index 82a5f39c..5d74a3c0 100644 --- a/src/taskq/backend/postgres.py +++ b/src/taskq/backend/postgres.py @@ -159,6 +159,14 @@ ) +def _cancel_notify_payload(job_id: UUID, worker_id: UUID) -> str: + return dumps_str({"type": "cancel", "job_id": str(job_id), "worker_id": str(worker_id)}) + + +def _cancel_notify_channels(schema: str, worker_id: UUID) -> list[str]: + return [events_channel(schema), worker_channel(schema, str(worker_id))] + + class PostgresBackend: """Production backend backed by Postgres. @@ -585,15 +593,8 @@ async def write_cancel_request( job_id=str(job_id), ) if _locked_by_worker is not None: - payload = dumps_str( - { - "type": "cancel", - "job_id": str(job_id), - "worker_id": str(_locked_by_worker), - } - ) - fleet_ch = events_channel(self._schema_name) - worker_ch = worker_channel(self._schema_name, str(_locked_by_worker)) + payload = _cancel_notify_payload(job_id, _locked_by_worker) + fleet_ch, worker_ch = _cancel_notify_channels(self._schema_name, _locked_by_worker) async with self._worker_pool.acquire() as notify_conn: await notify_conn.execute( "SELECT pg_notify($1, $2), pg_notify($3, $4)", @@ -633,28 +634,25 @@ async def cancel_where( channels: list[str] = [] payloads: list[str] = [] for target in notify_targets: - payload = dumps_str( - { - "type": "cancel", - "job_id": str(target.job_id), - "worker_id": str(target.worker_id), - } - ) - channels.extend( - [ - events_channel(self._schema_name), - worker_channel(self._schema_name, str(target.worker_id)), - ] - ) + payload = _cancel_notify_payload(target.job_id, target.worker_id) + ch = _cancel_notify_channels(self._schema_name, target.worker_id) + channels.extend(ch) payloads.extend([payload, payload]) - async with self._worker_pool.acquire() as notify_conn: - await notify_conn.execute( - "SELECT pg_notify(channel, payload) " - "FROM unnest($1::text[], $2::text[]) AS t(channel, payload)", - channels, - payloads, + try: + async with self._worker_pool.acquire() as notify_conn: + await notify_conn.execute( + "SELECT pg_notify(channel, payload) " + "FROM unnest($1::text[], $2::text[]) AS t(channel, payload)", + channels, + payloads, + ) + _cancel_notify_sent_counter.add(len(notify_targets), {"schema": self._schema_name}) + except Exception: + logger.warning( + "cancel_where_notify_failed", + notify_count=len(notify_targets), + exc_info=True, ) - _cancel_notify_sent_counter.add(len(notify_targets), {"schema": self._schema_name}) return result # ── Admin operations ────────────────────────────────────────────── diff --git a/src/taskq/client/_enqueuer.py b/src/taskq/client/_enqueuer.py index 60c35014..145396aa 100644 --- a/src/taskq/client/_enqueuer.py +++ b/src/taskq/client/_enqueuer.py @@ -329,6 +329,9 @@ async def enqueue_batch( idempotency_key=item.idempotency_key, idempotency_scope=item.idempotency_scope, identity_key=item.identity_key, + tags=list(item.tags) if item.tags else None, + inherit_tags=False, + start_to_close=item.start_to_close, ) handles.append(handle) except Exception as exc: diff --git a/src/taskq/client/_jobs.py b/src/taskq/client/_jobs.py index 09402324..7f6ee27e 100644 --- a/src/taskq/client/_jobs.py +++ b/src/taskq/client/_jobs.py @@ -761,15 +761,7 @@ async def cancel_where( """ from taskq.obs import record_cancel_requested - if not allow_empty_filter and ( - filter.queue is None - and filter.status is None - and filter.actor is None - and filter.identity_key is None - and filter.batch_id is None - and (filter.tags is None or len(filter.tags) == 0) - and filter.active is None - ): + if not allow_empty_filter and not filter.has_predicates(): raise EmptyFilterError() record_cancel_requested() diff --git a/src/taskq/worker/_consumer.py b/src/taskq/worker/_consumer.py index cbf5fb1c..4b9565bd 100644 --- a/src/taskq/worker/_consumer.py +++ b/src/taskq/worker/_consumer.py @@ -569,7 +569,6 @@ async def consume_one_job( ) finally: - _parent_tags_var.reset(_parent_tags_token) # Best-effort crash flush: ensures partial progress_state reaches PG # even when the actor raises unexpectedly (). if ( @@ -596,6 +595,7 @@ async def consume_one_job( await active_jobs.deregister(job.id) finally: + _parent_tags_var.reset(_parent_tags_token) if acquired and rate_limit_registry is not None: try: await asyncio.shield( diff --git a/src/taskq/worker/run.py b/src/taskq/worker/run.py index 2ce4d438..4fb9f1c6 100644 --- a/src/taskq/worker/run.py +++ b/src/taskq/worker/run.py @@ -348,60 +348,63 @@ async def consumer_loop_stub( _parent_tags_token = set_parent_tags(tuple(job.tags)) - ctx: JobContext[_StubPayload] = JobContext( - job_id=job.id, - actor=job.actor, - queue=job.queue, - attempt=job.attempt, - worker_id=worker_id, - payload=_StubPayload(), - jobs=SubJobEnqueuer( - loop_scope_resolved=None, - worker_pool=None, - backend=backend, - ), - log=bind_job_context( - _consumer_log, + try: + ctx: JobContext[_StubPayload] = JobContext( job_id=job.id, actor=job.actor, queue=job.queue, attempt=job.attempt, - identity_key=job.identity_key, - trace_id=job.trace_id or "", - ), - ) + worker_id=worker_id, + payload=_StubPayload(), + jobs=SubJobEnqueuer( + loop_scope_resolved=None, + worker_pool=None, + backend=backend, + ), + log=bind_job_context( + _consumer_log, + job_id=job.id, + actor=job.actor, + queue=job.queue, + attempt=job.attempt, + identity_key=job.identity_key, + trace_id=job.trace_id or "", + ), + ) - await deps.active_jobs.register(job.id, current_task, ctx) # type: ignore[arg-type] # Why: JobContext[_StubPayload] is a JobContext[BaseModel]; pyright cannot widen Generic[TChild] to Generic[TParent] without explicit covariance. + await deps.active_jobs.register(job.id, current_task, ctx) # type: ignore[arg-type] # Why: JobContext[_StubPayload] is a JobContext[BaseModel]; pyright cannot widen Generic[TChild] to Generic[TParent] without explicit covariance. - try: try: - await asyncio.wait_for( - ctx.cancel_event.wait(), - timeout=stub_work_timeout, - ) + try: + await asyncio.wait_for( + ctx.cancel_event.wait(), + timeout=stub_work_timeout, + ) + except asyncio.CancelledError: + with contextlib.suppress(asyncio.CancelledError): + await asyncio.shield(backend.mark_cancelled(job.id, worker_id)) + raise + except TimeoutError: + pass + + if ctx.cancellation_requested: + await asyncio.shield(backend.mark_cancelled(job.id, worker_id)) + else: + await asyncio.shield(backend.mark_succeeded(job.id, worker_id, None)) + except asyncio.CancelledError: with contextlib.suppress(asyncio.CancelledError): await asyncio.shield(backend.mark_cancelled(job.id, worker_id)) raise - except TimeoutError: - pass - if ctx.cancellation_requested: - await asyncio.shield(backend.mark_cancelled(job.id, worker_id)) - else: - await asyncio.shield(backend.mark_succeeded(job.id, worker_id, None)) - - except asyncio.CancelledError: - with contextlib.suppress(asyncio.CancelledError): - await asyncio.shield(backend.mark_cancelled(job.id, worker_id)) - raise + except Exception: + _consumer_log.exception("consumer-stub-error", job_id=str(job.id)) - except Exception: - _consumer_log.exception("consumer-stub-error", job_id=str(job.id)) + finally: + await deps.active_jobs.deregister(job.id) finally: _parent_tags_var.reset(_parent_tags_token) - await deps.active_jobs.deregister(job.id) async def di_consumer_loop( diff --git a/tests/test_cancel_where_client.py b/tests/test_cancel_where_client.py index c27ca7bf..f6e50176 100644 --- a/tests/test_cancel_where_client.py +++ b/tests/test_cancel_where_client.py @@ -82,6 +82,15 @@ async def test_client_cancel_where_empty_filter_raises() -> None: await client.cancel_where(JobFilter(), reason="oops") +async def test_client_cancel_where_empty_tags_tuple_raises() -> None: + """JobFilter(tags=()) is an empty filter — must raise EmptyFilterError.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + client = JobsClient(backend) + + with pytest.raises(EmptyFilterError): + await client.cancel_where(JobFilter(tags=()), reason="oops") + + async def test_client_cancel_where_empty_filter_override() -> None: """allow_empty_filter=True bypasses the guardrail.""" backend = InMemoryBackend(clock=FakeClock(_NOW)) diff --git a/tests/test_cancel_where_pg.py b/tests/test_cancel_where_pg.py index 2cb66172..b3f7a705 100644 --- a/tests/test_cancel_where_pg.py +++ b/tests/test_cancel_where_pg.py @@ -299,3 +299,50 @@ async def test_no_deadlock_no_retry(self) -> None: await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") assert conn.fetchrow.call_count == 1 + + async def test_deadlock_during_executemany_retries_correctly(self) -> None: + """Deadlock on executemany (after fetchrow succeeds) retries the + whole transaction — no phantom IDs from the aborted attempt.""" + row = self._success_row() + pool, conn = self._mock_pool_and_conn(fetch_row=row) + + call_count = [0] + + async def _flaky_executemany(query, args, *a, **kw): + call_count[0] += 1 + if call_count[0] <= 2: + raise asyncpg.DeadlockDetectedError() + return None + + conn.executemany = _flaky_executemany + + sql = MagicMock() + sql.insert_event = "INSERT INTO job_events VALUES ($1, $2, $3, $4)" + + result, _notify = await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") + + assert result.cancelled_directly == 1 + assert len(result.cancelled_ids) == 1 + + async def test_notify_target_filters_none_worker_id(self) -> None: + """A running job with NULL locked_by_worker is excluded from + notify_targets (no worker to NOTIFY).""" + jid = uuid4() + pool, _ = self._mock_pool_and_conn( + fetch_row={ + "cancelled_directly": 0, + "cancel_requested": 1, + "cancelled_ids": [], + "cancelled_prev_statuses": [], + "cancel_requested_ids": [jid], + "cancel_requested_workers": [None], + } + ) + + sql = MagicMock() + sql.insert_event = "INSERT INTO job_events VALUES ($1, $2, $3, $4)" + + result, notify = await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") + + assert result.cancel_requested == 1 + assert len(notify) == 0 diff --git a/tests/test_filter_sql.py b/tests/test_filter_sql.py index 9afcc33d..ec0ccc9e 100644 --- a/tests/test_filter_sql.py +++ b/tests/test_filter_sql.py @@ -6,10 +6,12 @@ order_by are ignored (they are handled by the caller). """ +import re from uuid import uuid4 from taskq.backend._filter_sql import build_filter_conditions -from taskq.backend._protocol import JobFilter +from taskq.backend._protocol import IdentityKey, JobFilter +from taskq.backend.statemachine import ACTIVE_STATUSES, TERMINAL_STATUSES class TestBuildFilterConditions: @@ -40,6 +42,26 @@ def test_active_true_filter(self) -> None: result = build_filter_conditions(JobFilter(active=True)) assert len(result.conditions) == 1 assert "status" in result.conditions[0] + assert set(result.params[0]) == set(ACTIVE_STATUSES) # type: ignore[arg-type] + + def test_active_false_filter(self) -> None: + result = build_filter_conditions(JobFilter(active=False)) + assert len(result.conditions) == 1 + assert "status" in result.conditions[0] + assert set(result.params[0]) == set(TERMINAL_STATUSES) # type: ignore[arg-type] + + def test_status_sequence_filter(self) -> None: + result = build_filter_conditions(JobFilter(status=["pending", "running"])) + assert len(result.conditions) == 1 + assert "ANY" in result.conditions[0] + assert result.params == (["pending", "running"],) + + def test_identity_key_filter(self) -> None: + key = IdentityKey("tenant-acme") + result = build_filter_conditions(JobFilter(identity_key=key)) + assert len(result.conditions) == 1 + assert "identity_key" in result.conditions[0] + assert result.params == (key,) def test_combined_filters(self) -> None: result = build_filter_conditions( @@ -48,13 +70,19 @@ def test_combined_filters(self) -> None: assert len(result.conditions) == 3 def test_cursor_and_order_by_ignored(self) -> None: - """cancel_where doesn't use cursor/order_by — the builder should - not include them in conditions.""" result = build_filter_conditions( JobFilter(cursor="some-cursor", order_by=None), ) assert result.conditions == () + def test_parameter_numbering_is_sequential(self) -> None: + """$N placeholders must be sequentially numbered and align + positionally with the params tuple.""" + result = build_filter_conditions(JobFilter(queue="q1", actor="a1", tags=("t1",))) + numbers = [int(re.search(r"\$(\d+)", c).group(1)) for c in result.conditions] + assert numbers == list(range(1, len(numbers) + 1)), numbers + assert result.params == ("q1", "a1", ["t1"]) + class TestSQLInjectionSafety: """Verify user-supplied values are always parameterized, never @@ -66,8 +94,6 @@ class TestSQLInjectionSafety: """ def test_queue_with_sql_metacharacters_is_parameterized(self) -> None: - """A queue name containing SQL injection payload must end up in - params, not in the condition string.""" payload = "'; DROP TABLE jobs; --" result = build_filter_conditions(JobFilter(queue=payload)) assert result.params == (payload,) @@ -90,8 +116,6 @@ def test_tags_with_sql_metacharacters_are_parameterized(self) -> None: assert payload[0] not in cond def test_identity_key_with_sql_metacharacters_is_parameterized(self) -> None: - from taskq.backend._protocol import IdentityKey - payload = IdentityKey("x'; DROP TABLE jobs; --") result = build_filter_conditions(JobFilter(identity_key=payload)) assert result.params == (payload,) diff --git a/tests/test_sub_job_tags.py b/tests/test_sub_job_tags.py index 446b5840..d833c35a 100644 --- a/tests/test_sub_job_tags.py +++ b/tests/test_sub_job_tags.py @@ -71,6 +71,21 @@ async def test_explicit_tags_no_inheritance(self) -> None: assert row is not None assert row.tags == ("alpha", "beta") + async def test_explicit_tags_no_parent_tags_inherit_true(self) -> None: + """inherit_tags=True (default) with no parent tags and explicit + tags returns only the explicit tags — no merge with empty parent.""" + backend = InMemoryBackend(clock=FakeClock(_NOW)) + enqueuer = _make_enqueuer(backend) + + handle = await enqueuer.enqueue( + _make_actor_ref(), + _Payload(), + tags=["alpha"], + ) + row = await backend.get(handle.job_id) + assert row is not None + assert row.tags == ("alpha",) + async def test_tags_validated(self) -> None: """Invalid tags raise ValueError.""" backend = InMemoryBackend(clock=FakeClock(_NOW)) From 20245b1b608a9481b4f6686dab5b258c70c08f1a Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 19:38:23 -0700 Subject: [PATCH 15/16] fix: address all review findings (C1-C2, H1, M1-M11, Low) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical: - C1: Redesign cancel_where SQL as two-statement pattern in same tx (pending/scheduled first, running second with fresh snapshot) — eliminates the race where pending→running escapes both CTEs - C2: Register pipeline_stage/tagged_pipeline_stage in worker_entry.py ACTORS dict — e2e tests were broken without this High: - H1: Fix all pyright errors in test files (typed dict generics, annotated mock params, guard re.search result) Medium: - M1: Add guardrail contract note to Backend.cancel_where docstring - M2: Fix has_predicates() docstring (manual sync, not automatic) - M3: Add parent_tags() context manager, drop _parent_tags_var from __all__, use context manager in run.py, document contract in module docstring - M4: CHANGELOG [Unreleased] entries + actors.md tag inheritance note - M9: BulkCancelResult ID fields are tuple[UUID, ...] not list[UUID] - M11: Fix false heartbeat_timeout 'actor default' docs claim Low: - Fix _filter_sql.py docstring (equivalence claim corrected) - Fix FilterSQL frozen docstring (params contains mutable lists) - Fix postgres.py module docstring (add _cancel_bulk, _filter_sql) - Add JobFilter docstring mention of cancel_where - Add pyproject.toml S608/SIM117/S311 rationale comments - Use public 'from taskq import JobFilter' in e2e tests and docs - Simplify _resolve_tags with dict.fromkeys dedup - README: add bulk cancel to capability list - Remove NotifyTarget from _cancel_bulk __all__ --- CHANGELOG.md | 24 +++++ README.md | 4 +- docs/guides/actors.md | 7 ++ docs/guides/cancellation.md | 2 +- docs/guides/jobs-clients.md | 10 +- pyproject.toml | 4 + src/taskq/backend/_cancel_bulk.py | 164 +++++++++++++++++++----------- src/taskq/backend/_filter_sql.py | 14 +-- src/taskq/backend/_protocol.py | 25 ++++- src/taskq/backend/postgres.py | 8 +- src/taskq/client/_enqueuer.py | 45 ++++++-- src/taskq/testing/_cancel_bulk.py | 4 +- src/taskq/worker/_consumer.py | 4 +- src/taskq/worker/run.py | 9 +- tests/e2e/test_cancellation.py | 3 +- tests/e2e/test_sub_job_tags.py | 2 +- tests/e2e/worker_entry.py | 4 + tests/test_bulk_cancel_types.py | 23 +++++ tests/test_cancel_where_pg.py | 85 ++++++++++------ tests/test_filter_sql.py | 6 +- 20 files changed, 307 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d4b1f0..97bc01b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +* `JobsClient.cancel_where(filter, reason)` — bulk cancel all jobs matching a + `JobFilter` in a single set-based operation. Pending/scheduled jobs go straight + to terminal `cancelled`; running jobs get cooperative cancel (`cancel_phase=1`). + Returns `BulkCancelResult` with counts and affected IDs. Empty filters are + rejected with `EmptyFilterError` unless `allow_empty_filter=True` is passed. +* `SubJobEnqueuer.enqueue()` now accepts `tags`, `inherit_tags`, + `schedule_to_close`, `start_to_close`, and `heartbeat_timeout` parameters. + Sub-jobs inherit the parent job's tags by default (`inherit_tags=True`); pass + `inherit_tags=False` to suppress inheritance for a specific sub-job. +* `BulkCancelResult` and `EmptyFilterError` exported from `taskq` top-level. + +### Changed + +* **Sub-jobs now inherit parent tags by default.** Every `ctx.jobs.enqueue()` + call inside an actor body now propagates the parent job's tags to the sub-job, + making sub-jobs findable by `JobFilter(tags=...)` and cancellable by + `cancel_where`. Pass `inherit_tags=False` per-call to opt out. This is a + behavior change for any code that relied on sub-job tags being empty — + inherited tags make sub-jobs visible to tag-based filters and bulk cancels. + ## [0.2.2](https://github.com/AZX-PBC-OSS/TaskQ/compare/v0.2.1...v0.2.2) (2026-07-22) diff --git a/README.md b/README.md index dbce89ae..97425f6c 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,9 @@ Async-native, Postgres-backed background job library for Python 3.12+. isn't inside an actor should instead poll `BatchHandle.status(db_connection)`. See [Jobs & Clients](docs/guides/jobs-clients.md#enqueue_batch). - **Cancellation** — cooperative cancellation with grace periods and - force-cancel sweeps; `ctx.check_cancelled()` inside actor bodies. + force-cancel sweeps; `ctx.check_cancelled()` inside actor bodies; bulk + `cancel_where(filter)` for set-based cancellation by tag, queue, actor, + or batch ID. - **Progress tracking** — `ctx.progress(...)` events buffered and published to subscribers and the admin UI. - **Workgroups** — multi-worker process supervision with a shared heartbeat diff --git a/docs/guides/actors.md b/docs/guides/actors.md index 2d1705e5..653fb203 100644 --- a/docs/guides/actors.md +++ b/docs/guides/actors.md @@ -689,6 +689,13 @@ INSERTs are part of the parent's database transaction: This is the correct default for fan-out patterns where sub-jobs should only exist if the parent completes successfully. +**Tag inheritance.** Sub-jobs enqueued via `ctx.jobs.enqueue()` inherit the parent +job's tags by default. This makes sub-jobs findable by `JobFilter(tags=...)` and +cancellable by `cancel_where`. Pass `inherit_tags=False` to suppress inheritance +for a specific sub-job, or pass explicit `tags=[...]` to merge with inherited +tags. See [Jobs & Clients — Sub-job tag inheritance](jobs-clients.md#tag-inheritance) +for the full semantics table. + **Autonomous fallback.** If no LOOP-scope `asyncpg.Connection` is registered in the DI container, `ctx.jobs.enqueue()` falls back to the worker pool and commits each INSERT independently. In this mode, sub-jobs are persisted even if the parent subsequently raises diff --git a/docs/guides/cancellation.md b/docs/guides/cancellation.md index 114ac29c..d7e53e7e 100644 --- a/docs/guides/cancellation.md +++ b/docs/guides/cancellation.md @@ -33,7 +33,7 @@ result = await handle.cancel(reason="deadline exceeded") ### Via `JobsClient.cancel_where()` ```python -from taskq.backend._protocol import JobFilter +from taskq import JobFilter result = await client.cancel_where( JobFilter(tags=("tenant-acme",), active=True), diff --git a/docs/guides/jobs-clients.md b/docs/guides/jobs-clients.md index f1c0e2d6..0caf9b9b 100644 --- a/docs/guides/jobs-clients.md +++ b/docs/guides/jobs-clients.md @@ -879,11 +879,11 @@ set per-item tags explicitly. #### `schedule_to_close` / `start_to_close` / `heartbeat_timeout` -These parameters are passed through to `build_enqueue_args` and override the -actor's declared defaults for this specific sub-job. Note that -`schedule_to_close` bounds total wall-clock time *including* time snoozed on -`wait_for_batch` — finalizer-style sub-jobs that snooze for long periods should -set it generously or not at all. +`schedule_to_close` and `start_to_close` override the actor's declared defaults for +this specific sub-job. `heartbeat_timeout` has no actor-level declaration — the +per-call value is the only source. Note that `schedule_to_close` bounds total +wall-clock time *including* time snoozed on `wait_for_batch` — finalizer-style +sub-jobs that snooze for long periods should set it generously or not at all. The per-call `max_pending=` argument is resolved against the operator-owned stored cap and the `@actor(...)` literal, not in place of them: against a non-NULL stored diff --git a/pyproject.toml b/pyproject.toml index 7a99a3ef..7d61d1ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,6 +202,10 @@ ignore = [ "src/taskq/backend/_terminal.py" = ["S608", "SIM117"] "src/taskq/backend/_enqueue.py" = ["S608", "SIM117"] "src/taskq/backend/_reads.py" = ["S608"] +# S608: schema name validated against _IDENT_RE before interpolation; same +# rationale as _reads.py. All user-supplied values use $N parameter binding. +# SIM117: pool.acquire() and conn.transaction() cannot be flattened. +# S311: random module used for jitter in deadlock retry backoff, not crypto. "src/taskq/backend/_cancel_bulk.py" = ["S608", "SIM117", "S311"] "src/taskq/backend/_dispatch.py" = ["S608", "SIM117"] "src/taskq/backend/_sql_templates.py" = ["S608"] diff --git a/src/taskq/backend/_cancel_bulk.py b/src/taskq/backend/_cancel_bulk.py index 0af85a0e..29f59839 100644 --- a/src/taskq/backend/_cancel_bulk.py +++ b/src/taskq/backend/_cancel_bulk.py @@ -1,8 +1,20 @@ """Bulk cancel SQL implementation for PostgresBackend. -Module-level function following the same pattern as ``_reads.py``, -``_terminal.py``, etc. The SQL uses two CTEs in a single statement -with EPQ-safe predicates duplicated in each UPDATE's WHERE clause. +Two-statement pattern within a single transaction, mirroring the +single-job ``write_cancel_request`` path: + +1. ``cancel_pending_scheduled`` — UPDATE pending/scheduled rows to + terminal ``cancelled`` with EPQ-safe predicates on the target table. +2. ``cancel_running`` — UPDATE running rows with ``cancel_phase=0`` to + ``cancel_phase=1`` (cooperative cancel), using a fresh snapshot that + catches jobs dispatched between statements. + +The two-statement approach eliminates the race where a job transitioning +``pending→running`` mid-statement escapes both CTEs in a single-shot +design: statement 1's EPQ guard rejects the now-running row, and +statement 2's fresh snapshot sees it as running and sets +``cancel_phase=1``. + Events are inserted via ``executemany`` within the same transaction. NOTIFY is sent by the caller (``PostgresBackend.cancel_where``) after commit because the ``taskq.cancel.notify_sent`` counter lives in @@ -42,11 +54,15 @@ async def _cancel_where( conditions_str = " AND ".join(filter_sql.conditions) if filter_sql.conditions else "TRUE" params = list(filter_sql.params) - cancel_sql = f""" + # Statement 1: cancel pending/scheduled → terminal 'cancelled' + # EPQ-safe: predicates on the target table (j.status) are re-evaluated + # for concurrently-modified rows. + cancel_ps_sql = f""" WITH matching AS ( - SELECT id, status, locked_by_worker + SELECT id, status FROM "{schema}".jobs WHERE {conditions_str} + AND status IN ('pending', 'scheduled') ORDER BY id ), cancelled AS ( @@ -55,27 +71,42 @@ async def _cancel_where( FROM ( SELECT id, status AS prev_status FROM matching - WHERE status IN ('pending', 'scheduled') ) AS prev WHERE j.id = prev.id AND j.status IN ('pending', 'scheduled') RETURNING j.id, prev.prev_status + ) + SELECT + (SELECT count(*)::int FROM cancelled) AS cancelled_directly, + (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, + (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses + """ + + # Statement 2: cooperative cancel for running jobs with cancel_phase=0 + # Fresh snapshot — catches jobs dispatched between statements 1 and 2. + cancel_running_sql = f""" + WITH matching AS ( + SELECT id, locked_by_worker + FROM "{schema}".jobs + WHERE {conditions_str} + AND status = 'running' + AND cancel_phase = 0 + ORDER BY id ), cancel_requested AS ( UPDATE "{schema}".jobs AS j SET cancel_requested_at = now(), cancel_phase = 1 - WHERE j.id IN ( - SELECT id FROM matching - WHERE cancel_phase = 0 - ) - AND j.status = 'running' AND j.cancel_phase = 0 - RETURNING j.id, j.locked_by_worker + FROM ( + SELECT id, locked_by_worker + FROM matching + ) AS prev + WHERE j.id = prev.id + AND j.status = 'running' + AND j.cancel_phase = 0 + RETURNING j.id, prev.locked_by_worker ) SELECT - (SELECT count(*)::int FROM cancelled) AS cancelled_directly, (SELECT count(*)::int FROM cancel_requested) AS cancel_requested, - (SELECT array_agg(id ORDER BY id) FROM cancelled) AS cancelled_ids, - (SELECT array_agg(prev_status ORDER BY id) FROM cancelled) AS cancelled_prev_statuses, (SELECT array_agg(id ORDER BY id) FROM cancel_requested) AS cancel_requested_ids, (SELECT array_agg(locked_by_worker ORDER BY id) FROM cancel_requested) AS cancel_requested_workers """ @@ -90,51 +121,62 @@ async def _cancel_where( try: async with pool.acquire() as conn: async with conn.transaction(): - row = await conn.fetchrow(cancel_sql, *params) - - if row is None: - raise RuntimeError("cancel_where: aggregate query returned no rows") - cancelled_ids = list(row["cancelled_ids"] or []) - cancel_requested_ids = list(row["cancel_requested_ids"] or []) - prev_statuses: dict[UUID, str] = dict( - zip(cancelled_ids, row["cancelled_prev_statuses"] or [], strict=True) - ) - notify_targets = [ - NotifyTarget(job_id=jid, worker_id=wid) - for jid, wid in zip( - cancel_requested_ids, - row["cancel_requested_workers"] or [], - strict=True, - ) - if wid is not None - ] - - if cancelled_ids: - await conn.executemany( - sql.insert_event, - [ - ( - jid, - "state_change", - jsonb_param( - { - "from_state": prev_statuses[jid], - "to_state": "cancelled", - } - ), - ) - for jid in cancelled_ids - ], - ) - await conn.executemany( - sql.insert_event, - [(jid, "cancel_request", cr_detail) for jid in cancelled_ids], - ) - if cancel_requested_ids: - await conn.executemany( - sql.insert_event, - [(jid, "cancel_request", cr_detail) for jid in cancel_requested_ids], + # Statement 1: pending/scheduled → cancelled + ps_row = await conn.fetchrow(cancel_ps_sql, *params) + if ps_row is not None: + cancelled_ids = list(ps_row["cancelled_ids"] or []) + prev_statuses: dict[UUID, str] = dict( + zip( + cancelled_ids, + ps_row["cancelled_prev_statuses"] or [], + strict=True, + ) ) + + if cancelled_ids: + await conn.executemany( + sql.insert_event, + [ + ( + jid, + "state_change", + jsonb_param( + { + "from_state": prev_statuses[jid], + "to_state": "cancelled", + } + ), + ) + for jid in cancelled_ids + ], + ) + await conn.executemany( + sql.insert_event, + [(jid, "cancel_request", cr_detail) for jid in cancelled_ids], + ) + + # Statement 2: running → cooperative cancel (fresh snapshot) + running_row = await conn.fetchrow(cancel_running_sql, *params) + if running_row is not None: + cancel_requested_ids = list(running_row["cancel_requested_ids"] or []) + notify_targets = [ + NotifyTarget(job_id=jid, worker_id=wid) + for jid, wid in zip( + cancel_requested_ids, + running_row["cancel_requested_workers"] or [], + strict=True, + ) + if wid is not None + ] + + if cancel_requested_ids: + await conn.executemany( + sql.insert_event, + [ + (jid, "cancel_request", cr_detail) + for jid in cancel_requested_ids + ], + ) break except asyncpg.DeadlockDetectedError: if attempt == 2: @@ -144,7 +186,7 @@ async def _cancel_where( result = BulkCancelResult( cancelled_directly=len(cancelled_ids), cancel_requested=len(cancel_requested_ids), - cancelled_ids=cancelled_ids, - cancel_requested_ids=cancel_requested_ids, + cancelled_ids=tuple(cancelled_ids), + cancel_requested_ids=tuple(cancel_requested_ids), ) return result, notify_targets diff --git a/src/taskq/backend/_filter_sql.py b/src/taskq/backend/_filter_sql.py index 371cef52..d9dc0d37 100644 --- a/src/taskq/backend/_filter_sql.py +++ b/src/taskq/backend/_filter_sql.py @@ -7,10 +7,9 @@ fields are NOT handled here — callers apply them separately. This module is SQL-only; the in-memory backend filters via its own -implementation in ``testing/_reads.py``. Backend equivalence is enforced -by the shared test suite (``test_backend_equivalence.py``), not shared -code — the two filter-matching strategies (SQL WHERE vs Python -predicates) do not share a trivial interface. +implementation in ``testing/_reads.py``. Filter semantics between the +two backends are verified by ``test_job_filter.py`` and the per-backend +cancel_where test suites. """ from dataclasses import dataclass @@ -26,9 +25,10 @@ class FilterSQL: """Built SQL fragments and parameters from a JobFilter. - ``conditions`` and ``params`` are stored as tuples so the frozen - contract is meaningful (mutable list fields in a frozen dataclass - only prevent reassignment, not in-place mutation). + ``conditions`` and ``params`` are stored as tuples so the container + itself is immutable (prevents reassignment of the field). Individual + param elements (e.g. ``list[str]`` for ``ANY()``) are inherently + mutable — callers consume them immediately without mutation. """ conditions: tuple[str, ...] = () diff --git a/src/taskq/backend/_protocol.py b/src/taskq/backend/_protocol.py index 539f2c0c..623efa49 100644 --- a/src/taskq/backend/_protocol.py +++ b/src/taskq/backend/_protocol.py @@ -424,7 +424,13 @@ class CancelFlag: @dataclass(frozen=True, slots=True) class JobFilter: - """Filter parameters for :meth:`Backend.list_jobs`. + """Filter parameters for :meth:`Backend.list_jobs` and + :meth:`Backend.cancel_where`. + + For ``cancel_where``, the ``limit``, ``cursor``, and ``order_by`` + fields are ignored — a bulk cancel is not paginated. Use + :meth:`has_predicates` to check whether the filter has at least one + predicate before passing it to ``cancel_where``. Heads-up: ``active=True`` is **not** Celery's 'active' — Celery's means 'currently executing' (``running`` only), TaskQ's means 'not @@ -529,7 +535,10 @@ def has_predicates(self) -> bool: Used by ``JobsClient.cancel_where`` to reject empty filters that would match the entire table. New predicate fields added to - ``JobFilter`` automatically participate in this check. + ``JobFilter`` MUST be added here and in + ``build_filter_conditions`` — the two are kept in sync manually. + Non-predicate fields (``limit``, ``cursor``, ``order_by``) are + excluded by design. """ return ( self.queue is not None @@ -651,10 +660,10 @@ class BulkCancelResult(BaseModel): cancel_requested: int """Count of running jobs with cancel_phase=1 set (cooperative cancel).""" - cancelled_ids: list[UUID] + cancelled_ids: tuple[UUID, ...] """IDs of jobs cancelled directly (pending/scheduled → cancelled).""" - cancel_requested_ids: list[UUID] + cancel_requested_ids: tuple[UUID, ...] """IDs of running jobs with cancel requested.""" @property @@ -1006,6 +1015,14 @@ async def cancel_where( The filter's ``limit``, ``cursor``, and ``order_by`` fields are ignored — this is a bulk write, not a paginated read. + **Guardrail:** the client layer (:meth:`JobsClient.cancel_where`) + rejects empty filters (no predicates) with + :class:`EmptyFilterError`. Backend implementations receive a + filter that has already been validated. A direct backend call + with ``JobFilter()`` renders ``WHERE TRUE`` and cancels the + entire table — callers using the backend directly are + responsible for validating the filter. + Returns a :class:`BulkCancelResult` with counts and affected IDs. """ ... diff --git a/src/taskq/backend/postgres.py b/src/taskq/backend/postgres.py index 5d74a3c0..f50d0783 100644 --- a/src/taskq/backend/postgres.py +++ b/src/taskq/backend/postgres.py @@ -9,9 +9,11 @@ (:mod:`taskq.backend._schedules`), terminal writes (:mod:`taskq.backend._terminal`), enqueue (:mod:`taskq.backend._enqueue`), reads (:mod:`taskq.backend._reads`), -and dispatch (:mod:`taskq.backend._dispatch`) live in companion -submodules; this module holds the cohesive core: ``__init__``, heartbeat, -cancel signals, NOTIFY, and schedule CRUD wiring. +bulk cancel (:mod:`taskq.backend._cancel_bulk`), filter SQL builder +(:mod:`taskq.backend._filter_sql`), and dispatch +(:mod:`taskq.backend._dispatch`) live in companion submodules; this +module holds the cohesive core: ``__init__``, heartbeat, cancel +signals, NOTIFY, and schedule CRUD wiring. """ import asyncio diff --git a/src/taskq/client/_enqueuer.py b/src/taskq/client/_enqueuer.py index 145396aa..b7a72a57 100644 --- a/src/taskq/client/_enqueuer.py +++ b/src/taskq/client/_enqueuer.py @@ -5,12 +5,19 @@ One instance per loop — survives across dispatches so the per-100-enqueue re-warning fires on the loop-level counter, not per-job. +Parent-tag propagation: the consumer sets the parent job's tags via +``set_parent_tags()`` before actor invocation and resets them after +(via ``parent_tags()`` context manager or manual token reset). The +``contextvars.ContextVar`` ensures concurrent consumers in the same +event loop each see their own parent tags — asyncio Tasks copy the +context at creation time. """ from __future__ import annotations +import contextlib import contextvars -from collections.abc import Mapping, Sequence +from collections.abc import Generator, Mapping, Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, cast from uuid import UUID @@ -40,7 +47,7 @@ from taskq.actor import ActorRef -__all__ = ["SubJobEnqueuer", "_parent_tags_var", "set_parent_tags"] +__all__ = ["SubJobEnqueuer", "parent_tags", "set_parent_tags"] _log: structlog.stdlib.BoundLogger = structlog.get_logger(__name__) @@ -54,11 +61,32 @@ def set_parent_tags(tags: tuple[str, ...]) -> contextvars.Token[tuple[str, ...]] """Set the parent job's tags for sub-job tag inheritance. Called by the consumer before actor invocation. The returned token - can be used to reset the context after the actor completes. + must be used to reset the context after the actor completes — use + ``_parent_tags_var.reset(token)`` or the ``parent_tags()`` context + manager. """ return _parent_tags_var.set(tags) +@contextlib.contextmanager +def parent_tags(tags: tuple[str, ...]) -> Generator[None, None, None]: + """Context manager that sets parent tags for the duration of the block. + + Ensures the ContextVar is reset on all exit paths (success, exception, + cancellation). Use this at worker entry points instead of manual + set/reset:: + + with parent_tags(tuple(job.tags)): + # actor invocation, sub-job enqueues, etc. + ... + """ + token = _parent_tags_var.set(tags) + try: + yield + finally: + _parent_tags_var.reset(token) + + class SubJobEnqueuer: """Enqueue sub-jobs from within an actor body. @@ -173,6 +201,9 @@ def _resolve_tags( """Resolve tags with parent inheritance. Returns a list suitable for build_enqueue_args, or None for empty. + Deduplication is order-preserving (parent first); the downstream + ``_validate_and_dedup_tags`` in ``build_enqueue_args`` also + deduplicates, but we do it here so the merge result is clean. """ parent_tags = _parent_tags_var.get() if inherit_tags else () @@ -187,13 +218,7 @@ def _resolve_tags( if not tags: return list(parent_tags) - seen: set[str] = set(parent_tags) - merged = list(parent_tags) - for tag in tags: - if tag not in seen: - seen.add(tag) - merged.append(tag) - return merged + return list(dict.fromkeys((*parent_tags, *tags))) def _resolve_connection( self, diff --git a/src/taskq/testing/_cancel_bulk.py b/src/taskq/testing/_cancel_bulk.py index db3ac05c..0458616a 100644 --- a/src/taskq/testing/_cancel_bulk.py +++ b/src/taskq/testing/_cancel_bulk.py @@ -66,6 +66,6 @@ async def _cancel_where( return BulkCancelResult( cancelled_directly=len(cancelled_ids), cancel_requested=len(cancel_requested_ids), - cancelled_ids=cancelled_ids, - cancel_requested_ids=cancel_requested_ids, + cancelled_ids=tuple(cancelled_ids), + cancel_requested_ids=tuple(cancel_requested_ids), ) diff --git a/src/taskq/worker/_consumer.py b/src/taskq/worker/_consumer.py index 4b9565bd..b6218c1c 100644 --- a/src/taskq/worker/_consumer.py +++ b/src/taskq/worker/_consumer.py @@ -29,7 +29,7 @@ JobRow, ) from taskq.backend.clock import Clock -from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags +from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var from taskq.constants import MAX_RESULT_BYTES from taskq.context import JobContext from taskq.exceptions import ( @@ -353,7 +353,7 @@ async def consume_one_job( _buf = _ProgressBuffer(job_id=job.id, base_seq=job.progress_seq) _progress_buffers[job.id] = _buf - _parent_tags_token = set_parent_tags(tuple(job.tags)) + _parent_tags_token = _parent_tags_var.set(tuple(job.tags)) try: validated_payload = ( diff --git a/src/taskq/worker/run.py b/src/taskq/worker/run.py index 4fb9f1c6..fb80c00e 100644 --- a/src/taskq/worker/run.py +++ b/src/taskq/worker/run.py @@ -46,7 +46,7 @@ from taskq.backend._protocol import Backend, JobRow from taskq.backend._records import jsonb_param from taskq.backend.clock import Clock -from taskq.client._enqueuer import SubJobEnqueuer, _parent_tags_var, set_parent_tags +from taskq.client._enqueuer import SubJobEnqueuer, parent_tags from taskq.constants import ( _IDENT_RE, # pyright: ignore[reportPrivateUsage] # Why: canonical identifier regex; copying would drift the validation pattern. ) @@ -346,9 +346,7 @@ async def consumer_loop_stub( if current_task is None: raise RuntimeError("consumer_loop_stub must run inside a TaskGroup") - _parent_tags_token = set_parent_tags(tuple(job.tags)) - - try: + with parent_tags(tuple(job.tags)): ctx: JobContext[_StubPayload] = JobContext( job_id=job.id, actor=job.actor, @@ -403,9 +401,6 @@ async def consumer_loop_stub( finally: await deps.active_jobs.deregister(job.id) - finally: - _parent_tags_var.reset(_parent_tags_token) - async def di_consumer_loop( deps: WorkerDeps, diff --git a/tests/e2e/test_cancellation.py b/tests/e2e/test_cancellation.py index a48e3c57..69f07da9 100644 --- a/tests/e2e/test_cancellation.py +++ b/tests/e2e/test_cancellation.py @@ -25,8 +25,7 @@ import pytest -from taskq import JobFailed -from taskq.backend._protocol import JobFilter +from taskq import JobFailed, JobFilter from taskq.batch import EnqueueItem from ._assertions import fetch_effects, wait_for_effects, wait_for_handle_status diff --git a/tests/e2e/test_sub_job_tags.py b/tests/e2e/test_sub_job_tags.py index 036789f5..8a59fcbd 100644 --- a/tests/e2e/test_sub_job_tags.py +++ b/tests/e2e/test_sub_job_tags.py @@ -12,7 +12,7 @@ import pytest -from taskq.backend._protocol import JobFilter +from taskq import JobFilter from ._assertions import wait_for_effects from .actors import ( diff --git a/tests/e2e/worker_entry.py b/tests/e2e/worker_entry.py index 71b8bce3..8b4855a8 100644 --- a/tests/e2e/worker_entry.py +++ b/tests/e2e/worker_entry.py @@ -22,12 +22,14 @@ import_contacts_csv, long_running_job, loop_blocker_job, + pipeline_stage, quick_result, rebuild_search_index, send_welcome_email, short_lived_job, slow_deliver_webhook, sync_user_profile, + tagged_pipeline_stage, ) from e2e.di import build_registry from taskq import ActorRef @@ -53,6 +55,8 @@ "cron_heartbeat": cron_heartbeat, "short_lived_job": short_lived_job, "concurrent_tracked_worker": concurrent_tracked_worker, + "pipeline_stage": pipeline_stage, + "tagged_pipeline_stage": tagged_pipeline_stage, } diff --git a/tests/test_bulk_cancel_types.py b/tests/test_bulk_cancel_types.py index d472c11d..6bd38d10 100644 --- a/tests/test_bulk_cancel_types.py +++ b/tests/test_bulk_cancel_types.py @@ -41,6 +41,29 @@ def test_zero_counts(self) -> None: ) assert result.total_affected == 0 + def test_ids_are_tuples(self) -> None: + """ID fields are tuples, not lists — frozen immutability.""" + result = BulkCancelResult( + cancelled_directly=1, + cancel_requested=0, + cancelled_ids=[uuid4()], + cancel_requested_ids=[], + ) + assert isinstance(result.cancelled_ids, tuple) + assert isinstance(result.cancel_requested_ids, tuple) + + def test_list_input_coerced_to_tuple(self) -> None: + """Pydantic v2 coerces list inputs to tuples.""" + ids = [uuid4() for _ in range(2)] + result = BulkCancelResult( + cancelled_directly=2, + cancel_requested=0, + cancelled_ids=ids, + cancel_requested_ids=[], + ) + assert isinstance(result.cancelled_ids, tuple) + assert len(result.cancelled_ids) == 2 + class TestEmptyFilterError: def test_is_taskq_error(self) -> None: diff --git a/tests/test_cancel_where_pg.py b/tests/test_cancel_where_pg.py index b3f7a705..36ac446d 100644 --- a/tests/test_cancel_where_pg.py +++ b/tests/test_cancel_where_pg.py @@ -219,21 +219,14 @@ class TestDeadlockRetry: @staticmethod def _mock_pool_and_conn( - fetch_row: dict | None = None, - fetch_side_effects: list[Exception] | None = None, + fetch_rows: list[dict[str, object] | None] | None = None, ) -> tuple[MagicMock, MagicMock]: conn = MagicMock() - if fetch_side_effects is not None: - fetch_mock = AsyncMock(side_effect=fetch_side_effects) - if fetch_row is not None: - fetch_mock.side_effect = [ - *fetch_side_effects, - fetch_row, - ] - conn.fetchrow = fetch_mock + if fetch_rows is not None: + conn.fetchrow = AsyncMock(side_effect=fetch_rows) else: - conn.fetchrow = AsyncMock(return_value=fetch_row) + conn.fetchrow = AsyncMock(return_value=None) conn.executemany = AsyncMock(return_value=None) @@ -250,22 +243,40 @@ def _mock_pool_and_conn( return pool, conn @staticmethod - def _success_row() -> dict: + def _ps_success_row() -> dict[str, object]: return { "cancelled_directly": 1, - "cancel_requested": 0, "cancelled_ids": [uuid4()], "cancelled_prev_statuses": ["pending"], + } + + @staticmethod + def _running_empty_row() -> dict[str, object]: + return { + "cancel_requested": 0, "cancel_requested_ids": [], "cancel_requested_workers": [], } + @staticmethod + def _running_success_row() -> dict[str, object]: + wid = uuid4() + return { + "cancel_requested": 1, + "cancel_requested_ids": [uuid4()], + "cancel_requested_workers": [wid], + } + async def test_deadlock_retry_succeeds_on_second_attempt(self) -> None: """_cancel_where retries on DeadlockDetectedError and succeeds.""" - row = self._success_row() + ps_row = self._ps_success_row() + running_row = self._running_empty_row() pool, _ = self._mock_pool_and_conn( - fetch_row=row, - fetch_side_effects=[asyncpg.DeadlockDetectedError()], + fetch_rows=[ + asyncpg.DeadlockDetectedError(), + ps_row, + running_row, + ] ) sql = MagicMock() @@ -280,7 +291,7 @@ async def test_deadlock_retry_succeeds_on_second_attempt(self) -> None: async def test_deadlock_retry_exhausted_raises(self) -> None: """_cancel_where raises after 3 failed attempts.""" pool, _ = self._mock_pool_and_conn( - fetch_side_effects=[asyncpg.DeadlockDetectedError()] * 3, + fetch_rows=[asyncpg.DeadlockDetectedError()] * 3, ) sql = MagicMock() @@ -290,29 +301,37 @@ async def test_deadlock_retry_exhausted_raises(self) -> None: async def test_no_deadlock_no_retry(self) -> None: """_cancel_where succeeds immediately without retry.""" - row = self._success_row() - pool, conn = self._mock_pool_and_conn(fetch_row=row) + ps_row = self._ps_success_row() + running_row = self._running_empty_row() + pool, conn = self._mock_pool_and_conn(fetch_rows=[ps_row, running_row]) sql = MagicMock() sql.insert_event = "INSERT INTO job_events VALUES ($1, $2, $3, $4)" await _cancel_where(pool, "taskq", sql, JobFilter(tags=("x",)), "test") - assert conn.fetchrow.call_count == 1 + assert conn.fetchrow.call_count == 2 async def test_deadlock_during_executemany_retries_correctly(self) -> None: """Deadlock on executemany (after fetchrow succeeds) retries the whole transaction — no phantom IDs from the aborted attempt.""" - row = self._success_row() - pool, conn = self._mock_pool_and_conn(fetch_row=row) + ps_row = self._ps_success_row() + running_row = self._running_empty_row() + # Attempt 1: fetchrow → ps_row, executemany → deadlock + # Attempt 2: fetchrow → ps_row, executemany → ok, fetchrow → running_row + pool, conn = self._mock_pool_and_conn(fetch_rows=[ps_row, ps_row, running_row]) call_count = [0] - async def _flaky_executemany(query, args, *a, **kw): + async def _flaky_executemany( + query: str, + args: list[tuple[object, ...]], + *a: object, + **kw: object, + ) -> None: call_count[0] += 1 - if call_count[0] <= 2: + if call_count[0] == 1: raise asyncpg.DeadlockDetectedError() - return None conn.executemany = _flaky_executemany @@ -329,14 +348,14 @@ async def test_notify_target_filters_none_worker_id(self) -> None: notify_targets (no worker to NOTIFY).""" jid = uuid4() pool, _ = self._mock_pool_and_conn( - fetch_row={ - "cancelled_directly": 0, - "cancel_requested": 1, - "cancelled_ids": [], - "cancelled_prev_statuses": [], - "cancel_requested_ids": [jid], - "cancel_requested_workers": [None], - } + fetch_rows=[ + None, + { + "cancel_requested": 1, + "cancel_requested_ids": [jid], + "cancel_requested_workers": [None], + }, + ] ) sql = MagicMock() diff --git a/tests/test_filter_sql.py b/tests/test_filter_sql.py index ec0ccc9e..1c8a681b 100644 --- a/tests/test_filter_sql.py +++ b/tests/test_filter_sql.py @@ -79,7 +79,11 @@ def test_parameter_numbering_is_sequential(self) -> None: """$N placeholders must be sequentially numbered and align positionally with the params tuple.""" result = build_filter_conditions(JobFilter(queue="q1", actor="a1", tags=("t1",))) - numbers = [int(re.search(r"\$(\d+)", c).group(1)) for c in result.conditions] + numbers: list[int] = [] + for c in result.conditions: + m = re.search(r"\$(\d+)", c) + assert m is not None, f"No $N placeholder in condition: {c!r}" + numbers.append(int(m.group(1))) assert numbers == list(range(1, len(numbers) + 1)), numbers assert result.params == ("q1", "a1", ["t1"]) From 2d310f197b90f6695e1f4721ea4e23064ce91501 Mon Sep 17 00:00:00 2001 From: Rich Evans Date: Wed, 29 Jul 2026 22:35:08 -0700 Subject: [PATCH 16/16] fix(e2e): correct stage-2 tag count in test_sub_job_explicit_tags_merge_with_parent Stage 3 inherits ALL parent tags (run-tag + stage-2) per the tag inheritance design (unit-tested in test_inherit_and_merge_tags). The e2e assertion expected only 1 job with stage-2 tag, but stage 3 legitimately inherited stage-2 from stage 2, so the correct count is 2. Updated the assertion to expect 2 jobs and verify the actual stage-2 job (excluding stage-3 which inherited it) carries both the parent run tag and the explicit stage-2 tag. --- tests/e2e/test_sub_job_tags.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_sub_job_tags.py b/tests/e2e/test_sub_job_tags.py index 8a59fcbd..8b5c15ba 100644 --- a/tests/e2e/test_sub_job_tags.py +++ b/tests/e2e/test_sub_job_tags.py @@ -103,6 +103,10 @@ async def test_sub_job_explicit_tags_merge_with_parent( ) stage2 = await e2e_client.list(JobFilter(tags=("stage-2",))) - assert len(stage2.jobs) == 1, f"Expected 1 job with stage-2 tag, found {len(stage2.jobs)}" - assert parent_tag in stage2.jobs[0].tags - assert "stage-2" in stage2.jobs[0].tags + assert len(stage2.jobs) == 2, ( + f"Expected 2 jobs with stage-2 tag (stage 2 + stage 3 inherited it), found {len(stage2.jobs)}" + ) + stage2_only = [j for j in stage2.jobs if "stage-3" not in j.tags] + assert len(stage2_only) == 1, "Expected exactly one job with stage-2 but not stage-3" + assert parent_tag in stage2_only[0].tags + assert "stage-2" in stage2_only[0].tags