From b7d42f2dfa2bff0d2eae6cfde3993a439118aa78 Mon Sep 17 00:00:00 2001 From: Sihan Wang Date: Tue, 4 Aug 2026 18:04:10 +0000 Subject: [PATCH] Revert "Merge pull request #28 from deepinfra/feat-trtllm-routing-priority" This reverts commit 82514b5749f24640321800d604becaf88c72d646, restoring handler_base.py and its tests to af778016d exactly. #28 wired routing.priority through to generate_async so per-request priority reached the TRT-LLM waiting queue. We are changing direction: priority will be implemented in the dynamo frontend's router queue only, and no priority value will be forwarded to the engines. Reverting keeps the worker on upstream behaviour (top-level `priority` remains the health-check-only path, pinning 1.0) rather than leaving a plumbed-but-unused engine path around. Nothing was built or deployed from the merge, so no image is affected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QqzQrZR2qrvxM6DsxqGciu --- .../trtllm/request_handlers/handler_base.py | 13 +--- .../trtllm/tests/test_trtllm_handler_base.py | 74 +------------------ 2 files changed, 5 insertions(+), 82 deletions(-) diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 1ee615f6e508..a7fca7289ada 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -1114,17 +1114,8 @@ async def _generate_locally_impl( f"Using dynamo router dp_rank={dp_rank} for TRTLLM attention DP scheduling" ) - # Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is - # 0.5. Real requests carry it in the routing hints. Clamp rather than - # let TRT-LLM reject the request outright on an out-of-range value. - priority = request.get("priority") - if priority is None: - routed = routing.get("priority") if routing else None - priority = ( - DEFAULT_REQUEST_PRIORITY - if routed is None - else min(1.0, max(0.0, float(routed))) - ) + # Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is 0.5. + priority = request.get("priority", DEFAULT_REQUEST_PRIORITY) cache_salt = request_cache_salt(request) try: diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index 34c95bbbe4d0..f14e407b2bf4 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -716,9 +716,9 @@ class TestHealthCheckPriority: """Verify generate_locally forwards the correct priority to generate_async. Health check requests (built by TrtllmHealthCheckPayload) must reach - the TRT-LLM engine at priority=1.0. Regular inference requests carry it - in routing hints; with no routing priority at all they fall back to - DEFAULT_REQUEST_PRIORITY (0.5). + the TRT-LLM engine at priority=1.0. Regular inference requests + (built by the Rust frontend as PreprocessedRequest, which has no + priority field) must fall back to DEFAULT_REQUEST_PRIORITY (0.5). """ def _make_handler(self) -> HandlerBase: @@ -803,74 +803,6 @@ async def test_regular_request_gets_default_priority(self): _, kwargs = handler.engine.llm.generate_async.call_args assert kwargs["priority"] == DEFAULT_REQUEST_PRIORITY - @pytest.mark.asyncio - @pytest.mark.parametrize("routed,expected", [(1, 1.0), (0, 0.0)]) - async def test_routing_priority_forwarded(self, routed, expected): - """The header is integer-typed, so only the [0, 1] rails arrive.""" - handler = self._make_handler() - generation_result = self._make_mock_generation_result() - handler.engine.llm.generate_async = MagicMock(return_value=generation_result) - - request = { - "token_ids": [1, 2, 3], - "stop_conditions": {"max_tokens": 10}, - "sampling_options": {"temperature": 0.7}, - "routing": {"priority": routed}, - } - - chunks = [ - c async for c in handler.generate_locally(request, self._make_context()) - ] - assert len(chunks) > 0 - - handler.engine.llm.generate_async.assert_called_once() - _, kwargs = handler.engine.llm.generate_async.call_args - assert kwargs["priority"] == pytest.approx(expected) - - @pytest.mark.asyncio - async def test_routing_priority_clamped_into_range(self): - """A value out of range must not reach TRT-LLM, which rejects it.""" - handler = self._make_handler() - generation_result = self._make_mock_generation_result() - handler.engine.llm.generate_async = MagicMock(return_value=generation_result) - - request = { - "token_ids": [1, 2, 3], - "stop_conditions": {"max_tokens": 10}, - "sampling_options": {"temperature": 0.7}, - "routing": {"priority": 99}, - } - - chunks = [ - c async for c in handler.generate_locally(request, self._make_context()) - ] - assert len(chunks) > 0 - - handler.engine.llm.generate_async.assert_called_once() - _, kwargs = handler.engine.llm.generate_async.call_args - assert kwargs["priority"] == 1.0 - - @pytest.mark.asyncio - async def test_health_check_priority_wins_over_routing(self): - """The top-level health-check priority is not overridden by routing.""" - handler = self._make_handler() - generation_result = self._make_mock_generation_result() - handler.engine.llm.generate_async = MagicMock(return_value=generation_result) - - request = TrtllmHealthCheckPayload( - disaggregation_mode=DisaggregationMode.AGGREGATED, - ).to_dict() - request["routing"] = {"priority": 0} - - chunks = [ - c async for c in handler.generate_locally(request, self._make_context()) - ] - assert len(chunks) > 0 - - handler.engine.llm.generate_async.assert_called_once() - _, kwargs = handler.engine.llm.generate_async.call_args - assert kwargs["priority"] == 1.0 - @pytest.mark.asyncio async def test_routing_cache_salt_forwarded_to_generate_async(self): handler = self._make_handler()