From d6a2587414fa02511fe407aada367802722cfe25 Mon Sep 17 00:00:00 2001 From: Amit Tsur Date: Tue, 27 Oct 2026 00:14:46 +0200 Subject: [PATCH 1/3] fix(vision): retain image ownership through request preparation Keep the FIFO and a reentrant image batch lock through encoding, validation and combined-file copying, defer eviction to the outermost batch, and trim back to the 64-entry bound afterward. Close encoder pipes on teardown; retry a native decoder refusal once through the existing Pillow normalization. Co-Authored-By: Claude Opus 5.5 --- terminal/serve/server.py | 204 ++++++++++-------- terminal/serve/test_server.py | 50 ++++- terminal/serve/test_vision_cache.py | 285 ++++++++++++++++++++++++++ terminal/serve/test_vision_elastic.py | 2 + 4 files changed, 456 insertions(+), 85 deletions(-) create mode 100644 terminal/serve/test_vision_cache.py diff --git a/terminal/serve/server.py b/terminal/serve/server.py index 9c53dcd400c..0622db3a1fa 100644 --- a/terminal/serve/server.py +++ b/terminal/serve/server.py @@ -2906,7 +2906,8 @@ def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = Fa self.reader = None # "own" (Strata's) or "mtmd" (llama.cpp's) from the READY line; None for an older reader self.proc = None self.stopped = True - self.lock = threading.Lock() + self.lock = threading.RLock() + self.cache_batches = 0 self.cache: dict[str, tuple[Path, int]] = {} # opt-in (the vision section's "elastic": true, with the config's "vram_elastic"): the encoder starts with the # first picture and stops after elastic_idle_s without one; while it runs, the engine's expert cache keeps @@ -2916,7 +2917,11 @@ def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = Fa self.elastic_mib = self.elastic_ask_mib(cfg, self.max_tokens) self.elastic_idle_s = float(cfg.get("elastic_idle_s") or 120) if not lazy and not self.elastic: - self._start() + try: + self._start() + except Exception: + self.shutdown() + raise # What a picture may become when the config does not say (setup writes the same numbers, setup.VISION): an old, hand-edited or migrated # config without vision.max_tokens left the reader unclamped, and #3690 saw one picture become 93,700 image tokens (I7). A key that is @@ -3240,12 +3245,12 @@ def _exif_rotated(data: bytes) -> bool: return False @staticmethod - def normalize(data: bytes) -> bytes: + def normalize(data: bytes, *, force: bool = False) -> bytes: """The formats strata-vision's decoder (stb_image) reads pass through; anything else is converted to PNG. A JPEG with an EXIF Orientation other than 1 is turned upright first (needs Pillow).""" jpeg = data[:3] == b"\xff\xd8\xff" - if (jpeg and not Vision._exif_rotated(data)) or data[:8] == b"\x89PNG\r\n\x1a\n" or data[:2] == b"BM" or \ - data[:6] in (b"GIF87a", b"GIF89a"): + if not force and ((jpeg and not Vision._exif_rotated(data)) or data[:8] == b"\x89PNG\r\n\x1a\n" or data[:2] == b"BM" or \ + data[:6] in (b"GIF87a", b"GIF89a")): return data try: import io @@ -3271,6 +3276,26 @@ def normalize(data: bytes) -> bytes: im.save(out, format="PNG") return out.getvalue() + def _trim_cache(self): + """The caller holds self.lock; keep at most 64 images once no batch still needs their paths.""" + while len(self.cache) > 64: + old = next(iter(self.cache)) + self.cache.pop(old)[0].unlink(missing_ok=True) + + + @contextlib.contextmanager + def batch(self): + """Keep encoded files until a request has copied them. Acquire the service FIFO before this lock.""" + with self.lock: + self.cache_batches += 1 + try: + yield + finally: + self.cache_batches -= 1 + if not self.cache_batches: + self._trim_cache() + + def encode_all(self, sources: list, cancel=None) -> list[tuple[Path, int]]: """encode() for every image of one request: the files of this request's earlier images are never evicted to make room for a later one (#1072: with more than 64 unique images the first was deleted before the request @@ -3303,6 +3328,13 @@ def encode(self, source: str | bytes, keep=()) -> tuple[Path, int]: self.proc.stdin.flush() try: line = self._readline(VISION_ENCODE_S, "the image could not be read").strip() + if line.startswith("ERR cannot read the image"): + converted = self.normalize(data, force=True) + if converted != data: + img.write_bytes(converted) + self.proc.stdin.write(f"ENC {img.name} {out.name}\n") + self.proc.stdin.flush() + line = self._readline(VISION_ENCODE_S, "the image could not be read").strip() except RuntimeError as e: # silent for VISION_ENCODE_S: the encoder was ended (#1317) self.stopped = True # alive() is False, so the next request starts it again raise ValueError(str(e)) from None @@ -3313,12 +3345,13 @@ def encode(self, source: str | bytes, keep=()) -> tuple[Path, int]: "the vision encoder stopped")) self.cache[key] = (out, int(line.split()[1])) self.warn_if_over_limit(self.cache[key][1]) - held = {str(p) for p in keep} | {str(out)} - for old in list(self.cache): # oldest first, but never one still in use - if len(self.cache) <= 64: - break - if str(self.cache[old][0]) not in held: - self.cache.pop(old)[0].unlink(missing_ok=True) + if not self.cache_batches: + held = {str(p) for p in keep} | {str(out)} + for old in list(self.cache): # oldest first, but never one still in use + if len(self.cache) <= 64: + break + if str(self.cache[old][0]) not in held: + self.cache.pop(old)[0].unlink(missing_ok=True) return self.cache[key] def warn_if_over_limit(self, tokens: int) -> None: @@ -3348,6 +3381,9 @@ def close(self): proc.wait(timeout=10) except (OSError, subprocess.TimeoutExpired): pass + for pipe in (proc.stdin, proc.stdout): + if pipe is not None: + pipe.close() self.stopped = True def shutdown(self): @@ -5129,7 +5165,9 @@ def fetch(src): # run on the GPU at the same time - an encode during a running request left that request stuck at # "reading the prompt" with CPU and GPU busy, for good (reproduced). So encoding takes its turn in the # same FIFO as the requests. - with self._turn(cancel): + with (self._turn(cancel) if images else contextlib.nullcontext()), \ + (self.vision.batch() if images else contextlib.nullcontext()): + if images: if not (hasattr(self.vision, "all_cached") and self.vision.all_cached(images)): self._vision_up() try: @@ -5137,78 +5175,78 @@ def fetch(src): else [self.vision.encode(src) for src in images]) finally: self.last_image_at = time.time() - # one <|image_pad|> per image -> one per image token. Only the markers the template writes for an image - # (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs, - # #150) is kept as plain text, or it took an image's place and the counts no longer matched. - literal = self.tok.encode(IMAGE_PAD, parse_special=False) - out, k = [], 0 - for j, t in enumerate(ids): - if t == pad and j > 0 and ids[j - 1] == start: - # more pairs than images: text parts that cut both markers apart (the literal marks keep whole ones text) - if k == len(encoded): - raise ValueError("the prompt and its images do not match") - out += [pad] * encoded[k][1] - k += 1 - elif t == pad: - out += literal - else: - out.append(t) - if k != len(encoded): - raise ValueError("the prompt and its images do not match") - ids = out - ctx = self.engine.max_context - if ctx <= 0: - if getattr(self.engine, "starting", False): # #344: (re)starting, not a prompt that is too long - raise EngineStarting("the engine is starting (a minute or two); try again shortly") - # a failed restart leaves max_context 0: check against the last known context, so the request reaches - # run() - which starts the engine again - instead of failing with "exceeds the context (0)" forever - ctx = getattr(self.engine, "known_ctx", 0) + # one <|image_pad|> per image -> one per image token. Only the markers the template writes for an image + # (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs, + # #150) is kept as plain text, or it took an image's place and the counts no longer matched. + literal = self.tok.encode(IMAGE_PAD, parse_special=False) + out, k = [], 0 + for j, t in enumerate(ids): + if t == pad and j > 0 and ids[j - 1] == start: + # more pairs than images: text parts that cut both markers apart (the literal marks keep whole ones text) + if k == len(encoded): + raise ValueError("the prompt and its images do not match") + out += [pad] * encoded[k][1] + k += 1 + elif t == pad: + out += literal + else: + out.append(t) + if k != len(encoded): + raise ValueError("the prompt and its images do not match") + ids = out + ctx = self.engine.max_context if ctx <= 0: - raise EngineStarting("the engine is starting (a minute or two); try again shortly") - room = ctx - CTX_SLACK - len(ids) - if max_new is None or max_new <= 0 or (self.fit_max_tokens and room < 1): - if room < 1: - # A prompt refused for the context is not kept (#567): the client cannot resend it unchanged. The - # other refusals (engine starting, images) keep it: the same prompt comes back and its encode is reused. - self.forget_prompt(encoded_ids) - raise ContextOverflow("request exceeds the context window: " - f"prompt ({len(ids)} tokens) leaves no room to answer in the context " - f"({ctx}); requests are never truncated", prompt=len(ids), context=ctx) - max_new = room - elif max_new > room: - floor = fit_floor(ctx) - if self.fit_max_tokens and room < floor: - # #5924: a clamp to a few hundred tokens left a thinking model no room to answer (or to write a tool - # call); agents resent the turn and wrote the same truncated sentence again and again. The context is - # all but full: a 400 that says so makes the client compact (Claude Code reads "prompt is too long"). - self.forget_prompt(encoded_ids) - raise ContextOverflow("request exceeds the context window: " - f"the prompt ({len(ids)} tokens) leaves {max(0, room)} of the {ctx}-token " - f"context, fewer than the {floor} an answer needs; Strata never truncates " - f"requests. Shorten the conversation, or send max_tokens {max(0, room)} or less", - prompt=len(ids), context=ctx, max_new=max_new, min_room=floor) - if not self.fit_max_tokens: - self.forget_prompt(encoded_ids) - raise ContextOverflow("request exceeds the context window: " - f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context " - f"({ctx}); requests are never truncated. Send a smaller " - f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the " - "model's strata-.json to shorten it to the room left (#545)", - prompt=len(ids), context=ctx, max_new=max_new) - print(f"[strata] max_tokens {max_new} shortened to {max(1, room)}: the prompt ({len(ids)} tokens) leaves that much " - f"of the context ({ctx}); \"fit_max_tokens\": false answers 400 instead", flush=True) - max_new = max(1, room) # a shorter completion beats a 400 (the default) - if images: - # The request's images in one file for GENI (~10 MB a picture), written once nothing above refuses the - # request: one refused after it (the engine starting, no room) left it in the vision directory for good, - # one more for every retry of a 503. run() deletes it; drop_embeddings() if run() never starts. - combined = combined_embeddings_path(self.vision.dir, sum(Path(p).stat().st_size for p, _ in encoded)) - self.embeddings.path = combined # first, so a half-written one is found as well - # #2228: each picture's key is its normalized bytes' hash (the encoder's file name), so the engine's - # conversation cache matches a picture by what it is, not by its embedding floats: a picture encoded again - # (an encoder restart, the cache's eviction) with other bits still matches its checkpoints - self.embeddings.keys = [Path(p).stem[:16] for p, _ in encoded] - write_temporary(combined, [p for p, _ in encoded]) + if getattr(self.engine, "starting", False): # #344: (re)starting, not a prompt that is too long + raise EngineStarting("the engine is starting (a minute or two); try again shortly") + # a failed restart leaves max_context 0: check against the last known context, so the request reaches + # run() - which starts the engine again - instead of failing with "exceeds the context (0)" forever + ctx = getattr(self.engine, "known_ctx", 0) + if ctx <= 0: + raise EngineStarting("the engine is starting (a minute or two); try again shortly") + room = ctx - CTX_SLACK - len(ids) + if max_new is None or max_new <= 0 or (self.fit_max_tokens and room < 1): + if room < 1: + # A prompt refused for the context is not kept (#567): the client cannot resend it unchanged. The + # other refusals (engine starting, images) keep it: the same prompt comes back and its encode is reused. + self.forget_prompt(encoded_ids) + raise ContextOverflow("request exceeds the context window: " + f"prompt ({len(ids)} tokens) leaves no room to answer in the context " + f"({ctx}); requests are never truncated", prompt=len(ids), context=ctx) + max_new = room + elif max_new > room: + floor = fit_floor(ctx) + if self.fit_max_tokens and room < floor: + # #5924: a clamp to a few hundred tokens left a thinking model no room to answer (or to write a tool + # call); agents resent the turn and wrote the same truncated sentence again and again. The context is + # all but full: a 400 that says so makes the client compact (Claude Code reads "prompt is too long"). + self.forget_prompt(encoded_ids) + raise ContextOverflow("request exceeds the context window: " + f"the prompt ({len(ids)} tokens) leaves {max(0, room)} of the {ctx}-token " + f"context, fewer than the {floor} an answer needs; Strata never truncates " + f"requests. Shorten the conversation, or send max_tokens {max(0, room)} or less", + prompt=len(ids), context=ctx, max_new=max_new, min_room=floor) + if not self.fit_max_tokens: + self.forget_prompt(encoded_ids) + raise ContextOverflow("request exceeds the context window: " + f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context " + f"({ctx}); requests are never truncated. Send a smaller " + f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the " + "model's strata-.json to shorten it to the room left (#545)", + prompt=len(ids), context=ctx, max_new=max_new) + print(f"[strata] max_tokens {max_new} shortened to {max(1, room)}: the prompt ({len(ids)} tokens) leaves that much " + f"of the context ({ctx}); \"fit_max_tokens\": false answers 400 instead", flush=True) + max_new = max(1, room) # a shorter completion beats a 400 (the default) + if images: + # The request's images in one file for GENI (~10 MB a picture), written once nothing above refuses the + # request: one refused after it (the engine starting, no room) left it in the vision directory for good, + # one more for every retry of a 503. run() deletes it; drop_embeddings() if run() never starts. + combined = combined_embeddings_path(self.vision.dir, sum(Path(p).stat().st_size for p, _ in encoded)) + self.embeddings.path = combined # first, so a half-written one is found as well + # #2228: each picture's key is its normalized bytes' hash (the encoder's file name), so the engine's + # conversation cache matches a picture by what it is, not by its embedding floats: a picture encoded again + # (an encoder restart, the cache's eviction) with other bits still matches its checkpoints + self.embeddings.keys = [Path(p).stem[:16] for p, _ in encoded] + write_temporary(combined, [p for p, _ in encoded]) if req is not None and req.get("strata_prefix") is not None: req["strata_prefix"] = self.resolve_prefix(req["strata_prefix"], messages, tools, kwargs, ids, bool(images)) return ids, kwargs.get("enable_thinking", True) is not False, max_new diff --git a/terminal/serve/test_server.py b/terminal/serve/test_server.py index 02e0a03e721..481ebeaa735 100644 --- a/terminal/serve/test_server.py +++ b/terminal/serve/test_server.py @@ -24,7 +24,7 @@ from serve.frontend import ChatTemplate, literal_tags, mark_think_literals, unmark_think_literals # noqa: E402 from serve.server import (ANSWER_RESERVE_MIN, CTX_SLACK, ByteTokenizer, EngineDied, EngineNonFinite, GpuBusy, MockEngine, # noqa: E402 PP_DONE_TAIL, Service, - StrataEngine, api_key_of, engine_args, key_matches, layer_split_value, prompt_progress, + StrataEngine, Vision, api_key_of, engine_args, key_matches, layer_split_value, prompt_progress, prompt_tokens_seen, request_stats, request_timings, serve, start_failure_hint) from types import SimpleNamespace # noqa: E402 @@ -35,6 +35,44 @@ # one token repeated, which the server ends at 256) +class VisionDecode(unittest.TestCase): + def test_native_decoder_failure_uses_existing_normalization_once(self): + for reply, expected_calls in (("ERR cannot read the image crop\nOK 450\n", 2), + ("ERR the vision encoder failed\n", 1)): + with self.subTest(reply=reply), tempfile.TemporaryDirectory() as temp: + vision = Vision.__new__(Vision) + vision.dir, vision.lock, vision.cache = Path(temp), threading.RLock(), {} + vision.cache_batches = 0 + vision.proc = SimpleNamespace(stdin=io.StringIO(), stdout=io.StringIO(reply)) + with mock.patch.object(Vision, 'load', return_value=b'original'), \ + mock.patch.object(Vision, 'normalize', side_effect=lambda data, force=False: + b'converted' if force else data) as normalize: + if expected_calls == 2: + encoded, tokens = vision.encode('crop') + encoded.write_bytes(b'fake encoder output') + self.assertEqual(tokens, 450) + self.assertEqual(vision.encode('crop')[1], 450) + self.assertEqual(vision.proc.stdin.getvalue().count('ENC '), 2) + self.assertEqual(normalize.call_args_list[1].kwargs, {'force': True}) + else: + with self.assertRaisesRegex(ValueError, 'vision encoder failed'): + vision.encode('crop') + self.assertEqual(normalize.call_count, 1) + self.assertFalse(list(Path(temp).glob('*.img'))) + + def test_failed_conversion_removes_temporary_image(self): + with tempfile.TemporaryDirectory() as temp: + vision = Vision.__new__(Vision) + vision.dir, vision.lock, vision.cache = Path(temp), threading.RLock(), {} + vision.cache_batches = 0 + vision.proc = SimpleNamespace(stdin=io.StringIO(), stdout=io.StringIO('ERR cannot read the image crop\n')) + with mock.patch.object(Vision, 'load', return_value=b'original'), \ + mock.patch.object(Vision, 'normalize', side_effect=[b'original', ValueError('bad image')]): + with self.assertRaisesRegex(ValueError, 'bad image'): + vision.encode('crop') + self.assertFalse(list(Path(temp).glob('*.img'))) + + class RecordingEngine(MockEngine): def generate(self, ids, max_new, sampling, cancel, embeddings=None): self.last_max_new = max_new @@ -371,6 +409,8 @@ class ImageMarkers(unittest.TestCase): """#150: the text "<|image_pad|>" inside a message is text, not an image's place.""" class FakeVision: + batch = staticmethod(contextlib.nullcontext) + def __init__(self, d): self.dir = Path(d) self.rows = self.dir / "img.sve" @@ -584,6 +624,7 @@ def flush(self): pipe = Pipe() v = Vision.__new__(Vision) v.dir, v.lock, v.cache = Path(tempfile.mkdtemp(prefix="strata-vision-test-")), threading.Lock(), {} + v.cache_batches = 0 v.proc = mock.Mock(stdin=pipe, stdout=mock.Mock(readline=lambda: "OK 7 1 1 1\n")) try: with mock.patch.object(Vision, "load", return_value=b""), \ @@ -6794,7 +6835,8 @@ class VisionCacheEviction(unittest.TestCase): def make(self, d): import serve.server as server v = server.Vision.__new__(server.Vision) - v.dir, v.cache, v.lock = d, {}, threading.Lock() + v.dir, v.cache, v.lock = d, {}, threading.RLock() + v.cache_batches = 0 v.load = lambda src: src class Pipe: @@ -6949,6 +6991,10 @@ def test_shutdown_removes_the_work_dir_but_close_keeps_it(self): class Proc: stdin = io.StringIO() + stdout = None + + def poll(self): + return 0 def wait(self, timeout=None): return 0 diff --git a/terminal/serve/test_vision_cache.py b/terminal/serve/test_vision_cache.py new file mode 100644 index 00000000000..5842079fb3f --- /dev/null +++ b/terminal/serve/test_vision_cache.py @@ -0,0 +1,285 @@ +"""#1072: image cache entries live until prepare() copies them, then return to the 64-entry bound. + +The real Vision cache and Service run against an in-memory ENC pipe; no GPU, model or network is needed. + python -m unittest serve.test_vision_cache -v +""" +import contextlib +import hashlib +import tempfile +import threading +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from serve import server +from serve.frontend import ChatTemplate + +ROOT = Path(__file__).resolve().parents[1] + + +def picture(i): + # PNG signature is enough for normalize()'s pass-through; no image decoder is under test. + return b"\x89PNG\r\n\x1a\n" + i.to_bytes(4, "big") + + +def record(data): + return hashlib.sha256(data).digest() + + +class EncoderPipe: + def __init__(self, root): + self.root, self.calls, self.fail_at = root, 0, None + self.answer = "" + + def write(self, line): + _, image, out = line.split() + self.calls += 1 + if self.calls == self.fail_at: + self.answer = "ERR injected encoding error\n" + else: + (self.root / out).write_bytes(record((self.root / image).read_bytes())) + self.answer = "OK 1 1 1 0\n" + + def flush(self): + pass + + def readline(self): + return self.answer + + +class TrackedLock: + """Report the second request's lock attempt without changing Lock's acquisition/release behavior.""" + def __init__(self): + self.lock = threading.Lock() + self.attempted = threading.Event() + + def acquire(self, *args, **kwargs): + if threading.current_thread().name == "image-B": + self.attempted.set() + return self.lock.acquire(*args, **kwargs) + + def release(self): + self.lock.release() + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, *args): + self.release() + + +class VisionCacheLifetime(unittest.TestCase): + @contextlib.contextmanager + def service(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + v = server.Vision.__new__(server.Vision) + v.dir, v.lock, v.cache, v.cache_batches = root, threading.RLock(), {}, 0 + v.stopped = False + pipe = EncoderPipe(root) + v.proc = SimpleNamespace(stdin=pipe, stdout=pipe, poll=lambda: None) + tok = server.ByteTokenizer() + svc = server.Service(server.MockEngine(tok, "ok", max_context=131072), tok, + ChatTemplate(ROOT / "serve/chat_template.jinja"), vision=v) + number = 0 + + def combined(*args): + nonlocal number + number += 1 + return root / f"req-{number}.sve" + + with mock.patch.object(server, "combined_embeddings_path", side_effect=combined), \ + mock.patch.object(server.Vision, "load", side_effect=lambda src: picture(int(src))): + try: + yield svc, v, pipe + finally: + svc.drop_embeddings() + + @staticmethod + def prepare(svc, values, **kwargs): + messages = [{"role": "user", "content": [{"type": "image", "source": str(i)} for i in values]}] + return svc.prepare(messages, None, {"enable_thinking": False}, **kwargs) + + def assert_idle(self, svc, v): + self.assertEqual(v.cache_batches, 0) + self.assertLessEqual(len(v.cache), 64) + self.assertEqual(len(list(v.dir.glob("*.sve"))), len(v.cache)) + self.assertEqual(list(v.dir.glob("*.img")), []) + # A different thread must be able to acquire both locks after every exit, including BaseException. + free = threading.Event() + + def take(): + with svc.fifo, v.lock: + free.set() + + thread = threading.Thread(target=take, daemon=True) + thread.start() + self.assertTrue(free.wait(2), "FIFO/vision lock leaked") + thread.join(2) + + def test_request_larger_than_cache_keeps_all_records_in_order(self): + for size in (64, 65, 70, 130): + with self.subTest(size=size), self.service() as (svc, v, pipe): + self.prepare(svc, range(size)) + self.assertEqual(svc.embeddings.path.read_bytes(), b"".join(record(picture(i)) for i in range(size))) + self.assertEqual(len(v.cache), 64) + self.assertEqual(pipe.calls, size) + svc.drop_embeddings() + self.assert_idle(svc, v) + + def test_repeated_images_encode_once_and_preserve_order(self): + with self.service() as (svc, v, pipe): + values = [0] * 70 + self.prepare(svc, values) + self.assertEqual(pipe.calls, 1) + self.assertEqual(svc.embeddings.path.read_bytes(), record(picture(0)) * 70) + svc.drop_embeddings() + values = list(range(70)) + [0, 4, 0, 69] + self.prepare(svc, values) + self.assertEqual(pipe.calls, 70) + self.assertEqual(svc.embeddings.path.read_bytes(), b"".join(record(picture(i)) for i in values)) + svc.drop_embeddings() + self.assert_idle(svc, v) + + def race(self, before_copy=False, tool=False): + with self.service() as (svc, v, pipe): + self.prepare(svc, range(64)) + svc.drop_embeddings() + svc.fifo = TrackedLock() + paused, resume, b_done = threading.Event(), threading.Event(), threading.Event() + results, errors = {}, [] + original = server.write_temporary if before_copy else svc.tok.encode + + def gated(*args, **kwargs): + at_gap = before_copy or args[0] == server.IMAGE_PAD and kwargs.get("parse_special") is False + if threading.current_thread().name == "image-A" and at_gap: + paused.set() + if not resume.wait(3): + raise AssertionError("test release did not arrive") + return original(*args, **kwargs) + + def request(name, value): + try: + if tool and name == "B": + # Tool-image preflight is another encoder caller and must follow the same lock order. + messages = [{"role": "tool", "content": [{"type": "image", "source": str(value)}]}] + svc.prepare(messages, None, {"enable_thinking": False}) + else: + self.prepare(svc, [value]) + results[name] = svc.embeddings.path.read_bytes() + except BaseException as exc: + errors.append(exc) + finally: + svc.drop_embeddings() + if name == "B": + b_done.set() + + target, attribute = (server, "write_temporary") if before_copy else (svc.tok, "encode") + with mock.patch.object(target, attribute, side_effect=gated): + a = threading.Thread(target=request, args=("A", 0), name="image-A", daemon=True) + b = threading.Thread(target=request, args=("B", 64), name="image-B", daemon=True) + a.start() + try: + self.assertTrue(paused.wait(2)) + b.start() + self.assertTrue(svc.fifo.attempted.wait(2)) + self.assertFalse(b_done.wait(0.1), "second encode ran while the first still needed its paths") + finally: + resume.set() + a.join(3) + if b.ident is not None: + b.join(3) + self.assertFalse(a.is_alive() or b.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, {"A": record(picture(0)), "B": record(picture(64))}) + self.assert_idle(svc, v) + + def test_one_image_requests_cannot_evict_before_stat(self): + self.race() + + def test_one_image_requests_cannot_evict_between_stat_and_copy(self): + self.race(before_copy=True) + + def test_tool_preflight_waits_for_active_copy(self): + self.race(before_copy=True, tool=True) + + def test_all_prepare_failures_trim_and_release(self): + class Abort(BaseException): + pass + + cases = ("encode", "markers", "starting", "no_room", "max_new", "write", "abort") + for case in cases: + with self.subTest(case=case), self.service() as (svc, v, pipe), contextlib.ExitStack() as stack: + error = ValueError + kwargs = {} + if case == "encode": + pipe.fail_at = 66 + elif case == "markers": + stack.enter_context(mock.patch.object(svc, "encode_prompt", return_value=[42])) + elif case == "starting": + svc.engine.max_context, svc.engine.starting = 0, True + error = server.EngineStarting + elif case == "no_room": + svc.engine.max_context = 64 + elif case == "max_new": + svc.fit_max_tokens = False + kwargs["max_new"] = 131072 + else: + error = OSError if case == "write" else Abort + def fail(path, parts): + path.write_bytes(b"partial") + raise error("injected copy failure") + stack.enter_context(mock.patch.object(server, "write_temporary", side_effect=fail)) + try: + with self.assertRaises(error): + self.prepare(svc, range(70), **kwargs) + finally: + svc.drop_embeddings() + self.assertEqual(len(v.cache), 64) + self.assert_idle(svc, v) + + def test_nested_batch_trims_only_on_outer_exit(self): + with self.service() as (svc, v, pipe): + with svc.fifo, v.batch(): + v.encode("0") + with v.batch(): + for i in range(1, 70): + v.encode(str(i)) + self.assertEqual(len(v.cache), 70) + self.assertTrue(all(path.exists() for path, _ in v.cache.values())) + self.assertEqual(len(v.cache), 64) + self.assert_idle(svc, v) + + def test_combined_file_survives_raw_eviction_and_run_cleanup(self): + with self.service() as (svc, v, pipe): + ids, thinking, max_new = self.prepare(svc, [0]) + combined = svc.embeddings.path + raw = next(iter(v.cache.values()))[0] + for i in range(1, 65): + v.encode(str(i)) + self.assertFalse(raw.exists()) + self.assertEqual(combined.read_bytes(), record(picture(0))) + gen = svc.run(ids, thinking, None, max_new, {}, threading.Event()) + next(gen) + self.assertTrue(combined.exists()) + gen.close() + self.assertFalse(combined.exists()) + self.assert_idle(svc, v) + + def test_no_run_drop_and_direct_encode_remain_bounded(self): + with self.service() as (svc, v, pipe): + for start in range(0, 210, 70): + self.prepare(svc, range(start, start + 70)) + svc.drop_embeddings() + svc.drop_embeddings() + self.assert_idle(svc, v) + for i in range(210, 300): + v.encode(str(i)) + self.assertLessEqual(len(v.cache), 64) + self.assert_idle(svc, v) + + +if __name__ == "__main__": + unittest.main() diff --git a/terminal/serve/test_vision_elastic.py b/terminal/serve/test_vision_elastic.py index 16b8d3152d7..7e837552717 100644 --- a/terminal/serve/test_vision_elastic.py +++ b/terminal/serve/test_vision_elastic.py @@ -27,6 +27,8 @@ def __init__(self, d, elastic=True): self.rows = self.dir / "img.sve" self.rows.write_bytes(b"rows") + batch = staticmethod(contextlib.nullcontext) + def alive(self): return self.up From b82bb5c8d4ebd341ae551eba713d63dea2d37ede Mon Sep 17 00:00:00 2001 From: Amit Tsur Date: Wed, 28 Oct 2026 12:36:09 +0200 Subject: [PATCH 2/3] fix(vision): keep request preparation working with an encoder that has no batch() A vision encoder without batch() (the HistoryReferences test stub) now prepares the request without the batch lock, as encode_all() already falls back to encode(). --- terminal/serve/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal/serve/server.py b/terminal/serve/server.py index 0622db3a1fa..da0a3050116 100644 --- a/terminal/serve/server.py +++ b/terminal/serve/server.py @@ -5166,7 +5166,7 @@ def fetch(src): # "reading the prompt" with CPU and GPU busy, for good (reproduced). So encoding takes its turn in the # same FIFO as the requests. with (self._turn(cancel) if images else contextlib.nullcontext()), \ - (self.vision.batch() if images else contextlib.nullcontext()): + (self.vision.batch() if images and hasattr(self.vision, "batch") else contextlib.nullcontext()): if images: if not (hasattr(self.vision, "all_cached") and self.vision.all_cached(images)): self._vision_up() From c82afd007963ea11ac118b0a8b4a33c74ef8bed5 Mon Sep 17 00:00:00 2001 From: Tsury Date: Fri, 20 Nov 2026 11:36:38 +0200 Subject: [PATCH 3/3] fix(vision): give Vision a class default for its batch counter Tests that build a reader with Vision.__new__ (test_vision_workdir) have no cache_batches, so encode() raised AttributeError. A class attribute keeps the counter at 0 until batch() sets it on the instance. Co-Authored-By: Claude Opus 5.5 --- terminal/serve/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terminal/serve/server.py b/terminal/serve/server.py index da0a3050116..83d4cff893a 100644 --- a/terminal/serve/server.py +++ b/terminal/serve/server.py @@ -2870,6 +2870,8 @@ class Vision: lines and writes each image's embeddings; results are cached by the image's hash, so a conversation that sends the same picture again (every turn, with most clients) encodes it once.""" + cache_batches = 0 # requests inside batch(): their encoded files stay until they copied them + @staticmethod def work_dir() -> Path: """#480: the directory for the encoder's files. The engine reads the embeddings file named in a `GENI` line @@ -2907,7 +2909,6 @@ def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = Fa self.proc = None self.stopped = True self.lock = threading.RLock() - self.cache_batches = 0 self.cache: dict[str, tuple[Path, int]] = {} # opt-in (the vision section's "elastic": true, with the config's "vram_elastic"): the encoder starts with the # first picture and stops after elastic_idle_s without one; while it runs, the engine's expert cache keeps