[Bugfix][V1][Multimodal] Recover from P0/P1 processor cache drift (#46747) (#46747)
Signed-off-by: William Zhao <williamzh@meta.com> Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude <noreply@anthropic.com>
WillZZZy committed
Aug 12, 2026 at 00:06 UTC
396204230423b7cc6798300926b8fa30190d26a9
13 files changed
+236
-113
rust/src/chat/tests/chat.rs
+2
-18
@@ -46,18 +46,9 @@ fn request_output(
46
EngineCoreOutput {
47
request_id: request_id.to_string(),
48
new_token_ids,
49
- new_logprobs: None,
50
- new_prompt_logprobs_tensors: None,
51
- pooling_output: None,
49
finish_reason,
50
stop_reason,
54
- events: None,
55
- kv_transfer_params: None,
56
- ec_transfer_params: None,
57
- trace_headers: None,
58
- prefill_stats: None,
59
- routed_experts: None,
60
- num_nans_in_logits: 0,
51
+ ..Default::default()
52
}
53
}
54
@@ -74,16 +65,9 @@ fn request_output_with_logprobs(
65
new_token_ids,
66
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
67
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
77
- pooling_output: None,
68
finish_reason,
69
stop_reason,
80
- events: None,
81
- kv_transfer_params: None,
82
- ec_transfer_params: None,
83
- trace_headers: None,
84
- prefill_stats: None,
85
- routed_experts: None,
86
- num_nans_in_logits: 0,
70
+ ..Default::default()
71
}
72
}
73
rust/src/engine-core-client/src/protocol/output.rs
+13
-10
@@ -112,6 +112,17 @@ pub struct EngineCoreOutput {
112
/// Number of NaNs seen in logits. Values above zero indicate corruption.
113
#[serde(default)]
114
pub num_nans_in_logits: u32,
115
+ /// Multi-modal hashes the engine could not find in its receiver cache,
116
+ /// because the frontend sent `data: None` for an item the engine had
117
+ /// already evicted. Non-empty makes the output retryable: a frontend that
118
+ /// keeps a metadata-only shadow of the engine cache drops these entries and
119
+ /// resends the request with the item data attached.
120
+ ///
121
+ /// TODO: always `None` here, since this client has no multi-modal processor
122
+ /// cache and always sends item data inline. Act on it once the Rust
123
+ /// frontend grows one.
124
+ #[serde(default)]
125
+ pub mm_cache_miss_hashes: Option<Vec<String>>,
126
}
127
128
impl EngineCoreOutput {
@@ -372,18 +383,9 @@ mod tests {
383
outputs: vec![EngineCoreOutput {
384
request_id: "req-1".to_string(),
385
new_token_ids: vec![42],
375
- new_logprobs: None,
376
- new_prompt_logprobs_tensors: None,
377
- pooling_output: None,
386
finish_reason: Some(EngineCoreFinishReason::Length),
387
stop_reason: Some(StopReason::Text("stop".to_string())),
380
- events: None,
381
- kv_transfer_params: None,
382
- ec_transfer_params: None,
383
- trace_headers: None,
384
- prefill_stats: None,
385
- routed_experts: None,
386
- num_nans_in_logits: 0,
388
+ ..Default::default()
389
}],
390
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
391
..Default::default()
@@ -437,6 +439,7 @@ mod tests {
439
prefill_stats: None,
440
routed_experts: None,
441
num_nans_in_logits: 0,
442
+ mm_cache_miss_hashes: None,
443
},
444
],
445
scheduler_stats: None,
rust/src/engine-core-client/src/tests/client.rs
+2
-11
@@ -223,18 +223,8 @@ fn request_output(
223
EngineCoreOutput {
224
request_id: request_id.to_string(),
225
new_token_ids,
226
- new_logprobs: None,
227
- new_prompt_logprobs_tensors: None,
228
- pooling_output: None,
226
finish_reason,
230
- stop_reason: None,
231
- events: None,
232
- kv_transfer_params: None,
233
- ec_transfer_params: None,
234
- trace_headers: None,
235
- prefill_stats: None,
236
- routed_experts: None,
237
- num_nans_in_logits: 0,
227
+ ..Default::default()
228
}
229
}
230
@@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2597
prefill_stats: None,
2598
routed_experts: None,
2599
num_nans_in_logits: 0,
2600
+ mm_cache_miss_hashes: None,
2601
},
2602
],
2603
scheduler_stats: None,
rust/src/engine-core-client/src/tests/python_compat.py
+1
@@ -97,6 +97,7 @@ class EngineCoreOutput(
97
prefill_stats: object | None = None
98
routed_experts: object | None = None
99
num_nans_in_logits: int = 0
100
+ mm_cache_miss_hashes: list[str] | None = None
101
102
103
class EngineCoreOutputs(
rust/src/llm/tests/generate.rs
+3
-26
@@ -47,18 +47,9 @@ fn request_output_with_events(
47
EngineCoreOutput {
48
request_id: request_id.to_string(),
49
new_token_ids,
50
- new_logprobs: None,
51
- new_prompt_logprobs_tensors: None,
52
- pooling_output: None,
50
finish_reason,
54
- stop_reason: None,
51
events,
56
- kv_transfer_params: None,
57
- ec_transfer_params: None,
58
- trace_headers: None,
59
- prefill_stats: None,
60
- routed_experts: None,
61
- num_nans_in_logits: 0,
52
+ ..Default::default()
53
}
54
}
55
@@ -74,16 +65,8 @@ fn request_output_with_logprobs(
65
new_token_ids,
66
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
67
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
77
- pooling_output: None,
68
finish_reason,
79
- stop_reason: None,
80
- events: None,
81
- kv_transfer_params: None,
82
- ec_transfer_params: None,
83
- trace_headers: None,
84
- prefill_stats: None,
85
- routed_experts: None,
86
- num_nans_in_logits: 0,
69
+ ..Default::default()
70
}
71
}
72
@@ -101,16 +84,10 @@ fn request_output_with_logprobs_and_kv(
84
new_token_ids,
85
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
86
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
104
- pooling_output: None,
87
finish_reason,
106
- stop_reason: None,
107
- events: None,
88
kv_transfer_params,
89
ec_transfer_params,
110
- trace_headers: None,
111
- prefill_stats: None,
112
- routed_experts: None,
113
- num_nans_in_logits: 0,
90
+ ..Default::default()
91
}
92
}
93
rust/src/server/src/grpc/tests.rs
+1
-11
@@ -114,18 +114,8 @@ fn request_output(
114
EngineCoreOutput {
115
request_id: request_id.to_string(),
116
new_token_ids,
117
- new_logprobs: None,
118
- new_prompt_logprobs_tensors: None,
119
- pooling_output: None,
117
finish_reason,
121
- stop_reason: None,
122
- events: None,
123
- kv_transfer_params: None,
124
- ec_transfer_params: None,
125
- trace_headers: None,
126
- prefill_stats: None,
127
- routed_experts: None,
128
- num_nans_in_logits: 0,
118
+ ..Default::default()
119
}
120
}
121
rust/src/server/src/routes/http_client_tests.rs
+1
-11
@@ -96,18 +96,8 @@ fn request_output(
96
EngineCoreOutput {
97
request_id: request_id.to_string(),
98
new_token_ids,
99
- new_logprobs: None,
100
- new_prompt_logprobs_tensors: None,
101
- pooling_output: None,
99
finish_reason,
103
- stop_reason: None,
104
- events: None,
105
- kv_transfer_params: None,
106
- ec_transfer_params: None,
107
- trace_headers: None,
108
- prefill_stats: None,
109
- routed_experts: None,
110
- num_nans_in_logits: 0,
100
+ ..Default::default()
101
}
102
}
103
rust/src/server/src/routes/tests.rs
+3
-24
@@ -76,18 +76,9 @@ fn request_output_with_stop_reason(
76
EngineCoreOutput {
77
request_id: request_id.to_string(),
78
new_token_ids,
79
- new_logprobs: None,
80
- new_prompt_logprobs_tensors: None,
81
- pooling_output: None,
79
finish_reason,
80
stop_reason,
84
- events: None,
85
- kv_transfer_params: None,
86
- ec_transfer_params: None,
87
- trace_headers: None,
88
- prefill_stats: None,
89
- routed_experts: None,
90
- num_nans_in_logits: 0,
81
+ ..Default::default()
82
}
83
}
84
@@ -104,16 +95,9 @@ fn request_output_with_logprobs(
95
new_token_ids,
96
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
97
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
107
- pooling_output: None,
98
finish_reason,
99
stop_reason,
110
- events: None,
111
- kv_transfer_params: None,
112
- ec_transfer_params: None,
113
- trace_headers: None,
114
- prefill_stats: None,
115
- routed_experts: None,
116
- num_nans_in_logits: 0,
100
+ ..Default::default()
101
}
102
}
103
@@ -132,16 +116,11 @@ fn request_output_with_logprobs_and_kv(
116
new_token_ids,
117
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
118
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
135
- pooling_output: None,
119
finish_reason,
120
stop_reason,
138
- events: None,
121
kv_transfer_params,
122
ec_transfer_params,
141
- trace_headers: None,
142
- prefill_stats: None,
143
- routed_experts: None,
144
- num_nans_in_logits: 0,
123
+ ..Default::default()
124
}
125
}
126
tests/multimodal/test_cache.py
+101
@@ -7,11 +7,13 @@ import pytest
7
import torch
8
9
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
10
+from vllm.config.multimodal import MultiModalConfig
11
from vllm.multimodal import MULTIMODAL_REGISTRY
12
from vllm.multimodal.cache import (
13
BaseMultiModalProcessorCache,
14
BaseMultiModalReceiverCache,
15
MultiModalCache,
16
+ MultiModalCacheMissError,
17
MultiModalProcessorCacheInItem,
18
MultiModalProcessorCacheItem,
19
MultiModalProcessorCacheItemMetadata,
@@ -232,6 +234,105 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
234
)
235
236
237
+class _StubModelConfig:
238
+ """Minimal model-config stand-in: the cache classes only call
239
+ get_multimodal_config(), so this lets us construct them directly without a
240
+ real model, keeping this unit test free of any Hugging Face / network lookup.
241
+ """
242
+
243
+ def __init__(self, mm_processor_cache_gb: float) -> None:
244
+ self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb)
245
+
246
+ def get_multimodal_config(self) -> MultiModalConfig:
247
+ return self._mm_config
248
+
249
+
250
+def test_mm_cache_miss_raises_and_recovers():
251
+ """A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
252
+
253
+ Reproduces the production failure: the P0 sender cache believes an item is
254
+ cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not
255
+ have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the
256
+ hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow
257
+ entry; and resending the item with data must repopulate both caches.
258
+
259
+ Caches are built directly (not via the registry) so the test needs no real
260
+ model or network.
261
+ """
262
+ model_config = _StubModelConfig(mm_processor_cache_gb=1)
263
+ p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
264
+ p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
265
+
266
+ mm_hash = "image_A"
267
+ item = MultiModalKwargsItem.dummy(nbytes=64)
268
+
269
+ # Drift: the item reaches P1 with no data (P0 would have sent data=None on a
270
+ # stale HIT) but P1 does not have it. Must raise a typed, retryable error
271
+ # carrying the hash -- not assert.
272
+ with pytest.raises(MultiModalCacheMissError) as exc_info:
273
+ p1.get_and_update_item(None, mm_hash)
274
+ assert exc_info.value.mm_hashes == [mm_hash]
275
+
276
+ # Give P0 a (now stale) shadow entry for the hash.
277
+ miss = p0.get_and_update_item((item, []), mm_hash)
278
+ assert miss[0] is item # MISS path returns the data to forward to P1
279
+ assert p0.is_cached_item(mm_hash)
280
+ # On the next request P0 short-circuits to data=None -- the drift bug when
281
+ # P1 lacks the item.
282
+ hit = p0.get_and_update_item((item, []), mm_hash)
283
+ assert hit[0] is None
284
+
285
+ # Recovery: drop the stale P0 entry so the client's resend-with-data takes
286
+ # the MISS path again and repopulates P1.
287
+ p0.invalidate(mm_hash)
288
+ assert not p0.is_cached_item(mm_hash)
289
+
290
+ resent = p0.get_and_update_item((item, []), mm_hash)
291
+ assert resent[0] is item # MISS again -> data is resent
292
+ assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
293
+ # A subsequent uuid-only request now succeeds on P1.
294
+ assert p1.get_and_update_item(None, mm_hash) == item
295
+
296
+
297
+def test_mm_cache_miss_batches_all_drifted_hashes():
298
+ """All hashes drifted within one request must surface in a single error.
299
+
300
+ Otherwise a request with k drifted items needs k client retries (each resend
301
+ un-shadows only the one reported hash). get_and_update_features collects every
302
+ miss and raises once, so a single retry recovers the whole request -- while the
303
+ non-drifted items in the same request are still ingested.
304
+ """
305
+ model_config = _StubModelConfig(mm_processor_cache_gb=1)
306
+ p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
307
+ item = MultiModalKwargsItem.dummy(nbytes=64)
308
+
309
+ def _feature(
310
+ mm_hash: str, data: MultiModalKwargsItem | None
311
+ ) -> MultiModalFeatureSpec:
312
+ return MultiModalFeatureSpec(
313
+ data=data,
314
+ modality="image",
315
+ identifier=mm_hash,
316
+ mm_position=PlaceholderRange(offset=0, length=1),
317
+ mm_hash=mm_hash,
318
+ )
319
+
320
+ # A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1
321
+ # has none of them -- all three must be reported together, in order.
322
+ features = [
323
+ _feature("A", item),
324
+ _feature("B", None),
325
+ _feature("C", item),
326
+ _feature("D", None),
327
+ _feature("E", None),
328
+ ]
329
+ with pytest.raises(MultiModalCacheMissError) as exc_info:
330
+ p1.get_and_update_features(features)
331
+ assert exc_info.value.mm_hashes == ["B", "D", "E"]
332
+ # The non-drifted items were still ingested into P1 before the raise.
333
+ assert "A" in p1._cache and "C" in p1._cache
334
+
335
+
336
def _run_test_cache_eviction_lru(
337
p0_cache: BaseMultiModalProcessorCache,
338
p1_cache: BaseMultiModalReceiverCache,
vllm/multimodal/cache.py
+52
-2
@@ -39,6 +39,29 @@ if TYPE_CHECKING:
39
logger = init_logger(__name__)
40
41
42
+class MultiModalCacheMissError(RuntimeError):
43
+ """Raised by the P1 receiver cache when items are requested with no data and
44
+ are not cached.
45
+
46
+ P0 (frontend) keeps a metadata-only shadow of P1 (engine) and sends
47
+ ``data=None`` on a shadow hit. The two caches are updated in different orders
48
+ across processes, so they can drift -- leaving P0 referencing items P1 has
49
+ evicted. Raising (instead of asserting) lets the engine return a retryable
50
+ response and have P0 drop the stale entries
51
+ (``BaseMultiModalProcessorCache.invalidate``) so the client resends the data.
52
+ Carries every drifted ``mm_hash`` in the request so P0 can drop them all in one
53
+ pass -- one retry then recovers the whole request, not one item per retry.
54
+ """
55
+
56
+ def __init__(self, mm_hashes: list[str]) -> None:
57
+ super().__init__(
58
+ f"Multi-modal items {mm_hashes} are not in the receiver (P1) cache and "
59
+ "no data was provided to recompute them (P0/P1 cache drift); the request "
60
+ "should be retried with the multi-modal data attached."
61
+ )
62
+ self.mm_hashes = mm_hashes
63
+
64
+
65
class MultiModalProcessorCacheItem:
66
"""
67
The data to store inside `MultiModalProcessorOnlyCache`.
@@ -295,6 +318,13 @@ class BaseMultiModalProcessorCache(
318
"""
319
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
320
321
+ def invalidate(self, mm_hash: str) -> None:
322
+ """Drop ``mm_hash`` from this P0 shadow cache to recover from P0/P1 drift.
323
+
324
+ No-op by default; shadow caches that can drift from P1 override this.
325
+ """
326
+ pass
327
+
328
def close(self) -> None:
329
"""Close the underlying cache, if needed."""
330
pass
@@ -433,6 +463,12 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
463
def make_stats(self, *, delta: bool = False) -> CacheInfo:
464
return self._cache.stat(delta=delta)
465
466
+ @override
467
+ def invalidate(self, mm_hash: str) -> None:
468
+ # Drop our stale shadow entry so the next request for this hash re-sends the
469
+ # data and repopulates P1 (see MultiModalCacheMissError).
470
+ self._cache.pop(mm_hash, None)
471
+
472
473
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
474
"""
@@ -602,9 +638,19 @@ class BaseMultiModalReceiverCache(
638
cache_key = feature.mm_hash or feature.identifier
639
self.touch_receiver_cache_item(cache_key, feature.data)
640
641
+ missing_mm_hashes: list[str] = []
642
for feature in mm_features:
643
cache_key = feature.mm_hash or feature.identifier
607
- feature.data = self.get_and_update_item(feature.data, cache_key)
644
+ try:
645
+ feature.data = self.get_and_update_item(feature.data, cache_key)
646
+ except MultiModalCacheMissError as e:
647
+ # Collect every drifted hash in this request before raising, so the
648
+ # engine can have P0 invalidate them all at once -- otherwise a
649
+ # request with k drifted items needs k client retries (each resend
650
+ # only un-shadows the one reported hash).
651
+ missing_mm_hashes.extend(e.mm_hashes)
652
+ if missing_mm_hashes:
653
+ raise MultiModalCacheMissError(missing_mm_hashes)
654
return mm_features
655
656
@abstractmethod
@@ -657,7 +703,11 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
703
if (cached_item := self._cache.get(mm_hash)) is not None:
704
return cached_item
705
660
- assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
706
+ # No data and not cached here: P0 sent data=None trusting its shadow, but
707
+ # the P0/P1 caches have drifted. Raise a retryable error (not assert) so the
708
+ # engine can have P0 drop the stale entry and the client resend the data.
709
+ if mm_item is None:
710
+ raise MultiModalCacheMissError([mm_hash])
711
712
self._cache[mm_hash] = mm_item
713
return mm_item
vllm/v1/engine/__init__.py
+5
@@ -211,6 +211,11 @@ class EngineCoreOutput(
211
# The number of NaNs in logits.
212
# A value greater than 0 indicates that the output is corrupted.
213
num_nans_in_logits: int = 0
214
+ # Multi-modal hashes missing from the P1 receiver cache (P0/P1 drift; see
215
+ # `MultiModalCacheMissError`). Non-empty => retryable: the frontend drops these
216
+ # from its sender cache and the request is resent with the data. Appended last
217
+ # so `array_like` positional serialization stays backward compatible.
218
+ mm_cache_miss_hashes: list[str] | None = None
219
220
@property
221
def finished(self) -> bool:
vllm/v1/engine/async_llm.py
+12
@@ -679,6 +679,8 @@ class AsyncLLM(EngineClient):
679
self._logger_ref = [self.logger_manager]
680
logger_ref = self._logger_ref
681
renderer = self.renderer
682
+ # P0 multi-modal sender ("shadow") cache; None for text-only models.
683
+ mm_processor_cache = renderer.mm_processor_cache
684
chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
685
686
async def output_handler():
@@ -706,6 +708,16 @@ class AsyncLLM(EngineClient):
708
# NOTE: RequestOutputs are pushed to their queues.
709
assert not processed_outputs.request_outputs
710
711
+ # 2b) Recover from P0/P1 cache drift: the engine flags hashes
712
+ # it couldn't find (mm_cache_miss_hashes); drop them from the
713
+ # P0 shadow so the client's retry resends the data and
714
+ # repopulates P1. Hot-path no-op (field is None otherwise).
715
+ if mm_processor_cache is not None:
716
+ for eco in outputs_slice:
717
+ if eco.mm_cache_miss_hashes:
718
+ for mm_hash in eco.mm_cache_miss_hashes:
719
+ mm_processor_cache.invalidate(mm_hash)
720
+
721
# Allow other asyncio tasks to run between chunks
722
if end < num_outputs:
723
await asyncio.sleep(0)
vllm/v1/engine/core.py
+40
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
32
from vllm.logging_utils.dump_input import dump_engine_exception
33
from vllm.lora.request import LoRARequest
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
+from vllm.multimodal.cache import MultiModalCacheMissError
36
from vllm.tasks import POOLING_TASKS, SupportedTask
37
from vllm.tracing import instrument, maybe_init_worker_tracer
38
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
@@ -1710,6 +1711,11 @@ class EngineCoreProc(EngineCore):
1711
req: EngineCoreRequest = add_request_decoder.decode(data_frames)
1712
try:
1713
request = self.preprocess_add_request(req)
1714
+ except MultiModalCacheMissError as e:
1715
+ # P0/P1 shadow drift -- return a retryable signal (P0
1716
+ # drops the stale entry, client resends with data).
1717
+ self._handle_mm_cache_miss(req, e)
1718
+ continue
1719
except Exception:
1720
self._handle_request_preproc_error(req)
1721
continue
@@ -1803,6 +1809,40 @@ class EngineCoreProc(EngineCore):
1809
# Limit the number of buffers to reuse.
1810
reuse_buffers.append(buffer)
1811
1812
+ def _handle_mm_cache_miss(
1813
+ self, request: EngineCoreRequest, err: MultiModalCacheMissError
1814
+ ) -> None:
1815
+ """Return a retryable response for a P0/P1 cache-drift miss.
1816
+
1817
+ Surfaces every drifted hash via ``EngineCoreOutput.mm_cache_miss_hashes`` so
1818
+ the frontend drops them from its sender cache and the client resends with
1819
+ data (see ``MultiModalCacheMissError``). Logged at warning, not exception,
1820
+ because it is expected and self-healing.
1821
+ """
1822
+ logger.warning(
1823
+ "Multi-modal cache miss for request %s (mm_hashes=%s): P0/P1 cache "
1824
+ "drift; returning a retryable response so the items are resent with data.",
1825
+ request.request_id,
1826
+ err.mm_hashes,
1827
+ )
1828
+ self.output_queue.put_nowait(
1829
+ (
1830
+ request.client_index,
1831
+ EngineCoreOutputs(
1832
+ engine_index=self.engine_index,
1833
+ finished_requests={request.request_id},
1834
+ outputs=[
1835
+ EngineCoreOutput(
1836
+ request_id=request.request_id,
1837
+ new_token_ids=[],
1838
+ finish_reason=FinishReason.ERROR,
1839
+ mm_cache_miss_hashes=err.mm_hashes,
1840
+ )
1841
+ ],
1842
+ ),
1843
+ )
1844
+ )
1845
+
1846
@staticmethod
1847
def _send_msg_tracking_payload(
1848
socket: zmq.Socket, buffers: Sequence[bytestr]