A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Bugfix] Copy encoder input ids to fix set iteration issue during VLM abort (#13173)
Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com>
Murali Andoorveedu committed
Feb 12, 2025 at 12:58 UTC
4c0d93f4b2de241336f4732cb5799cee8fedcb52
1 file changed
+1
-1
vllm/v1/core/encoder_cache_manager.py
+1
-1
@@ -54,7 +54,7 @@ class EncoderCacheManager:
54
55
def free(self, request: Request) -> None:
56
"""Free all cached input ids for the request."""
57
- input_ids = self.get_cached_input_ids(request)
57
+ input_ids = self.get_cached_input_ids(request).copy()
58
for input_id in input_ids:
59
self.free_encoder_input(request, input_id)
60