A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fixes assertion failure in prefix caching: the lora index mapping should respect prefix_len (#2688)
Signed-off-by: Tao He <sighingnow@gmail.com>
Tao He committed
Feb 1, 2024 at 01:00 UTC
d69ff0cbbbb07b571eeea62b4e2ce87b91cea387
1 file changed
+2
-2
vllm/worker/model_runner.py
+2
-2
@@ -142,10 +142,10 @@ class ModelRunner:
142
if lora_id > 0:
143
lora_requests.add(seq_group_metadata.lora_request)
144
145
- lora_index_mapping.append([lora_id] * prompt_len)
145
+ lora_index_mapping.append([lora_id] * (prompt_len - prefix_len))
146
lora_prompt_mapping.extend(
147
[lora_id] *
148
- (prompt_len
148
+ (prompt_len - prefix_len
149
if seq_group_metadata.sampling_params.prompt_logprobs else 1))
150
151
if seq_group_metadata.block_tables is None: