@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d69ff0cbbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fixes assertion failure in prefix caching: the lora index mapping should respect prefix_len (#2688)

Signed-off-by: Tao He <sighingnow@gmail.com>

Tao He committed Feb 1, 2024 at 01:00 UTC d69ff0cbbbb07b571eeea62b4e2ce87b91cea387
1 file changed +2 -2
vllm/worker/model_runner.py
+2 -2
@@ -142,10 +142,10 @@ class ModelRunner:
142 if lora_id > 0:
143 lora_requests.add(seq_group_metadata.lora_request)
144
145 - lora_index_mapping.append([lora_id] * prompt_len)
145 + lora_index_mapping.append([lora_id] * (prompt_len - prefix_len))
146 lora_prompt_mapping.extend(
147 [lora_id] *
148 - (prompt_len
148 + (prompt_len - prefix_len
149 if seq_group_metadata.sampling_params.prompt_logprobs else 1))
150
151 if seq_group_metadata.block_tables is None: