@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a56e1264f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Fix when max_model_len is not divisible by block_size (#10903)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 4, 2024 at 16:54 UTC 2a56e1264f3f0f32e25de42c32eac67cbc86a098
1 file changed +10 -3
vllm/v1/worker/gpu_model_runner.py
+10 -3
@@ -260,7 +260,8 @@ class GPUModelRunner:
260 # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
261 # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
262 # where M is the max_model_len.
263 - token_indices = positions_np + req_indices * self.max_model_len
263 + token_indices = (positions_np +
264 + req_indices * self.input_batch.token_ids_cpu.shape[1])
265 token_indices = torch.from_numpy(token_indices)
266 input_ids = torch.empty((total_num_scheduled_tokens, ),
267 dtype=torch.int32,
@@ -273,9 +274,15 @@ class GPUModelRunner:
274 out=input_ids)
275
276 # Calculate the slot mapping.
277 + # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
278 + # -> [0, 0, K, K, K + 1, K + 1, K + 2, 2 * K, 2 * K, 2 * K + 1]
279 + # where K is the max_num_blocks_per_req and the block size is 2.
280 + # NOTE(woosuk): We can't simply use `token_indices // block_size` here
281 + # because M (max_model_len) is not necessarily divisible by block_size.
282 block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
277 - token_indices // self.block_size]
278 - block_offsets = token_indices % self.block_size
283 + req_indices * self.max_num_blocks_per_req +
284 + positions_np // self.block_size]
285 + block_offsets = torch.from_numpy(positions_np % self.block_size)
286 slot_mapping = torch.empty((total_num_scheduled_tokens, ),
287 dtype=torch.int32,
288 device="cpu",