A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Fix when max_model_len is not divisible by block_size (#10903)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 4, 2024 at 16:54 UTC
2a56e1264f3f0f32e25de42c32eac67cbc86a098
1 file changed
+10
-3
vllm/v1/worker/gpu_model_runner.py
+10
-3
@@ -260,7 +260,8 @@ class GPUModelRunner:
260
# E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
261
# -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
262
# where M is the max_model_len.
263
- token_indices = positions_np + req_indices * self.max_model_len
263
+ token_indices = (positions_np +
264
+ req_indices * self.input_batch.token_ids_cpu.shape[1])
265
token_indices = torch.from_numpy(token_indices)
266
input_ids = torch.empty((total_num_scheduled_tokens, ),
267
dtype=torch.int32,
@@ -273,9 +274,15 @@ class GPUModelRunner:
274
out=input_ids)
275
276
# Calculate the slot mapping.
277
+ # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
278
+ # -> [0, 0, K, K, K + 1, K + 1, K + 2, 2 * K, 2 * K, 2 * K + 1]
279
+ # where K is the max_num_blocks_per_req and the block size is 2.
280
+ # NOTE(woosuk): We can't simply use `token_indices // block_size` here
281
+ # because M (max_model_len) is not necessarily divisible by block_size.
282
block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
277
- token_indices // self.block_size]
278
- block_offsets = token_indices % self.block_size
283
+ req_indices * self.max_num_blocks_per_req +
284
+ positions_np // self.block_size]
285
+ block_offsets = torch.from_numpy(positions_np % self.block_size)
286
slot_mapping = torch.empty((total_num_scheduled_tokens, ),
287
dtype=torch.int32,
288
device="cpu",