A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Use more persistent buffers to optimize input preparation overheads (#11111)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 11, 2024 at 23:14 UTC
f092153fbe349a9a1742940e3703bfcff6aa0a6d
2 files changed
+79
-59
vllm/v1/worker/gpu_input_batch.py
+14
-5
@@ -53,14 +53,23 @@ class InputBatch:
53
self.req_ids: List[Optional[str]] = [None] * max_num_reqs
54
self.req_id_to_index: Dict[str, int] = {}
55
56
- self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
57
- dtype=np.int32)
56
+ # TODO(woosuk): This buffer could be too large if max_model_len is big.
57
+ # Find a way to reduce the CPU memory usage.
58
+ self.token_ids_cpu_tensor = torch.zeros(
59
+ (max_num_reqs, max_model_len),
60
+ device="cpu",
61
+ dtype=torch.int32,
62
+ pin_memory=pin_memory,
63
+ )
64
+ self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
65
self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
66
67
# Attention-related.
61
- self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
62
- device=self.device,
63
- dtype=torch.int32)
68
+ self.block_table = torch.zeros(
69
+ (max_num_reqs, max_num_blocks_per_req),
70
+ device=self.device,
71
+ dtype=torch.int32,
72
+ )
73
self.block_table_cpu_tensor = torch.zeros(
74
(max_num_reqs, max_num_blocks_per_req),
75
device="cpu",
vllm/v1/worker/gpu_model_runner.py
+65
-54
@@ -67,6 +67,7 @@ class GPUModelRunner:
67
self.max_model_len = model_config.max_model_len
68
self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
69
self.max_num_tokens = scheduler_config.max_num_batched_tokens
70
+ self.max_num_reqs = scheduler_config.max_num_seqs
71
72
# Model-related.
73
self.num_attn_layers = model_config.get_num_layers_by_block_type(
@@ -88,7 +89,7 @@ class GPUModelRunner:
89
self.requests: Dict[str, CachedRequestState] = {}
90
# Persistent batch.
91
self.input_batch = InputBatch(
91
- max_num_reqs=self.scheduler_config.max_num_seqs,
92
+ max_num_reqs=self.max_num_reqs,
93
max_model_len=self.max_model_len,
94
max_num_blocks_per_req=self.max_num_blocks_per_req,
95
device=self.device,
@@ -117,6 +118,32 @@ class GPUModelRunner:
118
dtype=self.dtype,
119
device=self.device)
120
121
+ self.input_ids_cpu = torch.zeros(self.max_num_tokens,
122
+ dtype=torch.int32,
123
+ device="cpu",
124
+ pin_memory=self.pin_memory)
125
+ self.input_ids_np = self.input_ids_cpu.numpy()
126
+ self.positions_cpu = torch.zeros(self.max_num_tokens,
127
+ dtype=torch.int64,
128
+ device="cpu",
129
+ pin_memory=self.pin_memory)
130
+ self.positions_np = self.positions_cpu.numpy()
131
+ self.slot_mapping_cpu = torch.zeros(self.max_num_tokens,
132
+ dtype=torch.int32,
133
+ device="cpu",
134
+ pin_memory=self.pin_memory)
135
+ self.slot_mapping_np = self.slot_mapping_cpu.numpy()
136
+ self.query_start_loc_cpu = torch.zeros(self.max_num_reqs + 1,
137
+ dtype=torch.int32,
138
+ device="cpu",
139
+ pin_memory=self.pin_memory)
140
+ self.query_start_loc_np = self.query_start_loc_cpu.numpy()
141
+ self.seq_start_loc_cpu = torch.zeros(self.max_num_reqs + 1,
142
+ dtype=torch.int32,
143
+ device="cpu",
144
+ pin_memory=self.pin_memory)
145
+ self.seq_start_loc_np = self.seq_start_loc_cpu.numpy()
146
+
147
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
148
# Remove stopped requests from the cached states.
149
# Keep the states of the pre-empted requests.
@@ -241,22 +268,14 @@ class GPUModelRunner:
268
269
# Get request indices.
270
# E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
244
- indices = np.arange(num_reqs)
245
- req_indices = np.repeat(indices, num_scheduled_tokens)
271
+ req_indices = np.repeat(np.arange(num_reqs), num_scheduled_tokens)
272
273
# Get batched arange.
274
# E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
249
- arange_matrix = np.tile(np.arange(max_num_scheduled_tokens),
250
- (num_reqs, 1))
251
- mask = arange_matrix < num_scheduled_tokens[:, np.newaxis]
252
- arange = arange_matrix[mask]
275
+ arange = np.concatenate([np.arange(n) for n in num_scheduled_tokens])
276
277
# Get positions.
255
- positions = torch.empty((total_num_scheduled_tokens, ),
256
- dtype=torch.int32,
257
- device="cpu",
258
- pin_memory=self.pin_memory)
259
- positions_np = positions.numpy()
278
+ positions_np = self.positions_np[:total_num_scheduled_tokens]
279
np.add(self.input_batch.num_computed_tokens_cpu[req_indices],
280
arange,
281
out=positions_np)
@@ -267,16 +286,13 @@ class GPUModelRunner:
286
# where M is the max_model_len.
287
token_indices = (positions_np +
288
req_indices * self.input_batch.token_ids_cpu.shape[1])
270
- token_indices = torch.from_numpy(token_indices)
271
- input_ids = torch.empty((total_num_scheduled_tokens, ),
272
- dtype=torch.int32,
273
- device="cpu",
274
- pin_memory=self.pin_memory)
275
- torch.index_select(torch.from_numpy(
276
- self.input_batch.token_ids_cpu).flatten(),
289
+ # NOTE(woosuk): We use torch.index_select instead of np.take here
290
+ # because torch.index_select is much faster than np.take for large
291
+ # tensors.
292
+ torch.index_select(self.input_batch.token_ids_cpu_tensor.flatten(),
293
0,
278
- token_indices,
279
- out=input_ids)
294
+ torch.from_numpy(token_indices),
295
+ out=self.input_ids_cpu[:total_num_scheduled_tokens])
296
297
# Calculate the slot mapping.
298
# E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
@@ -284,45 +300,40 @@ class GPUModelRunner:
300
# where K is the max_num_blocks_per_req and the block size is 2.
301
# NOTE(woosuk): We can't simply use `token_indices // block_size` here
302
# because M (max_model_len) is not necessarily divisible by block_size.
287
- block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
288
- req_indices * self.max_num_blocks_per_req +
289
- positions_np // self.block_size]
290
- block_offsets = torch.from_numpy(positions_np % self.block_size)
291
- slot_mapping = torch.empty((total_num_scheduled_tokens, ),
292
- dtype=torch.int32,
293
- device="cpu",
294
- pin_memory=self.pin_memory)
295
- torch.add(block_numbers * self.block_size,
296
- block_offsets,
297
- out=slot_mapping)
303
+ block_table_indices = (req_indices * self.max_num_blocks_per_req +
304
+ positions_np // self.block_size)
305
+ # NOTE(woosuk): We use torch.index_select instead of np.take here
306
+ # because torch.index_select is much faster than np.take for large
307
+ # tensors.
308
+ block_numbers = (self.input_batch.block_table_cpu_tensor.flatten()
309
+ [block_table_indices].numpy())
310
+ block_offsets = positions_np % self.block_size
311
+ np.add(block_numbers * self.block_size,
312
+ block_offsets,
313
+ out=self.slot_mapping_np[:total_num_scheduled_tokens])
314
315
# Prepare the attention metadata.
300
- query_start_loc = torch.empty((num_reqs + 1, ),
301
- dtype=torch.int32,
302
- device="cpu",
303
- pin_memory=self.pin_memory)
304
- query_start_loc_np = query_start_loc.numpy()
305
- query_start_loc_np[0] = 0
306
- np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1:])
316
+ self.query_start_loc_np[0] = 0
317
+ np.cumsum(num_scheduled_tokens,
318
+ out=self.query_start_loc_np[1:num_reqs + 1])
319
320
seq_lens = (self.input_batch.num_computed_tokens_cpu[:num_reqs] +
321
num_scheduled_tokens)
322
max_seq_len = seq_lens.max()
311
- seq_start_loc = torch.empty((num_reqs + 1, ),
312
- dtype=torch.int32,
313
- device="cpu",
314
- pin_memory=self.pin_memory)
315
- seq_start_loc_np = seq_start_loc.numpy()
316
- seq_start_loc_np[0] = 0
317
- np.cumsum(seq_lens, out=seq_start_loc_np[1:])
318
-
319
- self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
320
- non_blocking=True)
321
- self.positions[:total_num_scheduled_tokens].copy_(positions,
322
- non_blocking=True)
323
- query_start_loc = query_start_loc.to(self.device, non_blocking=True)
324
- seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
325
- slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
323
+ self.seq_start_loc_np[0] = 0
324
+ np.cumsum(seq_lens, out=self.seq_start_loc_np[1:num_reqs + 1])
325
+
326
+ # Copy the tensors to the GPU.
327
+ self.input_ids[:total_num_scheduled_tokens].copy_(
328
+ self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
329
+ self.positions[:total_num_scheduled_tokens].copy_(
330
+ self.positions_cpu[:total_num_scheduled_tokens], non_blocking=True)
331
+ query_start_loc = self.query_start_loc_cpu[:num_reqs + 1].to(
332
+ self.device, non_blocking=True)
333
+ seq_start_loc = self.seq_start_loc_cpu[:num_reqs + 1].to(
334
+ self.device, non_blocking=True)
335
+ slot_mapping = self.slot_mapping_cpu[:total_num_scheduled_tokens].to(
336
+ self.device, non_blocking=True).long()
337
attn_metadata = FlashAttentionMetadata(
338
num_actual_tokens=total_num_scheduled_tokens,
339
max_query_len=max_num_scheduled_tokens,