@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f092153fbe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Use more persistent buffers to optimize input preparation overheads (#11111)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 11, 2024 at 23:14 UTC f092153fbe349a9a1742940e3703bfcff6aa0a6d
2 files changed +79 -59
vllm/v1/worker/gpu_input_batch.py
+14 -5
@@ -53,14 +53,23 @@ class InputBatch:
53 self.req_ids: List[Optional[str]] = [None] * max_num_reqs
54 self.req_id_to_index: Dict[str, int] = {}
55
56 - self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
57 - dtype=np.int32)
56 + # TODO(woosuk): This buffer could be too large if max_model_len is big.
57 + # Find a way to reduce the CPU memory usage.
58 + self.token_ids_cpu_tensor = torch.zeros(
59 + (max_num_reqs, max_model_len),
60 + device="cpu",
61 + dtype=torch.int32,
62 + pin_memory=pin_memory,
63 + )
64 + self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
65 self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
66
67 # Attention-related.
61 - self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
62 - device=self.device,
63 - dtype=torch.int32)
68 + self.block_table = torch.zeros(
69 + (max_num_reqs, max_num_blocks_per_req),
70 + device=self.device,
71 + dtype=torch.int32,
72 + )
73 self.block_table_cpu_tensor = torch.zeros(
74 (max_num_reqs, max_num_blocks_per_req),
75 device="cpu",
vllm/v1/worker/gpu_model_runner.py
+65 -54
@@ -67,6 +67,7 @@ class GPUModelRunner:
67 self.max_model_len = model_config.max_model_len
68 self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
69 self.max_num_tokens = scheduler_config.max_num_batched_tokens
70 + self.max_num_reqs = scheduler_config.max_num_seqs
71
72 # Model-related.
73 self.num_attn_layers = model_config.get_num_layers_by_block_type(
@@ -88,7 +89,7 @@ class GPUModelRunner:
89 self.requests: Dict[str, CachedRequestState] = {}
90 # Persistent batch.
91 self.input_batch = InputBatch(
91 - max_num_reqs=self.scheduler_config.max_num_seqs,
92 + max_num_reqs=self.max_num_reqs,
93 max_model_len=self.max_model_len,
94 max_num_blocks_per_req=self.max_num_blocks_per_req,
95 device=self.device,
@@ -117,6 +118,32 @@ class GPUModelRunner:
118 dtype=self.dtype,
119 device=self.device)
120
121 + self.input_ids_cpu = torch.zeros(self.max_num_tokens,
122 + dtype=torch.int32,
123 + device="cpu",
124 + pin_memory=self.pin_memory)
125 + self.input_ids_np = self.input_ids_cpu.numpy()
126 + self.positions_cpu = torch.zeros(self.max_num_tokens,
127 + dtype=torch.int64,
128 + device="cpu",
129 + pin_memory=self.pin_memory)
130 + self.positions_np = self.positions_cpu.numpy()
131 + self.slot_mapping_cpu = torch.zeros(self.max_num_tokens,
132 + dtype=torch.int32,
133 + device="cpu",
134 + pin_memory=self.pin_memory)
135 + self.slot_mapping_np = self.slot_mapping_cpu.numpy()
136 + self.query_start_loc_cpu = torch.zeros(self.max_num_reqs + 1,
137 + dtype=torch.int32,
138 + device="cpu",
139 + pin_memory=self.pin_memory)
140 + self.query_start_loc_np = self.query_start_loc_cpu.numpy()
141 + self.seq_start_loc_cpu = torch.zeros(self.max_num_reqs + 1,
142 + dtype=torch.int32,
143 + device="cpu",
144 + pin_memory=self.pin_memory)
145 + self.seq_start_loc_np = self.seq_start_loc_cpu.numpy()
146 +
147 def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
148 # Remove stopped requests from the cached states.
149 # Keep the states of the pre-empted requests.
@@ -241,22 +268,14 @@ class GPUModelRunner:
268
269 # Get request indices.
270 # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
244 - indices = np.arange(num_reqs)
245 - req_indices = np.repeat(indices, num_scheduled_tokens)
271 + req_indices = np.repeat(np.arange(num_reqs), num_scheduled_tokens)
272
273 # Get batched arange.
274 # E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
249 - arange_matrix = np.tile(np.arange(max_num_scheduled_tokens),
250 - (num_reqs, 1))
251 - mask = arange_matrix < num_scheduled_tokens[:, np.newaxis]
252 - arange = arange_matrix[mask]
275 + arange = np.concatenate([np.arange(n) for n in num_scheduled_tokens])
276
277 # Get positions.
255 - positions = torch.empty((total_num_scheduled_tokens, ),
256 - dtype=torch.int32,
257 - device="cpu",
258 - pin_memory=self.pin_memory)
259 - positions_np = positions.numpy()
278 + positions_np = self.positions_np[:total_num_scheduled_tokens]
279 np.add(self.input_batch.num_computed_tokens_cpu[req_indices],
280 arange,
281 out=positions_np)
@@ -267,16 +286,13 @@ class GPUModelRunner:
286 # where M is the max_model_len.
287 token_indices = (positions_np +
288 req_indices * self.input_batch.token_ids_cpu.shape[1])
270 - token_indices = torch.from_numpy(token_indices)
271 - input_ids = torch.empty((total_num_scheduled_tokens, ),
272 - dtype=torch.int32,
273 - device="cpu",
274 - pin_memory=self.pin_memory)
275 - torch.index_select(torch.from_numpy(
276 - self.input_batch.token_ids_cpu).flatten(),
289 + # NOTE(woosuk): We use torch.index_select instead of np.take here
290 + # because torch.index_select is much faster than np.take for large
291 + # tensors.
292 + torch.index_select(self.input_batch.token_ids_cpu_tensor.flatten(),
293 0,
278 - token_indices,
279 - out=input_ids)
294 + torch.from_numpy(token_indices),
295 + out=self.input_ids_cpu[:total_num_scheduled_tokens])
296
297 # Calculate the slot mapping.
298 # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
@@ -284,45 +300,40 @@ class GPUModelRunner:
300 # where K is the max_num_blocks_per_req and the block size is 2.
301 # NOTE(woosuk): We can't simply use `token_indices // block_size` here
302 # because M (max_model_len) is not necessarily divisible by block_size.
287 - block_numbers = self.input_batch.block_table_cpu_tensor.flatten()[
288 - req_indices * self.max_num_blocks_per_req +
289 - positions_np // self.block_size]
290 - block_offsets = torch.from_numpy(positions_np % self.block_size)
291 - slot_mapping = torch.empty((total_num_scheduled_tokens, ),
292 - dtype=torch.int32,
293 - device="cpu",
294 - pin_memory=self.pin_memory)
295 - torch.add(block_numbers * self.block_size,
296 - block_offsets,
297 - out=slot_mapping)
303 + block_table_indices = (req_indices * self.max_num_blocks_per_req +
304 + positions_np // self.block_size)
305 + # NOTE(woosuk): We use torch.index_select instead of np.take here
306 + # because torch.index_select is much faster than np.take for large
307 + # tensors.
308 + block_numbers = (self.input_batch.block_table_cpu_tensor.flatten()
309 + [block_table_indices].numpy())
310 + block_offsets = positions_np % self.block_size
311 + np.add(block_numbers * self.block_size,
312 + block_offsets,
313 + out=self.slot_mapping_np[:total_num_scheduled_tokens])
314
315 # Prepare the attention metadata.
300 - query_start_loc = torch.empty((num_reqs + 1, ),
301 - dtype=torch.int32,
302 - device="cpu",
303 - pin_memory=self.pin_memory)
304 - query_start_loc_np = query_start_loc.numpy()
305 - query_start_loc_np[0] = 0
306 - np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1:])
316 + self.query_start_loc_np[0] = 0
317 + np.cumsum(num_scheduled_tokens,
318 + out=self.query_start_loc_np[1:num_reqs + 1])
319
320 seq_lens = (self.input_batch.num_computed_tokens_cpu[:num_reqs] +
321 num_scheduled_tokens)
322 max_seq_len = seq_lens.max()
311 - seq_start_loc = torch.empty((num_reqs + 1, ),
312 - dtype=torch.int32,
313 - device="cpu",
314 - pin_memory=self.pin_memory)
315 - seq_start_loc_np = seq_start_loc.numpy()
316 - seq_start_loc_np[0] = 0
317 - np.cumsum(seq_lens, out=seq_start_loc_np[1:])
318 -
319 - self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
320 - non_blocking=True)
321 - self.positions[:total_num_scheduled_tokens].copy_(positions,
322 - non_blocking=True)
323 - query_start_loc = query_start_loc.to(self.device, non_blocking=True)
324 - seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
325 - slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
323 + self.seq_start_loc_np[0] = 0
324 + np.cumsum(seq_lens, out=self.seq_start_loc_np[1:num_reqs + 1])
325 +
326 + # Copy the tensors to the GPU.
327 + self.input_ids[:total_num_scheduled_tokens].copy_(
328 + self.input_ids_cpu[:total_num_scheduled_tokens], non_blocking=True)
329 + self.positions[:total_num_scheduled_tokens].copy_(
330 + self.positions_cpu[:total_num_scheduled_tokens], non_blocking=True)
331 + query_start_loc = self.query_start_loc_cpu[:num_reqs + 1].to(
332 + self.device, non_blocking=True)
333 + seq_start_loc = self.seq_start_loc_cpu[:num_reqs + 1].to(
334 + self.device, non_blocking=True)
335 + slot_mapping = self.slot_mapping_cpu[:total_num_scheduled_tokens].to(
336 + self.device, non_blocking=True).long()
337 attn_metadata = FlashAttentionMetadata(
338 num_actual_tokens=total_num_scheduled_tokens,
339 max_query_len=max_num_scheduled_tokens,