@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 31bff69151
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Make _prepare_sample non-blocking and use pinned memory for input buffers (#2207)

Hanzhi Zhou committed Dec 19, 2023 at 16:52 UTC 31bff69151606220e9db7ed37603e41b3f2e3230
1 file changed +38 -17
vllm/worker/model_runner.py
+38 -17
@@ -10,6 +10,7 @@ from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 from vllm.sampling_params import SamplingParams, SamplingType
12 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
13 +from vllm.utils import in_wsl
14
15 logger = init_logger(__name__)
16
@@ -52,6 +53,8 @@ class ModelRunner:
53 # The shape of the cached block table will be
54 # (max batch size to capture, max context len to capture / block size).
55 self.graph_block_tables = None # Set after initial profiling.
56 + # cache in_wsl result
57 + self.in_wsl = in_wsl()
58
59 def load_model(self) -> None:
60 self.model = get_model(self.model_config)
@@ -203,24 +206,29 @@ class ModelRunner:
206 # When using CUDA graph, we don't need to make the tensors on the GPU
207 # because they will be eventually copied to the designated GPU buffer.
208 device = "cpu" if use_captured_graph else "cuda"
209 + pin_memory = use_captured_graph and not self.in_wsl
210 input_tokens = _make_tensor_with_pad(input_tokens,
211 max_len=1,
212 pad=0,
213 dtype=torch.long,
210 - device=device)
214 + device=device,
215 + pin_memory=pin_memory)
216 input_positions = _make_tensor_with_pad(input_positions,
217 max_len=1,
218 pad=0,
219 dtype=torch.long,
215 - device=device)
220 + device=device,
221 + pin_memory=pin_memory)
222 slot_mapping = _make_tensor_with_pad(slot_mapping,
223 max_len=1,
224 pad=_PAD_SLOT_ID,
225 dtype=torch.long,
220 - device=device)
226 + device=device,
227 + pin_memory=pin_memory)
228 context_lens = torch.tensor(context_lens,
229 dtype=torch.int,
223 - device=device)
230 + device=device,
231 + pin_memory=pin_memory)
232
233 if use_captured_graph:
234 # The shape of graph_block_tables is
@@ -229,7 +237,7 @@ class ModelRunner:
237 for i, block_table in enumerate(block_tables):
238 if block_table:
239 input_block_tables[i, :len(block_table)] = block_table
232 - block_tables = torch.from_numpy(input_block_tables).to(device)
240 + block_tables = torch.tensor(input_block_tables, device=device)
241 else:
242 block_tables = _make_tensor_with_pad(
243 block_tables,
@@ -297,11 +305,11 @@ class ModelRunner:
305 categorized_sample_indices_start_idx + num_seqs))
306 categorized_sample_indices_start_idx += num_seqs
307
300 - selected_token_indices = torch.tensor(selected_token_indices,
301 - dtype=torch.long,
302 - device="cuda")
308 + selected_token_indices = _async_h2d(selected_token_indices,
309 + dtype=torch.long,
310 + pin_memory=not self.in_wsl)
311 categorized_sample_indices = {
304 - t: torch.tensor(seq_ids, dtype=torch.int, device="cuda")
312 + t: _async_h2d(seq_ids, dtype=torch.int, pin_memory=not self.in_wsl)
313 for t, seq_ids in categorized_sample_indices.items()
314 }
315
@@ -334,8 +342,6 @@ class ModelRunner:
342 else:
343 inputs = self._prepare_decode(seq_group_metadata_list)
344 input_tokens, input_positions, input_metadata = inputs
337 - sampling_metadata = self._prepare_sample(seq_group_metadata_list,
338 - input_metadata.prompt_lens)
345
346 # Execute the model.
347 if input_metadata.use_cuda_graph:
@@ -350,6 +356,9 @@ class ModelRunner:
356 input_metadata=input_metadata,
357 )
358
359 + sampling_metadata = self._prepare_sample(seq_group_metadata_list,
360 + input_metadata.prompt_lens)
361 +
362 # Sample the next token.
363 output = self.model.sample(
364 hidden_states=hidden_states,
@@ -502,11 +511,14 @@ class CUDAGraphRunner:
511 del kv_caches
512
513 # Copy the input tensors to the input buffers.
505 - self.input_buffers["input_ids"].copy_(input_ids)
506 - self.input_buffers["positions"].copy_(positions)
507 - self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping)
508 - self.input_buffers["context_lens"].copy_(input_metadata.context_lens)
509 - self.input_buffers["block_tables"].copy_(input_metadata.block_tables)
514 + self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
515 + self.input_buffers["positions"].copy_(positions, non_blocking=True)
516 + self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping,
517 + non_blocking=True)
518 + self.input_buffers["context_lens"].copy_(input_metadata.context_lens,
519 + non_blocking=True)
520 + self.input_buffers["block_tables"].copy_(input_metadata.block_tables,
521 + non_blocking=True)
522
523 # Run the graph.
524 self.graph.replay()
@@ -529,9 +541,13 @@ def _make_tensor_with_pad(
541 pad: int,
542 dtype: torch.dtype,
543 device: Union[str, torch.device] = "cuda",
544 + pin_memory: bool = False,
545 ) -> torch.Tensor:
546 padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
534 - return torch.tensor(padded_x, dtype=dtype, device=device)
547 + return torch.tensor(padded_x,
548 + dtype=dtype,
549 + device=device,
550 + pin_memory=pin_memory and str(device) == "cpu")
551
552
553 def _get_graph_batch_size(batch_size: int) -> int:
@@ -541,3 +557,8 @@ def _get_graph_batch_size(batch_size: int) -> int:
557 return 4
558 else:
559 return (batch_size + 7) // 8 * 8
560 +
561 +
562 +def _async_h2d(data: list, dtype, pin_memory):
563 + t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory)
564 + return t.to(device="cuda", non_blocking=True)