A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Core][Bugfix] Use correct device to initialize GPU data during CUDA-graph-capture (#11233)
Signed-off-by: Yan Burman <yanburman@users.noreply.github.com> Signed-off-by: Ido Asraff <idoa@atero.ai>
Yan Burman committed
Jan 4, 2025 at 08:50 UTC
300acb83472512b14ec7ba8cdf45efe07e8c8f68
5 files changed
+23
-15
tests/distributed/test_custom_all_reduce.py
+1
-1
@@ -50,7 +50,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
50
51
for sz in test_sizes:
52
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
53
- with graph_capture() as graph_capture_context:
53
+ with graph_capture(device=device) as graph_capture_context:
54
# use integers so result matches NCCL exactly
55
inp1 = torch.randint(1,
56
16, (sz, ),
tests/distributed/test_pynccl.py
+1
-1
@@ -107,7 +107,7 @@ def multiple_allreduce_with_vllm_worker_fn():
107
device = torch.device(f"cuda:{torch.distributed.get_rank()}")
108
ensure_model_parallel_initialized(2, 2)
109
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
110
- with graph_capture():
110
+ with graph_capture(device=device):
111
# two tp groups can communicate independently
112
if torch.distributed.get_rank() in [0, 1]:
113
tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/parallel_state.py
+4
-3
@@ -920,7 +920,7 @@ def get_kv_transfer_group() -> kv_transfer.KVTransferAgent:
920
921
922
@contextmanager
923
-def graph_capture():
923
+def graph_capture(device: torch.device):
924
"""
925
`graph_capture` is a context manager which should surround the code that
926
is capturing the CUDA graph. Its main purpose is to ensure that the
@@ -934,8 +934,9 @@ def graph_capture():
934
in order to explicitly distinguish the kernels to capture
935
from other kernels possibly launched on background in the default stream.
936
"""
937
- with get_tp_group().graph_capture() as context, get_pp_group(
938
- ).graph_capture(context):
937
+ context = GraphCaptureContext(torch.cuda.Stream(device=device))
938
+ with get_tp_group().graph_capture(context), get_pp_group().graph_capture(
939
+ context):
940
yield context
941
942
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -836,7 +836,7 @@ class GPUModelRunner:
836
# Trigger CUDA graph capture for specific shapes.
837
# Capture the large shapes first so that the smaller shapes
838
# can reuse the memory pool allocated for the large shapes.
839
- with graph_capture():
839
+ with graph_capture(device=self.device):
840
for num_tokens in reversed(self.cudagraph_batch_sizes):
841
for _ in range(self.vllm_config.compilation_config.
842
cudagraph_num_of_warmups):
vllm/worker/model_runner.py
+16
-9
@@ -1426,10 +1426,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1426
1427
# Prepare dummy inputs. These will be reused for all batch sizes.
1428
max_batch_size = self.max_batchsize_to_capture
1429
- input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1430
- input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1429
+ input_tokens = torch.zeros(max_batch_size,
1430
+ dtype=torch.long,
1431
+ device=self.device)
1432
+ input_positions = torch.zeros(max_batch_size,
1433
+ dtype=torch.long,
1434
+ device=self.device)
1435
if self.model_config.uses_mrope:
1432
- input_positions = torch.tile(input_positions, (3, 1))
1436
+ input_positions = torch.tile(input_positions,
1437
+ (3, 1)).cuda(device=self.device)
1438
# Prepare dummy previous_hidden_states only if needed by the model.
1439
# This is used by draft models such as EAGLE.
1440
previous_hidden_states = None
@@ -1448,8 +1453,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1453
dtype=self.model_config.dtype,
1454
device=self.device)
1455
1451
- with self.attn_state.graph_capture(
1452
- max_batch_size), graph_capture() as graph_capture_context:
1456
+ with self.attn_state.graph_capture(max_batch_size), graph_capture(
1457
+ self.device) as graph_capture_context:
1458
# NOTE: Capturing the largest batch size first may help reduce the
1459
# memory usage of CUDA graph.
1460
for virtual_engine in range(
@@ -1549,10 +1554,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1554
"""
1555
# During the decode phase encoder_input_ids and encoder_positions are
1556
# unset. Do the same thing for graph capture.
1552
- capture_inputs["encoder_input_ids"] = torch.tensor(
1553
- [], dtype=torch.long).cuda()
1554
- capture_inputs["encoder_positions"] = torch.tensor(
1555
- [], dtype=torch.long).cuda()
1557
+ capture_inputs["encoder_input_ids"] = torch.tensor([],
1558
+ dtype=torch.long,
1559
+ device=self.device)
1560
+ capture_inputs["encoder_positions"] = torch.tensor([],
1561
+ dtype=torch.long,
1562
+ device=self.device)
1563
1564
@property
1565
def vocab_size(self) -> int: