@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 300acb8347
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Core][Bugfix] Use correct device to initialize GPU data during CUDA-graph-capture (#11233)

Signed-off-by: Yan Burman <yanburman@users.noreply.github.com> Signed-off-by: Ido Asraff <idoa@atero.ai>

Yan Burman committed Jan 4, 2025 at 08:50 UTC 300acb83472512b14ec7ba8cdf45efe07e8c8f68
5 files changed +23 -15
tests/distributed/test_custom_all_reduce.py
+1 -1
@@ -50,7 +50,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
50
51 for sz in test_sizes:
52 for dtype in [torch.float32, torch.float16, torch.bfloat16]:
53 - with graph_capture() as graph_capture_context:
53 + with graph_capture(device=device) as graph_capture_context:
54 # use integers so result matches NCCL exactly
55 inp1 = torch.randint(1,
56 16, (sz, ),
tests/distributed/test_pynccl.py
+1 -1
@@ -107,7 +107,7 @@ def multiple_allreduce_with_vllm_worker_fn():
107 device = torch.device(f"cuda:{torch.distributed.get_rank()}")
108 ensure_model_parallel_initialized(2, 2)
109 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
110 - with graph_capture():
110 + with graph_capture(device=device):
111 # two tp groups can communicate independently
112 if torch.distributed.get_rank() in [0, 1]:
113 tensor = tensor_model_parallel_all_reduce(tensor)
vllm/distributed/parallel_state.py
+4 -3
@@ -920,7 +920,7 @@ def get_kv_transfer_group() -> kv_transfer.KVTransferAgent:
920
921
922 @contextmanager
923 -def graph_capture():
923 +def graph_capture(device: torch.device):
924 """
925 `graph_capture` is a context manager which should surround the code that
926 is capturing the CUDA graph. Its main purpose is to ensure that the
@@ -934,8 +934,9 @@ def graph_capture():
934 in order to explicitly distinguish the kernels to capture
935 from other kernels possibly launched on background in the default stream.
936 """
937 - with get_tp_group().graph_capture() as context, get_pp_group(
938 - ).graph_capture(context):
937 + context = GraphCaptureContext(torch.cuda.Stream(device=device))
938 + with get_tp_group().graph_capture(context), get_pp_group().graph_capture(
939 + context):
940 yield context
941
942
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -836,7 +836,7 @@ class GPUModelRunner:
836 # Trigger CUDA graph capture for specific shapes.
837 # Capture the large shapes first so that the smaller shapes
838 # can reuse the memory pool allocated for the large shapes.
839 - with graph_capture():
839 + with graph_capture(device=self.device):
840 for num_tokens in reversed(self.cudagraph_batch_sizes):
841 for _ in range(self.vllm_config.compilation_config.
842 cudagraph_num_of_warmups):
vllm/worker/model_runner.py
+16 -9
@@ -1426,10 +1426,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1426
1427 # Prepare dummy inputs. These will be reused for all batch sizes.
1428 max_batch_size = self.max_batchsize_to_capture
1429 - input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1430 - input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1429 + input_tokens = torch.zeros(max_batch_size,
1430 + dtype=torch.long,
1431 + device=self.device)
1432 + input_positions = torch.zeros(max_batch_size,
1433 + dtype=torch.long,
1434 + device=self.device)
1435 if self.model_config.uses_mrope:
1432 - input_positions = torch.tile(input_positions, (3, 1))
1436 + input_positions = torch.tile(input_positions,
1437 + (3, 1)).cuda(device=self.device)
1438 # Prepare dummy previous_hidden_states only if needed by the model.
1439 # This is used by draft models such as EAGLE.
1440 previous_hidden_states = None
@@ -1448,8 +1453,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1453 dtype=self.model_config.dtype,
1454 device=self.device)
1455
1451 - with self.attn_state.graph_capture(
1452 - max_batch_size), graph_capture() as graph_capture_context:
1456 + with self.attn_state.graph_capture(max_batch_size), graph_capture(
1457 + self.device) as graph_capture_context:
1458 # NOTE: Capturing the largest batch size first may help reduce the
1459 # memory usage of CUDA graph.
1460 for virtual_engine in range(
@@ -1549,10 +1554,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1554 """
1555 # During the decode phase encoder_input_ids and encoder_positions are
1556 # unset. Do the same thing for graph capture.
1552 - capture_inputs["encoder_input_ids"] = torch.tensor(
1553 - [], dtype=torch.long).cuda()
1554 - capture_inputs["encoder_positions"] = torch.tensor(
1555 - [], dtype=torch.long).cuda()
1557 + capture_inputs["encoder_input_ids"] = torch.tensor([],
1558 + dtype=torch.long,
1559 + device=self.device)
1560 + capture_inputs["encoder_positions"] = torch.tensor([],
1561 + dtype=torch.long,
1562 + device=self.device)
1563
1564 @property
1565 def vocab_size(self) -> int: