@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 5d8e90a966

[WideEP] Update NCCL to 2.30.7 to enable DeepEPv2 in the vllm/vllm-openai image (#45321)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Tyler Michael Smith <tyler@vllm.ai> Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Ilya Markov <ilmarkov@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Codex <noreply@openai.com>

Tyler Michael Smith committed Jul 24, 2026 at 16:00 UTC 5d8e90a96616c4fe339ff0b0c2a2d470f6eb24bf
13 files changed +338 -78
.buildkite/test_areas/misc.yaml
+1 -1
@@ -212,7 +212,7 @@ steps:
212 - vllm/multimodal
213 - examples/
214 commands:
215 - - pip install tensorizer # for tensorizer test
215 + - pip install --no-deps tensorizer # for tensorizer test
216 # for basic
217 - python3 basic/offline_inference/chat.py
218 - python3 basic/offline_inference/generate.py --model facebook/opt-125m
.buildkite/test_areas/model_runner_v2.yaml
+1 -1
@@ -41,7 +41,7 @@ steps:
41 commands:
42 - set -x
43 - export VLLM_USE_V2_MODEL_RUNNER=1
44 - - pip install tensorizer # for tensorizer test
44 + - pip install --no-deps tensorizer # for tensorizer test
45 - python3 basic/offline_inference/chat.py # for basic
46 - python3 basic/offline_inference/generate.py --model facebook/opt-125m
47 #- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 # TODO
docker/Dockerfile
+25 -8
@@ -25,6 +25,10 @@
25 ARG CUDA_VERSION=13.0.3
26 ARG PYTHON_VERSION=3.12
27 ARG UBUNTU_VERSION=22.04
28 +# DeepEPv2 requires NCCL >= 2.30.4 (GIN backend).
29 +# This version is only used for CUDA 13+ builds; CUDA 12 falls back to
30 +# the default NCCL version shipped with the base image.
31 +ARG NCCL_VERSION=2.30.7
32
33 # By parameterizing the base images, we allow third-party to use their own
34 # base images. One use case is hermetic builds with base images stored in
@@ -477,10 +481,17 @@ WORKDIR /workspace
481 # Build DeepEP wheels
482 COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh
483 # Defaults moved here from tools/ep_kernels/install_python_libraries.sh for centralized version management
480 -ARG DEEPEP_COMMIT_HASH=73b6ea4
484 +ARG DEEPEP_COMMIT_HASH=d4f41e4e93
485 ARG NVSHMEM_VER
486 +ARG NCCL_VERSION
487 RUN --mount=type=cache,target=/opt/uv/cache \
488 mkdir -p /tmp/ep_kernels_workspace/dist && \
489 + CUDA_MAJOR=$(echo $CUDA_VERSION | cut -d. -f1) && \
490 + if [ "$CUDA_MAJOR" -ge 13 ] && [ -n "$NCCL_VERSION" ]; then \
491 + echo "nvidia-nccl-cu${CUDA_MAJOR}==${NCCL_VERSION}" \
492 + > /tmp/nccl-override.txt && \
493 + export UV_OVERRIDE=/tmp/nccl-override.txt; \
494 + fi && \
495 export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \
496 /tmp/install_python_libraries.sh \
497 --workspace /tmp/ep_kernels_workspace \
@@ -644,6 +655,7 @@ FROM ${FINAL_BASE_IMAGE} AS vllm-base
655
656 ARG CUDA_VERSION
657 ARG PYTHON_VERSION
658 +ARG NCCL_VERSION
659 ARG DEADSNAKES_MIRROR_URL
660 ARG DEADSNAKES_GPGKEY_URL
661 ARG GET_PIP_URL
@@ -696,7 +708,6 @@ RUN apt-get update -y \
708 # Install CUDA development tools for runtime JIT compilation
709 # (FlashInfer, DeepGEMM, EP kernels all require compilation at runtime)
710 RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
699 - CUDA_VERSION_SHORT=$(echo $CUDA_VERSION | cut -d. -f1,2) && \
711 apt-get update -y && \
712 apt-get install -y --no-install-recommends --allow-change-held-packages \
713 cuda-nvcc-${CUDA_VERSION_DASH} \
@@ -709,12 +720,6 @@ RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
720 libnuma-dev \
721 # numactl CLI for NUMA binding at runtime
722 numactl && \
712 - # Fixes nccl_allocator requiring nccl.h at runtime
713 - # https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
714 - # NCCL packages don't use the cuda-MAJOR-MINOR naming convention,
715 - # so we pin the version to match our CUDA version
716 - NCCL_VER=$(apt-cache madison libnccl-dev | grep "+cuda${CUDA_VERSION_SHORT}" | head -1 | awk -F'|' '{gsub(/^ +| +$/, "", $2); print $2}') && \
717 - apt-get install -y --no-install-recommends --allow-change-held-packages libnccl-dev=${NCCL_VER} libnccl2=${NCCL_VER} && \
723 rm -rf /var/lib/apt/lists/*
724
725 # Install uv for faster pip installs
@@ -734,6 +739,18 @@ RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" \
739 && chgrp -R 0 /opt/uv \
740 && chmod -R g+rwX,a+rX /opt/uv
741
742 +# DeepEPv2 GIN requires NCCL >= 2.30.4 at both compile and runtime. torch pins
743 +# an older version as a transitive dep; this override forces uv to use our
744 +# pinned version whenever nvidia-nccl-cu* is resolved. Empty on CUDA 12 (no-op).
745 +RUN CUDA_MAJOR=$(echo $CUDA_VERSION | cut -d. -f1) && \
746 + if [ "$CUDA_MAJOR" -ge 13 ]; then \
747 + echo "nvidia-nccl-cu${CUDA_MAJOR}==${NCCL_VERSION}" \
748 + > /etc/uv-overrides.txt; \
749 + else \
750 + touch /etc/uv-overrides.txt; \
751 + fi
752 +ENV UV_OVERRIDE=/etc/uv-overrides.txt
753 +
754 # ----------------------------------------------------------------------
755 # Non-root support (opt-in)
756 # ----------------------------------------------------------------------
docker/versions.json
+4 -1
@@ -10,6 +10,9 @@
10 "UBUNTU_VERSION": {
11 "default": "22.04"
12 },
13 + "NCCL_VERSION": {
14 + "default": "2.30.7"
15 + },
16 "BUILD_BASE_IMAGE": {
17 "default": "nvidia/cuda:13.0.3-devel-ubuntu22.04"
18 },
@@ -56,7 +59,7 @@
59 "default": "cuda"
60 },
61 "DEEPEP_COMMIT_HASH": {
59 - "default": "73b6ea4"
62 + "default": "d4f41e4e93"
63 },
64 "GIT_REPO_CHECK": {
65 "default": "0"
docs/serving/expert_parallel_deployment.md
+5
@@ -12,6 +12,11 @@ Before using EP, you need to install the necessary dependencies. We are actively
12 2. **Install DeepGEMM library**: Follow the [official instructions](https://github.com/deepseek-ai/DeepGEMM#installation).
13 3. **For disaggregated serving**: Install `gdrcopy` by running the [`install_gdrcopy.sh`](../../tools/install_gdrcopy.sh) script (e.g., `install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "12.8" "x64"`). You can find available OS versions [here](https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2012.8/).
14
15 +!!! note "NCCL version (CUDA 13+)"
16 + The `deepep_v2` backend requires NCCL >= 2.30.4. PyTorch ships an older
17 + NCCL, so you must upgrade it before building or running DeepEP. See the
18 + [EP kernels guide](../../tools/ep_kernels) for instructions.
19 +
20 ### Backend Selection Guide
21
22 vLLM provides multiple communication backends for EP. Use `--all2all-backend` to select one:
tests/distributed/test_mnnvl_alltoall.py
+9 -3
@@ -73,7 +73,10 @@ def _spawn_workers(worker_fn, world_size, *, dp_size=None):
73 err_queue.close()
74 err_queue.join_thread()
75 if errors:
76 - pytest.fail("Worker(s) failed:\n" + "\n---\n".join(errors))
76 + combined = "\n---\n".join(errors)
77 + if "NCCL GIN" in combined:
78 + pytest.skip("NCCL GIN not available on this system")
79 + pytest.fail("Worker(s) failed:\n" + combined)
80
81
82 def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue):
@@ -916,8 +919,11 @@ def _deepep_v2_lifecycle_worker(rank, world_size):
919 DeepEPV2All2AllManager,
920 )
921
919 - cpu_group = get_ep_group().cpu_group
920 - manager = DeepEPV2All2AllManager(cpu_group)
922 + ep_group = get_ep_group()
923 + manager = DeepEPV2All2AllManager(
924 + ep_group.cpu_group,
925 + device_group=ep_group.device_group,
926 + )
927
928 assert manager.rank == rank
929 assert manager.world_size == world_size
tests/kernels/moe/parallel_utils.py
+36 -12
@@ -36,6 +36,10 @@ if has_deep_ep_v2():
36 P = ParamSpec("P")
37
38
39 +class GINNotAvailableError(RuntimeError):
40 + pass
41 +
42 +
43 @dataclasses.dataclass
44 class ProcessGroupInfo:
45 world_size: int
@@ -96,19 +100,27 @@ def parallel_launch(
100 **kwargs: P.kwargs,
101 ) -> None:
102 assert not kwargs
99 - spawn(
100 - _worker_parallel_launch,
101 - args=(
102 - world_size,
103 - world_size,
104 - 0,
105 - f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
106 - worker,
103 + try:
104 + spawn(
105 + _worker_parallel_launch,
106 + args=(
107 + world_size,
108 + world_size,
109 + 0,
110 + f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
111 + worker,
112 + )
113 + + args,
114 + nprocs=world_size,
115 + join=True,
116 )
108 - + args,
109 - nprocs=world_size,
110 - join=True,
111 - )
117 + except Exception as exc:
118 + # pytest.skip cannot propagate directly through torch.multiprocessing.
119 + if "GINNotAvailableError" in str(exc):
120 + import pytest
121 +
122 + pytest.skip("NCCL GIN not available (no IBGDA-capable hardware)")
123 + raise
124
125
126 ## DeepEP specific utils
@@ -225,6 +237,18 @@ def make_deepep_v2_a2a(
237 ):
238 import deep_ep
239
240 + from vllm.utils.nccl import query_nccl_gin_type
241 +
242 + # ElasticBuffer can segfault when GIN is unavailable. Initialize the
243 + # lazy communicator and reject unsupported systems before entering DeepEP.
244 + probe = torch.zeros(1, device=pgi.device)
245 + torch.distributed.all_reduce(probe, group=pg)
246 + gin_type = query_nccl_gin_type(pg)
247 + if gin_type is None:
248 + raise RuntimeError("Failed to determine NCCL GIN support")
249 + if gin_type == 0:
250 + raise GINNotAvailableError("NCCL GIN not available")
251 +
252 buffer = deep_ep.ElasticBuffer(
253 group=pg,
254 num_max_tokens_per_rank=v2_args.max_tokens_per_rank,
tests/kernels/moe/test_deepep_v2_moe.py
+101 -50
@@ -14,6 +14,7 @@ from torch.distributed import ProcessGroup
14 from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
15 from tests.kernels.utils import torch_experts
16 from vllm.config import VllmConfig, set_current_vllm_config
17 +from vllm.forward_context import set_forward_context
18 from vllm.model_executor.layers.fused_moe import TritonExperts
19 from vllm.model_executor.layers.fused_moe.activation import MoEActivation
20 from vllm.model_executor.layers.fused_moe.config import (
@@ -36,6 +37,14 @@ requires_deep_ep_v2 = pytest.mark.skipif(
37 )
38
39
40 +def assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
41 + close = torch.isclose(actual, expected, atol=2e-1, rtol=2e-1)
42 + close_fraction = close.float().mean().item()
43 + assert close_fraction > 0.99, (
44 + f"Only {close_fraction:.1%} of FP8 outputs are within tolerance"
45 + )
46 +
47 +
48 @dataclasses.dataclass
49 class TestConfig:
50 dtype: torch.dtype
@@ -50,6 +59,7 @@ class TestConfig:
59 class TestTensors:
60 rank_tokens: torch.Tensor
61 rank_token_scales: torch.Tensor | None
62 + intermediate_scales: torch.Tensor | None
63 topk: torch.Tensor
64 topk_weights: torch.Tensor
65 config: TestConfig
@@ -63,6 +73,12 @@ class TestTensors:
73 rank_tokens = (
74 torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
75 )
76 + if config.dtype == torch.float8_e4m3fn:
77 + rank_token_scales = torch.tensor(1 / 448, device="cuda")
78 + intermediate_scales = torch.tensor(8 / 448, device="cuda")
79 + else:
80 + rank_token_scales = None
81 + intermediate_scales = None
82
83 topk = torch.stack(
84 [
@@ -73,7 +89,8 @@ class TestTensors:
89 topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
90 return TestTensors(
91 rank_tokens=rank_tokens,
76 - rank_token_scales=None,
92 + rank_token_scales=rank_token_scales,
93 + intermediate_scales=intermediate_scales,
94 topk=topk,
95 topk_weights=topk_weights,
96 config=config,
@@ -124,7 +141,6 @@ def make_modular_kernel(
141 mk = FusedMoEKernel(
142 prepare_finalize=a2a,
143 fused_experts=fused_experts,
127 - inplace=False,
144 )
145 return mk
146
@@ -162,6 +178,7 @@ def deepep_v2_moe_impl(
178 w2_scale=w2_scale,
179 per_act_token_quant=per_act_token_quant,
180 a1_scale=test_tensors.rank_token_scales,
181 + a2_scale=test_tensors.intermediate_scales,
182 )
183
184 hidden_size = test_tensors.rank_tokens.size(1)
@@ -231,6 +248,8 @@ def _deep_ep_v2_moe(
248 test_tensors.topk,
249 w1_scale=w1_scale,
250 w2_scale=w2_scale,
251 + a1_scale=test_tensors.rank_token_scales,
252 + a2_scale=test_tensors.intermediate_scales,
253 quant_dtype=q_dtype,
254 per_act_token_quant=per_act_token_quant,
255 )
@@ -262,12 +281,15 @@ def _deep_ep_v2_moe(
281 per_act_token_quant,
282 )
283
265 - torch.testing.assert_close(
266 - torch_combined,
267 - deepep_combined,
268 - atol=6e-2,
269 - rtol=6e-2,
270 - )
284 + if is_quantized:
285 + assert_fp8_close(torch_combined, deepep_combined)
286 + else:
287 + torch.testing.assert_close(
288 + torch_combined,
289 + deepep_combined,
290 + atol=6e-2,
291 + rtol=6e-2,
292 + )
293
294
295 MNKs = [
@@ -355,17 +377,29 @@ def _deep_ep_v2_moe_cudagraph(
377 num_local_experts = config.num_experts // pgi.world_size
378 hidden_size = config.k
379
358 - # Create FP8 weights directly, then dequantize for bf16 reference.
359 - w1_fp8 = torch.randn(
360 - (config.num_experts, 2 * config.n, config.k),
361 - device="cuda",
362 - dtype=torch.bfloat16,
363 - ).to(torch.float8_e4m3fn)
364 - w2_fp8 = torch.randn(
365 - (config.num_experts, config.k, config.n),
366 - device="cuda",
367 - dtype=torch.bfloat16,
368 - ).to(torch.float8_e4m3fn)
380 + # All ranks must use the same global weights before taking their EP slice.
381 + w1_bf16 = (
382 + torch.randn(
383 + (config.num_experts, 2 * config.n, config.k),
384 + device="cuda",
385 + dtype=torch.bfloat16,
386 + )
387 + / 15
388 + )
389 + w2_bf16 = (
390 + torch.randn(
391 + (config.num_experts, config.k, config.n),
392 + device="cuda",
393 + dtype=torch.bfloat16,
394 + )
395 + / 15
396 + )
397 + torch.distributed.broadcast(w1_bf16, src=0, group=pg)
398 + torch.distributed.broadcast(w2_bf16, src=0, group=pg)
399 +
400 + # Round-trip through FP8 before constructing the reference and kernel weights.
401 + w1_fp8 = w1_bf16.to(torch.float8_e4m3fn)
402 + w2_fp8 = w2_bf16.to(torch.float8_e4m3fn)
403 w1_ref = w1_fp8.to(torch.bfloat16)
404 w2_ref = w2_fp8.to(torch.bfloat16)
405
@@ -385,21 +419,8 @@ def _deep_ep_v2_moe_cudagraph(
419 backend="nccl",
420 )
421 initialize_model_parallel(tensor_model_parallel_size=1)
388 - # Reference MoE using dequantized bf16 weights
389 - torch_combined = torch_experts(
390 - test_tensors.rank_tokens,
391 - w1_ref,
392 - w2_ref,
393 - test_tensors.topk_weights,
394 - test_tensors.topk,
395 - )
396 -
397 - # Use the production pipeline: make_fused_moe_layer creates
398 - # a FusedMoE layer, quantizes weights, runs
399 - # process_weights_after_loading (TrtLLM W31 swap + BlockMajorK
400 - # shuffle), and selects the kernel.
401 - # Quantize weights using production helper, EP-slice, then
402 - # convert to TrtLLM format.
422 + # Mirror production weight processing: quantize, EP-slice, then
423 + # convert to the TrtLLM BlockMajorK format.
424 from tests.kernels.moe.test_moe_layer import _quantize_fp8_halves
425 from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
426 TrtLlmFp8ExpertsModular,
@@ -411,14 +432,32 @@ def _deep_ep_v2_moe_cudagraph(
432
433 block_shape = [128, 128]
434 qw = _quantize_fp8_halves(w1_ref, w2_ref, block_shape)
435 + assert qw.w13_weight_scale is not None
436 + assert qw.w2_weight_scale is not None
437 +
438 + # Reference MoE using the same blockwise FP8 quantization scheme as
439 + # the production kernel. torch_experts quantizes activations before
440 + # both GEMMs and dequantizes the operands for the reference matmuls.
441 + reference_topk_weights = test_tensors.topk_weights.to(torch.bfloat16).to(
442 + torch.float32
443 + )
444 + torch_combined = torch_experts(
445 + test_tensors.rank_tokens,
446 + qw.w13_weight,
447 + qw.w2_weight,
448 + reference_topk_weights,
449 + test_tensors.topk,
450 + w1_scale=qw.w13_weight_scale,
451 + w2_scale=qw.w2_weight_scale,
452 + quant_dtype=torch.float8_e4m3fn,
453 + block_shape=block_shape,
454 + )
455
456 # EP-slice before format conversion
457 e_start = num_local_experts * pgi.rank
458 e_end = e_start + num_local_experts
459 w1_ep = qw.w13_weight[e_start:e_end]
460 w2_ep = qw.w2_weight[e_start:e_end]
420 - assert qw.w13_weight_scale is not None
421 - assert qw.w2_weight_scale is not None
461 w1_scale_ep = qw.w13_weight_scale[e_start:e_end]
462 w2_scale_ep = qw.w2_weight_scale[e_start:e_end]
463
@@ -452,11 +491,23 @@ def _deep_ep_v2_moe_cudagraph(
491 w2_scale=w2_scale_ep,
492 )
493 moe_config = make_dummy_moe_config(
455 - num_experts=num_local_experts,
494 + num_experts=config.num_experts,
495 + num_local_experts=num_local_experts,
496 experts_per_token=config.topk,
497 hidden_dim=hidden_size,
498 intermediate_size=config.n,
499 )
500 + moe_parallel_config = dataclasses.replace(
501 + moe_config.moe_parallel_config,
502 + ep_size=pgi.world_size,
503 + ep_rank=pgi.rank,
504 + use_ep=True,
505 + all2all_backend="deepep_v2",
506 + )
507 + moe_config = dataclasses.replace(
508 + moe_config,
509 + moe_parallel_config=moe_parallel_config,
510 + )
511 fused_experts = TrtLlmFp8ExpertsModular(
512 moe_config=moe_config,
513 quant_config=quant_config,
@@ -480,21 +531,21 @@ def _deep_ep_v2_moe_cudagraph(
531 mk_kernel = FusedMoEKernel(
532 prepare_finalize=a2a,
533 fused_experts=fused_experts,
483 - inplace=False,
534 )
535
486 - for _ in range(3):
487 - out = mk_kernel.apply(
488 - hidden_states=test_tensors.rank_tokens,
489 - w1=w1_ep,
490 - w2=w2_ep,
491 - topk_weights=test_tensors.topk_weights,
492 - topk_ids=test_tensors.topk,
493 - activation=MoEActivation.SILU,
494 - global_num_experts=config.num_experts,
495 - expert_map=None,
496 - apply_router_weight_on_input=False,
497 - )
536 + with set_forward_context(None, vllm_cfg):
537 + for _ in range(3):
538 + out = mk_kernel.apply(
539 + hidden_states=test_tensors.rank_tokens,
540 + w1=w1_ep,
541 + w2=w2_ep,
542 + topk_weights=test_tensors.topk_weights,
543 + topk_ids=test_tensors.topk,
544 + activation=MoEActivation.SILU,
545 + global_num_experts=config.num_experts,
546 + expert_map=None,
547 + apply_router_weight_on_input=False,
548 + )
549
550 torch.testing.assert_close(
551 torch_combined,
tools/ep_kernels/README.md
+32
@@ -11,6 +11,38 @@ Step 2 is necessary for multi-node deployment.
11
12 All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
13
14 +## NCCL version requirement (CUDA 13+)
15 +
16 +DeepEPv2 uses the NCCL GIN (GPU-Initiated Networking) backend, which requires
17 +NCCL >= 2.30.4 at both compile time and runtime. PyTorch 2.11 pins
18 +`nvidia-nccl-cu13==2.28.9` as a transitive dependency, so you need to
19 +override it.
20 +
21 +**With uv** (recommended):
22 +
23 +```bash
24 +# Create an override file
25 +echo "nvidia-nccl-cu13>=2.30.4" > /tmp/nccl-override.txt
26 +export UV_OVERRIDE=/tmp/nccl-override.txt
27 +
28 +# All subsequent uv pip install commands will respect the override
29 +uv pip install vllm
30 +```
31 +
32 +**With pip**:
33 +
34 +```bash
35 +pip install vllm
36 +pip install "nvidia-nccl-cu13>=2.30.4" --no-deps
37 +```
38 +
39 +The override / reinstall must happen before building DeepEP (for GIN device
40 +headers) and must remain in place at runtime. You can verify with:
41 +
42 +```bash
43 +python -c "from vllm.utils.import_utils import has_deep_ep_v2; print(has_deep_ep_v2())"
44 +```
45 +
46 ## Usage
47
48 ```bash
vllm/distributed/device_communicators/all2all.py
+27
@@ -978,6 +978,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
978 self._device_group = device_group
979 self.handle_cache = Cache()
980 self._num_sms: int | None = None
981 + self._gin_checked = False
982
983 def _make_all2all_kwargs(
984 self,
@@ -1000,11 +1001,37 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
1001 explicitly_destroy=True,
1002 )
1003
1004 + def _check_gin_support(self, group) -> None:
1005 + from vllm.utils.nccl import query_nccl_gin_type
1006 +
1007 + # ProcessGroupNCCL creates communicators lazily. Initialize this exact
1008 + # group before querying so a null comm pointer is not mistaken for
1009 + # missing GIN support.
1010 + probe = torch.zeros(1, device="cuda")
1011 + torch.distributed.all_reduce(probe, group=group)
1012 +
1013 + gin_type = query_nccl_gin_type(group)
1014 + if gin_type is None:
1015 + raise RuntimeError(
1016 + "DeepEPv2 communicator properties query failed; "
1017 + "networking capability could not be determined."
1018 + )
1019 + if gin_type == 0:
1020 + raise RuntimeError(
1021 + "DeepEPv2 requires NCCL GIN (GPU-Initiated Networking). "
1022 + "This usually means IBGDA-capable InfiniBand NICs or drivers "
1023 + "are not available. See tools/ep_kernels/README.md for "
1024 + "requirements."
1025 + )
1026 +
1027 def get_handle(self, kwargs):
1028 import deep_ep # type: ignore[import-not-found]
1029
1030 num_experts = kwargs.pop("num_experts", 256)
1031 buffer_kwargs = self._make_all2all_kwargs(**kwargs)
1032 + if not self._gin_checked:
1033 + self._check_gin_support(buffer_kwargs["group"])
1034 + self._gin_checked = True
1035 logger.debug("DeepEP v2 all2all args %s", buffer_kwargs)
1036 handle: deep_ep.ElasticBuffer = self.handle_cache.get_or_create(
1037 buffer_kwargs, deep_ep.ElasticBuffer
vllm/distributed/device_communicators/pynccl_allocator.py
+6 -2
@@ -14,7 +14,7 @@ from vllm import envs
14 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
15 from vllm.logger import init_logger
16 from vllm.platforms import current_platform
17 -from vllm.utils.nccl import find_nccl_include_paths
17 +from vllm.utils.nccl import find_nccl_include_paths, find_nccl_library_paths
18
19 logger = init_logger(__name__)
20
@@ -74,11 +74,15 @@ def compile_nccl_allocator():
74 out_dir = tempfile.gettempdir()
75 nccl_allocator_libname = "nccl_allocator"
76 nccl_include_paths = find_nccl_include_paths()
77 + ldflags = ["-l:libnccl.so.2"]
78 + nccl_lib_paths = find_nccl_library_paths()
79 + if nccl_lib_paths:
80 + ldflags = [f"-L{p}" for p in nccl_lib_paths] + ldflags
81 load_inline(
82 name=nccl_allocator_libname,
83 cpp_sources=nccl_allocator_source,
84 with_cuda=True,
81 - extra_ldflags=["-lnccl"],
85 + extra_ldflags=ldflags,
86 verbose=envs.VLLM_LOGGING_LEVEL == "DEBUG",
87 is_python_module=False,
88 build_directory=out_dir,
vllm/distributed/device_communicators/pynccl_wrapper.py
+29
@@ -51,6 +51,26 @@ class ncclUniqueId(ctypes.Structure):
51 _fields_ = [("internal", ctypes.c_byte * 128)]
52
53
54 +# NCCL 2.30+ ncclCommProperties_t. Only fields through ginType are read;
55 +# trailing fields keep the layout aligned with NCCL's versioned structure.
56 +class ncclCommProperties(ctypes.Structure):
57 + _fields_ = [
58 + ("size", ctypes.c_size_t),
59 + ("magic", ctypes.c_uint),
60 + ("version", ctypes.c_uint),
61 + ("rank", ctypes.c_int),
62 + ("nRanks", ctypes.c_int),
63 + ("cudaDev", ctypes.c_int),
64 + ("nvmlDev", ctypes.c_int),
65 + ("deviceApiSupport", ctypes.c_bool),
66 + ("multimemSupport", ctypes.c_bool),
67 + ("ginType", ctypes.c_int),
68 + ("nLsaTeams", ctypes.c_int),
69 + ("hostRmaSupport", ctypes.c_bool),
70 + ("railedGinType", ctypes.c_int),
71 + ]
72 +
73 +
74 cudaStream_t = ctypes.c_void_p
75 buffer_type = ctypes.c_void_p
76
@@ -317,6 +337,12 @@ class NCCLLibrary:
337 # ncclResult_t ncclCommWindowDeregister(
338 # ncclComm_t comm, ncclWindow_t win);
339 Function("ncclCommWindowDeregister", ncclResult_t, [ncclComm_t, ncclWindow_t]),
340 + # Query runtime properties of a specific initialized communicator.
341 + Function(
342 + "ncclCommQueryProperties",
343 + ncclResult_t,
344 + [ncclComm_t, ctypes.POINTER(ncclCommProperties)],
345 + ),
346 ]
347
348 # class attribute to store the mapping from the path to the library
@@ -375,6 +401,9 @@ class NCCLLibrary:
401 # Having an exception here on ROCm platform is
402 # not allowed during graph capturing
403 continue
404 + elif func.name == "ncclCommQueryProperties":
405 + # Optional on NCCL versions older than 2.29.
406 + continue
407 raise
408 NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
409 self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
vllm/utils/nccl.py
+62
@@ -3,6 +3,7 @@
3
4 from __future__ import annotations
5
6 +import ctypes
7 import importlib.util
8 import os
9
@@ -62,3 +63,64 @@ def find_nccl_include_paths() -> list[str] | None:
63 out.append(p)
64 seen.add(p)
65 return out or None
66 +
67 +
68 +def find_nccl_library_paths() -> list[str] | None:
69 + """Return possible library paths containing `libnccl.so`.
70 +
71 + Looks inside the `nvidia-nccl-cuXX` pip package.
72 + """
73 + paths: list[str] = []
74 + try:
75 + spec = importlib.util.find_spec("nvidia.nccl")
76 + if spec and (locs := getattr(spec, "submodule_search_locations", None)):
77 + for loc in locs:
78 + lib_dir = os.path.join(loc, "lib")
79 + if os.path.isdir(lib_dir):
80 + paths.append(lib_dir)
81 + except Exception as e:
82 + logger.debug("Failed to find nccl library path from nvidia.nccl package: %s", e)
83 + return paths or None
84 +
85 +
86 +def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None:
87 + """Return the GIN type for an initialized group, or ``None`` on failure."""
88 + from vllm.distributed.device_communicators.pynccl_wrapper import (
89 + NCCLLibrary,
90 + ncclCommProperties,
91 + )
92 +
93 + try:
94 + backend = group._get_backend(torch.device("cuda"))
95 + # GIN is a property of this initialized communicator, not just the
96 + # NCCL version. ncclCommQueryProperties requires its ncclComm_t.
97 + comm_ptr = backend._comm_ptr()
98 + if comm_ptr == 0:
99 + return None
100 + except Exception:
101 + logger.warning(
102 + "Failed to extract NCCL comm pointer from process group",
103 + exc_info=True,
104 + )
105 + return None
106 +
107 + try:
108 + nccl = NCCLLibrary()
109 + query_fn = nccl._funcs.get("ncclCommQueryProperties")
110 + if query_fn is None:
111 + return None
112 +
113 + props = ncclCommProperties()
114 + ctypes.memset(ctypes.addressof(props), 0, ctypes.sizeof(props))
115 + props.size = ctypes.sizeof(props)
116 + props.magic = 0xCAFEBEEF
117 + props.version = nccl.ncclGetRawVersion()
118 + result = query_fn(ctypes.c_void_p(comm_ptr), ctypes.byref(props))
119 + except Exception:
120 + logger.warning("Failed to query NCCL communicator properties", exc_info=True)
121 + return None
122 +
123 + if result != 0:
124 + logger.warning("ncclCommQueryProperties returned error %d", result)
125 + return None
126 + return props.ginType