A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
Relocate Encoder CUDA graph manager (#38116)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
Woosuk Kwon committed
Mar 25, 2026 at 20:52 UTC
144030c84ebf03b4f911ad8bab6432729418729c
4 files changed
+4
-6
tests/v1/cudagraph/test_encoder_cudagraph.py
+2
-2
@@ -14,8 +14,6 @@ from typing import Any
14
15
import pytest
16
import torch
17
-
18
-from vllm.platforms import current_platform
17
from vllm.v1.worker.gpu.mm.encoder_cudagraph import (
18
EncoderCudaGraphManager,
19
)
@@ -25,6 +23,8 @@ from vllm.v1.worker.gpu.mm.encoder_cudagraph_defs import (
23
EncoderCudaGraphReplayBuffers,
24
)
25
26
+from vllm.platforms import current_platform
27
+
28
# ---------------------------------------------------------------------------
29
# Helpers
30
# ---------------------------------------------------------------------------
vllm/v1/worker/encoder_cudagraph.py
renamed
vllm/v1/worker/encoder_cudagraph_defs.py
renamed
vllm/v1/worker/gpu_model_runner.py
+2
-4
@@ -208,7 +208,7 @@ from .utils import (
208
if TYPE_CHECKING:
209
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
210
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
211
- from vllm.v1.worker.gpu.mm.encoder_cudagraph import EncoderCudaGraphManager
211
+ from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
212
213
logger = init_logger(__name__)
214
@@ -5972,9 +5972,7 @@ class GPUModelRunner(
5972
SupportsEncoderCudaGraph,
5973
supports_encoder_cudagraph,
5974
)
5975
- from vllm.v1.worker.gpu.mm.encoder_cudagraph import (
5976
- EncoderCudaGraphManager,
5977
- )
5975
+ from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
5976
5977
raw_model = self.get_model()
5978
if supports_encoder_cudagraph(raw_model):