@tduncan / Nvidia-Optimization-Stack-VLLM / commits / de24e51908

[MM][V2] Enable encoder-only ViT CUDA graph capture (#56922)

Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Codex <noreply@openai.com>

jiangkuaixue123 committed Sep 17, 2026 at 16:12 UTC de24e5190820cc6640f2d55295405f228fffbc41

7 files changed +362 -5

.buildkite/test_areas/disaggregated_mooncake.yaml

+10

@@ -40,7 +40,14 @@ steps:

40 - vllm/v1/worker/gpu_worker.py

41 - vllm/v1/worker/gpu/ec_connector.py

42 - vllm/v1/worker/gpu/model_runner.py

43 + - vllm/v1/worker/mm_encoder_model_runner.py

44 + - vllm/v1/worker/encoder_cudagraph.py

45 + - vllm/v1/worker/encoder_cudagraph_defs.py

46 + - vllm/v1/attention/ops/vit_attn_wrappers.py

47 + - vllm/v1/worker/gpu/model_states/

48 - vllm/v1/worker/gpu/mm/

49 + - tests/v1/cudagraph/test_encoder_cudagraph.py

50 + - tests/models/multimodal/generation/test_vit_cudagraph.py

51 - examples/disaggregated/disaggregated_encoder/

52 - tests/v1/ec_connector/

53 - requirements/kv_connectors.txt

@@ -49,6 +56,9 @@ steps:

56 PYTHONUNBUFFERED: "1"

57 VLLM_HOST_IP: "127.0.0.1"

58 VLLM_USE_V2_MODEL_RUNNER: "1"

59 + E_CUDAGRAPH_MM_ENCODER: "1"

60 + # FA padding NaNs are tracked separately in #57136.

61 + E_MM_ENCODER_ATTN_BACKEND: "TRITON_ATTN"

62 MOONCAKE_EC_PROTOCOL: "tcp"

63 USE_MM_PROMPTS: "1"

64 SKIP_BASELINE: "0"

tests/models/multimodal/generation/test_vit_cudagraph.py

+115

@@ -499,3 +499,118 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):

499

500 # Ensure the output is a string

501 assert isinstance(output_text, str)

502 +

503 +

504 +def _check_eonly_encoder_outputs(worker, batches):

505 + """Compare real E-only replay with eager, retaining outputs across replay."""

506 + import torch

507 +

508 + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner

509 +

510 + runner = worker.model_runner

511 + assert isinstance(runner, MMEncoderModelRunner)

512 + manager = runner.model_state.encoder_runner.cudagraph_manager

513 + assert manager is not None and manager.is_captured()

514 + retained: list[tuple[torch.Tensor, torch.Tensor]] = []

515 + max_error = 0.0

516 + stats = []

517 + with torch.inference_mode():

518 + for batch in batches:

519 + kwargs = {key: value.to(runner.device) for key, value in batch.items()}

520 + actual = manager.execute(kwargs)

521 + expected = runner.model.embed_multimodal(**kwargs)

522 + assert len(actual) == len(expected)

523 + for output, reference in zip(actual, expected):

524 + assert torch.isfinite(output).all()

525 + # Two BF16 ulps at unit scale, fixed before model validation.

526 + torch.testing.assert_close(output, reference, rtol=0.016, atol=0.016)

527 + max_error = max(max_error, (output - reference).abs().max().item())

528 + for output, snapshot in retained:

529 + torch.testing.assert_close(output, snapshot, rtol=0, atol=0)

530 + retained.extend((output, output.clone()) for output in actual)

531 + stats.append(manager.get_cumulative_stats())

532 + assert stats[0]["graph_hits"] > 0

533 + assert stats[1]["graph_hits"] > stats[0]["graph_hits"]

534 + assert stats[2]["graph_misses"] > stats[1]["graph_misses"]

535 + assert stats[3]["graph_hits"] > stats[2]["graph_hits"]

536 + assert stats[4]["graph_hits"] > stats[3]["graph_hits"]

537 + assert stats[5]["graph_hits"] > stats[4]["graph_hits"]

538 + assert stats[6]["graph_hits"] > stats[5]["graph_hits"]

539 + return {"max_abs_error": max_error, "stats": stats}

540 +

541 +

542 +@pytest.mark.parametrize(

543 + "model_id",

544 + params_with_marks({key: MODEL_CONFIGS[key] for key in ("qwen3_vl", "qwen3_5")}),

545 +)

546 +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA graphs")

547 +def test_eonly_vit_cudagraph_outputs(

548 + model_id, vllm_runner, image_assets, video_assets, monkeypatch

549 +):

550 + """Exercise startup capture, image/video replay, fallback and output ownership."""

551 + import numpy as np

552 + from PIL import Image

553 + from transformers import AutoProcessor

554 +

555 + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")

556 + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

557 + config = MODEL_CONFIGS[model_id]

558 + token_budgets = [64, 256, 2048]

559 + processor = AutoProcessor.from_pretrained(config.model)

560 + first, second = [asset.pil_image for asset in image_assets]

561 + batches = [

562 + [first.resize((224, 224))],

563 + [second.resize((448, 224)), first.resize((224, 224))],

564 + [second.resize((1792, 1792))],

565 + [second.resize((224, 224))],

566 + [first.resize((1280, 720))],

567 + [first.resize((224, 224))],

568 + ]

569 + frames = sample_frames_from_video(video_assets[0].np_ndarrays, 2)

570 + video = np.stack(

571 + [np.asarray(Image.fromarray(frame).resize((224, 224))) for frame in frames]

572 + )

573 + with vllm_runner(

574 + config.model,

575 + dtype=config.dtype,

576 + mm_encoder_only=True,

577 + # FA padding NaNs are tracked separately in #57136.

578 + mm_encoder_attn_backend="FLASHINFER",

579 + enable_prefix_caching=False,

580 + max_model_len=4096,

581 + max_num_seqs=2,

582 + limit_mm_per_prompt={"image": 2, "video": 1},

583 + compilation_config={

584 + "cudagraph_mode": "NONE",

585 + "cudagraph_mm_encoder": True,

586 + "encoder_cudagraph_token_budgets": token_budgets,

587 + "encoder_cudagraph_max_vision_items_per_batch": 2,

588 + "encoder_cudagraph_max_frames_per_batch": 2,

589 + },

590 + ) as model:

591 + mm_config = model.llm.llm_engine.vllm_config.model_config.multimodal_config

592 + processor_kwargs = mm_config.merge_mm_processor_kwargs({})

593 + inputs = [

594 + dict(

595 + processor.image_processor(

596 + images=images, return_tensors="pt", **processor_kwargs

597 + )

598 + )

599 + for images in batches

600 + ]

601 + inputs.append(

602 + dict(

603 + processor.video_processor(

604 + videos=[video],

605 + do_sample_frames=False,

606 + return_tensors="pt",

607 + **processor_kwargs,

608 + )

609 + )

610 + )

611 + results = model.llm.collective_rpc(

612 + partial(_check_eonly_encoder_outputs, batches=inputs)

613 + )

614 + for result in results:

615 + assert result["stats"][0]["token_budgets"] == token_budgets

616 + print(f"E-only {model_id}: {results}")

tests/v1/cudagraph/test_encoder_cudagraph.py

+95

@@ -13,7 +13,9 @@ Test organization:

13 """

15 from collections.abc import Hashable

16 +from types import SimpleNamespace

17 from typing import Any

18 +from unittest.mock import MagicMock, patch

20 import pytest

21 import torch

@@ -575,6 +577,99 @@ class TestEncoderCudaGraphCaptureReplay:

577 assert out.shape == (4, _HIDDEN)

578

579

580 +# ---------------------------------------------------------------------------

581 +# E-only capture and output lifecycle

582 +# ---------------------------------------------------------------------------

583 +

584 +

585 +@pytest.mark.skipif(

586 + not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"

587 +)

588 +@pytest.mark.usefixtures("dist_init", "workspace_init")

589 +@pytest.mark.parametrize("profile_only", [False, True])

590 +@torch.inference_mode()

591 +def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only):

592 + """The E-only entry captures only the encoder and preserves cached outputs."""

593 + from vllm.distributed.ec_transfer.ec_connector.base import (

594 + ECConnectorBase,

595 + ECConnectorMetadata,

596 + )

597 + from vllm.v1.worker.gpu.ec_connector import ActiveECConnector

598 + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner

599 + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner

600 + from vllm.v1.worker.workspace import lock_workspace

601 +

602 + device = torch.device("cuda:0")

603 + dtype = torch.float16

604 + model = SimpleMockViTModel().to(device).half()

605 + manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype)

606 + encoder = object.__new__(EncoderRunner)

607 + encoder.device = device

608 + encoder.cudagraph_manager = manager

609 + runner = object.__new__(MMEncoderModelRunner)

610 + runner.model_state = SimpleNamespace(encoder_runner=encoder)

611 + # No decoder manager is installed: capture must be encoder-only.

612 + with patch(

613 + "vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace

614 + ) as lock:

615 + runner.capture_model(profile_only=profile_only)

616 + assert lock.call_count == int(not profile_only)

617 + assert len(manager.budget_graphs["default"]) == len(_BUDGETS)

618 +

619 + cache: dict[str, torch.Tensor] = {}

620 + pending_sends: dict[str, torch.Tensor] = {}

621 + connector = MagicMock(spec=ECConnectorBase)

622 + connector.is_producer = True

623 + connector.is_consumer = False

624 + connector.get_finished.return_value = (None, None)

625 + connector.save_caches.side_effect = lambda *, encoder_cache, mm_hash: (

626 + pending_sends.update({mm_hash: encoder_cache[mm_hash]})

627 + )

628 + with patch(

629 + "vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector

630 + ):

631 + ec = ActiveECConnector(SimpleNamespace(), cache)

632 + scheduled = SimpleNamespace(

633 + ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()

634 + )

635 + saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = []

636 + # Mixed sizes exercise packing order; 64 is the boundary, 81 falls back.

637 + for grids in ([[1, 8, 8], [1, 4, 4]], [[1, 16, 16]], [[1, 18, 18]], [[1, 4, 4]]):

638 + inputs = _make_mm_kwargs(grids, device, dtype)

639 + expected = model.encoder_eager_forward(inputs).split(

640 + [t * (h // 2) * (w // 2) for t, h, w in grids]

641 + )

642 + with ec.maybe_get_output(scheduled) as ec_output:

643 + outputs = manager.execute(inputs)

644 + assert outputs is not None

645 + cache[str(len(cache))] = outputs[0]

646 + assert ec_output.finished_sending is None

647 + assert outputs is not None

648 + for actual, eager in zip(outputs, expected):

649 + torch.testing.assert_close(actual, eager)

650 + for previous, snapshot in saved_outputs:

651 + torch.testing.assert_close(previous, snapshot, rtol=0, atol=0)

652 + saved_outputs.extend((output, output.clone()) for output in outputs)

653 + assert manager.graph_hits == 4

654 + assert manager.graph_misses == 1

655 + assert pending_sends.keys() == cache.keys()

656 + connector.get_finished.return_value = (set(cache), None)

657 + with ec.maybe_get_output(scheduled) as ec_output:

658 + pass

659 + assert ec_output.finished_sending == set(cache)

660 +

661 +

662 +def test_eonly_without_encoder_graph_skips_capture():

663 + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner

664 + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner

665 +

666 + encoder = object.__new__(EncoderRunner)

667 + encoder.cudagraph_manager = None

668 + runner = object.__new__(MMEncoderModelRunner)

669 + runner.model_state = SimpleNamespace(encoder_runner=encoder)

670 + assert runner.capture_model() == 0

671 +

672 +

673 # ---------------------------------------------------------------------------

674 # SimpleMockViTVideoModel — extends SimpleMockViTModel with video support

675 # ---------------------------------------------------------------------------

tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh

+35 -1

@@ -20,6 +20,8 @@

20 # SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE

21 # CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)

22 # MAX_MODEL_LEN context length (default 16384)

23 +# E_CUDAGRAPH_MM_ENCODER enable encoder CUDA graphs (default 0)

24 +# E_MM_ENCODER_ATTN_BACKEND optional encoder attention backend override

26 set -euo pipefail

@@ -35,6 +37,15 @@ if [[ "$USE_MM_PROMPTS" == "1" ]]; then

37 TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)

38 fi

39 MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"

40 +E_ATTN_ARGS=()

41 +if [[ -n "${E_MM_ENCODER_ATTN_BACKEND:-}" ]]; then

42 + E_ATTN_ARGS+=(--mm-encoder-attn-backend "$E_MM_ENCODER_ATTN_BACKEND")

43 +fi

44 +case "${E_CUDAGRAPH_MM_ENCODER:-0}" in

45 + 0) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":false}' ;;

46 + 1) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":true}' ;;

47 + *) echo "E_CUDAGRAPH_MM_ENCODER must be 0 or 1"; exit 1 ;;

48 +esac

50 GPU_SINGLE="${GPU_SINGLE:-0}"

51 GPU_E="${GPU_E:-0}"

@@ -203,7 +214,7 @@ run_epd_mooncake() {

214 cleanup_instances

215

216 echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"

206 - CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \

217 + VLLM_SERVER_DEV_MODE=1 CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \

218 --port "$ENCODE_PORT" \

219 --gpu-memory-utilization 0.35 \

220 --mm-tensor-ipc torch_shm \

@@ -215,6 +226,9 @@ run_epd_mooncake() {

226 --limit-mm-per-prompt '{"image":2,"video":0}' \

227 --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \

228 --ec-transfer-config "$ENC_EC_JSON" \

229 + --compilation-config "$E_COMPILATION_CONFIG" \

230 + "${E_ATTN_ARGS[@]}" \

231 + --worker-extension-cls tests.v1.ec_connector.integration.test_epd_correctness.EncoderGraphTestWorkerExtension \

232 >"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &

233 local ENCODER_PID=$!

234 PIDS+=("$ENCODER_PID")

@@ -266,6 +280,26 @@ run_epd_mooncake() {

280 --baseline_file "$BASELINE_FILE" \

281 "${TEST_ARGS[@]}"

282

283 + curl -fsS "http://localhost:${ENCODE_PORT}/collective_rpc" \

284 + -H 'Content-Type: application/json' \

285 + -d '{"method":"encoder_graph_stats"}' \

286 + >"${LOG_PATH}/encoder_graph_stats.json"

287 + "$PYTHON_BIN" - "${LOG_PATH}/encoder_graph_stats.json" "${E_CUDAGRAPH_MM_ENCODER:-0}" <<'PY'

288 +import json

289 +import sys

290 +

291 +with open(sys.argv[1]) as f:

292 + results = json.load(f)["results"]

293 +assert results, "No encoder workers returned graph statistics"

294 +for stats in results:

295 + if sys.argv[2] == "1":

296 + assert stats["is_captured"], stats

297 + assert stats["graph_hits"] > 0, stats

298 + else:

299 + assert stats["graph_hits"] == 0, stats

300 +print("Encoder graph statistics:", results)

301 +PY

302 +

303 cleanup_instances

304 }

305

tests/v1/ec_connector/integration/test_epd_correctness.py

+21

@@ -26,6 +26,7 @@ import json

26 import os

27 import time

28 from concurrent.futures import ThreadPoolExecutor

29 +from typing import TYPE_CHECKING

31 import openai

32 import requests

@@ -33,6 +34,26 @@ import requests

34 from vllm.assets.image import ImageAsset

35 from vllm.multimodal.utils import encode_image_url

37 +if TYPE_CHECKING:

38 + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner

39 +

40 +

41 +class EncoderGraphTestWorkerExtension:

42 + """Expose encoder replay evidence through the existing test RPC endpoint."""

43 +

44 + model_runner: "MMEncoderModelRunner"

45 +

46 + def encoder_graph_stats(self):

47 + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner

48 +

49 + runner = self.model_runner

50 + assert isinstance(runner, MMEncoderModelRunner)

51 + manager = runner.model_state.encoder_runner.cudagraph_manager

52 + if manager is None:

53 + return {"is_captured": False, "graph_hits": 0, "graph_misses": 0}

54 + return {"is_captured": manager.is_captured(), **manager.get_cumulative_stats()}

55 +

56 +

57 MAX_OUTPUT_LEN = 256

59 # Sample prompts with multimodal content

tests/v1/worker/test_encoder_runner.py

+54 -1

@@ -9,12 +9,15 @@ and tolerated (token-embedding fallback) when it is not, while a miss within

9 the processed range still fails loudly.

10 """

12 -from unittest.mock import MagicMock

12 +from types import SimpleNamespace

13 +from unittest.mock import MagicMock, patch

15 import numpy as np

16 import pytest

17 import torch

19 +from vllm.config.compilation import CUDAGraphMode

20 +from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph

21 from vllm.multimodal.inputs import (

22 MultiModalFeatureSpec,

23 MultiModalFieldElem,

@@ -31,6 +34,56 @@ pytestmark = pytest.mark.cpu_test

34 HIDDEN = 4

37 +@pytest.mark.parametrize(

38 + "enabled,enforce_eager,supported,expected",

39 + [

40 + (True, False, True, True),

41 + (False, False, True, False),

42 + (True, True, True, False),

43 + (True, False, False, False),

44 + ],

45 +)

46 +def test_encoder_graph_configuration(enabled, enforce_eager, supported, expected):

47 + """Encoder graph eligibility is independent of decoder graph mode."""

48 + config = MagicMock()

49 + config.model_config.enforce_eager = enforce_eager

50 + config.model_config.dtype = torch.float32

51 + config.model_config.get_inputs_embeds_size.return_value = HIDDEN

52 + config.scheduler_config.max_num_batched_tokens = 8

53 + config.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

54 + config.compilation_config.cudagraph_mm_encoder = enabled

55 + model = MagicMock(spec=SupportsEncoderCudaGraph) if supported else torch.nn.Module()

56 + state = _model_state(EncoderCache())

57 + with patch("vllm.v1.worker.gpu.model_states.interface.EncoderCudaGraphManager"):

58 + ModelState.__init__(state, config, model, state.encoder_cache, state.device)

59 + assert state.encoder_runner.has_cudagraph() is expected

60 +

61 +

62 +def test_encoder_graph_unsupported_modality_uses_eager_output():

63 + """An image graph must not intercept another modality's encoder output."""

64 + from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager

65 +

66 + runner = _make_runner([], [])

67 + output = torch.ones(2, HIDDEN)

68 + runner.model = MagicMock()

69 + runner.model.embed_multimodal.return_value = [output]

70 + manager = MagicMock(spec=EncoderCudaGraphManager)

71 + manager.config = SimpleNamespace(modalities=["image"])

72 + manager.is_captured.return_value = True

73 + manager.supports_modality.side_effect = lambda modality: (

74 + EncoderCudaGraphManager.supports_modality(manager, modality)

75 + )

76 + runner.cudagraph_manager = manager

77 + with patch(

78 + "vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs",

79 + return_value=[("audio", 1, {"audio_values": torch.zeros(2)})],

80 + ):

81 + result = runner.execute_mm_encoder([])

82 + assert len(result) == 1

83 + assert result[0] is output

84 + manager.execute.assert_not_called()

85 +

86 +

87 def _model_state(cache: EncoderCache) -> MagicMock:

88 """A mock ModelState backed by a real EncoderCache."""

89 state = MagicMock()

vllm/v1/worker/mm_encoder_model_runner.py

+32 -3

@@ -4,9 +4,10 @@

5 An encoder-only instance -- `--mm-encoder-only`, or the producer side of

6 encoder-cache disaggregation -- encodes the multi-modal items and publishes the

7 -embeddings. It runs no language model: no KV cache, no sampler, no CUDA graphs.

7 +embeddings. It runs no language model: no KV cache, sampler, or decoder graphs.

8 """

10 +import time

11 from collections.abc import Iterator

12 from contextlib import contextmanager

13 from typing import TYPE_CHECKING, Any

@@ -15,7 +16,9 @@ import torch

17 from vllm.config import VllmConfig

18 from vllm.config.compilation import CUDAGraphMode

19 +from vllm.logger import init_logger

20 from vllm.sequence import IntermediateTensors

21 +from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture

22 from vllm.v1.kv_cache_interface import KVCacheSpec

23 from vllm.v1.outputs import (

24 ModelRunnerOutput,

@@ -24,10 +27,13 @@ from vllm.v1.outputs import (

27 from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor

28 from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras

29 from vllm.v1.worker.gpu.model_runner import GPUModelRunner

30 +from vllm.v1.worker.workspace import lock_workspace

32 if TYPE_CHECKING:

33 from vllm.v1.core.sched.output import SchedulerOutput

35 +logger = init_logger(__name__)

36 +

38 class MMEncoderModelRunner(GPUModelRunner):

39 """Encoder-only variant of the V2 GPU model runner."""

@@ -67,8 +73,31 @@ class MMEncoderModelRunner(GPUModelRunner):

73 def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:

74 return {}

76 + @torch.inference_mode()

77 def capture_model(self, *, profile_only: bool = False) -> int:

71 - return 0

78 + encoder_runner = self.model_state.encoder_runner

79 + if not encoder_runner.has_cudagraph():

80 + return 0

81 +

82 + start_time = time.perf_counter()

83 + with freeze_gc_for_cudagraph_capture():

84 + torch.accelerator.empty_cache()

85 + start_free_memory = torch.accelerator.get_memory_info()[0]

86 + encoder_runner.capture()

87 + graph_memory = start_free_memory - torch.accelerator.get_memory_info()[0]

88 +

89 + if not profile_only:

90 + lock_workspace()

91 +

92 + manager = encoder_runner.cudagraph_manager

93 + assert manager is not None

94 + logger.info(

95 + "Encoder graph capturing finished in %.0f secs, took %.2f GiB (%d graphs)",

96 + time.perf_counter() - start_time,

97 + graph_memory / (1 << 30),

98 + manager.get_num_graphs_to_capture(),

99 + )

100 + return graph_memory

101

102 def needs_cudagraph_capture(self) -> bool:

103 return False

@@ -114,7 +143,7 @@ class MMEncoderModelRunner(GPUModelRunner):

143

144 batch_req_state, _ = self.gather_batch_req_state(scheduler_output, False)

145 assert batch_req_state is not None

117 - # No CUDA graph, and no DP peer to agree a padded shape with.

146 + # No decoder graph or DP padding; encoder graphs run independently.

147 batch_desc = BatchExecutionDescriptor(

148 cg_mode=CUDAGraphMode.NONE,

149 num_tokens=batch_req_state.num_tokens,