d215d1efca · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/benchmark_long_document_qa_throughput.py

+1 -2

@@ -42,7 +42,6 @@ details.

42

43 import random

44 import time

45 -from dataclasses import fields

45

46 from vllm import LLM, SamplingParams

47 from vllm.engine.arg_utils import EngineArgs

@@ -124,7 +123,7 @@ def main(args):

123

124 # Create the LLM engine

125 engine_args = EngineArgs.from_cli_args(args)

127 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

126 + llm = LLM.from_engine_args(engine_args)

127 sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)

128

129 print("------warm up------")

benchmarks/benchmark_prefix_caching.py

+1 -2

@@ -32,7 +32,6 @@ import dataclasses

32 import json

33 import random

34 import time

35 -from dataclasses import fields

35

36 from transformers import PreTrainedTokenizerBase

37

@@ -197,7 +196,7 @@ def main(args):

196

197 engine_args = EngineArgs.from_cli_args(args)

198

200 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

199 + llm = LLM.from_engine_args(engine_args)

200

201 sampling_params = SamplingParams(

202 temperature=0,

benchmarks/benchmark_prioritization.py

+1 -2

@@ -6,7 +6,6 @@ import argparse

6 import json

7 import random

8 import time

9 -from dataclasses import fields

9

10 from transformers import AutoTokenizer, PreTrainedTokenizerBase

11

@@ -79,7 +78,7 @@ def run_vllm(

78 ) -> float:

79 from vllm import LLM, SamplingParams

80

82 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

81 + llm = LLM.from_engine_args(engine_args)

82

83 assert all(

84 llm.llm_engine.model_config.max_model_len >= (request[1] + request[2])

examples/offline_inference/audio_language.py

+1 -2

@@ -9,7 +9,6 @@ on HuggingFace model repository.

9 """

10

11 import os

12 -from dataclasses import asdict

12 from typing import Any, NamedTuple

13

14 from huggingface_hub import snapshot_download

@@ -633,7 +632,7 @@ def main(args):

632 req_data.engine_args.limit_mm_per_prompt or {}

633 )

634

636 - engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

635 + engine_args = vars(req_data.engine_args) | {"seed": args.seed}

636 if args.tensor_parallel_size is not None:

637 engine_args["tensor_parallel_size"] = args.tensor_parallel_size

638 llm = LLM(**engine_args)

examples/offline_inference/encoder_decoder_multimodal.py

+5 -7

@@ -8,7 +8,6 @@ the explicit/implicit prompt format on enc-dec LMMs for text generation.

8 import os

9 import time

10 from collections.abc import Sequence

11 -from dataclasses import asdict

11 from typing import NamedTuple

12

13 from vllm import LLM, EngineArgs, PromptType, SamplingParams

@@ -91,13 +90,12 @@ def main(args):

90 req_data = model_example_map[model]()

91

92 # Disable other modalities to save memory

93 + engine_args = req_data.engine_args

94 default_limits = {"image": 0, "video": 0, "audio": 0}

95 - req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

96 - req_data.engine_args.limit_mm_per_prompt or {}

97 - )

98 -

99 - engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

100 - llm = LLM(**engine_args)

95 + limit_mm_per_prompt = default_limits | (engine_args.limit_mm_per_prompt or {})

96 + engine_args.limit_mm_per_prompt = limit_mm_per_prompt

97 + engine_args.seed = args.seed

98 + llm = LLM.from_engine_args(engine_args)

99

100 prompts = req_data.prompts

101

examples/offline_inference/load_sharded_state.py

+1 -3

@@ -20,8 +20,6 @@ python load_sharded_state.py \

20 --max-tokens 50

21 """

22

23 -import dataclasses

24 -

23 from vllm import LLM, EngineArgs, SamplingParams

24 from vllm.utils.argparse_utils import FlexibleArgumentParser

25

@@ -64,7 +62,7 @@ def main():

62 print(f"Tensor parallel size: {engine_args.tensor_parallel_size}")

63

64 # Load the model using engine args

67 - llm = LLM(**dataclasses.asdict(engine_args))

65 + llm = LLM.from_engine_args(engine_args)

66

67 # Prepare sampling parameters

68 sampling_params = SamplingParams(

examples/offline_inference/save_sharded_state.py

+1 -2

@@ -21,7 +21,6 @@ llm = LLM(

21 )

22 """

23

24 -import dataclasses

24 import os

25 import shutil

26 from pathlib import Path

@@ -60,7 +59,7 @@ def main(args):

59 if not Path(model_path).is_dir():

60 raise ValueError("model path must be a local directory")

61 # Create LLM instance from arguments

63 - llm = LLM(**dataclasses.asdict(engine_args))

62 + llm = LLM.from_engine_args(engine_args)

63 # Prepare output directory

64 Path(args.output).mkdir(exist_ok=True)

65 # Dump worker states to output directory

examples/offline_inference/vision_language.py

+6 -7

@@ -11,7 +11,6 @@ on HuggingFace model repository.

11 import os

12 import random

13 from contextlib import contextmanager

14 -from dataclasses import asdict

14 from typing import NamedTuple

15

16 from huggingface_hub import snapshot_download

@@ -2434,13 +2433,13 @@ def main(args):

2433 req_data.engine_args.limit_mm_per_prompt or {}

2434 )

2435

2437 - engine_args = asdict(req_data.engine_args) | {

2438 - "seed": args.seed,

2439 - "mm_processor_cache_gb": 0 if args.disable_mm_processor_cache else 4,

2440 - }

2436 + engine_args = req_data.engine_args

2437 + engine_args.seed = args.seed

2438 + mm_processor_cache_gb = 0 if args.disable_mm_processor_cache else 4

2439 + engine_args.mm_processor_cache_gb = mm_processor_cache_gb

2440 if args.tensor_parallel_size is not None:

2442 - engine_args["tensor_parallel_size"] = args.tensor_parallel_size

2443 - llm = LLM(**engine_args)

2441 + engine_args.tensor_parallel_size = args.tensor_parallel_size

2442 + llm = LLM.from_engine_args(engine_args)

2443

2444 # Don't want to check the flag multiple times, so just hijack `prompts`.

2445 prompts = (

examples/offline_inference/vision_language_multi_image.py

+8 -7

@@ -8,7 +8,6 @@ using the chat template defined by the model.

8

9 import os

10 from argparse import Namespace

11 -from dataclasses import asdict

11 from typing import NamedTuple

12

13 from huggingface_hub import snapshot_download

@@ -1481,10 +1480,11 @@ def run_generate(

1480 ):

1481 req_data = model_example_map[model](question, image_urls)

1482

1484 - engine_args = asdict(req_data.engine_args) | {"seed": seed}

1483 + engine_args = req_data.engine_args

1484 + engine_args.seed = seed

1485 if tensor_parallel_size is not None:

1486 - engine_args["tensor_parallel_size"] = tensor_parallel_size

1487 - llm = LLM(**engine_args)

1486 + engine_args.tensor_parallel_size = tensor_parallel_size

1487 + llm = LLM.from_engine_args(engine_args)

1488

1489 sampling_params = SamplingParams(

1490 temperature=0.0, max_tokens=256, stop_token_ids=req_data.stop_token_ids

@@ -1521,10 +1521,11 @@ def run_chat(

1521 req_data.engine_args.limit_mm_per_prompt or {}

1522 )

1523

1524 - engine_args = asdict(req_data.engine_args) | {"seed": seed}

1524 + engine_args = req_data.engine_args

1525 + engine_args.seed = seed

1526 if tensor_parallel_size is not None:

1526 - engine_args["tensor_parallel_size"] = tensor_parallel_size

1527 - llm = LLM(**engine_args)

1527 + engine_args.tensor_parallel_size = tensor_parallel_size

1528 + llm = LLM.from_engine_args(engine_args)

1529

1530 sampling_params = (

1531 SamplingParams(

examples/pooling/embed/vision_embedding_offline.py

+19 -25

@@ -10,12 +10,11 @@ on HuggingFace model repository.

10 """

11

12 import argparse

13 -from dataclasses import asdict

13 from pathlib import Path

14

15 from PIL.Image import Image

16

18 -from vllm import LLM, EngineArgs

17 +from vllm import LLM

18 from vllm.multimodal.utils import fetch_image

19 from vllm.utils.print_utils import print_embeddings

20

@@ -28,14 +27,13 @@ multi_modal_data = {"image": fetch_image(image_url)}

27

28

29 def run_clip(seed: int):

31 - engine_args = EngineArgs(

30 + llm = LLM(

31 model="openai/clip-vit-base-patch32",

32 runner="pooling",

33 limit_mm_per_prompt={"image": 1},

34 + seed=seed,

35 )

36

37 - llm = LLM(**asdict(engine_args) | {"seed": seed})

38 -

37 print("Text embedding output:")

38 outputs = llm.embed(text, use_tqdm=False)

39 print_embeddings(outputs[0].outputs.embedding)

@@ -53,15 +51,14 @@ def run_clip(seed: int):

51

52

53 def run_e5_v(seed: int):

56 - engine_args = EngineArgs(

54 + llm = LLM(

55 model="royokong/e5-v",

56 runner="pooling",

57 max_model_len=4096,

58 limit_mm_per_prompt={"image": 1},

59 + seed=seed,

60 )

61

63 - llm = LLM(**asdict(engine_args) | {"seed": seed})

64 -

62 llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501

63

64 print("Text embedding output:")

@@ -108,20 +105,20 @@ def run_qwen3_vl(seed: int):

105

106 multi_modal_data["image"] = post_process_image(multi_modal_data["image"])

107

111 - engine_args = EngineArgs(

112 - model="Qwen/Qwen3-VL-Embedding-2B",

113 - runner="pooling",

114 - max_model_len=8192,

115 - limit_mm_per_prompt={"image": 1},

116 - mm_processor_kwargs={"do_resize": False} if smart_resize is not None else None,

117 - )

108 default_instruction = "Represent the user's input."

109 image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

110 prompt_text = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant\n"

111 prompt_image = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}<|im_end|>\n<|im_start|>assistant\n"

112 prompt_image_text = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}{text}<|im_end|>\n<|im_start|>assistant\n"

113

124 - llm = LLM(**asdict(engine_args) | {"seed": seed})

114 + llm = LLM(

115 + model="Qwen/Qwen3-VL-Embedding-2B",

116 + runner="pooling",

117 + max_model_len=8192,

118 + limit_mm_per_prompt={"image": 1},

119 + mm_processor_kwargs={"do_resize": False} if smart_resize is not None else None,

120 + seed=seed,

121 + )

122

123 print("Text embedding output:")

124 outputs = llm.embed(prompt_text, use_tqdm=False)

@@ -149,14 +146,13 @@ def run_qwen3_vl(seed: int):

146

147

148 def run_siglip(seed: int):

152 - engine_args = EngineArgs(

149 + llm = LLM(

150 model="google/siglip-base-patch16-224",

151 runner="pooling",

152 limit_mm_per_prompt={"image": 1},

153 + seed=seed,

154 )

155

158 - llm = LLM(**asdict(engine_args) | {"seed": seed})

159 -

156 print("Text embedding output:")

157 outputs = llm.embed(text, use_tqdm=False)

158 print_embeddings(outputs[0].outputs.embedding)

@@ -174,16 +170,15 @@ def run_siglip(seed: int):

170

171

172 def run_vlm2vec_phi3v(seed: int):

177 - engine_args = EngineArgs(

173 + llm = LLM(

174 model="TIGER-Lab/VLM2Vec-Full",

175 runner="pooling",

176 max_model_len=4096,

177 trust_remote_code=True,

178 mm_processor_kwargs={"num_crops": 4},

179 limit_mm_per_prompt={"image": 1},

180 + seed=seed,

181 )

185 -

186 - llm = LLM(**asdict(engine_args) | {"seed": seed})

182 image_token = "<|image_1|>"

183

184 print("Text embedding output:")

@@ -259,7 +254,7 @@ def run_vlm2vec_qwen2vl(seed: int):

254 processor.save_pretrained(merged_path)

255 print("Done!")

256

262 - engine_args = EngineArgs(

257 + llm = LLM(

258 model=merged_path,

259 runner="pooling",

260 max_model_len=4096,

@@ -268,9 +263,8 @@ def run_vlm2vec_qwen2vl(seed: int):

263 "max_pixels": 12845056,

264 },

265 limit_mm_per_prompt={"image": 1},

266 + seed=seed,

267 )

272 -

273 - llm = LLM(**asdict(engine_args) | {"seed": seed})

268 image_token = "<|image_pad|>"

269

270 print("Text embedding output:")

examples/pooling/score/vision_reranker_offline.py

+1 -2

@@ -10,7 +10,6 @@ multimodal documents (text + images/videos).

10

11 from argparse import Namespace

12 from collections.abc import Callable

13 -from dataclasses import asdict

13 from pathlib import Path

14 from typing import NamedTuple

15

@@ -125,7 +124,7 @@ def main(args: Namespace):

124 model_request = model_example_map[args.model_name]()

125 engine_args = model_request.engine_args

126

128 - llm = LLM(**asdict(engine_args))

127 + llm = LLM.from_engine_args(engine_args)

128

129 print("Query: string & Document: string")

130 outputs = llm.score(query, document)

tests/compile/test_config.py

+6 -2

@@ -414,9 +414,12 @@ def test_cudagraph_sizes_post_init(

414 ctx,

415 patch("vllm.config.parallel.cuda_device_count_stateless", return_value=tp_size),

416 ):

417 + kwargs = {}

418 + if cudagraph_capture_sizes is not None:

419 + kwargs["cudagraph_capture_sizes"] = cudagraph_capture_sizes

420 + if max_cudagraph_capture_size is not None:

421 + kwargs["max_cudagraph_capture_size"] = max_cudagraph_capture_size

422 compilation_config = CompilationConfig(

418 - cudagraph_capture_sizes=cudagraph_capture_sizes,

419 - max_cudagraph_capture_size=max_cudagraph_capture_size,

423 pass_config=PassConfig(

424 enable_sp=enable_sp,

425 fuse_norm_quant=True,

@@ -425,6 +428,7 @@ def test_cudagraph_sizes_post_init(

428 sp_min_token_num=512 if enable_sp else None,

429 ),

430 cudagraph_mode=cudagraph_mode,

431 + **kwargs,

432 )

433 engine_args = EngineArgs(

434 model="facebook/opt-125m",

tests/entrypoints/offline_mode/test_offline_mode.py

+1 -4

@@ -2,7 +2,6 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """Tests for HF_HUB_OFFLINE mode"""

4

5 -import dataclasses

5 import importlib

6 import sys

7

@@ -12,7 +11,6 @@ import urllib3

11

12 from vllm import LLM

13 from vllm.distributed import cleanup_dist_env_and_memory

15 -from vllm.engine.arg_utils import EngineArgs

14

15 MODEL_CONFIGS = [

16 {

@@ -160,8 +158,7 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):

158 # Need to re-import huggingface_hub

159 # and friends to set up offline mode

160 _re_import_modules()

163 - engine_args = EngineArgs(model="facebook/opt-125m")

164 - LLM(**dataclasses.asdict(engine_args))

161 + LLM(model="facebook/opt-125m")

162 finally:

163 # Reset the environment after the test

164 # NB: Assuming tests are run in online mode

tests/models/multimodal/generation/test_keye.py

+8 -11

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from dataclasses import asdict

3 from typing import NamedTuple

4

5 import pytest

@@ -29,14 +28,6 @@ def test_keye_vl(image_assets, question: str):

28 images = [asset.pil_image for asset in image_assets]

29 image_urls = [encode_image_url(image) for image in images]

30

32 - engine_args = EngineArgs(

33 - model=MODEL_NAME,

34 - trust_remote_code=True,

35 - max_model_len=8192,

36 - max_num_seqs=5,

37 - limit_mm_per_prompt={"image": len(image_urls)},

38 - )

39 -

31 placeholders = [{"type": "image", "image": url} for url in image_urls]

32 messages = [

33 {

@@ -54,8 +45,14 @@ def test_keye_vl(image_assets, question: str):

45 messages, tokenize=False, add_generation_prompt=True

46 )

47

57 - engine_args = asdict(engine_args) | {"seed": 42}

58 - llm = LLM(**engine_args)

48 + llm = LLM(

49 + model=MODEL_NAME,

50 + trust_remote_code=True,

51 + max_model_len=8192,

52 + max_num_seqs=5,

53 + limit_mm_per_prompt={"image": len(image_urls)},

54 + seed=42,

55 + )

56

57 sampling_params = SamplingParams(

58 temperature=0.0, max_tokens=256, stop_token_ids=None

tests/models/multimodal/generation/test_vit_backend_functionality.py

+5 -10

@@ -7,13 +7,12 @@ This test validates that each multimodal model can successfully generate outputs

7 using different ViT attention backends. Tests are parametrized by model and backend.

8 """

9

10 -from dataclasses import asdict

10 from typing import Any

11

12 import pytest

13 from transformers import AutoProcessor

14

16 -from vllm import LLM, EngineArgs, SamplingParams

15 +from vllm import LLM, SamplingParams

16 from vllm.multimodal.utils import encode_image_url

17 from vllm.multimodal.video import sample_frames_from_video

18 from vllm.platforms import current_platform

@@ -274,7 +273,7 @@ def run_llm_generate_test(config, mm_encoder_attn_backend, image_assets):

273 limit_mm_per_prompt = config.get("limit_mm_per_prompt", {"image": len(images)})

274

275 # Create engine

277 - engine_args = EngineArgs(

276 + llm = LLM(

277 model=config["model_name"],

278 trust_remote_code=True,

279 max_model_len=config["max_model_len"],

@@ -283,11 +282,9 @@ def run_llm_generate_test(config, mm_encoder_attn_backend, image_assets):

282 mm_encoder_attn_backend=mm_encoder_attn_backend,

283 hf_overrides=dummy_hf_overrides,

284 load_format="dummy",

285 + seed=42,

286 )

287

288 - engine_dict = asdict(engine_args) | {"seed": 42}

289 - llm = LLM(**engine_dict)

290 -

288 # Generate

289 sampling_params = SamplingParams(**config["sampling_params"])

290 outputs = llm.generate(

@@ -318,7 +315,7 @@ def run_llm_chat_test(config, mm_encoder_attn_backend, image_assets):

315 messages = build_dots_ocr_prompt([stop_sign_image], config)

316

317 # Create engine

321 - engine_args = EngineArgs(

318 + llm = LLM(

319 model=config["model_name"],

320 trust_remote_code=True,

321 max_model_len=config["max_model_len"],

@@ -327,11 +324,9 @@ def run_llm_chat_test(config, mm_encoder_attn_backend, image_assets):

324 mm_encoder_attn_backend=mm_encoder_attn_backend,

325 hf_overrides=dummy_hf_overrides,

326 load_format="dummy",

327 + seed=42,

328 )

329

332 - engine_dict = asdict(engine_args) | {"seed": 42}

333 - llm = LLM(**engine_dict)

334 -

330 # Generate using chat

331 sampling_params = SamplingParams(**config["sampling_params"])

332 outputs = llm.chat(messages=messages, sampling_params=sampling_params)

tests/models/multimodal/generation/test_voxtral_realtime.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import contextlib

4 -from dataclasses import asdict

4

5 import pytest

6 import pytest_asyncio

@@ -75,7 +74,7 @@ def tokenizer() -> MistralTokenizer:

74 @pytest.fixture

75 def engine():

76 engine_args = EngineArgs(**ENGINE_CONFIG)

78 - llm = LLM(**asdict(engine_args))

77 + llm = LLM.from_engine_args(engine_args)

78 try:

79 yield llm

80 finally:

tests/v1/kv_connector/unit/test_example_connector.py

+15 -22

@@ -1,12 +1,11 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from dataclasses import asdict

3 from typing import NamedTuple

4

5 import pytest

6 from PIL import Image

7

9 -from vllm import LLM, EngineArgs, SamplingParams

8 +from vllm import LLM, SamplingParams

9 from vllm.assets.image import ImageAsset

10 from vllm.config import AttentionConfig, KVTransferConfig

11 from vllm.multimodal.utils import encode_image_url

@@ -129,24 +128,6 @@ def test_shared_storage_connector_hashes(tmp_path, attn_backend):

128 # Using tmp_path as the storage path to store KV

129 print(f"KV storage path at: {str(tmp_path)}")

130

132 - # Configure the ExampleConnector

133 - kv_transfer_config = KVTransferConfig(

134 - kv_connector="ExampleConnector",

135 - kv_role="kv_both",

136 - kv_connector_extra_config={"shared_storage_path": str(tmp_path)},

137 - )

138 -

139 - engine_args = EngineArgs(

140 - model=MODEL_NAME,

141 - max_model_len=8192,

142 - max_num_seqs=1,

143 - gpu_memory_utilization=0.4,

144 - attention_config=AttentionConfig(backend=attn_backend),

145 - enforce_eager=True,

146 - kv_transfer_config=kv_transfer_config,

147 - limit_mm_per_prompt={"image": 2},

148 - )

149 -

131 # don't put this import at the top level

132 # it will call torch.accelerator.device_count()

133 from transformers import AutoProcessor

@@ -163,8 +144,20 @@ def test_shared_storage_connector_hashes(tmp_path, attn_backend):

144 assert image_1 != image_2, "The images should not be identical"

145

146 # Create the LLM instance

166 - engine_args = asdict(engine_args)

167 - llm = LLM(**engine_args)

147 + llm = LLM(

148 + model=MODEL_NAME,

149 + max_model_len=8192,

150 + max_num_seqs=1,

151 + gpu_memory_utilization=0.4,

152 + attention_config=AttentionConfig(backend=attn_backend),

153 + enforce_eager=True,

154 + kv_transfer_config=KVTransferConfig(

155 + kv_connector="ExampleConnector",

156 + kv_role="kv_both",

157 + kv_connector_extra_config={"shared_storage_path": str(tmp_path)},

158 + ),

159 + limit_mm_per_prompt={"image": 2},

160 + )

161

162 # Prepare the input cases

163 input_cases = [

vllm/benchmarks/latency.py

+1 -2

@@ -6,7 +6,6 @@ import argparse

6 import json

7 import os

8 import time

9 -from dataclasses import fields

9 from typing import Any

10

11 import numpy as np

@@ -85,7 +84,7 @@ def main(args: argparse.Namespace):

84

85 # NOTE(woosuk): If the request cannot be processed in a single batch,

86 # the engine will automatically process the request in multiple batches.

88 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

87 + llm = LLM.from_engine_args(engine_args)

88 assert llm.llm_engine.model_config.max_model_len >= (

89 args.input_len + args.output_len

90 ), (

vllm/benchmarks/mm_processor.py

+1 -2

@@ -17,7 +17,6 @@ import argparse

17 import json

18 import time

19 from collections import defaultdict

20 -from dataclasses import fields

20 from datetime import datetime

21 from typing import TYPE_CHECKING, Any, Literal

22

@@ -225,7 +224,7 @@ def benchmark_multimodal_processor(

224 args.seed = 0

225

226 engine_args = EngineArgs.from_cli_args(args)

228 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

227 + llm = LLM.from_engine_args(engine_args)

228

229 tokenizer = llm.get_tokenizer()

230 requests = get_requests(args, tokenizer)

vllm/benchmarks/startup.py

+1 -2

@@ -16,7 +16,6 @@ import shutil

16 import tempfile

17 import time

18 from contextlib import contextmanager

19 -from dataclasses import fields

19 from typing import Any

20

21 import numpy as np

@@ -67,7 +66,7 @@ def run_startup_in_subprocess(engine_args, result_queue):

66 # Measure total startup time

67 start_time = time.perf_counter()

68

70 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

69 + llm = LLM.from_engine_args(engine_args)

70

71 total_startup_time = time.perf_counter() - start_time

72

vllm/benchmarks/throughput.py

+2 -3

@@ -8,7 +8,6 @@ import os

8 import random

9 import time

10 import warnings

11 -from dataclasses import fields

11 from typing import Any

12

13 import torch

@@ -53,7 +52,7 @@ def run_vllm(

52 ) -> tuple[float, list[RequestOutput] | None]:

53 from vllm import LLM, SamplingParams

54

56 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

55 + llm = LLM.from_engine_args(engine_args)

56 assert all(

57 llm.llm_engine.model_config.max_model_len

58 >= (request.prompt_len + request.expected_output_len)

@@ -141,7 +140,7 @@ def run_vllm_chat(

140 """

141 from vllm import LLM, SamplingParams

142

144 - llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

143 + llm = LLM.from_engine_args(engine_args)

144

145 assert all(

146 llm.llm_engine.model_config.max_model_len

vllm/config/compilation.py

+15 -20

@@ -116,29 +116,29 @@ class PassConfig:

116 """

117

118 # New flags

119 - fuse_norm_quant: bool | None = Field(default=None)

119 + fuse_norm_quant: bool = None # type: ignore[assignment]

120 """Fuse the custom RMSNorm + quant ops."""

121 - fuse_act_quant: bool | None = Field(default=None)

121 + fuse_act_quant: bool = None # type: ignore[assignment]

122 """Fuse the custom SiluMul + quant ops."""

123 - fuse_attn_quant: bool | None = Field(default=None)

123 + fuse_attn_quant: bool = None # type: ignore[assignment]

124 """Fuse the custom attention + quant ops."""

125 eliminate_noops: bool = Field(default=True)

126 """Eliminate no-op ops."""

127 - enable_sp: bool | None = Field(default=None)

127 + enable_sp: bool = None # type: ignore[assignment]

128 """Enable sequence parallelism. Requires TP>1. Automatically disabled

129 if the model's hidden_size is too small for SP to be beneficial

130 (threshold is device-capability dependent)."""

131 - fuse_gemm_comms: bool | None = Field(default=None)

131 + fuse_gemm_comms: bool = None # type: ignore[assignment]

132 """Enable async TP."""

133 - fuse_allreduce_rms: bool | None = Field(default=None)

133 + fuse_allreduce_rms: bool = None # type: ignore[assignment]

134 """Enable flashinfer allreduce fusion."""

135 enable_qk_norm_rope_fusion: bool = False

136 """Enable fused Q/K RMSNorm + RoPE pass."""

137

138 # ROCm/AITER specific fusions

139 - fuse_act_padding: bool | None = Field(default=None)

139 + fuse_act_padding: bool = None # type: ignore[assignment]

140 """Fuse the custom RMSNorm + padding ops."""

141 - fuse_rope_kvcache: bool | None = Field(default=None)

141 + fuse_rope_kvcache: bool = None # type: ignore[assignment]

142 """Fuse the QK rope + KV cache ops."""

143

144 rope_kvcache_fusion_max_token_num: int = 256

@@ -405,7 +405,7 @@ class CompilationConfig:

405 """

406

407 # Top-level Compilation control

408 - mode: CompilationMode = Field(default=None) # type: ignore[assignment]

408 + mode: CompilationMode = None # type: ignore[assignment]

409 """The compilation approach used for torch.compile-based compilation of the

410 model.

411

@@ -545,7 +545,7 @@ class CompilationConfig:

545 constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""

546

547 # CudaGraph compilation

548 - cudagraph_mode: CUDAGraphMode = Field(default=None) # type: ignore[assignment]

548 + cudagraph_mode: CUDAGraphMode = None # type: ignore[assignment]

549 """

550 The mode of the cudagraph:

551

@@ -586,7 +586,7 @@ class CompilationConfig:

586 It means the first several runs will be treated as warmup runs.

587 Only after that, the execution will be recorded, and the recorded

588 cudagraph will be used for subsequent runs."""

589 - cudagraph_capture_sizes: list[int] | None = None

589 + cudagraph_capture_sizes: list[int] = None # type: ignore[assignment]

590 """Sizes to capture cudagraph.

591 - None (default): capture sizes are inferred from vllm config.

592 - list[int]: capture sizes are specified as given."""

@@ -607,7 +607,7 @@ class CompilationConfig:

607 When `enable_lora` is False, this option has no effect.

608 """

609

610 - use_inductor_graph_partition: bool = Field(default=None) # type: ignore[assignment]

610 + use_inductor_graph_partition: bool = None # type: ignore[assignment]

611 """Use inductor graph partition to split the graph at cudagraph_unsafe ops.

612 This partition happens at inductor codegen time after all passes and fusions

613 are finished. It generates a single `call` function which wraps

@@ -630,7 +630,7 @@ class CompilationConfig:

630 pass_config: PassConfig = field(default_factory=PassConfig)

631 """Custom inductor passes, see PassConfig for more details"""

632

633 - max_cudagraph_capture_size: int | None = field(default=None)

633 + max_cudagraph_capture_size: int = None # type: ignore[assignment]

634 """The maximum cudagraph capture size.

635

636 If cudagraph_capture_sizes is specified, this will be set to the largest

@@ -750,7 +750,7 @@ class CompilationConfig:

750 return hash_factors(factors)

751

752 def __repr__(self) -> str:

753 - exclude = {

753 + exclude: dict[str, bool | dict[str, bool]] = {

754 "static_forward_context": True,

755 "enabled_custom_ops": True,

756 "disabled_custom_ops": True,

@@ -770,9 +770,7 @@ class CompilationConfig:

770 exclude["pass_config"] = pass_config_exclude

771

772 config = TypeAdapter(CompilationConfig).dump_python(

773 - self,

774 - exclude=exclude, # type: ignore[arg-type]

775 - exclude_unset=True,

773 + self, exclude=exclude, exclude_unset=True

774 )

775

776 return str(config)

@@ -1023,7 +1021,6 @@ class CompilationConfig:

1021 "Unrecognized size type in compile_sizes, "

1022 f"expect 'cudagraph_capture_sizes', got {x}"

1023 )

1026 - assert self.cudagraph_capture_sizes is not None

1024 computed_compile_sizes.extend(self.cudagraph_capture_sizes)

1025 else:

1026 assert isinstance(x, int)

@@ -1031,7 +1028,6 @@ class CompilationConfig:

1028 self.compile_sizes = computed_compile_sizes # type: ignore

1029

1030 # make sure the sizes are in ascending order

1034 - assert self.cudagraph_capture_sizes is not None

1031 self.cudagraph_capture_sizes.sort()

1032 if self.cudagraph_capture_sizes:

1033 assert self.cudagraph_capture_sizes[-1] == self.max_cudagraph_capture_size

@@ -1123,7 +1119,6 @@ class CompilationConfig:

1119

1120 def set_splitting_ops_for_attn_fusion(self):

1121 assert self.pass_config.fuse_attn_quant

1126 - assert self.cudagraph_mode is not None

1122 if self.splitting_ops is None:

1123 self.splitting_ops = []

1124 if self.cudagraph_mode.has_piecewise_cudagraphs():

vllm/config/device.py

+2 -2

@@ -13,8 +13,8 @@ from vllm.utils.hashing import safe_hash

13 Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]

14

15

16 -@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]

17 -class DeviceConfig: # type: ignore[misc]

16 +@config(config=ConfigDict(arbitrary_types_allowed=True))

17 +class DeviceConfig:

18 """Configuration for the device to use for vLLM execution."""

19

20 device: SkipValidation[Device | torch.device | None] = "auto"

vllm/config/kernel.py

+2 -2

@@ -4,7 +4,7 @@

4 from collections.abc import Callable

5 from typing import Any, Literal

6

7 -from pydantic import Field, field_validator

7 +from pydantic import field_validator

8

9 from vllm.config.utils import config

10 from vllm.utils.hashing import safe_hash

@@ -26,7 +26,7 @@ MoEBackend = Literal[

26 class KernelConfig:

27 """Configuration for kernel selection and warmup behavior."""

28

29 - enable_flashinfer_autotune: bool | None = Field(default=None)

29 + enable_flashinfer_autotune: bool = None # type: ignore[assignment]

30 """If True, run FlashInfer autotuning during kernel warmup."""

31

32 moe_backend: MoEBackend = "auto"

vllm/config/kv_events.py

+1 -3

@@ -4,8 +4,6 @@

4

5 from typing import Literal

6

7 -from pydantic import Field

8 -

7 from vllm.config.utils import config

8

9

@@ -18,7 +16,7 @@ class KVEventsConfig:

16 Events can be published externally by zmq using the event publisher config.

17 """

18

21 - publisher: Literal["null", "zmq"] | None = Field(default=None)

19 + publisher: Literal["null", "zmq"] = None # type: ignore[assignment]

20 """The publisher to use for publishing kv events. Can be "null", "zmq".

21 """

22

vllm/config/lora.py

+2 -2

@@ -25,8 +25,8 @@ MaxLoRARanks = Literal[1, 8, 16, 32, 64, 128, 256, 320, 512]

25 LoRAExtraVocabSize = Literal[256, 512]

26

27

28 -@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]

29 -class LoRAConfig: # type: ignore[misc]

28 +@config(config=ConfigDict(arbitrary_types_allowed=True))

29 +class LoRAConfig:

30 """Configuration for LoRA."""

31

32 max_lora_rank: MaxLoRARanks = 16

vllm/config/model.py

+6 -6

@@ -102,8 +102,8 @@ AttnTypeStr = Literal[

102 ]

103

104

105 -@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]

106 -class ModelConfig: # type: ignore[misc]

105 +@config(config=ConfigDict(arbitrary_types_allowed=True))

106 +class ModelConfig:

107 """Configuration for the model."""

108

109 model: str = "Qwen/Qwen3-0.6B"

@@ -121,7 +121,7 @@ class ModelConfig: # type: ignore[misc]

121 """Convert the model using adapters defined in

122 [vllm.model_executor.models.adapters][]. The most common use case is to

123 adapt a text generation model to be used for pooling tasks."""

124 - tokenizer: str = Field(default=None) # type: ignore[assignment]

124 + tokenizer: str = None # type: ignore[assignment]

125 """Name or path of the Hugging Face tokenizer to use. If unspecified, model

126 name or path will be used."""

127 tokenizer_mode: TokenizerMode | str = "auto"

@@ -583,7 +583,7 @@ class ModelConfig: # type: ignore[misc]

583 self.dtype,

584 is_pooling_model=self.runner_type == "pooling",

585 revision=self.revision,

586 - config_format=self.config_format, # type: ignore[arg-type]

586 + config_format=self.config_format,

587 )

588

589 self.original_max_model_len = self.max_model_len

@@ -733,7 +733,7 @@ class ModelConfig: # type: ignore[misc]

733

734 @property

735 def architectures(self) -> list[str]:

736 - return self.model_arch_config.architectures # type: ignore[return-value]

736 + return self.model_arch_config.architectures

737

738 @property

739 def architecture(self) -> str:

@@ -1944,7 +1944,7 @@ def _get_and_verify_dtype(

1944 *,

1945 is_pooling_model: bool,

1946 revision: str | None = None,

1947 - config_format: ConfigFormat = "hf",

1947 + config_format: str | ConfigFormat = "hf",

1948 ) -> torch.dtype:

1949 config_dtype = ModelArchConfigConvertorBase.get_torch_dtype(

1950 config, model_id, revision=revision, config_format=config_format

vllm/config/model_arch.py

+1 -1

@@ -16,7 +16,7 @@ class ModelArchitectureConfig:

16 Configuration for model architecture that required by vLLM runtime

17 """

18

19 - architectures: list[str] | None

19 + architectures: list[str]

20 """List of model architecture class names (e.g., ['LlamaForCausalLM']).

21 It can be None upon calling `vllm_config.with_hf_config(config.text_config)`"""

22

vllm/config/parallel.py

+1 -1

@@ -194,7 +194,7 @@ class ParallelConfig:

194 threshold, microbatching will be used. Otherwise, the request will be

195 processed in a single batch."""

196

197 - disable_nccl_for_dp_synchronization: bool | None = Field(default=None)

197 + disable_nccl_for_dp_synchronization: bool | None = None

198 """Forces the dp synchronization logic in vllm/v1/worker/dp_utils.py

199 to use Gloo instead of NCCL for its all reduce.

200

vllm/config/scheduler.py

+3 -3

@@ -52,7 +52,7 @@ class SchedulerConfig:

52 In real usage, this should be set in `EngineArgs.create_engine_config`.

53 """

54

55 - max_num_scheduled_tokens: int | None = Field(default=None)

55 + max_num_scheduled_tokens: int | None = None

56 """Maximum number of tokens that the scheduler may issue in a single iteration.

57

58 This is usually equal to max_num_batched_tokens, but can be smaller in cases

@@ -122,7 +122,7 @@ class SchedulerConfig:

122

123 # scheduler class or path. "vllm.v1.core.sched.scheduler.Scheduler"

124 # (default) or "mod.custom_class".

125 - scheduler_cls: str | type[object] | None = Field(default=None)

125 + scheduler_cls: str | type[object] | None = None

126 """The scheduler class to use. "vllm.v1.core.sched.scheduler.Scheduler" is

127 the default scheduler. Can be a class directly or the path to a class of

128 form "mod.custom_class"."""

@@ -141,7 +141,7 @@ class SchedulerConfig:

141 checking the first chunk. Prevents over-admission and KV cache thrashing

142 with chunked prefill."""

143

144 - async_scheduling: bool | None = Field(default=None)

144 + async_scheduling: bool | None = None

145 """If set to False, disable async scheduling. Async scheduling helps to

146 avoid gaps in GPU utilization, leading to better latency and throughput.

147 """

vllm/config/utils.py

+14 -4

@@ -11,13 +11,13 @@ import os

11 import pathlib

12 import textwrap

13 from collections.abc import Callable, Mapping, Sequence, Set

14 -from dataclasses import MISSING, dataclass, field, fields, is_dataclass

14 +from dataclasses import MISSING, field, fields, is_dataclass

15 from itertools import pairwise

16 -from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast

16 +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast, overload

17

18 import torch

19 from pydantic import ConfigDict

20 -from pydantic.dataclasses import dataclass as pydantic_dataclass

20 +from pydantic.dataclasses import dataclass

21 from pydantic.fields import Field as PydanticField

22 from pydantic.fields import FieldInfo

23 from typing_extensions import dataclass_transform, runtime_checkable

@@ -36,6 +36,16 @@ ConfigType = type[DataclassInstance]

36 ConfigT = TypeVar("ConfigT", bound=DataclassInstance)

37

38

39 +@overload

40 +def config(cls: type[ConfigT]) -> type[ConfigT]: ...

41 +

42 +

43 +@overload

44 +def config(

45 + *, config: ConfigDict | None = None, **kwargs: Any

46 +) -> Callable[[type[ConfigT]], type[ConfigT]]: ...

47 +

48 +

49 @dataclass_transform(field_specifiers=(PydanticField,))

50 def config(

51 cls: type[ConfigT] | None = None,

@@ -59,7 +69,7 @@ def config(

69 merged_config.update(config)

70

71 def decorator(cls: type[ConfigT]) -> type[ConfigT]:

62 - return pydantic_dataclass(cls, config=merged_config, **kwargs) # type: ignore[return-value]

72 + return dataclass(cls, config=merged_config, **kwargs) # type: ignore[return-value]

73

74 # Called with arguments: @config(config=...)

75 if cls is None:

vllm/config/vllm.py

+9 -14

@@ -246,15 +246,15 @@ OPTIMIZATION_LEVEL_TO_CONFIG = {

246 }

247

248

249 -@config(config=ConfigDict(arbitrary_types_allowed=True)) # type: ignore[arg-type,misc]

250 -class VllmConfig: # type: ignore[misc]

249 +@config(config=ConfigDict(arbitrary_types_allowed=True))

250 +class VllmConfig:

251 """Dataclass which contains all vllm-related configuration. This

252 simplifies passing around the distinct configurations in the codebase.

253 """

254

255 # TODO: use default_factory once default constructing ModelConfig doesn't

256 # try to download a model

257 - model_config: ModelConfig = Field(default=None) # type: ignore[assignment]

257 + model_config: ModelConfig = None # type: ignore[assignment]

258 """Model configuration."""

259 cache_config: CacheConfig = Field(default_factory=CacheConfig)

260 """Cache configuration."""

@@ -912,7 +912,8 @@ class VllmConfig: # type: ignore[misc]

912

913 tp_size = self.parallel_config.tensor_parallel_size

914 hidden_size = self.model_config.get_hidden_size()

915 - element_size = self.model_config.dtype.itemsize # type: ignore[union-attr]

915 + assert isinstance(self.model_config.dtype, torch.dtype)

916 + element_size = self.model_config.dtype.itemsize

917 pass_config.sp_min_token_num = get_sequence_parallelism_threshold(

918 hidden_size, tp_size, element_size

919 )

@@ -1246,14 +1247,6 @@ class VllmConfig: # type: ignore[misc]

1247 )

1248 self.compilation_config.debug_dump_path = env_path

1249

1249 - def has_blocked_weights(): # type: ignore[no-redef]

1250 - if self.quant_config is not None:

1251 - if hasattr(self.quant_config, "weight_block_size"):

1252 - return self.quant_config.weight_block_size is not None

1253 - elif hasattr(self.quant_config, "has_blocked_weights"):

1254 - return self.quant_config.has_blocked_weights()

1255 - return False

1256 -

1250 # Enable quant_fp8 CUDA ops (TODO disable in follow up)

1251 # On H100 the CUDA kernel is faster than

1252 # native implementation

@@ -1502,9 +1495,10 @@ class VllmConfig: # type: ignore[misc]

1495 tp_size = self.parallel_config.tensor_parallel_size

1496 max_size = compilation_config.pass_config.flashinfer_max_size(tp_size)

1497 if max_size is not None:

1498 + assert isinstance(self.model_config.dtype, torch.dtype)

1499 max_token_num = max_size // (

1500 self.model_config.get_hidden_size()

1507 - * self.model_config.dtype.itemsize # type: ignore[union-attr]

1501 + * self.model_config.dtype.itemsize

1502 )

1503 if compile_range_end is not None and max_token_num < compile_range_end:

1504 computed_compile_ranges_endpoints.append(max_token_num)

@@ -1527,7 +1521,8 @@ class VllmConfig: # type: ignore[misc]

1521

1522 tp_size = self.parallel_config.tensor_parallel_size

1523 hidden_size = self.model_config.get_hidden_size()

1530 - element_size = self.model_config.dtype.itemsize # type: ignore[union-attr]

1524 + assert isinstance(self.model_config.dtype, torch.dtype)

1525 + element_size = self.model_config.dtype.itemsize

1526 pass_config.sp_min_token_num = get_sequence_parallelism_threshold(

1527 hidden_size, tp_size, element_size

1528 )

vllm/engine/arg_utils.py

+1 -1

@@ -1935,7 +1935,7 @@ class EngineArgs:

1935 )

1936

1937 offload_config = OffloadConfig(

1938 - offload_backend=self.offload_backend, # type: ignore[arg-type]

1938 + offload_backend=self.offload_backend,

1939 uva=UVAOffloadConfig(

1940 cpu_offload_gb=self.cpu_offload_gb,

1941 cpu_offload_params=self.cpu_offload_params,

vllm/entrypoints/llm.py

+5

@@ -409,6 +409,11 @@ class LLM:

409 # Cache for __repr__ to avoid repeated collective_rpc calls

410 self._cached_repr: str | None = None

411

412 + @classmethod

413 + def from_engine_args(cls, engine_args: EngineArgs) -> "LLM":

414 + """Create an LLM instance from EngineArgs."""

415 + return cls(**vars(engine_args))

416 +

417 def get_tokenizer(self) -> TokenizerLike:

418 return self.llm_engine.get_tokenizer()

419

vllm/transformers_utils/model_arch_config_convertor.py

+5 -2

@@ -28,7 +28,10 @@ class ModelArchConfigConvertorBase:

28 self.hf_text_config = hf_text_config

29

30 def get_architectures(self) -> list[str]:

31 - return getattr(self.hf_config, "architectures", [])

31 + # Sometimes we get here from `vllm_config.with_hf_config(text_config)` where

32 + # `text_config` is a sub-config from a multi-modal model. If this is the case,

33 + # the sub-config will not have `architectures` and it will explicitly be `None`

34 + return getattr(self.hf_config, "architectures", None) or []

35

36 def get_num_hidden_layers(self) -> int:

37 return getattr(self.hf_text_config, "num_hidden_layers", 0)

@@ -128,7 +131,7 @@ class ModelArchConfigConvertorBase:

131 hf_config: PretrainedConfig,

132 model_id: str,

133 revision: str | None,

131 - config_format: ConfigFormat,

134 + config_format: str | ConfigFormat,

135 ):

136 # NOTE: getattr(config, "dtype", torch.float32) is not correct

137 # because config.dtype can be None.

[Mypy] Better fixes for the `mypy` issues in `vllm/config` (#37902)