f83b933b84 · tduncan/Nvidia-Optimization-Stack-VLLM

.pre-commit-config.yaml

+1 -1

@@ -55,7 +55,7 @@ repos:

55 language: python

56 types_or: [python, pyi]

57 require_serial: true

58 - additional_dependencies: ["mypy[faster-cache]==1.15.0", regex, types-cachetools, types-setuptools, types-PyYAML, types-requests, types-torch, pydantic]

58 + additional_dependencies: ["mypy[faster-cache]==1.19.1", regex, types-cachetools, types-setuptools, types-PyYAML, types-requests, types-torch, pydantic]

59 - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

60 name: Run mypy for Python 3.10

61 entry: python tools/pre_commit/mypy.py 1 "3.10"

tests/quantization/test_mixed_precision.py

+1

@@ -8,6 +8,7 @@ Run `pytest tests/quantization/test_mixed_precision.py`.

8

9 import importlib

10 import importlib.metadata

11 +import importlib.util

12 from dataclasses import dataclass

13

14 import lm_eval

vllm/distributed/device_communicators/shm_broadcast.py

+3

@@ -274,6 +274,7 @@ class ShmRingBuffer:

274 self.shared_memory = shared_memory.SharedMemory(

275 create=True, size=self.total_bytes_of_buffer

276 )

277 + assert self.shared_memory.buf is not None, "Buffer was not created"

278 # initialize the metadata section to 0

279 with self.shared_memory.buf[self.metadata_offset :] as metadata_buffer:

280 torch.frombuffer(metadata_buffer, dtype=torch.uint8).fill_(0)

@@ -325,6 +326,7 @@ class ShmRingBuffer:

326 def get_data(self, current_idx: int):

327 start = self.data_offset + current_idx * self.max_chunk_bytes

328 end = start + self.max_chunk_bytes

329 + assert self.shared_memory.buf is not None, "Buffer has been closed"

330 with self.shared_memory.buf[start:end] as buf:

331 yield buf

332

@@ -332,6 +334,7 @@ class ShmRingBuffer:

334 def get_metadata(self, current_idx: int):

335 start = self.metadata_offset + current_idx * self.metadata_size

336 end = start + self.metadata_size

337 + assert self.shared_memory.buf is not None, "Buffer has been closed"

338 with self.shared_memory.buf[start:end] as buf:

339 yield buf

340

vllm/distributed/device_communicators/shm_object_storage.py

+2

@@ -197,6 +197,7 @@ class SingleWriterShmRingBuffer:

197 """

198 assert self.is_writer, "Only the writer can allocate buffers."

199 assert size > 0, "Size must be greater than 0"

200 + assert self.shared_memory.buf is not None, "Buffer has been closed"

201 size += self.MD_SIZE # add metadata size to the buffer size

202 # reset to beginning if the buffer does have enough contiguous space

203 buffer_end_reset = self.data_buffer_end % self.data_buffer_size

@@ -239,6 +240,7 @@ class SingleWriterShmRingBuffer:

240

241 @contextmanager

242 def access_buf(self, address: int):

243 + assert self.shared_memory.buf is not None, "Buffer has been closed"

244 buf_idx = address % self.data_buffer_size

245

246 # read metadata

vllm/distributed/kv_transfer/kv_connector/utils.py

+1

@@ -351,6 +351,7 @@ class TpKVTopology:

351 include_num_layers_dimension=self._cross_layers_blocks

352 )

353 except (AttributeError, NotImplementedError):

354 + assert self.tensor_shape is not None

355 kv_cache_stride_order = tuple(range(len(self.tensor_shape)))

356

357 # In case of cross layers permute kv_cache_shape according to

vllm/distributed/parallel_state.py

+2

@@ -1964,6 +1964,7 @@ def in_the_same_node_as(

1964 if rank == source_rank:

1965 # create a shared memory segment

1966 shm = shared_memory.SharedMemory(create=True, size=128)

1967 + assert shm.buf is not None, "Buffer was not created"

1968 shm.buf[: len(magic_message)] = magic_message

1969 if isinstance(pg, ProcessGroup):

1970 torch.distributed.broadcast_object_list(

@@ -1990,6 +1991,7 @@ def in_the_same_node_as(

1991 lambda *args, **kwargs: None,

1992 ):

1993 shm = shared_memory.SharedMemory(name=name)

1994 + assert shm.buf is not None, "Buffer was not opened"

1995 if shm.buf[: len(magic_message)] == magic_message:

1996 is_in_the_same_node[rank] = 1

1997 except Exception as e:

vllm/lora/layers/base.py

+13 -1

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import TYPE_CHECKING

4 +from typing import TYPE_CHECKING, overload

5

6 import torch

7 import torch.nn as nn

@@ -14,12 +14,24 @@ if TYPE_CHECKING:

14

15

16 class BaseLayerWithLoRA(nn.Module):

17 + @overload

18 + def slice_lora_a(

19 + self, lora_a: list[torch.Tensor | None]

20 + ) -> list[torch.Tensor | None]: ...

21 + @overload

22 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: ...

23 def slice_lora_a(

24 self, lora_a: torch.Tensor | list[torch.Tensor | None]

25 ) -> torch.Tensor | list[torch.Tensor | None]:

26 """Slice lora a if splitting for tensor parallelism."""

27 ...

28

29 + @overload

30 + def slice_lora_b(

31 + self, lora_b: list[torch.Tensor | None]

32 + ) -> list[torch.Tensor | None]: ...

33 + @overload

34 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: ...

35 def slice_lora_b(

36 self, lora_b: torch.Tensor | list[torch.Tensor | None]

37 ) -> torch.Tensor | list[torch.Tensor | None]:

vllm/renderers/hf.py

+23 -1

@@ -5,7 +5,7 @@ import itertools

5 from collections import defaultdict, deque

6 from collections.abc import Set

7 from functools import lru_cache

8 -from typing import TYPE_CHECKING, Any, cast

8 +from typing import TYPE_CHECKING, Any, Literal, cast, overload

9

10 import jinja2

11 import jinja2.ext

@@ -439,6 +439,28 @@ def resolve_chat_template_kwargs(

439 return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars}

440

441

442 +@overload

443 +def safe_apply_chat_template(

444 + model_config: "ModelConfig",

445 + tokenizer: HfTokenizer,

446 + conversation: list[ConversationMessage],

447 + *,

448 + tools: list[dict[str, Any]] | None = ...,

449 + chat_template: str | None = ...,

450 + tokenize: Literal[True] = ...,

451 + **kwargs,

452 +) -> list[int]: ...

453 +@overload

454 +def safe_apply_chat_template(

455 + model_config: "ModelConfig",

456 + tokenizer: HfTokenizer,

457 + conversation: list[ConversationMessage],

458 + *,

459 + tools: list[dict[str, Any]] | None = ...,

460 + chat_template: str | None = ...,

461 + tokenize: Literal[False] = ...,

462 + **kwargs,

463 +) -> str: ...

464 def safe_apply_chat_template(

465 model_config: "ModelConfig",

466 tokenizer: HfTokenizer,

vllm/sampling_params.py

+1

@@ -533,6 +533,7 @@ class SamplingParams(

533 if eos_ids:

534 self._all_stop_token_ids.update(eos_ids)

535 if not self.ignore_eos:

536 + assert self.stop_token_ids is not None

537 eos_ids.update(self.stop_token_ids)

538 self.stop_token_ids = list(eos_ids)

539

vllm/transformers_utils/configs/funaudiochat.py

+16 -20

@@ -3,7 +3,7 @@

3

4 from __future__ import annotations

5

6 -from transformers import PretrainedConfig

6 +from transformers import CONFIG_MAPPING, PretrainedConfig

7

8 # NOTE: Temporary shim for FunAudioChat checkpoints.

9 # These checkpoints use `model_type="funaudiochat"`, which is not currently

@@ -92,28 +92,24 @@ class FunAudioChatConfig(PretrainedConfig):

92 self.audio_token_index = audio_token_index

93 self.ignore_index = ignore_index

94

95 - if isinstance(audio_config, dict):

96 - audio_config.setdefault(

97 - "model_type", FunAudioChatAudioEncoderConfig.model_type

98 - )

99 - audio_config = FunAudioChatAudioEncoderConfig(**audio_config)

100 - elif audio_config is None:

101 - audio_config = FunAudioChatAudioEncoderConfig()

102 - self.audio_config = audio_config

103 -

104 - if isinstance(text_config, dict):

95 + if audio_config is None:

96 + self.audio_config = FunAudioChatAudioEncoderConfig()

97 + elif isinstance(audio_config, dict):

98 + default_model_type = FunAudioChatAudioEncoderConfig.model_type

99 + audio_config.setdefault("model_type", default_model_type)

100 + self.audio_config = FunAudioChatAudioEncoderConfig(**audio_config)

101 + else:

102 + self.audio_config = audio_config

103 +

104 + if text_config is None:

105 + self.text_config = CONFIG_MAPPING["qwen2"]()

106 + elif isinstance(text_config, dict):

107 # Default to qwen2 for backwards compatibility; FunAudioChat uses

108 # qwen3 in practice for recent checkpoints.

109 text_config.setdefault("model_type", "qwen2")

108 - import transformers

109 -

110 - text_cls = transformers.CONFIG_MAPPING[text_config["model_type"]]

111 - text_config = text_cls(**text_config)

112 - elif text_config is None:

113 - import transformers

114 -

115 - text_config = transformers.CONFIG_MAPPING["qwen2"]()

116 - self.text_config = text_config

110 + self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)

111 + else:

112 + self.text_config = text_config

113

114 self.hidden_size = (

115 int(self.text_config.hidden_size)

vllm/transformers_utils/configs/kimi_k25.py

+8 -6

@@ -90,17 +90,19 @@ class KimiK25Config(PretrainedConfig):

90 ):

91 # Vision config

92 if vision_config is None:

93 - vision_config = KimiK25VisionConfig()

93 + self.vision_config = KimiK25VisionConfig()

94 elif isinstance(vision_config, dict):

95 - vision_config = KimiK25VisionConfig(**vision_config)

96 - self.vision_config: KimiK25VisionConfig = vision_config

95 + self.vision_config = KimiK25VisionConfig(**vision_config)

96 + else:

97 + self.vision_config = vision_config

98

99 # Text config

100 if text_config is None:

100 - text_config = DeepseekV3Config()

101 + self.text_config = DeepseekV3Config()

102 elif isinstance(text_config, dict):

102 - text_config = DeepseekV3Config(**text_config)

103 - self.text_config: DeepseekV3Config = text_config

103 + self.text_config = DeepseekV3Config(**text_config)

104 + else:

105 + self.text_config = text_config

106

107 # Set mm_hidden_size to text hidden size if not explicitly set

108 if self.vision_config.mm_hidden_size == self.vision_config.hidden_size:

vllm/transformers_utils/processors/ovis2_5.py

+1

@@ -412,6 +412,7 @@ class Ovis2_5Processor(ProcessorMixin):

412 images = video

413 else:

414 raise ValueError("Either images or video should be provided.")

415 + assert images is not None

416 min_pixels = min(

417 max_pixels if max_pixels is not None else MAX_PIXELS,

418 min_pixels if min_pixels is not None else MIN_PIXELS,

vllm/v1/engine/detokenizer.py

+3 -5

@@ -72,14 +72,12 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):

72 # Stop strings

73 params = request.sampling_params

74 assert params is not None

75 - stop_list: list[str]

75 if params.stop is None:

77 - stop_list = []

76 + self.stop = []

77 elif isinstance(params.stop, str):

79 - stop_list = [params.stop]

78 + self.stop = [params.stop]

79 else:

81 - stop_list = params.stop

82 - self.stop = stop_list

80 + self.stop = params.stop

81 self.min_tokens = params.min_tokens

82 self.include_stop_str_in_output = params.include_stop_str_in_output

83

vllm/v1/executor/ray_executor.py

+2 -2

@@ -282,8 +282,8 @@ class RayDistributedExecutor(Executor):

282 # driver_dummy_worker can be None when using ray spmd worker.

283 continue

284 worker_node_and_gpu_ids.append(

285 - ray.get(worker.get_node_and_gpu_ids.remote())

286 - ) # type: ignore[attr-defined]

285 + ray.get(worker.get_node_and_gpu_ids.remote()) # type: ignore[attr-defined]

286 + )

287

288 node_workers = defaultdict(list) # node id -> list of worker ranks

289 node_gpus = defaultdict(list) # node id -> list of gpu ids

[CI] Bump `mypy` version to 1.19.1 (#36104)