@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c0c68202c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Fix per file ruff ignores related to typing (#26254)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Oct 5, 2025 at 17:37 UTC 1c0c68202cc128e740223e033273caa949c45f15

32 files changed +258 -285

pyproject.toml

+1 -17

@@ -115,6 +115,7 @@ include = ["vllm*"]

115 "vllm/distributed/parallel_state.py" = ["SIM108"]

116 "vllm/entrypoints/chat_utils.py" = ["SIM108"]

117 "vllm/entrypoints/llm.py" = ["SIM108"]

118 +"vllm/executor/ray_distributed_executor.py" = ["SIM108", "SIM112"]

119 "vllm/model_executor/layers/batch_invariant.py" = ["SIM108"]

120 "vllm/model_executor/layers/fla/ops/chunk_o.py" = ["SIM108"]

121 "vllm/model_executor/layers/fused_moe/fused_moe.py" = ["SIM108"]

@@ -134,23 +135,6 @@ include = ["vllm*"]

135 "tools/profiler/print_layerwise_table.py" = ["SIM118"]

136 ## Loop variable binding issues

137 "tests/kernels/mamba/test_mamba_ssm_ssd.py" = ["B023"]

137 -## Type annotation modernization and other rules

138 -"vllm/attention/backends/abstract.py" = ["UP035", "UP006"]

139 -"vllm/attention/layer.py" = ["UP035", "UP006"]

140 -"vllm/attention/layers/chunked_local_attention.py" = ["UP035", "UP006"]

141 -"vllm/attention/ops/flashmla.py" = ["UP035", "UP006"]

142 -"vllm/attention/ops/paged_attn.py" = ["UP035", "UP006"]

143 -"vllm/engine/arg_utils.py" = ["UP035", "UP006"]

144 -"vllm/engine/metrics.py" = ["UP035", "UP006"]

145 -"vllm/engine/metrics_types.py" = ["UP035", "UP006"]

146 -"vllm/executor/executor_base.py" = ["UP035", "UP006"]

147 -"vllm/executor/msgspec_utils.py" = ["UP035", "UP006"]

148 -"vllm/executor/ray_distributed_executor.py" = ["UP035", "UP006", "SIM108", "SIM112"]

149 -"vllm/executor/ray_utils.py" = ["UP035", "UP006"]

150 -"vllm/executor/uniproc_executor.py" = ["UP035", "UP006"]

151 -"vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py" = ["UP035"]

152 -## Type comparison issues

153 -"vllm/multimodal/inputs.py" = ["E721"]

138 # End of temporary ignores

139

140 [tool.ruff.lint]

tests/compile/test_full_graph.py

+2 -2

@@ -5,7 +5,7 @@ from __future__ import annotations

6 import logging

7 import tempfile

8 -from typing import Any, Optional, Union

8 +from typing import Any, Union

10 import pytest

11 import torch

@@ -21,7 +21,7 @@ from vllm.utils import is_torch_equal_or_newer

21 from ..utils import create_new_process_for_each_test

24 -def models_list(*, all: bool = True, keywords: Optional[list[str]] = None):

24 +def models_list(*, all: bool = True, keywords: list[str] | None = None):

25 TEST_MODELS: list[tuple[str, dict[str, Any]]] = [

26 ("facebook/opt-125m", {}),

27 (

tests/entrypoints/openai/test_serving_chat.py

+3 -3

@@ -6,7 +6,7 @@ from __future__ import annotations

6 import asyncio

7 from contextlib import suppress

8 from dataclasses import dataclass, field

9 -from typing import TYPE_CHECKING, Any, Optional

9 +from typing import TYPE_CHECKING, Any

10 from unittest.mock import AsyncMock, MagicMock

12 import pytest

@@ -233,9 +233,9 @@ class MockModelConfig:

233 multimodal_config = MultiModalConfig()

234 hf_config = MockHFConfig()

235 logits_processor_pattern = None

236 - diff_sampling_param: Optional[dict] = None

236 + diff_sampling_param: dict | None = None

237 allowed_local_media_path: str = ""

238 - allowed_media_domains: Optional[list[str]] = None

238 + allowed_media_domains: list[str] | None = None

239 encoder_config = None

240 generation_config: str = "auto"

241 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

tests/plugins/prithvi_io_processor_plugin/prithvi_io_processor/prithvi_processor.py

+11 -11

@@ -9,7 +9,7 @@ import os

9 import tempfile

10 import urllib.request

11 from collections.abc import Sequence

12 -from typing import Any, Optional, Union

12 +from typing import Any, Union

14 import albumentations

15 import numpy as np

@@ -98,9 +98,9 @@ def _convert_np_uint8(float_image: torch.Tensor):

100 def read_geotiff(

101 - file_path: Optional[str] = None,

102 - path_type: Optional[str] = None,

103 - file_data: Optional[bytes] = None,

101 + file_path: str | None = None,

102 + path_type: str | None = None,

103 + file_data: bytes | None = None,

104 ) -> tuple[torch.Tensor, dict, tuple[float, float] | None]:

105 """Read all bands from *file_path* and return image + meta info.

106

@@ -114,8 +114,8 @@ def read_geotiff(

114

115 if all([x is None for x in [file_path, path_type, file_data]]):

116 raise Exception("All input fields to read_geotiff are None")

117 - write_to_file: Optional[bytes] = None

118 - path: Optional[str] = None

117 + write_to_file: bytes | None = None

118 + path: str | None = None

119 if file_data is not None:

120 # with tempfile.NamedTemporaryFile() as tmpfile:

121 # tmpfile.write(file_data)

@@ -162,9 +162,9 @@ def read_geotiff(

162 def load_image(

163 data: Union[list[str]],

164 path_type: str,

165 - mean: Optional[list[float]] = None,

166 - std: Optional[list[float]] = None,

167 - indices: Optional[Union[list[int], None]] = None,

165 + mean: list[float] | None = None,

166 + std: list[float] | None = None,

167 + indices: Union[list[int], None] | None = None,

168 ):

169 """Build an input example by loading images in *file_paths*.

170

@@ -278,7 +278,7 @@ class PrithviMultimodalDataProcessor(IOProcessor):

278 def pre_process(

279 self,

280 prompt: IOProcessorInput,

281 - request_id: Optional[str] = None,

281 + request_id: str | None = None,

282 **kwargs,

283 ) -> Union[PromptType, Sequence[PromptType]]:

284 image_data = dict(prompt)

@@ -359,7 +359,7 @@ class PrithviMultimodalDataProcessor(IOProcessor):

359 def post_process(

360 self,

361 model_output: Sequence[PoolingRequestOutput],

362 - request_id: Optional[str] = None,

362 + request_id: str | None = None,

363 **kwargs,

364 ) -> IOProcessorOutput:

365 pred_imgs_list = []

tests/v1/engine/test_llm_engine.py

+2 -2

@@ -3,7 +3,7 @@

3 from __future__ import annotations

5 import random

6 -from typing import TYPE_CHECKING, Optional

6 +from typing import TYPE_CHECKING

8 import pytest

@@ -78,7 +78,7 @@ def vllm_model_skip_tokenizer_init(vllm_runner, request, monkeypatch):

79 def _get_test_sampling_params(

80 prompt_list: list[str],

81 - seed: Optional[int] = 42,

81 + seed: int | None = 42,

82 structured_outputs: bool = False,

83 ) -> tuple[list[SamplingParams], list[int]]:

84 """Generate random sampling params for a batch."""

vllm/attention/backends/abstract.py

+6 -6

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from abc import ABC, abstractmethod

5 -from typing import Generic, List, Optional, Protocol, Tuple, Type, TypeVar

5 +from typing import Generic, Optional, Protocol, TypeVar

7 import torch

@@ -48,12 +48,12 @@ class AttentionBackend(ABC):

49 @staticmethod

50 @abstractmethod

51 - def get_impl_cls() -> Type["AttentionImpl"]:

51 + def get_impl_cls() -> type["AttentionImpl"]:

52 raise NotImplementedError

54 @staticmethod

55 @abstractmethod

56 - def get_metadata_cls() -> Type["AttentionMetadata"]:

56 + def get_metadata_cls() -> type["AttentionMetadata"]:

57 raise NotImplementedError

59 @classmethod

@@ -73,11 +73,11 @@ class AttentionBackend(ABC):

73 num_kv_heads: int,

74 head_size: int,

75 cache_dtype_str: str = "auto",

76 - ) -> Tuple[int, ...]:

76 + ) -> tuple[int, ...]:

77 raise NotImplementedError

79 @staticmethod

80 - def get_kv_cache_stride_order() -> Tuple[int, ...]:

80 + def get_kv_cache_stride_order() -> tuple[int, ...]:

81 raise NotImplementedError

83 @classmethod

@@ -147,7 +147,7 @@ class AttentionImpl(ABC, Generic[T]):

147 head_size: int,

148 scale: float,

149 num_kv_heads: Optional[int] = None,

150 - alibi_slopes: Optional[List[float]] = None,

150 + alibi_slopes: Optional[list[float]] = None,

151 sliding_window: Optional[int] = None,

152 kv_cache_dtype: str = "auto",

153 logits_soft_cap: Optional[float] = None,

vllm/attention/layer.py

+3 -3

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """Attention layer."""

5 -from typing import Callable, List, Optional

5 +from typing import Callable, Optional

7 import torch

8 import torch.nn as nn

@@ -126,7 +126,7 @@ class Attention(nn.Module, AttentionLayerBase):

126 head_size: int,

127 scale: float,

128 num_kv_heads: Optional[int] = None,

129 - alibi_slopes: Optional[List[float]] = None,

129 + alibi_slopes: Optional[list[float]] = None,

130 cache_config: Optional[CacheConfig] = None,

131 quant_config: Optional[QuantizationConfig] = None,

132 logits_soft_cap: Optional[float] = None,

@@ -586,7 +586,7 @@ def wait_for_kv_layer_from_connector(layer_name: str):

586

587 def maybe_save_kv_layer_to_connector(

588 layer_name: str,

589 - kv_cache_layer: List[torch.Tensor],

589 + kv_cache_layer: list[torch.Tensor],

590 ):

591 if not has_kv_transfer_group() or not is_v1_kv_transfer_group():

592 return

vllm/attention/layers/chunked_local_attention.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import functools

4 -from typing import ClassVar, List, Optional

4 +from typing import ClassVar, Optional

6 import torch

@@ -61,7 +61,7 @@ class ChunkedLocalAttention(Attention):

61 scale: float,

62 attention_chunk_size: int,

63 num_kv_heads: Optional[int] = None,

64 - alibi_slopes: Optional[List[float]] = None,

64 + alibi_slopes: Optional[list[float]] = None,

65 cache_config: Optional[CacheConfig] = None,

66 quant_config: Optional[QuantizationConfig] = None,

67 kv_sharing_target_layer_name: Optional[str] = None,

vllm/attention/ops/flashmla.py

+5 -5

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 # adapted from: https://github.com/deepseek-ai/FlashMLA/blob/main/flash_mla/flash_mla_interface.py

4 -from typing import Optional, Tuple

4 +from typing import Optional

6 import torch

@@ -31,7 +31,7 @@ else:

31 _flashmla_extension_C_AVAILABLE = False

34 -def is_flashmla_supported() -> Tuple[bool, Optional[str]]:

34 +def is_flashmla_supported() -> tuple[bool, Optional[str]]:

35 """

36 Return: is_supported_flag, unsupported_reason (optional).

37 """

@@ -57,7 +57,7 @@ def get_mla_metadata(

57 num_heads_q: Optional[int] = None,

58 is_fp8_kvcache: bool = False,

59 topk: Optional[int] = None,

60 -) -> Tuple[torch.Tensor, torch.Tensor]:

60 +) -> tuple[torch.Tensor, torch.Tensor]:

61 """

62 Arguments:

63 - cache_seqlens: (batch_size), dtype torch.int32.

@@ -101,7 +101,7 @@ def flash_mla_with_kvcache(

101 descale_k: Optional[torch.Tensor] = None,

102 is_fp8_kvcache: bool = False,

103 indices: Optional[torch.Tensor] = None,

104 -) -> Tuple[torch.Tensor, torch.Tensor]:

104 +) -> tuple[torch.Tensor, torch.Tensor]:

105 """

106 Arguments:

107 - q: (batch_size, seq_len_q, num_heads_q, head_dim).

@@ -183,7 +183,7 @@ def flash_mla_sparse_prefill(

183 indices: torch.Tensor,

184 sm_scale: float,

185 d_v: int = 512,

186 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

186 +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

187 """

188 Sparse attention prefill kernel

189

vllm/attention/ops/paged_attn.py

+5 -5

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from dataclasses import dataclass

5 -from typing import List, Optional, Tuple

5 +from typing import Optional

7 import torch

@@ -41,7 +41,7 @@ class PagedAttentionMetadata:

42 class PagedAttention:

43 @staticmethod

44 - def get_supported_head_sizes() -> List[int]:

44 + def get_supported_head_sizes() -> list[int]:

45 return [32, 64, 80, 96, 112, 120, 128, 192, 256]

47 @staticmethod

@@ -51,7 +51,7 @@ class PagedAttention:

51 num_kv_heads: int,

52 head_size: int,

53 cache_dtype_str: str = "auto",

54 - ) -> Tuple[int, ...]:

54 + ) -> tuple[int, ...]:

55 return (2, num_blocks, block_size * num_kv_heads * head_size)

57 @staticmethod

@@ -59,7 +59,7 @@ class PagedAttention:

59 kv_cache: torch.Tensor,

60 num_kv_heads: int,

61 head_size: int,

62 - ) -> Tuple[torch.Tensor, torch.Tensor]:

62 + ) -> tuple[torch.Tensor, torch.Tensor]:

63 x = 16 // kv_cache.element_size()

64 num_blocks = kv_cache.shape[1]

@@ -255,7 +255,7 @@ class PagedAttention:

255

256 @staticmethod

257 def copy_blocks(

258 - kv_caches: List[torch.Tensor],

258 + kv_caches: list[torch.Tensor],

259 src_to_dists: torch.Tensor,

260 ) -> None:

261 key_caches = [kv_cache[0] for kv_cache in kv_caches]

vllm/engine/arg_utils.py

+7 -10

@@ -14,11 +14,8 @@ from typing import (

14 Annotated,

15 Any,

16 Callable,

17 - Dict,

18 - List,

17 Literal,

18 Optional,

21 - Type,

19 TypeVar,

20 Union,

21 cast,

@@ -325,7 +322,7 @@ class EngineArgs:

322 """Arguments for vLLM engine."""

323

324 model: str = ModelConfig.model

328 - served_model_name: Optional[Union[str, List[str]]] = ModelConfig.served_model_name

325 + served_model_name: Optional[Union[str, list[str]]] = ModelConfig.served_model_name

326 tokenizer: Optional[str] = ModelConfig.tokenizer

327 hf_config_path: Optional[str] = ModelConfig.hf_config_path

328 runner: RunnerOption = ModelConfig.runner

@@ -350,7 +347,7 @@ class EngineArgs:

347 # is intended for expert use only. The API may change without

348 # notice.

349 distributed_executor_backend: Optional[

353 - Union[str, DistributedExecutorBackend, Type[ExecutorBase]]

350 + Union[str, DistributedExecutorBackend, type[ExecutorBase]]

351 ] = ParallelConfig.distributed_executor_backend

352 # number of P/D disaggregation (or other disaggregation) workers

353 pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size

@@ -418,7 +415,7 @@ class EngineArgs:

415 media_io_kwargs: dict[str, dict[str, Any]] = get_field(

416 MultiModalConfig, "media_io_kwargs"

417 )

421 - mm_processor_kwargs: Optional[Dict[str, Any]] = MultiModalConfig.mm_processor_kwargs

418 + mm_processor_kwargs: Optional[dict[str, Any]] = MultiModalConfig.mm_processor_kwargs

419 disable_mm_preprocessor_cache: bool = False # DEPRECATED

420 mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb

421 mm_processor_cache_type: Optional[MMCacheType] = (

@@ -436,7 +433,7 @@ class EngineArgs:

433 enable_lora_bias: bool = LoRAConfig.bias_enabled

434 max_loras: int = LoRAConfig.max_loras

435 max_lora_rank: int = LoRAConfig.max_lora_rank

439 - default_mm_loras: Optional[Dict[str, str]] = LoRAConfig.default_mm_loras

436 + default_mm_loras: Optional[dict[str, str]] = LoRAConfig.default_mm_loras

437 fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras

438 max_cpu_loras: Optional[int] = LoRAConfig.max_cpu_loras

439 lora_dtype: Optional[Union[str, torch.dtype]] = LoRAConfig.lora_dtype

@@ -446,7 +443,7 @@ class EngineArgs:

443 num_gpu_blocks_override: Optional[int] = CacheConfig.num_gpu_blocks_override

444 num_lookahead_slots: int = SchedulerConfig.num_lookahead_slots

445 model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")

449 - ignore_patterns: Optional[Union[str, List[str]]] = LoadConfig.ignore_patterns

446 + ignore_patterns: Optional[Union[str, list[str]]] = LoadConfig.ignore_patterns

447

448 enable_chunked_prefill: Optional[bool] = SchedulerConfig.enable_chunked_prefill

449 disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input

@@ -467,7 +464,7 @@ class EngineArgs:

464

465 logits_processor_pattern: Optional[str] = ModelConfig.logits_processor_pattern

466

470 - speculative_config: Optional[Dict[str, Any]] = None

467 + speculative_config: Optional[dict[str, Any]] = None

468

469 show_hidden_metrics_for_version: Optional[str] = (

470 ObservabilityConfig.show_hidden_metrics_for_version

@@ -477,7 +474,7 @@ class EngineArgs:

474 ObservabilityConfig.collect_detailed_traces

475 )

476 scheduling_policy: SchedulerPolicy = SchedulerConfig.policy

480 - scheduler_cls: Union[str, Type[object]] = SchedulerConfig.scheduler_cls

477 + scheduler_cls: Union[str, type[object]] = SchedulerConfig.scheduler_cls

478

479 pooler_config: Optional[PoolerConfig] = ModelConfig.pooler_config

480 override_pooler_config: Optional[Union[dict, PoolerConfig]] = (

vllm/engine/metrics.py

+21 -21

@@ -3,7 +3,7 @@

4 import time

5 from collections import Counter as CollectionsCounter

6 -from typing import Dict, List, Optional, Type, Union, cast

6 +from typing import Optional, Union, cast

8 import numpy as np

9 import prometheus_client

@@ -43,7 +43,7 @@ class Metrics:

43 _counter_cls = prometheus_client.Counter

44 _histogram_cls = prometheus_client.Histogram

46 - def __init__(self, labelnames: List[str], vllm_config: VllmConfig):

46 + def __init__(self, labelnames: list[str], vllm_config: VllmConfig):

47 # Unregister any existing vLLM collectors (for CI/CD)

48 self._unregister_vllm_metrics()

@@ -304,7 +304,7 @@ class _RayGaugeWrapper:

304 self,

305 name: str,

306 documentation: str = "",

307 - labelnames: Optional[List[str]] = None,

307 + labelnames: Optional[list[str]] = None,

308 multiprocess_mode: str = "",

309 ):

310 del multiprocess_mode

@@ -330,7 +330,7 @@ class _RayCounterWrapper:

330 prometheus_client.Counter"""

331

332 def __init__(

333 - self, name: str, documentation: str = "", labelnames: Optional[List[str]] = None

333 + self, name: str, documentation: str = "", labelnames: Optional[list[str]] = None

334 ):

335 labelnames_tuple = tuple(labelnames) if labelnames else None

336 self._counter = ray_metrics.Counter(

@@ -355,8 +355,8 @@ class _RayHistogramWrapper:

355 self,

356 name: str,

357 documentation: str = "",

358 - labelnames: Optional[List[str]] = None,

359 - buckets: Optional[List[float]] = None,

358 + labelnames: Optional[list[str]] = None,

359 + buckets: Optional[list[float]] = None,

360 ):

361 labelnames_tuple = tuple(labelnames) if labelnames else None

362 boundaries = buckets if buckets else []

@@ -381,17 +381,17 @@ class RayMetrics(Metrics):

381 Provides the same metrics as Metrics but uses Ray's util.metrics library.

382 """

383

384 - _gauge_cls: Type[prometheus_client.Gauge] = cast(

385 - Type[prometheus_client.Gauge], _RayGaugeWrapper

384 + _gauge_cls: type[prometheus_client.Gauge] = cast(

385 + type[prometheus_client.Gauge], _RayGaugeWrapper

386 )

387 - _counter_cls: Type[prometheus_client.Counter] = cast(

388 - Type[prometheus_client.Counter], _RayCounterWrapper

387 + _counter_cls: type[prometheus_client.Counter] = cast(

388 + type[prometheus_client.Counter], _RayCounterWrapper

389 )

390 - _histogram_cls: Type[prometheus_client.Histogram] = cast(

391 - Type[prometheus_client.Histogram], _RayHistogramWrapper

390 + _histogram_cls: type[prometheus_client.Histogram] = cast(

391 + type[prometheus_client.Histogram], _RayHistogramWrapper

392 )

393

394 - def __init__(self, labelnames: List[str], vllm_config: VllmConfig):

394 + def __init__(self, labelnames: list[str], vllm_config: VllmConfig):

395 if ray_metrics is None:

396 raise ImportError("RayMetrics requires Ray to be installed.")

397 super().__init__(labelnames, vllm_config)

@@ -401,14 +401,14 @@ class RayMetrics(Metrics):

401 pass

402

403

404 -def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:

404 +def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:

405 """

406 Builds a list of buckets with increasing powers of 10 multiplied by

407 mantissa values until the value exceeds the specified maximum.

408

409 """

410 exponent = 0

411 - buckets: List[int] = []

411 + buckets: list[int] = []

412 while True:

413 for m in mantissa_lst:

414 value = m * 10**exponent

@@ -419,7 +419,7 @@ def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:

419 exponent += 1

420

421

422 -def build_1_2_5_buckets(max_value: int) -> List[int]:

422 +def build_1_2_5_buckets(max_value: int) -> list[int]:

423 """

424 Example:

425 >>> build_1_2_5_buckets(100)

@@ -428,7 +428,7 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:

428 return build_buckets([1, 2, 5], max_value)

429

430

431 -def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:

431 +def build_1_2_3_5_8_buckets(max_value: int) -> list[int]:

432 """

433 Example:

434 >>> build_1_2_3_5_8_buckets(100)

@@ -442,7 +442,7 @@ def local_interval_elapsed(now: float, last_log: float, local_interval: float) -

442 return elapsed_time > local_interval

443

444

445 -def get_throughput(tracked_stats: List[int], now: float, last_log: float) -> float:

445 +def get_throughput(tracked_stats: list[int], now: float, last_log: float) -> float:

446 return float(np.sum(tracked_stats) / (now - last_log))

447

448

@@ -530,7 +530,7 @@ class PrometheusStatLogger(StatLoggerBase):

530 _gauge_cls = prometheus_client.Gauge

531

532 def __init__(

533 - self, local_interval: float, labels: Dict[str, str], vllm_config: VllmConfig

533 + self, local_interval: float, labels: dict[str, str], vllm_config: VllmConfig

534 ) -> None:

535 super().__init__(local_interval, vllm_config)

536 # Prometheus metrics

@@ -558,12 +558,12 @@ class PrometheusStatLogger(StatLoggerBase):

558 for label, count in data.items():

559 counter.labels(**{**self.labels, label_key: label}).inc(count)

560

561 - def _log_histogram(self, histogram, data: Union[List[int], List[float]]) -> None:

561 + def _log_histogram(self, histogram, data: Union[list[int], list[float]]) -> None:

562 # Convenience function for logging list to histogram.

563 for datum in data:

564 histogram.labels(**self.labels).observe(datum)

565

566 - def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:

566 + def _log_gauge_string(self, gauge, data: dict[str, str]) -> None:

567 gauge.labels(**data).set_to_current_time()

568

569 def _log_prometheus(self, stats: Stats) -> None:

vllm/engine/metrics_types.py

+17 -18

@@ -16,7 +16,6 @@ do this in Python code and lazily import prometheus_client.

16 import time

17 from abc import ABC, abstractmethod

18 from dataclasses import dataclass

19 -from typing import List

20 from vllm.config import SupportsMetricsInfo, VllmConfig

@@ -43,26 +42,26 @@ class Stats:

42 num_prompt_tokens_iter: int

43 num_generation_tokens_iter: int

44 num_tokens_iter: int

46 - time_to_first_tokens_iter: List[float]

47 - inter_token_latencies_iter: List[float]

45 + time_to_first_tokens_iter: list[float]

46 + inter_token_latencies_iter: list[float]

47 num_preemption_iter: int

49 # Request stats (should have _requests suffix)

50 # Latency

52 - time_e2e_requests: List[float]

53 - time_queue_requests: List[float]

54 - time_inference_requests: List[float]

55 - time_prefill_requests: List[float]

56 - time_decode_requests: List[float]

51 + time_e2e_requests: list[float]

52 + time_queue_requests: list[float]

53 + time_inference_requests: list[float]

54 + time_prefill_requests: list[float]

55 + time_decode_requests: list[float]

56 # Metadata

58 - num_prompt_tokens_requests: List[int]

59 - num_generation_tokens_requests: List[int]

60 - n_requests: List[int]

61 - max_num_generation_tokens_requests: List[int]

62 - max_tokens_requests: List[int]

63 - finished_reason_requests: List[str]

64 - waiting_lora_adapters: List[str]

65 - running_lora_adapters: List[str]

57 + num_prompt_tokens_requests: list[int]

58 + num_generation_tokens_requests: list[int]

59 + n_requests: list[int]

60 + max_num_generation_tokens_requests: list[int]

61 + max_tokens_requests: list[int]

62 + finished_reason_requests: list[str]

63 + waiting_lora_adapters: list[str]

64 + running_lora_adapters: list[str]

65 max_lora: str

@@ -71,8 +70,8 @@ class StatLoggerBase(ABC):

71 def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:

72 # Tracked stats over current local logging interval.

74 - self.num_prompt_tokens: List[int] = []

75 - self.num_generation_tokens: List[int] = []

73 + self.num_prompt_tokens: list[int] = []

74 + self.num_generation_tokens: list[int] = []

75 self.last_local_log = time.time()

76 self.local_interval = local_interval

vllm/entrypoints/harmony_utils.py

+12 -12

@@ -6,7 +6,7 @@ from __future__ import annotations

6 import datetime

7 import json

8 from collections.abc import Iterable, Sequence

9 -from typing import Literal, Optional, Union

9 +from typing import Literal, Union

11 from openai.types.responses import (

12 ResponseFunctionToolCall,

@@ -79,13 +79,13 @@ def get_encoding():

81 def get_system_message(

82 - model_identity: Optional[str] = None,

83 - reasoning_effort: Optional[Literal["high", "medium", "low"]] = None,

84 - start_date: Optional[str] = None,

85 - browser_description: Optional[str] = None,

86 - python_description: Optional[str] = None,

87 - container_description: Optional[str] = None,

88 - instructions: Optional[str] = None,

82 + model_identity: str | None = None,

83 + reasoning_effort: Literal["high", "medium", "low"] | None = None,

84 + start_date: str | None = None,

85 + browser_description: str | None = None,

86 + python_description: str | None = None,

87 + container_description: str | None = None,

88 + instructions: str | None = None,

89 with_custom_tools: bool = False,

90 ) -> Message:

91 sys_msg_content = SystemContent.new()

@@ -137,8 +137,8 @@ def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):

137

138

139 def get_developer_message(

140 - instructions: Optional[str] = None,

141 - tools: Optional[list[Union[Tool, ChatCompletionToolsParam]]] = None,

140 + instructions: str | None = None,

141 + tools: list[Union[Tool, ChatCompletionToolsParam]] | None = None,

142 ) -> Message:

143 dev_msg_content = DeveloperContent.new()

144 if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:

@@ -202,7 +202,7 @@ def parse_response_input(

202 msg = msg.with_channel("final")

203 elif response_msg["type"] == "function_call_output":

204 call_id = response_msg["call_id"]

205 - call_response: Optional[ResponseFunctionToolCall] = None

205 + call_response: ResponseFunctionToolCall | None = None

206 for prev_response in reversed(prev_responses):

207 if (

208 isinstance(prev_response, ResponseFunctionToolCall)

@@ -450,7 +450,7 @@ def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:

450

451 def parse_chat_output(

452 token_ids: Sequence[int],

453 -) -> tuple[Optional[str], Optional[str], bool]:

453 +) -> tuple[str | None, str | None, bool]:

454 parser = parse_output_into_messages(token_ids)

455 output_msgs = parser.messages

456 is_tool_call = False # TODO: update this when tool call is supported

vllm/executor/executor_base.py

+8 -8

@@ -6,7 +6,7 @@ import time

6 from abc import ABC, abstractmethod

7 from collections.abc import Awaitable

8 from functools import cached_property

9 -from typing import Any, Callable, List, Optional, Set, Union

9 +from typing import Any, Callable, Optional, Union

11 from typing_extensions import TypeVar

@@ -143,7 +143,7 @@ class ExecutorBase(ABC):

143

144 def execute_model(

145 self, execute_model_req: ExecuteModelRequest

146 - ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:

146 + ) -> Optional[list[Union[SamplerOutput, PoolerOutput]]]:

147 output = self.collective_rpc("execute_model", args=(execute_model_req,))

148 return output[0]

149

@@ -163,7 +163,7 @@ class ExecutorBase(ABC):

163 assert lora_id > 0, "lora_id must be greater than 0."

164 return all(self.collective_rpc("pin_lora", args=(lora_id,)))

165

166 - def list_loras(self) -> Set[int]:

166 + def list_loras(self) -> set[int]:

167 sets = self.collective_rpc("list_loras")

168 for s in sets:

169 assert s == sets[0], "All workers should have the same LORAs."

@@ -238,7 +238,7 @@ class ExecutorBase(ABC):

238

239 async def execute_model_async(

240 self, execute_model_req: ExecuteModelRequest

241 - ) -> List[SamplerOutput]:

241 + ) -> list[SamplerOutput]:

242 """Executes one model step on the given sequences."""

243 output = await make_async(self.execute_model)(execute_model_req)

244 return output

@@ -272,7 +272,7 @@ class DistributedExecutorBase(ExecutorBase):

272 def execute_model(

273 self,

274 execute_model_req: ExecuteModelRequest,

275 - ) -> List[SamplerOutput]:

275 + ) -> list[SamplerOutput]:

276 # TODO: unify into collective_rpc

277 if self.parallel_worker_tasks is None:

278 self.parallel_worker_tasks = self._run_workers(

@@ -299,7 +299,7 @@ class DistributedExecutorBase(ExecutorBase):

299 @abstractmethod

300 def _driver_execute_model(

301 self, execute_model_req: Optional[ExecuteModelRequest]

302 - ) -> Optional[List[SamplerOutput]]:

302 + ) -> Optional[list[SamplerOutput]]:

303 """Run execute_model in the driver worker.

304

305 Passing None will cause the driver to stop the model execution loop

@@ -346,7 +346,7 @@ class DistributedExecutorBase(ExecutorBase):

346

347 async def execute_model_async(

348 self, execute_model_req: ExecuteModelRequest

349 - ) -> List[SamplerOutput]:

349 + ) -> list[SamplerOutput]:

350 if self.parallel_worker_tasks is None:

351 # Start model execution loop running in the parallel workers

352 self.parallel_worker_tasks = asyncio.create_task(

@@ -371,7 +371,7 @@ class DistributedExecutorBase(ExecutorBase):

371 async def _driver_execute_model_async(

372 self,

373 execute_model_req: Optional[ExecuteModelRequest] = None,

374 - ) -> List[SamplerOutput]:

374 + ) -> list[SamplerOutput]:

375 """Execute the model asynchronously in the driver worker.

376

377 Passing None will cause the driver to stop the model execution

vllm/executor/msgspec_utils.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from array import array

5 -from typing import Any, Type

5 +from typing import Any

7 from vllm.multimodal.inputs import MultiModalKwargs

8 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE

@@ -23,7 +23,7 @@ def encode_hook(obj: Any) -> Any:

23 return dict(obj)

26 -def decode_hook(type: Type, obj: Any) -> Any:

26 +def decode_hook(type: type, obj: Any) -> Any:

27 """Custom msgspec dec hook that supports array types and MultiModalKwargs.

29 See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder

vllm/executor/ray_distributed_executor.py

+15 -15

@@ -5,7 +5,7 @@ import asyncio

5 import os

6 from collections import defaultdict

7 from dataclasses import dataclass

8 -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union

8 +from typing import TYPE_CHECKING, Any, Callable, Optional, Union

10 import cloudpickle

11 import msgspec

@@ -114,10 +114,10 @@ class RayDistributedExecutor(DistributedExecutorBase):

114 self._init_workers_ray(placement_group)

115

116 self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)

117 - self.output_decoder = msgspec.msgpack.Decoder(Optional[List[SamplerOutput]])

117 + self.output_decoder = msgspec.msgpack.Decoder(Optional[list[SamplerOutput]])

118 self.use_v1 = envs.VLLM_USE_V1

119

120 - self.pp_locks: Optional[List[asyncio.Lock]] = None

120 + self.pp_locks: Optional[list[asyncio.Lock]] = None

121 if not self.use_ray_compiled_dag:

122 self.driver_exec_method = make_async(self.driver_worker.execute_method)

123

@@ -137,7 +137,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

137 ray.kill(worker)

138 self.forward_dag = None

139

140 - def _configure_ray_workers_use_nsight(self, ray_remote_kwargs) -> Dict[str, Any]:

140 + def _configure_ray_workers_use_nsight(self, ray_remote_kwargs) -> dict[str, Any]:

141 # If nsight profiling is enabled, we need to set the profiling

142 # configuration for the ray workers as runtime env.

143 runtime_env = ray_remote_kwargs.setdefault("runtime_env", {})

@@ -164,12 +164,12 @@ class RayDistributedExecutor(DistributedExecutorBase):

164 # It holds the resource for the driver worker.

165 self.driver_dummy_worker: Optional[RayWorkerWrapper] = None

166 # The remaining workers are the actual ray actors.

167 - self.workers: List[RayWorkerWrapper] = []

167 + self.workers: list[RayWorkerWrapper] = []

168

169 # Used in ray compiled DAG: indexed first by PP rank,

170 # and then TP rank. In other words, the inner list is

171 # the TP group of workers for a PP rank.

172 - self.pp_tp_workers: List[List[RayWorkerWrapper]] = []

172 + self.pp_tp_workers: list[list[RayWorkerWrapper]] = []

173

174 if self.parallel_config.ray_workers_use_nsight:

175 ray_remote_kwargs = self._configure_ray_workers_use_nsight(

@@ -179,7 +179,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

179 logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)

180

181 # Create the workers.

182 - bundle_indices: List[int]

182 + bundle_indices: list[int]

183 if envs.VLLM_RAY_BUNDLE_INDICES:

184 # Use the bundle indices specified by the user.

185 bundle_indices = list(map(int, envs.VLLM_RAY_BUNDLE_INDICES.split(",")))

@@ -200,7 +200,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

200 bundle_indices.append(bundle_id)

201 bundle_indices = bundle_indices[: self.parallel_config.world_size]

202

203 - worker_metadata: List[RayWorkerMetaData] = []

203 + worker_metadata: list[RayWorkerMetaData] = []

204 driver_ip = get_ip()

205 for rank, bundle_id in enumerate(bundle_indices):

206 scheduling_strategy = PlacementGroupSchedulingStrategy(

@@ -262,7 +262,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

262 "the driver on a GPU node."

263 )

264

265 - ip_counts: Dict[str, int] = {}

265 + ip_counts: dict[str, int] = {}

266 for ip in worker_ips:

267 ip_counts[ip] = ip_counts.get(ip, 0) + 1

268

@@ -416,11 +416,11 @@ class RayDistributedExecutor(DistributedExecutorBase):

416 # This is the list of workers that are rank 0 of each TP group EXCEPT

417 # global rank 0. These are the workers that will broadcast to the

418 # rest of the workers.

419 - self.tp_driver_workers: List[RayWorkerWrapper] = []

419 + self.tp_driver_workers: list[RayWorkerWrapper] = []

420 # This is the list of workers that are not drivers and not the first

421 # worker in a TP group. These are the workers that will be

422 # broadcasted to.

423 - self.non_driver_workers: List[RayWorkerWrapper] = []

423 + self.non_driver_workers: list[RayWorkerWrapper] = []

424

425 # Enforce rank order for correct rank to return final output.

426 for index, worker in enumerate(self.workers):

@@ -433,7 +433,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

433

434 def _driver_execute_model(

435 self, execute_model_req: Optional[ExecuteModelRequest]

436 - ) -> Optional[List[SamplerOutput]]:

436 + ) -> Optional[list[SamplerOutput]]:

437 """Run execute_model in the driver worker.

438

439 Passing None will cause the driver to stop the model execution

@@ -446,7 +446,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

446

447 def execute_model(

448 self, execute_model_req: ExecuteModelRequest

449 - ) -> List[SamplerOutput]:

449 + ) -> list[SamplerOutput]:

450 if not self.use_ray_spmd_worker:

451 return super().execute_model(execute_model_req)

452

@@ -675,7 +675,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

675

676 async def execute_model_async(

677 self, execute_model_req: ExecuteModelRequest

678 - ) -> List[SamplerOutput]:

678 + ) -> list[SamplerOutput]:

679 if not self.use_ray_spmd_worker:

680 return await super().execute_model_async(execute_model_req)

681

@@ -689,7 +689,7 @@ class RayDistributedExecutor(DistributedExecutorBase):

689

690 async def _driver_execute_model_async(

691 self, execute_model_req: Optional[ExecuteModelRequest] = None

692 - ) -> List[SamplerOutput]:

692 + ) -> list[SamplerOutput]:

693 assert not self.use_ray_spmd_worker, (

694 "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1"

695 )

vllm/executor/ray_utils.py

+8 -8

@@ -4,7 +4,7 @@

4 import os

5 import time

6 from collections import defaultdict

7 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

7 +from typing import TYPE_CHECKING, Optional, Union

9 import msgspec

@@ -59,7 +59,7 @@ try:

59 def get_node_ip(self) -> str:

60 return get_ip()

62 - def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:

62 + def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:

63 node_id = ray.get_runtime_context().get_node_id()

64 device_key = vllm.platforms.current_platform.ray_device_key

65 if not device_key:

@@ -72,7 +72,7 @@ try:

73 def execute_model_spmd(

74 self,

75 - req_or_tuple: Union[bytes, Tuple[bytes, Optional[IntermediateTensors]]],

75 + req_or_tuple: Union[bytes, tuple[bytes, Optional[IntermediateTensors]]],

76 ) -> bytes:

77 """Execute model in SPMD fashion: used only when SPMD worker and

78 compiled DAG are both enabled.

@@ -126,10 +126,10 @@ try:

126 def execute_model_ray(

127 self,

128 scheduler_output: Union[

129 - "SchedulerOutput", Tuple["SchedulerOutput", "IntermediateTensors"]

129 + "SchedulerOutput", tuple["SchedulerOutput", "IntermediateTensors"]

130 ],

131 ) -> Union[

132 - "ModelRunnerOutput", Tuple["SchedulerOutput", "IntermediateTensors"]

132 + "ModelRunnerOutput", tuple["SchedulerOutput", "IntermediateTensors"]

133 ]:

134 # This method is used by Ray Compiled Graph to execute the model,

135 # and it needs a special logic of self.setup_device_if_necessary()

@@ -156,7 +156,7 @@ try:

156 output = output.get_output()

157 return output

158

159 - def override_env_vars(self, vars: Dict[str, str]):

159 + def override_env_vars(self, vars: dict[str, str]):

160 os.environ.update(vars)

161

162 ray_import_err = None

@@ -201,7 +201,7 @@ def _verify_bundles(

201 # bundle_idx -> bundle (e.g., {"GPU": 1})

202 bundles = pg_data["bundles"]

203 # node_id -> List of bundle (e.g., {"GPU": 1})

204 - node_id_to_bundle: Dict[str, List[Dict[str, float]]] = defaultdict(list)

204 + node_id_to_bundle: dict[str, list[dict[str, float]]] = defaultdict(list)

205

206 for bundle_idx, node_id in bundle_to_node_ids.items():

207 node_id_to_bundle[node_id].append(bundles[bundle_idx])

@@ -383,7 +383,7 @@ def initialize_ray_cluster(

383 device_str,

384 )

385 # Create a new placement group

386 - placement_group_specs: List[Dict[str, float]] = [

386 + placement_group_specs: list[dict[str, float]] = [

387 {device_str: 1.0} for _ in range(parallel_config.world_size)

388 ]

389

vllm/executor/uniproc_executor.py

+5 -5

@@ -4,7 +4,7 @@ import os

4 from concurrent.futures import Future, ThreadPoolExecutor

5 from functools import cached_property

6 from multiprocessing import Lock

7 -from typing import Any, Callable, Dict, List, Optional, Tuple, Union

7 +from typing import Any, Callable, Optional, Union

9 import torch

10 import torch.distributed as dist

@@ -68,10 +68,10 @@ class UniProcExecutor(ExecutorBase):

68 self,

69 method: Union[str, Callable],

70 timeout: Optional[float] = None,

71 - args: Tuple = (),

72 - kwargs: Optional[Dict] = None,

71 + args: tuple = (),

72 + kwargs: Optional[dict] = None,

73 non_block: bool = False,

74 - ) -> List[Any]:

74 + ) -> list[Any]:

75 if kwargs is None:

76 kwargs = {}

77 if self.mm_receiver_cache is not None and method == "execute_model":

@@ -158,7 +158,7 @@ class ExecutorWithExternalLauncher(UniProcExecutor):

158 local_rank = int(os.environ["LOCAL_RANK"])

159 return distributed_init_method, rank, local_rank

160

161 - def determine_num_available_blocks(self) -> Tuple[int, int]:

161 + def determine_num_available_blocks(self) -> tuple[int, int]:

162 """

163 Determine the number of available KV blocks.

164 Add an additional all_reduce to get the min across all ranks.

vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py

+2 -5

@@ -1,9 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from typing import (

4 - List, # noqa: UP035

5 - Optional,

6 -)

3 +from typing import Optional

5 import torch

@@ -32,7 +29,7 @@ def flashinfer_fused_moe_blockscale_fp8(

29 intermediate_size: int,

30 expert_offset: int,

31 local_num_experts: int,

35 - block_shape: List[int], # noqa: UP006

32 + block_shape: list[int],

33 routed_scaling: float = 1.0,

34 ) -> torch.Tensor:

35 from vllm.utils.flashinfer import flashinfer_trtllm_fp8_block_scale_moe

vllm/multimodal/inputs.py

+1 -1

@@ -289,7 +289,7 @@ class MultiModalFieldElem:

289 return (

290 (self.modality, self.key) == (other.modality, other.key)

291 and data_equal

292 - and type(self.field) == type(other.field)

292 + and type(self.field) is type(other.field)

293 ) # noqa: E721

294

295

vllm/plugins/io_processors/__init__.py

+1 -2

@@ -4,7 +4,6 @@

4 from __future__ import annotations

6 import logging

7 -from typing import Optional

8 from vllm.config import VllmConfig

9 from vllm.plugins import load_plugins_by_group

@@ -15,7 +14,7 @@ logger = logging.getLogger(__name__)

16 def get_io_processor(

18 - vllm_config: VllmConfig, plugin_from_init: Optional[str] = None

17 + vllm_config: VllmConfig, plugin_from_init: str | None = None

18 ) -> IOProcessor | None:

19 # Input.Output processors are loaded as plugins under the

20 # 'vllm.io_processor_plugins' group. Similar to platform

vllm/utils/__init__.py

+48 -51

@@ -68,7 +68,6 @@ from typing import (

68 Generic,

69 Literal,

70 NamedTuple,

71 - Optional,

71 TextIO,

72 TypeVar,

73 Union,

@@ -247,9 +246,7 @@ class CacheInfo(NamedTuple):

246

247

248 class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):

250 - def __init__(

251 - self, capacity: float, getsizeof: Optional[Callable[[_V], float]] = None

252 - ):

249 + def __init__(self, capacity: float, getsizeof: Callable[[_V], float] | None = None):

250 super().__init__(capacity, getsizeof)

251

252 self.pinned_items = set[_K]()

@@ -324,15 +321,15 @@ class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):

321 self._LRUCache__order[key] = None # type: ignore

322

323 @overload

327 - def get(self, key: _K, /) -> Optional[_V]: ...

324 + def get(self, key: _K, /) -> _V | None: ...

325

326 @overload

327 def get(self, key: _K, /, default: Union[_V, _T]) -> Union[_V, _T]: ...

328

329 def get(

333 - self, key: _K, /, default: Optional[Union[_V, _T]] = None

334 - ) -> Optional[Union[_V, _T]]:

335 - value: Optional[Union[_V, _T]]

330 + self, key: _K, /, default: Union[_V, _T] | None = None

331 + ) -> Union[_V, _T] | None:

332 + value: Union[_V, _T] | None

333 if key in self:

334 value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]

335

@@ -350,9 +347,9 @@ class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):

347 def pop(self, key: _K, default: Union[_V, _T]) -> Union[_V, _T]: ...

348

349 def pop(

353 - self, key: _K, default: Optional[Union[_V, _T]] = None

354 - ) -> Optional[Union[_V, _T]]:

355 - value: Optional[Union[_V, _T]]

350 + self, key: _K, default: Union[_V, _T] | None = None

351 + ) -> Union[_V, _T] | None:

352 + value: Union[_V, _T] | None

353 if key not in self:

354 return default

355

@@ -379,7 +376,7 @@ class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):

376 """

377 self.pinned_items.remove(key)

378

382 - def _on_remove(self, key: _K, value: Optional[_V]) -> None:

379 + def _on_remove(self, key: _K, value: _V | None) -> None:

380 pass

381

382 def remove_oldest(self, *, remove_pinned: bool = False) -> None:

@@ -705,7 +702,7 @@ def in_loop(event_loop: AbstractEventLoop) -> bool:

702

703

704 def make_async(

708 - func: Callable[P, T], executor: Optional[concurrent.futures.Executor] = None

705 + func: Callable[P, T], executor: concurrent.futures.Executor | None = None

706 ) -> Callable[P, Awaitable[T]]:

707 """Take a blocking function, and run it on in an executor thread.

708

@@ -940,7 +937,7 @@ def _get_open_port() -> int:

937 return s.getsockname()[1]

938

939

943 -def find_process_using_port(port: int) -> Optional[psutil.Process]:

940 +def find_process_using_port(port: int) -> psutil.Process | None:

941 # TODO: We can not check for running processes with network

942 # port on macOS. Therefore, we can not have a full graceful shutdown

943 # of vLLM. For now, let's not look for processes in this case.

@@ -1025,8 +1022,8 @@ def _generate_random_fp8(

1022

1023

1024 def get_kv_cache_torch_dtype(

1028 - cache_dtype: Optional[Union[str, torch.dtype]],

1029 - model_dtype: Optional[Union[str, torch.dtype]] = None,

1025 + cache_dtype: Union[str, torch.dtype] | None,

1026 + model_dtype: Union[str, torch.dtype] | None = None,

1027 ) -> torch.dtype:

1028 if isinstance(cache_dtype, str):

1029 if cache_dtype == "auto":

@@ -1053,11 +1050,11 @@ def create_kv_caches_with_random_flash(

1050 num_layers: int,

1051 num_heads: int,

1052 head_size: int,

1056 - cache_dtype: Optional[Union[str, torch.dtype]],

1057 - model_dtype: Optional[Union[str, torch.dtype]] = None,

1058 - seed: Optional[int] = None,

1059 - device: Optional[str] = "cuda",

1060 - cache_layout: Optional[str] = "NHD",

1053 + cache_dtype: Union[str, torch.dtype] | None,

1054 + model_dtype: Union[str, torch.dtype] | None = None,

1055 + seed: int | None = None,

1056 + device: str | None = "cuda",

1057 + cache_layout: str | None = "NHD",

1058 ) -> tuple[list[torch.Tensor], list[torch.Tensor]]:

1059 from vllm.platforms import current_platform

1060

@@ -1095,10 +1092,10 @@ def create_kv_caches_with_random(

1092 num_layers: int,

1093 num_heads: int,

1094 head_size: int,

1098 - cache_dtype: Optional[Union[str, torch.dtype]],

1099 - model_dtype: Optional[Union[str, torch.dtype]] = None,

1100 - seed: Optional[int] = None,

1101 - device: Optional[str] = "cuda",

1095 + cache_dtype: Union[str, torch.dtype] | None,

1096 + model_dtype: Union[str, torch.dtype] | None = None,

1097 + seed: int | None = None,

1098 + device: str | None = "cuda",

1099 ) -> tuple[list[torch.Tensor], list[torch.Tensor]]:

1100 if cache_dtype == "fp8" and head_size % 16:

1101 raise ValueError(

@@ -1156,7 +1153,7 @@ def is_uva_available() -> bool:

1153

1154

1155 class DeviceMemoryProfiler:

1159 - def __init__(self, device: Optional[torch.types.Device] = None):

1156 + def __init__(self, device: torch.types.Device | None = None):

1157 self.device = device

1158

1159 def current_memory_usage(self) -> float:

@@ -1184,7 +1181,7 @@ def make_ndarray_with_pad(

1181 pad: T,

1182 dtype: npt.DTypeLike,

1183 *,

1187 - max_len: Optional[int] = None,

1184 + max_len: int | None = None,

1185 ) -> npt.NDArray:

1186 """

1187 Make a padded array from 2D inputs.

@@ -1209,8 +1206,8 @@ def make_tensor_with_pad(

1206 pad: T,

1207 dtype: torch.dtype,

1208 *,

1212 - max_len: Optional[int] = None,

1213 - device: Optional[Union[str, torch.device]] = None,

1209 + max_len: int | None = None,

1210 + device: Union[str, torch.device] | None = None,

1211 pin_memory: bool = False,

1212 ) -> torch.Tensor:

1213 """

@@ -1405,7 +1402,7 @@ def find_nccl_library() -> str:

1402 return so_file

1403

1404

1408 -def find_nccl_include_paths() -> Optional[list[str]]:

1405 +def find_nccl_include_paths() -> list[str] | None:

1406 """

1407 We either use the nccl.h specified by the `VLLM_NCCL_INCLUDE_PATH`

1408 environment variable, or we find the library file brought by

@@ -1525,7 +1522,7 @@ F = TypeVar("F", bound=Callable[..., Any])

1522 def deprecate_args(

1523 start_index: int,

1524 is_deprecated: Union[bool, Callable[[], bool]] = True,

1528 - additional_message: Optional[str] = None,

1525 + additional_message: str | None = None,

1526 ) -> Callable[[F], F]:

1527 if not callable(is_deprecated):

1528 is_deprecated = partial(identity, is_deprecated)

@@ -1565,7 +1562,7 @@ def deprecate_args(

1562 def deprecate_kwargs(

1563 *kws: str,

1564 is_deprecated: Union[bool, Callable[[], bool]] = True,

1568 - additional_message: Optional[str] = None,

1565 + additional_message: str | None = None,

1566 ) -> Callable[[F], F]:

1567 deprecated_kws = set(kws)

1568

@@ -1598,7 +1595,7 @@ def deprecate_kwargs(

1595

1596

1597 @lru_cache(maxsize=8)

1601 -def _cuda_device_count_stateless(cuda_visible_devices: Optional[str] = None) -> int:

1598 +def _cuda_device_count_stateless(cuda_visible_devices: str | None = None) -> int:

1599 # Note: cuda_visible_devices is not used, but we keep it as an argument for

1600 # LRU Cache purposes.

1601

@@ -1746,7 +1743,7 @@ class FlexibleArgumentParser(ArgumentParser):

1743 ' --json-arg \'{"key4": ["value3", "value4", "value5"]}\'\n'

1744 " --json-arg.key4+ value3 --json-arg.key4+='value4,value5'\n\n"

1745 )

1749 - _search_keyword: Optional[str] = None

1746 + _search_keyword: str | None = None

1747

1748 def __init__(self, *args, **kwargs):

1749 # Set the default "formatter_class" to SortedHelpFormatter

@@ -2245,7 +2242,7 @@ def supports_kw(

2242

2243 def get_allowed_kwarg_only_overrides(

2244 callable: Callable[..., object],

2248 - overrides: Optional[Mapping[str, object]],

2245 + overrides: Mapping[str, object] | None,

2246 *,

2247 requires_kw_only: bool = True,

2248 allow_var_kwargs: bool = False,

@@ -2695,10 +2692,10 @@ vllm_lib = Library("vllm", "FRAGMENT") # noqa

2692 def direct_register_custom_op(

2693 op_name: str,

2694 op_func: Callable,

2698 - mutates_args: Optional[list[str]] = None,

2699 - fake_impl: Optional[Callable] = None,

2700 - target_lib: Optional[Library] = None,

2701 - dispatch_key: Optional[str] = None,

2695 + mutates_args: list[str] | None = None,

2696 + fake_impl: Callable | None = None,

2697 + target_lib: Library | None = None,

2698 + dispatch_key: str | None = None,

2699 tags: tuple[torch.Tag, ...] = (),

2700 ):

2701 """

@@ -3016,7 +3013,7 @@ def split_zmq_path(path: str) -> tuple[str, str, str]:

3013 return scheme, host, port

3014

3015

3019 -def make_zmq_path(scheme: str, host: str, port: Optional[int] = None) -> str:

3016 +def make_zmq_path(scheme: str, host: str, port: int | None = None) -> str:

3017 """Make a ZMQ path from its parts.

3018

3019 Args:

@@ -3039,9 +3036,9 @@ def make_zmq_socket(

3036 ctx: Union[zmq.asyncio.Context, zmq.Context], # type: ignore[name-defined]

3037 path: str,

3038 socket_type: Any,

3042 - bind: Optional[bool] = None,

3043 - identity: Optional[bytes] = None,

3044 - linger: Optional[int] = None,

3039 + bind: bool | None = None,

3040 + identity: bytes | None = None,

3041 + linger: int | None = None,

3042 ) -> Union[zmq.Socket, zmq.asyncio.Socket]: # type: ignore[name-defined]

3043 """Make a ZMQ socket with the proper bind/connect semantics."""

3044

@@ -3098,9 +3095,9 @@ def make_zmq_socket(

3095 def zmq_socket_ctx(

3096 path: str,

3097 socket_type: Any,

3101 - bind: Optional[bool] = None,

3098 + bind: bool | None = None,

3099 linger: int = 0,

3103 - identity: Optional[bytes] = None,

3100 + identity: bytes | None = None,

3101 ) -> Iterator[zmq.Socket]:

3102 """Context manager for a ZMQ socket"""

3103

@@ -3163,7 +3160,7 @@ def get_mp_context():

3160 def bind_kv_cache(

3161 ctx: dict[str, Any],

3162 kv_cache: list[list[torch.Tensor]], # [virtual_engine][layer_index]

3166 - shared_kv_cache_layers: Optional[dict[str, str]] = None,

3163 + shared_kv_cache_layers: dict[str, str] | None = None,

3164 ) -> None:

3165 # Bind the kv_cache tensor to Attention modules, similar to

3166 # ctx[layer_name].kv_cache[ve]=kv_cache[ve][extract_layer_index(layer_name)]

@@ -3379,7 +3376,7 @@ def swap_dict_values(obj: dict[_K, _V], key1: _K, key2: _K) -> None:

3376

3377

3378 @contextlib.contextmanager

3382 -def cprofile_context(save_file: Optional[str] = None):

3379 +def cprofile_context(save_file: str | None = None):

3380 """Run a cprofile

3381

3382 Args:

@@ -3401,7 +3398,7 @@ def cprofile_context(save_file: Optional[str] = None):

3398 prof.print_stats(sort="cumtime")

3399

3400

3404 -def cprofile(save_file: Optional[str] = None, enabled: bool = True):

3401 +def cprofile(save_file: str | None = None, enabled: bool = True):

3402 """Decorator to profile a Python method using cProfile.

3403

3404 Args:

@@ -3608,7 +3605,7 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:

3605 file.write = write_with_prefix # type: ignore[method-assign]

3606

3607

3611 -def decorate_logs(process_name: Optional[str] = None) -> None:

3608 +def decorate_logs(process_name: str | None = None) -> None:

3609 """

3610 Adds a process-specific prefix to each line of output written to stdout and

3611 stderr.

@@ -3631,8 +3628,8 @@ def decorate_logs(process_name: Optional[str] = None) -> None:

3628

3629

3630 def length_from_prompt_token_ids_or_embeds(

3634 - prompt_token_ids: Optional[list[int]],

3635 - prompt_embeds: Optional[torch.Tensor],

3631 + prompt_token_ids: list[int] | None,

3632 + prompt_embeds: torch.Tensor | None,

3633 ) -> int:

3634 """Calculate the request length (in number of tokens) give either

3635 prompt_token_ids or prompt_embeds.

vllm/utils/deep_gemm.py

+2 -2

@@ -10,7 +10,7 @@ from __future__ import annotations

10 import functools

11 import importlib

12 import os

13 -from typing import Any, Callable, NoReturn, Optional

13 +from typing import Any, Callable, NoReturn

15 import torch

@@ -325,7 +325,7 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):

325 def should_use_deepgemm_for_fp8_linear(

326 output_dtype: torch.dtype,

327 weight: torch.Tensor,

328 - supports_deep_gemm: Optional[bool] = None,

328 + supports_deep_gemm: bool | None = None,

329 ):

330 if supports_deep_gemm is None:

331 supports_deep_gemm = is_deep_gemm_supported()

vllm/utils/flashinfer.py

+4 -4

@@ -12,7 +12,7 @@ import functools

12 import importlib

13 import importlib.util

14 import os

15 -from typing import Any, Callable, NoReturn, Optional

15 +from typing import Any, Callable, NoReturn

17 import requests

18 import torch

@@ -202,14 +202,14 @@ def supports_trtllm_attention() -> bool:

202

203

204 @functools.cache

205 -def _force_use_trtllm_attention(env_value: Optional[bool]) -> Optional[bool]:

205 +def _force_use_trtllm_attention(env_value: bool | None) -> bool | None:

206 """Cache the env value for VLLM_USE_TRTLLM_ATTENTION"""

207 if env_value is not None:

208 logger.info_once("VLLM_USE_TRTLLM_ATTENTION is set to %s", env_value)

209 return env_value

210

211

212 -def force_use_trtllm_attention() -> Optional[bool]:

212 +def force_use_trtllm_attention() -> bool | None:

213 """

214 Return ``None`` if VLLM_USE_TRTLLM_ATTENTION is not set,

215 return ``True`` if TRTLLM attention is forced to be used,

@@ -401,7 +401,7 @@ def flashinfer_scaled_fp8_mm(

401 scale_a: torch.Tensor,

402 scale_b: torch.Tensor,

403 out_dtype: torch.dtype,

404 - bias: Optional[torch.Tensor] = None,

404 + bias: torch.Tensor | None = None,

405 ) -> torch.Tensor:

406 assert a.ndim == 2 and b.ndim == 2

407 assert a.shape[1] == b.shape[0]

vllm/v1/attention/backends/flashinfer.py

+25 -25

@@ -5,7 +5,7 @@

5 from __future__ import annotations

7 from dataclasses import dataclass

8 -from typing import ClassVar, Optional, Union

8 +from typing import ClassVar, Union

10 import numpy as np

11 import torch

@@ -254,12 +254,12 @@ class FlashInferMetadata:

254 # For cascade attention (CPU for planning).

255 use_cascade: bool

256

257 - prefill_wrapper: Optional[BatchPrefillWithPagedKVCacheWrapper] = None

258 - decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None

259 - cascade_wrapper: Optional[MultiLevelCascadeAttentionWrapper] = None

257 + prefill_wrapper: BatchPrefillWithPagedKVCacheWrapper | None = None

258 + decode_wrapper: BatchDecodeWithPagedKVCacheWrapper | None = None

259 + cascade_wrapper: MultiLevelCascadeAttentionWrapper | None = None

260

261 - qo_indptr_gpu: Optional[torch.Tensor] = None

262 - paged_kv_indptr_gpu: Optional[torch.Tensor] = None

261 + qo_indptr_gpu: torch.Tensor | None = None

262 + paged_kv_indptr_gpu: torch.Tensor | None = None

263

264

265 class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):

@@ -727,13 +727,13 @@ class FlashInferImpl(AttentionImpl):

727 head_size: int,

728 scale: float,

729 num_kv_heads: int,

730 - alibi_slopes: Optional[list[float]],

731 - sliding_window: Optional[int],

730 + alibi_slopes: list[float] | None,

731 + sliding_window: int | None,

732 kv_cache_dtype: str,

733 - logits_soft_cap: Optional[float] = None,

733 + logits_soft_cap: float | None = None,

734 attn_type: AttentionType = AttentionType.DECODER,

735 - kv_sharing_target_layer_name: Optional[int] = None,

736 - sinks: Optional[torch.Tensor] = None,

735 + kv_sharing_target_layer_name: int | None = None,

736 + sinks: torch.Tensor | None = None,

737 ) -> None:

738 self.num_heads = num_heads

739 self.head_size = head_size

@@ -763,7 +763,7 @@ class FlashInferImpl(AttentionImpl):

763 "FlashInferImpl"

764 )

765

766 - self.sinks: Optional[torch.Tensor] = None

766 + self.sinks: torch.Tensor | None = None

767 if sinks is not None:

768 if sinks.shape[0] != num_heads:

769 raise ValueError(

@@ -776,9 +776,9 @@ class FlashInferImpl(AttentionImpl):

776 self.support_trtllm_attn = (

777 supports_trtllm_attention() and num_heads % num_kv_heads == 0

778 )

779 - self.bmm1_scale: Optional[float] = None

780 - self.bmm2_scale: Optional[float] = None

781 - self.o_sf_scale: Optional[float] = None

779 + self.bmm1_scale: float | None = None

780 + self.bmm2_scale: float | None = None

781 + self.o_sf_scale: float | None = None

782

783 def fused_output_quant_supported(self, quant_key: QuantKey):

784 return (

@@ -795,9 +795,9 @@ class FlashInferImpl(AttentionImpl):

795 value: torch.Tensor,

796 kv_cache: torch.Tensor,

797 attn_metadata: FlashInferMetadata,

798 - output: Optional[torch.Tensor] = None,

799 - output_scale: Optional[torch.Tensor] = None,

800 - output_block_scale: Optional[torch.Tensor] = None,

798 + output: torch.Tensor | None = None,

799 + output_scale: torch.Tensor | None = None,

800 + output_block_scale: torch.Tensor | None = None,

801 ) -> torch.Tensor:

802 """Forward pass with FlashInfer.

803

@@ -1093,13 +1093,13 @@ def fast_plan_decode(

1093 page_size: int,

1094 pos_encoding_mode: str = "NONE",

1095 window_left: int = -1,

1096 - logits_soft_cap: Optional[float] = None,

1097 - q_data_type: Optional[Union[str, torch.dtype]] = "float16",

1098 - kv_data_type: Optional[Union[str, torch.dtype]] = None,

1099 - data_type: Optional[Union[str, torch.dtype]] = None,

1100 - sm_scale: Optional[float] = None,

1101 - rope_scale: Optional[float] = None,

1102 - rope_theta: Optional[float] = None,

1096 + logits_soft_cap: float | None = None,

1097 + q_data_type: Union[str, torch.dtype] | None = "float16",

1098 + kv_data_type: Union[str, torch.dtype] | None = None,

1099 + data_type: Union[str, torch.dtype] | None = None,

1100 + sm_scale: float | None = None,

1101 + rope_scale: float | None = None,

1102 + rope_theta: float | None = None,

1103 non_blocking: bool = True,

1104 ) -> None:

1105 """

vllm/v1/core/sched/output.py

+9 -9

@@ -4,7 +4,7 @@

4 from __future__ import annotations

6 from dataclasses import dataclass

7 -from typing import TYPE_CHECKING, Optional

7 +from typing import TYPE_CHECKING

9 from vllm._bc_linter import bc_linter_include

@@ -25,14 +25,14 @@ if TYPE_CHECKING:

25 @dataclass

26 class NewRequestData:

27 req_id: str

28 - prompt_token_ids: Optional[list[int]]

28 + prompt_token_ids: list[int] | None

29 mm_features: list[MultiModalFeatureSpec]

30 - sampling_params: Optional[SamplingParams]

31 - pooling_params: Optional[PoolingParams]

30 + sampling_params: SamplingParams | None

31 + pooling_params: PoolingParams | None

32 block_ids: tuple[list[int], ...]

33 num_computed_tokens: int

34 - lora_request: Optional[LoRARequest]

35 - prompt_embeds: Optional[torch.Tensor] = None

34 + lora_request: LoRARequest | None

35 + prompt_embeds: torch.Tensor | None = None

37 @classmethod

38 def from_request(

@@ -98,7 +98,7 @@ class CachedRequestData:

98 # NOTE(woosuk): new_token_ids is only used for pipeline parallelism.

99 # When PP is not used, new_token_ids will be empty.

100 new_token_ids: list[list[int]]

101 - new_block_ids: list[Optional[tuple[list[int], ...]]]

101 + new_block_ids: list[tuple[list[int], ...] | None]

102 num_computed_tokens: list[int]

103 num_output_tokens: list[int]

104

@@ -160,7 +160,7 @@ class SchedulerOutput:

160 # for filling the next token bitmask

161 structured_output_request_ids: dict[str, int]

162 # the bitmask for the whole batch

163 - grammar_bitmask: Optional[npt.NDArray[np.int32]]

163 + grammar_bitmask: npt.NDArray[np.int32] | None

164

165 # KV Cache Connector metadata.

166 - kv_connector_metadata: Optional[KVConnectorMetadata] = None

166 + kv_connector_metadata: KVConnectorMetadata | None = None

vllm/v1/core/sched/scheduler.py

+12 -12

@@ -7,7 +7,7 @@ import itertools

7 import time

8 from collections import defaultdict

9 from collections.abc import Iterable

10 -from typing import Any, Optional, Union

10 +from typing import Any, Union

12 from vllm.config import VllmConfig

13 from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch

@@ -64,7 +64,7 @@ class Scheduler(SchedulerInterface):

64 # request ids should be included in the EngineCoreOutputs returned

65 # by update_from_outputs(). This is currently used in the multi-engine

66 # case to track request lifetimes efficiently.

67 - self.finished_req_ids_dict: Optional[dict[int, set[str]]] = (

67 + self.finished_req_ids_dict: dict[int, set[str]] | None = (

68 defaultdict(set) if include_finished_set else None

69 )

@@ -708,7 +708,7 @@ class Scheduler(SchedulerInterface):

708 ) -> CachedRequestData:

709 req_ids: list[str] = []

710 new_token_ids: list[list[int]] = []

711 - new_block_ids: list[Optional[tuple[list[int], ...]]] = []

711 + new_block_ids: list[tuple[list[int], ...] | None] = []

712 num_computed_tokens: list[int] = []

713 num_output_tokens: list[int] = []

714

@@ -921,7 +921,7 @@ class Scheduler(SchedulerInterface):

921 kv_connector_output = model_runner_output.kv_connector_output

922

923 outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)

924 - spec_decoding_stats: Optional[SpecDecodingStats] = None

924 + spec_decoding_stats: SpecDecodingStats | None = None

925 kv_connector_stats = (

926 kv_connector_output.kv_connector_stats if kv_connector_output else None

927 )

@@ -1212,7 +1212,7 @@ class Scheduler(SchedulerInterface):

1212 request.status = finished_status

1213 self._free_request(request)

1214

1215 - def _free_request(self, request: Request) -> Optional[dict[str, Any]]:

1215 + def _free_request(self, request: Request) -> dict[str, Any] | None:

1216 assert request.is_finished()

1217

1218 delay_free_blocks, kv_xfer_params = self._connector_finished(request)

@@ -1243,9 +1243,9 @@ class Scheduler(SchedulerInterface):

1243

1244 def make_stats(

1245 self,

1246 - spec_decoding_stats: Optional[SpecDecodingStats] = None,

1247 - kv_connector_stats: Optional[KVConnectorStats] = None,

1248 - ) -> Optional[SchedulerStats]:

1246 + spec_decoding_stats: SpecDecodingStats | None = None,

1247 + kv_connector_stats: KVConnectorStats | None = None,

1248 + ) -> SchedulerStats | None:

1249 if not self.log_stats:

1250 return None

1251 prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()

@@ -1262,10 +1262,10 @@ class Scheduler(SchedulerInterface):

1262

1263 def make_spec_decoding_stats(

1264 self,

1265 - spec_decoding_stats: Optional[SpecDecodingStats],

1265 + spec_decoding_stats: SpecDecodingStats | None,

1266 num_draft_tokens: int,

1267 num_accepted_tokens: int,

1268 - ) -> Optional[SpecDecodingStats]:

1268 + ) -> SpecDecodingStats | None:

1269 if not self.log_stats:

1270 return None

1271 if spec_decoding_stats is None:

@@ -1285,12 +1285,12 @@ class Scheduler(SchedulerInterface):

1285 # KV Connector Related Methods

1286 ########################################################################

1287

1288 - def get_kv_connector(self) -> Optional[KVConnectorBase_V1]:

1288 + def get_kv_connector(self) -> KVConnectorBase_V1 | None:

1289 return self.connector

1290

1291 def _connector_finished(

1292 self, request: Request

1293 - ) -> tuple[bool, Optional[dict[str, Any]]]:

1293 + ) -> tuple[bool, dict[str, Any] | None]:

1294 """

1295 Invoke the KV connector request_finished() method if applicable.

1296

vllm/v1/structured_output/__init__.py

+5 -5

@@ -4,7 +4,7 @@ from __future__ import annotations

5 import multiprocessing

6 from concurrent.futures import Future, ThreadPoolExecutor

7 -from typing import TYPE_CHECKING, Optional

7 +from typing import TYPE_CHECKING

9 from vllm.config import VllmConfig

10 from vllm.logger import init_logger

@@ -35,11 +35,11 @@ class StructuredOutputManager:

35 """Engine-level manager for structured output requests."""

37 def __init__(self, vllm_config: VllmConfig):

38 - self.backend: Optional[StructuredOutputBackend] = None

39 - self.reasoner: Optional[ReasoningParser] = None

38 + self.backend: StructuredOutputBackend | None = None

39 + self.reasoner: ReasoningParser | None = None

40 self.vllm_config = vllm_config

42 - self._grammar_bitmask: Optional[torch.Tensor] = None

42 + self._grammar_bitmask: torch.Tensor | None = None

43 self._full_mask = torch.tensor(-1, dtype=torch.int32)

45 max_batch_size = self.vllm_config.scheduler_config.max_num_seqs

@@ -168,7 +168,7 @@ class StructuredOutputManager:

168 requests: dict[str, Request],

169 structured_output_request_ids: dict[str, int],

170 scheduled_spec_decode_tokens: dict[str, list[int]],

171 - ) -> Optional[npt.NDArray[np.int32]]:

171 + ) -> npt.NDArray[np.int32] | None:

172 # Prepare the structured output bitmask for this batch.

173 if not structured_output_request_ids:

174 return None

vllm/v1/structured_output/backend_guidance.py

+2 -2

@@ -7,7 +7,7 @@ import copy

7 import json

8 import os

9 from dataclasses import dataclass

10 -from typing import TYPE_CHECKING, Any, Optional, Union

10 +from typing import TYPE_CHECKING, Any, Union

12 import torch

@@ -252,7 +252,7 @@ def serialize_guidance_grammar(

252

253

254 def validate_guidance_grammar(

255 - sampling_params: SamplingParams, tokenizer: Optional[llguidance.LLTokenizer] = None

255 + sampling_params: SamplingParams, tokenizer: llguidance.LLTokenizer | None = None

256 ) -> None:

257 tp, grm = get_structured_output_key(sampling_params)

258 guidance_grm = serialize_guidance_grammar(tp, grm)

vllm/v1/structured_output/request.py

+5 -5

@@ -20,10 +20,10 @@ from vllm.v1.structured_output.backend_types import (

20 @dataclasses.dataclass

21 class StructuredOutputRequest:

22 sampling_params: SamplingParams

23 - _grammar: Optional[

24 - Union[Future[StructuredOutputGrammar], StructuredOutputGrammar]

25 - ] = None

26 - reasoning_ended: Optional[bool] = None

23 + _grammar: Union[Future[StructuredOutputGrammar], StructuredOutputGrammar] | None = (

24 + None

25 + )

26 + reasoning_ended: bool | None = None

28 def _check_grammar_completion(self) -> bool:

29 # NOTE: We have to lazy import to gate circular imports

@@ -43,7 +43,7 @@ class StructuredOutputRequest:

43 return self._check_grammar_completion()

45 @property

46 - def grammar(self) -> Optional[StructuredOutputGrammar]:

46 + def grammar(self) -> StructuredOutputGrammar | None:

47 completed = self._check_grammar_completion()

48 return (

49 cast(Optional[StructuredOutputGrammar], self._grammar)

vllm/v1/worker/worker_base.py

+7 -7

@@ -4,7 +4,7 @@

4 from __future__ import annotations

6 import os

7 -from typing import Any, Callable, Optional, TypeVar, Union

7 +from typing import Any, Callable, TypeVar, Union

9 import torch

10 import torch.nn as nn

@@ -78,8 +78,8 @@ class WorkerBase:

78 self.is_driver_worker = is_driver_worker

80 # Device and model state

81 - self.device: Optional[torch.device] = None

82 - self.model_runner: Optional[nn.Module] = None

81 + self.device: torch.device | None = None

82 + self.model_runner: nn.Module | None = None

84 def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:

85 """Get specifications for KV cache implementation."""

@@ -115,8 +115,8 @@ class WorkerBase:

115 raise NotImplementedError

116

117 def execute_model(

118 - self, execute_model_req: Optional[ExecuteModelRequest] = None

119 - ) -> Optional[list[SamplerOutput]]:

118 + self, execute_model_req: ExecuteModelRequest | None = None

119 + ) -> list[SamplerOutput] | None:

120 raise NotImplementedError

121

122 def start_worker_execution_loop(self) -> None:

@@ -198,8 +198,8 @@ class WorkerWrapperBase:

198 group.

199 """

200 self.rpc_rank = rpc_rank

201 - self.worker: Optional[WorkerBase] = None

202 - self.vllm_config: Optional[VllmConfig] = None

201 + self.worker: WorkerBase | None = None

202 + self.vllm_config: VllmConfig | None = None

203 # do not store this `vllm_config`, `init_worker` will set the final

204 # one. TODO: investigate if we can remove this field in

205 # `WorkerWrapperBase`, `init_cached_hf_modules` should be