1a7894dbdf · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/beam_search.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Any

6

7 from vllm.logprobs import Logprob

8 from vllm.lora.request import LoRARequest

@@ -27,7 +27,7 @@ class BeamSearchSequence:

27 text: str | None = None

28 finish_reason: str | None = None

29 stop_reason: int | str | None = None

30 - multi_modal_data: Optional["MultiModalDataDict"] = None

30 + multi_modal_data: "MultiModalDataDict | None" = None

31 mm_processor_kwargs: dict[str, Any] | None = None

32

33

vllm/distributed/kv_transfer/kv_connector/factory.py

+2 -2

@@ -3,7 +3,7 @@

3

4 import importlib

5 from collections.abc import Callable

6 -from typing import TYPE_CHECKING, Optional, cast

6 +from typing import TYPE_CHECKING, cast

7

8 from vllm.distributed.kv_transfer.kv_connector.base import (

9 KVConnectorBase,

@@ -44,7 +44,7 @@ class KVConnectorFactory:

44 cls,

45 config: "VllmConfig",

46 role: KVConnectorRole,

47 - kv_cache_config: Optional["KVCacheConfig"] = None,

47 + kv_cache_config: "KVCacheConfig | None" = None,

48 ) -> KVConnectorBase:

49 kv_transfer_config = config.kv_transfer_config

50 if kv_transfer_config is None:

vllm/distributed/kv_transfer/kv_connector/v1/base.py

+6 -6

@@ -41,7 +41,7 @@ The class provides the following primitives:

41 import enum

42 from abc import ABC, abstractmethod

43 from collections.abc import Callable, Iterable

44 -from typing import TYPE_CHECKING, Any, Literal, Optional

44 +from typing import TYPE_CHECKING, Any, Literal

45

46 import torch

47

@@ -161,7 +161,7 @@ class KVConnectorBase_V1(ABC):

161 self,

162 vllm_config: "VllmConfig",

163 role: KVConnectorRole,

164 - kv_cache_config: Optional["KVCacheConfig"] = None,

164 + kv_cache_config: "KVCacheConfig | None" = None,

165 ):

166 logger.warning(

167 "Initializing KVConnectorBase_V1. This API is experimental and "

@@ -383,13 +383,13 @@ class KVConnectorBase_V1(ABC):

383 """

384 return None

385

386 - def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:

386 + def get_kv_connector_stats(self) -> "KVConnectorStats | None":

387 """

388 Get the KV connector stats collected during the last interval.

389 """

390 return None

391

392 - def get_kv_connector_kv_cache_events(self) -> Optional["KVConnectorKVEvents"]:

392 + def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":

393 """

394 Get the KV connector kv cache events collected during the last interval.

395 This function should be called by the model runner every time after the

@@ -558,7 +558,7 @@ class KVConnectorBase_V1(ABC):

558 @classmethod

559 def build_kv_connector_stats(

560 cls, data: dict[str, Any] | None = None

561 - ) -> Optional["KVConnectorStats"]:

561 + ) -> "KVConnectorStats | None":

562 """

563 KVConnectorStats resolution method. This method allows dynamically

564 registered connectors to return their own KVConnectorStats object,

@@ -584,7 +584,7 @@ class KVConnectorBase_V1(ABC):

584 metric_types: dict[type["PromMetric"], type["PromMetricT"]],

585 labelnames: list[str],

586 per_engine_labelvalues: dict[int, list[object]],

587 - ) -> Optional["KVConnectorPromMetrics"]:

587 + ) -> "KVConnectorPromMetrics | None":

588 """

589 Create a KVConnectorPromMetrics subclass which should register

590 per-connector Prometheus metrics and implement observe() to

vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py

+2 -2

@@ -32,7 +32,7 @@ Usage:

32 """

33

34 from dataclasses import dataclass

35 -from typing import TYPE_CHECKING, Any, Optional

35 +from typing import TYPE_CHECKING, Any

36

37 import torch

38

@@ -84,7 +84,7 @@ class DecodeBenchConnector(KVConnectorBase_V1):

84 self,

85 vllm_config: "VllmConfig",

86 role: KVConnectorRole,

87 - kv_cache_config: Optional["KVCacheConfig"] = None,

87 + kv_cache_config: "KVCacheConfig | None" = None,

88 ):

89 super().__init__(vllm_config, role, kv_cache_config)

90

vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import os

4 from dataclasses import dataclass, field

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Any

6

7 import safetensors

8 import torch

@@ -91,7 +91,7 @@ class ExampleConnector(KVConnectorBase_V1):

91 self,

92 vllm_config: "VllmConfig",

93 role: KVConnectorRole,

94 - kv_cache_config: Optional["KVCacheConfig"] = None,

94 + kv_cache_config: "KVCacheConfig | None" = None,

95 ):

96 super().__init__(

97 vllm_config=vllm_config,

vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py

+2 -2

@@ -5,7 +5,7 @@ import os

5 import uuid

6 from collections.abc import Generator

7 from dataclasses import dataclass, field

8 -from typing import TYPE_CHECKING, Any, Optional

8 +from typing import TYPE_CHECKING, Any

9

10 import torch

11 from lmcache import utils

@@ -274,7 +274,7 @@ class ReqMeta:

274 load_spec: LoadSpec | None = None,

275 discard_partial_chunks: bool = True,

276 save_decode_cache: bool = False,

277 - ) -> Optional["ReqMeta"]:

277 + ) -> "ReqMeta | None":

278 """Create the request metadata from a request tracker.

279

280 Args:

vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py

+5 -5

@@ -3,7 +3,7 @@

3 import enum

4 from collections.abc import Iterable

5 from dataclasses import dataclass, field

6 -from typing import TYPE_CHECKING, Any, Literal, Optional, cast

6 +from typing import TYPE_CHECKING, Any, Literal, cast

7

8 import torch

9 import zmq

@@ -385,7 +385,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):

385 self,

386 vllm_config: "VllmConfig",

387 role: KVConnectorRole,

388 - kv_cache_config: Optional["KVCacheConfig"] = None,

388 + kv_cache_config: "KVCacheConfig | None" = None,

389 ):

390 super().__init__(vllm_config, role, kv_cache_config)

391

@@ -595,7 +595,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):

595 self.worker_adapter.shutdown()

596 return None

597

598 - def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:

598 + def get_kv_connector_stats(self) -> "KVConnectorStats | None":

599 """

600 Get the KV connector stats collected during the last interval.

601 """

@@ -810,7 +810,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):

810 @classmethod

811 def build_kv_connector_stats(

812 cls, data: dict[str, Any] | None = None

813 - ) -> Optional["KVConnectorStats"]:

813 + ) -> "KVConnectorStats | None":

814 """

815 KVConnectorStats resolution method. This method allows dynamically

816 registered connectors to return their own KVConnectorStats object,

@@ -825,7 +825,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):

825 metric_types: dict[type["PromMetric"], type["PromMetricT"]],

826 labelnames: list[str],

827 per_engine_labelvalues: dict[int, list[object]],

828 - ) -> Optional["KVConnectorPromMetrics"]:

828 + ) -> "KVConnectorPromMetrics | None":

829 """

830 Create a KVConnectorPromMetrics subclass which should register

831 per-connector Prometheus metrics and implement observe() to

vllm/distributed/kv_transfer/kv_connector/v1/mooncake_connector.py

+2 -2

@@ -6,7 +6,7 @@ import time

6 from collections import defaultdict

7 from concurrent.futures import ThreadPoolExecutor

8 from dataclasses import dataclass

9 -from typing import TYPE_CHECKING, Any, Optional

9 +from typing import TYPE_CHECKING, Any

10

11 import msgspec

12 import numpy as np

@@ -115,7 +115,7 @@ class MooncakeConnector(KVConnectorBase_V1):

115 self,

116 vllm_config: VllmConfig,

117 role: KVConnectorRole,

118 - kv_cache_config: Optional["KVCacheConfig"] = None,

118 + kv_cache_config: "KVCacheConfig | None" = None,

119 ):

120 super().__init__(vllm_config, role, kv_cache_config)

121

vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py

+2 -2

@@ -5,7 +5,7 @@ import threading

5 import time

6 from collections.abc import Iterator

7 from dataclasses import dataclass

8 -from typing import TYPE_CHECKING, Any, Optional

8 +from typing import TYPE_CHECKING, Any

9

10 import msgspec

11 import torch

@@ -101,7 +101,7 @@ class MoRIIOAgentMetadata(

101 class RoleManager:

102 """Manages role state across the connector."""

103

104 - _instance: Optional["RoleManager"] = None

104 + _instance: "RoleManager | None" = None

105 _lock = threading.Lock()

106

107 def __init__(self) -> None:

vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py

+3 -3

@@ -7,7 +7,7 @@ import threading

7 import time

8 from collections import defaultdict

9 from concurrent.futures import Future, ThreadPoolExecutor

10 -from typing import TYPE_CHECKING, Any, Optional

10 +from typing import TYPE_CHECKING, Any

11

12 import msgpack

13 import msgspec

@@ -90,7 +90,7 @@ class MoRIIOConnector(KVConnectorBase_V1):

90 self,

91 vllm_config: VllmConfig,

92 role: KVConnectorRole,

93 - kv_cache_config: Optional["KVCacheConfig"] = None,

93 + kv_cache_config: "KVCacheConfig | None" = None,

94 ):

95 super().__init__(vllm_config, role)

96 assert vllm_config.kv_transfer_config is not None, (

@@ -333,7 +333,7 @@ class MoRIIOConnectorScheduler:

333 request: "Request",

334 blocks: "KVCacheBlocks",

335 num_external_tokens: int,

336 - connector_worker: Optional["MoRIIOConnectorWorker"] = None,

336 + connector_worker: "MoRIIOConnectorWorker | None" = None,

337 ):

338 params = request.kv_transfer_params

339 if not params:

vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import threading

4 -from typing import TYPE_CHECKING, Any, Optional

4 +from typing import TYPE_CHECKING, Any

5 from weakref import ref as weakref_ref

6

7 import msgpack

@@ -340,7 +340,7 @@ class MoRIIOWrapper:

340

341 def __init__(

342 self,

343 - moriio_engine: Optional["IOEngine"] = None,

343 + moriio_engine: "IOEngine | None" = None,

344 tp_rank: int = 0,

345 dp_rank: int = 0,

346 ):

vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py

+2 -2

@@ -14,7 +14,7 @@ from collections import defaultdict

14 from collections.abc import Iterator

15 from concurrent.futures import Future, ThreadPoolExecutor

16 from dataclasses import dataclass

17 -from typing import TYPE_CHECKING, Any, Optional

17 +from typing import TYPE_CHECKING, Any

18

19 import msgspec

20 import numpy as np

@@ -302,7 +302,7 @@ class NixlConnector(KVConnectorBase_V1):

302 self,

303 vllm_config: VllmConfig,

304 role: KVConnectorRole,

305 - kv_cache_config: Optional["KVCacheConfig"] = None,

305 + kv_cache_config: "KVCacheConfig | None" = None,

306 ):

307 super().__init__(vllm_config, role, kv_cache_config)

308

vllm/distributed/kv_transfer/kv_connector/v1/p2p/p2p_nccl_connector.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Any

6

7 import regex as re

8 import torch

@@ -76,7 +76,7 @@ class P2pNcclConnector(KVConnectorBase_V1):

76 self,

77 vllm_config: "VllmConfig",

78 role: KVConnectorRole,

79 - kv_cache_config: Optional["KVCacheConfig"] = None,

79 + kv_cache_config: "KVCacheConfig | None" = None,

80 ):

81 super().__init__(

82 vllm_config=vllm_config,

vllm/distributed/kv_transfer/kv_transfer_state.py

+2 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from typing import TYPE_CHECKING, Optional

3 +from typing import TYPE_CHECKING

4

5 from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType

6 from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory

@@ -49,7 +49,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo

49

50

51 def ensure_kv_transfer_initialized(

52 - vllm_config: "VllmConfig", kv_cache_config: Optional["KVCacheConfig"] = None

52 + vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig | None" = None

53 ) -> None:

54 """

55 Initialize KV cache transfer parallel group.

vllm/distributed/parallel_state.py

+4 -4

@@ -33,7 +33,7 @@ from contextlib import contextmanager, nullcontext

33 from dataclasses import dataclass

34 from datetime import timedelta

35 from multiprocessing import shared_memory

36 -from typing import Any, Optional

36 +from typing import Any

37 from unittest.mock import patch

38

39 import torch

@@ -106,7 +106,7 @@ def _get_unique_name(name: str) -> str:

106 return newname

107

108

109 -_groups: dict[str, Callable[[], Optional["GroupCoordinator"]]] = {}

109 +_groups: dict[str, Callable[[], "GroupCoordinator | None"]] = {}

110

111

112 def _register_group(group: "GroupCoordinator") -> None:

@@ -784,7 +784,7 @@ class GroupCoordinator:

784 self,

785 tensor_dict: dict[str, torch.Tensor | Any],

786 dst: int | None = None,

787 - all_gather_group: Optional["GroupCoordinator"] = None,

787 + all_gather_group: "GroupCoordinator | None" = None,

788 all_gather_tensors: dict[str, bool] | None = None,

789 ) -> dict[str, torch.Tensor | Any] | None:

790 """Send the input tensor dictionary.

@@ -871,7 +871,7 @@ class GroupCoordinator:

871 def recv_tensor_dict(

872 self,

873 src: int | None = None,

874 - all_gather_group: Optional["GroupCoordinator"] = None,

874 + all_gather_group: "GroupCoordinator | None" = None,

875 all_gather_tensors: dict[str, bool] | None = None,

876 ) -> dict[str, torch.Tensor | Any] | None:

877 """Recv the input tensor dictionary.

vllm/entrypoints/anthropic/protocol.py

+2 -2

@@ -3,7 +3,7 @@

3 """Pydantic models for Anthropic API protocol"""

4

5 import time

6 -from typing import Any, Literal, Optional

6 +from typing import Any, Literal

7

8 from pydantic import BaseModel, field_validator

9

@@ -135,7 +135,7 @@ class AnthropicStreamEvent(BaseModel):

135 "ping",

136 "error",

137 ]

138 - message: Optional["AnthropicMessagesResponse"] = None

138 + message: "AnthropicMessagesResponse | None" = None

139 delta: AnthropicDelta | None = None

140 content_block: AnthropicContentBlock | None = None

141 index: int | None = None

vllm/lora/lora_weights.py

+2 -3

@@ -2,7 +2,6 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from collections.abc import Sequence as GenericSequence

5 -from typing import Optional

5

6 import torch

7 import torch.types

@@ -126,7 +125,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):

125

126 @classmethod

127 def pack(

129 - cls, loras: GenericSequence[Optional["LoRALayerWeights"]]

128 + cls, loras: GenericSequence["LoRALayerWeights | None"]

129 ) -> "PackedLoRALayerWeights":

130 """Pack a list of LoRAs into a single LoRA.

131

@@ -155,7 +154,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):

154 @classmethod

155 def pack_moe(

156 cls,

158 - loras: GenericSequence[Optional["LoRALayerWeights"]],

157 + loras: GenericSequence["LoRALayerWeights | None"],

158 module_name: str,

159 is_non_gated_moe: bool = False,

160 ) -> "PackedLoRALayerWeights":

vllm/lora/utils.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import os

5 -from typing import TYPE_CHECKING, Optional

5 +from typing import TYPE_CHECKING

6

7 import huggingface_hub

8 from huggingface_hub.utils import HfHubHTTPError, HFValidationError

@@ -131,7 +131,7 @@ def replace_submodule(

131

132

133 def parse_fine_tuned_lora_name(

134 - name: str, weights_mapper: Optional["WeightsMapper"] = None

134 + name: str, weights_mapper: "WeightsMapper | None" = None

135 ) -> tuple[str, bool]:

136 """Parse the name of lora weights.

137

vllm/model_executor/layers/fused_moe/config.py

+3 -3

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from dataclasses import dataclass

4 from enum import IntEnum

5 -from typing import Optional, Union

5 +from typing import Union

6

7 import torch

8

@@ -284,7 +284,7 @@ class FusedMoEQuantConfig:

284 return self._w1.bias

285

286 @property

287 - def w1_precision(self) -> Optional["PrecisionConfig"]:

287 + def w1_precision(self) -> "PrecisionConfig | None":

288 assert self._w1.scale is None or isinstance(self._w1.scale, PrecisionConfig)

289 return self._w1.scale

290

@@ -306,7 +306,7 @@ class FusedMoEQuantConfig:

306 return self._w2.bias

307

308 @property

309 - def w2_precision(self) -> Optional["PrecisionConfig"]:

309 + def w2_precision(self) -> "PrecisionConfig | None":

310 assert self._w2.scale is None or isinstance(self._w2.scale, PrecisionConfig)

311 return self._w2.scale

312

vllm/model_executor/layers/quantization/awq_marlin.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import TYPE_CHECKING, Any, Optional

4 +from typing import TYPE_CHECKING, Any

5

6 import torch

7 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

@@ -148,7 +148,7 @@ class AWQMarlinConfig(QuantizationConfig):

148 @classmethod

149 def override_quantization_method(

150 cls, hf_quant_cfg, user_quant

151 - ) -> Optional["QuantizationMethods"]:

151 + ) -> "QuantizationMethods | None":

152 can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)

153 is_valid_user_quant = (

154 user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"

@@ -173,7 +173,7 @@ class AWQMarlinConfig(QuantizationConfig):

173

174 def get_quant_method(

175 self, layer: torch.nn.Module, prefix: str

176 - ) -> Optional["QuantizeMethodBase"]:

176 + ) -> "QuantizeMethodBase | None":

177 if isinstance(layer, LinearBase) or (

178 isinstance(layer, ParallelLMHead) and self.lm_head_quantized

179 ):

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py

+3 -3

@@ -3,7 +3,7 @@

3

4 from contextlib import suppress

5 from functools import partial

6 -from typing import TYPE_CHECKING, Any, Literal, Optional, cast

6 +from typing import TYPE_CHECKING, Any, Literal, cast

7

8 import torch

9 from compressed_tensors.config import (

@@ -160,7 +160,7 @@ class CompressedTensorsConfig(QuantizationConfig):

160 self,

161 layer: torch.nn.Module,

162 prefix: str,

163 - ) -> Optional["QuantizeMethodBase"]:

163 + ) -> "QuantizeMethodBase | None":

164 if isinstance(layer, LinearBase):

165 # collect schemes

166 quant_scheme = self.get_scheme(layer=layer, layer_name=prefix)

@@ -691,7 +691,7 @@ class CompressedTensorsConfig(QuantizationConfig):

691

692 def get_scheme(

693 self, layer: torch.nn.Module, layer_name: str | None = None

694 - ) -> Optional["CompressedTensorsScheme"]:

694 + ) -> "CompressedTensorsScheme | None":

695 """

696 compressed-tensors supports non uniform in the following way:

697

vllm/model_executor/layers/quantization/cpu_wna16.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

@@ -105,7 +105,7 @@ class CPUAWQConfig(QuantizationConfig):

105 @classmethod

106 def override_quantization_method(

107 cls, hf_quant_cfg, user_quant

108 - ) -> Optional["QuantizationMethods"]:

108 + ) -> "QuantizationMethods | None":

109 quant_method = hf_quant_cfg.get("quant_method", "").lower()

110 if current_platform.is_cpu() and (quant_method == "awq"):

111 return cls.get_name()

@@ -113,7 +113,7 @@ class CPUAWQConfig(QuantizationConfig):

113

114 def get_quant_method(

115 self, layer: torch.nn.Module, prefix: str

116 - ) -> Optional["QuantizeMethodBase"]:

116 + ) -> "QuantizeMethodBase | None":

117 if isinstance(layer, LinearBase) or (

118 isinstance(layer, ParallelLMHead) and self.lm_head_quantized

119 ):

vllm/model_executor/layers/quantization/experts_int8.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7

@@ -52,7 +52,7 @@ class ExpertsInt8Config(QuantizationConfig):

52

53 def get_quant_method(

54 self, layer: torch.nn.Module, prefix: str

55 - ) -> Optional["QuantizeMethodBase"]:

55 + ) -> "QuantizeMethodBase | None":

56 if isinstance(layer, LinearBase):

57 return UnquantizedLinearMethod()

58 elif isinstance(layer, FusedMoE):

vllm/model_executor/layers/quantization/fbgemm_fp8.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7 from torch.nn import Module

@@ -78,7 +78,7 @@ class FBGEMMFp8Config(QuantizationConfig):

78

79 def get_quant_method(

80 self, layer: torch.nn.Module, prefix: str

81 - ) -> Optional["QuantizeMethodBase"]:

81 + ) -> "QuantizeMethodBase | None":

82 if isinstance(layer, LinearBase):

83 if is_layer_skipped(

84 prefix=prefix,

vllm/model_executor/layers/quantization/fp8.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import TYPE_CHECKING, Any, Optional

4 +from typing import TYPE_CHECKING, Any

5

6 import torch

7 from torch.nn import Module

@@ -182,7 +182,7 @@ class Fp8Config(QuantizationConfig):

182

183 def get_xpu_quant_method(

184 self, layer: torch.nn.Module, prefix: str

185 - ) -> Optional["QuantizeMethodBase"]:

185 + ) -> "QuantizeMethodBase | None":

186 from vllm.model_executor.layers.quantization.ipex_quant import (

187 XPUFp8LinearMethod,

188 XPUFp8MoEMethod,

@@ -218,7 +218,7 @@ class Fp8Config(QuantizationConfig):

218

219 def get_quant_method(

220 self, layer: torch.nn.Module, prefix: str

221 - ) -> Optional["QuantizeMethodBase"]:

221 + ) -> "QuantizeMethodBase | None":

222 if current_platform.is_xpu():

223 return self.get_xpu_quant_method(layer, prefix)

224 if isinstance(layer, LinearBase):

vllm/model_executor/layers/quantization/gguf.py

+2 -2

@@ -3,7 +3,7 @@

3

4 from collections.abc import Mapping

5 from types import MappingProxyType

6 -from typing import Any, Optional

6 +from typing import Any

7

8 import gguf

9 import torch

@@ -77,7 +77,7 @@ class GGUFConfig(QuantizationConfig):

77

78 def get_quant_method(

79 self, layer: torch.nn.Module, prefix: str

80 - ) -> Optional["QuantizeMethodBase"]:

80 + ) -> "QuantizeMethodBase | None":

81 if isinstance(layer, LinearBase):

82 if is_layer_skipped_gguf(

83 prefix, self.unquantized_modules, self.packed_modules_mapping

vllm/model_executor/layers/quantization/gptq_marlin.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from copy import deepcopy

5 -from typing import Any, Optional

5 +from typing import Any

6

7 import torch

8 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

@@ -240,7 +240,7 @@ class GPTQMarlinConfig(QuantizationConfig):

240

241 def get_quant_method(

242 self, layer: torch.nn.Module, prefix: str

243 - ) -> Optional["QuantizeMethodBase"]:

243 + ) -> "QuantizeMethodBase | None":

244 if isinstance(layer, FusedMoE):

245 from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config

246

vllm/model_executor/layers/quantization/inc.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from fractions import Fraction

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Any

6

7 import regex as re

8 import torch

@@ -456,7 +456,7 @@ class INCConfig(QuantizationConfig):

456 @classmethod

457 def override_quantization_method(

458 cls, hf_quant_cfg, user_quant

459 - ) -> Optional["QuantizationMethods"]:

459 + ) -> "QuantizationMethods | None":

460 """Override the `auto-round` method to `inc`."""

461 is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"

462 if is_auto_round_format:

vllm/model_executor/layers/quantization/ipex_quant.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7 from packaging import version

@@ -144,7 +144,7 @@ class IPEXConfig(QuantizationConfig):

144

145 def get_quant_method(

146 self, layer: torch.nn.Module, prefix: str

147 - ) -> Optional["LinearMethodBase"]:

147 + ) -> "LinearMethodBase | None":

148 if isinstance(layer, LinearBase):

149 if self.method == "awq":

150 if is_layer_skipped(

vllm/model_executor/layers/quantization/modelopt.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from fnmatch import fnmatch

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Any

6

7 import torch

8 from torch.nn import Module

@@ -181,7 +181,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):

181

182 def get_quant_method(

183 self, layer: torch.nn.Module, prefix: str

184 - ) -> Optional["QuantizeMethodBase"]:

184 + ) -> "QuantizeMethodBase | None":

185 # handle kv-cache first so we can focus only on weight quantization thereafter

186 if isinstance(layer, Attention):

187 return self.KVCacheMethodCls(self)

vllm/model_executor/layers/quantization/moe_wna16.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7

@@ -163,7 +163,7 @@ class MoeWNA16Config(QuantizationConfig):

163

164 def get_quant_method(

165 self, layer: torch.nn.Module, prefix: str

166 - ) -> Optional["QuantizeMethodBase"]:

166 + ) -> "QuantizeMethodBase | None":

167 if is_layer_skipped_quant(prefix, self.modules_to_not_convert):

168 if isinstance(layer, FusedMoE):

169 return UnquantizedFusedMoEMethod(layer.moe_config)

vllm/model_executor/layers/quantization/mxfp4.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from enum import Enum

4 -from typing import Optional

4

5 import torch

6 from torch.nn.parameter import Parameter

@@ -197,7 +196,7 @@ class Mxfp4Config(QuantizationConfig):

196

197 def get_quant_method(

198 self, layer: torch.nn.Module, prefix: str

200 - ) -> Optional["QuantizeMethodBase"]:

199 + ) -> "QuantizeMethodBase | None":

200 if isinstance(layer, LinearBase):

201 if self.ignored_layers and is_layer_skipped(

202 prefix=prefix,

vllm/model_executor/layers/quantization/petit.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py

4

5 -from typing import Any, Optional

5 +from typing import Any

6

7 import regex as re

8 import torch

@@ -159,7 +159,7 @@ class PetitNvFp4Config(QuantizationConfig):

159

160 def get_quant_method(

161 self, layer: torch.nn.Module, prefix: str

162 - ) -> Optional["QuantizeMethodBase"]:

162 + ) -> "QuantizeMethodBase | None":

163 exclude = self.require_exclude_modules()

164

165 if isinstance(layer, LinearBase):

vllm/model_executor/layers/quantization/ptpc_fp8.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -from typing import Any, Optional

4 +from typing import Any

5

6 import torch

7 from torch.nn.parameter import Parameter

@@ -67,7 +67,7 @@ class PTPCFp8Config(Fp8Config):

67

68 def get_quant_method(

69 self, layer: torch.nn.Module, prefix: str

70 - ) -> Optional["QuantizeMethodBase"]:

70 + ) -> "QuantizeMethodBase | None":

71 if isinstance(layer, LinearBase):

72 if is_layer_skipped(prefix, self.ignored_layers):

73 return UnquantizedLinearMethod()

vllm/model_executor/layers/quantization/quark/quark.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import fnmatch

5 -from typing import TYPE_CHECKING, Any, Optional, cast

5 +from typing import TYPE_CHECKING, Any, cast

6

7 import torch

8

@@ -102,7 +102,7 @@ class QuarkConfig(QuantizationConfig):

102

103 def get_quant_method(

104 self, layer: torch.nn.Module, prefix: str

105 - ) -> Optional["QuantizeMethodBase"]:

105 + ) -> "QuantizeMethodBase | None":

106 # Check if the layer is skipped for quantization.

107 exclude_layers = cast(list[str], self.quant_config.get("exclude"))

108 if should_ignore_layer(

vllm/model_executor/layers/quantization/torchao.py

+2 -2

@@ -4,7 +4,7 @@ import importlib

4 import json

5 import types

6 from importlib.util import find_spec

7 -from typing import Any, Optional

7 +from typing import Any

8

9 import regex as re

10 import torch

@@ -209,7 +209,7 @@ class TorchAOConfig(QuantizationConfig):

209

210 def get_quant_method(

211 self, layer: torch.nn.Module, prefix: str

212 - ) -> Optional["QuantizeMethodBase"]:

212 + ) -> "QuantizeMethodBase | None":

213 if not isinstance(layer, LinearBase):

214 return None

215

vllm/model_executor/layers/quantization/utils/petit_utils.py

+2 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from typing import TYPE_CHECKING, Optional

3 +from typing import TYPE_CHECKING

4

5 import torch

6

@@ -9,7 +9,7 @@ if TYPE_CHECKING:

9 from types import ModuleType

10

11 # 1. Create a global variable as a placeholder for the module

12 -_petit_kernel: Optional["ModuleType"] = None

12 +_petit_kernel: "ModuleType | None" = None

13

14 _PETIT_INSTALL_MSG = (

15 "Petit is not installed. Please install it with `pip install petit-kernel`."

vllm/model_executor/model_loader/tensorizer.py

+2 -2

@@ -12,7 +12,7 @@ import threading

12 import time

13 from collections.abc import Generator, MutableMapping

14 from dataclasses import asdict, dataclass, field, fields

15 -from typing import TYPE_CHECKING, Any, ClassVar, Optional

15 +from typing import TYPE_CHECKING, Any, ClassVar

16

17 import regex as re

18 import torch

@@ -323,7 +323,7 @@ class TensorizerConfig(MutableMapping):

323 " is unstable and may lead to errors."

324 )

325

326 - def open_stream(self, tensorizer_args: Optional["TensorizerArgs"] = None):

326 + def open_stream(self, tensorizer_args: "TensorizerArgs | None" = None):

327 if tensorizer_args is None:

328 tensorizer_args = self._construct_tensorizer_args()

329

vllm/multimodal/inputs.py

+2 -3

@@ -11,7 +11,6 @@ from typing import (

11 TYPE_CHECKING,

12 Any,

13 Literal,

14 - Optional,

14 TypeAlias,

15 TypedDict,

16 Union,

@@ -186,7 +185,7 @@ class PlaceholderRange:

185 length: int

186 """The length of the placeholder."""

187

189 - is_embed: Optional["torch.Tensor"] = None

188 + is_embed: "torch.Tensor | None" = None

189 """

190 A boolean mask of shape `(length,)` indicating which positions

191 between `offset` and `offset + length` to assign embeddings to.

@@ -341,7 +340,7 @@ class MultiModalFeatureSpec:

340 `MultiModalFeatureSpec` per item.

341 """

342

344 - data: Optional["MultiModalKwargsItem"]

343 + data: "MultiModalKwargsItem | None"

344 """

345 Represents multimodal data for this feature.

346

vllm/platforms/cuda.py

+2 -2

@@ -7,7 +7,7 @@ pynvml. However, it should not initialize cuda context.

7 import os

8 from collections.abc import Callable

9 from functools import cache, wraps

10 -from typing import TYPE_CHECKING, Optional, TypeVar

10 +from typing import TYPE_CHECKING, TypeVar

11

12 import torch

13 from typing_extensions import ParamSpec

@@ -382,7 +382,7 @@ class CudaPlatformBase(Platform):

382 cls,

383 head_size: int,

384 dtype: torch.dtype,

385 - backend: Optional["AttentionBackendEnum"] = None,

385 + backend: "AttentionBackendEnum | None" = None,

386 ) -> "AttentionBackendEnum":

387 if backend is not None:

388 assert backend in cls.get_supported_vit_attn_backends(), (

vllm/platforms/interface.py

+2 -2

@@ -7,7 +7,7 @@ import platform

7 import random

8 import sys

9 from datetime import timedelta

10 -from typing import TYPE_CHECKING, Any, NamedTuple, Optional

10 +from typing import TYPE_CHECKING, Any, NamedTuple

11

12 import numpy as np

13 import torch

@@ -243,7 +243,7 @@ class Platform:

243 cls,

244 head_size: int,

245 dtype: torch.dtype,

246 - backend: Optional["AttentionBackendEnum"] = None,

246 + backend: "AttentionBackendEnum | None" = None,

247 ) -> "AttentionBackendEnum":

248 """

249 Get the vision attention backend class of a device.

vllm/platforms/rocm.py

+2 -2

@@ -3,7 +3,7 @@

3

4 import os

5 from functools import cache, lru_cache, wraps

6 -from typing import TYPE_CHECKING, Optional

6 +from typing import TYPE_CHECKING

7

8 import torch

9

@@ -356,7 +356,7 @@ class RocmPlatform(Platform):

356 cls,

357 head_size: int,

358 dtype: torch.dtype,

359 - backend: Optional["AttentionBackendEnum"] = None,

359 + backend: "AttentionBackendEnum | None" = None,

360 ) -> "AttentionBackendEnum":

361 if backend is not None:

362 assert backend in cls.get_supported_vit_attn_backends(), (

vllm/platforms/xpu.py

+2 -2

@@ -3,7 +3,7 @@

3

4 import contextlib

5 import os

6 -from typing import TYPE_CHECKING, Optional

6 +from typing import TYPE_CHECKING

7

8 import torch

9

@@ -88,7 +88,7 @@ class XPUPlatform(Platform):

88 cls,

89 head_size: int,

90 dtype: torch.dtype,

91 - backend: Optional["AttentionBackendEnum"] = None,

91 + backend: "AttentionBackendEnum | None" = None,

92 ) -> "AttentionBackendEnum":

93 if backend is not None:

94 assert backend in cls.get_supported_vit_attn_backends(), (

vllm/pooling_params.py

+4 -4

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from copy import deepcopy

5 -from typing import Annotated, Any, Optional

5 +from typing import Annotated, Any

6

7 import msgspec

8

@@ -80,7 +80,7 @@ class PoolingParams(

80 return deepcopy(self)

81

82 def verify(

83 - self, task: PoolingTask, model_config: Optional["ModelConfig"] = None

83 + self, task: PoolingTask, model_config: "ModelConfig | None" = None

84 ) -> None:

85 if self.task is None:

86 self.task = task

@@ -106,7 +106,7 @@ class PoolingParams(

106 self._verify_valid_parameters()

107

108 def _merge_default_parameters(

109 - self, model_config: Optional["ModelConfig"] = None

109 + self, model_config: "ModelConfig | None" = None

110 ) -> None:

111 if model_config is None:

112 return

@@ -160,7 +160,7 @@ class PoolingParams(

160 if getattr(self, k, None) is None:

161 setattr(self, k, getattr(pooler_config, k))

162

163 - def _set_default_parameters(self, model_config: Optional["ModelConfig"]):

163 + def _set_default_parameters(self, model_config: "ModelConfig | None"):

164 if self.task in ["embed", "token_embed"]:

165 if self.use_activation is None:

166 self.use_activation = True

vllm/profiler/layerwise_profile.py

+2 -2

@@ -5,7 +5,7 @@ import copy

5 from collections import defaultdict

6 from collections.abc import Callable

7 from dataclasses import asdict, dataclass, field

8 -from typing import Any, Optional, TypeAlias

8 +from typing import Any, TypeAlias

9

10 from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult

11 from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent

@@ -31,7 +31,7 @@ except ImportError:

31 @dataclass

32 class _ModuleTreeNode:

33 event: _ProfilerEvent

34 - parent: Optional["_ModuleTreeNode"] = None

34 + parent: "_ModuleTreeNode | None" = None

35 children: list["_ModuleTreeNode"] = field(default_factory=list)

36 trace: str = ""

37

vllm/transformers_utils/s3_utils.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import fnmatch

5 -from typing import TYPE_CHECKING, Optional

5 +from typing import TYPE_CHECKING

6

7 from vllm.utils.import_utils import PlaceholderModule

8

@@ -32,7 +32,7 @@ def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]:

32

33

34 def glob(

35 - s3: Optional["BaseClient"] = None,

35 + s3: "BaseClient | None" = None,

36 path: str = "",

37 allow_pattern: list[str] | None = None,

38 ) -> list[str]:

vllm/v1/attention/backends/mla/flashmla_sparse.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from dataclasses import dataclass

4 -from typing import TYPE_CHECKING, ClassVar, Optional

4 +from typing import TYPE_CHECKING, ClassVar

5

6 import numpy as np

7 import torch

@@ -707,7 +707,7 @@ class FlashMLASparseImpl(MLACommonBaseImpl[FlashMLASparseMetadata]):

707 kv_sharing_target_layer_name: str | None,

708 # MLA Specific Arguments

709 topk_indice_buffer: torch.Tensor | None = None,

710 - indexer: Optional["Indexer"] = None,

710 + indexer: "Indexer | None" = None,

711 **mla_args,

712 ) -> None:

713 super().__init__(

vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, ClassVar, Optional

5 +from typing import TYPE_CHECKING, ClassVar

6

7 import numpy as np

8 import torch

@@ -284,7 +284,7 @@ class ROCMAiterMLASparseImpl(MLACommonBaseImpl[ROCMAiterMLASparseMetadata]):

284 kv_sharing_target_layer_name: str | None,

285 # MLA Specific Arguments

286 topk_indice_buffer: torch.Tensor | None = None,

287 - indexer: Optional["Indexer"] = None,

287 + indexer: "Indexer | None" = None,

288 **mla_args,

289 ) -> None:

290 super().__init__(

vllm/v1/attention/backends/tree_attn.py

+5 -5

@@ -4,7 +4,7 @@

4

5 import ast

6 from dataclasses import dataclass

7 -from typing import ClassVar, Optional

7 +from typing import ClassVar

8

9 import torch

10

@@ -87,11 +87,11 @@ class TreeAttentionMetadata:

87 tree_attn_bias: torch.Tensor | None = None

88

89 # Cached Prefill/decode metadata.

90 - _cached_prefill_metadata: Optional["TreeAttentionMetadata"] = None

91 - _cached_decode_metadata: Optional["TreeAttentionMetadata"] = None

90 + _cached_prefill_metadata: "TreeAttentionMetadata | None" = None

91 + _cached_decode_metadata: "TreeAttentionMetadata | None" = None

92

93 @property

94 - def prefill_metadata(self) -> Optional["TreeAttentionMetadata"]:

94 + def prefill_metadata(self) -> "TreeAttentionMetadata | None":

95 if self.num_prefills == 0:

96 return None

97

@@ -116,7 +116,7 @@ class TreeAttentionMetadata:

116 return self._cached_prefill_metadata

117

118 @property

119 - def decode_metadata(self) -> Optional["TreeAttentionMetadata"]:

119 + def decode_metadata(self) -> "TreeAttentionMetadata | None":

120 if self.num_decode_tokens == 0:

121 return None

122

vllm/v1/core/sched/interface.py

+3 -3

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from abc import ABC, abstractmethod

4 from collections.abc import Iterable

5 -from typing import TYPE_CHECKING, Optional

5 +from typing import TYPE_CHECKING

6

7 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

8

@@ -189,7 +189,7 @@ class SchedulerInterface(ABC):

189 raise NotImplementedError

190

191 @abstractmethod

192 - def make_stats(self) -> Optional["SchedulerStats"]:

192 + def make_stats(self) -> "SchedulerStats | None":

193 """Make a SchedulerStats object for logging.

194

195 The SchedulerStats object is created for every scheduling step.

@@ -201,5 +201,5 @@ class SchedulerInterface(ABC):

201 """Shutdown the scheduler."""

202 raise NotImplementedError

203

204 - def get_kv_connector(self) -> Optional["KVConnectorBase_V1"]:

204 + def get_kv_connector(self) -> "KVConnectorBase_V1 | None":

205 return None

vllm/v1/engine/parallel_sampling.py

+2 -2

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from copy import copy

5 -from typing import Optional, cast

5 +from typing import cast

6

7 from vllm.outputs import CompletionOutput

8 from vllm.sampling_params import RequestOutputKind, SamplingParams

@@ -133,7 +133,7 @@ class ParentRequest:

133

134 @staticmethod

135 def observe_finished_request(

136 - parent_req: Optional["ParentRequest"],

136 + parent_req: "ParentRequest | None",

137 iteration_stats: IterationStats,

138 num_generation_tokens: int,

139 ):

vllm/v1/request.py

+2 -2

@@ -7,7 +7,7 @@ from collections import deque

7 from collections.abc import Callable, Mapping

8 from dataclasses import dataclass

9 from functools import partial

10 -from typing import TYPE_CHECKING, Any, Optional

10 +from typing import TYPE_CHECKING, Any

11

12 import torch

13

@@ -68,7 +68,7 @@ class Request:

68 arrival_time: float | None = None,

69 prompt_embeds: torch.Tensor | None = None,

70 mm_features: list[MultiModalFeatureSpec] | None = None,

71 - lora_request: Optional["LoRARequest"] = None,

71 + lora_request: "LoRARequest | None" = None,

72 cache_salt: str | None = None,

73 priority: int = 0,

74 trace_headers: Mapping[str, str] | None = None,

vllm/v1/sample/logits_processor/interface.py

+2 -2

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod

4 from collections.abc import Sequence

5 from dataclasses import dataclass

6 from enum import Enum, auto

7 -from typing import TYPE_CHECKING, Optional

7 +from typing import TYPE_CHECKING

8

9 import torch

10

@@ -94,7 +94,7 @@ class LogitsProcessor(ABC):

94 @abstractmethod

95 def update_state(

96 self,

97 - batch_update: Optional["BatchUpdate"],

97 + batch_update: "BatchUpdate | None",

98 ) -> None:

99 """Called when there are new output tokens, prior

100 to each forward pass.

vllm/v1/utils.py

+1 -2

@@ -14,7 +14,6 @@ from typing import (

14 TYPE_CHECKING,

15 Any,

16 Generic,

17 - Optional,

17 TypeVar,

18 Union,

19 overload,

@@ -229,7 +228,7 @@ def wait_for_completion_or_failure(

228 api_server_manager: APIServerProcessManager,

229 engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]

230 | None = None,

232 - coordinator: Optional["DPCoordinator"] = None,

231 + coordinator: "DPCoordinator | None" = None,

232 ) -> None:

233 """Wait for all processes to complete or detect if any fail.

234

vllm/v1/worker/ubatching.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import threading

4 -from typing import Optional

4

5 import torch

6

@@ -15,7 +14,7 @@ logger = init_logger(__name__)

14 _THREAD_ID_TO_CONTEXT: dict = {}

15 # Here we hardcode the number of microbatches to 2 for default.

16 _NUM_UBATCHES: int = 2

18 -_CURRENT_CONTEXTS: list[Optional["UBatchContext"]] = []

17 +_CURRENT_CONTEXTS: list["UBatchContext | None"] = []

18

19

20 class UBatchContext:

vllm/v1/worker/workspace.py

+1 -2

@@ -5,7 +5,6 @@ import inspect

5 import os

6 from itertools import accumulate

7 from math import prod

8 -from typing import Optional

8

9 import torch

10

@@ -26,7 +25,7 @@ _MB = 1024**2

25 _GiB = 1024**3

26

27 # Global workspace manager instance

29 -_manager: Optional["WorkspaceManager"] = None

28 +_manager: "WorkspaceManager | None" = None

29

30

31 class WorkspaceManager:

[Misc] Replace Optional[X] with X | None syntax (#33332)