@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e09d1753ec

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Remove Python 3.9 support ahead of PyTorch 2.9 in v0.11.1 (#26416)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Oct 8, 2025 at 18:40 UTC e09d1753ecfeeba061cf9b8d6ff5f7dd89beb1e7

20 files changed +45 -87

.pre-commit-config.yaml

+5 -5

@@ -55,11 +55,6 @@ repos:

55 types_or: [python, pyi]

56 require_serial: true

57 additional_dependencies: [mypy==1.11.1, regex, types-cachetools, types-setuptools, types-PyYAML, types-requests, types-torch, pydantic]

58 - - id: mypy-3.9 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

59 - name: Run mypy for Python 3.9

60 - entry: python tools/pre_commit/mypy.py 1 "3.9"

61 - <<: *mypy_common

62 - stages: [manual] # Only run in CI

58 - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

59 name: Run mypy for Python 3.10

60 entry: python tools/pre_commit/mypy.py 1 "3.10"

@@ -75,6 +70,11 @@ repos:

70 entry: python tools/pre_commit/mypy.py 1 "3.12"

71 <<: *mypy_common

72 stages: [manual] # Only run in CI

73 + - id: mypy-3.13 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward

74 + name: Run mypy for Python 3.13

75 + entry: python tools/pre_commit/mypy.py 1 "3.13"

76 + <<: *mypy_common

77 + stages: [manual] # Only run in CI

78 - id: shellcheck

79 name: Lint shell scripts

80 entry: tools/shellcheck.sh

CMakeLists.txt

+1 -1

@@ -34,7 +34,7 @@ install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)

34 # Supported python versions. These versions will be searched in order, the

35 # first match will be selected. These should be kept in sync with setup.py.

36 #

37 -set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12" "3.13")

37 +set(PYTHON_SUPPORTED_VERSIONS "3.10" "3.11" "3.12" "3.13")

39 # Supported AMD GPU architectures.

40 set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151")

benchmarks/multi_turn/benchmark_serving_multi_turn.py

+15 -23

@@ -13,7 +13,7 @@ from datetime import datetime

13 from enum import Enum

14 from http import HTTPStatus

15 from statistics import mean

16 -from typing import NamedTuple, Optional, Union

16 +from typing import NamedTuple, Union

18 import aiohttp # type: ignore

19 import numpy as np # type: ignore

@@ -46,9 +46,9 @@ class ConversationSampling(str, Enum):

47 class ClientArgs(NamedTuple):

48 seed: int

49 - max_num_requests: Optional[int]

49 + max_num_requests: int | None

50 skip_first_turn: bool

51 - max_turns: Optional[int]

51 + max_turns: int | None

52 max_active_conversations: int

53 verbose: bool

54 print_content: bool

@@ -109,9 +109,9 @@ class RequestStats(NamedTuple):

109

110 class MetricStats:

111 def __init__(self) -> None:

112 - self.min: Optional[float] = None

113 - self.max: Optional[float] = None

114 - self.avg: Optional[float] = None

112 + self.min: float | None = None

113 + self.max: float | None = None

114 + self.avg: float | None = None

115 self.sum = 0.0

116 self.count = 0

117

@@ -143,7 +143,7 @@ class MovingAverage:

143 self.index = 0

144 self.sum = 0.0

145 self.count = 0

146 - self.avg: Optional[float] = None

146 + self.avg: float | None = None

147

148 def update(self, new_value: float) -> None:

149 if self.count < self.window_size:

@@ -198,14 +198,6 @@ class DebugStats:

198 self.logger.info("-" * 50)

199

200

201 -# Must support Python 3.8, we can't use str.removeprefix(prefix)

202 -# introduced in Python 3.9

203 -def remove_prefix(text: str, prefix: str) -> str:

204 - if text.startswith(prefix):

205 - return text[len(prefix) :]

206 - return text

207 -

208 -

201 def nanosec_to_millisec(value: float) -> float:

202 return value / 1000000.0

203

@@ -220,8 +212,8 @@ async def send_request(

212 chat_url: str,

213 model: str,

214 stream: bool = True,

223 - min_tokens: Optional[int] = None,

224 - max_tokens: Optional[int] = None,

215 + min_tokens: int | None = None,

216 + max_tokens: int | None = None,

217 ) -> ServerResponse:

218 payload = {

219 "model": model,

@@ -250,9 +242,9 @@ async def send_request(

242 timeout = aiohttp.ClientTimeout(total=timeout_sec)

243

244 valid_response = True

253 - ttft: Optional[float] = None

245 + ttft: float | None = None

246 chunk_delay: list[int] = []

255 - latency: Optional[float] = None

247 + latency: float | None = None

248 first_chunk = ""

249 generated_text = ""

250

@@ -269,7 +261,7 @@ async def send_request(

261 if not chunk_bytes:

262 continue

263

272 - chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ")

264 + chunk = chunk_bytes.decode("utf-8").removeprefix("data: ")

265 if chunk == "[DONE]":

266 # End of stream

267 latency = time.perf_counter_ns() - start_time

@@ -364,7 +356,7 @@ async def send_turn(

356 req_args: RequestArgs,

357 verbose: bool,

358 verify_output: bool,

367 -) -> Optional[RequestStats]:

359 +) -> RequestStats | None:

360 assert messages_to_use > 0

361 assert messages_to_use <= len(conversation_messages)

362

@@ -769,7 +761,7 @@ def get_client_config(

761 "Number of conversations must be equal or larger than the number of clients"

762 )

763

772 - max_req_per_client: Optional[int] = None

764 + max_req_per_client: int | None = None

765 if args.max_num_requests is not None:

766 # Max number of requests per client

767 req_per_client = args.max_num_requests // args.num_clients

@@ -1032,7 +1024,7 @@ def process_statistics(

1024 warmup_percentages: list[float],

1025 test_params: dict,

1026 verbose: bool,

1035 - gen_conv_args: Optional[GenConvArgs] = None,

1027 + gen_conv_args: GenConvArgs | None = None,

1028 excel_output: bool = False,

1029 ) -> None:

1030 if len(client_metrics) == 0:

docker/Dockerfile.cpu

+1 -1

@@ -13,7 +13,7 @@

13 # vllm-dev: used for development

14 #

15 # Build arguments:

16 -# PYTHON_VERSION=3.12 (default)|3.11|3.10|3.9

16 +# PYTHON_VERSION=3.13|3.12 (default)|3.11|3.10

17 # VLLM_CPU_DISABLE_AVX512=false (default)|true

18 # VLLM_CPU_AVX512BF16=false (default)|true

19 # VLLM_CPU_AVX512VNNI=false (default)|true

docs/contributing/README.md

+2 -2

@@ -54,7 +54,7 @@ For more details about installing from source and installing for other hardware,

54 For an optimized workflow when iterating on C++/CUDA kernels, see the [Incremental Compilation Workflow](./incremental_build.md) for recommendations.

56 !!! tip

57 - vLLM is compatible with Python versions 3.9 to 3.12. However, vLLM's default [Dockerfile](gh-file:docker/Dockerfile) ships with Python 3.12 and tests in CI (except `mypy`) are run with Python 3.12.

57 + vLLM is compatible with Python versions 3.10 to 3.13. However, vLLM's default [Dockerfile](gh-file:docker/Dockerfile) ships with Python 3.12 and tests in CI (except `mypy`) are run with Python 3.12.

59 Therefore, we recommend developing with Python 3.12 to minimise the chance of your local environment clashing with our CI environment.

@@ -83,7 +83,7 @@ vLLM's `pre-commit` hooks will now run automatically every time you commit.

84 ```bash

85 pre-commit run --hook-stage manual markdownlint

86 - pre-commit run --hook-stage manual mypy-3.9

86 + pre-commit run --hook-stage manual mypy-3.10

87 ```

89 ### Documentation

docs/getting_started/installation/cpu.md

+1 -1

@@ -20,7 +20,7 @@ vLLM is a Python library that supports the following CPU variants. Select your C

21 ## Requirements

23 -- Python: 3.9 -- 3.12

23 +- Python: 3.10 -- 3.13

25 === "Intel/AMD x86"

docs/getting_started/installation/gpu.md

+1 -1

@@ -17,7 +17,7 @@ vLLM is a Python library that supports the following GPU variants. Select your G

17 ## Requirements

19 - OS: Linux

20 -- Python: 3.9 -- 3.12

20 +- Python: 3.10 -- 3.13

22 !!! note

23 vLLM does not support Windows natively. To run vLLM on Windows, you can use the Windows Subsystem for Linux (WSL) with a compatible Linux distribution, or use some community-maintained forks, e.g. [https://github.com/SystemPanic/vllm-windows](https://github.com/SystemPanic/vllm-windows).

docs/getting_started/quickstart.md

+1 -1

@@ -8,7 +8,7 @@ This guide will help you quickly get started with vLLM to perform:

8 ## Prerequisites

10 - OS: Linux

11 -- Python: 3.9 -- 3.13

11 +- Python: 3.10 -- 3.13

13 ## Installation

examples/online_serving/structured_outputs/pyproject.toml

+1 -1

@@ -1,6 +1,6 @@

1 [project]

2 name = "examples-online-structured-outputs"

3 -requires-python = ">=3.9, <3.13"

3 +requires-python = ">=3.10, <3.14"

4 dependencies = ["openai==1.78.1", "pydantic==2.11.4"]

5 version = "0.0.0"

pyproject.toml

+7 -2

@@ -20,7 +20,6 @@ license-files = ["LICENSE"]

20 readme = "README.md"

21 description = "A high-throughput and memory-efficient inference and serving engine for LLMs"

22 classifiers = [

23 - "Programming Language :: Python :: 3.9",

23 "Programming Language :: Python :: 3.10",

24 "Programming Language :: Python :: 3.11",

25 "Programming Language :: Python :: 3.12",

@@ -31,7 +30,7 @@ classifiers = [

30 "Topic :: Scientific/Engineering :: Artificial Intelligence",

31 "Topic :: Scientific/Engineering :: Information Analysis",

32 ]

34 -requires-python = ">=3.9,<3.14"

33 +requires-python = ">=3.10,<3.14"

34 dynamic = [ "version", "dependencies", "optional-dependencies"]

36 [project.urls]

@@ -79,12 +78,18 @@ ignore = [

78 "F405", "F403",

79 # lambda expression assignment

80 "E731",

81 + # zip without `strict=`

82 + "B905",

83 # Loop control variable not used within loop body

84 "B007",

85 # f-string format

86 "UP032",

87 # Can remove once 3.10+ is the minimum Python version

88 "UP007",

89 + "UP027",

90 + "UP035",

91 + "UP038",

92 + "UP045",

93 ]

95 [tool.ruff.format]

requirements/cpu.txt

+1 -2

@@ -1,8 +1,7 @@

1 # Common dependencies

2 -r common.txt

4 -numba == 0.60.0; python_version == '3.9' and platform_machine != "s390x" # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

5 -numba == 0.61.2; python_version > '3.9' and platform_machine != "s390x"

4 +numba == 0.61.2; platform_machine != "s390x" # Required for N-gram speculative decoding

6 # Dependencies for CPUs

7 packaging>=24.2

requirements/cuda.txt

+1 -2

@@ -1,8 +1,7 @@

1 # Common dependencies

2 -r common.txt

4 -numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

5 -numba == 0.61.2; python_version > '3.9'

4 +numba == 0.61.2 # Required for N-gram speculative decoding

6 # Dependencies for NVIDIA GPUs

7 ray[cgraph]>=2.48.0 # Ray Compiled Graph, required for pipeline parallelism in V1.

requirements/nightly_torch_test.txt

+1 -2

@@ -40,8 +40,7 @@ buildkite-test-collector==0.1.9

40 genai_perf==0.0.8

41 tritonclient==2.51.0

43 -numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

44 -numba == 0.61.2; python_version > '3.9'

43 +numba == 0.61.2 # Required for N-gram speculative decoding

44 numpy

45 runai-model-streamer[s3,gcs]==0.14.0

46 fastsafetensors>=0.1.10

requirements/rocm.txt

+1 -2

@@ -1,8 +1,7 @@

1 # Common dependencies

2 -r common.txt

4 -numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

5 -numba == 0.61.2; python_version > '3.9'

4 +numba == 0.61.2 # Required for N-gram speculative decoding

6 # Dependencies for AMD GPUs

7 datasets

requirements/test.in

+1 -2

@@ -48,8 +48,7 @@ buildkite-test-collector==0.1.9

48 genai_perf==0.0.8

49 tritonclient==2.51.0

51 -numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

52 -numba == 0.61.2; python_version > '3.9'

51 +numba == 0.61.2 # Required for N-gram speculative decoding

52 numpy

53 runai-model-streamer[s3,gcs]==0.14.0

54 fastsafetensors>=0.1.10

requirements/xpu.txt

+1 -1

@@ -9,7 +9,7 @@ setuptools>=77.0.3,<80.0.0

9 wheel

10 jinja2>=3.1.6

11 datasets # for benchmark scripts

12 -numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding

12 +numba == 0.61.2 # Required for N-gram speculative decoding

13 nixl==0.3.0 # for PD disaggregation

14 torch==2.8.0+xpu

15 torchaudio

tools/validate_config.py

+1 -13

@@ -8,6 +8,7 @@ and that each field has a docstring.

8 import ast

9 import inspect

10 import sys

11 +from itertools import pairwise

13 import regex as re

@@ -20,19 +21,6 @@ def get_attr_docs(cls_node: ast.ClassDef) -> dict[str, str]:

21 https://davidism.com/mit-license/

22 """

23 - def pairwise(iterable):

24 - """

25 - Manually implement https://docs.python.org/3/library/itertools.html#itertools.pairwise

26 -

27 - Can be removed when Python 3.9 support is dropped.

28 - """

29 - iterator = iter(iterable)

30 - a = next(iterator, None)

31 -

32 - for b in iterator:

33 - yield a, b

34 - a = b

35 -

24 out = {}

26 # Consider each pair of nodes.

vllm/config/utils.py

+1 -13

@@ -7,6 +7,7 @@ import inspect

7 import textwrap

8 from collections.abc import Iterable

9 from dataclasses import MISSING, Field, field, fields, is_dataclass, replace

10 +from itertools import pairwise

11 from typing import TYPE_CHECKING, Any, Protocol, TypeVar

13 import regex as re

@@ -102,19 +103,6 @@ def get_attr_docs(cls: type[Any]) -> dict[str, str]:

103 https://davidism.com/mit-license/

104 """

105

105 - def pairwise(iterable):

106 - """

107 - Manually implement https://docs.python.org/3/library/itertools.html#itertools.pairwise

108 -

109 - Can be removed when Python 3.9 support is dropped.

110 - """

111 - iterator = iter(iterable)

112 - a = next(iterator, None)

113 -

114 - for b in iterator:

115 - yield a, b

116 - a = b

117 -

106 try:

107 cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]

108 except (OSError, KeyError, TypeError):

vllm/plugins/__init__.py

+1 -6

@@ -15,12 +15,7 @@ plugins_loaded = False

17 def load_plugins_by_group(group: str) -> dict[str, Callable[[], Any]]:

18 - import sys

19 -

20 - if sys.version_info < (3, 10):

21 - from importlib_metadata import entry_points

22 - else:

23 - from importlib.metadata import entry_points

18 + from importlib.metadata import entry_points

20 allowed_plugins = envs.VLLM_PLUGINS

vllm/v1/sample/logits_processor/__init__.py

+1 -6

@@ -55,12 +55,7 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [

55 def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:

56 """Load all installed logit processor plugins"""

58 - import sys

59 -

60 - if sys.version_info < (3, 10):

61 - from importlib_metadata import entry_points

62 - else:

63 - from importlib.metadata import entry_points

58 + from importlib.metadata import entry_points

60 installed_logitsprocs_plugins = entry_points(group=LOGITSPROCS_GROUP)

61 if len(installed_logitsprocs_plugins) == 0: