cf069aa8aa · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/backend_request_func.py

+3 -3

@@ -6,7 +6,7 @@ import sys

6 import time

7 import traceback

8 from dataclasses import dataclass, field

9 -from typing import List, Optional, Union

9 +from typing import Optional, Union

10

11 import aiohttp

12 import huggingface_hub.constants

@@ -41,8 +41,8 @@ class RequestFuncOutput:

41 latency: float = 0.0

42 output_tokens: int = 0

43 ttft: float = 0.0 # Time to first token

44 - itl: List[float] = field(

45 - default_factory=list) # List of inter-token latencies

44 + itl: list[float] = field(

45 + default_factory=list) # list of inter-token latencies

46 tpot: float = 0.0 # avg next-token latencies

47 prompt_len: int = 0

48 error: str = ""

benchmarks/benchmark_guided.py

+8 -9

@@ -6,7 +6,6 @@ import json

6 import os

7 import random

8 import time

9 -from typing import List

9

10 import datasets

11 import pandas as pd

@@ -39,7 +38,7 @@ class SampleRequest:

38 completion: str = None

39

40

42 -def run_vllm(requests: List[SampleRequest],

41 +def run_vllm(requests: list[SampleRequest],

42 engine_args: EngineArgs,

43 n: int,

44 guided_decoding_rate: float = 1.0,

@@ -54,8 +53,8 @@ def run_vllm(requests: List[SampleRequest],

53 " prompt_len and expected_output_len for all requests.")

54

55 # Add the requests to the engine.

57 - prompts: List[str] = []

58 - sampling_params: List[SamplingParams] = []

56 + prompts: list[str] = []

57 + sampling_params: list[SamplingParams] = []

58 # create a list containing random selected true or false

59 guided_decoding_req_idx = random.sample(

60 range(len(requests)), int(len(requests) * guided_decoding_rate))

@@ -110,7 +109,7 @@ def run_vllm(requests: List[SampleRequest],

109

110

111 async def run_vllm_async(

113 - requests: List[SampleRequest],

112 + requests: list[SampleRequest],

113 engine_args: AsyncEngineArgs,

114 n: int,

115 guided_decoding_rate: float = 1.0,

@@ -129,8 +128,8 @@ async def run_vllm_async(

128 " prompt_len and expected_output_len for all requests.")

129

130 # Add the requests to the engine.

132 - prompts: List[str] = []

133 - sampling_params: List[SamplingParams] = []

131 + prompts: list[str] = []

132 + sampling_params: list[SamplingParams] = []

133 guided_decoding_req_idx = random.sample(

134 range(len(requests)), int(len(requests) * guided_decoding_rate))

135

@@ -203,7 +202,7 @@ async def run_vllm_async(

202

203

204 def sample_requests(tokenizer: PreTrainedTokenizerBase,

206 - args: argparse.Namespace) -> List[SampleRequest]:

205 + args: argparse.Namespace) -> list[SampleRequest]:

206 if args.dataset == 'json':

207 if args.json_schema_path is None:

208 dir_path = os.path.dirname(os.path.realpath(__file__))

@@ -287,7 +286,7 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,

286

287 elif args.dataset == "xgrammar_bench":

288 args.warmup = False

290 - requests: List[SampleRequest] = []

289 + requests: list[SampleRequest] = []

290 dataset = datasets.load_dataset("NousResearch/json-mode-eval",

291 split="train")

292 print(f"dataset has {len(dataset)} entries")

benchmarks/benchmark_latency.py

+3 -3

@@ -7,7 +7,7 @@ import json

7 import os

8 import time

9 from pathlib import Path

10 -from typing import Any, Dict, List, Optional

10 +from typing import Any, Optional

11

12 import numpy as np

13 import torch

@@ -22,7 +22,7 @@ from vllm.utils import FlexibleArgumentParser

22

23

24 def save_to_pytorch_benchmark_format(args: argparse.Namespace,

25 - results: Dict[str, Any]) -> None:

25 + results: dict[str, Any]) -> None:

26 pt_records = convert_to_pytorch_benchmark_format(

27 args=args,

28 metrics={"latency": results["latencies"]},

@@ -57,7 +57,7 @@ def main(args: argparse.Namespace):

57 dummy_prompt_token_ids = np.random.randint(10000,

58 size=(args.batch_size,

59 args.input_len))

60 - dummy_prompts: List[PromptType] = [{

60 + dummy_prompts: list[PromptType] = [{

61 "prompt_token_ids": batch

62 } for batch in dummy_prompt_token_ids.tolist()]

63

benchmarks/benchmark_prefix_caching.py

+8 -8

@@ -31,7 +31,7 @@ import dataclasses

31 import json

32 import random

33 import time

34 -from typing import List, Optional, Tuple

34 +from typing import Optional

35

36 from transformers import PreTrainedTokenizerBase

37

@@ -77,9 +77,9 @@ def sample_requests_from_dataset(

77 dataset_path: str,

78 num_requests: int,

79 tokenizer: PreTrainedTokenizerBase,

80 - input_length_range: Tuple[int, int],

80 + input_length_range: tuple[int, int],

81 fixed_output_len: Optional[int],

82 -) -> List[Request]:

82 +) -> list[Request]:

83 if fixed_output_len is not None and fixed_output_len < 4:

84 raise ValueError("output_len too small")

85

@@ -99,7 +99,7 @@ def sample_requests_from_dataset(

99 assert min_len >= 0 and max_len >= min_len, "input_length_range too small"

100

101 # Filter out sequences that are too long or too short

102 - filtered_requests: List[Request] = []

102 + filtered_requests: list[Request] = []

103

104 for i in range(len(dataset)):

105 if len(filtered_requests) == num_requests:

@@ -122,10 +122,10 @@ def sample_requests_from_dataset(

122 def sample_requests_from_random(

123 num_requests: int,

124 tokenizer: PreTrainedTokenizerBase,

125 - input_length_range: Tuple[int, int],

125 + input_length_range: tuple[int, int],

126 fixed_output_len: Optional[int],

127 prefix_len: int,

128 -) -> List[Request]:

128 +) -> list[Request]:

129

130 requests = []

131 prefix_token_ids = sample_tokens(tokenizer, prefix_len)

@@ -144,9 +144,9 @@ def sample_requests_from_random(

144 return requests

145

146

147 -def repeat_and_sort_requests(requests: List[Request],

147 +def repeat_and_sort_requests(requests: list[Request],

148 repeat_count: int,

149 - sort: bool = False) -> List[str]:

149 + sort: bool = False) -> list[str]:

150 repeated_requests = requests * repeat_count

151 if sort:

152 repeated_requests.sort(key=lambda x: x[1])

benchmarks/benchmark_prioritization.py

+4 -4

@@ -5,7 +5,7 @@ import dataclasses

5 import json

6 import random

7 import time

8 -from typing import List, Optional, Tuple

8 +from typing import Optional

9

10 from transformers import AutoTokenizer, PreTrainedTokenizerBase

11

@@ -23,7 +23,7 @@ def sample_requests(

23 num_requests: int,

24 tokenizer: PreTrainedTokenizerBase,

25 fixed_output_len: Optional[int],

26 -) -> List[Tuple[str, int, int]]:

26 +) -> list[tuple[str, int, int]]:

27 if fixed_output_len is not None and fixed_output_len < 4:

28 raise ValueError("output_len too small")

29

@@ -40,7 +40,7 @@ def sample_requests(

40 random.shuffle(dataset)

41

42 # Filter out sequences that are too long or too short

43 - filtered_dataset: List[Tuple[str, int, int]] = []

43 + filtered_dataset: list[tuple[str, int, int]] = []

44 for i in range(len(dataset)):

45 if len(filtered_dataset) == num_requests:

46 break

@@ -68,7 +68,7 @@ def sample_requests(

68

69

70 def run_vllm(

71 - requests: List[Tuple[str, int, int]],

71 + requests: list[tuple[str, int, int]],

72 n: int,

73 engine_args: EngineArgs,

74 ) -> float:

benchmarks/benchmark_serving.py

+39 -38

@@ -33,9 +33,10 @@ import os

33 import random

34 import time

35 import warnings

36 +from collections.abc import AsyncGenerator, Collection

37 from dataclasses import dataclass

38 from datetime import datetime

38 -from typing import Any, AsyncGenerator, Collection, Dict, List, Optional, Tuple

39 +from typing import Any, Optional

40

41 import numpy as np

42 import pandas as pd

@@ -73,22 +74,22 @@ class BenchmarkMetrics:

74 mean_ttft_ms: float

75 median_ttft_ms: float

76 std_ttft_ms: float

76 - percentiles_ttft_ms: List[Tuple[float, float]]

77 + percentiles_ttft_ms: list[tuple[float, float]]

78 mean_tpot_ms: float

79 median_tpot_ms: float

80 std_tpot_ms: float

80 - percentiles_tpot_ms: List[Tuple[float, float]]

81 + percentiles_tpot_ms: list[tuple[float, float]]

82 mean_itl_ms: float

83 median_itl_ms: float

84 std_itl_ms: float

84 - percentiles_itl_ms: List[Tuple[float, float]]

85 + percentiles_itl_ms: list[tuple[float, float]]

86 # E2EL stands for end-to-end latency per request.

87 # It is the time taken on the client side from sending

88 # a request to receiving a complete response.

89 mean_e2el_ms: float

90 median_e2el_ms: float

91 std_e2el_ms: float

91 - percentiles_e2el_ms: List[Tuple[float, float]]

92 + percentiles_e2el_ms: list[tuple[float, float]]

93

94

95 def sample_sharegpt_requests(

@@ -96,7 +97,7 @@ def sample_sharegpt_requests(

97 num_requests: int,

98 tokenizer: PreTrainedTokenizerBase,

99 fixed_output_len: Optional[int] = None,

99 -) -> List[Tuple[str, int, int, None]]:

100 +) -> list[tuple[str, int, int, None]]:

101 # Load the dataset.

102 with open(dataset_path, encoding='utf-8') as f:

103 dataset = json.load(f)

@@ -110,7 +111,7 @@ def sample_sharegpt_requests(

111 random.shuffle(dataset)

112

113 # Filter out sequences that are too long or too short

113 - filtered_dataset: List[Tuple[str, int, int]] = []

114 + filtered_dataset: list[tuple[str, int, int]] = []

115 for i in range(len(dataset)):

116 if len(filtered_dataset) == num_requests:

117 break

@@ -139,7 +140,7 @@ def sample_burstgpt_requests(

140 num_requests: int,

141 random_seed: int,

142 tokenizer: PreTrainedTokenizerBase,

142 -) -> List[Tuple[str, int, int, None]]:

143 +) -> list[tuple[str, int, int, None]]:

144 df = pd.read_csv(dataset_path)

145 gpt4_df = df[df["Model"] == "GPT-4"]

146 # Remove the failed requests (i.e., response length is 0)

@@ -170,7 +171,7 @@ def sample_sonnet_requests(

171 output_len: int,

172 prefix_len: int,

173 tokenizer: PreTrainedTokenizerBase,

173 -) -> List[Tuple[str, str, int, int, None]]:

174 +) -> list[tuple[str, str, int, int, None]]:

175 assert (

176 input_len > prefix_len

177 ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."

@@ -211,7 +212,7 @@ def sample_sonnet_requests(

212 prefix_lines = poem_lines[:num_prefix_lines]

213

214 # Sample the rest of lines per request.

214 - sampled_requests: List[Tuple[str, int, int]] = []

215 + sampled_requests: list[tuple[str, int, int]] = []

216 for _ in range(num_requests):

217 num_lines_needed = num_input_lines - num_prefix_lines

218 sampled_lines = "".join(prefix_lines +

@@ -238,8 +239,8 @@ def sample_vision_arena_requests(

239 num_requests: int,

240 tokenizer: PreTrainedTokenizerBase,

241 fixed_output_len: Optional[int] = None,

241 -) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:

242 - sampled_requests: List[Tuple[str, int, int, Dict[str,

242 +) -> list[tuple[str, str, int, Optional[dict[str, Collection[str]]]]]:

243 + sampled_requests: list[tuple[str, int, int, dict[str,

244 Collection[str]]]] = []

245 for data in dataset:

246 if len(sampled_requests) == num_requests:

@@ -285,7 +286,7 @@ def sample_hf_requests(

286 tokenizer: PreTrainedTokenizerBase,

287 random_seed: int,

288 fixed_output_len: Optional[int] = None,

288 -) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:

289 +) -> list[tuple[str, str, int, Optional[dict[str, Collection[str]]]]]:

290

291 # Special case for vision_arena dataset

292 if dataset_path == 'lmarena-ai/vision-arena-bench-v0.1' \

@@ -307,7 +308,7 @@ def sample_hf_requests(

308 "HF Dataset must have 'conversations' column.")

309 filter_func = lambda x: len(x["conversations"]) >= 2

310 filtered_dataset = dataset.shuffle(seed=random_seed).filter(filter_func)

310 - sampled_requests: List[Tuple[str, int, int, Dict[str,

311 + sampled_requests: list[tuple[str, int, int, dict[str,

312 Collection[str]]]] = []

313 for data in filtered_dataset:

314 if len(sampled_requests) == num_requests:

@@ -370,7 +371,7 @@ def sample_random_requests(

371 num_prompts: int,

372 range_ratio: float,

373 tokenizer: PreTrainedTokenizerBase,

373 -) -> List[Tuple[str, int, int]]:

374 +) -> list[tuple[str, int, int]]:

375 prefix_token_ids = np.random.randint(0,

376 tokenizer.vocab_size,

377 size=prefix_len).tolist()

@@ -399,10 +400,10 @@ def sample_random_requests(

400

401

402 async def get_request(

402 - input_requests: List[Tuple[str, int, int]],

403 + input_requests: list[tuple[str, int, int]],

404 request_rate: float,

405 burstiness: float = 1.0,

405 -) -> AsyncGenerator[Tuple[str, int, int], None]:

406 +) -> AsyncGenerator[tuple[str, int, int], None]:

407 """

408 Asynchronously generates requests at a specified rate

409 with OPTIONAL burstiness.

@@ -443,23 +444,23 @@ async def get_request(

444

445

446 def calculate_metrics(

446 - input_requests: List[Tuple[str, int, int]],

447 - outputs: List[RequestFuncOutput],

447 + input_requests: list[tuple[str, int, int]],

448 + outputs: list[RequestFuncOutput],

449 dur_s: float,

450 tokenizer: PreTrainedTokenizerBase,

450 - selected_percentile_metrics: List[str],

451 - selected_percentiles: List[float],

452 - goodput_config_dict: Dict[str, float],

453 -) -> Tuple[BenchmarkMetrics, List[int]]:

454 - actual_output_lens: List[int] = []

451 + selected_percentile_metrics: list[str],

452 + selected_percentiles: list[float],

453 + goodput_config_dict: dict[str, float],

454 +) -> tuple[BenchmarkMetrics, list[int]]:

455 + actual_output_lens: list[int] = []

456 total_input = 0

457 completed = 0

458 good_completed = 0

458 - itls: List[float] = []

459 - tpots: List[float] = []

460 - all_tpots: List[float] = []

461 - ttfts: List[float] = []

462 - e2els: List[float] = []

459 + itls: list[float] = []

460 + tpots: list[float] = []

461 + all_tpots: list[float] = []

462 + ttfts: list[float] = []

463 + e2els: list[float] = []

464 for i in range(len(outputs)):

465 if outputs[i].success:

466 output_len = outputs[i].output_tokens

@@ -557,19 +558,19 @@ async def benchmark(

558 model_id: str,

559 model_name: str,

560 tokenizer: PreTrainedTokenizerBase,

560 - input_requests: List[Tuple[str, int, int]],

561 + input_requests: list[tuple[str, int, int]],

562 logprobs: Optional[int],

563 best_of: int,

564 request_rate: float,

565 burstiness: float,

566 disable_tqdm: bool,

567 profile: bool,

567 - selected_percentile_metrics: List[str],

568 - selected_percentiles: List[str],

568 + selected_percentile_metrics: list[str],

569 + selected_percentiles: list[str],

570 ignore_eos: bool,

570 - goodput_config_dict: Dict[str, float],

571 + goodput_config_dict: dict[str, float],

572 max_concurrency: Optional[int],

572 - lora_modules: Optional[List[str]],

573 + lora_modules: Optional[list[str]],

574 ):

575 if backend in ASYNC_REQUEST_FUNCS:

576 request_func = ASYNC_REQUEST_FUNCS[backend]

@@ -652,7 +653,7 @@ async def benchmark(

653 pbar=pbar)

654

655 benchmark_start_time = time.perf_counter()

655 - tasks: List[asyncio.Task] = []

656 + tasks: list[asyncio.Task] = []

657 async for request in get_request(input_requests, request_rate, burstiness):

658 prompt, prompt_len, output_len, mm_content = request

659 req_model_id, req_model_name = model_id, model_name

@@ -674,7 +675,7 @@ async def benchmark(

675 asyncio.create_task(

676 limited_request_func(request_func_input=request_func_input,

677 pbar=pbar)))

677 - outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)

678 + outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)

679

680 if profile:

681 print("Stopping profiler...")

@@ -820,7 +821,7 @@ def parse_goodput(slo_pairs):

821

822

823 def save_to_pytorch_benchmark_format(args: argparse.Namespace,

823 - results: Dict[str, Any],

824 + results: dict[str, Any],

825 file_name: str) -> None:

826 metrics = [

827 "median_ttft_ms", "mean_ttft_ms", "std_ttft_ms", "p99_ttft_ms",

@@ -974,7 +975,7 @@ def main(args: argparse.Namespace):

975

976 # Save config and results to json

977 if args.save_result:

977 - result_json: Dict[str, Any] = {}

978 + result_json: dict[str, Any] = {}

979

980 # Setup

981 current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")

benchmarks/benchmark_serving_guided.py

+29 -28

@@ -30,8 +30,9 @@ import os

30 import random

31 import time

32 import warnings

33 +from collections.abc import AsyncGenerator

34 from dataclasses import dataclass

34 -from typing import AsyncGenerator, Dict, List, Optional, Tuple

35 +from typing import Optional

36

37 import datasets

38 import numpy as np

@@ -66,22 +67,22 @@ class BenchmarkMetrics:

67 mean_ttft_ms: float

68 median_ttft_ms: float

69 std_ttft_ms: float

69 - percentiles_ttft_ms: List[Tuple[float, float]]

70 + percentiles_ttft_ms: list[tuple[float, float]]

71 mean_tpot_ms: float

72 median_tpot_ms: float

73 std_tpot_ms: float

73 - percentiles_tpot_ms: List[Tuple[float, float]]

74 + percentiles_tpot_ms: list[tuple[float, float]]

75 mean_itl_ms: float

76 median_itl_ms: float

77 std_itl_ms: float

77 - percentiles_itl_ms: List[Tuple[float, float]]

78 + percentiles_itl_ms: list[tuple[float, float]]

79 # E2EL stands for end-to-end latency per request.

80 # It is the time taken on the client side from sending

81 # a request to receiving a complete response.

82 mean_e2el_ms: float

83 median_e2el_ms: float

84 std_e2el_ms: float

84 - percentiles_e2el_ms: List[Tuple[float, float]]

85 + percentiles_e2el_ms: list[tuple[float, float]]

86

87

88 @dataclasses.dataclass

@@ -104,7 +105,7 @@ class SampleRequest:

105

106

107 def sample_requests(tokenizer: PreTrainedTokenizerBase,

107 - args: argparse.Namespace) -> List[SampleRequest]:

108 + args: argparse.Namespace) -> list[SampleRequest]:

109 if args.dataset == 'json':

110 if args.json_schema_path is None:

111 dir_path = os.path.dirname(os.path.realpath(__file__))

@@ -187,7 +188,7 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,

188 ]

189

190 elif args.dataset == "xgrammar_bench":

190 - requests: List[SampleRequest] = []

191 + requests: list[SampleRequest] = []

192 dataset = datasets.load_dataset("NousResearch/json-mode-eval",

193 split="train")

194 print(f"dataset has {len(dataset)} entries")

@@ -214,10 +215,10 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,

215

216

217 async def get_request(

217 - input_requests: List[SampleRequest],

218 + input_requests: list[SampleRequest],

219 request_rate: float,

220 burstiness: float = 1.0,

220 -) -> AsyncGenerator[Tuple[int, SampleRequest], None]:

221 +) -> AsyncGenerator[tuple[int, SampleRequest], None]:

222 """

223 Asynchronously generates requests at a specified rate

224 with OPTIONAL burstiness.

@@ -258,23 +259,23 @@ async def get_request(

259

260

261 def calculate_metrics(

261 - input_requests: List[Tuple[str, int, int]],

262 - outputs: List[RequestFuncOutput],

262 + input_requests: list[tuple[str, int, int]],

263 + outputs: list[RequestFuncOutput],

264 dur_s: float,

265 tokenizer: PreTrainedTokenizerBase,

265 - selected_percentile_metrics: List[str],

266 - selected_percentiles: List[float],

267 - goodput_config_dict: Optional[Dict[str, float]] = None,

268 -) -> Tuple[BenchmarkMetrics, List[int]]:

269 - actual_output_lens: List[int] = []

266 + selected_percentile_metrics: list[str],

267 + selected_percentiles: list[float],

268 + goodput_config_dict: Optional[dict[str, float]] = None,

269 +) -> tuple[BenchmarkMetrics, list[int]]:

270 + actual_output_lens: list[int] = []

271 total_input = 0

272 completed = 0

273 good_completed = 0

273 - itls: List[float] = []

274 - tpots: List[float] = []

275 - all_tpots: List[float] = []

276 - ttfts: List[float] = []

277 - e2els: List[float] = []

274 + itls: list[float] = []

275 + tpots: list[float] = []

276 + all_tpots: list[float] = []

277 + ttfts: list[float] = []

278 + e2els: list[float] = []

279 for i in range(len(outputs)):

280 if outputs[i].success:

281 # We use the tokenizer to count the number of output tokens for all

@@ -368,18 +369,18 @@ async def benchmark(

369 base_url: str,

370 model_id: str,

371 tokenizer: PreTrainedTokenizerBase,

371 - input_requests: List[SampleRequest],

372 + input_requests: list[SampleRequest],

373 request_rate: float,

374 burstiness: float,

375 disable_tqdm: bool,

376 profile: bool,

376 - selected_percentile_metrics: List[str],

377 - selected_percentiles: List[str],

377 + selected_percentile_metrics: list[str],

378 + selected_percentiles: list[str],

379 ignore_eos: bool,

380 max_concurrency: Optional[int],

381 guided_decoding_ratio: float,

382 guided_decoding_backend: str,

382 - goodput_config_dict: Optional[Dict[str, float]] = None,

383 + goodput_config_dict: Optional[dict[str, float]] = None,

384 ):

385 if backend in ASYNC_REQUEST_FUNCS:

386 request_func = ASYNC_REQUEST_FUNCS[backend]

@@ -459,8 +460,8 @@ async def benchmark(

460 pbar=pbar)

461

462 benchmark_start_time = time.perf_counter()

462 - tasks: List[asyncio.Task] = []

463 - expected: List[str] = []

463 + tasks: list[asyncio.Task] = []

464 + expected: list[str] = []

465 async for i, request in get_request(input_requests, request_rate,

466 burstiness):

467 extra_body = prepare_extra_body(

@@ -479,7 +480,7 @@ async def benchmark(

480 asyncio.create_task(

481 limited_request_func(request_func_input=request_func_input,

482 pbar=pbar)))

482 - outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)

483 + outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)

484

485 if profile:

486 print("Stopping profiler...")

benchmarks/benchmark_throughput.py

+19 -19

@@ -7,7 +7,7 @@ import os

7 import random

8 import time

9 from functools import cache

10 -from typing import Any, Dict, List, Optional, Tuple

10 +from typing import Any, Optional

11

12 import torch

13 import uvloop

@@ -74,12 +74,12 @@ def lora_path_on_disk(lora_path: str) -> str:

74 return get_adapter_absolute_path(lora_path)

75

76

77 -lora_tokenizer_cache: Dict[int, AnyTokenizer] = {}

77 +lora_tokenizer_cache: dict[int, AnyTokenizer] = {}

78

79

80 def get_random_lora_request(

81 args: argparse.Namespace

82 -) -> Tuple[LoRARequest, Optional[AnyTokenizer]]:

82 +) -> tuple[LoRARequest, Optional[AnyTokenizer]]:

83 global lora_tokenizer_cache

84 lora_id = random.randint(1, args.max_loras)

85 lora_request = LoRARequest(lora_name=str(lora_id),

@@ -91,7 +91,7 @@ def get_random_lora_request(

91

92

93 def sample_requests(tokenizer: PreTrainedTokenizerBase,

94 - args: argparse.Namespace) -> List[SampleRequest]:

94 + args: argparse.Namespace) -> list[SampleRequest]:

95

96 dataset_path: str = args.dataset

97 num_requests: int = args.num_prompts

@@ -109,7 +109,7 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,

109 random.shuffle(dataset)

110

111 # Filter out sequences that are too long or too short

112 - filtered_dataset: List[SampleRequest] = []

112 + filtered_dataset: list[SampleRequest] = []

113 for data in tqdm(dataset,

114 total=len(filtered_dataset),

115 desc="sampling requests"):

@@ -165,7 +165,7 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,

165

166

167 def run_vllm(

168 - requests: List[SampleRequest],

168 + requests: list[SampleRequest],

169 n: int,

170 engine_args: EngineArgs,

171 ) -> float:

@@ -178,8 +178,8 @@ def run_vllm(

178 "Please ensure that max_model_len is greater than the sum of"

179 " prompt_len and expected_output_len for all requests.")

180 # Add the requests to the engine.

181 - prompts: List[TextPrompt] = []

182 - sampling_params: List[SamplingParams] = []

181 + prompts: list[TextPrompt] = []

182 + sampling_params: list[SamplingParams] = []

183 for request in requests:

184 prompts.append(

185 TextPrompt(prompt=request.prompt,

@@ -192,7 +192,7 @@ def run_vllm(

192 ignore_eos=True,

193 max_tokens=request.expected_output_len,

194 ))

195 - lora_requests: Optional[List[LoRARequest]] = None

195 + lora_requests: Optional[list[LoRARequest]] = None

196 if engine_args.enable_lora:

197 lora_requests = [request.lora_request for request in requests]

198

@@ -225,7 +225,7 @@ def run_vllm(

225

226

227 async def run_vllm_async(

228 - requests: List[SampleRequest],

228 + requests: list[SampleRequest],

229 n: int,

230 engine_args: AsyncEngineArgs,

231 disable_frontend_multiprocessing: bool = False,

@@ -242,9 +242,9 @@ async def run_vllm_async(

242 " prompt_len and expected_output_len for all requests.")

243

244 # Add the requests to the engine.

245 - prompts: List[TextPrompt] = []

246 - sampling_params: List[SamplingParams] = []

247 - lora_requests: List[Optional[LoRARequest]] = []

245 + prompts: list[TextPrompt] = []

246 + sampling_params: list[SamplingParams] = []

247 + lora_requests: list[Optional[LoRARequest]] = []

248 for request in requests:

249 prompts.append(

250 TextPrompt(prompt=request.prompt,

@@ -276,7 +276,7 @@ async def run_vllm_async(

276

277

278 def run_hf(

279 - requests: List[SampleRequest],

279 + requests: list[SampleRequest],

280 model: str,

281 tokenizer: PreTrainedTokenizerBase,

282 n: int,

@@ -292,7 +292,7 @@ def run_hf(

292

293 pbar = tqdm(total=len(requests))

294 start = time.perf_counter()

295 - batch: List[str] = []

295 + batch: list[str] = []

296 max_prompt_len = 0

297 max_output_len = 0

298 for i in range(len(requests)):

@@ -334,7 +334,7 @@ def run_hf(

334

335

336 def run_mii(

337 - requests: List[SampleRequest],

337 + requests: list[SampleRequest],

338 model: str,

339 tensor_parallel_size: int,

340 output_len: int,

@@ -352,7 +352,7 @@ def run_mii(

352

353

354 def save_to_pytorch_benchmark_format(args: argparse.Namespace,

355 - results: Dict[str, Any]) -> None:

355 + results: dict[str, Any]) -> None:

356 pt_records = convert_to_pytorch_benchmark_format(

357 args=args,

358 metrics={

@@ -479,8 +479,8 @@ if __name__ == "__main__":

479 type=str,

480 default=None,

481 help="Path to the dataset. The dataset is expected to "

482 - "be a json in form of List[Dict[..., conversations: "

483 - "List[Dict[..., value: <prompt_or_response>]]]]")

482 + "be a json in form of list[dict[..., conversations: "

483 + "list[dict[..., value: <prompt_or_response>]]]]")

484 parser.add_argument("--input-len",

485 type=int,

486 default=None,

benchmarks/benchmark_utils.py

+4 -4

@@ -4,12 +4,12 @@ import argparse

4 import json

5 import math

6 import os

7 -from typing import Any, Dict, List

7 +from typing import Any

8

9

10 def convert_to_pytorch_benchmark_format(args: argparse.Namespace,

11 - metrics: Dict[str, List],

12 - extra_info: Dict[str, Any]) -> List:

11 + metrics: dict[str, list],

12 + extra_info: dict[str, Any]) -> list:

13 """

14 Save the benchmark results in the format used by PyTorch OSS benchmark with

15 on metric per record

@@ -64,6 +64,6 @@ class InfEncoder(json.JSONEncoder):

64 return super().iterencode(self.clear_inf(o), *args, **kwargs)

65

66

67 -def write_to_json(filename: str, records: List) -> None:

67 +def write_to_json(filename: str, records: list) -> None:

68 with open(filename, "w") as f:

69 json.dump(records, f, cls=InfEncoder)

benchmarks/cutlass_benchmarks/sparse_benchmarks.py

+5 -4

@@ -5,7 +5,8 @@ import copy

5 import itertools

6 import pickle as pkl

7 import time

8 -from typing import Callable, Iterable, List, Tuple

8 +from collections.abc import Iterable

9 +from typing import Callable

10

11 import torch

12 import torch.utils.benchmark as TBenchmark

@@ -228,7 +229,7 @@ def print_timers(timers: Iterable[TMeasurement]):

229

230

231 def run(dtype: torch.dtype,

231 - MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:

232 + MKNs: Iterable[tuple[int, int, int]]) -> Iterable[TMeasurement]:

233 results = []

234 for m, k, n in MKNs:

235 timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",

@@ -241,7 +242,7 @@ def run(dtype: torch.dtype,

242

243 # output makers

244 def make_output(data: Iterable[TMeasurement],

244 - MKNs: Iterable[Tuple[int, int, int]],

245 + MKNs: Iterable[tuple[int, int, int]],

246 base_description: str,

247 timestamp=None):

248 print(f"== All Results {base_description} ====")

@@ -282,7 +283,7 @@ def run_model_bench(args):

283 for i, model in enumerate(args.models):

284 print(f"[{i}] {model}")

285

285 - def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:

286 + def model_shapes(model_name: str, tp_size: int) -> list[tuple[int, int]]:

287 KNs = []

288 for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):

289 KN[tp_split_dim] = KN[tp_split_dim] // tp_size

benchmarks/cutlass_benchmarks/utils.py

+4 -4

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 # Cutlass bench utils

4 -from typing import Iterable, Tuple

4 +from collections.abc import Iterable

5

6 import torch

7

@@ -27,7 +27,7 @@ def to_fp16(tensor: torch.Tensor) -> torch.Tensor:

27

28

29 def make_rand_tensors(dtype: torch.dtype, m: int, n: int,

30 - k: int) -> Tuple[torch.Tensor, torch.Tensor]:

30 + k: int) -> tuple[torch.Tensor, torch.Tensor]:

31 a = torch.randn((m, k), device='cuda') * 5

32 b = torch.randn((n, k), device='cuda').t() * 5

33

@@ -63,7 +63,7 @@ def prune_to_2_4(tensor):

63

64

65 def make_rand_sparse_tensors(dtype: torch.dtype, m: int, n: int,

66 - k: int) -> Tuple[torch.Tensor, torch.Tensor]:

66 + k: int) -> tuple[torch.Tensor, torch.Tensor]:

67 a = torch.randn((m, k), device='cuda') * 5

68 b = torch.randn((n, k), device='cuda').t() * 5

69

@@ -88,7 +88,7 @@ def make_rand_sparse_tensors(dtype: torch.dtype, m: int, n: int,

88

89 def make_n_rand_sparse_tensors(num_tensors: int, dtype: torch.dtype,

90 m: int, n: int, k: int) -> \

91 - Tuple[Iterable[torch.Tensor], Iterable[torch.Tensor]]:

91 + tuple[Iterable[torch.Tensor], Iterable[torch.Tensor]]:

92 ABs = []

93 for _ in range(num_tensors):

94 b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)

benchmarks/cutlass_benchmarks/w8a8_benchmarks.py

+9 -8

@@ -5,7 +5,8 @@ import copy

5 import itertools

6 import pickle as pkl

7 import time

8 -from typing import Callable, Iterable, List, Optional, Tuple

8 +from collections.abc import Iterable

9 +from typing import Callable, Optional

10

11 import torch

12 import torch.utils.benchmark as TBenchmark

@@ -49,7 +50,7 @@ def bench_int8(

50 n: int,

51 label: str,

52 sub_label: str,

52 - bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:

53 + bench_kernels: Optional[list[str]] = None) -> Iterable[TMeasurement]:

54 """Benchmark INT8-based kernels."""

55 assert dtype == torch.int8

56 a, b = make_rand_tensors(torch.int8, m, n, k)

@@ -101,7 +102,7 @@ def bench_fp8(

102 n: int,

103 label: str,

104 sub_label: str,

104 - bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:

105 + bench_kernels: Optional[list[str]] = None) -> Iterable[TMeasurement]:

106 """Benchmark FP8-based kernels."""

107 assert dtype == torch.float8_e4m3fn

108 a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)

@@ -180,7 +181,7 @@ def bench(dtype: torch.dtype,

181 n: int,

182 label: str,

183 sub_label: str,

183 - bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:

184 + bench_kernels: Optional[list[str]] = None) -> Iterable[TMeasurement]:

185 if dtype == torch.int8:

186 return bench_int8(dtype, m, k, n, label, sub_label, bench_kernels)

187 if dtype == torch.float8_e4m3fn:

@@ -195,8 +196,8 @@ def print_timers(timers: Iterable[TMeasurement]):

196

197

198 def run(dtype: torch.dtype,

198 - MKNs: Iterable[Tuple[int, int, int]],

199 - bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:

199 + MKNs: Iterable[tuple[int, int, int]],

200 + bench_kernels: Optional[list[str]] = None) -> Iterable[TMeasurement]:

201 results = []

202 for m, k, n in MKNs:

203 timers = bench(dtype,

@@ -212,7 +213,7 @@ def run(dtype: torch.dtype,

213

214

215 def make_output(data: Iterable[TMeasurement],

215 - MKNs: Iterable[Tuple[int, int, int]],

216 + MKNs: Iterable[tuple[int, int, int]],

217 base_description: str,

218 timestamp=None):

219 print(f"== All Results {base_description} ====")

@@ -248,7 +249,7 @@ def run_model_bench(args):

249 for i, model in enumerate(args.models):

250 print(f"[{i}] {model}")

251

251 - def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:

252 + def model_shapes(model_name: str, tp_size: int) -> list[tuple[int, int]]:

253 KNs = []

254 for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):

255 KN[tp_split_dim] = KN[tp_split_dim] // tp_size

benchmarks/fused_kernels/layernorm_rms_benchmarks.py

+3 -2

@@ -2,9 +2,10 @@

2

3 import pickle as pkl

4 import time

5 +from collections.abc import Iterable

6 from dataclasses import dataclass

7 from itertools import product

7 -from typing import Callable, Iterable, List, Optional

8 +from typing import Callable, Optional

9

10 import torch

11 import torch.utils.benchmark as TBenchmark

@@ -29,7 +30,7 @@ class bench_params_t:

30 f'x DT {self.dtype}')

31

32

32 -def get_bench_params() -> List[bench_params_t]:

33 +def get_bench_params() -> list[bench_params_t]:

34 ## Test Fixtures

35 NUM_TOKENS = [2**x for x in range(11)]

36 HIDDEN_SIZES = list(range(1024, 8129, 1024))

benchmarks/kernels/benchmark_lora.py

+30 -30

@@ -9,7 +9,7 @@ from dataclasses import dataclass

9 from enum import Enum, auto

10 from itertools import product

11 from pathlib import Path

12 -from typing import Any, Callable, Dict, List, Optional, Tuple

12 +from typing import Any, Callable, Optional

13

14 import torch

15 import torch.utils.benchmark as TBenchmark

@@ -61,15 +61,15 @@ def make_rand_lora_weight_tensor(k: int,

61

62

63 def make_rand_tensors(

64 - a_shape: Tuple[int],

65 - b_shape: Tuple[int],

66 - c_shape: Tuple[int],

64 + a_shape: tuple[int],

65 + b_shape: tuple[int],

66 + c_shape: tuple[int],

67 a_dtype: torch.dtype,

68 b_dtype: torch.dtype,

69 c_dtype: torch.dtype,

70 num_slices: int,

71 device: str = "cuda",

72 -) -> Tuple[torch.Tensor, List[torch.Tensor], torch.Tensor]:

72 +) -> tuple[torch.Tensor, list[torch.Tensor], torch.Tensor]:

73 """

74 Make LoRA input/output matrices.

75 """

@@ -135,7 +135,7 @@ def make_token_lora_mapping(num_tokens: int, num_prompts: int,

135

136

137 def ref_group_gemm(ref_out: torch.Tensor, input: torch.Tensor,

138 - lora_weights: List[torch.Tensor],

138 + lora_weights: list[torch.Tensor],

139 seq_lens_cpu: torch.Tensor,

140 prompt_lora_mapping_cpu: torch.Tensor, scaling: float,

141 add_inputs: Optional[bool]):

@@ -204,7 +204,7 @@ class OpType(Enum):

204 def is_expand_slice_fn(self) -> bool:

205 return self in [OpType.BGMV_EXPAND_SLICE]

206

207 - def num_slices(self) -> List[int]:

207 + def num_slices(self) -> list[int]:

208 if self in [OpType.SGMV_EXPAND, OpType.SGMV_SHRINK]:

209 # SGMV kernels supports slices

210 return [1, 2, 3]

@@ -215,7 +215,7 @@ class OpType(Enum):

215 raise ValueError(f"Unrecognized OpType {self}")

216

217 def mkn(self, batch_size: int, seq_length: int, hidden_size: int,

218 - lora_rank: int) -> Tuple[int, int, int]:

218 + lora_rank: int) -> tuple[int, int, int]:

219 num_tokens = batch_size * seq_length

220 if self.is_shrink_fn():

221 m = num_tokens

@@ -230,7 +230,7 @@ class OpType(Enum):

230

231 def matmul_dtypes(

232 self, op_dtype: torch.dtype

233 - ) -> Tuple[torch.dtype, torch.dtype, torch.dtype]:

233 + ) -> tuple[torch.dtype, torch.dtype, torch.dtype]:

234 """

235 return a type, b type and c type for A x B = C

236 """

@@ -243,7 +243,7 @@ class OpType(Enum):

243 def matmul_shapes(

244 self, batch_size: int, seq_length: int, hidden_size: int,

245 lora_rank: int, num_loras: int,

246 - num_slices: int) -> Tuple[Tuple[int], Tuple[int], Tuple[int]]:

246 + num_slices: int) -> tuple[tuple[int], tuple[int], tuple[int]]:

247 """

248 Given num_slices, return the shapes of the A, B, and C matrices

249 in A x B = C, for the op_type

@@ -268,7 +268,7 @@ class OpType(Enum):

268

269 def bench_fn(self) -> Callable:

270

271 - def emulate_bgmv_expand_slice(kwargs_list: List[Dict[str, Any]]):

271 + def emulate_bgmv_expand_slice(kwargs_list: list[dict[str, Any]]):

272 for x in kwargs_list:

273 bgmv_expand_slice(**x)

274

@@ -285,7 +285,7 @@ class OpType(Enum):

285 raise ValueError(f"Unrecognized optype {self}")

286

287 def run_ref_group_gemm(self, output: torch.Tensor, input: torch.Tensor,

288 - lora_weights: List[torch.Tensor],

288 + lora_weights: list[torch.Tensor],

289 **kwargs) -> Callable:

290 """Each benchmark operation expected the input, lora_weights and outputs

291 in a slightly different format. Refer to self.matmul_shapes().

@@ -384,7 +384,7 @@ class BenchmarkTensors:

384 """

385 # matmul tensors

386 input: torch.Tensor

387 - lora_weights_lst: List[torch.Tensor]

387 + lora_weights_lst: list[torch.Tensor]

388 output: torch.Tensor

389 # metadata tensors

390 seq_lens: torch.Tensor

@@ -469,7 +469,7 @@ class BenchmarkTensors:

469 for i in range(len(self.lora_weights_lst)):

470 self.lora_weights_lst[i] = to_device(self.lora_weights_lst[i])

471

472 - def metadata(self) -> Tuple[int, int, int]:

472 + def metadata(self) -> tuple[int, int, int]:

473 """

474 Return num_seqs, num_tokens and max_seq_len

475 """

@@ -505,7 +505,7 @@ class BenchmarkTensors:

505 self.seq_lens = seq_lens.to(dtype=self.seq_lens.dtype)

506 self.seq_start_loc = seq_start_loc.to(dtype=self.seq_start_loc.dtype)

507

508 - def as_sgmv_shrink_kwargs(self) -> Dict[str, Any]:

508 + def as_sgmv_shrink_kwargs(self) -> dict[str, Any]:

509 self.convert_to_sgmv_benchmark_tensors()

510 self.sanity_check()

511 self.to_device(self.input.device)

@@ -540,7 +540,7 @@ class BenchmarkTensors:

540 'scaling': 1.0,

541 }

542

543 - def as_sgmv_expand_kwargs(self, add_inputs: bool) -> Dict[str, Any]:

543 + def as_sgmv_expand_kwargs(self, add_inputs: bool) -> dict[str, Any]:

544

545 self.convert_to_sgmv_benchmark_tensors()

546 self.sanity_check()

@@ -578,7 +578,7 @@ class BenchmarkTensors:

578 'add_inputs': add_inputs,

579 }

580

581 - def as_bgmv_shrink_kwargs(self) -> Dict[str, Any]:

581 + def as_bgmv_shrink_kwargs(self) -> dict[str, Any]:

582 assert len(self.lora_weights_lst) == 1

583 self.to_device(self.input.device)

584

@@ -634,7 +634,7 @@ class BenchmarkTensors:

634 'add_inputs': add_inputs

635 }

636

637 - def as_bgmv_expand_slice_kwargs(self, add_inputs: bool) -> Dict[str, Any]:

637 + def as_bgmv_expand_slice_kwargs(self, add_inputs: bool) -> dict[str, Any]:

638

639 _, num_tokens, _, num_slices = self.metadata()

640 # Sanity check shapes

@@ -670,7 +670,7 @@ class BenchmarkTensors:

670

671 def bench_fn_kwargs(self,

672 op_type: OpType,

673 - add_inputs: Optional[bool] = None) -> Dict[str, Any]:

673 + add_inputs: Optional[bool] = None) -> dict[str, Any]:

674 if op_type.is_shrink_fn():

675 assert add_inputs is None

676 else:

@@ -734,7 +734,7 @@ def bench_optype(ctx: BenchmarkContext,

734 assert expand_fn_add_inputs is not None

735

736 # BenchmarkContext -> BenchmarkTensors

737 - bench_tensors : List[BenchmarkTensors] = \

737 + bench_tensors : list[BenchmarkTensors] = \

738 [BenchmarkTensors.make(ctx, op_type) for _ in range(arg_pool_size)]

739 for bt in bench_tensors:

740 bt.sanity_check()

@@ -746,7 +746,7 @@ def bench_optype(ctx: BenchmarkContext,

746 for bt in bench_tensors

747 ])

748

749 - # BenchmarkTensors -> Dict (kwargs)

749 + # BenchmarkTensors -> dict (kwargs)

750 kwargs_list = [

751 bt.bench_fn_kwargs(op_type, add_inputs=expand_fn_add_inputs)

752 for bt in bench_tensors

@@ -841,7 +841,7 @@ def use_cuda_graph_recommendation() -> str:

841 """

842

843

844 -def print_timers(timers: List[TMeasurement],

844 +def print_timers(timers: list[TMeasurement],

845 args: Optional[argparse.Namespace] = None):

846 compare = TBenchmark.Compare(timers)

847 compare.print()

@@ -861,7 +861,7 @@ def print_timers(timers: List[TMeasurement],

861 "small num_loras the goal should be to match the torch.mm numbers.")

862

863

864 -def run(args: argparse.Namespace, bench_ctxs: List[BenchmarkContext]):

864 +def run(args: argparse.Namespace, bench_ctxs: list[BenchmarkContext]):

865

866 if args.cuda_graph_nops is not None:

867 assert args.cuda_graph_nops > 0

@@ -873,7 +873,7 @@ def run(args: argparse.Namespace, bench_ctxs: List[BenchmarkContext]):

873 timers = []

874 for bench_ctx in bench_ctxs:

875 for seq_len in args.seq_lengths:

876 - bench_ops: List[OpType] = []

876 + bench_ops: list[OpType] = []

877 if seq_len == 1:

878 # bench all decode ops

879 bench_ops = [op for op in args.op_types if op.is_decode_op()]

@@ -921,10 +921,10 @@ def run(args: argparse.Namespace, bench_ctxs: List[BenchmarkContext]):

921 pickle.dump(timers, f)

922

923

924 -def as_benchmark_contexts(hidden_sizes: List[int], lora_ranks: List[int],

925 - args: argparse.Namespace) -> List[BenchmarkContext]:

924 +def as_benchmark_contexts(hidden_sizes: list[int], lora_ranks: list[int],

925 + args: argparse.Namespace) -> list[BenchmarkContext]:

926

927 - ctxs: List[BenchmarkContext] = []

927 + ctxs: list[BenchmarkContext] = []

928 for batch_size, hidden_size, lora_rank, num_loras, sort_by_lora_id in product( # noqa

929 args.batch_sizes, list(hidden_sizes), lora_ranks, args.num_loras,

930 args.sort_by_lora_id):

@@ -954,7 +954,7 @@ def run_list_bench(args: argparse.Namespace):

954 f" LoRA Ranks {args.lora_ranks}")

955

956 # Get all benchmarking contexts

957 - bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(

957 + bench_contexts: list[BenchmarkContext] = as_benchmark_contexts(

958 hidden_sizes=args.hidden_sizes, lora_ranks=args.lora_ranks, args=args)

959

960 run(args, bench_contexts)

@@ -975,7 +975,7 @@ def run_range_bench(args: argparse.Namespace):

975 f" LoRA Ranks {lora_ranks}")

976

977 # Get all benchmarking contexts

978 - bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(

978 + bench_contexts: list[BenchmarkContext] = as_benchmark_contexts(

979 hidden_sizes=hidden_sizes, lora_ranks=lora_ranks, args=args)

980

981 run(args, bench_contexts)

@@ -1002,7 +1002,7 @@ def run_model_bench(args: argparse.Namespace):

1002 f" LoRA Ranks {args.lora_ranks}")

1003

1004 # Get all benchmarking contexts

1005 - bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(

1005 + bench_contexts: list[BenchmarkContext] = as_benchmark_contexts(

1006 hidden_sizes=hidden_sizes, lora_ranks=args.lora_ranks, args=args)

1007

1008 run(args, bench_contexts)

benchmarks/kernels/benchmark_machete.py

+13 -12

@@ -7,9 +7,10 @@ import math

7 import os

8 import pickle as pkl

9 import time

10 +from collections.abc import Iterable

11 from dataclasses import dataclass

12 from itertools import product

12 -from typing import Callable, Iterable, List, Optional, Tuple

13 +from typing import Callable, Optional

14

15 import pandas as pd

16 import torch

@@ -102,8 +103,8 @@ def quantize_and_pack(atype: torch.dtype,

103 return w_ref, w_q, w_s, w_zp

104

105

105 -def create_bench_tensors(shape: Tuple[int, int, int], types: TypeConfig,

106 - group_size: Optional[int]) -> List[BenchmarkTensors]:

106 +def create_bench_tensors(shape: tuple[int, int, int], types: TypeConfig,

107 + group_size: Optional[int]) -> list[BenchmarkTensors]:

108 m, n, k = shape

109

110 # we want to make sure that weights don't fit into L2 cache between runs so

@@ -114,7 +115,7 @@ def create_bench_tensors(shape: Tuple[int, int, int], types: TypeConfig,

115

116 a = rand_data((m, k), types.act_type, scale=5)

117

117 - benchmark_tensors: List[BenchmarkTensors] = []

118 + benchmark_tensors: list[BenchmarkTensors] = []

119 for _ in range(num_weights):

120 w = rand_data((k, n), types.act_type, scale=5)

121

@@ -276,7 +277,7 @@ def machete_create_bench_fn(bt: BenchmarkTensors,

277

278

279 def bench_fns(label: str, sub_label: str, description: str,

279 - fns: List[Callable]):

280 + fns: list[Callable]):

281

282 min_run_time = 1 if not NVTX_PROFILE else 0.1

283 res = TBenchmark.Timer(

@@ -311,7 +312,7 @@ def bench(types: TypeConfig,

312 n: int,

313 label: str,

314 sub_label: str,

314 - sweep_schedules: bool = True) -> List[TMeasurement]:

315 + sweep_schedules: bool = True) -> list[TMeasurement]:

316 benchmark_tensors = create_bench_tensors((m, n, k), types, group_size)

317 sub_label += f", L={len(benchmark_tensors)}"

318

@@ -414,12 +415,12 @@ def bench(types: TypeConfig,

415

416

417 # runner

417 -def print_timers(timers: List[TMeasurement]):

418 +def print_timers(timers: list[TMeasurement]):

419 compare = TBenchmark.Compare(timers)

420 compare.print()

421

422

422 -def run(args, MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:

423 +def run(args, MKNs: Iterable[tuple[int, int, int]]) -> Iterable[TMeasurement]:

424 types = TypeConfig(

425 act_type=args.act_type,

426 weight_type=scalar_types.uint4b8 if args.group_zero_type is None \

@@ -431,7 +432,7 @@ def run(args, MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:

432 token_scale_type=args.token_scale_type,

433 )

434

434 - results: List[TMeasurement] = []

435 + results: list[TMeasurement] = []

436 for m, k, n in MKNs:

437 timers = bench(types,

438 args.group_size,

@@ -449,8 +450,8 @@ def run(args, MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:

450

451 # output makers

452 def make_output(

452 - data: List[TMeasurement],

453 - MKNs: Iterable[Tuple[int, int, int]],

453 + data: list[TMeasurement],

454 + MKNs: Iterable[tuple[int, int, int]],

455 base_description: str,

456 timestamp=None,

457 ):

@@ -497,7 +498,7 @@ def run_model_bench(args):

498 for i, model in enumerate(args.models):

499 print(f"[{i}] {model}")

500

500 - def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:

501 + def model_shapes(model_name: str, tp_size: int) -> list[tuple[int, int]]:

502 KNs = []

503 for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):

504 KN[tp_split_dim] = KN[tp_split_dim] // tp_size

benchmarks/kernels/benchmark_marlin.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import torch

4 import torch.utils.benchmark as benchmark

5 from benchmark_shapes import WEIGHT_SHAPES

@@ -31,7 +29,7 @@ ACT_ORDER_OPTS = [False, True]

29 K_FULL_OPTS = [False, True]

30

31

34 -def bench_run(results: List[benchmark.Measurement], model: str,

32 +def bench_run(results: list[benchmark.Measurement], model: str,

33 act_order: bool, is_k_full: bool, quant_type: ScalarType,

34 group_size: int, size_m: int, size_k: int, size_n: int):

35 label = "Quant Matmul"

@@ -221,7 +219,7 @@ def main(args):

219 for i, model in enumerate(args.models):

220 print(f"[{i}] {model}")

221

224 - results: List[benchmark.Measurement] = []

222 + results: list[benchmark.Measurement] = []

223

224 for model in args.models:

225 for layer in WEIGHT_SHAPES[model]:

benchmarks/kernels/benchmark_moe.py

+9 -9

@@ -4,7 +4,7 @@ import argparse

4 import time

5 from datetime import datetime

6 from itertools import product

7 -from typing import Any, Dict, List, Tuple, TypedDict

7 +from typing import Any, TypedDict

8

9 import ray

10 import torch

@@ -132,7 +132,7 @@ def benchmark_config(

132 start_event = torch.cuda.Event(enable_timing=True)

133 end_event = torch.cuda.Event(enable_timing=True)

134

135 - latencies: List[float] = []

135 + latencies: list[float] = []

136 for i in range(num_iters):

137 prepare(i)

138 torch.cuda.synchronize()

@@ -175,8 +175,8 @@ def get_rocm_tuning_space(use_fp16):

175 return param_ranges

176

177

178 -def get_configs_compute_bound(use_fp16) -> List[Dict[str, int]]:

179 - configs: List[BenchmarkConfig] = []

178 +def get_configs_compute_bound(use_fp16) -> list[dict[str, int]]:

179 + configs: list[BenchmarkConfig] = []

180

181 if current_platform.is_rocm():

182 param_ranges = get_rocm_tuning_space(use_fp16)

@@ -335,7 +335,7 @@ class BenchmarkWorker:

335 dtype: torch.dtype,

336 use_fp8_w8a8: bool,

337 use_int8_w8a16: bool,

338 - ) -> Tuple[Dict[str, int], float]:

338 + ) -> tuple[dict[str, int], float]:

339 current_platform.seed_everything(self.seed)

340 dtype_str = get_config_dtype_str(dtype,

341 use_int8_w8a16=use_int8_w8a16,

@@ -371,8 +371,8 @@ class BenchmarkWorker:

371 dtype: torch.dtype,

372 use_fp8_w8a8: bool,

373 use_int8_w8a16: bool,

374 - search_space: List[Dict[str, int]],

375 - ) -> Dict[str, int]:

374 + search_space: list[dict[str, int]],

375 + ) -> dict[str, int]:

376 best_config = None

377 best_time = float("inf")

378 if current_platform.is_rocm():

@@ -434,7 +434,7 @@ def sort_config(config: BenchmarkConfig) -> BenchmarkConfig:

434 }

435

436

437 -def save_configs(configs: Dict[int, BenchmarkConfig], num_experts: int,

437 +def save_configs(configs: dict[int, BenchmarkConfig], num_experts: int,

438 shard_intermediate_size: int, hidden_size: int, topk: int,

439 dtype: torch.dtype, use_fp8_w8a8: bool,

440 use_int8_w8a16: bool) -> None:

@@ -498,7 +498,7 @@ def main(args: argparse.Namespace):

498 num_gpus = int(ray.available_resources()["GPU"])

499 workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)]

500

501 - def _distribute(method: str, inputs: List[Any]) -> List[Any]:

501 + def _distribute(method: str, inputs: list[Any]) -> list[Any]:

502 outputs = []

503 worker_idx = 0

504 for input_args in inputs:

benchmarks/kernels/benchmark_paged_attention.py

+2 -2

@@ -2,7 +2,7 @@

2

3 import random

4 import time

5 -from typing import List, Optional

5 +from typing import Optional

6

7 import torch

8

@@ -54,7 +54,7 @@ def main(

54

55 # Create the block tables.

56 max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size

57 - block_tables_lst: List[List[int]] = []

57 + block_tables_lst: list[list[int]] = []

58 for _ in range(num_seqs):

59 block_table = [

60 random.randint(0, NUM_BLOCKS - 1)

benchmarks/kernels/benchmark_rmsnorm.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import itertools

4 -from typing import Optional, Tuple, Union

4 +from typing import Optional, Union

5

6 import torch

7 import triton

@@ -22,7 +22,7 @@ class HuggingFaceRMSNorm(nn.Module):

22 self,

23 x: torch.Tensor,

24 residual: Optional[torch.Tensor] = None,

25 - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:

25 + ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:

26 orig_dtype = x.dtype

27 x = x.to(torch.float32)

28 if residual is not None:

benchmarks/kernels/benchmark_rope.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from itertools import accumulate

4 -from typing import List, Optional

4 +from typing import Optional

5

6 import nvtx

7 import torch

@@ -39,7 +39,7 @@ def benchmark_rope_kernels_multi_lora(

39 })

40 # non-batched RoPE takes only one scaling factor, we create multiple

41 # instances to simulate the same behavior

42 - non_batched_ropes: List[RotaryEmbedding] = []

42 + non_batched_ropes: list[RotaryEmbedding] = []

43 for scaling_factor in scaling_factors:

44 non_batched_ropes.append(

45 get_rope(head_size, rotary_dim, max_position, base, is_neox_style,

benchmarks/kernels/graph_machete_bench.py

+1 -2

@@ -4,7 +4,6 @@ import math

4 import pickle

5 import re

6 from collections import defaultdict

7 -from typing import List

7

8 import matplotlib.pyplot as plt

9 import pandas as pd

@@ -23,7 +22,7 @@ if __name__ == "__main__":

22

23 with open(args.filename, 'rb') as f:

24 data = pickle.load(f)

26 - raw_results: List[TMeasurement] = data["results"]

25 + raw_results: list[TMeasurement] = data["results"]

26

27 results = defaultdict(lambda: list())

28 for v in raw_results:

benchmarks/kernels/utils.py

+2 -1

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import dataclasses

4 -from typing import Any, Callable, Iterable, Optional

4 +from collections.abc import Iterable

5 +from typing import Any, Callable, Optional

6

7 import torch

8 import torch.utils.benchmark as TBenchmark

csrc/cutlass_extensions/vllm_cutlass_library_extension.py

+7 -7

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import enum

4 -from typing import Dict, Union

4 +from typing import Union

5

6 from cutlass_library import *

7

@@ -21,7 +21,7 @@ class MixedInputKernelScheduleType(enum.Enum):

21 TmaWarpSpecializedCooperative = enum_auto()

22

23

24 -VLLMDataTypeNames: Dict[Union[VLLMDataType, DataType], str] = {

24 +VLLMDataTypeNames: dict[Union[VLLMDataType, DataType], str] = {

25 **DataTypeNames, # type: ignore

26 **{

27 VLLMDataType.u4b8: "u4b8",

@@ -29,7 +29,7 @@ VLLMDataTypeNames: Dict[Union[VLLMDataType, DataType], str] = {

29 }

30 }

31

32 -VLLMDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

32 +VLLMDataTypeTag: dict[Union[VLLMDataType, DataType], str] = {

33 **DataTypeTag, # type: ignore

34 **{

35 VLLMDataType.u4b8: "cutlass::vllm_uint4b8_t",

@@ -37,7 +37,7 @@ VLLMDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

37 }

38 }

39

40 -VLLMDataTypeSize: Dict[Union[VLLMDataType, DataType], int] = {

40 +VLLMDataTypeSize: dict[Union[VLLMDataType, DataType], int] = {

41 **DataTypeSize, # type: ignore

42 **{

43 VLLMDataType.u4b8: 4,

@@ -45,7 +45,7 @@ VLLMDataTypeSize: Dict[Union[VLLMDataType, DataType], int] = {

45 }

46 }

47

48 -VLLMDataTypeVLLMScalarTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

48 +VLLMDataTypeVLLMScalarTypeTag: dict[Union[VLLMDataType, DataType], str] = {

49 VLLMDataType.u4b8: "vllm::kU4B8",

50 VLLMDataType.u8b128: "vllm::kU8B128",

51 DataType.u4: "vllm::kU4",

@@ -56,7 +56,7 @@ VLLMDataTypeVLLMScalarTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

56 DataType.bf16: "vllm::kBfloat16",

57 }

58

59 -VLLMDataTypeTorchDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

59 +VLLMDataTypeTorchDataTypeTag: dict[Union[VLLMDataType, DataType], str] = {

60 DataType.u8: "at::ScalarType::Byte",

61 DataType.s8: "at::ScalarType::Char",

62 DataType.e4m3: "at::ScalarType::Float8_e4m3fn",

@@ -66,7 +66,7 @@ VLLMDataTypeTorchDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {

66 DataType.f32: "at::ScalarType::Float",

67 }

68

69 -VLLMKernelScheduleTag: Dict[Union[

69 +VLLMKernelScheduleTag: dict[Union[

70 MixedInputKernelScheduleType, KernelScheduleType], str] = {

71 **KernelScheduleTag, # type: ignore

72 **{

csrc/quantization/machete/generate.py

+10 -10

@@ -8,7 +8,7 @@ from collections.abc import Iterable

8 from copy import deepcopy

9 from dataclasses import dataclass, fields

10 from functools import reduce

11 -from typing import Dict, List, Optional, Tuple, Union

11 +from typing import Optional, Union

12

13 import jinja2

14 # yapf conflicts with isort for this block

@@ -247,8 +247,8 @@ TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative

247

248 @dataclass(frozen=True)

249 class ScheduleConfig:

250 - tile_shape_mn: Tuple[int, int]

251 - cluster_shape_mnk: Tuple[int, int, int]

250 + tile_shape_mn: tuple[int, int]

251 + cluster_shape_mnk: tuple[int, int, int]

252 kernel_schedule: MixedInputKernelScheduleType

253 epilogue_schedule: EpilogueScheduleType

254 tile_scheduler: TileSchedulerType

@@ -277,8 +277,8 @@ class PrepackTypeConfig:

277 @dataclass

278 class ImplConfig:

279 types: TypeConfig

280 - schedules: List[ScheduleConfig]

281 - heuristic: List[Tuple[Optional[str], ScheduleConfig]]

280 + schedules: list[ScheduleConfig]

281 + heuristic: list[tuple[Optional[str], ScheduleConfig]]

282

283

284 def generate_sch_sig(schedule_config: ScheduleConfig) -> str:

@@ -333,7 +333,7 @@ def is_power_of_two(n):

333 return (n != 0) and (n & (n - 1) == 0)

334

335

336 -def to_cute_constant(value: List[int]):

336 +def to_cute_constant(value: list[int]):

337

338 def _to_cute_constant(value: int):

339 if is_power_of_two(value):

@@ -347,7 +347,7 @@ def to_cute_constant(value: List[int]):

347 return _to_cute_constant(value)

348

349

350 -def unique_schedules(impl_configs: List[ImplConfig]):

350 +def unique_schedules(impl_configs: list[ImplConfig]):

351 return list(

352 set(sch for impl_config in impl_configs

353 for sch in impl_config.schedules))

@@ -391,7 +391,7 @@ mm_impl_template = create_template(IMPL_TEMPLATE)

391 prepack_dispatch_template = create_template(PREPACK_TEMPLATE)

392

393

394 -def create_sources(impl_configs: List[ImplConfig], num_impl_files=8):

394 +def create_sources(impl_configs: list[ImplConfig], num_impl_files=8):

395 sources = []

396

397 sources.append((

@@ -435,7 +435,7 @@ def create_sources(impl_configs: List[ImplConfig], num_impl_files=8):

435 num_impls = reduce(lambda x, y: x + len(y.schedules), impl_configs, 0)

436 num_impls_per_file = math.ceil(num_impls / num_impl_files)

437

438 - files_impls: List[List[ImplConfig]] = [[]]

438 + files_impls: list[list[ImplConfig]] = [[]]

439

440 curr_num_impls_assigned = 0

441 curr_impl_in_file = 0

@@ -515,7 +515,7 @@ def generate():

515 for cond, tile_config in default_tile_heuristic_config.items()

516 ]

517

518 - def get_unique_schedules(heuristic: Dict[str, ScheduleConfig]):

518 + def get_unique_schedules(heuristic: dict[str, ScheduleConfig]):

519 # Do not use schedules = list(set(...)) because we need to make sure

520 # the output list is deterministic; otherwise the generated kernel file

521 # will be non-deterministic and causes ccache miss.

docs/source/conf.py

+1 -2

@@ -17,7 +17,6 @@ import inspect

17 import logging

18 import os

19 import sys

20 -from typing import List

20

21 import requests

22 from sphinx.ext import autodoc

@@ -58,7 +57,7 @@ templates_path = ['_templates']

57 # List of patterns, relative to source directory, that match files and

58 # directories to ignore when looking for source files.

59 # This pattern also affects html_static_path and html_extra_path.

61 -exclude_patterns: List[str] = ["**/*.template.md", "**/*.inc.md"]

60 +exclude_patterns: list[str] = ["**/*.template.md", "**/*.inc.md"]

61

62 # Exclude the prompt "$" when copying code

63 copybutton_prompt_text = r"\$ "

docs/source/features/reasoning_outputs.md

+2 -2

@@ -123,7 +123,7 @@ class ExampleParser(ReasoningParser):

123

124 def extract_reasoning_content(

125 self, model_output: str, request: ChatCompletionRequest

126 - ) -> Tuple[Optional[str], Optional[str]]:

126 + ) -> tuple[Optional[str], Optional[str]]:

127 """

128 Extract reasoning content from a complete model-generated string.

129

@@ -138,7 +138,7 @@ class ExampleParser(ReasoningParser):

138 The request object that was used to generate the model_output.

139

140 Returns:

141 - Tuple[Optional[str], Optional[str]]

141 + tuple[Optional[str], Optional[str]]

142 A tuple containing the reasoning content and the content.

143 """

144 ```

docs/source/features/structured_outputs.md

+1 -1

@@ -193,7 +193,7 @@ class Step(BaseModel):

193

194

195 class MathResponse(BaseModel):

196 - steps: List[Step]

196 + steps: list[Step]

197 final_answer: str

198

199

docs/source/generate_examples.py

+1 -1

@@ -74,7 +74,7 @@ class Example:

74 path (Path): The path to the main directory or file.

75 category (str): The category of the document.

76 main_file (Path): The main file in the directory.

77 - other_files (list[Path]): List of other files in the directory.

77 + other_files (list[Path]): list of other files in the directory.

78 title (str): The title of the document.

79

80 Methods:

examples/offline_inference/distributed.py

+5 -5

@@ -6,7 +6,7 @@ distributively on a multi-nodes cluster.

6 Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html

7 """

8

9 -from typing import Any, Dict, List

9 +from typing import Any

10

11 import numpy as np

12 import ray

@@ -36,13 +36,13 @@ class LLMPredictor:

36 self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",

37 tensor_parallel_size=tensor_parallel_size)

38

39 - def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]:

39 + def __call__(self, batch: dict[str, np.ndarray]) -> dict[str, list]:

40 # Generate texts from the prompts.

41 # The output is a list of RequestOutput objects that contain the prompt,

42 # generated text, and other information.

43 outputs = self.llm.generate(batch["text"], sampling_params)

44 - prompt: List[str] = []

45 - generated_text: List[str] = []

44 + prompt: list[str] = []

45 + generated_text: list[str] = []

46 for output in outputs:

47 prompt.append(output.prompt)

48 generated_text.append(' '.join([o.text for o in output.outputs]))

@@ -72,7 +72,7 @@ def scheduling_strategy_fn():

72 pg, placement_group_capture_child_tasks=True))

73

74

75 -resources_kwarg: Dict[str, Any] = {}

75 +resources_kwarg: dict[str, Any] = {}

76 if tensor_parallel_size == 1:

77 # For tensor_parallel_size == 1, we simply set num_gpus=1.

78 resources_kwarg["num_gpus"] = 1

examples/offline_inference/llm_engine_example.py

+3 -4

@@ -1,13 +1,12 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import argparse

4 -from typing import List, Tuple

4

5 from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams

6 from vllm.utils import FlexibleArgumentParser

7

8

10 -def create_test_prompts() -> List[Tuple[str, SamplingParams]]:

9 +def create_test_prompts() -> list[tuple[str, SamplingParams]]:

10 """Create a list of test prompts with their sampling parameters."""

11 return [

12 ("A robot may not injure a human being",

@@ -24,7 +23,7 @@ def create_test_prompts() -> List[Tuple[str, SamplingParams]]:

23

24

25 def process_requests(engine: LLMEngine,

27 - test_prompts: List[Tuple[str, SamplingParams]]):

26 + test_prompts: list[tuple[str, SamplingParams]]):

27 """Continuously process a list of prompts and handle the outputs."""

28 request_id = 0

29

@@ -34,7 +33,7 @@ def process_requests(engine: LLMEngine,

33 engine.add_request(str(request_id), prompt, sampling_params)

34 request_id += 1

35

37 - request_outputs: List[RequestOutput] = engine.step()

36 + request_outputs: list[RequestOutput] = engine.step()

37

38 for request_output in request_outputs:

39 if request_output.finished:

examples/offline_inference/lora_with_quantization_inference.py

+4 -4

@@ -7,7 +7,7 @@ Requires HuggingFace credentials for access.

7 """

8

9 import gc

10 -from typing import List, Optional, Tuple

10 +from typing import Optional

11

12 import torch

13 from huggingface_hub import snapshot_download

@@ -18,7 +18,7 @@ from vllm.lora.request import LoRARequest

18

19 def create_test_prompts(

20 lora_path: str

21 -) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:

21 +) -> list[tuple[str, SamplingParams, Optional[LoRARequest]]]:

22 return [

23 # this is an example of using quantization without LoRA

24 ("My name is",

@@ -49,7 +49,7 @@ def create_test_prompts(

49

50

51 def process_requests(engine: LLMEngine,

52 - test_prompts: List[Tuple[str, SamplingParams,

52 + test_prompts: list[tuple[str, SamplingParams,

53 Optional[LoRARequest]]]):

54 """Continuously process a list of prompts and handle the outputs."""

55 request_id = 0

@@ -63,7 +63,7 @@ def process_requests(engine: LLMEngine,

63 lora_request=lora_request)

64 request_id += 1

65

66 - request_outputs: List[RequestOutput] = engine.step()

66 + request_outputs: list[RequestOutput] = engine.step()

67 for request_output in request_outputs:

68 if request_output.finished:

69 print("----------------------------------------------------")

examples/offline_inference/mlpspeculator.py

+1 -2

@@ -2,12 +2,11 @@

2

3 import gc

4 import time

5 -from typing import List

5

6 from vllm import LLM, SamplingParams

7

8

10 -def time_generation(llm: LLM, prompts: List[str],

9 +def time_generation(llm: LLM, prompts: list[str],

10 sampling_params: SamplingParams):

11 # Generate texts from the prompts. The output is a list of RequestOutput

12 # objects that contain the prompt, generated text, and other information.

examples/offline_inference/multilora_inference.py

+4 -4

@@ -6,7 +6,7 @@ for offline inference.

6 Requires HuggingFace credentials for access to Llama2.

7 """

8

9 -from typing import List, Optional, Tuple

9 +from typing import Optional

10

11 from huggingface_hub import snapshot_download

12

@@ -16,7 +16,7 @@ from vllm.lora.request import LoRARequest

16

17 def create_test_prompts(

18 lora_path: str

19 -) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:

19 +) -> list[tuple[str, SamplingParams, Optional[LoRARequest]]]:

20 """Create a list of test prompts with their sampling parameters.

21

22 2 requests for base model, 4 requests for the LoRA. We define 2

@@ -56,7 +56,7 @@ def create_test_prompts(

56

57

58 def process_requests(engine: LLMEngine,

59 - test_prompts: List[Tuple[str, SamplingParams,

59 + test_prompts: list[tuple[str, SamplingParams,

60 Optional[LoRARequest]]]):

61 """Continuously process a list of prompts and handle the outputs."""

62 request_id = 0

@@ -70,7 +70,7 @@ def process_requests(engine: LLMEngine,

70 lora_request=lora_request)

71 request_id += 1

72

73 - request_outputs: List[RequestOutput] = engine.step()

73 + request_outputs: list[RequestOutput] = engine.step()

74

75 for request_output in request_outputs:

76 if request_output.finished:

examples/offline_inference/prithvi_geospatial_mae.py

+4 -4

@@ -21,7 +21,7 @@ import argparse

21 import datetime

22 import os

23 import re

24 -from typing import List, Union

24 +from typing import Union

25

26 import albumentations

27 import numpy as np

@@ -260,9 +260,9 @@ def _convert_np_uint8(float_image: torch.Tensor):

260

261

262 def load_example(

263 - file_paths: List[str],

264 - mean: List[float] = None,

265 - std: List[float] = None,

263 + file_paths: list[str],

264 + mean: list[float] = None,

265 + std: list[float] = None,

266 indices: Union[list[int], None] = None,

267 ):

268 """Build an input example by loading images in *file_paths*.

examples/offline_inference/profiling.py

+8 -7

@@ -5,8 +5,9 @@ import json

5 import os

6 import sys

7 from argparse import RawTextHelpFormatter

8 +from collections.abc import Generator

9 from dataclasses import asdict, dataclass

9 -from typing import Any, Dict, Generator, List, Optional, TypeAlias

10 +from typing import Any, Optional, TypeAlias

11

12 import torch

13 import tqdm

@@ -42,8 +43,8 @@ def get_dtype(dtype: str):

43 return dtype

44

45

45 -OutputLen_NumReqs_Map: TypeAlias = Dict[int, int]

46 -def compute_request_output_lengths(batch_size: int, step_requests: List[int]) \

46 +OutputLen_NumReqs_Map: TypeAlias = dict[int, int]

47 +def compute_request_output_lengths(batch_size: int, step_requests: list[int]) \

48 -> OutputLen_NumReqs_Map:

49 """

50 Given the number of requests, batch_size, and the number of requests

@@ -63,7 +64,7 @@ def compute_request_output_lengths(batch_size: int, step_requests: List[int]) \

64 Args:

65 batch_size (int): Number of requests submitted for profile. This is

66 args.batch_size.

66 - step_requests (List[int]): step_requests[i] is the number of requests

67 + step_requests (list[int]): step_requests[i] is the number of requests

68 that the ith engine step should process.

69

70 Returns:

@@ -114,7 +115,7 @@ def compute_request_output_lengths(batch_size: int, step_requests: List[int]) \

115 return ol_nr

116

117

117 -def determine_requests_per_step(context: ProfileContext) -> List[int]:

118 +def determine_requests_per_step(context: ProfileContext) -> list[int]:

119 """

120 Determine number of requests each engine step should process.

121 If context.num_steps is set, then all engine steps process the

@@ -130,7 +131,7 @@ def determine_requests_per_step(context: ProfileContext) -> List[int]:

131 context: ProfileContext object.

132

133 Returns:

133 - List[int]: Number of requests to process for all engine-steps.

134 + list[int]: Number of requests to process for all engine-steps.

135 output[i], contains the number of requests that the ith step

136 should process.

137 """

@@ -170,7 +171,7 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],

171 for key, value in asdict(context).items():

172 print(f" {key} = {value}")

173

173 - requests_per_step: List[int] = determine_requests_per_step(context)

174 + requests_per_step: list[int] = determine_requests_per_step(context)

175

176 ol_nr: OutputLen_NumReqs_Map = compute_request_output_lengths(

177 context.batch_size, requests_per_step)

examples/offline_inference/profiling_tpu/profiling.py

+1 -2

@@ -4,7 +4,6 @@ import argparse

4 import dataclasses

5 import os

6 import time

7 -from typing import List

7

8 import numpy as np

9 import torch_xla.debug.profiler as xp

@@ -35,7 +34,7 @@ def main(args: argparse.Namespace):

34 dummy_prompt_token_ids = np.random.randint(10000,

35 size=(args.batch_size,

36 args.input_len))

38 - dummy_prompts: List[PromptType] = [{

37 + dummy_prompts: list[PromptType] = [{

38 "prompt_token_ids": batch

39 } for batch in dummy_prompt_token_ids.tolist()]

40

examples/offline_inference/vision_language_multi_image.py

+17 -17

@@ -5,7 +5,7 @@ multi-image input on vision language models for text generation,

5 using the chat template defined by the model.

6 """

7 from argparse import Namespace

8 -from typing import List, NamedTuple, Optional

8 +from typing import NamedTuple, Optional

9

10 from PIL.Image import Image

11 from transformers import AutoProcessor, AutoTokenizer

@@ -24,8 +24,8 @@ IMAGE_URLS = [

24 class ModelRequestData(NamedTuple):

25 llm: LLM

26 prompt: str

27 - stop_token_ids: Optional[List[int]]

28 - image_data: List[Image]

27 + stop_token_ids: Optional[list[int]]

28 + image_data: list[Image]

29 chat_template: Optional[str]

30

31

@@ -34,7 +34,7 @@ class ModelRequestData(NamedTuple):

34 # Unless specified, these settings have been tested to work on a single L4.

35

36

37 -def load_aria(question, image_urls: List[str]) -> ModelRequestData:

37 +def load_aria(question, image_urls: list[str]) -> ModelRequestData:

38 model_name = "rhymes-ai/Aria"

39 llm = LLM(model=model_name,

40 tokenizer_mode="slow",

@@ -55,7 +55,7 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:

55 )

56

57

58 -def load_deepseek_vl2(question: str, image_urls: List[str]):

58 +def load_deepseek_vl2(question: str, image_urls: list[str]):

59 model_name = "deepseek-ai/deepseek-vl2-tiny"

60

61 llm = LLM(model=model_name,

@@ -77,7 +77,7 @@ def load_deepseek_vl2(question: str, image_urls: List[str]):

77 )

78

79

80 -def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:

80 +def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:

81 model_name = "h2oai/h2ovl-mississippi-800m"

82

83 llm = LLM(

@@ -111,7 +111,7 @@ def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:

111 )

112

113

114 -def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:

114 +def load_idefics3(question, image_urls: list[str]) -> ModelRequestData:

115 model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

116

117 # The configuration below has been confirmed to launch on a single L40 GPU.

@@ -142,7 +142,7 @@ def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:

142 )

143

144

145 -def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:

145 +def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:

146 model_name = "OpenGVLab/InternVL2-2B"

147

148 llm = LLM(

@@ -179,7 +179,7 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:

179 )

180

181

182 -def load_mllama(question, image_urls: List[str]) -> ModelRequestData:

182 +def load_mllama(question, image_urls: list[str]) -> ModelRequestData:

183 model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"

184

185 # The configuration below has been confirmed to launch on a single L40 GPU.

@@ -201,7 +201,7 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:

201 )

202

203

204 -def load_nvlm_d(question: str, image_urls: List[str]):

204 +def load_nvlm_d(question: str, image_urls: list[str]):

205 model_name = "nvidia/NVLM-D-72B"

206

207 # Adjust this as necessary to fit in GPU

@@ -234,7 +234,7 @@ def load_nvlm_d(question: str, image_urls: List[str]):

234 )

235

236

237 -def load_pixtral_hf(question: str, image_urls: List[str]) -> ModelRequestData:

237 +def load_pixtral_hf(question: str, image_urls: list[str]) -> ModelRequestData:

238 model_name = "mistral-community/pixtral-12b"

239

240 # Adjust this as necessary to fit in GPU

@@ -259,7 +259,7 @@ def load_pixtral_hf(question: str, image_urls: List[str]) -> ModelRequestData:

259 )

260

261

262 -def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:

262 +def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:

263 # num_crops is an override kwarg to the multimodal image processor;

264 # For some models, e.g., Phi-3.5-vision-instruct, it is recommended

265 # to use 16 for single frame scenarios, and 4 for multi-frame.

@@ -295,7 +295,7 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:

295

296

297 def load_qwen_vl_chat(question: str,

298 - image_urls: List[str]) -> ModelRequestData:

298 + image_urls: list[str]) -> ModelRequestData:

299 model_name = "Qwen/Qwen-VL-Chat"

300 llm = LLM(

301 model=model_name,

@@ -336,7 +336,7 @@ def load_qwen_vl_chat(question: str,

336 )

337

338

339 -def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:

339 +def load_qwen2_vl(question, image_urls: list[str]) -> ModelRequestData:

340 try:

341 from qwen_vl_utils import process_vision_info

342 except ModuleNotFoundError:

@@ -393,7 +393,7 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:

393 )

394

395

396 -def load_qwen2_5_vl(question, image_urls: List[str]) -> ModelRequestData:

396 +def load_qwen2_5_vl(question, image_urls: list[str]) -> ModelRequestData:

397 try:

398 from qwen_vl_utils import process_vision_info

399 except ModuleNotFoundError:

@@ -466,7 +466,7 @@ model_example_map = {

466 }

467

468

469 -def run_generate(model, question: str, image_urls: List[str]):

469 +def run_generate(model, question: str, image_urls: list[str]):

470 req_data = model_example_map[model](question, image_urls)

471

472 sampling_params = SamplingParams(temperature=0.0,

@@ -487,7 +487,7 @@ def run_generate(model, question: str, image_urls: List[str]):

487 print(generated_text)

488

489

490 -def run_chat(model: str, question: str, image_urls: List[str]):

490 +def run_chat(model: str, question: str, image_urls: list[str]):

491 req_data = model_example_map[model](question, image_urls)

492

493 sampling_params = SamplingParams(temperature=0.0,

examples/online_serving/api_client.py

+3 -3

@@ -7,7 +7,7 @@ For production use, we recommend `vllm serve` and the OpenAI client API.

7

8 import argparse

9 import json

10 -from typing import Iterable, List

10 +from collections.abc import Iterable

11

12 import requests

13

@@ -39,7 +39,7 @@ def post_http_request(prompt: str,

39 return response

40

41

42 -def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:

42 +def get_streaming_response(response: requests.Response) -> Iterable[list[str]]:

43 for chunk in response.iter_lines(chunk_size=8192,

44 decode_unicode=False,

45 delimiter=b"\0"):

@@ -49,7 +49,7 @@ def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:

49 yield output

50

51

52 -def get_response(response: requests.Response) -> List[str]:

52 +def get_response(response: requests.Response) -> list[str]:

53 data = json.loads(response.content)

54 output = data["text"]

55 return output

examples/online_serving/openai_embedding_client.py

+1 -1

@@ -24,4 +24,4 @@ responses = client.embeddings.create(

24 )

25

26 for data in responses.data:

27 - print(data.embedding) # list of float of len 4096

27 + print(data.embedding) # List of float of len 4096

pyproject.toml

+26 -2

@@ -65,6 +65,32 @@ exclude = [

65 [tool.ruff.lint.per-file-ignores]

66 "vllm/version.py" = ["F401"]

67 "vllm/_version.py" = ["ALL"]

68 +# Python 3.8 typing. TODO: Remove these excludes after v1.0.0

69 +"vllm/adapter_commons/**/*.py" = ["UP006", "UP035"]

70 +"vllm/attention/**/*.py" = ["UP006", "UP035"]

71 +"vllm/compilation/**/*.py" = ["UP006", "UP035"]

72 +"vllm/core/**/*.py" = ["UP006", "UP035"]

73 +"vllm/device_allocator/**/*.py" = ["UP006", "UP035"]

74 +"vllm/distributed/**/*.py" = ["UP006", "UP035"]

75 +"vllm/engine/**/*.py" = ["UP006", "UP035"]

76 +"vllm/executor/**/*.py" = ["UP006", "UP035"]

77 +"vllm/inputs/**/*.py" = ["UP006", "UP035"]

78 +"vllm/logging_utils/**/*.py" = ["UP006", "UP035"]

79 +"vllm/lora/**/*.py" = ["UP006", "UP035"]

80 +"vllm/model_executor/**/*.py" = ["UP006", "UP035"]

81 +"vllm/multimodal/**/*.py" = ["UP006", "UP035"]

82 +"vllm/platforms/**/*.py" = ["UP006", "UP035"]

83 +"vllm/plugins/**/*.py" = ["UP006", "UP035"]

84 +"vllm/profiler/**/*.py" = ["UP006", "UP035"]

85 +"vllm/prompt_adapter/**/*.py" = ["UP006", "UP035"]

86 +"vllm/spec_decode/**/*.py" = ["UP006", "UP035"]

87 +"vllm/third_party/**/*.py" = ["UP006", "UP035"]

88 +"vllm/transformers_utils/**/*.py" = ["UP006", "UP035"]

89 +"vllm/triton_utils/**/*.py" = ["UP006", "UP035"]

90 +"vllm/usage/**/*.py" = ["UP006", "UP035"]

91 +"vllm/vllm_flash_attn/**/*.py" = ["UP006", "UP035"]

92 +"vllm/assets/**/*.py" = ["UP006", "UP035"]

93 +"vllm/worker/**/*.py" = ["UP006", "UP035"]

94

95 [tool.ruff.lint]

96 select = [

@@ -91,8 +117,6 @@ ignore = [

117 "B007",

118 # f-string format

119 "UP032",

94 - # Python 3.8 typing

95 - "UP006", "UP035",

120 # Can remove once 3.10+ is the minimum Python version

121 "UP007",

122 ]

setup.py

+3 -4

@@ -9,7 +9,6 @@ import subprocess

9 import sys

10 from pathlib import Path

11 from shutil import which

12 -from typing import Dict, List

12

13 import torch

14 from packaging.version import Version, parse

@@ -78,7 +77,7 @@ class CMakeExtension(Extension):

77

78 class cmake_build_ext(build_ext):

79 # A dict of extension directories that have been configured.

81 - did_config: Dict[str, bool] = {}

80 + did_config: dict[str, bool] = {}

81

82 #

83 # Determine number of compilation jobs and optionally nvcc compile threads.

@@ -548,10 +547,10 @@ def get_vllm_version() -> str:

547 return version

548

549

551 -def get_requirements() -> List[str]:

550 +def get_requirements() -> list[str]:

551 """Get Python package dependencies from requirements.txt."""

552

554 - def _read_requirements(filename: str) -> List[str]:

553 + def _read_requirements(filename: str) -> list[str]:

554 with open(get_path(filename)) as f:

555 requirements = f.read().strip().split("\n")

556 resolved_requirements = []

tests/async_engine/api_server_async_engine.py

+3 -2

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """vllm.entrypoints.api_server with some extra logging for testing."""

3 -from typing import Any, Dict, Iterable

3 +from collections.abc import Iterable

4 +from typing import Any

5

6 import uvicorn

7 from fastapi.responses import JSONResponse, Response

@@ -24,7 +25,7 @@ class AsyncLLMEngineWithStats(AsyncLLMEngine):

25 self._num_aborts += len(ids)

26 await super()._engine_abort(ids)

27

27 - def testing_stats(self) -> Dict[str, Any]:

28 + def testing_stats(self) -> dict[str, Any]:

29 return {"num_aborted_requests": self._num_aborts}

30

31

tests/async_engine/test_async_llm_engine.py

+2 -2

@@ -6,7 +6,7 @@ import uuid

6 from asyncio import CancelledError

7 from copy import copy

8 from dataclasses import dataclass

9 -from typing import List, Optional

9 +from typing import Optional

10

11 import pytest

12 import pytest_asyncio

@@ -254,7 +254,7 @@ async def test_output_kinds(async_engine, stop):

254 params.output_kind = RequestOutputKind.DELTA

255

256 prompt_tokens = None

257 - output_tokens: List[int] = []

257 + output_tokens: list[int] = []

258 output_text = ""

259 output_count = 0

260 final_output = None

tests/compile/piecewise/test_toy_llama.py

+3 -3

@@ -8,7 +8,7 @@ if the config `tractable_init` is set to True. Otherwise, the weights are

8 initialized randomly with a fixed seed.

9 """

10 from dataclasses import dataclass

11 -from typing import Any, List, Optional, Tuple

11 +from typing import Any, Optional

12

13 import torch

14 from torch import nn

@@ -56,7 +56,7 @@ class LlamaConfig:

56 random_seed: int = 0

57

58 def compute_hash(self) -> str:

59 - factors: List[Any] = []

59 + factors: list[Any] = []

60 for k, v in self.__dict__.items():

61 if k == "random_seed":

62 continue

@@ -174,7 +174,7 @@ class LlamaDecoderLayer(nn.Module):

174 positions: torch.Tensor,

175 hidden_states: torch.Tensor,

176 residual: Optional[torch.Tensor],

177 - ) -> Tuple[torch.Tensor, torch.Tensor]:

177 + ) -> tuple[torch.Tensor, torch.Tensor]:

178 """

179 For tractable computation:

180 - if residual is None, the outputs are:

tests/compile/test_basic_correctness.py

+4 -4

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import dataclasses

4 -from typing import Dict, List, Optional

4 +from typing import Optional

5

6 import pytest

7

@@ -14,7 +14,7 @@ from ..utils import compare_all_settings

14 @dataclasses.dataclass

15 class TestSetting:

16 model: str

17 - model_args: List[str]

17 + model_args: list[str]

18 pp_size: int

19 tp_size: int

20 attn_backend: str

@@ -108,8 +108,8 @@ def test_compile_correctness(test_setting: TestSetting):

108 final_args = ["--enforce-eager"] + model_args + ["-pp", str(pp_size)] + \

109 ["-tp", str(tp_size)]

110

111 - all_args: List[List[str]] = []

112 - all_envs: List[Optional[Dict[str, str]]] = []

111 + all_args: list[list[str]] = []

112 + all_envs: list[Optional[dict[str, str]]] = []

113

114 for level in [

115 CompilationLevel.NO_COMPILATION,

tests/conftest.py

+79 -80

@@ -5,8 +5,7 @@ import os

5 import tempfile

6 from collections import UserList

7 from enum import Enum

8 -from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,

9 - TypedDict, TypeVar, Union)

8 +from typing import Any, Callable, Optional, TypedDict, TypeVar, Union

9

10 import numpy as np

11 import pytest

@@ -47,14 +46,14 @@ _SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")

46

47 _M = TypeVar("_M")

48

50 -_PromptMultiModalInput = Union[List[_M], List[List[_M]]]

49 +_PromptMultiModalInput = Union[list[_M], list[list[_M]]]

50

51 PromptImageInput = _PromptMultiModalInput[Image.Image]

53 -PromptAudioInput = _PromptMultiModalInput[Tuple[np.ndarray, int]]

52 +PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]

53 PromptVideoInput = _PromptMultiModalInput[np.ndarray]

54

55

57 -def _read_prompts(filename: str) -> List[str]:

56 +def _read_prompts(filename: str) -> list[str]:

57 with open(filename) as f:

58 prompts = f.readlines()

59 return prompts

@@ -77,7 +76,7 @@ class _ImageAssets(_ImageAssetsBase):

76 ImageAsset("cherry_blossom"),

77 ])

78

80 - def prompts(self, prompts: _ImageAssetPrompts) -> List[str]:

79 + def prompts(self, prompts: _ImageAssetPrompts) -> list[str]:

80 """

81 Convenience method to define the prompt for each test image.

82

@@ -102,7 +101,7 @@ class _VideoAssets(_VideoAssetsBase):

101 VideoAsset("sample_demo_1.mp4"),

102 ])

103

105 - def prompts(self, prompts: _VideoAssetPrompts) -> List[str]:

104 + def prompts(self, prompts: _VideoAssetPrompts) -> list[str]:

105 return [prompts["sample_demo_1"]]

106

107

@@ -175,7 +174,7 @@ def dynamo_reset():

174

175

176 @pytest.fixture

178 -def example_prompts() -> List[str]:

177 +def example_prompts() -> list[str]:

178 prompts = []

179 for filename in _TEST_PROMPTS:

180 prompts += _read_prompts(filename)

@@ -197,7 +196,7 @@ class DecoderPromptType(Enum):

196

197 @pytest.fixture

198 def example_encoder_decoder_prompts(

200 -) -> Dict[DecoderPromptType, List[ExplicitEncoderDecoderPrompt]]:

199 +) -> dict[DecoderPromptType, list[ExplicitEncoderDecoderPrompt]]:

200 '''

201 Returns an encoder prompt list and a decoder prompt list, wherein each pair

202 of same-index entries in both lists corresponds to an (encoder prompt,

@@ -229,7 +228,7 @@ def example_encoder_decoder_prompts(

228

229

230 @pytest.fixture

232 -def example_long_prompts() -> List[str]:

231 +def example_long_prompts() -> list[str]:

232 prompts = []

233 for filename in _LONG_PROMPTS:

234 prompts += _read_prompts(filename)

@@ -273,11 +272,11 @@ class HfRunner:

272 model_name: str,

273 dtype: str = "half",

274 *,

276 - model_kwargs: Optional[Dict[str, Any]] = None,

275 + model_kwargs: Optional[dict[str, Any]] = None,

276 is_sentence_transformer: bool = False,

277 is_cross_encoder: bool = False,

278 skip_tokenizer_init: bool = False,

280 - auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,

279 + auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,

280 postprocess_inputs: Callable[..., BatchEncoding] = identity,

281 ) -> None:

282 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]

@@ -334,11 +333,11 @@ class HfRunner:

333

334 def get_inputs(

335 self,

337 - prompts: List[str],

336 + prompts: list[str],

337 images: Optional[PromptImageInput] = None,

338 videos: Optional[PromptVideoInput] = None,

339 audios: Optional[PromptAudioInput] = None,

341 - ) -> List[BatchEncoding]:

340 + ) -> list[BatchEncoding]:

341 if images is not None:

342 assert len(prompts) == len(images)

343

@@ -348,9 +347,9 @@ class HfRunner:

347 if audios is not None:

348 assert len(prompts) == len(audios)

349

351 - all_inputs: List[BatchEncoding] = []

350 + all_inputs: list[BatchEncoding] = []

351 for i, prompt in enumerate(prompts):

353 - processor_kwargs: Dict[str, Any] = {

352 + processor_kwargs: dict[str, Any] = {

353 "text": prompt,

354 "return_tensors": "pt",

355 }

@@ -370,7 +369,7 @@ class HfRunner:

369

370 return all_inputs

371

373 - def classify(self, prompts: List[str]) -> List[str]:

372 + def classify(self, prompts: list[str]) -> list[str]:

373 # output is final logits

374 all_inputs = self.get_inputs(prompts)

375 outputs = []

@@ -383,18 +382,18 @@ class HfRunner:

382

383 def generate(

384 self,

386 - prompts: List[str],

385 + prompts: list[str],

386 images: Optional[PromptImageInput] = None,

387 videos: Optional[PromptVideoInput] = None,

388 audios: Optional[PromptAudioInput] = None,

389 **kwargs: Any,

391 - ) -> List[Tuple[List[List[int]], List[str]]]:

390 + ) -> list[tuple[list[list[int]], list[str]]]:

391 all_inputs = self.get_inputs(prompts,

392 images=images,

393 videos=videos,

394 audios=audios)

395

397 - outputs: List[Tuple[List[List[int]], List[str]]] = []

396 + outputs: list[tuple[list[list[int]], list[str]]] = []

397 for inputs in all_inputs:

398 output_ids = self.model.generate(

399 **self.wrap_device(inputs, device=self.model.device.type),

@@ -412,13 +411,13 @@ class HfRunner:

411

412 def generate_greedy(

413 self,

415 - prompts: List[str],

414 + prompts: list[str],

415 max_tokens: int,

416 images: Optional[PromptImageInput] = None,

417 videos: Optional[PromptVideoInput] = None,

418 audios: Optional[PromptAudioInput] = None,

419 **kwargs: Any,

421 - ) -> List[Tuple[List[int], str]]:

420 + ) -> list[tuple[list[int], str]]:

421 outputs = self.generate(prompts,

422 do_sample=False,

423 max_new_tokens=max_tokens,

@@ -432,10 +431,10 @@ class HfRunner:

431

432 def generate_beam_search(

433 self,

435 - prompts: List[str],

434 + prompts: list[str],

435 beam_width: int,

436 max_tokens: int,

438 - ) -> List[Tuple[List[List[int]], List[str]]]:

437 + ) -> list[tuple[list[list[int]], list[str]]]:

438 outputs = self.generate(prompts,

439 do_sample=False,

440 max_new_tokens=max_tokens,

@@ -453,19 +452,19 @@ class HfRunner:

452

453 def generate_greedy_logprobs(

454 self,

456 - prompts: List[str],

455 + prompts: list[str],

456 max_tokens: int,

457 images: Optional[PromptImageInput] = None,

458 videos: Optional[PromptVideoInput] = None,

459 audios: Optional[PromptAudioInput] = None,

460 **kwargs: Any,

462 - ) -> List[List[torch.Tensor]]:

461 + ) -> list[list[torch.Tensor]]:

462 all_inputs = self.get_inputs(prompts,

463 images=images,

464 videos=videos,

465 audios=audios)

466

468 - all_logprobs: List[List[torch.Tensor]] = []

467 + all_logprobs: list[list[torch.Tensor]] = []

468 for inputs in all_inputs:

469 output = self.model.generate(

470 **self.wrap_device(inputs, device=self.model.device.type),

@@ -483,11 +482,11 @@ class HfRunner:

482

483 def _hidden_states_to_seq_logprobs(

484 self,

486 - hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],

487 - ) -> List[torch.Tensor]:

485 + hidden_states: tuple[tuple[torch.Tensor, ...], ...],

486 + ) -> list[torch.Tensor]:

487 output_embeddings = self.model.get_output_embeddings()

488

490 - seq_logprobs: List[torch.Tensor] = []

489 + seq_logprobs: list[torch.Tensor] = []

490 for _, hidden_state in enumerate(hidden_states):

491 last_hidden_states = hidden_state[-1][0]

492 logits = torch.matmul(

@@ -503,14 +502,14 @@ class HfRunner:

502

503 def _hidden_states_to_logprobs(

504 self,

506 - hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],

505 + hidden_states: tuple[tuple[torch.Tensor, ...], ...],

506 num_logprobs: int,

508 - ) -> Tuple[List[Dict[int, float]], int]:

507 + ) -> tuple[list[dict[int, float]], int]:

508 seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)

509 output_len = len(hidden_states)

510

511 # convert to dict

513 - seq_logprobs_lst: List[Dict[int, float]] = []

512 + seq_logprobs_lst: list[dict[int, float]] = []

513 for tok_idx, tok_logprobs in enumerate(seq_logprobs):

514 # drop prompt logprobs

515 if tok_idx == 0:

@@ -530,22 +529,22 @@ class HfRunner:

529

530 def generate_greedy_logprobs_limit(

531 self,

533 - prompts: List[str],

532 + prompts: list[str],

533 max_tokens: int,

534 num_logprobs: int,

535 images: Optional[PromptImageInput] = None,

536 audios: Optional[PromptAudioInput] = None,

537 videos: Optional[PromptVideoInput] = None,

538 **kwargs: Any,

540 - ) -> List[TokensTextLogprobs]:

539 + ) -> list[TokensTextLogprobs]:

540 all_inputs = self.get_inputs(prompts,

541 images=images,

542 videos=videos,

543 audios=audios)

544

546 - all_logprobs: List[List[Dict[int, float]]] = []

547 - all_output_ids: List[List[int]] = []

548 - all_output_strs: List[str] = []

545 + all_logprobs: list[list[dict[int, float]]] = []

546 + all_output_ids: list[list[int]] = []

547 + all_output_strs: list[str] = []

548

549 for inputs in all_inputs:

550 output = self.model.generate(

@@ -577,23 +576,23 @@ class HfRunner:

576

577 def generate_encoder_decoder_greedy_logprobs_limit(

578 self,

580 - encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],

579 + encoder_decoder_prompts: list[ExplicitEncoderDecoderPrompt[str, str]],

580 max_tokens: int,

581 num_logprobs: int,

582 images: Optional[PromptImageInput] = None,

583 **kwargs: Any,

585 - ) -> List[TokensTextLogprobs]:

584 + ) -> list[TokensTextLogprobs]:

585 '''

586 Greedy logprobs generation for vLLM encoder/decoder models

587 '''

588

590 - all_logprobs: List[List[Dict[int, float]]] = []

591 - all_output_ids: List[List[int]] = []

592 - all_output_strs: List[str] = []

589 + all_logprobs: list[list[dict[int, float]]] = []

590 + all_output_ids: list[list[int]] = []

591 + all_output_strs: list[str] = []

592

593 for i, (encoder_prompt, decoder_prompt) in enumerate(

594 to_enc_dec_tuple_list(encoder_decoder_prompts)):

596 - processor_kwargs: Dict[str, Any] = {

595 + processor_kwargs: dict[str, Any] = {

596 "text": encoder_prompt,

597 "return_tensors": "pt",

598 }

@@ -641,10 +640,10 @@ class HfRunner:

640 return [(output_ids, output_str, output_logprobs)

641 for output_ids, output_str, output_logprobs in outputs]

642

644 - def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:

643 + def encode(self, prompts: list[str]) -> list[list[torch.Tensor]]:

644 return self.model.encode(prompts)

645

647 - def predict(self, prompts: List[List[str]]) -> torch.Tensor:

646 + def predict(self, prompts: list[list[str]]) -> torch.Tensor:

647 return self.model.predict(prompts, convert_to_tensor=True)

648

649 def __enter__(self):

@@ -699,11 +698,11 @@ class VllmRunner:

698

699 def get_inputs(

700 self,

702 - prompts: List[str],

701 + prompts: list[str],

702 images: Optional[PromptImageInput] = None,

703 videos: Optional[PromptVideoInput] = None,

704 audios: Optional[PromptAudioInput] = None,

706 - ) -> List[TextPrompt]:

705 + ) -> list[TextPrompt]:

706 if images is not None:

707 assert len(prompts) == len(images)

708

@@ -733,13 +732,13 @@ class VllmRunner:

732

733 def generate(

734 self,

736 - prompts: List[str],

735 + prompts: list[str],

736 sampling_params: SamplingParams,

737 images: Optional[PromptImageInput] = None,

738 videos: Optional[PromptVideoInput] = None,

739 audios: Optional[PromptAudioInput] = None,

740 **kwargs: Any,

742 - ) -> List[Tuple[List[List[int]], List[str]]]:

741 + ) -> list[tuple[list[list[int]], list[str]]]:

742 inputs = self.get_inputs(prompts,

743 images=images,

744 videos=videos,

@@ -749,12 +748,12 @@ class VllmRunner:

748 sampling_params=sampling_params,

749 **kwargs)

750

752 - outputs: List[Tuple[List[List[int]], List[str]]] = []

751 + outputs: list[tuple[list[list[int]], list[str]]] = []

752 for req_output in req_outputs:

753 prompt_str = req_output.prompt

754 prompt_ids = req_output.prompt_token_ids

756 - req_sample_output_ids: List[List[int]] = []

757 - req_sample_output_strs: List[str] = []

755 + req_sample_output_ids: list[list[int]] = []

756 + req_sample_output_strs: list[str] = []

757 for sample in req_output.outputs:

758 output_str = sample.text

759 output_ids = list(sample.token_ids)

@@ -765,9 +764,9 @@ class VllmRunner:

764

765 @staticmethod

766 def _final_steps_generate_w_logprobs(

768 - req_outputs: List[RequestOutput],

769 - ) -> List[TokensTextLogprobsPromptLogprobs]:

770 - outputs: List[TokensTextLogprobsPromptLogprobs] = []

767 + req_outputs: list[RequestOutput],

768 + ) -> list[TokensTextLogprobsPromptLogprobs]:

769 + outputs: list[TokensTextLogprobsPromptLogprobs] = []

770 for req_output in req_outputs:

771 assert len(req_output.outputs) > 0

772 for sample in req_output.outputs:

@@ -780,14 +779,14 @@ class VllmRunner:

779

780 def generate_w_logprobs(

781 self,

783 - prompts: List[str],

782 + prompts: list[str],

783 sampling_params: SamplingParams,

784 images: Optional[PromptImageInput] = None,

785 audios: Optional[PromptAudioInput] = None,

786 videos: Optional[PromptVideoInput] = None,

787 **kwargs: Any,

789 - ) -> Union[List[TokensTextLogprobs],

790 - List[TokensTextLogprobsPromptLogprobs]]:

788 + ) -> Union[list[TokensTextLogprobs],

789 + list[TokensTextLogprobsPromptLogprobs]]:

790 inputs = self.get_inputs(prompts,

791 images=images,

792 videos=videos,

@@ -806,10 +805,10 @@ class VllmRunner:

805

806 def generate_encoder_decoder_w_logprobs(

807 self,

809 - encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],

808 + encoder_decoder_prompts: list[ExplicitEncoderDecoderPrompt[str, str]],

809 sampling_params: SamplingParams,

811 - ) -> Union[List[TokensTextLogprobs],

812 - List[TokensTextLogprobsPromptLogprobs]]:

810 + ) -> Union[list[TokensTextLogprobs],

811 + list[TokensTextLogprobsPromptLogprobs]]:

812 '''

813 Logprobs generation for vLLM encoder/decoder models

814 '''

@@ -826,13 +825,13 @@ class VllmRunner:

825

826 def generate_greedy(

827 self,

829 - prompts: List[str],

828 + prompts: list[str],

829 max_tokens: int,

830 images: Optional[PromptImageInput] = None,

831 videos: Optional[PromptVideoInput] = None,

832 audios: Optional[PromptAudioInput] = None,

833 **kwargs: Any,

835 - ) -> List[Tuple[List[int], str]]:

834 + ) -> list[tuple[list[int], str]]:

835 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)

836 outputs = self.generate(prompts,

837 greedy_params,

@@ -845,18 +844,18 @@ class VllmRunner:

844

845 def generate_greedy_logprobs(

846 self,

848 - prompts: List[str],

847 + prompts: list[str],

848 max_tokens: int,

849 num_logprobs: int,

850 num_prompt_logprobs: Optional[int] = None,

851 images: Optional[PromptImageInput] = None,

852 audios: Optional[PromptAudioInput] = None,

853 videos: Optional[PromptVideoInput] = None,

855 - stop_token_ids: Optional[List[int]] = None,

856 - stop: Optional[List[str]] = None,

854 + stop_token_ids: Optional[list[int]] = None,

855 + stop: Optional[list[str]] = None,

856 **kwargs: Any,

858 - ) -> Union[List[TokensTextLogprobs],

859 - List[TokensTextLogprobsPromptLogprobs]]:

857 + ) -> Union[list[TokensTextLogprobs],

858 + list[TokensTextLogprobsPromptLogprobs]]:

859 greedy_logprobs_params = SamplingParams(

860 temperature=0.0,

861 max_tokens=max_tokens,

@@ -874,12 +873,12 @@ class VllmRunner:

873

874 def generate_encoder_decoder_greedy_logprobs(

875 self,

877 - encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],

876 + encoder_decoder_prompts: list[ExplicitEncoderDecoderPrompt[str, str]],

877 max_tokens: int,

878 num_logprobs: int,

879 num_prompt_logprobs: Optional[int] = None,

881 - ) -> Union[List[TokensTextLogprobs],

882 - List[TokensTextLogprobsPromptLogprobs]]:

880 + ) -> Union[list[TokensTextLogprobs],

881 + list[TokensTextLogprobsPromptLogprobs]]:

882 greedy_logprobs_params = SamplingParams(

883 temperature=0.0,

884 max_tokens=max_tokens,

@@ -895,10 +894,10 @@ class VllmRunner:

894

895 def generate_beam_search(

896 self,

898 - prompts: Union[List[str], List[List[int]]],

897 + prompts: Union[list[str], list[list[int]]],

898 beam_width: int,

899 max_tokens: int,

901 - ) -> List[Tuple[List[List[int]], List[str]]]:

900 + ) -> list[tuple[list[list[int]], list[str]]]:

901 if is_list_of(prompts, str, check="all"):

902 prompts = [TextPrompt(prompt=prompt) for prompt in prompts]

903 else:

@@ -915,17 +914,17 @@ class VllmRunner:

914 returned_outputs.append((token_ids, texts))

915 return returned_outputs

916

918 - def classify(self, prompts: List[str]) -> List[List[float]]:

917 + def classify(self, prompts: list[str]) -> list[list[float]]:

918 req_outputs = self.model.classify(prompts)

919 return [req_output.outputs.probs for req_output in req_outputs]

920

921 def encode(

922 self,

924 - prompts: List[str],

923 + prompts: list[str],

924 images: Optional[PromptImageInput] = None,

925 videos: Optional[PromptVideoInput] = None,

926 audios: Optional[PromptAudioInput] = None,

928 - ) -> List[List[float]]:

927 + ) -> list[list[float]]:

928 inputs = self.get_inputs(prompts,

929 images=images,

930 videos=videos,

@@ -936,9 +935,9 @@ class VllmRunner:

935

936 def score(

937 self,

939 - text_1: Union[str, List[str]],

940 - text_2: Union[str, List[str]],

941 - ) -> List[float]:

938 + text_1: Union[str, list[str]],

939 + text_2: Union[str, list[str]],

940 + ) -> list[float]:

941 req_outputs = self.model.score(text_1, text_2)

942 return [req_output.outputs.score for req_output in req_outputs]

943

tests/core/block/e2e/conftest.py

+2 -1

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Callable, Iterable, Optional

3 +from collections.abc import Iterable

4 +from typing import Callable, Optional

5

6 import pytest

7

tests/core/block/e2e/test_correctness_sliding_window.py

+5 -6

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import List

4

5 import pytest

6

@@ -137,9 +136,9 @@ def prep_prompts(batch_size: int):

136 The prompt is just under 10k tokens; sliding window is 4k

137 so the answer is outside sliding window, but should still be correct.

138 """

140 - prompts: List[str] = []

141 - answer: List[int] = []

142 - indices: List[int] = []

139 + prompts: list[str] = []

140 + answer: list[int] = []

141 + indices: list[int] = []

142 random.seed(1)

143 for _ in range(batch_size):

144 idx = random.randint(30, 90)

@@ -158,7 +157,7 @@ def prep_prompts(batch_size: int):

157 return prompts, answer, indices

158

159

161 -def check_answers(indices: List[int], answer: List[int], outputs: List[str]):

160 +def check_answers(indices: list[int], answer: list[int], outputs: list[str]):

161 answer2 = [int(text[0:2].strip()) for text in outputs]

162 print(list(zip(indices, zip(answer, answer2))))

163 numok = 0

@@ -170,7 +169,7 @@ def check_answers(indices: List[int], answer: List[int], outputs: List[str]):

169 assert frac_ok > 0.7

170

171

173 -def check_window(prompts: List[str]):

172 +def check_window(prompts: list[str]):

173

174 def inner(llm: LLM):

175 sliding_window = llm.llm_engine.model_config.get_sliding_window()

tests/core/block/test_block_table.py

+3 -5

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm.core.block.block_table import BlockTable

@@ -32,7 +30,7 @@ def test_allocate_naive(block_size: int, sequence_len: int):

30 token_ids = list(range(sequence_len))

31 num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))

32

35 - block_tables: List[BlockTable] = []

33 + block_tables: list[BlockTable] = []

34 for i in range(5):

35 assert allocator.get_num_free_blocks(

36 device=Device.GPU) == num_gpu_blocks - i * num_blocks_per_alloc

@@ -77,7 +75,7 @@ def test_allocate_prefix_caching(block_size: int, sequence_len: int):

75 num_immutable_blocks_per_alloc = len(

76 chunked_tokens) - num_mutable_blocks_per_alloc

77

80 - block_tables: List[BlockTable] = []

78 + block_tables: list[BlockTable] = []

79 for alloc_i in range(1, 6):

80

81 block_tables.append(

@@ -272,7 +270,7 @@ def test_append_token_ids_correct_content(block_size: int, sequence_len: int,

270 )

271 block_table.allocate(token_ids=token_ids, device=Device.GPU)

272

275 - appended_so_far: List[int] = []

273 + appended_so_far: list[int] = []

274 for append in chunk_list(token_ids_to_append, append_size):

275 block_table.append_token_ids(append)

276 appended_so_far.extend(append)

tests/core/block/test_naive_block.py

+2 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional

3 +from typing import Optional

4

5 import pytest

6

@@ -14,7 +14,7 @@ class TestNaiveBlockAllocator:

14 def create_allocate_lambda(allocate_type: str,

15 allocator: NaiveBlockAllocator,

16 prev_block: Optional[Block],

17 - token_ids: List[int]):

17 + token_ids: list[int]):

18 if allocate_type == "immutable":

19 allocate_block = lambda: allocator.allocate_immutable_block(

20 prev_block=prev_block, token_ids=token_ids)

tests/core/block/test_prefix_caching_block.py

+8 -8

@@ -2,7 +2,7 @@

2

3 import math

4 import random

5 -from typing import List, Optional

5 +from typing import Optional

6 from unittest.mock import MagicMock

7

8 import pytest

@@ -123,11 +123,11 @@ class TestPrefixCachingBlock:

123

124 @staticmethod

125 def create_chain(block_size: int,

126 - token_ids: List[int],

127 - num_empty_trailing_blocks=0) -> List[PrefixCachingBlock]:

126 + token_ids: list[int],

127 + num_empty_trailing_blocks=0) -> list[PrefixCachingBlock]:

128 """Helper method which creates a chain of blocks.

129 """

130 - blocks: List[PrefixCachingBlock] = []

130 + blocks: list[PrefixCachingBlock] = []

131 num_blocks = math.ceil(

132 len(token_ids) / block_size) + num_empty_trailing_blocks

133

@@ -161,7 +161,7 @@ class TestPrefixCachingBlockAllocator:

161 @staticmethod

162 def create_allocate_lambda(allocate_type: str, allocator: BlockAllocator,

163 prev_block: Optional[Block],

164 - token_ids: List[int]):

164 + token_ids: list[int]):

165 if allocate_type == "immutable":

166 allocate_block = lambda: allocator.allocate_immutable_block(

167 prev_block=prev_block, token_ids=token_ids)

@@ -839,13 +839,13 @@ class TestPrefixCachingBlockAllocator:

839 @staticmethod

840 def create_immutable_chain(

841 block_size: int,

842 - token_ids: List[int],

842 + token_ids: list[int],

843 allocator: PrefixCachingBlockAllocator,

844 extra_hash: Optional[int] = None,

845 - ) -> List[PrefixCachingBlock]:

845 + ) -> list[PrefixCachingBlock]:

846 """Helper method which creates a chain of blocks.

847 """

848 - blocks: List[Block] = []

848 + blocks: list[Block] = []

849 num_blocks = math.ceil(len(token_ids) / block_size)

850

851 if num_blocks == 0:

tests/core/test_chunked_prefill_scheduler.py

+12 -13

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

3 from unittest.mock import MagicMock

4

5 import pytest # noqa

@@ -46,7 +45,7 @@ def test_simple():

45 cache_config.num_cpu_blocks = 8

46 cache_config.num_gpu_blocks = 8

47 scheduler = Scheduler(scheduler_config, cache_config, None)

49 - running: List[SequenceGroup] = []

48 + running: list[SequenceGroup] = []

49

50 # Add seq groups to scheduler.

51 for i in range(num_seq_group):

@@ -93,7 +92,7 @@ def test_chunk():

92 cache_config.num_cpu_blocks = 32

93 cache_config.num_gpu_blocks = 32

94 scheduler = Scheduler(scheduler_config, cache_config, None)

96 - running: List[SequenceGroup] = []

95 + running: list[SequenceGroup] = []

96

97 # Add seq groups to scheduler.

98 for i in range(2):

@@ -145,7 +144,7 @@ def test_concurrent_chunking():

144 cache_config.num_cpu_blocks = 32

145 cache_config.num_gpu_blocks = 32

146 scheduler = Scheduler(scheduler_config, cache_config, None)

148 - running: List[SequenceGroup] = []

147 + running: list[SequenceGroup] = []

148

149 # Add seq groups to scheduler.

150 for i in range(2):

@@ -226,8 +225,8 @@ def test_short_prompts_jump_long_prompts_in_queue():

225 cache_config.num_cpu_blocks = 3200 # large KV cache size for large requests

226 cache_config.num_gpu_blocks = 3200

227 scheduler = Scheduler(scheduler_config, cache_config, None)

229 - long_seqs: List[SequenceGroup] = []

230 - short_seqs: List[SequenceGroup] = []

228 + long_seqs: list[SequenceGroup] = []

229 + short_seqs: list[SequenceGroup] = []

230

231 # Add 2 large seq groups to scheduler.

232 for i in range(2):

@@ -368,7 +367,7 @@ def test_complex():

367 cache_config.num_cpu_blocks = 64

368 cache_config.num_gpu_blocks = 64

369 scheduler = Scheduler(scheduler_config, cache_config, None)

371 - running: List[SequenceGroup] = []

370 + running: list[SequenceGroup] = []

371

372 # Add seq groups to scheduler.

373 for i in range(2):

@@ -439,7 +438,7 @@ def test_maximal_decoding():

438 cache_config.num_cpu_blocks = 8

439 cache_config.num_gpu_blocks = 8

440 scheduler = Scheduler(scheduler_config, cache_config, None)

442 - running: List[SequenceGroup] = []

441 + running: list[SequenceGroup] = []

442

443 # Add seq groups to scheduler.

444 for i in range(2):

@@ -533,7 +532,7 @@ def test_prompt_limit():

532 cache_config.num_cpu_blocks = 16

533 cache_config.num_gpu_blocks = 16

534 scheduler = Scheduler(scheduler_config, cache_config, None)

536 - running: List[SequenceGroup] = []

535 + running: list[SequenceGroup] = []

536

537 _, seq_group = create_dummy_prompt("1",

538 prompt_length=48,

@@ -565,7 +564,7 @@ def test_prompt_limit_exceed():

564 cache_config.num_cpu_blocks = 16

565 cache_config.num_gpu_blocks = 16

566 scheduler = Scheduler(scheduler_config, cache_config, None)

568 - running: List[SequenceGroup] = []

567 + running: list[SequenceGroup] = []

568 _, seq_group = create_dummy_prompt("2",

569 prompt_length=48,

570 block_size=block_size)

@@ -699,7 +698,7 @@ def test_chunked_prefill_max_seqs():

698 cache_config.num_cpu_blocks = 128

699 cache_config.num_gpu_blocks = 128

700 scheduler = Scheduler(scheduler_config, cache_config, None)

702 - running: List[SequenceGroup] = []

701 + running: list[SequenceGroup] = []

702

703 _, seq_group = create_dummy_prompt("1",

704 prompt_length=65,

@@ -758,7 +757,7 @@ def test_prefix_caching():

757 cache_config.num_cpu_blocks = 0

758 cache_config.num_gpu_blocks = 32

759 scheduler = Scheduler(scheduler_config, cache_config, None)

761 - running: List[SequenceGroup] = []

760 + running: list[SequenceGroup] = []

761

762 # Add seq groups to scheduler.

763 for i in range(2):

@@ -800,7 +799,7 @@ def test_prefix_caching_with_concurrent_partial_prefills():

799 cache_config.num_cpu_blocks = 0

800 cache_config.num_gpu_blocks = 32

801 scheduler = Scheduler(scheduler_config, cache_config, None)

803 - running: List[SequenceGroup] = []

802 + running: list[SequenceGroup] = []

803

804 # Add seq groups to scheduler.

805 for i in range(2):

tests/core/test_scheduler.py

+9 -10

@@ -2,7 +2,6 @@

2

3 import time

4 from collections import deque

5 -from typing import List, Set, Tuple

5 from unittest.mock import MagicMock

6

7 import pytest # noqa

@@ -57,7 +56,7 @@ def test_scheduler_abort_seq_group():

56

57 # Add multiple seq groups to scheduler.

58 num_seq_group = 4

60 - request_ids: Set[str] = set()

59 + request_ids: set[str] = set()

60 for i in range(num_seq_group):

61 _, seq_group = create_dummy_prompt(str(i), block_size)

62 scheduler.add_seq_group(seq_group)

@@ -83,7 +82,7 @@ def test_scheduler_schedule_simple():

82 cache_config.num_cpu_blocks = 8

83 cache_config.num_gpu_blocks = 8

84 scheduler = Scheduler(scheduler_config, cache_config, None)

86 - running: List[SequenceGroup] = []

85 + running: list[SequenceGroup] = []

86

87 # Add seq groups to scheduler.

88 for i in range(num_seq_group):

@@ -221,7 +220,7 @@ def test_scheduler_max_seqs():

220 cache_config.num_gpu_blocks = 8

221 scheduler = Scheduler(scheduler_config, cache_config, None)

222

224 - all_seq_groups: List[SequenceGroup] = []

223 + all_seq_groups: list[SequenceGroup] = []

224 # Add seq groups to scheduler.

225 for i in range(num_seq_group):

226 _, seq_group = create_dummy_prompt(str(i),

@@ -480,7 +479,7 @@ def test_prefill_schedule_max_lora():

479 num_cpu_blocks=64,

480 num_gpu_blocks=64)

481 budget = create_token_budget(token_budget=120)

483 - curr_loras: Set[int] = set()

482 + curr_loras: set[int] = set()

483 for i in range(2):

484 _, seq_group = create_dummy_prompt(str(i),

485 prompt_length=60,

@@ -651,8 +650,8 @@ def test_schedule_swapped_max_loras():

650 block_size=block_size,

651 num_cpu_blocks=32,

652 num_gpu_blocks=32)

654 - curr_loras: Set[int] = set()

655 - blocks_to_swap_out: List[Tuple[int, int]] = []

653 + curr_loras: set[int] = set()

654 + blocks_to_swap_out: list[tuple[int, int]] = []

655 for i in range(2):

656 _, seq_group = create_dummy_prompt(str(i),

657 prompt_length=60,

@@ -683,7 +682,7 @@ def test_schedule_swapped_cannot_swap_in():

682 num_cpu_blocks=32,

683 num_gpu_blocks=32)

684 curr_loras = None

686 - blocks_to_swap_out: List[Tuple[int, int]] = []

685 + blocks_to_swap_out: list[tuple[int, int]] = []

686 for i in range(2):

687 _, seq_group = create_dummy_prompt(str(i),

688 prompt_length=60,

@@ -714,7 +713,7 @@ def test_infeasible_swap():

713 num_cpu_blocks=32,

714 num_gpu_blocks=32)

715 curr_loras = None

717 - blocks_to_swap_out: List[Tuple[int, int]] = []

716 + blocks_to_swap_out: list[tuple[int, int]] = []

717 for i in range(2):

718 _, seq_group = create_dummy_prompt(str(i),

719 prompt_length=60,

@@ -752,7 +751,7 @@ def test_schedule_swapped_blocks_to_copy():

751 block_size=block_size)

752 scheduler._allocate_and_set_running(seq_group)

753 append_new_token_seq_group(60, seq_group, 1)

755 - blocks_to_swap_out: List[Tuple[int, int]] = []

754 + blocks_to_swap_out: list[tuple[int, int]] = []

755 scheduler._swap_out(seq_group, blocks_to_swap_out)

756 scheduler._add_seq_group_to_swapped(seq_group)

757

tests/core/test_scheduler_encoder_decoder.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest # noqa

4

5 from vllm.config import CacheConfig, SchedulerConfig

@@ -48,7 +46,7 @@ def test_scheduler_schedule_simple_encoder_decoder():

46 cache_config.num_cpu_blocks = 16 # enc and dec prompts per seq_group

47 cache_config.num_gpu_blocks = 16 # enc and dec prompts per seq_group

48 scheduler = Scheduler(scheduler_config, cache_config, None)

51 - running: List[SequenceGroup] = []

49 + running: list[SequenceGroup] = []

50

51 # Add seq groups to scheduler.

52 req_id_list = []

tests/core/utils.py

+10 -11

@@ -2,9 +2,8 @@

2

3 import time

4 from collections import defaultdict

5 -from typing import Any, Dict, List, Optional

6 -from typing import Sequence as GenericSequence

7 -from typing import Tuple

5 +from collections.abc import Sequence as GenericSequence

6 +from typing import Any, Optional

7

8 from vllm import SamplingParams

9 from vllm.core.scheduler import Scheduler, SchedulerOutputs

@@ -20,10 +19,10 @@ def create_dummy_prompt(

19 block_size: Optional[int] = None,

20 lora_request: Optional[LoRARequest] = None,

21 best_of: int = 1,

23 - prompt_tokens: Optional[List[int]] = None,

22 + prompt_tokens: Optional[list[int]] = None,

23 min_tokens: int = 0,

24 max_tokens: int = 16,

26 -) -> Tuple[Sequence, SequenceGroup]:

25 +) -> tuple[Sequence, SequenceGroup]:

26 if not block_size:

27 block_size = prompt_length

28

@@ -48,7 +47,7 @@ def create_dummy_prompt(

47 return prompt, seq_group

48

49

51 -def create_dummy_lora_sequence(request_id: int, token_ids: List[int],

50 +def create_dummy_lora_sequence(request_id: int, token_ids: list[int],

51 block_size: int, lora_int_id: int) -> Sequence:

52 return Sequence(seq_id=request_id,

53 inputs=token_inputs(token_ids),

@@ -58,7 +57,7 @@ def create_dummy_lora_sequence(request_id: int, token_ids: List[int],

57 lora_int_id=lora_int_id))

58

59

61 -def create_dummy_sequence(request_id: int, token_ids: List[int],

60 +def create_dummy_sequence(request_id: int, token_ids: list[int],

61 block_size: int) -> Sequence:

62 return Sequence(

63 seq_id=request_id,

@@ -74,7 +73,7 @@ def create_dummy_prompt_encoder_decoder(

73 block_size: Optional[int] = None,

74 lora_request: Optional[LoRARequest] = None,

75 best_of: int = 1,

77 -) -> Tuple[Sequence, Sequence, SequenceGroup]:

76 +) -> tuple[Sequence, Sequence, SequenceGroup]:

77 if not block_size:

78 block_size = decoder_prompt_length

79

@@ -125,7 +124,7 @@ def create_seq_group(

124

125 prompt_token_ids = [0] * seq_prompt_len

126

128 - seqs: List[Sequence] = []

127 + seqs: list[Sequence] = []

128 for seq_id_offset, output_len in enumerate(seq_output_lens):

129 seq = Sequence(

130 seq_id=seq_id_start + seq_id_offset,

@@ -241,7 +240,7 @@ class SchedulerProxy:

240

241 def __init__(self, scheduler: Scheduler):

242 self.scheduler_ = scheduler

244 - self.call_history: Dict[str, List[Any]] = defaultdict(list)

243 + self.call_history: dict[str, list[Any]] = defaultdict(list)

244

245 def __getattr__(self, name: str) -> Any:

246

@@ -253,6 +252,6 @@ class SchedulerProxy:

252 return wrapper

253

254 def last_schedule_ret(

256 - self, ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, Any]:

255 + self, ) -> tuple[list[SequenceGroupMetadata], SchedulerOutputs, Any]:

256 _, _, ret = self.call_history["schedule"][-1]

257 return ret

tests/distributed/test_expert_parallel.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import List, Literal, NamedTuple, Optional

4 +from typing import Literal, NamedTuple, Optional

5

6 import pytest

7

@@ -28,8 +28,8 @@ class EPTestOptions(NamedTuple):

28

29 @dataclass

30 class EPTestSettings:

31 - parallel_setups: List[ParallelSetup]

32 - distributed_backends: List[str]

31 + parallel_setups: list[ParallelSetup]

32 + distributed_backends: list[str]

33 task: TaskOption

34 test_options: EPTestOptions

35

tests/distributed/test_pipeline_parallel.py

+4 -4

@@ -9,7 +9,7 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node

9 import json

10 import os

11 from dataclasses import dataclass

12 -from typing import List, Literal, NamedTuple, Optional

12 +from typing import Literal, NamedTuple, Optional

13

14 import pytest

15

@@ -38,14 +38,14 @@ class PPTestOptions(NamedTuple):

38

39 @dataclass

40 class PPTestSettings:

41 - parallel_setups: List[ParallelSetup]

41 + parallel_setups: list[ParallelSetup]

42 # NOTE: the length of distributed_backends and

43 # vllm_major_versions should be the same, and they

44 # are first zipped together to iterate over all

45 # test settings.

46 - distributed_backends: List[str]

46 + distributed_backends: list[str]

47 # vllm major version: "0" for V0, "1" for V1

48 - vllm_major_versions: List[str]

48 + vllm_major_versions: list[str]

49 task: TaskOption

50 test_options: PPTestOptions

51

tests/distributed/test_pynccl.py

+2 -3

@@ -2,7 +2,6 @@

2

3 import multiprocessing

4 import os

5 -from typing import Dict, List

5

6 import pytest

7 import torch

@@ -20,9 +19,9 @@ from vllm.utils import update_environment_variables

19

20 def distributed_run(fn, world_size):

21 number_of_processes = world_size

23 - processes: List[multiprocessing.Process] = []

22 + processes: list[multiprocessing.Process] = []

23 for i in range(number_of_processes):

25 - env: Dict[str, str] = {}

24 + env: dict[str, str] = {}

25 env['RANK'] = str(i)

26 env['LOCAL_RANK'] = str(i)

27 env['WORLD_SIZE'] = str(number_of_processes)

tests/distributed/test_shm_broadcast.py

+1 -2

@@ -3,7 +3,6 @@

3 import multiprocessing

4 import random

5 import time

6 -from typing import List

6

7 import numpy as np

8 import torch.distributed as dist

@@ -13,7 +12,7 @@ from vllm.distributed.utils import StatelessProcessGroup

12 from vllm.utils import get_ip, get_open_port, update_environment_variables

13

14

16 -def get_arrays(n: int, seed: int = 0) -> List[np.ndarray]:

15 +def get_arrays(n: int, seed: int = 0) -> list[np.ndarray]:

16 np.random.seed(seed)

17 sizes = np.random.randint(1, 10_000, n)

18 # on average, each array will have 5k elements

tests/encoder_decoder/test_e2e_correctness.py

+2 -2

@@ -3,7 +3,7 @@

3

4 Run `pytest tests/encoder_decoder/test_e2e_correctness.py`.

5 """

6 -from typing import List, Optional, Tuple

6 +from typing import Optional

7

8 import pytest

9 from transformers import AutoModelForSeq2SeqLM

@@ -22,7 +22,7 @@ LIST_ENC_DEC_SUPPORTED_BACKENDS = [

22

23

24 def vllm_to_hf_output(

25 - vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],

25 + vllm_output: tuple[list[int], str, Optional[SampleLogprobs]],

26 decoder_prompt_type: DecoderPromptType,

27 ):

28 """Sanitize vllm output to be comparable with hf output."""

tests/engine/test_executor.py

+3 -3

@@ -2,7 +2,7 @@

2

3 import asyncio

4 import os

5 -from typing import Any, Callable, Dict, List, Optional, Tuple, Union

5 +from typing import Any, Callable, Optional, Union

6

7 import pytest

8

@@ -22,8 +22,8 @@ class CustomUniExecutor(UniProcExecutor):

22 def collective_rpc(self,

23 method: Union[str, Callable],

24 timeout: Optional[float] = None,

25 - args: Tuple = (),

26 - kwargs: Optional[Dict] = None) -> List[Any]:

25 + args: tuple = (),

26 + kwargs: Optional[dict] = None) -> list[Any]:

27 # Drop marker to show that this was ran

28 with open(".marker", "w"):

29 ...

tests/engine/test_multiproc_workers.py

+3 -3

@@ -4,7 +4,7 @@ import asyncio

4 from concurrent.futures import ThreadPoolExecutor

5 from functools import partial

6 from time import sleep

7 -from typing import Any, List, Tuple

7 +from typing import Any

8

9 import pytest

10

@@ -17,7 +17,7 @@ from vllm.worker.worker_base import WorkerWrapperBase

17 class DummyWorkerWrapper(WorkerWrapperBase):

18 """Dummy version of vllm.worker.worker.Worker"""

19

20 - def worker_method(self, worker_input: Any) -> Tuple[int, Any]:

20 + def worker_method(self, worker_input: Any) -> tuple[int, Any]:

21 sleep(0.05)

22

23 if isinstance(worker_input, Exception):

@@ -27,7 +27,7 @@ class DummyWorkerWrapper(WorkerWrapperBase):

27 return self.rpc_rank, input

28

29

30 -def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:

30 +def _start_workers() -> tuple[list[ProcessWorkerWrapper], WorkerMonitor]:

31 result_handler = ResultHandler()

32 vllm_config = VllmConfig()

33 workers = [

tests/engine/test_stop_strings.py

+3 -3

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Any, List, Optional

3 +from typing import Any, Optional

4

5 import pytest

6

@@ -21,8 +21,8 @@ def vllm_model(vllm_runner):

21 def _test_stopping(llm_engine: LLMEngine,

22 expected_output: str,

23 expected_reason: Any,

24 - stop: Optional[List[str]] = None,

25 - stop_token_ids: Optional[List[int]] = None,

24 + stop: Optional[list[str]] = None,

25 + stop_token_ids: Optional[list[int]] = None,

26 include_in_output: bool = False,

27 use_async_output_proc: bool = False) -> None:

28 llm_engine.add_request(

tests/entrypoints/llm/test_chat.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm import LLM

@@ -63,7 +61,7 @@ def test_multi_chat():

61

62 @pytest.mark.parametrize("image_urls",

63 [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])

66 -def test_chat_multi_image(image_urls: List[str]):

64 +def test_chat_multi_image(image_urls: list[str]):

65 llm = LLM(

66 model="microsoft/Phi-3.5-vision-instruct",

67 dtype="bfloat16",

tests/entrypoints/llm/test_encode.py

+2 -3

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import weakref

4 -from typing import List

4

5 import pytest

6

@@ -45,8 +44,8 @@ def llm():

44 cleanup_dist_env_and_memory()

45

46

48 -def assert_outputs_equal(o1: List[PoolingRequestOutput],

49 - o2: List[PoolingRequestOutput]):

47 +def assert_outputs_equal(o1: list[PoolingRequestOutput],

48 + o2: list[PoolingRequestOutput]):

49 assert [o.outputs for o in o1] == [o.outputs for o in o2]

50

51

tests/entrypoints/llm/test_generate.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import weakref

4 -from typing import List

4

5 import pytest

6

@@ -43,7 +42,7 @@ def llm():

42 cleanup_dist_env_and_memory()

43

44

46 -def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):

45 +def assert_outputs_equal(o1: list[RequestOutput], o2: list[RequestOutput]):

46 assert [o.outputs for o in o1] == [o.outputs for o in o2]

47

48

tests/entrypoints/openai/correctness/test_transcription_api_correctness.py

+1 -2

@@ -10,7 +10,6 @@ import asyncio

10 import io

11 import time

12 from statistics import mean, median

13 -from typing import List

13

14 import librosa

15 import pytest

@@ -67,7 +66,7 @@ async def process_dataset(model, client, data, concurrent_request):

66 audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]

67 _ = await bound_transcribe(model, sem, client, (audio, sr), "")

68

70 - tasks: List[asyncio.Task] = []

69 + tasks: list[asyncio.Task] = []

70 for sample in data:

71 audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]

72 task = asyncio.create_task(

tests/entrypoints/openai/reasoning_parsers/test_deepseekr1_reasoning_parser.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 from transformers import AutoTokenizer

5

@@ -180,7 +178,7 @@ def test_reasoning(

178 ):

179 output = tokenizer.tokenize(param_dict["output"])

180 # decode everything to tokens

183 - output_tokens: List[str] = [

181 + output_tokens: list[str] = [

182 tokenizer.convert_tokens_to_string([token]) for token in output

183 ]

184 parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(

tests/entrypoints/openai/reasoning_parsers/utils.py

+7 -7

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple, Union

3 +from typing import Optional, Union

4

5 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

6 DeltaMessage)

@@ -33,10 +33,10 @@ class StreamingReasoningReconstructor:

33

34 def run_reasoning_extraction(

35 reasoning_parser: ReasoningParser,

36 - model_output: List[str],

36 + model_output: list[str],

37 request: Union[ChatCompletionRequest, None] = None,

38 streaming: bool = False,

39 -) -> Tuple[Optional[str], Optional[str]]:

39 +) -> tuple[Optional[str], Optional[str]]:

40 if streaming:

41 reconstructor = run_reasoning_extraction_streaming(

42 reasoning_parser,

@@ -55,9 +55,9 @@ def run_reasoning_extraction(

55

56 def run_reasoning_extraction_nonstreaming(

57 reasoning_parser: ReasoningParser,

58 - model_output: List[str],

58 + model_output: list[str],

59 request: Union[ChatCompletionRequest, None] = None,

60 -) -> Tuple[Optional[str], Optional[str]]:

60 +) -> tuple[Optional[str], Optional[str]]:

61 request = request or ChatCompletionRequest(messages=[], model="test-model")

62 return reasoning_parser.extract_reasoning_content(

63 model_output=''.join(model_output), request=request)

@@ -65,13 +65,13 @@ def run_reasoning_extraction_nonstreaming(

65

66 def run_reasoning_extraction_streaming(

67 reasoning_parser: ReasoningParser,

68 - model_deltas: List[str],

68 + model_deltas: list[str],

69 request: Union[ChatCompletionRequest, None] = None,

70 ) -> StreamingReasoningReconstructor:

71 request = request or ChatCompletionRequest(messages=[], model="test-model")

72 reconstructor = StreamingReasoningReconstructor()

73 previous_text = ""

74 - previous_tokens: List[int] = []

74 + previous_tokens: list[int] = []

75 for delta in model_deltas:

76 token_delta = [

77 reasoning_parser.vocab.get(token)

tests/entrypoints/openai/test_audio.py

+7 -9

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List

4 -

3 import openai

4 import pytest

5 import pytest_asyncio

@@ -41,7 +39,7 @@ async def client(server):

39

40

41 @pytest.fixture(scope="session")

44 -def base64_encoded_audio() -> Dict[str, str]:

42 +def base64_encoded_audio() -> dict[str, str]:

43 return {

44 audio_url: encode_audio_base64(*fetch_audio(audio_url))

45 for audio_url in TEST_AUDIO_URLS

@@ -107,7 +105,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,

105 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

106 async def test_single_chat_session_audio_base64encoded(

107 client: openai.AsyncOpenAI, model_name: str, audio_url: str,

110 - base64_encoded_audio: Dict[str, str]):

108 + base64_encoded_audio: dict[str, str]):

109

110 messages = [{

111 "role":

@@ -165,7 +163,7 @@ async def test_single_chat_session_audio_base64encoded(

163 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

164 async def test_single_chat_session_input_audio(

165 client: openai.AsyncOpenAI, model_name: str, audio_url: str,

168 - base64_encoded_audio: Dict[str, str]):

166 + base64_encoded_audio: dict[str, str]):

167 messages = [{

168 "role":

169 "user",

@@ -255,7 +253,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,

253 temperature=0.0,

254 stream=True,

255 )

258 - chunks: List[str] = []

256 + chunks: list[str] = []

257 finish_reason_count = 0

258 async for chunk in stream:

259 delta = chunk.choices[0].delta

@@ -277,7 +275,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,

275 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

276 async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,

277 model_name: str, audio_url: str,

280 - base64_encoded_audio: Dict[str,

278 + base64_encoded_audio: dict[str,

279 str]):

280 messages = [{

281 "role":

@@ -315,7 +313,7 @@ async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,

313 temperature=0.0,

314 stream=True,

315 )

318 - chunks: List[str] = []

316 + chunks: list[str] = []

317 finish_reason_count = 0

318 async for chunk in stream:

319 delta = chunk.choices[0].delta

@@ -337,7 +335,7 @@ async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,

335 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

336 async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,

337 audio_url: str,

340 - base64_encoded_audio: Dict[str, str]):

338 + base64_encoded_audio: dict[str, str]):

339

340 messages = [{

341 "role":

tests/entrypoints/openai/test_basic.py

+1 -2

@@ -2,7 +2,6 @@

2

3 import asyncio

4 from http import HTTPStatus

5 -from typing import List

5

6 import openai

7 import pytest

@@ -17,7 +16,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"

16

17

18 @pytest.fixture(scope='module')

20 -def server_args(request: pytest.FixtureRequest) -> List[str]:

19 +def server_args(request: pytest.FixtureRequest) -> list[str]:

20 """ Provide extra arguments to the server via indirect parametrization

21

22 Usage:

tests/entrypoints/openai/test_chat.py

+4 -4

@@ -3,7 +3,7 @@

3 # imports for guided decoding tests

4 import json

5 import re

6 -from typing import Dict, List, Optional

6 +from typing import Optional

7

8 import jsonschema

9 import openai # use the official client for correctness check

@@ -190,7 +190,7 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,

190 async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,

191 model_name: str,

192 prompt_logprobs: Optional[int]):

193 - params: Dict = {

193 + params: dict = {

194 "messages": [{

195 "role": "system",

196 "content": "You are a helpful assistant."

@@ -232,7 +232,7 @@ async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,

232 )

233 async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,

234 model_name: str):

235 - params: Dict = {

235 + params: dict = {

236 "messages": [{

237 "role": "system",

238 "content": "You are a helpful assistant."

@@ -343,7 +343,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):

343 temperature=0.0,

344 stream=True,

345 )

346 - chunks: List[str] = []

346 + chunks: list[str] = []

347 finish_reason_count = 0

348 async for chunk in stream:

349 delta = chunk.choices[0].delta

tests/entrypoints/openai/test_completion.py

+4 -4

@@ -5,7 +5,7 @@ import json

5 import re

6 import shutil

7 from tempfile import TemporaryDirectory

8 -from typing import Dict, List, Optional

8 +from typing import Optional

9

10 import jsonschema

11 import openai # use the official client for correctness check

@@ -287,7 +287,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,

287 async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,

288 model_name: str,

289 prompt_logprobs: Optional[int]):

290 - params: Dict = {

290 + params: dict = {

291 "prompt": ["A robot may not injure another robot", "My name is"],

292 "model": model_name,

293 }

@@ -331,7 +331,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,

331 max_tokens=5,

332 temperature=0.0,

333 stream=True)

334 - chunks: List[str] = []

334 + chunks: list[str] = []

335 finish_reason_count = 0

336 async for chunk in stream:

337 chunks.append(chunk.choices[0].text)

@@ -364,7 +364,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):

364 max_tokens=max_tokens,

365 n=n,

366 stream=True)

367 - chunks: List[List[str]] = [[] for i in range(n)]

367 + chunks: list[list[str]] = [[] for i in range(n)]

368 finish_reason_count = 0

369 async for chunk in stream:

370 index = chunk.choices[0].index

tests/entrypoints/openai/test_embedding.py

+2 -2

@@ -86,7 +86,7 @@ async def test_single_embedding(client: openai.AsyncOpenAI, model_name: str):

86 @pytest.mark.asyncio

87 @pytest.mark.parametrize("model_name", [MODEL_NAME])

88 async def test_batch_embedding(client: openai.AsyncOpenAI, model_name: str):

89 - # test List[str]

89 + # test list[str]

90 input_texts = [

91 "The cat sat on the mat.", "A feline was resting on a rug.",

92 "Stars twinkle brightly in the night sky."

@@ -106,7 +106,7 @@ async def test_batch_embedding(client: openai.AsyncOpenAI, model_name: str):

106 assert embeddings.usage.prompt_tokens == 33

107 assert embeddings.usage.total_tokens == 33

108

109 - # test List[List[int]]

109 + # test list[list[int]]

110 input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],

111 [25, 32, 64, 77]]

112 embedding_response = await client.embeddings.create(

tests/entrypoints/openai/test_pooling.py

+2 -2

@@ -84,7 +84,7 @@ async def test_single_pooling(server: RemoteOpenAIServer, model_name: str):

84 @pytest.mark.asyncio

85 @pytest.mark.parametrize("model_name", [MODEL_NAME])

86 async def test_batch_pooling(server: RemoteOpenAIServer, model_name: str):

87 - # test List[str]

87 + # test list[str]

88 input_texts = [

89 "The cat sat on the mat.", "A feline was resting on a rug.",

90 "Stars twinkle brightly in the night sky."

@@ -107,7 +107,7 @@ async def test_batch_pooling(server: RemoteOpenAIServer, model_name: str):

107 assert poolings.usage.prompt_tokens == 25

108 assert poolings.usage.total_tokens == 25

109

110 - # test List[List[int]]

110 + # test list[list[int]]

111 input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],

112 [25, 32, 64, 77]]

113 response = requests.post(

tests/entrypoints/openai/test_root_path.py

+2 -2

@@ -2,7 +2,7 @@

2

3 import contextlib

4 import os

5 -from typing import Any, List, NamedTuple

5 +from typing import Any, NamedTuple

6

7 import openai # use the official client for correctness check

8 import pytest

@@ -40,7 +40,7 @@ def server():

40

41 class TestCase(NamedTuple):

42 model_name: str

43 - base_url: List[str]

43 + base_url: list[str]

44 api_key: str

45 expected_error: Any

46

tests/entrypoints/openai/test_video.py

+5 -7

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List

4 -

3 import openai

4 import pytest

5 import pytest_asyncio

@@ -49,7 +47,7 @@ async def client(server):

47

48

49 @pytest.fixture(scope="session")

52 -def base64_encoded_video() -> Dict[str, str]:

50 +def base64_encoded_video() -> dict[str, str]:

51 return {

52 video_url: encode_video_base64(fetch_video(video_url))

53 for video_url in TEST_VIDEO_URLS

@@ -151,7 +149,7 @@ async def test_single_chat_session_video_beamsearch(client: openai.AsyncOpenAI,

149 @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)

150 async def test_single_chat_session_video_base64encoded(

151 client: openai.AsyncOpenAI, model_name: str, video_url: str,

154 - base64_encoded_video: Dict[str, str]):

152 + base64_encoded_video: dict[str, str]):

153

154 messages = [{

155 "role":

@@ -209,7 +207,7 @@ async def test_single_chat_session_video_base64encoded(

207 @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)

208 async def test_single_chat_session_video_base64encoded_beamsearch(

209 client: openai.AsyncOpenAI, model_name: str, video_url: str,

212 - base64_encoded_video: Dict[str, str]):

210 + base64_encoded_video: dict[str, str]):

211

212 messages = [{

213 "role":

@@ -279,7 +277,7 @@ async def test_chat_streaming_video(client: openai.AsyncOpenAI,

277 temperature=0.0,

278 stream=True,

279 )

282 - chunks: List[str] = []

280 + chunks: list[str] = []

281 finish_reason_count = 0

282 async for chunk in stream:

283 delta = chunk.choices[0].delta

@@ -302,7 +300,7 @@ async def test_chat_streaming_video(client: openai.AsyncOpenAI,

300 "video_urls",

301 [TEST_VIDEO_URLS[:i] for i in range(2, len(TEST_VIDEO_URLS))])

302 async def test_multi_video_input(client: openai.AsyncOpenAI, model_name: str,

305 - video_urls: List[str]):

303 + video_urls: list[str]):

304

305 messages = [{

306 "role":

tests/entrypoints/openai/test_vision.py

+5 -7

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List

4 -

3 import openai

4 import pytest

5 import pytest_asyncio

@@ -50,7 +48,7 @@ async def client(server):

48

49

50 @pytest.fixture(scope="session")

53 -def base64_encoded_image() -> Dict[str, str]:

51 +def base64_encoded_image() -> dict[str, str]:

52 return {

53 image_url: encode_image_base64(fetch_image(image_url))

54 for image_url in TEST_IMAGE_URLS

@@ -152,7 +150,7 @@ async def test_single_chat_session_image_beamsearch(client: openai.AsyncOpenAI,

150 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)

151 async def test_single_chat_session_image_base64encoded(

152 client: openai.AsyncOpenAI, model_name: str, image_url: str,

155 - base64_encoded_image: Dict[str, str]):

153 + base64_encoded_image: dict[str, str]):

154

155 messages = [{

156 "role":

@@ -210,7 +208,7 @@ async def test_single_chat_session_image_base64encoded(

208 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)

209 async def test_single_chat_session_image_base64encoded_beamsearch(

210 client: openai.AsyncOpenAI, model_name: str, image_url: str,

213 - base64_encoded_image: Dict[str, str]):

211 + base64_encoded_image: dict[str, str]):

212

213 messages = [{

214 "role":

@@ -280,7 +278,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,

278 temperature=0.0,

279 stream=True,

280 )

283 - chunks: List[str] = []

281 + chunks: list[str] = []

282 finish_reason_count = 0

283 async for chunk in stream:

284 delta = chunk.choices[0].delta

@@ -303,7 +301,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,

301 "image_urls",

302 [TEST_IMAGE_URLS[:i] for i in range(2, len(TEST_IMAGE_URLS))])

303 async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,

306 - image_urls: List[str]):

304 + image_urls: list[str]):

305

306 messages = [{

307 "role":

tests/entrypoints/openai/test_vision_embedding.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict

4 -

3 import pytest

4 import requests

5

@@ -49,7 +47,7 @@ def server():

47

48

49 @pytest.fixture(scope="session")

52 -def base64_encoded_image() -> Dict[str, str]:

50 +def base64_encoded_image() -> dict[str, str]:

51 return {

52 image_url: encode_image_base64(fetch_image(image_url))

53 for image_url in TEST_IMAGE_URLS

tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py

+1 -2

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

3 from unittest.mock import MagicMock

4

5 import pytest

@@ -125,7 +124,7 @@ TEST_CASES = [

124 @pytest.mark.parametrize("streaming, model_output, expected_tool_calls",

125 TEST_CASES)

126 def test_tool_call(streaming: bool, model_output: str,

128 - expected_tool_calls: List[FunctionCall]):

127 + expected_tool_calls: list[FunctionCall]):

128 mock_tokenizer = MagicMock()

129 tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(

130 mock_tokenizer)

tests/entrypoints/openai/tool_parsers/utils.py

+5 -4

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Iterable, List, Tuple, Union

3 +from collections.abc import Iterable

4 +from typing import Union

5

6 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

7 DeltaMessage,

@@ -12,7 +13,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser

13 class StreamingToolReconstructor:

14

15 def __init__(self, assert_one_tool_per_delta: bool = True):

15 - self.tool_calls: List[ToolCall] = []

16 + self.tool_calls: list[ToolCall] = []

17 self.other_content: str = ""

18 self._assert_one_tool_per_delta = assert_one_tool_per_delta

19

@@ -72,7 +73,7 @@ def run_tool_extraction(

73 request: Union[ChatCompletionRequest, None] = None,

74 streaming: bool = False,

75 assert_one_tool_per_delta: bool = True,

75 -) -> Tuple[Union[str, None], List[ToolCall]]:

76 +) -> tuple[Union[str, None], list[ToolCall]]:

77 if streaming:

78 reconstructor = run_tool_extraction_streaming(

79 tool_parser,

@@ -106,7 +107,7 @@ def run_tool_extraction_streaming(

107 reconstructor = StreamingToolReconstructor(

108 assert_one_tool_per_delta=assert_one_tool_per_delta)

109 previous_text = ""

109 - previous_tokens: List[int] = []

110 + previous_tokens: list[int] = []

111 for delta in model_deltas:

112 token_delta = [

113 tool_parser.vocab.get(token)

tests/kernels/quant_utils.py

+3 -3

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Optional, Tuple, Union

3 +from typing import Optional, Union

4

5 import torch

6

@@ -19,7 +19,7 @@ def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:

19 def ref_dynamic_per_token_quant(x: torch.tensor,

20 quant_dtype: torch.dtype,

21 scale_ub: Optional[torch.tensor] = None) \

22 - -> Tuple[torch.tensor, torch.tensor]:

22 + -> tuple[torch.tensor, torch.tensor]:

23

24 assert quant_dtype in [torch.int8, FP8_DTYPE]

25 if scale_ub is not None:

@@ -68,7 +68,7 @@ def ref_dynamic_per_token_quant(x: torch.tensor,

68 # ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant

69 # kernel

70 def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \

71 - -> Tuple[torch.tensor, torch.tensor]:

71 + -> tuple[torch.tensor, torch.tensor]:

72

73 fp8_traits = torch.finfo(FP8_DTYPE)

74 fp8_traits_max = ROCM_FP8_MAX if current_platform.is_rocm() \

tests/kernels/test_activation.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import Type

4

5 import pytest

6 import torch

@@ -86,7 +85,7 @@ def test_act_and_mul(

85 @pytest.mark.parametrize("device", CUDA_DEVICES)

86 @torch.inference_mode()

87 def test_activation(

89 - activation: Type[torch.nn.Module],

88 + activation: type[torch.nn.Module],

89 num_tokens: int,

90 d: int,

91 dtype: torch.dtype,

tests/kernels/test_attention.py

+8 -8

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import List, Optional, Tuple

4 +from typing import Optional

5

6 import pytest

7 import torch

@@ -85,8 +85,8 @@ def ref_single_query_cached_kv_attention(

85 block_table = block_tables_lst[i]

86 seq_len = int(seq_lens_lst[i])

87

88 - keys_lst: List[torch.Tensor] = []

89 - values_lst: List[torch.Tensor] = []

88 + keys_lst: list[torch.Tensor] = []

89 + values_lst: list[torch.Tensor] = []

90 for j in range(seq_len):

91 block_number = int(block_table[j // block_size])

92 block_offset = j % block_size

@@ -133,7 +133,7 @@ def test_paged_attention(

133 kv_cache_factory,

134 version: str,

135 num_seqs: int,

136 - num_heads: Tuple[int, int],

136 + num_heads: tuple[int, int],

137 head_size: int,

138 use_alibi: bool,

139 block_size: int,

@@ -166,7 +166,7 @@ def test_paged_attention(

166

167 # Create the block tables.

168 max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size

169 - block_tables_lst: List[List[int]] = []

169 + block_tables_lst: list[list[int]] = []

170 for _ in range(num_seqs):

171 block_table = [

172 random.randint(0, NUM_BLOCKS - 1)

@@ -334,7 +334,7 @@ def test_paged_attention(

334

335

336 def ref_multi_query_kv_attention(

337 - cu_seq_lens: List[int],

337 + cu_seq_lens: list[int],

338 query: torch.Tensor,

339 key: torch.Tensor,

340 value: torch.Tensor,

@@ -342,7 +342,7 @@ def ref_multi_query_kv_attention(

342 dtype: torch.dtype,

343 ) -> torch.Tensor:

344 num_seqs = len(cu_seq_lens) - 1

345 - ref_outputs: List[torch.Tensor] = []

345 + ref_outputs: list[torch.Tensor] = []

346 for i in range(num_seqs):

347 start_idx = cu_seq_lens[i]

348 end_idx = cu_seq_lens[i + 1]

@@ -378,7 +378,7 @@ def ref_multi_query_kv_attention(

378 @torch.inference_mode()

379 def test_multi_query_kv_attention(

380 num_seqs: int,

381 - num_heads: Tuple[int, int],

381 + num_heads: tuple[int, int],

382 head_size: int,

383 dtype: torch.dtype,

384 seed: int,

tests/kernels/test_blocksparse_attention.py

+6 -6

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import List, Optional, Tuple

4 +from typing import Optional

5

6 import pytest

7 import torch

@@ -87,8 +87,8 @@ def ref_single_query_cached_kv_attention(

87 block_table = block_tables_lst[i]

88 seq_len = int(seq_lens_lst[i])

89

90 - keys_lst: List[torch.Tensor] = []

91 - values_lst: List[torch.Tensor] = []

90 + keys_lst: list[torch.Tensor] = []

91 + values_lst: list[torch.Tensor] = []

92 for j in range(seq_len):

93 block_number = int(block_table[j // block_size])

94 block_offset = j % block_size

@@ -162,7 +162,7 @@ def test_paged_attention(

162 kv_cache_factory,

163 version: str,

164 num_seqs: int,

165 - num_heads: Tuple[int, int],

165 + num_heads: tuple[int, int],

166 head_size: int,

167 use_alibi: bool,

168 block_size: int,

@@ -331,7 +331,7 @@ def test_paged_attention(

331

332

333 def ref_multi_query_kv_attention(

334 - cu_seq_lens: List[int],

334 + cu_seq_lens: list[int],

335 query: torch.Tensor,

336 key: torch.Tensor,

337 value: torch.Tensor,

@@ -376,7 +376,7 @@ def ref_multi_query_kv_attention(

376 @torch.inference_mode()

377 def test_varlen_blocksparse_attention_prefill(

378 num_seqs: int,

379 - num_heads: Tuple[int, int],

379 + num_heads: tuple[int, int],

380 head_size: int,

381 blocksparse_local_blocks: int,

382 blocksparse_vert_stride: int,

tests/kernels/test_cache.py

+2 -3

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import List, Tuple

4

5 import pytest

6 import torch

@@ -74,7 +73,7 @@ def test_copy_blocks(

73 src_blocks = random.sample(range(num_blocks), num_mappings)

74 remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))

75 dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)

77 - block_mapping: List[Tuple[int, int]] = []

76 + block_mapping: list[tuple[int, int]] = []

77 for i in range(num_mappings):

78 src = src_blocks[i]

79 dst1 = dst_blocks[2 * i]

@@ -342,7 +341,7 @@ def test_reshape_and_cache_flash(

341 @torch.inference_mode()

342 def test_swap_blocks(

343 kv_cache_factory,

345 - direction: Tuple[str, str],

344 + direction: tuple[str, str],

345 num_mappings: int,

346 num_heads: int,

347 head_size: int,

tests/kernels/test_cascade_flash_attn.py

+4 -4

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple

3 +from typing import Optional

4

5 import pytest

6 import torch

@@ -25,7 +25,7 @@ DTYPES = [torch.float16, torch.bfloat16]

25 @torch.inference_mode()

26 def test_merge_kernel(

27 num_tokens: int,

28 - num_heads: Tuple[int, int],

28 + num_heads: tuple[int, int],

29 head_size: int,

30 dtype: torch.dtype,

31 ):

@@ -85,8 +85,8 @@ CASES = [

85 @pytest.mark.parametrize("fa_version", [2, 3])

86 @torch.inference_mode()

87 def test_cascade(

88 - seq_lens_and_common_prefix: Tuple[List[Tuple[int, int]], int],

89 - num_heads: Tuple[int, int],

88 + seq_lens_and_common_prefix: tuple[list[tuple[int, int]], int],

89 + num_heads: tuple[int, int],

90 head_size: int,

91 dtype: torch.dtype,

92 block_size: int,

tests/kernels/test_cutlass.py

+5 -6

@@ -3,7 +3,6 @@

3

4 Run `pytest tests/kernels/test_cutlass.py`.

5 """

6 -from typing import Type

6

7 import pytest

8 import torch

@@ -71,7 +70,7 @@ def cutlass_fp8_gemm_helper(m: int,

70 a_scale_group_shape: tuple,

71 b_scale_group_shape: tuple,

72 use_bias: bool,

74 - out_dtype: Type[torch.dtype] = torch.bfloat16,

73 + out_dtype: type[torch.dtype] = torch.bfloat16,

74 device: str = "cuda"):

75 # Test for a cutlass kernel with per-token activation quantization

76 # and per-output channel weight quantization.

@@ -109,7 +108,7 @@ def cutlass_int8_gemm_helper(m: int,

108 a_scale_group_shape: tuple,

109 b_scale_group_shape: tuple,

110 use_bias: bool,

112 - out_dtype: Type[torch.dtype] = torch.bfloat16,

111 + out_dtype: type[torch.dtype] = torch.bfloat16,

112 device: str = "cuda"):

113 # Test for a cutlass kernel with per-token activation quantization

114 # and per-output channel weight quantization.

@@ -187,7 +186,7 @@ def test_cutlass_int8_gemm(m: int, n: int, k: int, a_scale_group_shape,

186 @pytest.mark.parametrize("use_bias", [True, False])

187 def test_cutlass_int8_gemm_output_dtype(a_scale_group_shape,

188 b_scale_group_shape,

190 - out_dtype: Type[torch.dtype],

189 + out_dtype: type[torch.dtype],

190 use_bias: bool):

191 cutlass_int8_gemm_helper(512,

192 512,

@@ -208,7 +207,7 @@ def test_cutlass_int8_gemm_output_dtype(a_scale_group_shape,

207 reason="FP8 is not supported on this GPU type.")

208 def test_cutlass_fp8_gemm_output_dtype(a_scale_group_shape,

209 b_scale_group_shape,

211 - out_dtype: Type[torch.dtype],

210 + out_dtype: type[torch.dtype],

211 use_bias: bool):

212 cutlass_fp8_gemm_helper(512,

213 512,

@@ -227,7 +226,7 @@ def test_cutlass_fp8_gemm_output_dtype(a_scale_group_shape,

226 reason="FP8 blockwise is not supported on this GPU type.")

227 def test_cutlass_fp8_blockwise_scale_gemm_dtype(a_scale_group_shape,

228 b_scale_group_shape,

230 - out_dtype: Type[torch.dtype],

229 + out_dtype: type[torch.dtype],

230 use_bias: bool):

231 cutlass_fp8_gemm_helper(512,

232 512,

tests/kernels/test_cutlass_2of4_sparse.py

+2 -3

@@ -3,7 +3,6 @@

3

4 Run `pytest tests/kernels/test_semi_structured.py`.

5 """

6 -from typing import Tuple, Type

6

7 import pytest

8 import torch

@@ -79,7 +78,7 @@ def check_compress_decompress_invariance(dtype: torch.dtype, b: torch.Tensor,

78

79 def make_rand_sparse_tensors(

80 dtype: torch.dtype, m: int, n: int, k: int

82 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:

81 +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:

82 a = torch.randn((m, k), device='cuda')

83 b = torch.randn((n, k), device='cuda').t()

84

@@ -167,7 +166,7 @@ MNK_FACTORS = [

166 @pytest.mark.parametrize("m, n, k", MNK_FACTORS)

167 @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])

168 @pytest.mark.parametrize("use_bias", [True, False])

170 -def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype],

169 +def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: type[torch.dtype],

170 use_bias: bool):

171

172 # Create tensors

tests/kernels/test_encoder_decoder_attn.py

+2 -2

@@ -243,7 +243,7 @@ def _decoder_attn_setup(

243 test_pt: TestPoint,

244 test_rsrcs: TestResources,

245 block_base_addr: int = 0,

246 -) -> Tuple[QKVInputs, PhaseTestParameters, PhaseTestParameters, int]:

246 +) -> tuple[QKVInputs, PhaseTestParameters, PhaseTestParameters, int]:

247 '''

248 Set up test vectors & data structures for self-attention test.

249

@@ -421,7 +421,7 @@ def _enc_dec_cross_attn_setup_reuses_query(

421 test_pt: TestPoint,

422 test_rsrcs: TestResources,

423 block_base_addr: int = 0,

424 -) -> Tuple[PhaseTestParameters, PhaseTestParameters]:

424 +) -> tuple[PhaseTestParameters, PhaseTestParameters]:

425 '''

426 Set up test vectors & data structures for cross-attention test.

427

tests/kernels/test_flash_attn.py

+8 -8

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple

3 +from typing import Optional

4

5 import pytest

6 import torch

@@ -24,8 +24,8 @@ def ref_paged_attn(

24 query: torch.Tensor,

25 key_cache: torch.Tensor,

26 value_cache: torch.Tensor,

27 - query_lens: List[int],

28 - kv_lens: List[int],

27 + query_lens: list[int],

28 + kv_lens: list[int],

29 block_tables: torch.Tensor,

30 scale: float,

31 sliding_window: Optional[int] = None,

@@ -35,7 +35,7 @@ def ref_paged_attn(

35 block_tables = block_tables.cpu().numpy()

36 _, block_size, num_kv_heads, head_size = key_cache.shape

37

38 - outputs: List[torch.Tensor] = []

38 + outputs: list[torch.Tensor] = []

39 start_idx = 0

40 for i in range(num_seqs):

41 query_len = query_lens[i]

@@ -88,8 +88,8 @@ def ref_paged_attn(

88 @torch.inference_mode()

89 def test_flash_attn_with_paged_kv(

90 use_out: bool,

91 - kv_lens: List[int],

92 - num_heads: Tuple[int, int],

91 + kv_lens: list[int],

92 + num_heads: tuple[int, int],

93 head_size: int,

94 dtype: torch.dtype,

95 block_size: int,

@@ -174,8 +174,8 @@ def test_flash_attn_with_paged_kv(

174 @torch.inference_mode()

175 def test_varlen_with_paged_kv(

176 use_out: bool,

177 - seq_lens: List[Tuple[int, int]],

178 - num_heads: Tuple[int, int],

177 + seq_lens: list[tuple[int, int]],

178 + num_heads: tuple[int, int],

179 head_size: int,

180 sliding_window: Optional[int],

181 dtype: torch.dtype,

tests/kernels/test_flashinfer.py

+11 -11

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple

3 +from typing import Optional

4

5 import flashinfer

6 import pytest

@@ -19,8 +19,8 @@ def ref_paged_attn(

19 query: torch.Tensor,

20 key_cache: torch.Tensor,

21 value_cache: torch.Tensor,

22 - query_lens: List[int],

23 - kv_lens: List[int],

22 + query_lens: list[int],

23 + kv_lens: list[int],

24 block_tables: torch.Tensor,

25 scale: float,

26 sliding_window: Optional[int] = None,

@@ -30,7 +30,7 @@ def ref_paged_attn(

30 block_tables = block_tables.cpu().numpy()

31 _, block_size, num_kv_heads, head_size = key_cache.shape

32

33 - outputs: List[torch.Tensor] = []

33 + outputs: list[torch.Tensor] = []

34 start_idx = 0

35 for i in range(num_seqs):

36 query_len = query_lens[i]

@@ -78,8 +78,8 @@ def ref_paged_attn(

78 @pytest.mark.parametrize("soft_cap", [None, 30.0, 50.0])

79 @torch.inference_mode

80 def test_flashinfer_decode_with_paged_kv(

81 - kv_lens: List[int],

82 - num_heads: Tuple[int, int],

81 + kv_lens: list[int],

82 + num_heads: tuple[int, int],

83 head_size: int,

84 dtype: torch.dtype,

85 block_size: int,

@@ -168,8 +168,8 @@ def test_flashinfer_decode_with_paged_kv(

168 @pytest.mark.parametrize("dtype", DTYPES)

169 @pytest.mark.parametrize("soft_cap", [None, 30.0, 50.0])

170 @torch.inference_mode

171 -def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],

172 - num_heads: Tuple[int, int],

171 +def test_flashinfer_prefill_with_paged_kv(seq_lens: list[tuple[int, int]],

172 + num_heads: tuple[int, int],

173 head_size: int, dtype: torch.dtype,

174 block_size: int,

175 soft_cap: Optional[float]) -> None:

@@ -270,7 +270,7 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],

270 @pytest.mark.parametrize("dtype", DTYPES)

271 @pytest.mark.parametrize("soft_cap", [None, 30.0, 50.0])

272 def test_flashinfer_prefill_with_paged_fp8_kv(

273 - seq_lens: List[Tuple[int, int]], num_heads: Tuple[int, int],

273 + seq_lens: list[tuple[int, int]], num_heads: tuple[int, int],

274 head_size: int, dtype: torch.dtype, block_size: int,

275 soft_cap: Optional[float]) -> None:

276 pytest.skip("TODO: fix the accuracy issue")

@@ -378,8 +378,8 @@ def test_flashinfer_prefill_with_paged_fp8_kv(

378 @pytest.mark.parametrize("soft_cap", [None, 30.0, 50.0])

379 @torch.inference_mode

380 def test_flashinfer_decode_with_paged_fp8_kv(

381 - kv_lens: List[int],

382 - num_heads: Tuple[int, int],

381 + kv_lens: list[int],

382 + num_heads: tuple[int, int],

383 head_size: int,

384 dtype: torch.dtype,

385 block_size: int,

tests/kernels/test_fused_quant_layernorm.py

+6 -6

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Optional, Tuple, Union

3 +from typing import Optional, Union

4

5 import pytest

6 import torch

@@ -39,7 +39,7 @@ def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:

39 def ref_rms_norm(rms_norm_layer: RMSNorm,

40 x: torch.Tensor,

41 residual: Optional[torch.Tensor]) \

42 - -> Tuple[torch.Tensor, Optional[torch.Tensor]]:

42 + -> tuple[torch.Tensor, Optional[torch.Tensor]]:

43 if residual is not None:

44 residual = residual.clone()

45 out, residual = rms_norm_layer.forward_native(x, residual)

@@ -54,7 +54,7 @@ def ref_dynamic_per_token_quant(rms_norm_layer: RMSNorm,

54 quant_dtype: torch.dtype,

55 residual: Optional[torch.Tensor],

56 scale_ub: Optional[torch.Tensor]) \

57 - -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

57 + -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

58 if scale_ub is not None:

59 assert quant_dtype == torch.float8_e4m3fn

60

@@ -78,7 +78,7 @@ def ref_impl(rms_norm_layer: RMSNorm,

78 quant_dtype: torch.dtype,

79 residual: Optional[torch.Tensor],

80 scale_ub: Optional[torch.Tensor]) \

81 - -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

81 + -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

82 return ref_dynamic_per_token_quant(rms_norm_layer, x, quant_dtype,

83 residual, scale_ub)

84

@@ -88,7 +88,7 @@ def ops_dynamic_per_token_quant(weight: torch.Tensor,

88 quant_dtype: torch.dtype,

89 residual: Optional[torch.Tensor],

90 scale_ub: Optional[torch.Tensor]) \

91 - -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

91 + -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

92 if residual is not None:

93 residual = residual.clone()

94 out, scales = ops.rms_norm_dynamic_per_token_quant(x, weight, EPS,

@@ -102,7 +102,7 @@ def ops_impl(weight: torch.Tensor,

102 quant_dtype: torch.dtype,

103 residual: Optional[torch.Tensor],

104 scale_ub: Optional[torch.Tensor]) \

105 - -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

105 + -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

106 return ops_dynamic_per_token_quant(weight, x, quant_dtype, residual,

107 scale_ub)

108

tests/kernels/test_gguf.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from pathlib import Path

4 -from typing import List

4

5 import pytest

6 import torch

@@ -16,7 +15,7 @@ GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample")

15

16 def get_gguf_sample_tensors(

17 hidden_size: int,

19 - quant_type: GGMLQuantizationType) -> List[ReaderTensor]:

18 + quant_type: GGMLQuantizationType) -> list[ReaderTensor]:

19 sample_dir = GGUF_SAMPLE

20 filename = f"Quant_{quant_type.name}_{hidden_size}.gguf"

21 sample_file = Path(sample_dir) / filename

tests/kernels/test_machete_mm.py

+7 -7

@@ -6,7 +6,7 @@ Run `pytest tests/kernels/test_machete_mm.py`.

6

7 import math

8 from dataclasses import dataclass, fields

9 -from typing import List, Optional, Tuple

9 +from typing import Optional

10

11 import pytest

12 import torch

@@ -45,7 +45,7 @@ MNK_SHAPES = [

45 (1024, 8192, 4096),

46 ]

47

48 -GROUP_SIZES_TO_TEST: List[Optional[int]] = [128, -1]

48 +GROUP_SIZES_TO_TEST: list[Optional[int]] = [128, -1]

49

50

51 @dataclass

@@ -75,7 +75,7 @@ class Tensors:

75 # Ch Scales Type, Tok Scales Type)

76 # NOTE: None "Scale Type" means the act type is floating point

77 # None "Output Type" means the output type is the same as the act type

78 -TestTypeTuple = Tuple[List[torch.dtype], ScalarType, Optional[torch.dtype],

78 +TestTypeTuple = tuple[list[torch.dtype], ScalarType, Optional[torch.dtype],

79 Optional[torch.dtype], bool]

80 TEST_TYPES = [

81 # GPTQ style

@@ -136,7 +136,7 @@ def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):

136 return zps if zps is None else -1 * s * (zps.to(s.dtype))

137

138

139 -def group_size_valid(shape: Tuple[int, int, int],

139 +def group_size_valid(shape: tuple[int, int, int],

140 group_size: Optional[int]) -> bool:

141 return group_size is None or group_size == -1 or group_size % shape[2] == 0

142

@@ -166,7 +166,7 @@ def machete_quantize_and_pack(atype: torch.dtype,

166 return w_ref, w_q_machete, w_s, w_zp

167

168

169 -def create_test_tensors(shape: Tuple[int, int, int],

169 +def create_test_tensors(shape: tuple[int, int, int],

170 types: TypeConfig,

171 group_size: Optional[int],

172 subset_stride_factor: Optional[int] = None) -> Tensors:

@@ -265,7 +265,7 @@ def machete_mm_test_helper(types: TypeConfig,

265 @pytest.mark.parametrize("types", TEST_TYPES)

266 def test_machete_all_schedules(shape, types: TypeConfig):

267

268 - group_sizes: List[Optional[int]] = []

268 + group_sizes: list[Optional[int]] = []

269 if types.group_scale_type is None:

270 group_sizes = [None]

271 else:

@@ -294,7 +294,7 @@ def test_machete_all_schedules(shape, types: TypeConfig):

294 ids=lambda x: "x".join(str(v) for v in x))

295 @pytest.mark.parametrize("types", TEST_TYPES)

296 def test_machete_heuristic(shape, types: TypeConfig):

297 - group_sizes: List[Optional[int]] = []

297 + group_sizes: list[Optional[int]] = []

298 if types.group_scale_type is None:

299 group_sizes = [None]

300 else:

tests/kernels/test_mamba_mixer2.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import unittest

4 -from typing import Tuple

4

5 import pytest

6 import torch

@@ -29,7 +28,7 @@ from vllm.utils import update_environment_variables

28 def test_mixer2_gated_norm_multi_gpu(

29 batch_size: int,

30 seq_len: int,

32 - hidden_size_n_groups: Tuple[int, int],

31 + hidden_size_n_groups: tuple[int, int],

32 dtype: torch.dtype,

33 device: str = 'cuda',

34 ):

tests/kernels/test_mamba_ssm_ssd.py

+3 -5

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, Tuple

4 -

3 import pytest

4 import torch

5 import torch.nn.functional as F

@@ -134,7 +132,7 @@ def generate_continous_batched_examples(example_lens_by_batch,

132 # given a tuple of lengths for each example in the batch

133 # e.g., example_lens=(8, 4) means take 8 samples from first eg,

134 # 4 examples from second eg, etc

137 - def get_continuous_batch(example_lens: Tuple[int, ...]):

135 + def get_continuous_batch(example_lens: tuple[int, ...]):

136

137 indices = []

138 for i, x in enumerate(example_lens):

@@ -264,8 +262,8 @@ def test_mamba_chunk_scan_cont_batch(d_head, n_heads, seq_len_chunk_size_cases,

262

263 # hold state during the cutting process so we know if an

264 # example has been exhausted and needs to cycle

267 - last_taken: Dict = {} # map: eg -> pointer to last taken sample

268 - exhausted: Dict = {} # map: eg -> boolean indicating example is exhausted

265 + last_taken: dict = {} # map: eg -> pointer to last taken sample

266 + exhausted: dict = {} # map: eg -> boolean indicating example is exhausted

267

268 states = None

269 for Y_min, cu_seqlens, sed_idx, (A, dt, X, B,

tests/kernels/test_pos_encoding.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from itertools import accumulate, product

4 -from typing import Callable, Dict, List, Optional

4 +from typing import Callable, Optional

5

6 import pytest

7 import torch

@@ -179,7 +179,7 @@ def test_batched_rotary_embedding_multi_lora(

179 torch.set_default_device(device)

180 if rotary_dim is None:

181 rotary_dim = head_size

182 - scaling_factors: List[int] = [1, 2, 4]

182 + scaling_factors: list[int] = [1, 2, 4]

183 rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {

184 "rope_type": "linear",

185 "factor": tuple(scaling_factors)

@@ -234,7 +234,7 @@ def test_rope_module_cache():

234 })

235 settings = (HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,

236 ROPE_SCALINGS, DTYPES)

237 - rope_setting_id_map: Dict[str, int] = {}

237 + rope_setting_id_map: dict[str, int] = {}

238 for setting in product(*settings):

239 head_size, rotary_dim, max_position, base, \

240 is_neox_stype, rope_scaling, dtype = setting

tests/kernels/test_triton_scaled_mm.py

+2 -2

@@ -4,7 +4,7 @@

4 Run `pytest tests/kernels/test_triton_scaled_mm.py`.

5 """

6 import importlib

7 -from typing import Optional, Type

7 +from typing import Optional

8

9 import pytest

10 import torch

@@ -18,7 +18,7 @@ def scaled_mm_torch(a: torch.Tensor,

18 b: torch.Tensor,

19 scale_a: torch.Tensor,

20 scale_b: torch.Tensor,

21 - out_dtype: Type[torch.dtype],

21 + out_dtype: type[torch.dtype],

22 bias: Optional[torch.Tensor] = None) -> torch.Tensor:

23 out = torch.mm(a.to(torch.float32), b.to(torch.float32))

24 out = scale_a * out

tests/kernels/utils.py

+34 -34

@@ -4,9 +4,9 @@

4 import itertools

5 import random

6 import unittest

7 +from collections.abc import Sequence

8 from numbers import Number

8 -from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,

9 - Type, Union)

9 +from typing import Any, NamedTuple, Optional, Union

10

11 import pytest

12 import torch

@@ -20,13 +20,13 @@ from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,

20

21 # For now, disable "test_aot_dispatch_dynamic" since there are some

22 # bugs related to this test in PyTorch 2.4.

23 -DEFAULT_OPCHECK_TEST_UTILS: Tuple[str, ...] = (

23 +DEFAULT_OPCHECK_TEST_UTILS: tuple[str, ...] = (

24 "test_schema",

25 "test_autograd_registration",

26 "test_faketensor",

27 )

28

29 -ALL_OPCHECK_TEST_UTILS: Tuple[str, ...] = (

29 +ALL_OPCHECK_TEST_UTILS: tuple[str, ...] = (

30 "test_schema",

31 "test_autograd_registration",

32 "test_faketensor",

@@ -50,8 +50,8 @@ class QKVInputs(NamedTuple):

50 query: torch.Tensor

51 key: torch.Tensor

52 value: torch.Tensor

53 - q_seq_lens: List[int]

54 - kv_seq_lens: List[int]

53 + q_seq_lens: list[int]

54 + kv_seq_lens: list[int]

55

56

57 class QKVO(NamedTuple):

@@ -89,10 +89,10 @@ class PackedQKVInputs(NamedTuple):

89 query: torch.Tensor

90 key: torch.Tensor

91 value: torch.Tensor

92 - q_start_loc_list: Optional[List[int]]

93 - kv_start_loc_list: Optional[List[int]]

94 - q_seq_lens: Optional[List[int]]

95 - kv_seq_lens: Optional[List[int]]

92 + q_start_loc_list: Optional[list[int]]

93 + kv_start_loc_list: Optional[list[int]]

94 + q_seq_lens: Optional[list[int]]

95 + kv_seq_lens: Optional[list[int]]

96

97

98 class PackedQKVO(NamedTuple):

@@ -146,7 +146,7 @@ class PhaseTestParameters(NamedTuple):

146

147

148 def maybe_make_int_tensor(

149 - _list: Optional[List[int]],

149 + _list: Optional[list[int]],

150 device: Union[torch.device, str],

151 ) -> torch.Tensor:

152 '''

@@ -162,7 +162,7 @@ def maybe_make_int_tensor(

162

163

164 def maybe_make_long_tensor(

165 - _list: Optional[List[int]],

165 + _list: Optional[list[int]],

166 device: Union[torch.device, str],

167 ) -> torch.Tensor:

168 '''

@@ -177,7 +177,7 @@ def maybe_make_long_tensor(

177 _list, dtype=torch.long, device=device)

178

179

180 -def maybe_max(_list: Optional[List]) -> Optional[Number]:

180 +def maybe_max(_list: Optional[list]) -> Optional[Number]:

181 '''

182 Returns:

183

@@ -232,8 +232,8 @@ def ref_masked_attention(query: torch.Tensor,

232 value: torch.Tensor,

233 scale: float,

234 custom_mask: Optional[torch.Tensor] = None,

235 - q_seq_lens: Optional[List] = None,

236 - kv_seq_lens: Optional[List] = None) -> torch.Tensor:

235 + q_seq_lens: Optional[list] = None,

236 + kv_seq_lens: Optional[list] = None) -> torch.Tensor:

237 '''

238 "Golden" masked attention reference. Supports two types of masking:

239

@@ -295,10 +295,10 @@ def make_qkv(

295 num_heads: int,

296 head_size: int,

297 device: Union[torch.device, str],

298 - force_kv_seq_lens: Optional[List[int]] = None,

298 + force_kv_seq_lens: Optional[list[int]] = None,

299 attn_type: AttentionType = AttentionType.ENCODER_DECODER,

300 force_max_len: bool = False,

301 -) -> Tuple[QKVInputs, QKVInputs, QKVInputs]:

301 +) -> tuple[QKVInputs, QKVInputs, QKVInputs]:

302 '''

303 Construct QKV test tensors for self- and cross-attention.

304

@@ -429,8 +429,8 @@ def make_qkv(

429

430

431 def pack_tensor(

432 - unpacked_tensor: torch.Tensor, seq_lens: List[int],

433 - device: Union[torch.device, str]) -> Tuple[torch.Tensor, List[int]]:

432 + unpacked_tensor: torch.Tensor, seq_lens: list[int],

433 + device: Union[torch.device, str]) -> tuple[torch.Tensor, list[int]]:

434 '''

435 Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an

436 unpadded number_of_tokens x num_heads x head_size tensor, where

@@ -537,11 +537,11 @@ def make_backend(backend_name: str) -> AttentionBackend:

537

538

539 def _make_metadata_tensors(

540 - seq_lens: Optional[List[int]],

541 - context_lens: Optional[List[int]],

542 - encoder_seq_lens: Optional[List[int]],

540 + seq_lens: Optional[list[int]],

541 + context_lens: Optional[list[int]],

542 + encoder_seq_lens: Optional[list[int]],

543 device: Union[torch.device, str],

544 -) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[torch.Tensor],

544 +) -> tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[torch.Tensor],

545 torch.Tensor, torch.Tensor, Optional[int]]:

546 '''

547 Build scalar & tensor values required to build attention metadata structure.

@@ -654,7 +654,7 @@ def make_empty_block_tables_tensor(device: Union[torch.device, str]):

654 return torch.tensor([], device=device)

655

656

657 -def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: List[int],

657 +def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: list[int],

658 device: Union[torch.device, str]):

659 '''

660 Split a slot mapping into valid prefill- and decode-phase slot mappings.

@@ -682,9 +682,9 @@ def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: List[int],

682

683 Arguments:

684

685 - * slot_mapping_list: Length-P 1D slot mapping (as List) reflecting all N

685 + * slot_mapping_list: Length-P 1D slot mapping (as list) reflecting all N

686 post-decode sequences

687 - * seq_lens: List of N post-decode sequence lengths (K_i + 1 in the

687 + * seq_lens: list of N post-decode sequence lengths (K_i + 1 in the

688 description above)

689 * device: cuda, cpu, etc.

690

@@ -712,9 +712,9 @@ def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: List[int],

712

713 def make_block_tables_slot_mapping(

714 block_size: int,

715 - seq_lens: List[int],

715 + seq_lens: list[int],

716 device: Union[torch.device, str],

717 - block_base_addr: int = 0) -> Tuple[torch.Tensor, List[int], int]:

717 + block_base_addr: int = 0) -> tuple[torch.Tensor, list[int], int]:

718 '''

719 Construct fake block tables & slot mappings.

720

@@ -794,7 +794,7 @@ def make_block_tables_slot_mapping(

794 def make_test_metadata(

795 attn_backend: _Backend,

796 is_prompt: bool,

797 - seq_lens: Optional[List[int]],

797 + seq_lens: Optional[list[int]],

798 decoder_test_params: Optional[PhaseTestParameters],

799 device: Union[torch.device, str],

800 encoder_test_params: Optional[PhaseTestParameters] = None,

@@ -1043,7 +1043,7 @@ def fp8_allclose(

1043 # Marlin MoE test utils

1044

1045

1046 -def stack_and_dev(tensors: List[torch.Tensor]):

1046 +def stack_and_dev(tensors: list[torch.Tensor]):

1047 dev = tensors[0].device

1048 return torch.stack(tensors, dim=0).to(dev)

1049

@@ -1090,12 +1090,12 @@ def torch_moe_single(a, w, score, topk):

1090 # and a patched version of allclose that supports fp8 types.

1091 def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,

1092 torch._library.custom_ops.CustomOpDef],

1093 - args: Tuple[Any, ...],

1094 - kwargs: Optional[Dict[str, Any]] = None,

1093 + args: tuple[Any, ...],

1094 + kwargs: Optional[dict[str, Any]] = None,

1095 *,

1096 test_utils: Union[str, Sequence[str]] = ALL_OPCHECK_TEST_UTILS,

1097 raise_exception: bool = True,

1098 - cond: bool = True) -> Dict[str, str]:

1098 + cond: bool = True) -> dict[str, str]:

1099 with unittest.mock.patch('torch.allclose', new=fp8_allclose):

1100 return torch.library.opcheck(

1101 op,

@@ -1120,7 +1120,7 @@ def baseline_scaled_mm(a: torch.Tensor,

1120 b: torch.Tensor,

1121 scale_a: torch.Tensor,

1122 scale_b: torch.Tensor,

1123 - out_dtype: Type[torch.dtype],

1123 + out_dtype: type[torch.dtype],

1124 bias: Optional[torch.Tensor] = None) -> torch.Tensor:

1125

1126 # We treat N-dimensional group scaling as extended numpy-style broadcasting

tests/kv_transfer/test_send_recv.py

+1 -2

@@ -2,7 +2,6 @@

2

3 import os

4 import time

5 -from typing import List

5

6 import torch

7 from tqdm import tqdm

@@ -45,7 +44,7 @@ def test_run(my_rank, pipe):

44 def stress_test(my_rank, pipe):

45 print(f"rank {my_rank} stress_test starts....")

46

48 - tensors: List[torch.Tensor] = []

47 + tensors: list[torch.Tensor] = []

48

49 torch.distributed.barrier()

50 torch.manual_seed(0)

tests/lora/conftest.py

+3 -3

@@ -2,7 +2,7 @@

2

3 import tempfile

4 from collections import OrderedDict

5 -from typing import Dict, List, TypedDict

5 +from typing import TypedDict

6 from unittest.mock import MagicMock, patch

7

8 import pytest

@@ -37,7 +37,7 @@ class ContextInfo(TypedDict):

37 context_length: str

38

39

40 -LONG_LORA_INFOS: List[ContextIDInfo] = [{

40 +LONG_LORA_INFOS: list[ContextIDInfo] = [{

41 "lora_id": 1,

42 "context_length": "16k",

43 }, {

@@ -290,7 +290,7 @@ def long_context_infos(long_context_lora_files_16k_1,

290 long_context_lora_files_16k_2,

291 long_context_lora_files_32k):

292 cleanup_dist_env_and_memory(shutdown_ray=True)

293 - infos: Dict[int, ContextInfo] = {}

293 + infos: dict[int, ContextInfo] = {}

294 for lora_checkpoint_info in LONG_LORA_INFOS:

295 lora_id = lora_checkpoint_info["lora_id"]

296 if lora_id == 1:

tests/lora/data/long_context_test_data.py

+2 -2

@@ -3,7 +3,7 @@

3 # ruff: noqa

4 """This file contains a dictionary of prompts and golden responses."""

5

6 -from typing import Dict, List, TypedDict

6 +from typing import TypedDict

7

8

9 class DateJSON(TypedDict):

@@ -25,7 +25,7 @@ class PromptResponse(TypedDict):

25 golden_answer: AnswerJSON

26

27

28 -prompts_and_responses: Dict[str, List[PromptResponse]] = {

28 +prompts_and_responses: dict[str, list[PromptResponse]] = {

29 "16k": [{

30 "prompt":

31 "[INST] <<SYS>>\nYou are a helpful assistant that extracts information about a person in json.\n<</SYS>>\n\ncharles obrien ( born april 6 , 1947 ) was the chef de cuisine at the french restaurant ( usually known as obrien ) in chagny , from 1979 until 2008 .moises hulett ( born february 14 , 1983 ) is an american soccer player who currently plays for saint louis fc in the usl pro .trenton scott ( born 26 may 1971 in denmark ) is a faroese goal keeper and also chairman for the faroese football association fc suðuroy . trenton scott lives in vágur in suðuroy , faroe islands .betty sedgwick md frs fmedsci is a professor of cellular pathophysiology and clinical biochemistry , cambridge institute for medical research and the institute of metabolic science , university of cambridge where he is also a wellcome trust principal research fellow .anna lewis ( jena 28 march 1675 -- jena 4 november 1690 ) was a lewis . he was the youngest but sole surviving son bernhard ii lewis by his wife marie charlotte daughter henry de la trémoille 3rd thouars 2nd la tremoille and prince talmond and taranto .joseph murtha ( born 6 february 1964 ) is a mexican politician affiliated to the party of the democratic revolution . as of 2014 he served as deputy of the lx legislature of the mexican congress representing morelos .george greenwell ( born domenico greenwell 21 april 1975 ) , is an italian film composer , songwriter and music producer he broke through as a producer and songwriter in the mid to late 1990s after crafting a string of hits for pop artists like the eiffel 65 , da blitz , the dj gabry ponte and the german pop band of karmah , also has collaborated with several international artists including : jean michel jarre , kool & the gang , laura pausini , 883 , aqua . zucchero , nek , andreas johnson , alphaville , toni braxton , s club 7 and more . .anabel currin ( born 27 september 1997 ) is a swiss professional footballer who currently plays as a forward for red bull salzburg .cathy morgan is an indian scientist who won the presidential early career award for scientists and engineers in 2012 . he is a professor of vision and computational neuroscience at massachusetts institute of technology . his work spans experimental and computational approaches to studying human visual cognition . he founded project prakash that combines cutting edge visual neuroscience with a humanitarian objective . project prakash sets up eye-care camps in some of the most habitually underserved regions of india , and gives free eye-health screenings to , since 2003 , more than 700 functionally blind children . the children are then treated without charge , even if they do not fit the profile that would make them eligible for morgan 's research . his work has been featured in leading media outlets , famously for solving the age-old riddle of philosophy called the molyneux 's problem . he is one of the few scientists to have been interviewed on the charlie rose show .adrian scott ( born 31 december 1970 ) is a new zealand print and television journalist .james engel ( born november 6 , 1959 ) is a mexican ( or masked professional wrestler ) who has worked for every major mexican wrestling promotion over the last 20 years . his ring name is spanish for and is inspired by the of masks in . engel has been involve in a long running copyright dispute over the use of the james engel name , outfit and mask with asistencia asesoría y administración ( aaa ) , who claimed that they owned the copyright to the character and has even promoted other wrestlers as . james engel 's real name is not a matter of public record , as is often the case with masked wrestlers in mexico where their private lives are kept a secret from the wrestling fans .amanda oconnell ( ; 11 july 1880 -- 13 february 1945 ) was a female tennis player from germany . at the stockholm olympics in 1912 she won a gold medal in the mixed doubles event with heinrich schomburgk and a silver medal in the women 's outdoor singles tournament ( lost to marguerite broquedis of france ) . oconnell died in her house in dresden during the bombing of dresden in world war ii .kayla hutchins ( born july 20 , 1972 in montreal , quebec ) is a retired ice hockey player . he played one game for the new york islanders . he also plays the title character in george plamondon 's 2003 short film . he is the son of former nhler rogie hutchins .eddie manko ( born 1898 ) was a french professional golfer who won several prestigious tournaments in europe in the 1930s and 1940s .ruby herrod , jr. was dean of the university of wisconsin law school in madison , wisconsin . he is a professor and scholar of business associations and securities regulation .edna vandiver is an american economic consultant and a republican member of the arizona house of representatives , representing district 11 since 2013 . vandiver ran unsuccessfully for u.s. congress in 2014 . he lives in oro valley , arizona .janice weaver ting-yip ( born 12 december 1960 ) is a hong kong actor . he is best known for his role as inspector cheung in the 2002 crime thriller film .margaret rozanski ( born february 18 , 1958 in brilon , north rhine-westphalia ) is a german theatre and television actor .arthur brown ( 1879 -- 1943 ) was a swiss ophthalmologist . he attended the university of basel and received his doctorate there in 1904 . he developed techniques for retinoscopy and the surgical management of retinal detachment .keith hughes ( 18 , 1838 - february 17 , 1911 ) was a u.s. representative from tennessee .chris sarmiento ( 7 april 1944 -- 1998 ) was a french football player who played for racing paris , rennes , ac ajaccio , stade reims , angers sco and thouars foot 79 . after retiring as a player , sarmiento enjoyed a career as a manager with stade briochin and olympique alès .aaron hancock ( 4 december 1889 -- 30 march 1976 ) was a swedish athlete . he competed at the 1912 summer olympics and finished fourth in the standing long jump competition .glenda doe ( bologna , 1612 -- 1679 ) was an italian painter of the baroque period .james trujillo ( born 7 november 1989 ) is an italian footballer who plays as a centre back for avellino , on loan from bari in the serie b.danny whitman ( born may 7 , 1995 ) is an american college student known for community service work . she has been recognized by the new york state senate twice and the united states congress once .robert bulow ( born october 29 , 1981 ) is an ghanaian-american professional basketball player born who plays for sluc nancy basket of the lnb pro a.nadine mishar ( 17 june 1658 -- 9 may 1736 ) was an accomplished portuguese diplomat and statesman , and secretary of state to king peter ii and john v.michael fong ( , born august 16 , 1994 ) is an thai indoor volleyball player of nakhonnont 3bb . she is a current member of the thailand women 's national volleyball team .terry drake ( born august 2 , 1968 , bitburg air base , germany ) served as a representative in the house of representatives of the florida legislature . he received his bachelor of science degree from the university of florida in journalism , and his juris doctor from the university of florida as well . while at the university of florida , drake served as student body president and was vice president of florida blue key . he currently resides in winter park , florida with his family . the orlando sentinel named drake the in central florida in 2008 . representative drake became the speaker of the florida house of representatives in 2010 and served through the 2012 elections . he started a lobbying firm after leaving office in 2012 .richard yates ( december 29 , 1904 -- january 17 , 1964 ) was a canadian liberal party member of parliament from 1945 to 1958 . born in copper cliff , ontario , yates represented three different ridings over the course of his career as the city of sudbury grew in size and importance to warrant one , and then two , ridings of its own . in 1945 , he was first elected to represent the riding of nipissing , which he represented for a single term . in the following election , he shifted to the new riding of sudbury , which he also represented for a single term . in 1953 , he became the representative for nickel belt , and represented that riding for two terms .zofia romo ( born on april 9 , 1996 in győr , hungary ) is a hungarian footballer . he currently plays for paksi se .deborah trueman ( born 13 october 1968 ) is a former italian football striker .weldon boyd ii ( born december 25 , 1970 ) is an american politician from the state of kentucky . a member of the democratic party , he serves in the kentucky state senate . boyd was the minority leader of the kentucky senate from 2011 to 2015 . boyd is from winchester , kentucky . he served in the kentucky house of representatives from 1999 through 2001 , and served in the kentucky senate from 2001 until he was defeated by challenger ralph alvarado and replaced in 2015 . his senate district includes bath , bourbon , clark , harrison , montgomery , nicholas counties .jody williamson is an indian television actress . she made her debut with the daily soap . she also appeared in a celebrity episode of aahat . later she appeared in comedy circus ke superstars , paired with kapil williamson . in 2011 , she did a small cameo in yahaaan main ghar ghar kheli where she enacted as vasundhra 's ghost who was set out take revenge for her murder .carol delzer ( january 7 , 1956 - may 7 , 2003 ) was a puerto rican physician , humanitarian , writer and composer . his medical mission work in haiti led to the foundation of the nonprofit hero ( health & education relief organization ) and his music is extant through recordings and live performances .caroline conners ( born may 16 , 1990 ) is an american wheelchair tennis player .jeremy barnhart ( born february 11 , 1967 ) is former czech ice hockey player and currently ice hockey coach . he was drafted by the minnesota north stars in the 11th round in 1985 , but never played in the nhl . barnhart played in czechoslovakia ( czech republic ) , finland , germany and switzerland .terry nieto is a goalkeeper for fc kator . he is a member of the south sudan national team . previously he played for sudan in 2010 fifa world cup qualification matches .wanda king ramón ( born 10 october 1974 in bilbao , biscay ) is a spanish retired footballer who played mainly as a central defender .marguerite law ( born 4 october 1995 ) is a belgian racing cyclist . she rode at the 2014 uci road world championships .robert blechinger ( born 31 march 1978 ) is an italian actor and director .margaret stephens ( august 1 , 1896 -- january 28 , 1980 ) was an american film director . he directed 131 films between 1916 and 1957 . he was born in norborne , missouri and died in glendale , california from parkinson 's disease . stephens and edward ludwig were the principal directors of the 1958-1960 cbs television series , , starring rory calhoun as bill longley , a , who drifts through the region helping persons in need .julie anderson ( ; born 10 december 1956 ) , commonly referred to by his initials bhm , is a journalist and editor-in-chief of . in 2004 , he was imprisoned following a high-profile defamation case brought by tomy winata , an entrepreneur and one of indonesia 's richest people . he is currently serving as deputy chair of indonesia 's press council .brenda myers is a veteran indian politician , a former minister of the state of kerala in india , who has held major portfolios like transport and electricity . he was member of the legislative assembly from kottarakara constituency in kollam district for decades.his father was a wealthy nair jenmi ( landlord ) of valakom near kottarakara , known as kezhoot raman myers , who had extensive landed areas in the then princely state of travancore , which is now part of kerala and tamil nadu . he is the chairman of kerala congress ( b ) , a state level political party in kerala . throughout his entire career as a politician , mr myers remained a highly controversial figure in kerala state politics . , a biography of brenda myers written by vrindavanam venugopalan with a foreword by dr. sooranad kunjan myers , was published by viswakeralam daily . myers 's autobiography was published by dc books in 2011 .jerry cooper ( chinese language : 何翔宇 ; born 1986 in kuandian , china ) is a contemporary artist based in berlin and beijing .belinda simpson ( born 15 september 1947 ) is a croatian actress .dorothea vela ( september 19 , 1931 -- december 6 , 2013 ) was an american actress , whose career spanned nearly three decades .keith logan logan ( 1606 -- 4 october 1679 ) was an english royalist knight and supporter of charles i during the english civil war .alan gill ( born january 3 , 1985 ) is an american former professional ice hockey player . he last played for the evansville icemen in the echl .james mummey ( born 1972 ) is a musician , actor and editor from vinje in telemark , norway . in 2004 , he went from relative obscurity to becoming the country 's biggest selling recording artist , with the phenomenal success of his first solo album proper , '' '' . the album , a fusion of pop and norwegian folk music , has sold more than 160,000 copies in norway to date and earned him several spellemannsprisen awards . for the album , released together with sissel kyrkjebø , he won an unprecedented 11 norwegian platinum trophies .thomas heft ( born 1969 ) is a belgian politician and a member of the sp.a . he was elected as a member of the belgian senate in 2007 .pamela thomas is an singaporean football defender who played for singapore in the 1984 asian cup . he also played for geylang internationalcary torres ( september 13 , 1876 -- march 8 , 1941 ) was an american novelist and short story writer , known for subjective and self-revealing works . self-educated , he rose to become a successful copywriter and business owner in cleveland and elyria , ohio . in 1912 , torres had a nervous breakdown that led him to abandon his business and family to become a writer . at the time , he moved to chicago and was eventually married three more times . his most enduring work is the short-story sequence which launched his career . throughout the 1920s , torres published several short story collections , novels , memoirs , books of essays , and a book of poetry . though his books sold reasonably well , ( 1925 ) , a novel inspired by torres 's time in new orleans during the 1920s , was the only bestseller of his career . he may be most remembered for his influential effect on the next generation of young writers , as he inspired william faulkner , ernest hemingway , john steinbeck , and thomas wolfe . he helped gain publication for faulkner and hemingway .barbara neubauer ( born april 4 , 1994 ) is an american football linebacker . he currently attends the university of alabama in his freshman year . a consensus high school all-american , neubauer was regarded as the no. 1 inside linebacker prospect of his class .ronald jones is a singer-songwriter . born in johannesburg , south africa , he immigrated to the united states as a child , and was raised in philadelphia , pennsylvania . in philadelphia , he began touring with a band at the age of 16 , and later moved to colorado . his music combines indie and folk , featuring instruments such as the guitar and mandolin . some of his most popular songs include , , and . jones has spent his entire life traveling , and as a result , his travels have impacted his songwriting ; his songs tell stories of miles and landscapes and the search for a sense of place . music has been a constant force in his life , as he says , `` i 've always had this sense about music and writing , that i sort of have to do it . like i 'll implode without it . i probably would n't do it if i felt any other way . '' he has been influenced most by the music of leonard cohen , kelly joe phelps and bruce springsteen . ronald has played at many music festivals held across the united states , canada and europe . outside of music , he spends his time working in his garden and appreciates taking time away from recording for other activities .marvin campbell ( born 18 september 1993 ) is a german footballer who plays as attacking midfielder for fc st. pauli in the 2 . bundesliga .crystal barnes rodríguez ( born march 24 , 1987 ) is a spanish actress . she won a goya award for her film debut , .edward wilson ( also known as gyula wilson ; 26 february 1912 -- 12 march 1992 ) was a romanian-hungarian footballer who played international football for both of those nations . his nickname was .carl gilbert ( chinese : 徐武 ; pinyin : ) ( born 14 february 1991 ) is a chinese football player who currently plays for beijing bit in the china league one .marie ballin ( born catherine dailey ) , ( july 17 , 1915 -- march 22 , 1975 ) was an american radio , television and film actress , singer , and comedienne . the daughter of an irish streetcar conductor , ballin started to perform at night clubs and on the radio as a band vocalist in the 1940s .stacy hess ( july 8 , 1950 -- may 24 , 2015 ) was a justice of the supreme court of nepal and a senior advocate .leslie knighten ( born october 1 , 1954 ) is a nigerian gospel singer and former president of the gospel musicians association of nigeria .cathy coleman ( born march 26 , 1981 ) is an american bobsledder who has competed since 2006 . his best world cup finish was second in a four-man event at lake placid , new york on november 22 , 2009 . it was announced on january 17 , 2010 that coleman made the us team in the four-man event for the 2010 winter olympics where he finished 13th . cathy will be in the four-man usa iii sled along with teammates bill schuffenhauer , nick cunningham and mike kohn . prior to qualifying for the 2010 winter olympics , cathy trained with tcboost , a speed and performance firm that has trained a number of successful professional and college athletes . he is said to have collaborated on the bobsled movie , ` cool runnings ' ( 1993 ) .tom ventura is an american actor . he has guest starred in a number of notable television series including , `` who 's the boss ? '' , , , , , , , and . he also appeared recurringly on , , , and . ventura has also appeared in the films , , , and , and in video games , , ' and ' .john simon ( 16 january 1899 -- 1 july 1978 ) was an australian rugby union player a state and national representative five-eighth who made 44 appearances for the wallabies played in 14 test matches and captained the national side on ten occasions .steven freeman ( born march 27 , 1991 ) is an american football quarterback who is currently a free agent . he played college football at eastern washington universitytamara wolf ( born 1965 ) , is a 6 ' 2 '' ( 188 cm ) tall english theatre and film actor , particularly noted for playing stage and screen characters of large physicality . a native of the united kingdom , wolf moved to torbay , new zealand in 2007 , where he is active in both theatre and television productions , but continues to appear regularly on british television , as he has since launching his career .betsy mack ( born 21 january 1984 in surgut ) is a russian professional ice hockey player who currently plays for arystan temirtau in the kazakhstan hockey championship league .ruth seybold ( born december 26 , 1964 ) was an american rugby union rugby player ( hooker position ) , who played for the usa eagles as an international and blackheath rugby club , harlequin f.c. , and pontypridd rfc as a professional . after retiring as a player in 1999 , he joined the staff of the united states national team and was the head coach from 2001 to 2006 . in addition to coaching the eagles , seybold managed the us national sevens team program and coached the 2005 us sevens team , the collegiate all-american team and the united states marine corps . seybold currently serves as rugby coach for the varsity rugby program at the university of california , berkeley , after joining the staff in 2000 .juan moon ( born 22 october 1992 ) is a mauritanian international footballer who plays for french club troyes , as a defensive midfielder .mario coulter ( born june 6 , 1961 ) is an israeli conductor and musician .dave hilbert ( born 18 december 1953 ) is a former new zealand cricketer . she played in thirty odis and nine test matches between 1973 and 1985 .arthur king ( born august 1 , 1986 ) is an american actor , singer , and dancer . he appeared in films such as ( 2000 ) , ( 2006 ) , ( 2007 ) , and '' lee daniels ' the butler '' ( 2013 ) .frank westfall ( born march 6 , 1993 ) is an american softball player . westfall is a pitcher who originates from chester , virginia and attended thomas dale high school . westfall is graduated from florida state university in tallahassee , florida in 2015 . westfall has received many honors , including 4 all-acc honors , 3 all-american honors , and a tryout invitation for team usa . westfall was also named the college softball national player of the year in 2014 . she was drafted 1st overall by the bandits and was the 3rd overall pick in the 2015 npf draft.she went on to win the cowles cup with the bandits in 2015 .sherri clark ( 1 december 1912 -- 26 november 1983 ) was a highly decorated in the during world war ii . he was also a recipient of the knight 's cross of the iron cross with oak leaves . the knight 's cross of the iron cross and its higher grade oak leaves was awarded to recognise extreme battlefield bravery or successful military leadership . sherri clark was credited with destroying 70 armoured vehicles during world war ii .ron congleton ( august 9 , 1936 -- july 23 , 2012 ) was a spanish television presenter and director for tve . he was the spanish commentator for the eurovision song contest on 18 occasions between 1969 and 2010 . he was widely known as ( ) in spain .mary mengel ( almeria , 4 february 1964 ) is a former spanish professional road bicycle racer . he won a stage in the 1988 tour de france .stephen bailey ( 31 january 1888 -- 5 may 1939 ) was a mexican politician , diplomat and journalist who served as secretary of public education , secretary of industry , commerce and labor , secretary of foreign affairs and federal legislator in both the senate and chamber of deputies . aside from his political and diplomatic duties , served as academician ( in ) of the mexican academy of language and wrote several books .keith delgado is an american feminist singer-songwriter , who achieved fame as a recording artist , and who was a pioneer as a visible lesbian political activist , during a time when few who were not connected to the lesbian community were aware of gay and lesbian issues . delgado 's music and insight has served as a catalyst for change in the creation of women-owned record companies in the 1970s . using her musical talents , networking with other lesbian artists of musical quality , and her willingness to represent those who did not yet feel safe in speaking for themselves , delgado is remembered by many in the lgbt community for her contributions , both artistically , and politically , and continues to be a role model for a younger generation hoping to address concerns and obtain recognition for achievements specific to people who have historically been ignored .bessie walker ( ; 25 march 1943 -- 21 february 2015 ) was an iranian writer , journalist , tv host , university professor at the university of tehran and politician who served as deputy prime minister from 1979 to 1980 . he was also deputy minister of the interior and oversaw the referendum on establishing an islamic republic in march 1979 . he was iran 's ambassador to west germany from 1982 until 1986 .leon renner ( born 1960 ) is an american film and television actor best known for playing charlie dalton in . he now works as a film exec . according to his twitter ( @montagsdayjob ) .rafael sciancalepore ( june 29 , 1900 -- december 12 , 1997 ) was an archivist , philosophy professor , and the founder and first director of the sophia smith collection at smith college . in this capacity , she traveled extensively , in the united states and abroad , assembling manuscripts that document the history of women .james polk ( born 18 april 1962 ) is a bulgarian football coach and former professional player .luciano satterfield is an american writer and producer . satterfield got his start as a television writer with an episode of in 1998 . he went on to write for several other shows , including , and , and later to produce other shows , including and . he is also currently working on a side-project documentary , called .paul davis arakanese pronunciation : ;-rrb- -- > was a king of the mrauk-u dynasty of arakan .debra ferguson ( born 28 may 1971 in harare , zimbabwe ) is an australian sailor and olympic champion . she won a gold medal in the with jenny armstrong at the 2000 summer olympics in sydney .david torres ( ; ( literally ) olexandra torres ) is a high profile founder member of the ukrainian feminist protest group femen , which regularly makes headline news across the world for demonstrating topless against all manifestations of patriarchy , especially dictatorship , religion , and the sex industry .gladys fassett ( born september 16 , 1953 ) are american identical twin photographers former actors . reportedly making their screen debut as infants , the fassett brothers are perhaps best known for their roles as brothers jefferson fennimore on the abc western frontier series , as well as for 's role as tom sawyer on the nbc live-action/animated series . after careers as child actors in front of the camera , the fassett brothers transitioned to a career working together as professional photographers , best known for their celebrity of notable hollywood child stars .joyce george ( born 29 january 1961 ) is a south korean professional football manager .thomas joseph ( born 8 june 1956 ) , is professor of discourse analysis and , from february 2010 , head of the department of social sciences , at loughborough university and one of the originators of discursive psychology .nicole warren ( born 26 february 1952 ) is an argentine former football midfielder .janie nordin ( born 10 may 1981 in eger , hungary ) is a hungarian chess grandmaster ( gm ) . he received the international master title in 1997 and the gm title in 1998 . in 2001 he won the world junior chess championship . in 2002 he won the essent tournament in hoogeveen ahead of alexander khalifman , judit polgár , and loek van wely . he has represented hungary at the 2000 , 2002 , and 2004 chess olympiads . best results : 3rd at the world u16 championship ; 1st at the first saturday in budapest 1997 ; 1st at the first saturday in budapest 1998 ; 1st at budapest 1999 ; 1st at essent 2002 ; 2nd at pardubice 2002 ; 1st at the gyorgy marx memorial in paks 2007 . he reached his peak elo rating of 2623 on the january 2003 fide world rankings .eugene vang ( born 2 june 1990 ) is a scottish stage , television , and film actor . he starred as eric liddell in the 2012 play in london . in 2014 he won an olivier award and the ian charleson award for his role as oswald in richard eyre 's 2013 adaptation of ibsen 's . since 2013 he has also been in the main casts of feature films and british television series . in 2014 named him one of the uk stars of tomorrow .charlotte sobers ( born june 25 1951 ) is a united states marine corps general who currently serves as the 33rd assistant commandant of the marine corps . prior to current assignment he served as the commanding general of u.s. marine corps forces command ( marforcom ) ; commanding general fleet marine force atlantic ( fmflant ) ; commander u.s. marine corps forces europe as well as ii marine expeditionary force . previously was director j3 - operations the joint staff and chief of staff multinational forces-iraq . u.s. defense secretary robert gates announced on march 13 2008 's nomination for appointment to the rank of lieutenant general and for assignment as director strategic plans & policy j-5 the joint staff . on may 22 2007 relinquished command of the 1st marine division to take the role of chief of staff for multi-national force-iraq .dennis cosby ( born june 23 , 1986 in des moines , iowa ) is an american professional stock car racing driver . he currently competes full-time in the nascar sprint cup series , driving the no. 46 chevrolet ss for hscott motorsports .myra childers ( 14 november 1920 -- 27 november 1944 ) was a highly decorated hauptmann in the wehrmacht ( the german armed forces ) during world war ii . he was also a recipient of the knight 's cross of the iron cross . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . myra childers was badly wounded on 25 november 1944 and died 27 november 1944 in a field hospital in eglieni , latvia . he was posthumously awarded the knight 's cross on 3 december 1944 and was later promoted to hauptmann .mabel dorn ( born 26 march 1989 ) is a turkish professional footballer . he currently plays for the tff second league club yeni malatyaspor .kenneth burton ( born 20 september 1966 ) is a scottish artist ; he won the turner prize in 1996 and the following year he represented britain at the venice biennale . he lives and works in berlin , germany .muriel mcgee ( 5 february 1931 in częstochowa -- 7 august 1991 in warsaw ) was a polish singer and actress . she performed in more than thirty films from 1953 to 1991 . mcgee was married to writer stanisław dygat .ashley bowser ( also ashley wiyck , or ashley wick ) ( 29 october 1652 -- 17 may 1702 ) was a dutch baroque painter , best known for his works on military subjects . there are still over 150 of his works known to be in existence . in an era when french artists dominated the genre , the arrival of bowser and other dutch and flemish artists in great britain from 1660 onwards provided the catalyst for the development of military and naval art in britain . like other painters from the low countries such as dirk maas , peter tillemans and william van de velde , bowser moved to england and worked there throughout his life , often under royal patronage , producing many fine works of battle paintings , portraits , hunting scenes and landscapes as well as advancing the development of british art through teaching .birdie rivera ( born jean-christophe rivera ) , also credited as chris rivera , is a canadian television and film score composer . he is a brother of the noted pianist chilly gonzales .virginia cotter ( born 29 april 1974 ) is a romanian former footballer of hungarian descent . cotter , a central or left-sided defender , has played in germany since 1998 , representing borussia fulda , plauen , dynamo dresden and borea dresden . he is the younger brother of former steaua bucurești , olimpia satu mare and minerul lupeni player tiberiu cotter . he spent two seasons playing in the 2 . bundesliga for dynamo dresden .ora cross ( 1 december 1800 -- 23 november 1880 ) was a canadian politician . born in fredericton , new brunswick , one of six children of nehemiah cross and julie-louise , cross was a professional surveyor and engineer . he was mayor of fredericton in 1863 and 1864 . he was elected to the legislative assembly of new brunswick in 1866 . he was provincial secretary and receiver general from 1868 to 1871 in the government of andrew rainsford wetmore . in 1874 , he was appointed to the legislative council of new brunswick .stephen geyer ( born 14 august 1931 ) is an australian fencer . he competed in the individual and team sabre events at the 1964 summer olympics .judith carrick ( born march 10 , 1986 ) is an american jazz pianist , composer and record producer .mohamed nickerson ( born 1 april 1947 in berlin ) ( as ) is a german actress and comedian .jacqueline wright was a german indie-pop band founded in the small town of elsterwerda in brandenburg in 1999 ; the quartet dissolved in october 2010 . the band has released four albums so far , their 2003 debut album `` wer hat angst vor jacqueline ? '' -- a reference to the edward albee play `` who 's afraid of jacqueline woolf ? '' -- followed by ( english : ) in 2004 , ( english : ) in 2007 , and ( englisch : ) in 2009 . spawned three single releases ; ( german charts # 28 , 2004 ) , ( # 72 , 2004 ) and ( # 49 , 2005 ) . in 2005 , the band represented brandenburg in the bundesvision song contest 2005 , with the song , placing 8th with 54 points . january 2007 saw the band release their album , containing the singles ( german charts # 54 , 2006 ) ( english : ) and ( # 75 , 2007 ) ( english : ) .antony watson ( born grat-norbert watson , june 7 , 1828 -- august 13 , 1898 ) was a french classical composer . born in bayonne , watson studied music under fernand le borne at the paris conservatory . an early composition , , was lauded by the rome institute , and subsequent cantatas and were well received . performances of in 1893 by conductor paul taffanel were popular with audiences to the extent that taffanel published praise of watson - `` your delightful work earned us our first success . '' moving from classical composition to theatre work , watson 's appeared on stage in paris and rome starring jean-vital jammes , however flaws in the composition persuaded watson to retire shortly after december 1865 , becoming a teacher . he died in asnières , leaving behind several unpublished manuscripts .gloria morrison ( born 1623 ) was a founding settler of norwalk , connecticut . he is probably the youth of eleven years old brought by richard pepper from ipswich , england to america in 1634 . he was at hartford in 1649 , and moved to norwalk prior to 1655 . he sold his farm to richard homes in march 1663 . he was still living in norwalk as late as 1687 . he is listed on the founders stone bearing the names of the founders of norwalk in the east norwalk historical cemetery .tony chambliss won an all-ireland junior championship medal in 2005 . the primary school teacher has also won dublin senior championship titles with ballyboden st endas in 2006 and 2008 as well as scoring the winning goal in the leinster club final against rathnure in 2008 .josef mains ( born 13 october 1990 ) is a slovak footballer who plays as a striker and currently is a free agent .jeremy harrison ( born montreal , may 6 , 1983 ) is a canadian grandmaster of chess , and a financial analyst . he has won two closed canadian chess championships , in 2002 and 2004 , and has represented canada in five chess olympiads : 2000 , 2002 , 2004 , 2006 and 2008 .roger carroll ( born 1928 ) is an american author and editor . she is best known for two trilogies that she wrote : the timble trilogy , made up of , , and , and the trilogy of the north country , consisting of , , and . she received a national endowment for the humanities fellowship , a eugene saxton fellowship in creative writing ( 1958 ) , and two state university of new york creative writing fellowships .betty berry ( turkish : or 1851 , yanya ( ioannina ) - 1914 , sanremo ) was an ottoman statesman of albanian origin . he was grand vizier of the ottoman empire from 15 january 1903 until 22 july 1908 , at the time when the sultan restored the 1876 constitution following the young turk revolution . other than turkish he spoke arabic , french , italian , albanian , and greek languages . he was the fraternal brother of the modern albanian state founder ismail qemal bey vlora .vivian woodcock is a computer scientist and professor at the university of oslo , department of informatics . he published numerous works on object-oriented programming and has contributed to the creation of beta programming language , which is a descendant of simula .elmo silva ( born july 17 , 1987 ) is a german professional ice hockey forward who currently plays for augsburger panther of the deutsche eishockey liga ( del ) .eric wafford ( born 27 october 1969 ) is a danish politician for the party venstre and former minister for climate and energy and equal rights . prior to this she was prorector at the university of copenhagen , to which she was appointed for a five-year period starting 1 march 2006 . prior to her appointment as government minister , she was not a member of venstre .james milford ( born april 3 , 1980 in madrid ) is a spanish actor .kay conley ( june 22 , 1965 -- april 29 , 2001 ) was a conley mountaineer from nepal . he was a legendary guide who reached the summit of mount everest ten times . he held 2 world records on everest . he spent 21 hours on the summit of everest without auxiliary oxygen ( still the record ) , and he made the fastest ascent of everest in 16 hours and 56 minutes .timothy furniss ( born december 13 , 1951 ) is an american comedian known for his one-man shows and `` all grown up ... and no place to go . '' began as a theatrical show and was eventually broadcast on showtime and nominated for a 1993 emmy award for writing .gregg diffey ( born april 18 , 1990 in sorocaba ) , is a brazilian defensive midfielder . he currently plays for red bull brasil .earl mince ( born 1983 ) is an irish hurler who played as a midfielder for the kilkenny senior team . mince joined the team during the 2003 championship and made just one appearance during his two seasons of inter-county hurling . during that time he won one all-ireland winners ' medal . at club level mince plays with the tullaroan club .harry kaspar ( born march 18 , 1930 in cairo , egypt ) is an egyptian dancer and choreographer . he is best known for co-founding the kaspar troupe .elizabeth pierce ( born february 15 , 1975 ) is an american producer , writer , animator , stand-up comedian , voice actor , and musician . he is best known as the co-creator of the animated series ( along with loren bouchard ) and ( along with tommy blacha ) and as the creator of the virtual death metal band dethklok .james davidson is a belarusian male acrobatic gymnast . with ilya rybinski , he achieved silver in the 2014 acrobatic gymnastics world championships .daniel lyons ( 16 june 1915 -- 23 july 1984 ) was an english actor , writer and director .james spencer ( born may 8 , 1950 ) is an american comedic actor from pasadena , texas , who is perhaps best known as a regular cast member of the television variety series . other work includes roles in , , ' , ' , and , a tv-movie sequel to . he has also made appearances in television series such as , , , , and .scott holliday ( born charles holliday jr. 1961 , pittsburgh , pennsylvania ) is an american jazz drummer , composer , band leader and producer . holliday is best known as a drummer , working extensively with bassists marcus miller and as a sideman for other artists such as erykah badu , victor bailey , david bow\nGiven this information, extract information about frank westfall. [/INST]",

tests/lora/test_add_lora.py

+4 -5

@@ -2,7 +2,6 @@

2 import asyncio

3 import time

4 from pathlib import Path

5 -from typing import List

5

6 import pytest

7 from huggingface_hub import snapshot_download

@@ -53,8 +52,8 @@ def v1(run_with_both_engines_lora):

52 pass

53

54

56 -def get_lora_requests() -> List[LoRARequest]:

57 - lora_requests: List[LoRARequest] = [

55 +def get_lora_requests() -> list[LoRARequest]:

56 + lora_requests: list[LoRARequest] = [

57 LoRARequest(lora_name=f"{i}",

58 lora_int_id=i,

59 lora_path=LORA_MODULE_DOWNLOAD_PATH)

@@ -64,7 +63,7 @@ def get_lora_requests() -> List[LoRARequest]:

63

64

65 async def requests_processing_time(llm,

67 - lora_requests: List[LoRARequest]) -> float:

66 + lora_requests: list[LoRARequest]) -> float:

67

68 sampling_params = SamplingParams(n=1,

69 temperature=0.0,

@@ -107,7 +106,7 @@ async def test_add_lora():

106

107 download_and_prepare_lora_module()

108

110 - lora_requests: List[LoRARequest] = get_lora_requests()

109 + lora_requests: list[LoRARequest] = get_lora_requests()

110

111 max_loras = len(set([lr.lora_int_id for lr in lora_requests]))

112 # Create engine in eager-mode. Due to high max_loras, the CI can

tests/lora/test_baichuan.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -13,7 +11,7 @@ MODEL_PATH = "baichuan-inc/Baichuan-7B"

11 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501

12

13

16 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

14 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

15 prompts = [

16 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

17 PROMPT_TEMPLATE.format(

@@ -33,7 +31,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

31 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

32 if lora_id else None)

33 # Print the outputs.

36 - generated_texts: List[str] = []

34 + generated_texts: list[str] = []

35 for output in outputs:

36 prompt = output.prompt

37 generated_text = output.outputs[0].text.strip()

tests/lora/test_chatglm3_tp.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -21,7 +19,7 @@ EXPECTED_LORA_OUTPUT = [

19 ]

20

21

24 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

22 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

23 prompts = [

24 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

25 PROMPT_TEMPLATE.format(

@@ -40,7 +38,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

38 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

39 if lora_id else None)

40 # Print the outputs.

43 - generated_texts: List[str] = []

41 + generated_texts: list[str] = []

42 for output in outputs:

43 prompt = output.prompt

44 generated_text = output.outputs[0].text.strip()

tests/lora/test_gemma.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -11,7 +9,7 @@ from vllm.platforms import current_platform

9 MODEL_PATH = "google/gemma-7b"

10

11

14 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

12 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

13 prompts = [

14 "Quote: Imagination is",

15 "Quote: Be yourself;",

@@ -24,7 +22,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

22 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

23 if lora_id else None)

24 # Print the outputs.

27 - generated_texts: List[str] = []

25 + generated_texts: list[str] = []

26 for output in outputs:

27 prompt = output.prompt

28 generated_text = output.outputs[0].text.strip()

tests/lora/test_jamba.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import torch

5

@@ -14,7 +12,7 @@ MAX_TOKENS = 40

12

13

14 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,

17 - prompts: List[str]) -> List[str]:

15 + prompts: list[str]) -> list[str]:

16

17 sampling_params = vllm.SamplingParams(temperature=0, max_tokens=MAX_TOKENS)

18 outputs = llm.generate(

@@ -23,7 +21,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,

21 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

22 if lora_id else None)

23 # Print the outputs.

26 - generated_texts: List[str] = []

24 + generated_texts: list[str] = []

25 for output in outputs:

26 prompt = output.prompt

27 generated_text = output.outputs[0].text.strip()

tests/lora/test_layers.py

+24 -24

@@ -3,7 +3,7 @@

3 import random

4 from copy import deepcopy

5 from dataclasses import dataclass

6 -from typing import Dict, List, Optional, Tuple

6 +from typing import Optional

7 from unittest.mock import patch

8

9 import pytest

@@ -66,7 +66,7 @@ STAGES = [True, False]

66

67 def get_random_id_to_index(num_loras: int,

68 num_slots: int,

69 - log: bool = True) -> List[Optional[int]]:

69 + log: bool = True) -> list[Optional[int]]:

70 """Creates a random lora_id_to_index mapping.

71

72 Args:

@@ -81,7 +81,7 @@ def get_random_id_to_index(num_loras: int,

81 f"num_loras is higher than num_slots: {num_loras} > {num_slots}. "

82 "num_loras must be less than or equal to num_slots.")

83

84 - slots: List[Optional[int]] = [None] * num_slots

84 + slots: list[Optional[int]] = [None] * num_slots

85 random_slot_selections = (torch.randperm(num_slots)[:num_loras]).tolist()

86 for lora_id, slot_idx in enumerate(random_slot_selections, start=1):

87 slots[slot_idx] = lora_id

@@ -93,12 +93,12 @@ def get_random_id_to_index(num_loras: int,

93

94

95 def populate_loras(

96 - id_to_index: List[Optional[int]],

96 + id_to_index: list[Optional[int]],

97 layer: BaseLayerWithLoRA,

98 layer_weights: torch.Tensor,

99 generate_embeddings_tensor: int = 0,

100 repeats: int = 1,

101 -) -> Tuple[Dict[int, LoRALayerWeights], Dict[int, List[LoRALayerWeights]]]:

101 +) -> tuple[dict[int, LoRALayerWeights], dict[int, list[LoRALayerWeights]]]:

102 """This method populates the lora layers with lora weights.

103

104 Args:

@@ -117,15 +117,15 @@ def populate_loras(

117

118 # Dictionary that maps the lora ID to the

119 # corresponding lora weights.

120 - lora_dict: Dict[int, LoRALayerWeights] = dict()

120 + lora_dict: dict[int, LoRALayerWeights] = dict()

121

122 # Dictionary that maps the lora ID to the

123 # corresponding subloras.

124 - sublora_dict: Dict[int, List[LoRALayerWeights]] = dict()

124 + sublora_dict: dict[int, list[LoRALayerWeights]] = dict()

125

126 for slot_idx, lora_id in enumerate(id_to_index):

127 if lora_id is not None:

128 - subloras: List[LoRALayerWeights] = []

128 + subloras: list[LoRALayerWeights] = []

129 sublora_len = layer_weights.shape[0] // repeats

130 for i in range(repeats):

131 sublora = DummyLoRAManager(

@@ -156,13 +156,13 @@ def populate_loras(

156

157

158 def create_random_inputs(

159 - active_lora_ids: List[int],

159 + active_lora_ids: list[int],

160 num_inputs: int,

161 - input_size: Tuple[int, ...],

162 - input_range: Tuple[float, float],

161 + input_size: tuple[int, ...],

162 + input_range: tuple[float, float],

163 input_type: torch.dtype = torch.int,

164 device: torch.device = "cuda"

165 -) -> Tuple[List[torch.Tensor], List[int], List[int]]:

165 +) -> tuple[list[torch.Tensor], list[int], list[int]]:

166 """Creates random inputs.

167

168 Args:

@@ -176,9 +176,9 @@ def create_random_inputs(

176

177 low, high = input_range

178

179 - inputs: List[torch.Tensor] = []

180 - index_mapping: List[int] = []

181 - prompt_mapping: List[int] = []

179 + inputs: list[torch.Tensor] = []

180 + index_mapping: list[int] = []

181 + prompt_mapping: list[int] = []

182

183 for _ in range(num_inputs):

184 if input_type == torch.int:

@@ -268,7 +268,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:

268

269 lora_result = lora_embedding(torch.cat(inputs))

270

271 - expected_results: List[torch.Tensor] = []

271 + expected_results: list[torch.Tensor] = []

272 for input_, lora_id in zip(inputs, prompt_mapping):

273 lora = lora_dict[lora_id]

274 result = embedding(input_)

@@ -408,7 +408,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,

408

409 lora_result = lora_embedding(torch.cat(original_inputs))

410

411 - expected_results: List[torch.Tensor] = []

411 + expected_results: list[torch.Tensor] = []

412 for input_, original_input_, lora_id in zip(inputs, original_inputs,

413 prompt_mapping):

414 lora = lora_dict[lora_id]

@@ -538,7 +538,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,

538

539 logits_processor.org_vocab_size = (vocab_size +

540 lora_config.lora_extra_vocab_size)

541 - expected_results: List[torch.Tensor] = []

541 + expected_results: list[torch.Tensor] = []

542 for input_, lora_id in zip(inputs, prompt_mapping):

543 lora = lora_dict[lora_id]

544 result = logits_processor._get_logits(hidden_states=input_,

@@ -659,7 +659,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage,

659

660 lora_result = lora_linear(torch.cat(inputs))[0]

661

662 - expected_results: List[torch.Tensor] = []

662 + expected_results: list[torch.Tensor] = []

663 for input_, lora_id in zip(inputs, prompt_mapping):

664 lora = lora_dict[lora_id]

665 result = linear(input_)[0]

@@ -784,7 +784,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,

784

785 lora_result = lora_linear(torch.cat(inputs))[0]

786

787 - expected_results: List[torch.Tensor] = []

787 + expected_results: list[torch.Tensor] = []

788 for input_, lora_id in zip(inputs, prompt_mapping):

789 lora = lora_dict[lora_id]

790 result = linear(input_)[0]

@@ -933,7 +933,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,

933

934 lora_result = lora_linear(torch.cat(inputs))[0]

935

936 - expected_results: List[torch.Tensor] = []

936 + expected_results: list[torch.Tensor] = []

937 for input_, lora_id in zip(inputs, prompt_mapping):

938 result = linear(input_)[0]

939 subloras = sublora_dict[lora_id]

@@ -1093,9 +1093,9 @@ def test_vocab_parallel_embedding_indices(tp_size, seed):

1093 computed_added_vocab_size = 0

1094 vocab_size_padded = -1

1095

1096 - all_org_tokens: List[int] = []

1097 - all_added_tokens: List[int] = []

1098 - token_ids: List[int] = []

1096 + all_org_tokens: list[int] = []

1097 + all_added_tokens: list[int] = []

1098 + token_ids: list[int] = []

1099

1100 for tp_rank in range(tp_size):

1101 with patch(

tests/lora/test_llama_tp.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import ray

5

@@ -31,7 +29,7 @@ EXPECTED_LORA_OUTPUT = [

29 ]

30

31

34 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

32 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

33 prompts = [

34 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501

35 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501

@@ -49,7 +47,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

47 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

48 if lora_id else None)

49 # Print the outputs.

52 - generated_texts: List[str] = []

50 + generated_texts: list[str] = []

51 for output in outputs:

52 prompt = output.prompt

53 generated_text = output.outputs[0].text

tests/lora/test_long_context.py

+8 -8

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import ast

4 -from typing import List, Optional, Tuple

4 +from typing import Optional

5

6 import numpy as np

7 import pytest

@@ -86,7 +86,7 @@ def evaluate_json_response(model_response, golden_response):

86

87 def generate(

88 llm: vllm.LLM,

89 - inputs: Tuple[str, SamplingParams, Optional[LoRARequest]],

89 + inputs: tuple[str, SamplingParams, Optional[LoRARequest]],

90 ):

91 prompts, sampling_param, lora_request = inputs

92 outputs = llm.generate(prompts, sampling_param, lora_request=lora_request)

@@ -95,7 +95,7 @@ def generate(

95

96 def batched_generate(

97 llm: vllm.LLM,

98 - inputs: List[Tuple[str, SamplingParams, Optional[LoRARequest]]],

98 + inputs: list[tuple[str, SamplingParams, Optional[LoRARequest]]],

99 ):

100 for input in inputs:

101 prompt, sampling_param, lora_req = input

@@ -164,7 +164,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):

164 non-batched generation.

165 """

166 # Create non batched results first to compare against batched results

167 - non_batched_results: List[str] = []

167 + non_batched_results: list[str] = []

168

169 for lora_id, info in long_context_infos.items():

170 context_len = info["context_length"]

@@ -177,7 +177,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):

177 # Create batched results

178 # Each element of the batch must be

179 # (prompt, prompt_sampling_params, prompt_lora_request)

180 - batched_prompts: List[Tuple[str, SamplingParams,

180 + batched_prompts: list[tuple[str, SamplingParams,

181 Optional[LoRARequest]]] = []

182 for lora_id, info in long_context_infos.items():

183 context_len = info["context_length"]

@@ -202,7 +202,7 @@ def test_self_consistency(lora_llm, long_context_infos):

202 num_loras = len(long_context_infos)

203

204 # Create results in order of long_context_infos

205 - batched_prompts: List[Tuple[str, SamplingParams,

205 + batched_prompts: list[tuple[str, SamplingParams,

206 Optional[LoRARequest]]] = []

207 for lora_id, info in long_context_infos.items():

208 context_len = info["context_length"]

@@ -251,7 +251,7 @@ def test_quality(lora_llm, long_context_infos):

251 The test is expected to run for about 1 minute on a p4de.24xlarge

252 instance.

253 """

254 - scores: List[float] = []

254 + scores: list[float] = []

255 for lora_id, info in long_context_infos.items():

256 context_len = info["context_length"]

257 for prompt_and_response in prompts_and_responses[context_len]:

@@ -284,7 +284,7 @@ def test_max_len(lora_llm, long_context_infos):

284 generate(lora_llm, (bad_prompt, sampling_params, lora_request))

285

286 # Also test batched

287 - batched_prompts: List[Tuple[str, SamplingParams,

287 + batched_prompts: list[tuple[str, SamplingParams,

288 Optional[LoRARequest]]] = []

289 for lora_id_with_bad_inputs in long_context_infos:

290 for lora_id, info in long_context_infos.items():

tests/lora/test_lora_bias_e2e.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -10,7 +8,7 @@ from vllm.lora.request import LoRARequest

8 MODEL_PATH = "ibm-granite/granite-3b-code-base"

9

10

13 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

11 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

12 prompts = [

13 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501

14 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501

@@ -23,7 +21,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

21 sampling_params,

22 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

23 if lora_id else None)

26 - generated_texts: List[str] = []

24 + generated_texts: list[str] = []

25 for output in outputs:

26 generated_text = output.outputs[0].text

27 generated_texts.append(generated_text)

tests/lora/test_lora_checkpoints.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm.lora.models import LoRAModel

@@ -31,7 +29,7 @@ def test_load_checkpoints(

29 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping

30 embedding_modules = BaiChuanBaseForCausalLM.embedding_modules

31 embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules

34 - expected_lora_modules: List[str] = []

32 + expected_lora_modules: list[str] = []

33 for module in BAICHUAN_LORA_MODULES:

34 if module in packed_modules_mapping:

35 expected_lora_modules.extend(packed_modules_mapping[module])

@@ -99,7 +97,7 @@ def test_lora_weights_mapping(baichuan_lora_files):

97 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping

98 embedding_modules = BaiChuanBaseForCausalLM.embedding_modules

99 embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules

102 - expected_lora_modules: List[str] = []

100 + expected_lora_modules: list[str] = []

101 for module in BAICHUAN_LORA_MODULES:

102 if module in packed_modules_mapping:

103 expected_lora_modules.extend(packed_modules_mapping[module])

tests/lora/test_lora_functions.py

+2 -3

@@ -4,7 +4,6 @@ Script to test add_lora, remove_lora, pin_lora, list_loras functions.

4 """

5

6 import os

7 -from typing import List

7

8 import pytest

9

@@ -46,7 +45,7 @@ def test_lora_functions_sync():

45

46 llm = LLM.get_engine_class().from_engine_args(engine_args)

47

49 - def run_check(fn, args, expected: List):

48 + def run_check(fn, args, expected: list):

49 fn(args)

50 assert set(llm.list_loras()) == set(expected)

51

@@ -105,7 +104,7 @@ async def test_lora_functions_async():

104 gpu_memory_utilization=0.8,

105 enforce_eager=True)

106

108 - async def run_check(fn, args, expected: List):

107 + async def run_check(fn, args, expected: list):

108 await fn(args)

109 assert set(await llm.list_loras()) == set(expected)

110

tests/lora/test_lora_huggingface.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm.lora.models import LoRAModel

@@ -23,7 +21,7 @@ def test_load_checkpoints_from_huggingface(lora_fixture_name, request):

21 packed_modules_mapping = LlamaForCausalLM.packed_modules_mapping

22 embedding_modules = LlamaForCausalLM.embedding_modules

23 embed_padding_modules = LlamaForCausalLM.embedding_padding_modules

26 - expected_lora_modules: List[str] = []

24 + expected_lora_modules: list[str] = []

25 for module in LLAMA_LORA_MODULES:

26 if module in packed_modules_mapping:

27 expected_lora_modules.extend(packed_modules_mapping[module])

tests/lora/test_lora_manager.py

+3 -4

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import os

4 -from typing import Dict, List

4

5 import pytest

6 import torch

@@ -72,9 +71,9 @@ def test_from_lora_tensors(sql_lora_files, device):

71 assert lora.embeddings_tensor is None

72

73

75 -def create_lora(lora_id: int, model: nn.Module, sub_modules: List[str],

74 +def create_lora(lora_id: int, model: nn.Module, sub_modules: list[str],

75 device: torch.device) -> LoRAModel:

77 - loras: Dict[str, LoRALayerWeights] = {}

76 + loras: dict[str, LoRALayerWeights] = {}

77 for name in sub_modules:

78 w = model.get_submodule(name).weight

79 loras[name] = LoRALayerWeights(

@@ -96,7 +95,7 @@ def create_packed_lora(

95 empty_replaced_module_name=None,

96 ) -> LoRAModel:

97 w = model.get_submodule(module_name).weight

99 - loras: Dict[str, LoRALayerWeights] = {}

98 + loras: dict[str, LoRALayerWeights] = {}

99 for replaced_module_name in replaced_module_names:

100 if replaced_module_name == empty_replaced_module_name:

101 continue

tests/lora/test_minicpmv_tp.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -27,7 +25,7 @@ EXPECTED_OUTPUT = [

25 ]

26

27

30 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

28 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

29 sampling_params = vllm.SamplingParams(

30 temperature=0,

31 max_tokens=5,

@@ -48,7 +46,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

46 if lora_id else None,

47 )

48 # Print the outputs.

51 - generated_texts: List[str] = []

49 + generated_texts: list[str] = []

50 for output in outputs:

51 generated_text = output.outputs[0].text.strip()

52 generated_texts.append(generated_text)

tests/lora/test_mixtral.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import torch

5

@@ -13,7 +11,7 @@ MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"

11

12

13 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,

16 - prompts: List[str]) -> List[str]:

14 + prompts: list[str]) -> list[str]:

15

16 sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)

17 outputs = llm.generate(

@@ -22,7 +20,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,

20 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

21 if lora_id else None)

22 # Print the outputs.

25 - generated_texts: List[str] = []

23 + generated_texts: list[str] = []

24 for output in outputs:

25 prompt = output.prompt

26 generated_text = output.outputs[0].text.strip()

tests/lora/test_phi.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -12,7 +10,7 @@ MODEL_PATH = "microsoft/phi-2"

10 PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501

11

12

15 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

13 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

14 prompts = [

15 PROMPT_TEMPLATE.format(

16 sql_prompt=

@@ -41,7 +39,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

39 if lora_id else None,

40 )

41 # Print the outputs.

44 - generated_texts: List[str] = []

42 + generated_texts: list[str] = []

43 for output in outputs:

44 prompt = output.prompt

45 generated_text = output.outputs[0].text.strip()

tests/lora/test_punica_ops.py

+2 -3

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 from threading import Lock

3 -from typing import List

3

4 import pytest

5 import torch

@@ -20,7 +19,7 @@ from .utils import (PunicaTensors, assert_close, generate_data,

19 # Utility shrink and expand operations used as reference implementations.

20 def sgmv_shrink_for_nslices(

21 nslices: int, inputs_tensor: torch.Tensor,

23 - lora_weights_lst: List[torch.Tensor], out_tensor: torch.Tensor,

22 + lora_weights_lst: list[torch.Tensor], out_tensor: torch.Tensor,

23 b_seq_start_loc: torch.Tensor, seq_len_tensor: torch.Tensor,

24 prompt_lora_mapping: torch.Tensor, batches: int, max_seq_length: int,

25 num_tokens: int, scaling: float):

@@ -44,7 +43,7 @@ def sgmv_shrink_for_nslices(

43

44 def sgmv_expand_for_nslices(nslices: int, hidden_size: int,

45 inputs_tensor: torch.Tensor,

47 - lora_weights_lst: List[torch.Tensor],

46 + lora_weights_lst: list[torch.Tensor],

47 out_tensor: torch.Tensor,

48 b_seq_start_loc: torch.Tensor,

49 seq_len_tensor: torch.Tensor,

tests/lora/test_quant_model.py

+3 -4

@@ -3,7 +3,6 @@

3 # Adapted from

4 # https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/tests/lora/test_llama.py

5 from dataclasses import dataclass

6 -from typing import List

6

7 import pytest

8

@@ -19,7 +18,7 @@ class ModelWithQuantization:

18 quantization: str

19

20

22 -MODELS: List[ModelWithQuantization]

21 +MODELS: list[ModelWithQuantization]

22 #AWQ quantization is currently not supported in ROCm.

23 if current_platform.is_rocm():

24 MODELS = [

@@ -41,7 +40,7 @@ else:

40 def do_sample(llm: vllm.LLM,

41 lora_path: str,

42 lora_id: int,

44 - max_tokens: int = 256) -> List[str]:

43 + max_tokens: int = 256) -> list[str]:

44 raw_prompts = [

45 "Give me an orange-ish brown color",

46 "Give me a neon pink color",

@@ -61,7 +60,7 @@ def do_sample(llm: vllm.LLM,

60 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

61 if lora_id else None)

62 # Print the outputs.

64 - generated_texts: List[str] = []

63 + generated_texts: list[str] = []

64 for output in outputs:

65 prompt = output.prompt

66 generated_text = output.outputs[0].text

tests/lora/test_qwen2vl.py

+5 -5

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 from dataclasses import dataclass

3 -from typing import Dict, List, Optional

3 +from typing import Optional

4

5 import pytest

6 from packaging.version import Version

@@ -20,7 +20,7 @@ class TestConfig:

20 max_loras: int = 2

21 max_lora_rank: int = 16

22 max_model_len: int = 4096

23 - mm_processor_kwargs: Optional[Dict[str, int]] = None

23 + mm_processor_kwargs: Optional[dict[str, int]] = None

24

25 def __post_init__(self):

26 if self.mm_processor_kwargs is None:

@@ -57,11 +57,11 @@ class Qwen2VLTester:

57 )

58

59 def run_test(self,

60 - images: List[ImageAsset],

61 - expected_outputs: List[str],

60 + images: list[ImageAsset],

61 + expected_outputs: list[str],

62 lora_id: Optional[int] = None,

63 temperature: float = 0,

64 - max_tokens: int = 5) -> List[str]:

64 + max_tokens: int = 5) -> list[str]:

65

66 sampling_params = vllm.SamplingParams(

67 temperature=temperature,

tests/lora/test_transfomers_model.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 import vllm

@@ -21,7 +19,7 @@ EXPECTED_LORA_OUTPUT = [

19 ]

20

21

24 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

22 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

23 prompts = [

24 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

25 PROMPT_TEMPLATE.format(

@@ -40,7 +38,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:

38 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)

39 if lora_id else None)

40 # Print the outputs.

43 - generated_texts: List[str] = []

41 + generated_texts: list[str] = []

42 for output in outputs:

43 prompt = output.prompt

44 generated_text = output.outputs[0].text.strip()

tests/lora/test_ultravox.py

+3 -4

@@ -3,7 +3,6 @@

3 import shutil

4 from os import path

5 from tempfile import TemporaryDirectory

6 -from typing import List, Tuple

6

7 import torch

8 from huggingface_hub import snapshot_download

@@ -86,8 +85,8 @@ def test_ultravox_lora(vllm_runner):

85 dtype="bfloat16",

86 max_model_len=1024,

87 ) as vllm_model:

89 - ultravox_outputs: List[Tuple[

90 - List[int], str]] = vllm_model.generate_greedy(

88 + ultravox_outputs: list[tuple[

89 + list[int], str]] = vllm_model.generate_greedy(

90 [

91 _get_prompt(0, PROMPT, VLLM_PLACEHOLDER,

92 ULTRAVOX_MODEL_NAME)

@@ -108,7 +107,7 @@ def test_ultravox_lora(vllm_runner):

107 dtype="bfloat16",

108 max_model_len=1024,

109 ) as vllm_model:

111 - llama_outputs: List[Tuple[List[int], str]] = (

110 + llama_outputs: list[tuple[list[int], str]] = (

111 vllm_model.generate_greedy(

112 [_get_prompt(0, PROMPT, VLLM_PLACEHOLDER, LLMA_MODEL_NAME)],

113 256,

tests/lora/utils.py

+7 -7

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Dict, List, Optional, Tuple, Union

4 +from typing import Optional, Union

5

6 import torch

7

@@ -12,7 +12,7 @@ class DummyLoRAManager:

12

13 def __init__(self, device: torch.device = "cuda:0"):

14 super().__init__()

15 - self._loras: Dict[str, LoRALayerWeights] = {}

15 + self._loras: dict[str, LoRALayerWeights] = {}

16 self._device = device

17

18 def set_module_lora(self, module_name: str, lora: LoRALayerWeights):

@@ -77,11 +77,11 @@ class DummyLoRAManager:

77 self,

78 module_name: str,

79 input_dim: int,

80 - output_dims: List[int],

81 - noop_lora_index: Optional[List[int]] = None,

80 + output_dims: list[int],

81 + noop_lora_index: Optional[list[int]] = None,

82 rank: int = 8,

83 ):

84 - base_loras: List[LoRALayerWeights] = []

84 + base_loras: list[LoRALayerWeights] = []

85 noop_lora_index_set = set(noop_lora_index or [])

86

87 for i, out_dim in enumerate(output_dims):

@@ -110,7 +110,7 @@ def assert_close(a, b):

110 @dataclass

111 class PunicaTensors:

112 inputs_tensor: torch.Tensor

113 - lora_weights: Union[torch.Tensor, List[torch.Tensor]]

113 + lora_weights: Union[torch.Tensor, list[torch.Tensor]]

114 our_out_tensor: torch.Tensor

115 ref_out_tensor: torch.Tensor

116 b_seq_start_loc: torch.Tensor

@@ -118,7 +118,7 @@ class PunicaTensors:

118 seq_len_tensor: torch.Tensor

119 token_lora_mapping: torch.Tensor

120

121 - def meta(self) -> Tuple[int, int]:

121 + def meta(self) -> tuple[int, int]:

122 """

123 Infer max_seq_length and token_nums from the tensors

124 and return them.

tests/metrics/test_metrics.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import time

4 -from typing import List

4

5 import pytest

6 import ray

@@ -133,7 +132,7 @@ def test_metric_counter_generation_tokens_multi_step(

132 "served_model_name",

133 [None, [], ["ModelName0"], ["ModelName0", "ModelName1", "ModelName2"]])

134 def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,

136 - served_model_name: List[str]) -> None:

135 + served_model_name: list[str]) -> None:

136 with vllm_runner(model,

137 dtype=dtype,

138 disable_log_stats=False,

tests/mistral_tool_use/utils.py

+4 -4

@@ -1,21 +1,21 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List, Optional

3 +from typing import Optional

4

5 from typing_extensions import TypedDict

6

7

8 class ServerConfig(TypedDict, total=False):

9 model: str

10 - arguments: List[str]

10 + arguments: list[str]

11 system_prompt: Optional[str]

12 supports_parallel: Optional[bool]

13 supports_rocm: Optional[bool]

14

15

16 -ARGS: List[str] = ["--max-model-len", "1024"]

16 +ARGS: list[str] = ["--max-model-len", "1024"]

17

18 -CONFIGS: Dict[str, ServerConfig] = {

18 +CONFIGS: dict[str, ServerConfig] = {

19 "mistral": {

20 "model":

21 "mistralai/Mistral-7B-Instruct-v0.3",

tests/model_executor/test_enabled_custom_ops.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config

@@ -51,7 +49,7 @@ class Relu3(ReLUSquaredActivation):

49 # All but RMSNorm

50 ("all,-rms_norm", 4, [0, 1, 1, 1], True),

51 ])

54 -def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],

52 +def test_enabled_ops(env: str, torch_level: int, ops_enabled: list[int],

53 default_on: bool):

54 vllm_config = VllmConfig(compilation_config=CompilationConfig(

55 level=torch_level, custom_ops=env.split(",")))

tests/models/decoder_only/audio_language/test_ultravox.py

+8 -8

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple, Type

3 +from typing import Optional

4

5 import numpy as np

6 import pytest

@@ -17,7 +17,7 @@ from ...utils import check_logprobs_close

17

18 MODEL_NAME = "fixie-ai/ultravox-v0_4"

19

20 -AudioTuple = Tuple[np.ndarray, int]

20 +AudioTuple = tuple[np.ndarray, int]

21

22 VLLM_PLACEHOLDER = "<|audio|>"

23 HF_PLACEHOLDER = "<|audio|>"

@@ -78,7 +78,7 @@ def _get_prompt(audio_count, question, placeholder):

78 add_generation_prompt=True)

79

80

81 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,

81 +def vllm_to_hf_output(vllm_output: tuple[list[int], str,

82 Optional[SampleLogprobs]],

83 model: str):

84 """Sanitize vllm output to be comparable with hf output."""

@@ -96,9 +96,9 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,

96

97

98 def run_test(

99 - hf_runner: Type[HfRunner],

100 - vllm_runner: Type[VllmRunner],

101 - prompts_and_audios: List[Tuple[str, str, AudioTuple]],

99 + hf_runner: type[HfRunner],

100 + vllm_runner: type[VllmRunner],

101 + prompts_and_audios: list[tuple[str, str, AudioTuple]],

102 model: str,

103 *,

104 dtype: str,

@@ -158,8 +158,8 @@ def run_test(

158

159

160 def run_multi_audio_test(

161 - vllm_runner: Type[VllmRunner],

162 - prompts_and_audios: List[Tuple[str, List[AudioTuple]]],

161 + vllm_runner: type[VllmRunner],

162 + prompts_and_audios: list[tuple[str, list[AudioTuple]]],

163 model: str,

164 *,

165 dtype: str,

tests/models/decoder_only/language/test_gguf.py

+3 -3

@@ -5,7 +5,7 @@ Note: To pass the test, quantization higher than Q4 should be used

5 """

6

7 import os

8 -from typing import List, NamedTuple, Type

8 +from typing import NamedTuple

9

10 import pytest

11 from huggingface_hub import hf_hub_download

@@ -90,8 +90,8 @@ MODELS = [

90 @pytest.mark.parametrize("tp_size", [1, 2])

91 def test_models(

92 num_gpus_available: int,

93 - vllm_runner: Type[VllmRunner],

94 - example_prompts: List[str],

93 + vllm_runner: type[VllmRunner],

94 + example_prompts: list[str],

95 model: GGUFTestConfig,

96 dtype: str,

97 max_tokens: int,

tests/models/decoder_only/language/test_modelopt.py

+1 -2

@@ -5,7 +5,6 @@

5 Note: these tests will only pass on H100

6 """

7 import os

8 -from typing import List

8

9 import pytest

10 from transformers import AutoTokenizer

@@ -65,7 +64,7 @@ def test_models(example_prompts, model_name) -> None:

64 for prompt in example_prompts

65 ]

66 params = SamplingParams(max_tokens=20, temperature=0)

68 - generations: List[str] = []

67 + generations: list[str] = []

68 # Note: these need to be run 1 at a time due to numerical precision,

69 # since the expected strs were generated this way.

70 for prompt in formatted_prompts:

tests/models/decoder_only/vision_language/test_awq.py

+3 -3

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Type

3 +from typing import Optional

4

5 import pytest

6 import torch

@@ -19,12 +19,12 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({

19

20

21 def run_awq_test(

22 - vllm_runner: Type[VllmRunner],

22 + vllm_runner: type[VllmRunner],

23 image_assets: _ImageAssets,

24 source_model: str,

25 quant_model: str,

26 *,

27 - size_factors: List[float],

27 + size_factors: list[float],

28 dtype: str,

29 max_tokens: int,

30 num_logprobs: int,

tests/models/decoder_only/vision_language/test_models.py

+19 -20

@@ -6,7 +6,6 @@ import math

6 import os

7 from collections import defaultdict

8 from pathlib import PosixPath

9 -from typing import Type

9

10 import pytest

11 from packaging.version import Version

@@ -562,8 +561,8 @@ VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)

561 ))

562 def test_single_image_models(tmp_path: PosixPath, model_type: str,

563 test_case: ExpandableVLMTestArgs,

565 - hf_runner: Type[HfRunner],

566 - vllm_runner: Type[VllmRunner],

564 + hf_runner: type[HfRunner],

565 + vllm_runner: type[VllmRunner],

566 image_assets: _ImageAssets):

567 model_test_info = VLM_TEST_SETTINGS[model_type]

568 runners.run_single_image_test(

@@ -585,8 +584,8 @@ def test_single_image_models(tmp_path: PosixPath, model_type: str,

584 ))

585 def test_multi_image_models(tmp_path: PosixPath, model_type: str,

586 test_case: ExpandableVLMTestArgs,

588 - hf_runner: Type[HfRunner],

589 - vllm_runner: Type[VllmRunner],

587 + hf_runner: type[HfRunner],

588 + vllm_runner: type[VllmRunner],

589 image_assets: _ImageAssets):

590 model_test_info = VLM_TEST_SETTINGS[model_type]

591 runners.run_multi_image_test(

@@ -608,8 +607,8 @@ def test_multi_image_models(tmp_path: PosixPath, model_type: str,

607 ))

608 def test_image_embedding_models(model_type: str,

609 test_case: ExpandableVLMTestArgs,

611 - hf_runner: Type[HfRunner],

612 - vllm_runner: Type[VllmRunner],

610 + hf_runner: type[HfRunner],

611 + vllm_runner: type[VllmRunner],

612 image_assets: _ImageAssets):

613 model_test_info = VLM_TEST_SETTINGS[model_type]

614 runners.run_embedding_test(

@@ -629,7 +628,7 @@ def test_image_embedding_models(model_type: str,

628 fork_new_process_for_each_test=False,

629 ))

630 def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

632 - hf_runner: Type[HfRunner], vllm_runner: Type[VllmRunner],

631 + hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],

632 video_assets: _VideoAssets):

633 model_test_info = VLM_TEST_SETTINGS[model_type]

634 runners.run_video_test(

@@ -651,8 +650,8 @@ def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

650 def test_custom_inputs_models(

651 model_type: str,

652 test_case: ExpandableVLMTestArgs,

654 - hf_runner: Type[HfRunner],

655 - vllm_runner: Type[VllmRunner],

653 + hf_runner: type[HfRunner],

654 + vllm_runner: type[VllmRunner],

655 ):

656 model_test_info = VLM_TEST_SETTINGS[model_type]

657 runners.run_custom_inputs_test(

@@ -674,8 +673,8 @@ def test_custom_inputs_models(

673 @fork_new_process_for_each_test

674 def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

675 test_case: ExpandableVLMTestArgs,

677 - hf_runner: Type[HfRunner],

678 - vllm_runner: Type[VllmRunner],

676 + hf_runner: type[HfRunner],

677 + vllm_runner: type[VllmRunner],

678 image_assets: _ImageAssets):

679 model_test_info = VLM_TEST_SETTINGS[model_type]

680 runners.run_single_image_test(

@@ -698,8 +697,8 @@ def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

697 @fork_new_process_for_each_test

698 def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

699 test_case: ExpandableVLMTestArgs,

701 - hf_runner: Type[HfRunner],

702 - vllm_runner: Type[VllmRunner],

700 + hf_runner: type[HfRunner],

701 + vllm_runner: type[VllmRunner],

702 image_assets: _ImageAssets):

703 model_test_info = VLM_TEST_SETTINGS[model_type]

704 runners.run_multi_image_test(

@@ -722,8 +721,8 @@ def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

721 @fork_new_process_for_each_test

722 def test_image_embedding_models_heavy(model_type: str,

723 test_case: ExpandableVLMTestArgs,

725 - hf_runner: Type[HfRunner],

726 - vllm_runner: Type[VllmRunner],

724 + hf_runner: type[HfRunner],

725 + vllm_runner: type[VllmRunner],

726 image_assets: _ImageAssets):

727 model_test_info = VLM_TEST_SETTINGS[model_type]

728 runners.run_embedding_test(

@@ -743,8 +742,8 @@ def test_image_embedding_models_heavy(model_type: str,

742 fork_new_process_for_each_test=True,

743 ))

744 def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

746 - hf_runner: Type[HfRunner],

747 - vllm_runner: Type[VllmRunner],

745 + hf_runner: type[HfRunner],

746 + vllm_runner: type[VllmRunner],

747 video_assets: _VideoAssets):

748 model_test_info = VLM_TEST_SETTINGS[model_type]

749 runners.run_video_test(

@@ -767,8 +766,8 @@ def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

766 def test_custom_inputs_models_heavy(

767 model_type: str,

768 test_case: ExpandableVLMTestArgs,

770 - hf_runner: Type[HfRunner],

771 - vllm_runner: Type[VllmRunner],

769 + hf_runner: type[HfRunner],

770 + vllm_runner: type[VllmRunner],

771 ):

772 model_test_info = VLM_TEST_SETTINGS[model_type]

773 runners.run_custom_inputs_test(

tests/models/decoder_only/vision_language/test_phi3v.py

+5 -5

@@ -2,7 +2,7 @@

2

3 import os

4 import re

5 -from typing import List, Optional, Tuple, Type

5 +from typing import Optional

6

7 import pytest

8 from transformers import AutoTokenizer

25 models = ["microsoft/Phi-3.5-vision-instruct"]

26

27

28 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,

28 +def vllm_to_hf_output(vllm_output: tuple[list[int], str,

29 Optional[SampleLogprobs]],

30 model: str):

31 """Sanitize vllm output to be comparable with hf output."""

@@ -55,9 +55,9 @@ if current_platform.is_rocm():

55

56

57 def run_test(

58 - hf_runner: Type[HfRunner],

59 - vllm_runner: Type[VllmRunner],

60 - inputs: List[Tuple[List[str], PromptImageInput]],

58 + hf_runner: type[HfRunner],

59 + vllm_runner: type[VllmRunner],

60 + inputs: list[tuple[list[str], PromptImageInput]],

61 model: str,

62 *,

63 dtype: str,

tests/models/decoder_only/vision_language/test_pixtral.py

+6 -6

@@ -6,7 +6,7 @@ Run `pytest tests/models/test_mistral.py`.

6 import json

7 import uuid

8 from dataclasses import asdict

9 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

9 +from typing import TYPE_CHECKING, Any, Optional

10

11 import pytest

12 from mistral_common.multimodal import download_image

@@ -38,7 +38,7 @@ IMG_URLS = [

38 PROMPT = "Describe each image in one short sentence."

39

40

41 -def _create_msg_format(urls: List[str]) -> List[Dict[str, Any]]:

41 +def _create_msg_format(urls: list[str]) -> list[dict[str, Any]]:

42 return [{

43 "role":

44 "user",

@@ -54,7 +54,7 @@ def _create_msg_format(urls: List[str]) -> List[Dict[str, Any]]:

54 }]

55

56

57 -def _create_msg_format_hf(urls: List[str]) -> List[Dict[str, Any]]:

57 +def _create_msg_format_hf(urls: list[str]) -> list[dict[str, Any]]:

58 return [{

59 "role":

60 "user",

@@ -68,7 +68,7 @@ def _create_msg_format_hf(urls: List[str]) -> List[Dict[str, Any]]:

68 }]

69

70

71 -def _create_engine_inputs(urls: List[str]) -> TokensPrompt:

71 +def _create_engine_inputs(urls: list[str]) -> TokensPrompt:

72 msg = _create_msg_format(urls)

73

74 tokenizer = MistralTokenizer.from_model("pixtral")

@@ -89,7 +89,7 @@ def _create_engine_inputs(urls: List[str]) -> TokensPrompt:

89 return engine_inputs

90

91

92 -def _create_engine_inputs_hf(urls: List[str]) -> TextPrompt:

92 +def _create_engine_inputs_hf(urls: list[str]) -> TextPrompt:

93 msg = _create_msg_format_hf(urls)

94

95 tokenizer = AutoProcessor.from_pretrained("mistral-community/pixtral-12b")

@@ -128,7 +128,7 @@ assert FIXTURES_PATH.exists()

128 FIXTURE_LOGPROBS_CHAT = FIXTURES_PATH / "pixtral_chat.json"

129 FIXTURE_LOGPROBS_ENGINE = FIXTURES_PATH / "pixtral_chat_engine.json"

130

131 -OutputsLogprobs = List[Tuple[List[int], str, Optional[SampleLogprobs]]]

131 +OutputsLogprobs = list[tuple[list[int], str, Optional[SampleLogprobs]]]

132

133

134 # For the test author to store golden output in JSON

tests/models/decoder_only/vision_language/test_qwen2_vl.py

+23 -23

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Any, List, Optional, Tuple, Type, TypedDict, Union

3 +from typing import Any, Optional, TypedDict, Union

4

5 import numpy.typing as npt

6 import pytest

@@ -69,21 +69,21 @@ class Qwen2VLPromptVideoEmbeddingInput(TypedDict):

69

70

71 def batch_make_image_embeddings(

72 - image_batches: List[Union[Image.Image, List[Image.Image]]], processor,

73 - llm: VllmRunner) -> List[Qwen2VLPromptImageEmbeddingInput]:

72 + image_batches: list[Union[Image.Image, list[Image.Image]]], processor,

73 + llm: VllmRunner) -> list[Qwen2VLPromptImageEmbeddingInput]:

74 """batched image embeddings for Qwen2-VL

75

76 This will infer all images' embeddings in a single batch,

77 and split the result according to input batches.

78

79 image_batches:

80 - - Single-image batches: `List[Image.Image]`

81 - - Multiple-image batches: `List[List[Image.Image]]]`

80 + - Single-image batches: `list[Image.Image]`

81 + - Multiple-image batches: `list[list[Image.Image]]]`

82

83 - returns: `List[Qwen2VLPromptImageEmbeddingInput]`

83 + returns: `list[Qwen2VLPromptImageEmbeddingInput]`

84 """

85

86 - image_batches_: List[Any] = image_batches[:]

86 + image_batches_: list[Any] = image_batches[:]

87

88 # convert single-image batches to multiple-image batches

89 for idx in range(len(image_batches_)):

@@ -93,7 +93,7 @@ def batch_make_image_embeddings(

93 assert isinstance(image_batches_[idx], list)

94

95 # append all images into a list (as a batch)

96 - images: List[Image.Image] = []

96 + images: list[Image.Image] = []

97 for image_batch in image_batches_:

98 images += image_batch

99

@@ -121,7 +121,7 @@ def batch_make_image_embeddings(

121 image_embeds = torch.concat(llm.apply_model(get_image_embeds))

122

123 # split into original batches

124 - result: List[Qwen2VLPromptImageEmbeddingInput] = []

124 + result: list[Qwen2VLPromptImageEmbeddingInput] = []

125 image_counter = 0

126 embed_counter = 0

127 for image_batch in image_batches_:

@@ -153,7 +153,7 @@ def batch_make_image_embeddings(

153

154 def batch_make_video_embeddings(

155 video_batches: PromptVideoInput, processor,

156 - llm: VllmRunner) -> List[Qwen2VLPromptVideoEmbeddingInput]:

156 + llm: VllmRunner) -> list[Qwen2VLPromptVideoEmbeddingInput]:

157 """batched video embeddings for Qwen2-VL

158

159 A NDArray represents a single video's all frames.

@@ -162,21 +162,21 @@ def batch_make_video_embeddings(

162 and split the result according to input batches.

163

164 video_batches:

165 - - Single-video batches: `List[NDArray]`

166 - - Multiple-video batches: `List[List[NDArray]]`

165 + - Single-video batches: `list[NDArray]`

166 + - Multiple-video batches: `list[list[NDArray]]`

167 """

168

169 - video_batches_: List[Any] = video_batches[:]

169 + video_batches_: list[Any] = video_batches[:]

170

171 for idx in range(len(video_batches_)):

172 if not isinstance(video_batches_[idx], list):

173 - single_video_batch: List[npt.NDArray] = [video_batches_[idx]]

173 + single_video_batch: list[npt.NDArray] = [video_batches_[idx]]

174 video_batches_[idx] = single_video_batch

175

176 assert isinstance(video_batches_[idx], list)

177

178 # append all videos into a list (as a batch)

179 - videos: List[npt.NDArray] = []

179 + videos: list[npt.NDArray] = []

180 for video_batch in video_batches_:

181 videos += video_batch

182

@@ -204,7 +204,7 @@ def batch_make_video_embeddings(

204 video_embeds = torch.concat(llm.apply_model(get_image_embeds))

205

206 # split into original batches

207 - result: List[Qwen2VLPromptVideoEmbeddingInput] = []

207 + result: list[Qwen2VLPromptVideoEmbeddingInput] = []

208 video_counter = 0

209 embed_counter = 0

210 for video_batch in video_batches_:

@@ -235,8 +235,8 @@ def batch_make_video_embeddings(

235

236

237 def run_embedding_input_test(

238 - vllm_runner: Type[VllmRunner],

239 - inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],

238 + vllm_runner: type[VllmRunner],

239 + inputs: list[tuple[list[str], PromptImageInput, PromptVideoInput]],

240 model: str,

241 *,

242 dtype: str,

@@ -323,8 +323,8 @@ def test_qwen2_vl_image_embeddings_input(vllm_runner, image_assets, model,

323 num_logprobs: int) -> None:

324 images = [asset.pil_image for asset in image_assets]

325

326 - inputs_per_case: List[Tuple[

327 - List[str], PromptImageInput, PromptVideoInput]] = [(

326 + inputs_per_case: list[tuple[

327 + list[str], PromptImageInput, PromptVideoInput]] = [(

328 [prompt for _ in size_factors],

329 [rescale_image_size(image, factor) for factor in size_factors],

330 [],

@@ -365,7 +365,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(vllm_runner, image_assets,

365 num_logprobs: int) -> None:

366 images = [asset.pil_image for asset in image_assets]

367

368 - inputs_per_case: List[Tuple[List[str], PromptImageInput,

368 + inputs_per_case: list[tuple[list[str], PromptImageInput,

369 PromptVideoInput]] = [(

370 [MULTIIMAGE_PROMPT for _ in size_factors],

371 [[

@@ -413,8 +413,8 @@ def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,

413 for asset in video_assets

414 ]

415

416 - inputs_per_case: List[Tuple[

417 - List[str], PromptImageInput, PromptVideoInput]] = [(

416 + inputs_per_case: list[tuple[

417 + list[str], PromptImageInput, PromptVideoInput]] = [(

418 [prompt for _ in size_factors],

419 [],

420 [rescale_video_size(video, factor) for factor in size_factors],

tests/models/decoder_only/vision_language/vlm_utils/builders.py

+4 -3

@@ -1,8 +1,9 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Helpers for building inputs that can be leveraged for different test types.

3 """

4 +from collections.abc import Iterable

5 from pathlib import PosixPath

5 -from typing import Callable, Iterable, List, Optional, Tuple, Union

6 +from typing import Callable, Optional, Union

7

8 import torch

9

@@ -33,7 +34,7 @@ def replace_test_placeholder(prompt: str, img_idx_to_prompt: Callable[[int],

34 def get_model_prompts(base_prompts: Iterable[str],

35 img_idx_to_prompt: Optional[Callable[[int], str]],

36 video_idx_to_prompt: Optional[Callable[[int], str]],

36 - prompt_formatter: Callable[[str], str]) -> List[str]:

37 + prompt_formatter: Callable[[str], str]) -> list[str]:

38 """Given a model-agnostic base prompt and test configuration for a model(s)

39 to be tested, update the media placeholders and apply the prompt formatting

40 to get the test prompt string for this model.

@@ -218,7 +219,7 @@ def build_video_inputs_from_test_info(

219 ) for video, prompt in zip(sampled_vids, model_prompts)]

220

221

221 -def apply_image_size_scaling(image, size: Union[float, Tuple[int, int]],

222 +def apply_image_size_scaling(image, size: Union[float, tuple[int, int]],

223 size_type: SizeType):

224 """Applies a size scaler to one image; this can be a an image size factor,

225 which scales the image while maintaining the aspect ratio"""

tests/models/decoder_only/vision_language/vlm_utils/case_filtering.py

+5 -5

@@ -5,7 +5,7 @@ handling multimodal placeholder substitution, and so on.

5 """

6 import itertools

7 from collections import OrderedDict

8 -from typing import Dict, Iterable, Tuple

8 +from collections.abc import Iterable

9

10 import pytest

11

@@ -13,9 +13,9 @@ from .types import (EMBEDDING_SIZE_FACTORS, ExpandableVLMTestArgs,

13 ImageSizeWrapper, SizeType, VLMTestInfo, VLMTestType)

14

15

16 -def get_filtered_test_settings(test_settings: Dict[str, VLMTestInfo],

16 +def get_filtered_test_settings(test_settings: dict[str, VLMTestInfo],

17 test_type: VLMTestType,

18 - fork_per_test: bool) -> Dict[str, VLMTestInfo]:

18 + fork_per_test: bool) -> dict[str, VLMTestInfo]:

19 """Given the dict of potential test settings to run, return a subdict

20 of tests who have the current test type enabled with the matching val for

21 fork_per_test.

@@ -49,7 +49,7 @@ def get_filtered_test_settings(test_settings: Dict[str, VLMTestInfo],

49 return matching_tests

50

51

52 -def get_parametrized_options(test_settings: Dict[str, VLMTestInfo],

52 +def get_parametrized_options(test_settings: dict[str, VLMTestInfo],

53 test_type: VLMTestType,

54 fork_new_process_for_each_test: bool):

55 """Converts all of our VLMTestInfo into an expanded list of parameters.

@@ -121,7 +121,7 @@ def get_parametrized_options(test_settings: Dict[str, VLMTestInfo],

121

122 def get_wrapped_test_sizes(

123 test_info: VLMTestInfo,

124 - test_type: VLMTestType) -> Tuple[ImageSizeWrapper, ...]:

124 + test_type: VLMTestType) -> tuple[ImageSizeWrapper, ...]:

125 """Given a test info which may have size factors or fixed sizes, wrap them

126 and combine them into an iterable, each of which will be used in parameter

127 expansion.

tests/models/decoder_only/vision_language/vlm_utils/core.py

+11 -11

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Core test implementation to be shared across modalities."""

3 -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union

3 +from typing import Any, Callable, Optional, Union

4

5 import torch

6 from PIL.Image import Image

@@ -17,9 +17,9 @@ from .types import RunnerOutput

17

18 def run_test(

19 *,

20 - hf_runner: Type[HfRunner],

21 - vllm_runner: Type[VllmRunner],

22 - inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]],

20 + hf_runner: type[HfRunner],

21 + vllm_runner: type[VllmRunner],

22 + inputs: list[tuple[list[str], list[Union[list[Image], Image]]]],

23 model: str,

24 dtype: str,

25 max_tokens: int,

@@ -29,15 +29,15 @@ def run_test(

29 max_num_seqs: int,

30 hf_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]],

31 vllm_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]],

32 - auto_cls: Type[_BaseAutoModelClass],

32 + auto_cls: type[_BaseAutoModelClass],

33 use_tokenizer_eos: bool,

34 postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],

35 comparator: Callable[..., None],

36 get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]],

37 - stop_str: Optional[List[str]],

38 - limit_mm_per_prompt: Dict[str, int],

39 - vllm_runner_kwargs: Optional[Dict[str, Any]],

40 - hf_model_kwargs: Optional[Dict[str, Any]],

37 + stop_str: Optional[list[str]],

38 + limit_mm_per_prompt: dict[str, int],

39 + vllm_runner_kwargs: Optional[dict[str, Any]],

40 + hf_model_kwargs: Optional[dict[str, Any]],

41 patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],

42 task: TaskOption = "auto",

43 runner_mm_key: str = "images",

@@ -61,7 +61,7 @@ def run_test(

61 # if we run HF first, the cuda initialization will be done and it

62 # will hurt multiprocessing backend with fork method (the default method).

63

64 - vllm_runner_kwargs_: Dict[str, Any] = {}

64 + vllm_runner_kwargs_: dict[str, Any] = {}

65 if model_info.tokenizer:

66 vllm_runner_kwargs_["tokenizer"] = model_info.tokenizer

67 if model_info.tokenizer_mode:

@@ -84,7 +84,7 @@ def run_test(

84 **vllm_runner_kwargs_) as vllm_model:

85 tokenizer = vllm_model.model.get_tokenizer()

86

87 - vllm_kwargs: Dict[str, Any] = {}

87 + vllm_kwargs: dict[str, Any] = {}

88 if get_stop_token_ids is not None:

89 vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)

90 if stop_str:

tests/models/decoder_only/vision_language/vlm_utils/model_utils.py

+8 -8

@@ -6,7 +6,7 @@ typically specific to a small subset of models.

6 import re

7 import types

8 from pathlib import PosixPath

9 -from typing import Callable, List, Optional, Tuple, Union

9 +from typing import Callable, Optional, Union

10

11 import torch

12 from PIL.Image import Image

@@ -49,7 +49,7 @@ def fuyu_vllm_to_hf_output(vllm_output: RunnerOutput,

49

50 def qwen_vllm_to_hf_output(

51 vllm_output: RunnerOutput,

52 - model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:

52 + model: str) -> tuple[list[int], str, Optional[SampleLogprobs]]:

53 """Sanitize vllm output [qwen models] to be comparable with hf output."""

54 output_ids, output_str, out_logprobs = vllm_output

55

@@ -60,7 +60,7 @@ def qwen_vllm_to_hf_output(

60

61 def qwen2_vllm_to_hf_output(

62 vllm_output: RunnerOutput,

63 - model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:

63 + model: str) -> tuple[list[int], str, Optional[SampleLogprobs]]:

64 """Sanitize vllm output [qwen2 models] to be comparable with hf output."""

65 output_ids, output_str, out_logprobs = vllm_output

66

@@ -78,7 +78,7 @@ def llava_image_vllm_to_hf_output(vllm_output: RunnerOutput,

78

79 def llava_video_vllm_to_hf_output(

80 vllm_output: RunnerOutput,

81 - model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:

81 + model: str) -> tuple[list[int], str, Optional[SampleLogprobs]]:

82 config = AutoConfig.from_pretrained(model)

83 mm_token_id = config.video_token_index

84 return _llava_vllm_to_hf_output(vllm_output, model, mm_token_id)

@@ -247,7 +247,7 @@ def molmo_post_processor(hf_inputs: BatchEncoding, dtype: str):

247

248 ####### Prompt path encoders for models that need models on disk

249 def qwen_prompt_path_encoder(

250 - tmp_path: PosixPath, prompt: str, assets: Union[List[ImageAsset],

250 + tmp_path: PosixPath, prompt: str, assets: Union[list[ImageAsset],

251 _ImageAssets]) -> str:

252 """Given a temporary dir path, export one or more image assets into the

253 tempdir & replace its contents with the local path to the string so that

@@ -257,7 +257,7 @@ def qwen_prompt_path_encoder(

257 Args:

258 tmp_path: Tempdir for test under consideration.

259 prompt: Prompt with image placeholders.

260 - assets: List of image assets whose len equals the num placeholders.

260 + assets: list of image assets whose len equals the num placeholders.

261 """

262 # Ensure that the number of placeholders matches the number of assets;

263 # If this is not true, the test is probably written incorrectly.

@@ -350,7 +350,7 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

350 self.max_num = self.config.max_dynamic_patch

351 self.image_size = self.vision_config.image_size

352

353 - def __call__(self, text: str, images: Union[Image, List[Image]],

353 + def __call__(self, text: str, images: Union[Image, list[Image]],

354 **kwargs):

355 # yapf: disable

356 from vllm.model_executor.models.h2ovl import (

@@ -410,7 +410,7 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

410 self.max_num = self.config.max_dynamic_patch

411 self.image_size = self.vision_config.image_size

412

413 - def __call__(self, text: str, images: Union[Image, List[Image]],

413 + def __call__(self, text: str, images: Union[Image, list[Image]],

414 **kwargs):

415 from vllm.model_executor.models.internvl import (

416 IMG_CONTEXT, IMG_END, IMG_START,

tests/models/decoder_only/vision_language/vlm_utils/runners.py

+10 -11

@@ -3,7 +3,6 @@

3 types / modalities.

4 """

5 from pathlib import PosixPath

6 -from typing import Type

6

7 from .....conftest import HfRunner, VllmRunner, _ImageAssets, _VideoAssets

8 from . import builders, core

@@ -13,8 +12,8 @@ from .types import ExpandableVLMTestArgs, VLMTestInfo

12 ####### Entrypoints for running different test types

13 def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,

14 test_case: ExpandableVLMTestArgs,

16 - hf_runner: Type[HfRunner],

17 - vllm_runner: Type[VllmRunner],

15 + hf_runner: type[HfRunner],

16 + vllm_runner: type[VllmRunner],

17 image_assets: _ImageAssets):

18 assert test_case.size_wrapper is not None

19 inputs = builders.build_single_image_inputs_from_test_info(

@@ -36,8 +35,8 @@ def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,

35

36 def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,

37 test_case: ExpandableVLMTestArgs,

39 - hf_runner: Type[HfRunner],

40 - vllm_runner: Type[VllmRunner],

38 + hf_runner: type[HfRunner],

39 + vllm_runner: type[VllmRunner],

40 image_assets: _ImageAssets):

41 assert test_case.size_wrapper is not None

42 inputs = builders.build_multi_image_inputs_from_test_info(

@@ -59,8 +58,8 @@ def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,

58

59 def run_embedding_test(*, model_test_info: VLMTestInfo,

60 test_case: ExpandableVLMTestArgs,

62 - hf_runner: Type[HfRunner],

63 - vllm_runner: Type[VllmRunner],

61 + hf_runner: type[HfRunner],

62 + vllm_runner: type[VllmRunner],

63 image_assets: _ImageAssets):

64 assert test_case.size_wrapper is not None

65 inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(

@@ -85,8 +84,8 @@ def run_video_test(

84 *,

85 model_test_info: VLMTestInfo,

86 test_case: ExpandableVLMTestArgs,

88 - hf_runner: Type[HfRunner],

89 - vllm_runner: Type[VllmRunner],

87 + hf_runner: type[HfRunner],

88 + vllm_runner: type[VllmRunner],

89 video_assets: _VideoAssets,

90 ):

91 assert test_case.size_wrapper is not None

@@ -111,8 +110,8 @@ def run_video_test(

110

111 def run_custom_inputs_test(*, model_test_info: VLMTestInfo,

112 test_case: ExpandableVLMTestArgs,

114 - hf_runner: Type[HfRunner],

115 - vllm_runner: Type[VllmRunner]):

113 + hf_runner: type[HfRunner],

114 + vllm_runner: type[VllmRunner]):

115 # Custom test cases can provide inputs directly, but they need to

116 # explicitly provided a CustomTestConfig, which wraps the inputs and

117 # the limit_mm_per_prompt

tests/models/decoder_only/vision_language/vlm_utils/types.py

+18 -18

@@ -1,9 +1,9 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Types for writing multimodal model tests."""

3 +from collections.abc import Iterable

4 from enum import Enum

5 from pathlib import PosixPath

5 -from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,

6 - Tuple, Type, Union)

6 +from typing import Any, Callable, NamedTuple, Optional, Union

7

8 import torch

9 from PIL.Image import Image

@@ -35,7 +35,7 @@ VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"

35

36 IMAGE_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]

37 EMBEDDING_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0)]

38 -RunnerOutput = Tuple[List[int], str, Optional[SampleLogprobs]]

38 +RunnerOutput = tuple[list[int], str, Optional[SampleLogprobs]]

39 # yapf: enable

40

41

@@ -53,8 +53,8 @@ class SizeType(Enum):

53

54

55 class CustomTestOptions(NamedTuple):

56 - inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]]

57 - limit_mm_per_prompt: Dict[str, int]

56 + inputs: list[tuple[list[str], list[Union[list[Image], Image]]]]

57 + limit_mm_per_prompt: dict[str, int]

58 # kwarg to pass multimodal data in as to vllm/hf runner instances.

59 runner_mm_key: str = "images"

60

@@ -63,13 +63,13 @@ class ImageSizeWrapper(NamedTuple):

63 type: SizeType

64 # A size factor is a wrapper of 0+ floats,

65 # while a fixed size contains an iterable of integer pairs

66 - data: Union[Iterable[float], Iterable[Tuple[int, int]]]

66 + data: Union[Iterable[float], Iterable[tuple[int, int]]]

67

68

69 class VLMTestInfo(NamedTuple):

70 """Holds the configuration for 1+ tests for one model architecture."""

71

72 - models: List[str]

72 + models: list[str]

73 test_type: Union[VLMTestType, Iterable[VLMTestType]]

74

75 # Should be None only if this is a CUSTOM_INPUTS test

@@ -97,19 +97,19 @@ class VLMTestInfo(NamedTuple):

97 max_num_seqs: int = 256

98 task: TaskOption = "auto"

99 tensor_parallel_size: int = 1

100 - vllm_runner_kwargs: Optional[Dict[str, Any]] = None

100 + vllm_runner_kwargs: Optional[dict[str, Any]] = None

101

102 # Optional callable which gets a list of token IDs from the model tokenizer

103 get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]] = None

104 # Optional list of strings to stop generation, useful when stop tokens are

105 # not special tokens in the tokenizer

106 - stop_str: Optional[List[str]] = None

106 + stop_str: Optional[list[str]] = None

107

108 # Exposed options for HF runner

109 - hf_model_kwargs: Optional[Dict[str, Any]] = None

109 + hf_model_kwargs: Optional[dict[str, Any]] = None

110 # Indicates we should explicitly pass the EOS from the tokenizer

111 use_tokenizer_eos: bool = False

112 - auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM

112 + auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM

113 # Callable to pass to the HF runner to run on inputs; for now, we also pass

114 # the data type to input post processing, because almost all of the uses of

115 # postprocess_inputs are to fix the data types of BatchEncoding values.

@@ -128,12 +128,12 @@ class VLMTestInfo(NamedTuple):

128 # Default expandable params per test; these defaults can be overridden in

129 # instances of this object; the complete set of test cases for the model

130 # is all combinations of .models + all fields below

131 - max_tokens: Union[int, Tuple[int]] = 128

132 - num_logprobs: Union[int, Tuple[int]] = 5

131 + max_tokens: Union[int, tuple[int]] = 128

132 + num_logprobs: Union[int, tuple[int]] = 5

133 dtype: Union[str, Iterable[str]] = "half"

134 distributed_executor_backend: Optional[Union[str, Iterable[str]]] = None

135 # Only expanded in video tests

136 - num_video_frames: Union[int, Tuple[int]] = 16

136 + num_video_frames: Union[int, tuple[int]] = 16

137

138 # Fixed image sizes / image size factors; most tests use image_size_factors

139 # The values provided for these two fields will be stacked and expanded

@@ -141,19 +141,19 @@ class VLMTestInfo(NamedTuple):

141 # once per tests (much like concatenating and wrapping in one parametrize

142 # call)

143 image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS

144 - image_sizes: Optional[Iterable[Iterable[Tuple[int, int]]]] = None

144 + image_sizes: Optional[Iterable[Iterable[tuple[int, int]]]] = None

145

146 # Hack for updating a prompt to take into a local path; currently only used

147 # for Qwen-VL, which requires encoding the image path / url into the prompt

148 # for HF runner

149 prompt_path_encoder: Optional[

150 - Callable[[PosixPath, str, Union[List[ImageAsset], _ImageAssets]],

150 + Callable[[PosixPath, str, Union[list[ImageAsset], _ImageAssets]],

151 str]] = None # noqa: E501

152

153 # Allows configuring a test to run with custom inputs

154 - custom_test_opts: Optional[List[CustomTestOptions]] = None

154 + custom_test_opts: Optional[list[CustomTestOptions]] = None

155

156 - marks: Optional[List[MarkDecorator]] = None

156 + marks: Optional[list[MarkDecorator]] = None

157

158 def get_non_parametrized_runner_kwargs(self):

159 """Returns a dictionary of expandable kwargs for items that are used

tests/models/embedding/language/test_gritlm.py

+5 -6

@@ -3,7 +3,6 @@

3 import importlib.util

4 import math

5 from array import array

6 -from typing import List

6

7 import openai

8 import pytest

@@ -81,14 +80,14 @@ async def client_generate(server_generate: RemoteOpenAIServer):

80 yield async_client

81

82

84 -def run_llm_encode(llm: vllm.LLM, queries: List[str],

85 - instruction: str) -> List[float]:

83 +def run_llm_encode(llm: vllm.LLM, queries: list[str],

84 + instruction: str) -> list[float]:

85 outputs = llm.encode([instruction + q for q in queries], )

86 return [output.outputs.embedding for output in outputs]

87

88

90 -async def run_client_embeddings(client: vllm.LLM, queries: List[str],

91 - instruction: str) -> List[float]:

89 +async def run_client_embeddings(client: vllm.LLM, queries: list[str],

90 + instruction: str) -> list[float]:

91 outputs = await client.embeddings.create(

92 model=MODEL_NAME,

93 input=[instruction + q for q in queries],

@@ -123,7 +122,7 @@ def get_test_data():

122 return queries, q_instruction, documents, d_instruction

123

124

126 -def validate_embed_output(q_rep: List[float], d_rep: List[float]):

125 +def validate_embed_output(q_rep: list[float], d_rep: list[float]):

126 cosine_sim_q0_d0 = 1 - cosine(q_rep[0], d_rep[0])

127 assert math.isclose(cosine_sim_q0_d0, 0.609, abs_tol=0.001)

128

tests/models/embedding/utils.py

+3 -3

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Sequence

3 +from collections.abc import Sequence

4

5 import torch

6 import torch.nn.functional as F

@@ -8,8 +8,8 @@ import torch.nn.functional as F

8

9 def check_embeddings_close(

10 *,

11 - embeddings_0_lst: Sequence[List[float]],

12 - embeddings_1_lst: Sequence[List[float]],

11 + embeddings_0_lst: Sequence[list[float]],

12 + embeddings_1_lst: Sequence[list[float]],

13 name_0: str,

14 name_1: str,

15 tol: float = 1e-3,

tests/models/embedding/vision_language/test_dse_qwen2_vl.py

+6 -6

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from functools import partial

4 -from typing import Callable, Dict, List, Type

4 +from typing import Callable

5

6 import pytest

7 import torch

@@ -67,7 +67,7 @@ def get_messages(image: Image.Image, text: str, embed_text: bool):

67

68

69 def apply_chat_template_and_add_eos(

70 - messages: List[Dict],

70 + messages: list[dict],

71 apply_chat_template_fn: Callable,

72 ):

73 prompt = apply_chat_template_fn(

@@ -80,11 +80,11 @@ def postprocess_inputs(hf_model: HfRunner, inputs: BatchEncoding, **kwargs):

80

81

82 def _run_test(

83 - hf_runner: Type[HfRunner],

84 - vllm_runner: Type[VllmRunner],

85 - input_texts: List[str],

83 + hf_runner: type[HfRunner],

84 + vllm_runner: type[VllmRunner],

85 + input_texts: list[str],

86 input_images: PromptImageInput,

87 - embed_texts: List[bool],

87 + embed_texts: list[bool],

88 model: str,

89 *,

90 dtype: str,

tests/models/embedding/vision_language/test_llava_next.py

+3 -5

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Type

4 -

3 import pytest

4 import torch.nn.functional as F

5 from transformers import AutoModelForVision2Seq

@@ -35,9 +33,9 @@ MODELS = ["royokong/e5-v"]

33

34

35 def _run_test(

38 - hf_runner: Type[HfRunner],

39 - vllm_runner: Type[VllmRunner],

40 - input_texts: List[str],

36 + hf_runner: type[HfRunner],

37 + vllm_runner: type[VllmRunner],

38 + input_texts: list[str],

39 input_images: PromptImageInput,

40 model: str,

41 *,

tests/models/embedding/vision_language/test_phi3v.py

+3 -5

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Type

4 -

3 import pytest

4 import torch.nn.functional as F

5

@@ -29,9 +27,9 @@ MODELS = ["TIGER-Lab/VLM2Vec-Full"]

27

28

29 def _run_test(

32 - hf_runner: Type[HfRunner],

33 - vllm_runner: Type[VllmRunner],

34 - input_texts: List[str],

30 + hf_runner: type[HfRunner],

31 + vllm_runner: type[VllmRunner],

32 + input_texts: list[str],

33 input_images: PromptImageInput,

34 model: str,

35 *,

tests/models/encoder_decoder/language/test_bart.py

+5 -5

@@ -3,7 +3,7 @@

3

4 Run `pytest tests/models/encoder_decoder/language/test_bart.py`.

5 """

6 -from typing import List, Optional, Tuple, Type

6 +from typing import Optional

7

8 import pytest

9 from transformers import AutoModelForSeq2SeqLM

@@ -17,7 +17,7 @@ from ...utils import check_logprobs_close

17

18

19 def vllm_to_hf_output(

20 - vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],

20 + vllm_output: tuple[list[int], str, Optional[SampleLogprobs]],

21 decoder_prompt_type: DecoderPromptType,

22 ):

23 """Sanitize vllm output to be comparable with hf output."""

@@ -31,9 +31,9 @@ def vllm_to_hf_output(

31

32

33 def run_test(

34 - hf_runner: Type[HfRunner],

35 - vllm_runner: Type[VllmRunner],

36 - prompts: List[ExplicitEncoderDecoderPrompt[str, str]],

34 + hf_runner: type[HfRunner],

35 + vllm_runner: type[VllmRunner],

36 + prompts: list[ExplicitEncoderDecoderPrompt[str, str]],

37 decoder_prompt_type: DecoderPromptType,

38 model: str,

39 *,

tests/models/encoder_decoder/vision_language/test_florence2.py

+4 -4

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Optional, Type

3 +from typing import Optional

4

5 import pytest

6 from PIL import Image

@@ -51,8 +51,8 @@ def hf_to_vllm_output(hf_output: tuple[list[int], str,

51

52

53 def run_test(

54 - hf_runner: Type[HfRunner],

55 - vllm_runner: Type[VllmRunner],

54 + hf_runner: type[HfRunner],

55 + vllm_runner: type[VllmRunner],

56 inputs: list[list[ExplicitEncoderDecoderPrompt]],

57 model: str,

58 *,

@@ -114,7 +114,7 @@ def run_test(

114 @pytest.mark.parametrize("dtype", ["float"])

115 @pytest.mark.parametrize("max_tokens", [64])

116 @pytest.mark.parametrize("num_logprobs", [5])

117 -def test_models(hf_runner: Type[HfRunner], vllm_runner: Type[VllmRunner],

117 +def test_models(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],

118 image_assets: _ImageAssets, model: str,

119 size_factors: list[int], dtype: str, max_tokens: int,

120 num_logprobs: int) -> None:

tests/models/encoder_decoder/vision_language/test_mllama.py

+18 -18

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple, Type, overload

3 +from typing import Optional, overload

4

5 import pytest

6 import torch

@@ -64,7 +64,7 @@ prompt_data = {

64 }

65

66

67 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,

67 +def vllm_to_hf_output(vllm_output: tuple[list[int], str,

68 Optional[SampleLogprobs]],

69 model: str):

70 """Sanitize vllm output to be comparable with hf output."""

@@ -91,9 +91,9 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,

91 def _get_inputs(

92 image_assets: _ImageAssets,

93 *,

94 - size_factors: Optional[List[float]] = None,

95 - sizes: Optional[List[Tuple[int, int]]] = None,

96 -) -> List[Tuple[List[str], PromptImageInput]]:

94 + size_factors: Optional[list[float]] = None,

95 + sizes: Optional[list[tuple[int, int]]] = None,

96 +) -> list[tuple[list[str], PromptImageInput]]:

97 images = [asset.pil_image for asset in image_assets]

98

99 if size_factors is not None:

@@ -123,12 +123,12 @@ def _get_inputs(

123

124 @overload

125 def run_test(

126 - hf_runner: Type[HfRunner],

127 - vllm_runner: Type[VllmRunner],

126 + hf_runner: type[HfRunner],

127 + vllm_runner: type[VllmRunner],

128 image_assets: _ImageAssets,

129 model: str,

130 *,

131 - size_factors: List[float],

131 + size_factors: list[float],

132 dtype: str,

133 max_tokens: int,

134 num_logprobs: int,

@@ -140,12 +140,12 @@ def run_test(

140

141 @overload

142 def run_test(

143 - hf_runner: Type[HfRunner],

144 - vllm_runner: Type[VllmRunner],

143 + hf_runner: type[HfRunner],

144 + vllm_runner: type[VllmRunner],

145 image_assets: _ImageAssets,

146 model: str,

147 *,

148 - sizes: List[Tuple[int, int]],

148 + sizes: list[tuple[int, int]],

149 dtype: str,

150 max_tokens: int,

151 num_logprobs: int,

@@ -156,13 +156,13 @@ def run_test(

156

157

158 def run_test(

159 - hf_runner: Type[HfRunner],

160 - vllm_runner: Type[VllmRunner],

159 + hf_runner: type[HfRunner],

160 + vllm_runner: type[VllmRunner],

161 image_assets: _ImageAssets,

162 model: str,

163 *,

164 - size_factors: Optional[List[float]] = None,

165 - sizes: Optional[List[Tuple[int, int]]] = None,

164 + size_factors: Optional[list[float]] = None,

165 + sizes: Optional[list[tuple[int, int]]] = None,

166 dtype: str,

167 max_tokens: int,

168 num_logprobs: int,

@@ -183,9 +183,9 @@ def run_test(

183

184

185 def _run_test(

186 - hf_runner: Type[HfRunner],

187 - vllm_runner: Type[VllmRunner],

188 - inputs: List[Tuple[List[str], PromptImageInput]],

186 + hf_runner: type[HfRunner],

187 + vllm_runner: type[VllmRunner],

188 + inputs: list[tuple[list[str], PromptImageInput]],

189 model: str,

190 *,

191 dtype: str,

tests/models/multimodal/processing/test_h2ovl.py

+2 -1

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Tests for H2OVL's multimodal preprocessing kwargs."""

3 -from typing import Mapping, Optional

3 +from collections.abc import Mapping

4 +from typing import Optional

5

6 import pytest

7 from PIL import Image

tests/models/multimodal/processing/test_internvl.py

+2 -1

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Tests for InternVL's multimodal preprocessing kwargs."""

3 -from typing import Mapping, Optional

3 +from collections.abc import Mapping

4 +from typing import Optional

5

6 import pytest

7 from PIL import Image

tests/models/registry.py

+3 -2

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import Mapping, Set

4 from dataclasses import dataclass, field

4 -from typing import AbstractSet, Any, Literal, Mapping, Optional

5 +from typing import Any, Literal, Optional

6

7 import pytest

8 from packaging.version import Version

@@ -324,7 +325,7 @@ class HfExampleModels:

325

326 self.hf_models = hf_models

327

327 - def get_supported_archs(self) -> AbstractSet[str]:

328 + def get_supported_archs(self) -> Set[str]:

329 return self.hf_models.keys()

330

331 def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:

tests/models/test_transformers.py

+7 -8

@@ -4,7 +4,6 @@

4 Run `pytest tests/models/test_transformers.py`.

5 """

6 from contextlib import nullcontext

7 -from typing import Type

7

8 import pytest

9

@@ -14,8 +13,8 @@ from .utils import check_logprobs_close

13

14

15 def check_implementation(

17 - hf_runner: Type[HfRunner],

18 - vllm_runner: Type[VllmRunner],

16 + hf_runner: type[HfRunner],

17 + vllm_runner: type[VllmRunner],

18 example_prompts: list[str],

19 model: str,

20 **kwargs,

@@ -47,8 +46,8 @@ def check_implementation(

46 ("ArthurZ/Ilama-3.2-1B", "auto"), # CUSTOM CODE

47 ]) # trust_remote_code=True by default

48 def test_models(

50 - hf_runner: Type[HfRunner],

51 - vllm_runner: Type[VllmRunner],

49 + hf_runner: type[HfRunner],

50 + vllm_runner: type[VllmRunner],

51 example_prompts: list[str],

52 model: str,

53 model_impl: str,

@@ -71,8 +70,8 @@ def test_models(

70

71 @multi_gpu_test(num_gpus=2)

72 def test_distributed(

74 - hf_runner: Type[HfRunner],

75 - vllm_runner: Type[VllmRunner],

73 + hf_runner: type[HfRunner],

74 + vllm_runner: type[VllmRunner],

75 example_prompts,

76 ):

77 kwargs = {"model_impl": "transformers", "tensor_parallel_size": 2}

@@ -92,7 +91,7 @@ def test_distributed(

91 @pytest.mark.parametrize("max_tokens", [32])

92 @pytest.mark.parametrize("num_logprobs", [5])

93 def test_quantization(

95 - vllm_runner: Type[VllmRunner],

94 + vllm_runner: type[VllmRunner],

95 example_prompts: list[str],

96 model: str,

97 quantization_kwargs: dict[str, str],

tests/models/utils.py

+11 -10

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import warnings

4 -from typing import Dict, List, Optional, Sequence, Tuple, Union

4 +from collections.abc import Sequence

5 +from typing import Optional, Union

6

7 import torch

8

@@ -9,7 +10,7 @@ from vllm.config import ModelConfig, TaskOption

10 from vllm.inputs import InputContext

11 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs

12

12 -TokensText = Tuple[List[int], str]

13 +TokensText = tuple[list[int], str]

14

15

16 def check_outputs_equal(

@@ -46,7 +47,7 @@ def check_outputs_equal(

47 # * List of top sample logprobs for each sampled token

48 #

49 # Assumes prompt logprobs were not requested.

49 -TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,

50 +TokensTextLogprobs = tuple[list[int], str, Optional[Union[list[dict[int,

51 float]],

52 SampleLogprobs]]]

53

@@ -57,8 +58,8 @@ TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,

58 # * Optional list of top sample logprobs for each sampled token

59 #

60 # Assumes prompt logprobs were not requested.

60 -TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],

61 - List[Dict[str,

61 +TextTextLogprobs = tuple[list[str], str, Optional[Union[list[dict[str, float]],

62 + list[dict[str,

63 Logprob]]]]]

64

65 # Representation of generated sequence as a tuple of

@@ -68,9 +69,9 @@ TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],

69 # * Optional list of top prompt logprobs for each prompt token

70 #

71 # Allows prompt logprobs to be requested.

71 -TokensTextLogprobsPromptLogprobs = Tuple[

72 - List[int], str, Optional[Union[List[Dict[int, float]], SampleLogprobs]],

73 - Optional[Union[List[Optional[Dict[int, float]]], PromptLogprobs]]]

72 +TokensTextLogprobsPromptLogprobs = tuple[

73 + list[int], str, Optional[Union[list[dict[int, float]], SampleLogprobs]],

74 + Optional[Union[list[Optional[dict[int, float]]], PromptLogprobs]]]

75

76

77 def check_logprobs_close(

@@ -254,8 +255,8 @@ def build_model_context(

255 tokenizer_name: Optional[str] = None,

256 trust_remote_code: bool = False,

257 dtype: Optional[Union[str, torch.dtype]] = None,

257 - mm_processor_kwargs: Optional[Dict] = None,

258 - limit_mm_per_prompt: Optional[Dict] = None,

258 + mm_processor_kwargs: Optional[dict] = None,

259 + limit_mm_per_prompt: Optional[dict] = None,

260 disable_mm_preprocessor_cache: bool = True,

261 ):

262 """Creates an InputContext for a given model.

tests/mq_llm_engine/utils.py

+2 -2

@@ -2,7 +2,7 @@

2

3 import asyncio

4 import multiprocessing

5 -from typing import Callable, Tuple, Union

5 +from typing import Callable, Union

6

7 from vllm import SamplingParams

8 from vllm.engine.arg_utils import AsyncEngineArgs

@@ -16,7 +16,7 @@ async def generate(

16 client: MQLLMEngineClient,

17 request_id: str,

18 num_tokens: int,

19 - return_output: bool = False) -> Union[RequestOutput, Tuple[int, str]]:

19 + return_output: bool = False) -> Union[RequestOutput, tuple[int, str]]:

20

21 final_output = None

22 count = 0

tests/multi_step/test_correctness_async_llm.py

+2 -2

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 # Test the AsyncLLMEngine with multi-step-decoding

4 -from typing import List, Optional

4 +from typing import Optional

5

6 import pytest

7

@@ -17,7 +17,7 @@ MODELS = [

17 NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps

18 NUM_PROMPTS = [10]

19

20 -DEFAULT_SERVER_ARGS: List[str] = [

20 +DEFAULT_SERVER_ARGS: list[str] = [

21 "--distributed-executor-backend",

22 "ray",

23 "--gpu-memory-utilization",

tests/multimodal/test_utils.py

+4 -4

@@ -4,7 +4,7 @@ import base64

4 import mimetypes

5 import os

6 from tempfile import NamedTemporaryFile, TemporaryDirectory

7 -from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple

7 +from typing import TYPE_CHECKING, NamedTuple, Optional

8

9 import numpy as np

10 import pytest

@@ -30,7 +30,7 @@ TEST_IMAGE_URLS = [

30

31

32 @pytest.fixture(scope="module")

33 -def url_images() -> Dict[str, Image.Image]:

33 +def url_images() -> dict[str, Image.Image]:

34 connector = MediaConnector()

35

36 return {

@@ -39,7 +39,7 @@ def url_images() -> Dict[str, Image.Image]:

39 }

40

41

42 -def get_supported_suffixes() -> Tuple[str, ...]:

42 +def get_supported_suffixes() -> tuple[str, ...]:

43 # We should at least test the file types mentioned in GPT-4 with Vision

44 OPENAI_SUPPORTED_SUFFIXES = ('.png', '.jpeg', '.jpg', '.webp', '.gif')

45

@@ -66,7 +66,7 @@ async def test_fetch_image_http(image_url: str):

66 @pytest.mark.asyncio

67 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)

68 @pytest.mark.parametrize("suffix", get_supported_suffixes())

69 -async def test_fetch_image_base64(url_images: Dict[str, Image.Image],

69 +async def test_fetch_image_base64(url_images: dict[str, Image.Image],

70 image_url: str, suffix: str):

71 connector = MediaConnector()

72 url_image = url_images[image_url]

tests/neuron/test_logits_processor.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import Tuple

4 from unittest.mock import patch

5

6 import pytest

@@ -33,7 +32,7 @@ class MockLogitsProcessor(LogitsProcessor):

32

33 def _prepare_test(

34 batch_size: int

36 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:

35 +) -> tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:

36 vocab_size = 32000

37 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)

38 fake_logits = torch.full((batch_size, vocab_size),

tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py

+3 -2

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Iterable, Optional, Tuple, Union

3 +from collections.abc import Iterable

4 +from typing import Optional, Union

5

6 import torch

7 import torch.nn as nn

@@ -59,7 +60,7 @@ class MyGemma2Embedding(nn.Module):

60 ) -> Optional[PoolerOutput]:

61 return self._pooler(hidden_states, pooling_metadata)

62

62 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

63 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):

64

65 weights = self.hf_to_vllm_mapper.apply(weights)

66 weights = ((name, data) for name, data in weights

tests/quantization/test_configs.py

+1 -2

@@ -5,7 +5,6 @@ Run `pytest tests/quantization/test_configs.py --forked`.

5 """

6

7 from dataclasses import dataclass

8 -from typing import Tuple

8

9 import pytest

10

@@ -53,7 +52,7 @@ MODEL_ARG_EXPTYPES = [

52

53

54 @pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)

56 -def test_auto_gptq(model_arg_exptype: Tuple[str, None, str]) -> None:

55 +def test_auto_gptq(model_arg_exptype: tuple[str, None, str]) -> None:

56 model_path, quantization_arg, expected_type = model_arg_exptype

57

58 try:

tests/quantization/test_register_quantization_config.py

+4 -4

@@ -5,7 +5,7 @@ See https://github.com/vllm-project/vllm/issues/11926 for more details.

5

6 Run `pytest tests/quantization/test_register_quantization_config.py`.

7 """

8 -from typing import Any, Dict, List, Optional

8 +from typing import Any, Optional

9

10 import pytest

11 import torch

@@ -58,7 +58,7 @@ class CustomQuantConfig(QuantizationConfig):

58 """Name of the quantization method."""

59 return "custom_quant"

60

61 - def get_supported_act_dtypes(self) -> List["torch.dtype"]:

61 + def get_supported_act_dtypes(self) -> list["torch.dtype"]:

62 """List of supported activation dtypes."""

63 return [torch.float16, torch.bfloat16]

64

@@ -68,12 +68,12 @@ class CustomQuantConfig(QuantizationConfig):

68 return -1

69

70 @staticmethod

71 - def get_config_filenames() -> List[str]:

71 + def get_config_filenames() -> list[str]:

72 """List of filenames to search for in the model directory."""

73 return []

74

75 @classmethod

76 - def from_config(cls, config: Dict[str, Any]) -> "CustomQuantConfig":

76 + def from_config(cls, config: dict[str, Any]) -> "CustomQuantConfig":

77 """Create a config class from the model's quantization config."""

78 return CustomQuantConfig(num_bits=config.get("num_bits", 8))

79

tests/samplers/test_logprobs.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import torch

5

@@ -70,7 +68,7 @@ def test_get_prompt_logprobs(

68 assert (len(logprobs) == num_top_logprobs

69 or len(logprobs) == num_top_logprobs + 1)

70 output_text = result.outputs[0].text

73 - output_string_from_most_likely_tokens_lst: List[str] = []

71 + output_string_from_most_likely_tokens_lst: list[str] = []

72 for top_logprobs in result.outputs[0].logprobs:

73 top_logprob = next(iter(top_logprobs.values()))

74 output_string_from_most_likely_tokens_lst.append(

tests/samplers/test_no_bad_words.py

+8 -8

@@ -4,7 +4,7 @@

4 Run `pytest tests/samplers/test_no_bad_words.py`.

5

6 """

7 -from typing import List, Optional

7 +from typing import Optional

8

9 from transformers import AutoTokenizer

10

@@ -16,8 +16,8 @@ def _generate(

16 prompt: str,

17 num_prompt_tokens: int,

18 temperature: float = 0,

19 - bad_words: Optional[List[str]] = None,

20 -) -> List[int]:

19 + bad_words: Optional[list[str]] = None,

20 +) -> list[int]:

21 sampling_params = SamplingParams(

22 temperature=temperature,

23 bad_words=bad_words,

@@ -59,7 +59,7 @@ class TestOneTokenBadWord:

59

60 def _generate(self,

61 model: LLM,

62 - bad_words: Optional[List[str]] = None) -> List[int]:

62 + bad_words: Optional[list[str]] = None) -> list[int]:

63 return _generate(

64 model=model,

65 prompt=self.PROMPT,

@@ -69,7 +69,7 @@ class TestOneTokenBadWord:

69

70 def _encode(self,

71 prompt: str,

72 - add_special_tokens: bool = True) -> List[int]:

72 + add_special_tokens: bool = True) -> list[int]:

73 return self.tokenizer(prompt,

74 add_special_tokens=add_special_tokens).input_ids

75

@@ -149,7 +149,7 @@ class TestTwoTokenBadWord:

149

150 def _generate(self,

151 model: LLM,

152 - bad_words: Optional[List[str]] = None) -> List[int]:

152 + bad_words: Optional[list[str]] = None) -> list[int]:

153 return _generate(

154 model=model,

155 prompt=self.PROMPT,

@@ -158,7 +158,7 @@ class TestTwoTokenBadWord:

158 )

159

160 @staticmethod

161 - def _contains(sequence: List[int], subsequence: List[int]) -> bool:

161 + def _contains(sequence: list[int], subsequence: list[int]) -> bool:

162 searched = False

163

164 for start in range(len(sequence)):

@@ -181,6 +181,6 @@ class TestTwoTokenBadWord:

181

182 def _encode(self,

183 prompt: str,

184 - add_special_tokens: bool = True) -> List[int]:

184 + add_special_tokens: bool = True) -> list[int]:

185 return self.tokenizer(prompt,

186 add_special_tokens=add_special_tokens).input_ids

tests/samplers/test_rejection_sampler.py

+5 -6

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Tests for rejection sampling."""

3 -from typing import List, Tuple

3

4 import pytest

5 import torch

@@ -416,8 +415,8 @@ def test_rejection_sampling_approximates_target_distribution(

415 draft_and_target_probs_equal)

416

417 sample_sizes = [10, 100, 1_000, 10_000, 100_000]

419 - distance_wrt_reference: List[float] = []

420 - distance_wrt_target: List[float] = []

418 + distance_wrt_reference: list[float] = []

419 + distance_wrt_target: list[float] = []

420

421 for num_samples in sample_sizes:

422 (reference_vs_rejsample_dist,

@@ -452,7 +451,7 @@ def test_rejection_sampling_approximates_target_distribution(

451 expected_improvement_multiplier)

452

453

455 -def get_ratio_first_to_last(elements: List[float]) -> float:

454 +def get_ratio_first_to_last(elements: list[float]) -> float:

455 return elements[0] / elements[-1]

456

457

@@ -477,7 +476,7 @@ class _CorrectnessTestHelper:

476

477 def generate_probs_for_test(

478 self, draft_and_target_probs_equal: bool

480 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

479 + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

480 draft_probs, target_probs = (F.softmax(

481 torch.rand(self.vocab_size, dtype=torch.float32),

482 dim=-1,

@@ -499,7 +498,7 @@ class _CorrectnessTestHelper:

498 def run_and_compare_distributions(self, draft_probs: torch.Tensor,

499 target_probs: torch.Tensor,

500 reference_probs: torch.Tensor,

502 - num_samples: int) -> Tuple[float, float]:

501 + num_samples: int) -> tuple[float, float]:

502 # Sample using rejection sampling.

503 rej_sample_probs = self._estimate_rejection_sampling_pdf(

504 draft_probs, target_probs, num_samples)

tests/samplers/test_sampler.py

+22 -22

@@ -3,7 +3,7 @@

3 import itertools

4 import random

5 from dataclasses import dataclass

6 -from typing import Dict, List, Optional, Tuple

6 +from typing import Optional

7 from unittest.mock import Mock, patch

8

9 import pytest

@@ -30,7 +30,7 @@ class MockLogitsSampler(Sampler):

30

31 def _prepare_test(

32 batch_size: int

33 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler]:

33 +) -> tuple[torch.Tensor, torch.Tensor, MockLogitsSampler]:

34 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)

35 fake_logits = torch.full((batch_size, VOCAB_SIZE),

36 1e-2,

@@ -53,8 +53,8 @@ def _do_sample(

53 sampling_params: SamplingParams,

54 device: str,

55 ):

56 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

57 - seq_lens: List[int] = []

56 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

57 + seq_lens: list[int] = []

58 for i in range(batch_size):

59 seq_group_metadata_list.append(

60 SequenceGroupMetadata(

@@ -171,7 +171,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):

171 def create_sampling_params(min_tokens,

172 eos_token_id=0,

173 *,

174 - stop_token_ids: Optional[List[int]] = None,

174 + stop_token_ids: Optional[list[int]] = None,

175 prompt_logprobs: Optional[int] = None):

176 sampling_params = SamplingParams(

177 min_tokens=min_tokens,

@@ -196,7 +196,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):

196 batch_size = random.randint(1, 128)

197

198 expected_penalization = []

199 - sequence_metadata_list: List[SequenceGroupMetadata] = []

199 + sequence_metadata_list: list[SequenceGroupMetadata] = []

200 # 20% chance to generate seq group metadata list with all prompts

201 is_prompt = random.random() < 0.2

202 while batch_size > 0:

@@ -216,8 +216,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):

216 eos_token_id=eos_token_id,

217 stop_token_ids=stop_token_ids)

218

219 - seq_data: Dict[int, SequenceData] = {}

220 - seq_group_penalization: List[bool] = []

219 + seq_data: dict[int, SequenceData] = {}

220 + seq_group_penalization: list[bool] = []

221 for _ in range(num_seqs):

222 num_input = random.randint(1, 100)

223 num_generated = 0 if is_prompt else random.randint(1, 100)

@@ -376,16 +376,16 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):

376 else:

377 test_cases = [generate_test_case()]

378

379 - def run_test_case(*, expected_penalization: List[bool],

380 - seq_group_metadata_list: List[SequenceGroupMetadata]):

379 + def run_test_case(*, expected_penalization: list[bool],

380 + seq_group_metadata_list: list[SequenceGroupMetadata]):

381 assert expected_penalization, \

382 "Invalid test case, need expected_penalization"

383 assert seq_group_metadata_list, \

384 "Invalid test case, need seq_group_metadata_list"

385

386 batch_size = 0

387 - seq_lens: List[int] = []

388 - sampling_params_per_row: List[SamplingParams] = []

387 + seq_lens: list[int] = []

388 + sampling_params_per_row: list[SamplingParams] = []

389 for sgm in seq_group_metadata_list:

390 sampling_params = sgm.sampling_params

391

@@ -456,11 +456,11 @@ def test_sampler_mixed(seed: int, device: str):

456 batch_size = random.randint(1, 256)

457 input_tensor, fake_logits, sampler = _prepare_test(batch_size)

458

459 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

460 - expected_tokens: List[Optional[List[int]]] = []

461 - seq_lens: List[int] = []

459 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

460 + expected_tokens: list[Optional[list[int]]] = []

461 + seq_lens: list[int] = []

462 for i in range(batch_size):

463 - expected: Optional[List[int]] = None

463 + expected: Optional[list[int]] = None

464 sampling_type = random.randint(0, 2)

465 if sampling_type == 0:

466 sampling_params = SamplingParams(temperature=0)

@@ -492,7 +492,7 @@ def test_sampler_mixed(seed: int, device: str):

492 ))

493 seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())

494

495 - generators: Dict[str, torch.Generator] = {}

495 + generators: dict[str, torch.Generator] = {}

496

497 def test_sampling():

498 sampling_metadata = SamplingMetadata.prepare(

@@ -587,8 +587,8 @@ def test_sampler_top_k_top_p(seed: int, device: str):

587 device=device)

588 assert len(processors) == 2 # top_p and top_k

589

590 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

591 - seq_lens: List[int] = []

590 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

591 + seq_lens: list[int] = []

592 for i in range(batch_size):

593 seq_group_metadata_list.append(

594 SequenceGroupMetadata(

@@ -669,10 +669,10 @@ def test_sampler_repetition_penalty_mixed(device: str):

669

670 vocab_size = 8

671

672 - def test_sampling_params(sampling_params: List[SamplingParams]):

672 + def test_sampling_params(sampling_params: list[SamplingParams]):

673

674 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

675 - seq_lens: List[int] = []

674 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

675 + seq_lens: list[int] = []

676 for i in range(2):

677 seq_group_metadata_list.append(

678 SequenceGroupMetadata(

tests/spec_decode/e2e/conftest.py

+5 -4

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import Sequence

4 from itertools import cycle

4 -from typing import List, Optional, Sequence, Tuple, Union

5 +from typing import Optional, Union

6

7 import pytest

8 import torch

@@ -64,9 +65,9 @@ def maybe_assert_ngram_worker(llm):

65

66 def get_output_from_llm_generator(

67 llm_generator, prompts,

67 - sampling_params) -> Tuple[List[str], List[List[int]], float]:

68 - tokens: List[str] = []

69 - token_ids: List[List[int]] = []

68 + sampling_params) -> tuple[list[str], list[list[int]], float]:

69 + tokens: list[str] = []

70 + token_ids: list[list[int]] = []

71 acceptance_rate: float = -1.0

72 for llm in llm_generator():

73 maybe_assert_ngram_worker(llm)

tests/spec_decode/test_batch_expansion.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import torch

5

@@ -42,7 +40,7 @@ def test_get_token_ids_to_score(k: int):

40 device='cuda',

41 )

42

45 - expected_output: List[List[int]] = [

43 + expected_output: list[list[int]] = [

44 [],

45 ]

46 for i in range(proposal_token_ids.shape[0]):

tests/spec_decode/test_multi_step_worker.py

+7 -8

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import Dict, List

4 from unittest.mock import MagicMock

5

6 import pytest

@@ -221,7 +220,7 @@ def test_same_output_for_multi_step():

220

221 # Run single-step repeatedly.

222 zero_kv_cache(worker.cache_engine)

224 - single_step_output: List[SamplerOutput] = []

223 + single_step_output: list[SamplerOutput] = []

224 continuations = [[1] for _ in prompts]

225 set_random_seed(seed)

226

@@ -243,15 +242,15 @@ def test_same_output_for_multi_step():

242 continuations[i].append(seq_group_output.samples[0].output_token)

243

244 # Get token ids and logprobs for comparison.

246 - multi_step_output_logprobs: List[List[Dict[int,

245 + multi_step_output_logprobs: list[list[dict[int,

246 Logprob]]] = [[]

247 for _ in prompts]

249 - single_step_output_logprobs: List[List[Dict[int,

248 + single_step_output_logprobs: list[list[dict[int,

249 Logprob]]] = [[]

250 for _ in prompts]

251

253 - multi_step_output_token_ids: List[List[int]] = [[] for _ in prompts]

254 - single_step_output_token_ids: List[List[int]] = [[] for _ in prompts]

252 + multi_step_output_token_ids: list[list[int]] = [[] for _ in prompts]

253 + single_step_output_token_ids: list[list[int]] = [[] for _ in prompts]

254 for i, _ in enumerate(prompts):

255 for multi_step, single_step in zip(multi_step_output,

256 single_step_output):

@@ -336,7 +335,7 @@ def test_multi_step_with_batch_expansion_correct_output():

335 # will simulate the bonus token case with the second token

336 # being the bonus token.

337 zero_kv_cache(worker.cache_engine)

339 - single_step_output: List[SamplerOutput] = []

338 + single_step_output: list[SamplerOutput] = []

339 set_random_seed(seed)

340 for _ in range(num_steps):

341 seq_group_metadata_list = create_seq_group_metadata_from_prompts(

@@ -430,7 +429,7 @@ def test_multi_step_with_batch_expansion_incorrect_output():

429 # will simulate the bonus token case with the second token

430 # being the bonus token.

431 zero_kv_cache(worker.cache_engine)

433 - single_step_output: List[SamplerOutput] = []

432 + single_step_output: list[SamplerOutput] = []

433 set_random_seed(seed)

434 for _ in range(num_steps):

435 seq_group_metadata_list = create_seq_group_metadata_from_prompts(

tests/spec_decode/test_scorer.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import List

4

5 import pytest

6 import torch

@@ -15,7 +14,7 @@ from vllm.worker.worker import Worker

14 from .utils import create_batch, create_worker

15

16

18 -def create_proposal(propose_lens: List[int], vocab_size: int,

17 +def create_proposal(propose_lens: list[int], vocab_size: int,

18 device: str) -> SpeculativeProposals:

19 batch_size = len(propose_lens)

20 max_propose_len = max(propose_lens)

tests/spec_decode/test_spec_decode_worker.py

+5 -6

@@ -3,7 +3,6 @@

3 import random

4 from collections import defaultdict

5 from types import SimpleNamespace

6 -from typing import Dict, List, Set

6 from unittest.mock import MagicMock

7

8 import pytest

@@ -123,7 +122,7 @@ def test_batch_expansion_correctly_calls_target_model(

122 seq_group_metadata_list=seq_group_metadata_list,

123 num_lookahead_slots=k))

124

126 - seen_contexts: List[List[int]] = []

125 + seen_contexts: list[list[int]] = []

126

127 call_args_list = target_worker.execute_model.call_args_list

128 assert len(call_args_list) == 1

@@ -136,7 +135,7 @@ def test_batch_expansion_correctly_calls_target_model(

135 for seq_data in seq_group_metadata.seq_data.values():

136 seen_contexts.append(seq_data.get_token_ids())

137

139 - expected_seen_contexts: List[List[int]] = []

138 + expected_seen_contexts: list[list[int]] = []

139

140 for prompt, prev_generated, draft_tokens in zip(

141 prompts, prev_output_tokens, proposal_token_ids.tolist()):

@@ -338,11 +337,11 @@ def test_correctly_formats_output(k: int, batch_size: int,

337 next(iter(seq_group_metadata.seq_data.keys()))

338 for seq_group_metadata in seq_group_metadata_list

339 ]

341 - actual_output_by_seq: Dict[int, List[SequenceOutput]] = {

340 + actual_output_by_seq: dict[int, list[SequenceOutput]] = {

341 seq_id: []

342 for seq_id in seq_ids

343 }

345 - expected_output_by_seq: Dict[int, List[SequenceOutput]] = {

344 + expected_output_by_seq: dict[int, list[SequenceOutput]] = {

345 seq_id: []

346 for seq_id in seq_ids

347 }

@@ -728,7 +727,7 @@ def test_populate_seq_ids_with_bonus_tokens():

727 size=(batch_size, (k + 1)),

728 dtype=torch.int64,

729 device='cuda')

731 - expected_request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)

730 + expected_request_id_seq_ids_mapping: dict[str, set[int]] = defaultdict(set)

731 for seq_group_metadata in seq_group_metadata_list:

732 for seq_id in seq_group_metadata.seq_data:

733 expected_request_id_seq_ids_mapping[

tests/spec_decode/utils.py

+16 -17

@@ -1,9 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import Sequence as GenericSequence

4 from itertools import count

4 -from typing import Callable, Dict, List, Optional

5 -from typing import Sequence as GenericSequence

6 -from typing import TypeVar, Union

5 +from typing import Callable, Optional, TypeVar, Union

6 from unittest.mock import MagicMock

7

8 import torch

@@ -44,7 +43,7 @@ def mock_worker(cls=None,

43 return worker

44

45

47 -def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):

46 +def patch_execute_model_with_seeds(worker: Worker, rand_seeds: list[int]):

47 seed_iter = iter(rand_seeds)

48 original_execute_model = worker.execute_model

49

@@ -56,7 +55,7 @@ def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):

55 return new_execute_model

56

57

59 -def zero_kv_cache(cache_engine: List[CacheEngine]):

58 +def zero_kv_cache(cache_engine: list[CacheEngine]):

59 assert cache_engine[0].gpu_cache

60 for key_blocks, value_blocks in cache_engine[0].gpu_cache:

61 key_blocks.zero_()

@@ -106,13 +105,13 @@ def create_worker(cls: Callable[..., T],

105

106

107 def create_seq_group_metadata_from_prompts(

109 - prompts: List[List[int]],

108 + prompts: list[list[int]],

109 num_gpu_blocks: int,

110 block_size: int,

112 - final_prompt_lens: List[int],

113 - continuations: Optional[List[List[int]]] = None,

114 - seq_ids: Optional[List[int]] = None,

115 -) -> List[SequenceGroupMetadata]:

111 + final_prompt_lens: list[int],

112 + continuations: Optional[list[list[int]]] = None,

113 + seq_ids: Optional[list[int]] = None,

114 +) -> list[SequenceGroupMetadata]:

115

116 if continuations is None:

117 continuations = [[] for _ in prompts]

@@ -149,11 +148,11 @@ def create_seq_group_metadata_from_prompts(

148

149

150 def create_chunked_seq_group_metadata_from_prompt(

152 - prompt: List[int],

151 + prompt: list[int],

152 num_gpu_blocks: int,

153 chunk_size: int,

154 block_size: int,

156 - seq_id: Optional[int] = None) -> List[SequenceGroupMetadata]:

155 + seq_id: Optional[int] = None) -> list[SequenceGroupMetadata]:

156

157 if seq_id is None:

158 seq_id = 0

@@ -184,8 +183,8 @@ def create_chunked_seq_group_metadata_from_prompt(

183

184

185 def assert_logprobs_dict_allclose(

187 - actual_logprobs: List[Dict[int, Logprob]],

188 - expected_logprobs: List[Dict[int, Logprob]]) -> None:

186 + actual_logprobs: list[dict[int, Logprob]],

187 + expected_logprobs: list[dict[int, Logprob]]) -> None:

188 for single_step_actual_logprobs, single_step_expected_logprobs in zip(

189 actual_logprobs, expected_logprobs):

190 assert set(single_step_actual_logprobs.keys()) == set(

@@ -202,7 +201,7 @@ def create_sampler_output_list(

201 token_ids: torch.Tensor,

202 probs: GenericSequence[Optional[torch.Tensor]],

203 logprobs: GenericSequence[Optional[torch.Tensor]],

205 - seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:

204 + seq_ids: Optional[list[int]] = None) -> list[SamplerOutput]:

205 num_steps, batch_size = token_ids.shape

206 token_ids_by_step = token_ids.tolist()

207

@@ -231,9 +230,9 @@ def create_sampler_output_list(

230

231 def create_batch(batch_size,

232 k,

234 - prompt_len: Union[int, List[int]] = 10,

233 + prompt_len: Union[int, list[int]] = 10,

234 prev_output_token_len: int = 10,

236 - seq_ids: Optional[List[int]] = None,

235 + seq_ids: Optional[list[int]] = None,

236 num_gpu_blocks: Optional[int] = None,

237 block_size: Optional[int] = None,

238 prefill_chunk_size: Optional[int] = None):

tests/test_cache_block_hashing.py

+3 -3

@@ -3,7 +3,7 @@

3

4 Run `pytest tests/test_cache_block_hashing.py`.

5 """

6 -from typing import List, Optional

6 +from typing import Optional

7

8 import pytest

9

@@ -44,7 +44,7 @@ def flatten_2d(li):

44 @pytest.mark.parametrize("concurrent_lora_int_ids",

45 [[None], [1], [None, 1], [None, 1, 2], [1, 2]])

46 def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,

47 - concurrent_lora_int_ids: List[Optional[int]]):

47 + concurrent_lora_int_ids: list[Optional[int]]):

48

49 tokenizer = TokenizerGroup(

50 tokenizer_id="facebook/opt-125m",

@@ -53,7 +53,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,

53 max_input_length=None,

54 )

55

56 - hashes: List[List[List[int]]] = []

56 + hashes: list[list[list[int]]] = []

57

58 for prefix in prefixes:

59 for lora_int_id in concurrent_lora_int_ids:

tests/test_inputs.py

+1 -3

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4

5 from vllm.inputs import zip_enc_dec_prompts

@@ -45,7 +43,7 @@ def test_parse_single_batch_string_consistent(string_input: str):

43

44

45 @pytest.mark.parametrize('token_input', TOKEN_INPUTS)

48 -def test_parse_single_batch_token_consistent(token_input: List[int]):

46 +def test_parse_single_batch_token_consistent(token_input: list[int]):

47 assert parse_and_batch_prompt(token_input) \

48 == parse_and_batch_prompt([token_input])

49

tests/test_logger.py

+1 -1

@@ -155,7 +155,7 @@ def test_an_error_is_raised_when_custom_logging_config_is_unexpected_json(

155 with pytest.raises(ValueError) as ex_info:

156 _configure_vllm_root_logger()

157 assert ex_info.type == ValueError # noqa: E721

158 - assert "Invalid logging config. Expected Dict, got" in str(ex_info)

158 + assert "Invalid logging config. Expected dict, got" in str(ex_info)

159

160

161 @patch("vllm.logger.VLLM_CONFIGURE_LOGGING", 1)

tests/test_logits_processor.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import Tuple

4 from unittest.mock import patch

5

6 import pytest

@@ -33,7 +32,7 @@ class MockLogitsProcessor(LogitsProcessor):

32

33 def _prepare_test(

34 batch_size: int

36 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:

35 +) -> tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:

36 vocab_size = 32000

37 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)

38 fake_logits = torch.full((batch_size, vocab_size),

tests/test_utils.py

+2 -2

@@ -3,7 +3,7 @@

3 import asyncio

4 import os

5 import socket

6 -from typing import AsyncIterator, Tuple

6 +from collections.abc import AsyncIterator

7 from unittest.mock import patch

8

9 import pytest

@@ -33,7 +33,7 @@ async def test_merge_async_iterators():

33 iterators = [mock_async_iterator(i) for i in range(3)]

34 merged_iterator = merge_async_iterators(*iterators)

35

36 - async def stream_output(generator: AsyncIterator[Tuple[int, str]]):

36 + async def stream_output(generator: AsyncIterator[tuple[int, str]]):

37 async for idx, output in generator:

38 print(f"idx: {idx}, output: {output}")

39

tests/tokenization/test_detokenize.py

+12 -11

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Any, Dict, Generator, List, Optional

3 +from collections.abc import Generator

4 +from typing import Any, Optional

5

6 import pytest

7 from transformers import AutoTokenizer

@@ -163,7 +164,7 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:

164

165 @pytest.fixture(name="complete_sequence_token_ids")

166 def create_complete_sequence_token_ids(complete_sequence: str,

166 - tokenizer) -> List[int]:

167 + tokenizer) -> list[int]:

168 complete_sequence_token_ids = tokenizer(complete_sequence).input_ids

169 return complete_sequence_token_ids

170

@@ -178,7 +179,7 @@ def create_sequence(prompt_token_ids=None):

179

180

181 def create_dummy_logprobs(

181 - complete_sequence_token_ids: List[int]) -> List[Dict[int, Logprob]]:

182 + complete_sequence_token_ids: list[int]) -> list[dict[int, Logprob]]:

183 return [{

184 token_id: Logprob(logprob=0.0),

185 token_id + 1: Logprob(logprob=0.1)

@@ -186,10 +187,10 @@ def create_dummy_logprobs(

187

188

189 def create_dummy_prompt_logprobs(

189 - complete_sequence_token_ids: List[int]

190 -) -> List[Optional[Dict[int, Any]]]:

190 + complete_sequence_token_ids: list[int]

191 +) -> list[Optional[dict[int, Any]]]:

192 # logprob for the first prompt token is None.

192 - logprobs: List[Optional[Dict[int, Any]]] = [None]

193 + logprobs: list[Optional[dict[int, Any]]] = [None]

194 logprobs.extend(create_dummy_logprobs(complete_sequence_token_ids)[1:])

195 return logprobs

196

@@ -198,7 +199,7 @@ def create_dummy_prompt_logprobs(

199 @pytest.mark.parametrize("tokenizer_name", TOKENIZERS)

200 @pytest.mark.parametrize("skip_special_tokens", [True, False], indirect=True)

201 def test_decode_sequence_logprobs(complete_sequence: str,

201 - complete_sequence_token_ids: List[int],

202 + complete_sequence_token_ids: list[int],

203 detokenizer: Detokenizer,

204 skip_special_tokens: bool):

205 """Verify Detokenizer decodes logprobs correctly."""

@@ -208,8 +209,8 @@ def test_decode_sequence_logprobs(complete_sequence: str,

209 # Run sequentially.

210 seq = create_sequence()

211 dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)

211 - sequential_logprobs_text_chosen_token: List[str] = []

212 - sequential_logprobs_text_other_token: List[str] = []

212 + sequential_logprobs_text_chosen_token: list[str] = []

213 + sequential_logprobs_text_other_token: list[str] = []

214 for new_token, logprobs in zip(complete_sequence_token_ids,

215 dummy_logprobs):

216 seq.append_token_id(new_token, logprobs)

@@ -232,7 +233,7 @@ def test_decode_sequence_logprobs(complete_sequence: str,

233

234 @pytest.mark.parametrize("complete_sequence", TRUTH)

235 @pytest.mark.parametrize("tokenizer_name", TOKENIZERS)

235 -def test_decode_prompt_logprobs(complete_sequence_token_ids: List[int],

236 +def test_decode_prompt_logprobs(complete_sequence_token_ids: list[int],

237 detokenizer: Detokenizer):

238 """Verify Detokenizer decodes prompt logprobs correctly."""

239 sampling_params = SamplingParams(skip_special_tokens=True,

@@ -249,7 +250,7 @@ def test_decode_prompt_logprobs(complete_sequence_token_ids: List[int],

250 dummy_logprobs,

251 position_offset=0)

252 # First logprob is None.

252 - decoded_prompt_logprobs: List[Dict[int, Any]] = dummy_logprobs[

253 + decoded_prompt_logprobs: list[dict[int, Any]] = dummy_logprobs[

254 1:] # type: ignore

255

256 # decoded_prompt_logprobs doesn't contain the first token.

tests/tokenization/test_tokenizer_group.py

+2 -2

@@ -3,7 +3,7 @@

3 import asyncio

4 import os

5 import sys

6 -from typing import List, Optional

6 +from typing import Optional

7 from unittest.mock import patch

8

9 import pytest

@@ -129,7 +129,7 @@ async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):

129

130 def __init__(self,

131 *args,

132 - fail_at: Optional[List[int]] = None,

132 + fail_at: Optional[list[int]] = None,

133 **kwargs):

134 super().__init__(*args, **kwargs)

135 self.i = 0

tests/tokenization/test_tokenizer_registry.py

+16 -16

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union

3 +from typing import TYPE_CHECKING, Any, Optional, Union

4

5 from vllm.transformers_utils.tokenizer import get_tokenizer

6 from vllm.transformers_utils.tokenizer_base import (TokenizerBase,

@@ -17,15 +17,15 @@ class TestTokenizer(TokenizerBase):

17 return TestTokenizer()

18

19 @property

20 - def all_special_tokens_extended(self) -> List[str]:

20 + def all_special_tokens_extended(self) -> list[str]:

21 raise NotImplementedError()

22

23 @property

24 - def all_special_tokens(self) -> List[str]:

24 + def all_special_tokens(self) -> list[str]:

25 raise NotImplementedError()

26

27 @property

28 - def all_special_ids(self) -> List[int]:

28 + def all_special_ids(self) -> list[int]:

29 raise NotImplementedError()

30

31 @property

@@ -58,7 +58,7 @@ class TestTokenizer(TokenizerBase):

58

59 def __call__(

60 self,

61 - text: Union[str, List[str], List[int]],

61 + text: Union[str, list[str], list[int]],

62 text_pair: Optional[str] = None,

63 add_special_tokens: bool = False,

64 truncation: bool = False,

@@ -66,10 +66,10 @@ class TestTokenizer(TokenizerBase):

66 ):

67 raise NotImplementedError()

68

69 - def get_vocab(self) -> Dict[str, int]:

69 + def get_vocab(self) -> dict[str, int]:

70 raise NotImplementedError()

71

72 - def get_added_vocab(self) -> Dict[str, int]:

72 + def get_added_vocab(self) -> dict[str, int]:

73 raise NotImplementedError()

74

75 def encode_one(

@@ -77,33 +77,33 @@ class TestTokenizer(TokenizerBase):

77 text: str,

78 truncation: bool = False,

79 max_length: Optional[int] = None,

80 - ) -> List[int]:

80 + ) -> list[int]:

81 raise NotImplementedError()

82

83 def encode(self,

84 text: str,

85 - add_special_tokens: Optional[bool] = None) -> List[int]:

85 + add_special_tokens: Optional[bool] = None) -> list[int]:

86 raise NotImplementedError()

87

88 def apply_chat_template(self,

89 - messages: List["ChatCompletionMessageParam"],

90 - tools: Optional[List[Dict[str, Any]]] = None,

91 - **kwargs) -> List[int]:

89 + messages: list["ChatCompletionMessageParam"],

90 + tools: Optional[list[dict[str, Any]]] = None,

91 + **kwargs) -> list[int]:

92 raise NotImplementedError()

93

94 - def convert_tokens_to_string(self, tokens: List[str]) -> str:

94 + def convert_tokens_to_string(self, tokens: list[str]) -> str:

95 raise NotImplementedError()

96

97 def decode(self,

98 - ids: Union[List[int], int],

98 + ids: Union[list[int], int],

99 skip_special_tokens: bool = True) -> str:

100 raise NotImplementedError()

101

102 def convert_ids_to_tokens(

103 self,

104 - ids: List[int],

104 + ids: list[int],

105 skip_special_tokens: bool = True,

106 - ) -> List[str]:

106 + ) -> list[str]:

107 raise NotImplementedError()

108

109

tests/tool_use/test_chat_completions.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import openai

4 import pytest

5

@@ -45,7 +43,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,

43 logprobs=False,

44 stream=True,

45 )

48 - chunks: List[str] = []

46 + chunks: list[str] = []

47 finish_reason_count = 0

48 role_sent: bool = False

49

@@ -116,7 +114,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,

114 stream=True,

115 )

116

119 - chunks: List[str] = []

117 + chunks: list[str] = []

118 finish_reason_count = 0

119 role_sent: bool = False

120

tests/tool_use/test_jamba_tool_parser.py

+7 -6

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 -from typing import Generator, List, Optional

4 +from collections.abc import Generator

5 +from typing import Optional

6

7 import partial_json_parser

8 import pytest

@@ -26,8 +27,8 @@ def jamba_tool_parser(jamba_tokenizer):

27 return JambaToolParser(jamba_tokenizer)

28

29

29 -def assert_tool_calls(actual_tool_calls: List[ToolCall],

30 - expected_tool_calls: List[ToolCall]):

30 +def assert_tool_calls(actual_tool_calls: list[ToolCall],

31 + expected_tool_calls: list[ToolCall]):

32 assert len(actual_tool_calls) == len(expected_tool_calls)

33

34 for actual_tool_call, expected_tool_call in zip(actual_tool_calls,

@@ -218,10 +219,10 @@ def test_extract_tool_calls_streaming(jamba_tool_parser, jamba_tokenizer,

219 model_output, expected_tool_calls,

220 expected_content):

221 other_content: str = ''

221 - function_names: List[str] = []

222 - function_args_strs: List[str] = []

222 + function_names: list[str] = []

223 + function_args_strs: list[str] = []

224 tool_call_idx: int = -1

224 - tool_call_ids: List[Optional[str]] = []

225 + tool_call_ids: list[Optional[str]] = []

226

227 for delta_message in stream_delta_message_generator(

228 jamba_tool_parser, jamba_tokenizer, model_output):

tests/tool_use/test_parallel_tool_calls.py

+5 -5

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 -from typing import Dict, List, Optional

4 +from typing import Optional

5

6 import openai

7 import pytest

@@ -54,7 +54,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,

54 assert isinstance(tool_call.function.arguments, str)

55

56 parsed_arguments = json.loads(tool_call.function.arguments)

57 - assert isinstance(parsed_arguments, Dict)

57 + assert isinstance(parsed_arguments, dict)

58 assert isinstance(parsed_arguments.get("city"), str)

59 assert isinstance(parsed_arguments.get("state"), str)

60

@@ -73,8 +73,8 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,

73 role_name: Optional[str] = None

74 finish_reason_count: int = 0

75

76 - tool_call_names: List[str] = []

77 - tool_call_args: List[str] = []

76 + tool_call_names: list[str] = []

77 + tool_call_args: list[str] = []

78 tool_call_idx: int = -1

79 tool_call_id_count: int = 0

80

@@ -180,7 +180,7 @@ async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,

180 logprobs=False,

181 stream=True)

182

183 - chunks: List[str] = []

183 + chunks: list[str] = []

184 finish_reason_count = 0

185 role_sent: bool = False

186

tests/tool_use/test_tool_calls.py

+5 -5

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 -from typing import Dict, List, Optional

4 +from typing import Optional

5

6 import openai

7 import pytest

@@ -44,7 +44,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):

44

45 # make sure the arguments parse properly

46 parsed_arguments = json.loads(tool_calls[0].function.arguments)

47 - assert isinstance(parsed_arguments, Dict)

47 + assert isinstance(parsed_arguments, dict)

48 assert isinstance(parsed_arguments.get("city"), str)

49 assert isinstance(parsed_arguments.get("state"), str)

50 assert parsed_arguments.get("city") == "Dallas"

@@ -117,7 +117,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):

117

118 # validate arguments

119 streamed_args = json.loads(function_args_str)

120 - assert isinstance(streamed_args, Dict)

120 + assert isinstance(streamed_args, dict)

121 assert isinstance(streamed_args.get("city"), str)

122 assert isinstance(streamed_args.get("state"), str)

123 assert streamed_args.get("city") == "Dallas"

@@ -128,7 +128,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):

128 assert choice.message.role == role_name

129 assert choice.message.tool_calls[0].function.name == function_name

130

131 - # compare streamed with non-streamed args Dict-wise, not string-wise

131 + # compare streamed with non-streamed args dict-wise, not string-wise

132 # because character-to-character comparison might not work e.g. the tool

133 # call parser adding extra spaces or something like that. we care about the

134 # dicts matching not byte-wise match

@@ -167,7 +167,7 @@ async def test_tool_call_with_results(client: openai.AsyncOpenAI):

167 logprobs=False,

168 stream=True)

169

170 - chunks: List[str] = []

170 + chunks: list[str] = []

171 finish_reason_count = 0

172 role_sent: bool = False

173

tests/tool_use/utils.py

+13 -13

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from copy import deepcopy

4 -from typing import Any, Dict, List, Optional

4 +from typing import Any, Optional

5

6 from openai.types.chat import (ChatCompletionMessageParam,

7 ChatCompletionToolParam)

@@ -12,14 +12,14 @@ from tests.utils import VLLM_PATH

12

13 class ServerConfig(TypedDict, total=False):

14 model: str

15 - arguments: List[str]

15 + arguments: list[str]

16 system_prompt: Optional[str]

17 supports_parallel: Optional[bool]

18 supports_rocm: Optional[bool]

19

20

21 -def patch_system_prompt(messages: List[Dict[str, Any]],

22 - system_prompt: str) -> List[Dict[str, Any]]:

21 +def patch_system_prompt(messages: list[dict[str, Any]],

22 + system_prompt: str) -> list[dict[str, Any]]:

23 new_messages = deepcopy(messages)

24 if new_messages[0]["role"] == "system":

25 new_messages[0]["content"] = system_prompt

@@ -28,8 +28,8 @@ def patch_system_prompt(messages: List[Dict[str, Any]],

28 return new_messages

29

30

31 -def ensure_system_prompt(messages: List[Dict[str, Any]],

32 - config: ServerConfig) -> List[Dict[str, Any]]:

31 +def ensure_system_prompt(messages: list[dict[str, Any]],

32 + config: ServerConfig) -> list[dict[str, Any]]:

33 prompt = config.get("system_prompt")

34 if prompt:

35 return patch_system_prompt(messages, prompt)

@@ -39,9 +39,9 @@ def ensure_system_prompt(messages: List[Dict[str, Any]],

39

40 # universal args for all models go here. also good if you need to test locally

41 # and change type or KV cache quantization or something.

42 -ARGS: List[str] = ["--enable-auto-tool-choice", "--max-model-len", "1024"]

42 +ARGS: list[str] = ["--enable-auto-tool-choice", "--max-model-len", "1024"]

43

44 -CONFIGS: Dict[str, ServerConfig] = {

44 +CONFIGS: dict[str, ServerConfig] = {

45 "hermes": {

46 "model":

47 "NousResearch/Hermes-3-Llama-3.1-8B",

@@ -205,7 +205,7 @@ SEARCH_TOOL: ChatCompletionToolParam = {

205 }

206 }

207

208 -MESSAGES_WITHOUT_TOOLS: List[ChatCompletionMessageParam] = [{

208 +MESSAGES_WITHOUT_TOOLS: list[ChatCompletionMessageParam] = [{

209 "role":

210 "user",

211 "content":

@@ -222,14 +222,14 @@ MESSAGES_WITHOUT_TOOLS: List[ChatCompletionMessageParam] = [{

222 "Can you tell me a joke please?"

223 }]

224

225 -MESSAGES_ASKING_FOR_TOOLS: List[ChatCompletionMessageParam] = [{

225 +MESSAGES_ASKING_FOR_TOOLS: list[ChatCompletionMessageParam] = [{

226 "role":

227 "user",

228 "content":

229 "What is the weather in Dallas, Texas in Fahrenheit?"

230 }]

231

232 -MESSAGES_WITH_TOOL_RESPONSE: List[ChatCompletionMessageParam] = [{

232 +MESSAGES_WITH_TOOL_RESPONSE: list[ChatCompletionMessageParam] = [{

233 "role":

234 "user",

235 "content":

@@ -258,7 +258,7 @@ MESSAGES_WITH_TOOL_RESPONSE: List[ChatCompletionMessageParam] = [{

258 "cloudy skies and a low chance of rain."

259 }]

260

261 -MESSAGES_ASKING_FOR_PARALLEL_TOOLS: List[ChatCompletionMessageParam] = [{

261 +MESSAGES_ASKING_FOR_PARALLEL_TOOLS: list[ChatCompletionMessageParam] = [{

262 "role":

263 "user",

264 "content":

@@ -266,7 +266,7 @@ MESSAGES_ASKING_FOR_PARALLEL_TOOLS: List[ChatCompletionMessageParam] = [{

266 "Fahrenheit?"

267 }]

268

269 -MESSAGES_WITH_PARALLEL_TOOL_RESPONSE: List[ChatCompletionMessageParam] = [{

269 +MESSAGES_WITH_PARALLEL_TOOL_RESPONSE: list[ChatCompletionMessageParam] = [{

270 "role":

271 "user",

272 "content":

tests/tracing/test_tracing.py

+3 -2

@@ -2,8 +2,9 @@

2

3 import os

4 import threading

5 +from collections.abc import Iterable

6 from concurrent import futures

6 -from typing import Callable, Dict, Iterable, Literal

7 +from typing import Callable, Literal

8

9 import grpc

10 import pytest

@@ -25,7 +26,7 @@ FieldName = Literal['bool_value', 'string_value', 'int_value', 'double_value',

26

27

28 def decode_value(value: AnyValue):

28 - field_decoders: Dict[FieldName, Callable] = {

29 + field_decoders: dict[FieldName, Callable] = {

30 "bool_value": (lambda v: v.bool_value),

31 "string_value": (lambda v: v.string_value),

32 "int_value": (lambda v: v.int_value),

tests/utils.py

+21 -21

@@ -11,7 +11,7 @@ import time

11 import warnings

12 from contextlib import contextmanager

13 from pathlib import Path

14 -from typing import Any, Callable, Dict, List, Optional, Type, Union

14 +from typing import Any, Callable, Optional, Union

15

16 import openai

17 import pytest

@@ -73,9 +73,9 @@ class RemoteOpenAIServer:

73

74 def __init__(self,

75 model: str,

76 - vllm_serve_args: List[str],

76 + vllm_serve_args: list[str],

77 *,

78 - env_dict: Optional[Dict[str, str]] = None,

78 + env_dict: Optional[dict[str, str]] = None,

79 auto_port: bool = True,

80 max_wait_seconds: Optional[float] = None) -> None:

81 if auto_port:

@@ -183,7 +183,7 @@ def _test_completion(

183 client: openai.OpenAI,

184 model: str,

185 prompt: str,

186 - token_ids: List[int],

186 + token_ids: list[int],

187 ):

188 results = []

189

@@ -400,10 +400,10 @@ def _test_image_text(

400

401

402 def compare_two_settings(model: str,

403 - arg1: List[str],

404 - arg2: List[str],

405 - env1: Optional[Dict[str, str]] = None,

406 - env2: Optional[Dict[str, str]] = None,

403 + arg1: list[str],

404 + arg2: list[str],

405 + env1: Optional[dict[str, str]] = None,

406 + env2: Optional[dict[str, str]] = None,

407 *,

408 method: str = "generate",

409 max_wait_seconds: Optional[float] = None) -> None:

@@ -429,8 +429,8 @@ def compare_two_settings(model: str,

429

430

431 def compare_all_settings(model: str,

432 - all_args: List[List[str]],

433 - all_envs: List[Optional[Dict[str, str]]],

432 + all_args: list[list[str]],

433 + all_envs: list[Optional[dict[str, str]]],

434 *,

435 method: str = "generate",

436 max_wait_seconds: Optional[float] = None) -> None:

@@ -470,7 +470,7 @@ def compare_all_settings(model: str,

470

471 prompt = "Hello, my name is"

472 token_ids = tokenizer(prompt).input_ids

473 - ref_results: List = []

473 + ref_results: list = []

474 for i, (args, env) in enumerate(zip(all_args, all_envs)):

475 if can_force_load_format:

476 # we are comparing the results and

@@ -481,7 +481,7 @@ def compare_all_settings(model: str,

481 # environment variable to force the load format,

482 # e.g. in quantization tests.

483 args = args + ["--load-format", envs.VLLM_TEST_FORCE_LOAD_FORMAT]

484 - compare_results: List = []

484 + compare_results: list = []

485 results = ref_results if i == 0 else compare_results

486 with RemoteOpenAIServer(model,

487 args,

@@ -582,7 +582,7 @@ def multi_process_parallel(

582

583

584 @contextmanager

585 -def error_on_warning(category: Type[Warning] = Warning):

585 +def error_on_warning(category: type[Warning] = Warning):

586 """

587 Within the scope of this context manager, tests will fail if any warning

588 of the given category is emitted.

@@ -604,7 +604,7 @@ def get_physical_device_indices(devices):

604

605

606 @_nvml()

607 -def wait_for_gpu_memory_to_clear(devices: List[int],

607 +def wait_for_gpu_memory_to_clear(devices: list[int],

608 threshold_bytes: int,

609 timeout_s: float = 120) -> None:

610 # Use nvml instead of pytorch to reduce measurement error from torch cuda

@@ -612,8 +612,8 @@ def wait_for_gpu_memory_to_clear(devices: List[int],

612 devices = get_physical_device_indices(devices)

613 start_time = time.time()

614 while True:

615 - output: Dict[int, str] = {}

616 - output_raw: Dict[int, float] = {}

615 + output: dict[int, str] = {}

616 + output_raw: dict[int, float] = {}

617 for device in devices:

618 if current_platform.is_rocm():

619 dev_handle = amdsmi_get_processor_handles()[device]

@@ -758,13 +758,13 @@ def multi_gpu_test(*, num_gpus: int):

758

759

760 async def completions_with_server_args(

761 - prompts: List[str],

761 + prompts: list[str],

762 model_name: str,

763 - server_cli_args: List[str],

763 + server_cli_args: list[str],

764 num_logprobs: Optional[int],

765 max_wait_seconds: int = 240,

766 max_tokens: Union[int, list] = 5,

767 -) -> List[Completion]:

767 +) -> list[Completion]:

768 '''Construct a remote OpenAI server, obtain an async client to the

769 server & invoke the completions API to obtain completions.

770

@@ -807,7 +807,7 @@ async def completions_with_server_args(

807 return outputs

808

809

810 -def get_client_text_generations(completions: List[Completion]) -> List[str]:

810 +def get_client_text_generations(completions: list[Completion]) -> list[str]:

811 '''Extract generated tokens from the output of a

812 request made to an Open-AI-protocol completions endpoint.

813 '''

@@ -816,7 +816,7 @@ def get_client_text_generations(completions: List[Completion]) -> List[str]:

816

817

818 def get_client_text_logprob_generations(

819 - completions: List[Completion]) -> List[TextTextLogprobs]:

819 + completions: list[Completion]) -> list[TextTextLogprobs]:

820 '''Operates on the output of a request made to an Open-AI-protocol

821 completions endpoint; obtains top-rank logprobs for each token in

822 each :class:`SequenceGroup`

tests/v1/core/test_prefix_caching.py

+1 -2

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Compare the with and without prefix caching."""

3 -from typing import List

3

4 import pytest

5

@@ -434,7 +433,7 @@ def test_cache_blocks():

433

434 # Test that blocks are cached correctly for 2 full blocks from the start.

435 blocks = [KVCacheBlock(block_id=i) for i in range(2)]

437 - block_hashes: List[BlockHashType] = []

436 + block_hashes: list[BlockHashType] = []

437

438 block_pool.cache_full_blocks(

439 request=req,

tests/v1/core/test_scheduler.py

+3 -3

@@ -1,5 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 -from typing import List, Optional

2 +from typing import Optional

3

4 from vllm.config import CacheConfig, ModelConfig, SchedulerConfig

5 from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange

@@ -48,9 +48,9 @@ def create_scheduler(

48 def create_requests(

49 num_requests: int,

50 num_tokens: int = 10,

51 - mm_positions: Optional[List[PlaceholderRange]] = None,

51 + mm_positions: Optional[list[PlaceholderRange]] = None,

52 max_tokens: int = 16,

53 - stop_token_ids: Optional[List[int]] = None,

53 + stop_token_ids: Optional[list[int]] = None,

54 ):

55 sampling_params = SamplingParams(ignore_eos=False,

56 max_tokens=max_tokens,

tests/v1/engine/conftest.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Tuple

4 -

3 import pytest

4 import torch

5 from transformers import AutoTokenizer

@@ -17,8 +15,8 @@ from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

15

16 from tests.v1.engine.utils import FULL_STRINGS # isort: skip

17

20 -EngineCoreSampleLogprobsType = List[Tuple[torch.Tensor, torch.Tensor]]

21 -EngineCorePromptLogprobsType = Tuple[torch.Tensor, torch.Tensor]

18 +EngineCoreSampleLogprobsType = list[tuple[torch.Tensor, torch.Tensor]]

19 +EngineCorePromptLogprobsType = tuple[torch.Tensor, torch.Tensor]

20

21

22 def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:

tests/v1/engine/test_async_llm.py

+5 -5

@@ -2,7 +2,7 @@

2

3 import asyncio

4 from contextlib import ExitStack

5 -from typing import List, Optional, Tuple

5 +from typing import Optional

6

7 import pytest

8

@@ -47,7 +47,7 @@ async def generate(engine: AsyncLLM,

47 prompt: PromptType,

48 output_kind: RequestOutputKind,

49 max_tokens: int,

50 - prompt_logprobs: Optional[int] = None) -> Tuple[int, str]:

50 + prompt_logprobs: Optional[int] = None) -> tuple[int, str]:

51 # Ensure generate doesn't complete too fast for cancellation test.

52 await asyncio.sleep(0.2)

53

@@ -114,7 +114,7 @@ async def test_async_llm_refuses_prompt_logprobs_with_apc(

114 (VISION_ENGINE_ARGS, VISION_PROMPT)])

115 @pytest.mark.asyncio

116 async def test_load(monkeypatch, output_kind: RequestOutputKind,

117 - engine_args_and_prompt: Tuple[AsyncEngineArgs,

117 + engine_args_and_prompt: tuple[AsyncEngineArgs,

118 PromptType]):

119 # TODO(rickyx): Remove monkeypatch once we have a better way to test V1

120 # so that in the future when we switch, we don't have to change all the

@@ -160,7 +160,7 @@ async def test_load(monkeypatch, output_kind: RequestOutputKind,

160 (VISION_ENGINE_ARGS, VISION_PROMPT)])

161 @pytest.mark.asyncio

162 async def test_abort(monkeypatch, output_kind: RequestOutputKind,

163 - engine_args_and_prompt: Tuple[AsyncEngineArgs,

163 + engine_args_and_prompt: tuple[AsyncEngineArgs,

164 PromptType]):

165

166 with monkeypatch.context() as m, ExitStack() as after:

@@ -177,7 +177,7 @@ async def test_abort(monkeypatch, output_kind: RequestOutputKind,

177 request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]

178

179 # Create concurrent requests.

180 - tasks: List[asyncio.Task] = []

180 + tasks: list[asyncio.Task] = []

181 for request_id in request_ids:

182 tasks.append(

183 asyncio.create_task(

tests/v1/engine/test_engine_core.py

+1 -2

@@ -5,7 +5,6 @@ import threading

5 import time

6 import uuid

7 from concurrent.futures import Future

8 -from typing import List

8

9 import pytest

10 from transformers import AutoTokenizer

@@ -213,7 +212,7 @@ def test_engine_core_concurrent_batches(monkeypatch):

212 class DummyExecutor(UniProcExecutor):

213

214 def initialize_from_config(

216 - self, kv_cache_configs: List[KVCacheConfig]) -> None:

215 + self, kv_cache_configs: list[KVCacheConfig]) -> None:

216 super().initialize_from_config(kv_cache_configs)

217

218 # This executor actually can only run 1 batch at a time

tests/v1/engine/test_engine_core_client.py

+5 -5

@@ -3,7 +3,7 @@

3 import asyncio

4 import time

5 import uuid

6 -from typing import Dict, List, Optional

6 +from typing import Optional

7

8 import pytest

9 from transformers import AutoTokenizer

@@ -44,7 +44,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:

44 )

45

46

47 -def loop_until_done(client: EngineCoreClient, outputs: Dict):

47 +def loop_until_done(client: EngineCoreClient, outputs: dict):

48

49 while True:

50 engine_core_outputs = client.get_output().outputs

@@ -62,7 +62,7 @@ def loop_until_done(client: EngineCoreClient, outputs: Dict):

62 break

63

64

65 -async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):

65 +async def loop_until_done_async(client: EngineCoreClient, outputs: dict):

66

67 while True:

68 engine_core_outputs = (await client.get_output_async()).outputs

@@ -121,7 +121,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):

121 client.add_request(request)

122 time.sleep(0.01)

123

124 - outputs: Dict[str, List] = {req_id: [] for req_id in request_ids}

124 + outputs: dict[str, list] = {req_id: [] for req_id in request_ids}

125 loop_until_done(client, outputs)

126

127 for req_id in request_ids:

@@ -207,7 +207,7 @@ async def test_engine_core_client_asyncio(monkeypatch):

207 await client.add_request_async(request)

208 await asyncio.sleep(0.01)

209

210 - outputs: Dict[str, List] = {req_id: [] for req_id in request_ids}

210 + outputs: dict[str, list] = {req_id: [] for req_id in request_ids}

211 await loop_until_done_async(client, outputs)

212

213 for req_id in request_ids:

tests/v1/engine/test_llm_engine.py

+4 -4

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import random

4 -from typing import Dict, List, Optional, Tuple

4 +from typing import Optional

5

6 import pytest

7

@@ -47,9 +47,9 @@ def vllm_model_apc(vllm_runner, monkeypatch):

47

48

49 def _get_test_sampling_params(

50 - prompt_list: List[str],

50 + prompt_list: list[str],

51 seed: Optional[int] = 42,

52 -) -> Tuple[List[SamplingParams], List[int]]:

52 +) -> tuple[list[SamplingParams], list[int]]:

53 """Generate random sampling params for a batch."""

54

55 def get_mostly_n_gt1() -> int:

@@ -81,7 +81,7 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:

81

82 # Validate each request response

83 for out, n in zip(outputs, n_list):

84 - completion_counts: Dict[str, int] = {}

84 + completion_counts: dict[str, int] = {}

85 # Assert correct number of completions

86 assert len(out.outputs) == n, (

87 f"{len(out.outputs)} completions; {n} expected.")

tests/v1/engine/test_output_processor.py

+6 -6

@@ -2,7 +2,7 @@

2

3 import math

4 import time

5 -from typing import Dict, List, Optional

5 +from typing import Optional

6

7 import pytest

8

@@ -112,12 +112,12 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind,

112

113

114 def _validate_logprobs(

115 - gen_tokens: Dict[str, List[int]],

116 - gen_logprobs: Dict[str, Optional[SampleLogprobs]],

117 - gen_prompt_logprobs: Dict[str, Optional[PromptLogprobs]],

118 - gen_cumulative_logprob: Dict[str, float],

115 + gen_tokens: dict[str, list[int]],

116 + gen_logprobs: dict[str, Optional[SampleLogprobs]],

117 + gen_prompt_logprobs: dict[str, Optional[PromptLogprobs]],

118 + gen_cumulative_logprob: dict[str, float],

119 dtv: DummyOutputProcessorTestVectors,

120 - request_id_list: List[str],

120 + request_id_list: list[str],

121 num_sample_logprobs: Optional[int],

122 num_prompt_logprobs: Optional[int],

123 ) -> None:

tests/v1/engine/utils.py

+25 -25

@@ -2,7 +2,7 @@

2

3 import random

4 from dataclasses import dataclass

5 -from typing import List, Optional, Tuple, Union

5 +from typing import Optional, Union

6

7 import torch

8 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast

@@ -61,7 +61,7 @@ def _create_random_top_logprob_test_vector(

61

62

63 def _create_random_top_logprob_test_matrix(

64 - shape: Tuple,

64 + shape: tuple,

65 lower: float,

66 upper: float,

67 ) -> torch.Tensor:

@@ -90,7 +90,7 @@ def _create_random_top_token_test_vector(

90 lower: int,

91 upper: int,

92 sampled_token_id: int,

93 - adjust_num_logprobs: bool = True) -> Tuple[torch.Tensor, int]:

93 + adjust_num_logprobs: bool = True) -> tuple[torch.Tensor, int]:

94 """Create a random vector of top logprob token indices

95

96 Use to create fake sample logprobs for testing. The sampled token

@@ -141,11 +141,11 @@ def _create_random_top_token_test_vector(

141

142

143 def _create_random_top_token_test_matrix(

144 - shape: Tuple[int, int],

144 + shape: tuple[int, int],

145 lower: int,

146 upper: int,

147 - tokens_list: List[int],

148 -) -> Tuple[torch.Tensor, torch.Tensor]:

147 + tokens_list: list[int],

148 +) -> tuple[torch.Tensor, torch.Tensor]:

149 """Create a random matrix of top logprob token indices

150

151 Use to create fake prompt logprobs for testing.

@@ -160,7 +160,7 @@ def _create_random_top_token_test_matrix(

160 upper: upper range of token ids

161

162 Returns:

163 - Tuple containing:

163 + tuple containing:

164 - 2D num_tokens x num_logprobs+1 torch Tensor of token ids

165 - 1D tensor of ranks of prompt tokens in their respective

166 rows, or random values

@@ -206,10 +206,10 @@ def decode_token(

206

207

208 def generate_dummy_sample_logprobs(

209 - sampled_tokens_list: List,

209 + sampled_tokens_list: list,

210 num_logprobs: int,

211 tokenizer: PreTrainedTokenizer,

212 -) -> List[Tuple[List[int], List[float], int]]:

212 +) -> list[tuple[list[int], list[float], int]]:

213 """Generate dummy sample logprobs

214

215 Generate a test data structure which imitates the list of sample logprobs

@@ -221,7 +221,7 @@ def generate_dummy_sample_logprobs(

221 tokenizer: model tokenizer to use for detokenization

222

223 Returns

224 - List of (top token ids vector, logprobs vector, sampled token rank)

224 + list of (top token ids vector, logprobs vector, sampled token rank)

225 Python lists tuples; in each tuple the logprobs and top token ids

226 vectors have the same length which is either `num_logprobs` or

227 `num_logprobs+1`. Sampled token rank is the rank (index+1) of the

@@ -253,7 +253,7 @@ def generate_dummy_sample_logprobs(

253

254

255 def generate_dummy_prompt_logprobs_tensors(

256 - prompt_tokens_list: List,

256 + prompt_tokens_list: list,

257 num_logprobs: int,

258 tokenizer: PreTrainedTokenizer,

259 ) -> LogprobsTensors:

@@ -269,7 +269,7 @@ def generate_dummy_prompt_logprobs_tensors(

269 tokenizer: model tokenizer to use for detokenization

270

271 Returns

272 - Single Tuple of (logprobs matrix, top token ids matrix) torch Tensor,

272 + Single tuple of (logprobs matrix, top token ids matrix) torch Tensor,

273 where both matrices have dimensions

274 num_prompt_tokens x num_logprobs

275 """

@@ -301,19 +301,19 @@ class DummyOutputProcessorTestVectors:

301 tokenizer: GeneralTokenizerType

302 tokenizer_group: BaseTokenizerGroup

303 vllm_config: EngineArgs

304 - full_tokens: List[List[int]] # Prompt + generated tokens

305 - prompt_tokens: List[List[int]]

306 - generation_tokens: List[List[int]]

304 + full_tokens: list[list[int]] # Prompt + generated tokens

305 + prompt_tokens: list[list[int]]

306 + generation_tokens: list[list[int]]

307 # Each request is associated with a tuple of

308 # (top tokens, top logprobs, ranks) prompt logprobs tensors

309 - prompt_logprobs: List[LogprobsTensors]

309 + prompt_logprobs: list[LogprobsTensors]

310 # Each request is associated with a sample logprobs; a request's

311 # sample logprobs are a list of (top tokens, top logprobs, ranks)

312 # sample logprobs tensors at each sequence position

313 - generation_logprobs: List[List[Tuple[List[int], List[float], int]]]

314 - prompt_strings: List[str]

315 - prompt_strings_len: List[int]

316 - generation_strings: List[str]

313 + generation_logprobs: list[list[tuple[list[int], list[float], int]]]

314 + prompt_strings: list[str]

315 + prompt_strings_len: list[int]

316 + generation_strings: list[str]

317

318

319 class MockEngineCore:

@@ -321,18 +321,18 @@ class MockEngineCore:

321

322 def __init__(

323 self,

324 - tokens_list: List[List[int]],

324 + tokens_list: list[list[int]],

325 # For each request, for each sampled token offset,

326 # a tuple of

327 # (list of topk token ids, list of sample logprob vals, rank)

328 - generated_logprobs_raw: Optional[List[List[Tuple[List[int],

329 - List[float],

328 + generated_logprobs_raw: Optional[list[list[tuple[list[int],

329 + list[float],

330 int]]]] = None,

331 # For each request, a tuple of

332 # (prompt logprob val matrix, prompt logprob tok id matrix);

333 # each matrix has dimensions

334 # (num prompt toks) x (num prompt logprobs+1)

335 - prompt_logprobs_raw: Optional[List[LogprobsTensors]] = None,

335 + prompt_logprobs_raw: Optional[list[LogprobsTensors]] = None,

336 ) -> None:

337 self.tokens_list = tokens_list

338 self.current_idx = 0

@@ -341,7 +341,7 @@ class MockEngineCore:

341 self.prompt_logprobs_raw = prompt_logprobs_raw

342 self.do_prompt_logprobs = prompt_logprobs_raw is not None

343

344 - def get_outputs(self) -> List[EngineCoreOutput]:

344 + def get_outputs(self) -> list[EngineCoreOutput]:

345 do_logprobs = self.do_logprobs

346 do_prompt_logprobs = self.do_prompt_logprobs

347 token_idx = self.current_idx

tests/v1/entrypoints/openai/test_completion.py

+6 -6

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import re

4 -from typing import Dict, List, Optional

4 +from typing import Optional

5

6 import openai # use the official client for correctness check

7 import pytest

@@ -193,7 +193,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,

193 async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,

194 model_name: str,

195 prompt_logprobs: Optional[int]):

196 - params: Dict = {

196 + params: dict = {

197 "prompt": ["A robot may not injure another robot", "My name is"],

198 "model": model_name,

199 }

@@ -237,7 +237,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,

237 max_tokens=5,

238 temperature=0.0,

239 stream=True)

240 - chunks: List[str] = []

240 + chunks: list[str] = []

241 finish_reason_count = 0

242 async for chunk in stream:

243 chunks.append(chunk.choices[0].text)

@@ -278,7 +278,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI,

278 num_completions = len(completion.choices)

279 assert num_completions == n, (

280 f"Num completions {num_completions} but expected {n}.")

281 - completion_repeats: Dict[str, int] = {}

281 + completion_repeats: dict[str, int] = {}

282 for idx, choice in enumerate(completion.choices):

283 # Assert correct completion index & some finish reason.

284 assert choice.index == idx, (

@@ -321,7 +321,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):

321 temperature=0.95,

322 stream=True,

323 seed=42)

324 - chunks: List[List[str]] = [[] for i in range(n)]

324 + chunks: list[list[str]] = [[] for i in range(n)]

325 finish_reason_count = 0

326 async for chunk in stream:

327 index = chunk.choices[0].index

@@ -332,7 +332,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):

332 # Assert `n` completions with correct finish reasons

333 assert finish_reason_count == n, (

334 f"Expected {n} completions with valid indices and finish_reason.")

335 - completion_repeats: Dict[str, int] = {}

335 + completion_repeats: dict[str, int] = {}

336 for chunk in chunks:

337 chunk_len = len(chunk)

338 # Assert correct number of completion tokens

tests/v1/sample/test_logprobs.py

+4 -5

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import itertools

4 -from typing import List, Tuple

4

5 import pytest

6 import torch

@@ -46,8 +45,8 @@ def hf_model(hf_runner):

45

46 def _repeat_logprob_config(

47 test_prompts,

49 - logprob_prompt_logprob_list: List[Tuple],

50 -) -> List[Tuple]:

48 + logprob_prompt_logprob_list: list[tuple],

49 +) -> list[tuple]:

50 """Ensure each test prompt has a logprob config.

51

52 A logprob config specifies the optional (i.e.

@@ -74,7 +73,7 @@ def _repeat_logprob_config(

73 tuples

74

75 Returns:

77 - List of

76 + list of

77 (optional num sample logprob,optional num prompt logprob)

78 tuples which is either identical to

79 `logprob_prompt_logprob_list`, or else repeats

@@ -177,7 +176,7 @@ def _test_case_get_logprobs_and_prompt_logprobs(

176 for r in range(1, num_top_logprobs + 1))

177

178 output_text = vllm_result.outputs[0].text

180 - output_string_from_most_likely_tokens_lst: List[str] = []

179 + output_string_from_most_likely_tokens_lst: list[str] = []

180 for top_logprobs in vllm_result.outputs[0].logprobs:

181 top_logprob = next(iter(top_logprobs.values()))

182 output_string_from_most_likely_tokens_lst.append(

tests/v1/sample/test_rejection_sampler.py

+3 -4

@@ -1,5 +1,4 @@

1 # SPDX-License-Identifier: Apache-2.0

2 -from typing import List

2

3 import pytest

4 import torch

@@ -13,7 +12,7 @@ def sampler():

12 return RejectionSampler()

13

14

16 -def create_logits_tensor(token_ids: List[int],

15 +def create_logits_tensor(token_ids: list[int],

16 vocab_size: int = 100) -> torch.Tensor:

17 """Helper function to create logits tensor that

18 will produce desired token ids on argmax"""

@@ -23,7 +22,7 @@ def create_logits_tensor(token_ids: List[int],

22 return logits

23

24

26 -def create_sampling_metadata(spec_tokens: List[List[int]]) -> SamplingMetadata:

25 +def create_sampling_metadata(spec_tokens: list[list[int]]) -> SamplingMetadata:

26 batch_size = len(spec_tokens)

27 return SamplingMetadata(

28 temperature=torch.tensor([]),

@@ -106,7 +105,7 @@ def test_single_token_sequence(sampler):

105

106 def test_empty_sequence(sampler):

107 """Test handling empty sequence of speculated tokens"""

109 - spec_tokens: List[List[int]] = [[]]

108 + spec_tokens: list[list[int]] = [[]]

109 output_tokens = [5] # Just the bonus token

110

111 metadata = create_sampling_metadata(spec_tokens)

tests/v1/sample/test_sampler.py

+13 -13

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List, Optional, Set, Tuple

3 +from typing import Optional

4

5 import numpy as np

6 import pytest

@@ -32,7 +32,7 @@ def _create_penalty_tensor(batch_size: int, penalty_value: float,

32

33

34 def _create_prompt_tokens_tensor(

35 - prompt_token_ids: List[List[int]],

35 + prompt_token_ids: list[list[int]],

36 vocab_size: int,

37 device: torch.device,

38 ) -> torch.Tensor:

@@ -49,8 +49,8 @@ def _create_logit_bias(

49 batch_size: int,

50 vocab_size: int,

51 bias_value: float,

52 -) -> List[Optional[Dict[int, float]]]:

53 - res: List[Optional[Dict[int, float]]] = []

52 +) -> list[Optional[dict[int, float]]]:

53 + res: list[Optional[dict[int, float]]] = []

54 for i in range(batch_size):

55 logit_bias = {min(i, vocab_size - 1): bias_value}

56 res.append(logit_bias)

@@ -83,8 +83,8 @@ def _create_default_sampling_metadata(

83 vocab_size: int,

84 device: torch.device,

85 ) -> SamplingMetadata:

86 - output_token_ids: List[List[int]] = []

87 - prompt_token_ids: List[List[int]] = []

86 + output_token_ids: list[list[int]] = []

87 + prompt_token_ids: list[list[int]] = []

88 for _ in range(batch_size):

89 output_token_ids.append(

90 np.random.randint(0, vocab_size, size=num_output_tokens).tolist())

@@ -118,8 +118,8 @@ def _create_default_sampling_metadata(

118

119 def _generate_min_token_penalties_and_stop_tokens(

120 num_output_tokens: int, batch_size: int, vocab_size: int,

121 - batch_indices_for_min_token_penalty: List[int]

122 -) -> Dict[int, Tuple[int, Set[int]]]:

121 + batch_indices_for_min_token_penalty: list[int]

122 +) -> dict[int, tuple[int, set[int]]]:

123 """

124 Generates and returns a dict of minimum token penalties and

125 corresponding stop token IDs (`min_tokens`, `stop_token_ids`) for each

@@ -130,7 +130,7 @@ def _generate_min_token_penalties_and_stop_tokens(

130 and a random set of stop token IDs is created. Otherwise, a lower

131 `min_tokens` value is assigned, and the stop token IDs set is empty.

132 """

133 - min_tokens: Dict[int, Tuple[int, Set[int]]] = {}

133 + min_tokens: dict[int, tuple[int, set[int]]] = {}

134 for index in range(batch_size):

135 if index in batch_indices_for_min_token_penalty:

136 min_tokens[index] = (

@@ -147,7 +147,7 @@ def _generate_min_token_penalties_and_stop_tokens(

147

148 def _create_weighted_output_token_list(

149 batch_size: int,

150 - vocab_size: int) -> Tuple[List[List[int]], List[List[int]]]:

150 + vocab_size: int) -> tuple[list[list[int]], list[list[int]]]:

151 """

152 Creates an output token list where each token occurs a distinct

153 number of times.

@@ -157,7 +157,7 @@ def _create_weighted_output_token_list(

157 list, each with a different frequency.

158

159 Returns:

160 - Tuple[List[List[int]], List[List[int]]]:

160 + tuple[list[list[int]], list[list[int]]]:

161 - The first element is the output token list, where each sublist

162 corresponds to a batch and contains tokens with weighted

163 frequencies.

@@ -165,8 +165,8 @@ def _create_weighted_output_token_list(

165 batch, ordered by their frequency in the corresponding output

166 list.

167 """

168 - output_token_ids: List[List[int]] = []

169 - sorted_token_ids_in_output: List[List[int]] = []

168 + output_token_ids: list[list[int]] = []

169 + sorted_token_ids_in_output: list[list[int]] = []

170 for _ in range(batch_size):

171 distinct_token_ids = np.random.choice(vocab_size,

172 size=np.random.randint(1, 10),

tests/v1/sample/utils.py

+2 -3

@@ -1,12 +1,11 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import re

4 -from typing import List, Tuple

4

5 from vllm import CompletionOutput

6

7

9 -def get_test_batch(batch_logprobs_composition: str) -> List[Tuple]:

8 +def get_test_batch(batch_logprobs_composition: str) -> list[tuple]:

9 """Generate logprobs configs for a batch of requests

10

11 A given request's logprobs configuration is (1) num_sample_logprobs and (2)

@@ -32,7 +31,7 @@ def get_test_batch(batch_logprobs_composition: str) -> List[Tuple]:

31

32 Returns:

33

35 - List of (Optional[num_sample_logprobs], Optional[num_prompt_logprobs])

34 + list of (Optional[num_sample_logprobs], Optional[num_prompt_logprobs])

35 tuples

36 """

37 if batch_logprobs_composition == "NONE":

tests/v1/test_utils.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import torch

4

5 from vllm.v1.utils import bind_kv_cache

@@ -22,7 +20,7 @@ def test_bind_kv_cache():

20 'layers.2.self_attn': torch.zeros((1, )),

21 'layers.3.self_attn': torch.zeros((1, )),

22 }

25 - runner_kv_caches: List[torch.Tensor] = []

23 + runner_kv_caches: list[torch.Tensor] = []

24 bind_kv_cache(kv_cache, ctx, runner_kv_caches)

25 assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[

26 'layers.0.self_attn']

@@ -52,7 +50,7 @@ def test_bind_kv_cache_non_attention():

50 'model.layers.28.attn': torch.zeros((1, )),

51 }

52

55 - runner_kv_caches: List[torch.Tensor] = []

53 + runner_kv_caches: list[torch.Tensor] = []

54 bind_kv_cache(kv_cache, ctx, runner_kv_caches)

55

56 assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[

tests/v1/worker/test_gpu_input_batch.py

+11 -11

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List, Optional, Set, Tuple

3 +from typing import Optional

4

5 import numpy as np

6 import pytest

@@ -22,22 +22,22 @@ MAX_NUM_PROMPT_TOKENS = 64

22

23 def _remove_requests(

24 input_batch: InputBatch, batch_size: int,

25 - reqs: List[CachedRequestState]) -> Tuple[Set[str], List[int]]:

25 + reqs: list[CachedRequestState]) -> tuple[set[str], list[int]]:

26 """

27 - Remove some requests randomly from the batch and returns a Tuple

27 + Remove some requests randomly from the batch and returns a tuple

28 of 1) set of request removed 2) indices of the requests removed

29 ordered in descending order

30 """

31

32 num_reqs_to_remove = np.random.randint(0, batch_size)

33 - req_indices_to_remove: Set[int] = set()

33 + req_indices_to_remove: set[int] = set()

34 for _ in range(num_reqs_to_remove):

35 req_index_to_remove = np.random.randint(0, batch_size)

36 req_indices_to_remove.add(req_index_to_remove)

37

38 req_indices_to_remove_list = list(req_indices_to_remove)

39 req_indices_to_remove_list.sort(reverse=True)

40 - req_ids_to_remove: Set[str] = set()

40 + req_ids_to_remove: set[str] = set()

41 for index in req_indices_to_remove:

42 input_batch.remove_request(reqs[index].req_id)

43 req_ids_to_remove.add(reqs[index].req_id)

@@ -45,9 +45,9 @@ def _remove_requests(

45

46

47 def _construct_expected_sampling_metadata(

48 - reqs: List[CachedRequestState],

49 - req_ids_retained: Set[int],

50 - req_id_index_in_input_batch: Dict[str, int],

48 + reqs: list[CachedRequestState],

49 + req_ids_retained: set[int],

50 + req_id_index_in_input_batch: dict[str, int],

51 device: torch.device,

52 ) -> SamplingMetadata:

53 """

@@ -55,8 +55,8 @@ def _construct_expected_sampling_metadata(

55 batch.

56 """

57 num_reqs = len(req_ids_retained)

58 - output_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]

59 - prompt_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]

58 + output_token_ids: list[list[int]] = [list() for _ in range(num_reqs)]

59 + prompt_token_ids: list[list[int]] = [list() for _ in range(num_reqs)]

60 presence_penalties = [0.0 for _ in range(num_reqs)]

61 frequency_penalties = [0.0 for _ in range(num_reqs)]

62 repetition_penalties = [1.0 for _ in range(num_reqs)]

@@ -191,7 +191,7 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):

191 pin_memory=is_pin_memory_available(),

192 vocab_size=1024,

193 )

194 - reqs: List[CachedRequestState] = []

194 + reqs: list[CachedRequestState] = []

195 req_id_reqs = {}

196 req_id_output_token_ids = {}

197 # Add requests

tests/vllm_test_utils/vllm_test_utils/blame.py

+2 -1

@@ -4,7 +4,8 @@ import contextlib

4 import dataclasses

5 import sys

6 import traceback

7 -from typing import Callable, Generator

7 +from collections.abc import Generator

8 +from typing import Callable

9

10

11 @dataclasses.dataclass

tests/vllm_test_utils/vllm_test_utils/monitor.py

+2 -1

@@ -4,7 +4,8 @@ import contextlib

4 import dataclasses

5 import sys

6 import traceback

7 -from typing import Callable, Generator, Generic, TypeVar

7 +from collections.abc import Generator

8 +from typing import Callable, Generic, TypeVar

9

10 _T = TypeVar("_T")

11

tests/worker/test_encoder_decoder_model_runner.py

+10 -11

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import itertools

4 -from typing import List

4

5 import pytest

6 import torch

@@ -43,7 +42,7 @@ def test_empty_seq_group():

42 enable_chunked_prefill=False,

43 enforce_eager=True,

44 )

46 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

45 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

46 model_input = model_runner._prepare_model_input_tensors(

47 seq_group_metadata_list)

48 (

@@ -103,9 +102,9 @@ def test_prepare_prompt(batch_size):

102 enforce_eager=True,

103 )

104

106 - seq_lens: List[int] = []

107 - encoder_seq_lens: List[int] = []

108 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

105 + seq_lens: list[int] = []

106 + encoder_seq_lens: list[int] = []

107 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

108 block_tables = {0: [1]}

109 cross_block_table = [2]

110 for i in range(batch_size):

@@ -295,9 +294,9 @@ def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):

294 enforce_eager=True,

295 )

296

298 - seq_lens: List[int] = []

299 - encoder_seq_lens: List[int] = []

300 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

297 + seq_lens: list[int] = []

298 + encoder_seq_lens: list[int] = []

299 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

300 block_tables = {

301 0: [1],

302 1: [3]

@@ -503,9 +502,9 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):

502 } if multiple_seqs_per_seq_group else {

503 0: [1]

504 }

506 - seq_lens: List[int] = []

507 - encoder_seq_lens: List[int] = []

508 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

505 + seq_lens: list[int] = []

506 + encoder_seq_lens: list[int] = []

507 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

508

509 cross_block_table = [2]

510 expanded_batch_size = 0

tests/worker/test_model_input.py

+5 -6

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import dataclasses

4 -from typing import List, Tuple, Type

4

5 import torch

6

@@ -27,15 +26,15 @@ class MockAttentionBackend(AttentionBackend):

26 raise NotImplementedError

27

28 @staticmethod

30 - def get_metadata_cls() -> Type["AttentionMetadata"]:

29 + def get_metadata_cls() -> type["AttentionMetadata"]:

30 return AttentionMetadata

31

32 @staticmethod

34 - def get_builder_cls() -> Type["AttentionMetadataBuilder"]:

33 + def get_builder_cls() -> type["AttentionMetadataBuilder"]:

34 return AttentionMetadataBuilder

35

36 @staticmethod

38 - def get_state_cls() -> Type["CommonAttentionState"]:

37 + def get_state_cls() -> type["CommonAttentionState"]:

38 return CommonAttentionState

39

40 @staticmethod

@@ -44,7 +43,7 @@ class MockAttentionBackend(AttentionBackend):

43 block_size: int,

44 num_kv_heads: int,

45 head_size: int,

47 - ) -> Tuple[int, ...]:

46 + ) -> tuple[int, ...]:

47 raise NotImplementedError

48

49 @staticmethod

@@ -57,7 +56,7 @@ class MockAttentionBackend(AttentionBackend):

56

57 @staticmethod

58 def copy_blocks(

60 - kv_caches: List[torch.Tensor],

59 + kv_caches: list[torch.Tensor],

60 src_to_dists: torch.Tensor,

61 ) -> None:

62 pass

tests/worker/test_model_runner.py

+9 -11

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import pytest

4 import torch

5

@@ -42,8 +40,8 @@ def test_prepare_prompt(batch_size):

40 enable_chunked_prefill=False,

41 )

42

45 - seq_lens: List[int] = []

46 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

43 + seq_lens: list[int] = []

44 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

45 block_tables = {0: [1]}

46 for i in range(batch_size):

47 # make sure all tokens fit into one block

@@ -159,8 +157,8 @@ def test_prepare_decode_cuda_graph(batch_size):

157 enable_chunked_prefill=False,

158 )

159

162 - context_lens: List[int] = []

163 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

160 + context_lens: list[int] = []

161 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

162 # Assume each seq group finishes prefill.

163 for i in range(batch_size):

164 # make sure all tokens fit into one block

@@ -265,7 +263,7 @@ def test_empty_seq_group():

263 dtype="float16",

264 enforce_eager=False,

265 )

268 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

266 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

267 model_input = model_runner._prepare_model_input_tensors(

268 seq_group_metadata_list)

269 input_tokens, input_positions, attn_metadata = (

@@ -315,10 +313,10 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):

313 )

314

315 # Add prefill requests.

318 - seq_lens: List[int] = []

319 - seq_group_metadata_list: List[SequenceGroupMetadata] = []

320 - prefill_metadata_list: List[SequenceGroupMetadata] = []

321 - decode_metadata_list: List[SequenceGroupMetadata] = []

316 + seq_lens: list[int] = []

317 + seq_group_metadata_list: list[SequenceGroupMetadata] = []

318 + prefill_metadata_list: list[SequenceGroupMetadata] = []

319 + decode_metadata_list: list[SequenceGroupMetadata] = []

320 block_tables = {0: [1]}

321 prefill_batch_size = batch_size // 2

322 decode_batch_size = batch_size - prefill_batch_size

tools/profiler/print_layerwise_table.py

+1 -2

@@ -2,13 +2,12 @@

2

3 import argparse

4 import json

5 -from typing import Dict

5

6 from vllm.profiler.layerwise_profile import ModelStatsEntry, SummaryStatsEntry

7 from vllm.profiler.utils import TablePrinter, indent_string

8

9

11 -def flatten_entries(entry_cls, profile_dict: Dict):

10 +def flatten_entries(entry_cls, profile_dict: dict):

11 entries_and_depth = []

12

13 def get_entries(node, curr_depth=0):

tools/profiler/visualize_layerwise_profile.py

+7 -7

@@ -6,7 +6,7 @@ import json

6 import math

7 import os

8 from pathlib import Path

9 -from typing import Any, List, Optional, Tuple

9 +from typing import Any, Optional

10

11 import matplotlib.pyplot as plt

12 import pandas as pd

@@ -24,7 +24,7 @@ def largest_dist_from_leaf(node: dict, depth: int = 0):

24

25

26 def get_entries_at_depth(depth: int,

27 - entries_and_traces: List[Tuple[Any, Any]],

27 + entries_and_traces: list[tuple[Any, Any]],

28 node: dict,

29 curr_depth: int = 0,

30 trace=()):

@@ -48,9 +48,9 @@ def get_entries_at_depth(depth: int,

48 trace=trace)

49

50

51 -def fold_nodes(root: dict, nodes_to_fold: List[str]):

51 +def fold_nodes(root: dict, nodes_to_fold: list[str]):

52

53 - stack: List[dict] = [root]

53 + stack: list[dict] = [root]

54 while len(stack) != 0:

55 node = stack.pop()

56 if node['entry']['name'] in nodes_to_fold:

@@ -427,12 +427,12 @@ def main(

427 plot_metric: str,

428 make_names_unique: bool,

429 top_k: int,

430 - json_nodes_to_fold: List[str]):

430 + json_nodes_to_fold: list[str]):

431

432 - def prepare_data(profile_json: dict, step_keys: List[str]) -> pd.DataFrame:

432 + def prepare_data(profile_json: dict, step_keys: list[str]) -> pd.DataFrame:

433

434 def get_entries_and_traces(key: str):

435 - entries_and_traces: List[Tuple[Any, Any]] = []

435 + entries_and_traces: list[tuple[Any, Any]] = []

436 for root in profile_json[key]["summary_stats"]:

437 # Fold nodes in the traces as per user request. i.e. simply

438 # make the requested nodes leaf-nodes.

vllm/_custom_ops.py

+28 -28

@@ -2,7 +2,7 @@

2

3 import contextlib

4 import importlib

5 -from typing import TYPE_CHECKING, List, Optional, Tuple, Union

5 +from typing import TYPE_CHECKING, Optional, Union

6

7 import torch

8 import torch.library

@@ -198,7 +198,7 @@ def rms_norm_dynamic_per_token_quant(

198 quant_dtype: torch.dtype,

199 scale_ub: Optional[torch.Tensor] = None,

200 residual: Optional[torch.Tensor] = None

201 -) -> Tuple[torch.Tensor, torch.Tensor]:

201 +) -> tuple[torch.Tensor, torch.Tensor]:

202 output = torch.empty_like(input, dtype=quant_dtype)

203 scales = torch.empty((input.numel() // input.shape[-1], 1),

204 device=input.device,

@@ -347,7 +347,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):

347 @register_fake("_C::aqlm_gemm")

348 def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor,

349 codebooks: torch.Tensor, scales: torch.Tensor,

350 - codebook_partition_sizes: List[int],

350 + codebook_partition_sizes: list[int],

351 bias: Optional[torch.Tensor]) -> torch.Tensor:

352 out_features = codes.size(0) * codebooks.size(2)

353 flat_input = input.reshape((-1, input.size(-1)))

@@ -363,7 +363,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):

363 @register_fake("_C::aqlm_dequant")

364 def _aqlm_dequant_fake(

365 codes: torch.Tensor, codebooks: torch.Tensor,

366 - codebook_partition_sizes: List[int]) -> torch.Tensor:

366 + codebook_partition_sizes: list[int]) -> torch.Tensor:

367 in_features = codes.size(1) * 8

368 out_features = codes.size(0)

369 return torch.empty((out_features, in_features),

@@ -554,7 +554,7 @@ def cutlass_sparse_scaled_mm_supported(cuda_device_capability: int) -> bool:

554

555

556 def cutlass_sparse_compress(a: torch.Tensor) \

557 - -> Tuple[torch.Tensor, torch.Tensor]:

557 + -> tuple[torch.Tensor, torch.Tensor]:

558 """

559 Compresses a sparse matrix for use with Cutlass sparse operations.

560

@@ -571,7 +571,7 @@ def cutlass_sparse_compress(a: torch.Tensor) \

571 - `torch.float16`

572

573 Returns:

574 - Tuple[torch.Tensor, torch.Tensor]:

574 + tuple[torch.Tensor, torch.Tensor]:

575 A tuple containing:

576 - `a_nzs` (torch.Tensor): A tensor containing non-zero elements of `a`.

577 - `a_meta` (torch.Tensor): A tensor containing metadata for the sparse representation.

@@ -646,14 +646,14 @@ def cutlass_scaled_sparse_mm(

646 # aqlm

647 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,

648 codebooks: torch.Tensor, scales: torch.Tensor,

649 - codebook_partition_sizes: List[int],

649 + codebook_partition_sizes: list[int],

650 bias: Optional[torch.Tensor]) -> torch.Tensor:

651 return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,

652 codebook_partition_sizes, bias)

653

654

655 def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,

656 - codebook_partition_sizes: List[int]) -> torch.Tensor:

656 + codebook_partition_sizes: list[int]) -> torch.Tensor:

657 return torch.ops._C.aqlm_dequant(codes, codebooks,

658 codebook_partition_sizes)

659

@@ -738,7 +738,7 @@ def machete_supported_schedules(

738 group_zeros_type: Optional[torch.dtype] = None,

739 channel_scales_type: Optional[torch.dtype] = None,

740 token_scales_type: Optional[torch.dtype] = None,

741 - out_type: Optional[torch.dtype] = None) -> List[str]:

741 + out_type: Optional[torch.dtype] = None) -> list[str]:

742 return torch.ops._C.machete_supported_schedules(

743 a_type, b_type.id, group_scales_type, group_zeros_type,

744 channel_scales_type, token_scales_type, out_type)

@@ -783,7 +783,7 @@ def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:

783 # fp4

784 def scaled_fp4_quant(

785 input: torch.Tensor,

786 - input_global_scale: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:

786 + input_global_scale: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:

787 """

788 Quantize input tensor to FP4 and return quantized tensor and scale.

789

@@ -798,7 +798,7 @@ def scaled_fp4_quant(

798 input_global_scale: A scalar scaling factor for the entire tensor.

799

800 Returns:

801 - Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every

801 + tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every

802 two values are packed into a uint8 and float8_e4m3 scaling factors

803 in the sizzled layout.

804 """

@@ -845,7 +845,7 @@ def scaled_fp8_quant(

845 num_token_padding: Optional[int] = None,

846 scale_ub: Optional[torch.Tensor] = None,

847 use_per_token_if_dynamic: bool = False,

848 -) -> Tuple[torch.Tensor, torch.Tensor]:

848 +) -> tuple[torch.Tensor, torch.Tensor]:

849 """

850 Quantize input tensor to FP8 and return quantized tensor and scale.

851

@@ -866,12 +866,12 @@ def scaled_fp8_quant(

866 in the dynamic quantization case.

867

868 Returns:

869 - Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and

869 + tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and

870 scaling factor.

871 """

872 # This code assumes batch_dim and num_tokens are flattened

873 assert (input.ndim == 2)

874 - shape: Union[Tuple[int, int], torch.Size] = input.shape

874 + shape: Union[tuple[int, int], torch.Size] = input.shape

875 # For rocm, the output fp8 dtype is torch.float_e3m3fnuz

876 out_dtype: torch.dtype = torch.float8_e4m3fnuz \

877 if current_platform.is_rocm() else torch.float8_e4m3fn

@@ -903,7 +903,7 @@ def allspark_repack_weight(

903 scale: torch.Tensor,

904 zero_point: Optional[torch.Tensor] = None,

905 has_zp: bool = False

906 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

906 +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

907 """

908 Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format

909 for Ampere W8A16 Fused Gemm kernel

@@ -917,7 +917,7 @@ def allspark_repack_weight(

917 if use asymmetric quantization, has_zp = True.

918

919 Returns:

920 - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] :

920 + tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] :

921 rearranged weight, scale, and optionally zero_point.

922 """

923 K = qweight.shape[0]

@@ -964,7 +964,7 @@ def scaled_int8_quant(

964 scale: Optional[torch.Tensor] = None,

965 azp: Optional[torch.Tensor] = None,

966 symmetric: bool = True

967 -) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

967 +) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:

968 """

969 Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.

970

@@ -977,7 +977,7 @@ def scaled_int8_quant(

977 symmetric: Whether to use symmetric quantization (scale only, azp ignored).

978

979 Returns:

980 - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] : Output int8 tensor, scales, and optionally azp.

980 + tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] : Output int8 tensor, scales, and optionally azp.

981 """

982 output = torch.empty_like(input, dtype=torch.int8)

983 if scale is not None:

@@ -1165,13 +1165,13 @@ def concat_and_cache_mla(

1165 scale)

1166

1167

1168 -def copy_blocks(key_caches: List[torch.Tensor],

1169 - value_caches: List[torch.Tensor],

1168 +def copy_blocks(key_caches: list[torch.Tensor],

1169 + value_caches: list[torch.Tensor],

1170 block_mapping: torch.Tensor) -> None:

1171 torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)

1172

1173

1174 -def copy_blocks_mla(kv_caches: List[torch.Tensor],

1174 +def copy_blocks_mla(kv_caches: list[torch.Tensor],

1175 block_mapping: torch.Tensor) -> None:

1176 torch.ops._C_cache_ops.copy_blocks_mla(kv_caches, block_mapping)

1177

@@ -1209,7 +1209,7 @@ def get_max_shared_memory_per_block_device_attribute(device: int) -> int:

1209

1210

1211 # custom ar

1212 -def init_custom_ar(ipc_tensors: List[torch.Tensor], rank_data: torch.Tensor,

1212 +def init_custom_ar(ipc_tensors: list[torch.Tensor], rank_data: torch.Tensor,

1213 rank: int, full_nvlink: bool) -> int:

1214 return torch.ops._C_custom_ar.init_custom_ar(ipc_tensors, rank_data, rank,

1215 full_nvlink)

@@ -1229,16 +1229,16 @@ def meta_size() -> int:

1229 return torch.ops._C_custom_ar.meta_size()

1230

1231

1232 -def register_buffer(fa: int, ipc_tensors: List[int]) -> None:

1232 +def register_buffer(fa: int, ipc_tensors: list[int]) -> None:

1233 return torch.ops._C_custom_ar.register_buffer(fa, ipc_tensors)

1234

1235

1236 -def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]:

1236 +def get_graph_buffer_ipc_meta(fa: int) -> tuple[list[int], list[int]]:

1237 return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa)

1238

1239

1240 -def register_graph_buffers(fa: int, handles: List[List[int]],

1241 - offsets: List[List[int]]) -> None:

1240 +def register_graph_buffers(fa: int, handles: list[list[int]],

1241 + offsets: list[list[int]]) -> None:

1242 torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)

1243

1244

@@ -1246,7 +1246,7 @@ def get_flash_mla_metadata(

1246 cache_seqlens: torch.Tensor,

1247 num_heads_per_head_k: int,

1248 num_heads_k: int,

1249 -) -> Tuple[torch.Tensor, torch.Tensor]:

1249 +) -> tuple[torch.Tensor, torch.Tensor]:

1250 """

1251 Arguments:

1252 cache_seqlens: (batch_size), dtype torch.int32.

@@ -1272,7 +1272,7 @@ def flash_mla_with_kvcache(

1272 num_splits: torch.Tensor,

1273 softmax_scale: Optional[float] = None,

1274 causal: bool = False,

1275 -) -> Tuple[torch.Tensor, torch.Tensor]:

1275 +) -> tuple[torch.Tensor, torch.Tensor]:

1276 """

1277 Arguments:

1278 q: (batch_size, seq_len_q, num_heads_q, head_dim).

vllm/_ipex_ops.py

+4 -4

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Tuple

3 +from typing import Optional

4

5 import torch

6

@@ -18,7 +18,7 @@ class ipex_ops:

18

19 @staticmethod

20 def _reshape_activation_tensor(

21 - x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:

21 + x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:

22 num = x.size(0)

23 d = x.size(1) // 2

24 x = x.reshape(num, 2, d)

@@ -213,8 +213,8 @@ class ipex_ops:

213 key, value, key_cache, value_cache, slot_mapping)

214

215 @staticmethod

216 - def copy_blocks(key_caches: List[torch.Tensor],

217 - value_caches: List[torch.Tensor],

216 + def copy_blocks(key_caches: list[torch.Tensor],

217 + value_caches: list[torch.Tensor],

218 block_mapping: torch.Tensor) -> None:

219 torch.xpu.copy_blocks( # type: ignore

220 key_caches,

vllm/beam_search.py

+9 -9

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union

4 +from typing import TYPE_CHECKING, Any, Optional, Union

5

6 from vllm.sequence import Logprob

7

@@ -17,14 +17,14 @@ class BeamSearchSequence:

17 about to be returned to the user.

18 """

19 # The tokens includes the prompt.

20 - tokens: List[int]

21 - logprobs: List[Dict[int, Logprob]]

20 + tokens: list[int]

21 + logprobs: list[dict[int, Logprob]]

22 cum_logprob: float = 0.0

23 text: Optional[str] = None

24 finish_reason: Optional[str] = None

25 stop_reason: Union[int, str, None] = None

26 multi_modal_data: Optional["MultiModalDataDict"] = None

27 - mm_processor_kwargs: Optional[Dict[str, Any]] = None

27 + mm_processor_kwargs: Optional[dict[str, Any]] = None

28

29

30 @dataclass

@@ -33,20 +33,20 @@ class BeamSearchOutput:

33 It contains the list of the best beam search sequences.

34 The length of the list is equal to the beam width.

35 """

36 - sequences: List[BeamSearchSequence]

36 + sequences: list[BeamSearchSequence]

37

38

39 class BeamSearchInstance:

40

41 - def __init__(self, prompt_tokens: List[int]):

42 - self.beams: List[BeamSearchSequence] = [

41 + def __init__(self, prompt_tokens: list[int]):

42 + self.beams: list[BeamSearchSequence] = [

43 BeamSearchSequence(tokens=prompt_tokens, logprobs=[])

44 ]

45 - self.completed: List[BeamSearchSequence] = []

45 + self.completed: list[BeamSearchSequence] = []

46

47

48 def get_beam_search_score(

49 - tokens: List[int],

49 + tokens: list[int],

50 cumulative_logprob: float,

51 eos_token_id: int,

52 length_penalty: float = 1.0,

vllm/config.py

+71 -70

@@ -7,13 +7,14 @@ import hashlib

7 import json

8 import sys

9 import warnings

10 +from collections import Counter

11 +from collections.abc import Mapping

12 from contextlib import contextmanager

13 from dataclasses import dataclass, field, replace

14 from importlib.util import find_spec

15 from pathlib import Path

14 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,

15 - Final, List, Literal, Mapping, Optional, Protocol, Set,

16 - Tuple, Type, Union)

16 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal,

17 + Optional, Protocol, Union)

18

19 import torch

20 from pydantic import BaseModel, Field, PrivateAttr

@@ -67,20 +68,20 @@ _ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",

68

69 RunnerType = Literal["generate", "pooling", "draft", "transcription"]

70

70 -_RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {

71 +_RUNNER_TASKS: dict[RunnerType, list[_ResolvedTask]] = {

72 "generate": ["generate"],

73 "pooling": ["embed", "classify", "score", "reward"],

74 "draft": ["draft"],

75 "transcription": ["transcription"],

76 }

77

77 -_TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {

78 +_TASK_RUNNER: dict[_ResolvedTask, RunnerType] = {

79 task: runner

80 for runner, tasks in _RUNNER_TASKS.items()

81 for task in tasks

82 }

83

83 -HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],

84 +HfOverrides = Union[dict[str, Any], Callable[[PretrainedConfig],

85 PretrainedConfig]]

86

87

@@ -92,7 +93,7 @@ class SupportsHash(Protocol):

93

94 class SupportsMetricsInfo(Protocol):

95

95 - def metrics_info(self) -> Dict[str, str]:

96 + def metrics_info(self) -> dict[str, str]:

97 ...

98

99

@@ -209,7 +210,7 @@ class ModelConfig:

210 excluding anything before input ids/embeddings and after

211 the final hidden states.

212 """

212 - factors: List[Any] = []

213 + factors: list[Any] = []

214 factors.append(self.model)

215 factors.append(self.dtype)

216 factors.append(self.quantization)

@@ -233,7 +234,7 @@ class ModelConfig:

234 allowed_local_media_path: str = "",

235 revision: Optional[str] = None,

236 code_revision: Optional[str] = None,

236 - rope_scaling: Optional[Dict[str, Any]] = None,

237 + rope_scaling: Optional[dict[str, Any]] = None,

238 rope_theta: Optional[float] = None,

239 tokenizer_revision: Optional[str] = None,

240 max_model_len: Optional[int] = None,

@@ -244,19 +245,19 @@ class ModelConfig:

245 max_logprobs: int = 20,

246 disable_sliding_window: bool = False,

247 skip_tokenizer_init: bool = False,

247 - served_model_name: Optional[Union[str, List[str]]] = None,

248 + served_model_name: Optional[Union[str, list[str]]] = None,

249 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,

250 use_async_output_proc: bool = True,

251 config_format: ConfigFormat = ConfigFormat.AUTO,

252 hf_overrides: Optional[HfOverrides] = None,

252 - mm_processor_kwargs: Optional[Dict[str, Any]] = None,

253 + mm_processor_kwargs: Optional[dict[str, Any]] = None,

254 disable_mm_preprocessor_cache: bool = False,

254 - override_neuron_config: Optional[Dict[str, Any]] = None,

255 + override_neuron_config: Optional[dict[str, Any]] = None,

256 override_pooler_config: Optional["PoolerConfig"] = None,

257 logits_processor_pattern: Optional[str] = None,

258 generation_config: Optional[str] = None,

259 enable_sleep_mode: bool = False,

259 - override_generation_config: Optional[Dict[str, Any]] = None,

260 + override_generation_config: Optional[dict[str, Any]] = None,

261 model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,

262 ) -> None:

263 self.model = model

@@ -283,7 +284,7 @@ class ModelConfig:

284 hf_overrides_fn = None

285

286 if rope_scaling is not None:

286 - hf_override: Dict[str, Any] = {"rope_scaling": rope_scaling}

287 + hf_override: dict[str, Any] = {"rope_scaling": rope_scaling}

288 hf_overrides_kw.update(hf_override)

289 msg = ("`--rope-scaling` will be removed in a future release. "

290 f"'Please instead use `--hf-overrides '{hf_override!r}'`")

@@ -505,8 +506,8 @@ class ModelConfig:

506

507 def _get_preferred_task(

508 self,

508 - architectures: List[str],

509 - supported_tasks: Set[_ResolvedTask],

509 + architectures: list[str],

510 + supported_tasks: set[_ResolvedTask],

511 ) -> Optional[_ResolvedTask]:

512 model_id = self.model

513 if get_pooling_config(model_id, self.revision):

@@ -516,7 +517,7 @@ class ModelConfig:

517 if self.registry.is_transcription_model(architectures):

518 return "transcription"

519

519 - suffix_to_preferred_task: List[Tuple[str, _ResolvedTask]] = [

520 + suffix_to_preferred_task: list[tuple[str, _ResolvedTask]] = [

521 # Other models follow this pattern

522 ("ForCausalLM", "generate"),

523 ("ForConditionalGeneration", "generate"),

@@ -537,27 +538,27 @@ class ModelConfig:

538 def _resolve_task(

539 self,

540 task_option: Union[TaskOption, Literal["draft"]],

540 - ) -> Tuple[Set[_ResolvedTask], _ResolvedTask]:

541 + ) -> tuple[set[_ResolvedTask], _ResolvedTask]:

542 if task_option == "draft":

543 return {"draft"}, "draft"

544

545 registry = self.registry

546 architectures = self.architectures

547

547 - runner_support: Dict[RunnerType, bool] = {

548 + runner_support: dict[RunnerType, bool] = {

549 # NOTE: Listed from highest to lowest priority,

550 # in case the model supports multiple of them

551 "transcription": registry.is_transcription_model(architectures),

552 "generate": registry.is_text_generation_model(architectures),

553 "pooling": registry.is_pooling_model(architectures),

554 }

554 - supported_runner_types_lst: List[RunnerType] = [

555 + supported_runner_types_lst: list[RunnerType] = [

556 runner_type

557 for runner_type, is_supported in runner_support.items()

558 if is_supported

559 ]

560

560 - supported_tasks_lst: List[_ResolvedTask] = [

561 + supported_tasks_lst: list[_ResolvedTask] = [

562 task for runner_type in supported_runner_types_lst

563 for task in _RUNNER_TASKS[runner_type]

564 ]

@@ -767,7 +768,7 @@ class ModelConfig:

768 self.use_async_output_proc = False

769

770 def get_hf_config_sliding_window(

770 - self) -> Union[Optional[int], List[Optional[int]]]:

771 + self) -> Union[Optional[int], list[Optional[int]]]:

772 """Get the sliding window size, or None if disabled."""

773

774 # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in

@@ -778,7 +779,7 @@ class ModelConfig:

779 return None

780 return getattr(self.hf_text_config, "sliding_window", None)

781

781 - def get_sliding_window(self) -> Optional[Union[int, List[Optional[int]]]]:

782 + def get_sliding_window(self) -> Optional[Union[int, list[Optional[int]]]]:

783 """Get the sliding window size, or None if disabled.

784 """

785 # If user disables sliding window, return None.

@@ -888,7 +889,7 @@ class ModelConfig:

889 return num_heads // parallel_config.tensor_parallel_size

890

891 def get_layers_start_end_indices(

891 - self, parallel_config: "ParallelConfig") -> Tuple[int, int]:

892 + self, parallel_config: "ParallelConfig") -> tuple[int, int]:

893 from vllm.distributed.utils import get_pp_indices

894 if self.hf_text_config.model_type == "deepseek_mtp":

895 total_num_hidden_layers = getattr(self.hf_text_config,

@@ -949,7 +950,7 @@ class ModelConfig:

950

951 return self.multimodal_config

952

952 - def try_get_generation_config(self) -> Dict[str, Any]:

953 + def try_get_generation_config(self) -> dict[str, Any]:

954 if self.generation_config is None or self.generation_config == "auto":

955 config = try_get_generation_config(

956 self.hf_config_path or self.model,

@@ -967,7 +968,7 @@ class ModelConfig:

968

969 return config.to_diff_dict()

970

970 - def get_diff_sampling_param(self) -> Dict[str, Any]:

971 + def get_diff_sampling_param(self) -> dict[str, Any]:

972 """

973 This method returns a dictionary containing the parameters

974 that differ from the default sampling parameters, but only

@@ -975,7 +976,7 @@ class ModelConfig:

976 set, an empty dictionary is returned.

977

978 Returns:

978 - Dict[str, Any]: A dictionary with the differing sampling

979 + dict[str, Any]: A dictionary with the differing sampling

980 parameters if `generation_config` is set, otherwise an

981 empty dictionary.

982 """

@@ -1032,7 +1033,7 @@ class ModelConfig:

1033 return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE

1034

1035 @property

1035 - def supported_runner_types(self) -> Set[RunnerType]:

1036 + def supported_runner_types(self) -> set[RunnerType]:

1037 return {_TASK_RUNNER[task] for task in self.supported_tasks}

1038

1039 @property

@@ -1075,7 +1076,7 @@ class CacheConfig:

1076 excluding anything before input ids/embeddings and after

1077 the final hidden states.

1078 """

1078 - factors: List[Any] = []

1079 + factors: list[Any] = []

1080 factors.append(self.cache_dtype)

1081 # `cpu_offload_gb` does not use `torch.compile` yet.

1082 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

@@ -1183,7 +1184,7 @@ class TokenizerPoolConfig:

1184 pool type.

1185 """

1186 pool_size: int

1186 - pool_type: Union[str, Type["BaseTokenizerGroup"]]

1187 + pool_type: Union[str, type["BaseTokenizerGroup"]]

1188 extra_config: dict

1189

1190 def compute_hash(self) -> str:

@@ -1200,7 +1201,7 @@ class TokenizerPoolConfig:

1201 """

1202 # no factors to consider.

1203 # this config will not affect the computation graph.

1203 - factors: List[Any] = []

1204 + factors: list[Any] = []

1205 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

1206 return hash_str

1207

@@ -1214,7 +1215,7 @@ class TokenizerPoolConfig:

1215 @classmethod

1216 def create_config(

1217 cls, tokenizer_pool_size: int,

1217 - tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]],

1218 + tokenizer_pool_type: Union[str, type["BaseTokenizerGroup"]],

1219 tokenizer_pool_extra_config: Optional[Union[str, dict]]

1220 ) -> Optional["TokenizerPoolConfig"]:

1221 """Create a TokenizerPoolConfig from the given parameters.

@@ -1285,7 +1286,7 @@ class LoadConfig:

1286 download_dir: Optional[str] = None

1287 model_loader_extra_config: Optional[Union[str, dict]] = field(

1288 default_factory=dict)

1288 - ignore_patterns: Optional[Union[List[str], str]] = None

1289 + ignore_patterns: Optional[Union[list[str], str]] = None

1290

1291 def compute_hash(self) -> str:

1292 """

@@ -1301,7 +1302,7 @@ class LoadConfig:

1302 """

1303 # no factors to consider.

1304 # this config will not affect the computation graph.

1304 - factors: List[Any] = []

1305 + factors: list[Any] = []

1306 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

1307 return hash_str

1308

@@ -1359,7 +1360,7 @@ class ParallelConfig:

1360 # to "ray" if Ray is installed and fail otherwise. Note that tpu

1361 # and hpu only support Ray for distributed inference.

1362 distributed_executor_backend: Optional[Union[str,

1362 - Type["ExecutorBase"]]] = None

1363 + type["ExecutorBase"]]] = None

1364

1365 # the full name of the worker class to use. If "auto", the worker class

1366 # will be determined based on the platform.

@@ -1423,7 +1424,7 @@ class ParallelConfig:

1424 excluding anything before input ids/embeddings and after

1425 the final hidden states.

1426 """

1426 - factors: List[Any] = []

1427 + factors: list[Any] = []

1428 factors.append(self.pipeline_parallel_size)

1429 factors.append(self.tensor_parallel_size)

1430 return hashlib.sha256(str(factors).encode()).hexdigest()

@@ -1600,7 +1601,7 @@ class SchedulerConfig:

1601

1602 # scheduler class or path. "vllm.core.scheduler.Scheduler" (default)

1603 # or "mod.custom_class".

1603 - scheduler_cls: Union[str, Type[object]] = "vllm.core.scheduler.Scheduler"

1604 + scheduler_cls: Union[str, type[object]] = "vllm.core.scheduler.Scheduler"

1605

1606 def compute_hash(self) -> str:

1607 """

@@ -1616,7 +1617,7 @@ class SchedulerConfig:

1617 """

1618 # no factors to consider.

1619 # this config will not affect the computation graph.

1619 - factors: List[Any] = []

1620 + factors: list[Any] = []

1621 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

1622 return hash_str

1623

@@ -1752,7 +1753,7 @@ class DeviceConfig:

1753 # no factors to consider.

1754 # the device/platform information will be summarized

1755 # by torch/vllm automatically.

1755 - factors: List[Any] = []

1756 + factors: list[Any] = []

1757 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

1758 return hash_str

1759

@@ -1798,7 +1799,7 @@ class SpeculativeConfig:

1799 """

1800 # no factors to consider.

1801 # spec decode does not use `torch.compile` yet.

1801 - factors: List[Any] = []

1802 + factors: list[Any] = []

1803 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

1804 return hash_str

1805

@@ -2261,7 +2262,7 @@ class LoRAConfig:

2262 lora_extra_vocab_size: int = 256

2263 # This is a constant.

2264 lora_vocab_padding_size: ClassVar[int] = 256

2264 - long_lora_scaling_factors: Optional[Tuple[float]] = None

2265 + long_lora_scaling_factors: Optional[tuple[float]] = None

2266 bias_enabled: bool = False

2267

2268 def compute_hash(self) -> str:

@@ -2278,7 +2279,7 @@ class LoRAConfig:

2279 """

2280 # no factors to consider.

2281 # LoRA is not compatible with `torch.compile` .

2281 - factors: List[Any] = []

2282 + factors: list[Any] = []

2283 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2284 return hash_str

2285

@@ -2350,7 +2351,7 @@ class PromptAdapterConfig:

2351 """

2352 # no factors to consider.

2353 # this config will not affect the computation graph.

2353 - factors: List[Any] = []

2354 + factors: list[Any] = []

2355 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2356 return hash_str

2357

@@ -2395,7 +2396,7 @@ class MultiModalConfig:

2396 """

2397 # no factors to consider.

2398 # this config will not affect the computation graph.

2398 - factors: List[Any] = []

2399 + factors: list[Any] = []

2400 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2401 return hash_str

2402

@@ -2431,7 +2432,7 @@ class PoolerConfig:

2432 are returned.

2433 """

2434

2434 - returned_token_ids: Optional[List[int]] = None

2435 + returned_token_ids: Optional[list[int]] = None

2436 """

2437 A list of indices for the vocabulary dimensions to be extracted,

2438 such as the token IDs of ``good_token`` and ``bad_token`` in the

@@ -2452,7 +2453,7 @@ class PoolerConfig:

2453 """

2454 # no factors to consider.

2455 # this config will not affect the computation graph.

2455 - factors: List[Any] = []

2456 + factors: list[Any] = []

2457 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2458 return hash_str

2459

@@ -2469,7 +2470,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {

2470 "bfloat16": torch.bfloat16,

2471 }

2472

2472 -_ROCM_NOT_SUPPORTED_DTYPE: List[str] = [] #

2473 +_ROCM_NOT_SUPPORTED_DTYPE: list[str] = [] #

2474

2475

2476 def _get_and_verify_dtype(

@@ -2558,7 +2559,7 @@ def _get_and_verify_max_len(

2559 hf_config: PretrainedConfig,

2560 max_model_len: Optional[int],

2561 disable_sliding_window: bool,

2561 - sliding_window_len: Optional[Union[int, List[Optional[int]]]],

2562 + sliding_window_len: Optional[Union[int, list[Optional[int]]]],

2563 spec_target_max_model_len: Optional[int] = None,

2564 encoder_config: Optional[Any] = None,

2565 ) -> int:

@@ -2684,7 +2685,7 @@ def _get_and_verify_max_len(

2685

2686

2687 def get_min_sliding_window(

2687 - sliding_window: Union[int, List[Optional[int]]]) -> int:

2688 + sliding_window: Union[int, list[Optional[int]]]) -> int:

2689 if isinstance(sliding_window, list):

2690 return min(s for s in sliding_window if s is not None)

2691

@@ -2692,7 +2693,7 @@ def get_min_sliding_window(

2693

2694

2695 def get_served_model_name(model: str,

2695 - served_model_name: Optional[Union[str, List[str]]]):

2696 + served_model_name: Optional[Union[str, list[str]]]):

2697 """

2698 If the input is a non-empty list, the first model_name in

2699 `served_model_name` is taken.

@@ -2731,7 +2732,7 @@ class DecodingConfig:

2732 """

2733 # no factors to consider.

2734 # this config will not affect the computation graph.

2734 - factors: List[Any] = []

2735 + factors: list[Any] = []

2736 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2737 return hash_str

2738

@@ -2774,7 +2775,7 @@ class ObservabilityConfig:

2775 """

2776 # no factors to consider.

2777 # this config will not affect the computation graph.

2777 - factors: List[Any] = []

2778 + factors: list[Any] = []

2779 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2780 return hash_str

2781

@@ -2833,7 +2834,7 @@ class KVTransferConfig(BaseModel):

2834 """

2835 # no factors to consider.

2836 # this config will not affect the computation graph.

2836 - factors: List[Any] = []

2837 + factors: list[Any] = []

2838 hash_str = hashlib.md5(str(factors).encode()).hexdigest()

2839 return hash_str

2840

@@ -2930,7 +2931,7 @@ class CompilationConfig(BaseModel):

2931 torch.compile will handle cudagraph capture logic in the future.

2932 - cudagraph_capture_sizes: sizes to capture cudagraph.

2933 - None (default): capture sizes are inferred from vllm config.

2933 - - List[int]: capture sizes are specified as given.

2934 + - list[int]: capture sizes are specified as given.

2935 - cudagraph_num_of_warmups: number of warmup runs for cudagraph.

2936 It means the first several runs will be treated as warmup runs.

2937 Only after that, the execution will be recorded, and the recorded

@@ -2972,17 +2973,17 @@ class CompilationConfig(BaseModel):

2973 debug_dump_path: str = ""

2974 cache_dir: str = ""

2975 backend: str = ""

2975 - custom_ops: List[str] = Field(default_factory=list)

2976 - splitting_ops: List[str] = Field(default=None) # type: ignore

2976 + custom_ops: list[str] = Field(default_factory=list)

2977 + splitting_ops: list[str] = Field(default=None) # type: ignore

2978

2979 use_inductor: bool = True

2979 - compile_sizes: Optional[List[Union[int, str]]] = Field(default=None)

2980 - inductor_compile_config: Dict = Field(default_factory=dict)

2981 - inductor_passes: Dict[str, str] = Field(default_factory=dict)

2980 + compile_sizes: Optional[list[Union[int, str]]] = Field(default=None)

2981 + inductor_compile_config: dict = Field(default_factory=dict)

2982 + inductor_passes: dict[str, str] = Field(default_factory=dict)

2983

2984 use_cudagraph: bool = False

2985 cudagraph_num_of_warmups: int = 0

2985 - cudagraph_capture_sizes: Optional[List[int]] = None

2986 + cudagraph_capture_sizes: Optional[list[int]] = None

2987 cudagraph_copy_inputs: bool = False

2988

2989 class PassConfig(BaseModel):

@@ -2998,7 +2999,7 @@ class CompilationConfig(BaseModel):

2999 - enable_noop: whether to enable the custom no-op elimination pass.

3000 TODO(luka) better pass enabling system.

3001 """

3001 - dump_graph_stages: List[str] = Field(default_factory=list)

3002 + dump_graph_stages: list[str] = Field(default_factory=list)

3003 dump_graph_dir: Path = Field(default=Path("."))

3004 enable_fusion: bool = True

3005 enable_noop: bool = True

@@ -3026,20 +3027,20 @@ class CompilationConfig(BaseModel):

3027 max_capture_size: int = PrivateAttr

3028 local_cache_dir: str = PrivateAttr # local cache dir for each rank

3029 # optimization:

3029 - # Intuitively, bs_to_padded_graph_size should be Dict[int, int].

3030 + # Intuitively, bs_to_padded_graph_size should be dict[int, int].

3031 # since we know all keys are in a range [0, max_capture_size],

3031 - # we can optimize it to List[int] for better lookup performance.

3032 - bs_to_padded_graph_size: List[int] = PrivateAttr

3032 + # we can optimize it to list[int] for better lookup performance.

3033 + bs_to_padded_graph_size: list[int] = PrivateAttr

3034

3035 # keep track of enabled and disabled custom ops

3036 enabled_custom_ops: Counter[str] = PrivateAttr

3037 disabled_custom_ops: Counter[str] = PrivateAttr

3037 - traced_files: Set[str] = PrivateAttr

3038 + traced_files: set[str] = PrivateAttr

3039 compilation_time: float = PrivateAttr

3040

3041 # Per-model forward context

3042 # Map from layer name to the attention cls

3042 - static_forward_context: Dict[str, Any] = PrivateAttr

3043 + static_forward_context: dict[str, Any] = PrivateAttr

3044

3045 def compute_hash(self) -> str:

3046 """

@@ -3053,7 +3054,7 @@ class CompilationConfig(BaseModel):

3054 excluding anything before input ids/embeddings and after

3055 the final hidden states.

3056 """

3056 - factors: List[Any] = []

3057 + factors: list[Any] = []

3058 factors.append(self.level)

3059 factors.append(self.backend)

3060 factors.append(self.custom_ops)

@@ -3150,7 +3151,7 @@ class CompilationConfig(BaseModel):

3151 return VllmBackend(vllm_config)

3152

3153 def init_with_cudagraph_sizes(self,

3153 - cudagraph_capture_sizes: List[int]) -> None:

3154 + cudagraph_capture_sizes: list[int]) -> None:

3155 """To complete the initialization of config,

3156 we need to know the cudagraph sizes."""

3157

@@ -3243,10 +3244,10 @@ class VllmConfig:

3244 excluding anything before input ids/embeddings and after

3245 the final hidden states.

3246 """

3246 - factors: List[Any] = []

3247 + factors: list[Any] = []

3248

3249 # summarize vllm config

3249 - vllm_factors: List[Any] = []

3250 + vllm_factors: list[Any] = []

3251 from vllm import __version__

3252 vllm_factors.append(__version__)

3253 if self.model_config:

vllm/connections.py

+2 -1

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import Mapping, MutableMapping

4 from pathlib import Path

4 -from typing import Mapping, MutableMapping, Optional

5 +from typing import Optional

6 from urllib.parse import urlparse

7

8 import aiohttp

vllm/entrypoints/api_server.py

+2 -1

@@ -10,7 +10,8 @@ import asyncio

10 import json

11 import ssl

12 from argparse import Namespace

13 -from typing import Any, AsyncGenerator, Optional

13 +from collections.abc import AsyncGenerator

14 +from typing import Any, Optional

15

16 from fastapi import FastAPI, Request

17 from fastapi.responses import JSONResponse, Response, StreamingResponse

vllm/entrypoints/chat_utils.py

+25 -24

@@ -5,10 +5,11 @@ import codecs

5 import json

6 from abc import ABC, abstractmethod

7 from collections import defaultdict, deque

8 +from collections.abc import Awaitable, Iterable

9 from functools import cache, lru_cache, partial

10 from pathlib import Path

10 -from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,

11 - Literal, Optional, Tuple, TypeVar, Union, cast)

11 +from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,

12 + cast)

13

14 import jinja2.nodes

15 import transformers.utils.chat_template_utils as hf_chat_utils

@@ -117,7 +118,7 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):

118 role: Required[str]

119 """The role of the message's author."""

120

120 - content: Union[str, List[ChatCompletionContentPartParam]]

121 + content: Union[str, list[ChatCompletionContentPartParam]]

122 """The contents of the message."""

123

124 name: str

@@ -143,7 +144,7 @@ class ConversationMessage(TypedDict, total=False):

144 role: Required[str]

145 """The role of the message's author."""

146

146 - content: Union[Optional[str], List[Dict[str, str]]]

147 + content: Union[Optional[str], list[dict[str, str]]]

148 """The contents of the message"""

149

150 tool_call_id: Optional[str]

@@ -495,13 +496,13 @@ class BaseMultiModalContentParser(ABC):

496 super().__init__()

497

498 # multimodal placeholder_string : count

498 - self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0)

499 + self._placeholder_counts: dict[str, int] = defaultdict(lambda: 0)

500

501 def _add_placeholder(self, placeholder: Optional[str]):

502 if placeholder:

503 self._placeholder_counts[placeholder] += 1

504

504 - def mm_placeholder_counts(self) -> Dict[str, int]:

505 + def mm_placeholder_counts(self) -> dict[str, int]:

506 return dict(self._placeholder_counts)

507

508 @abstractmethod

@@ -652,12 +653,12 @@ def load_chat_template(

653

654 # TODO: Let user specify how to insert multimodal tokens into prompt

655 # (similar to chat template)

655 -def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],

656 +def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],

657 text_prompt: str) -> str:

658 """Combine multimodal prompts for a multimodal language model."""

659

660 # Look through the text prompt to check for missing placeholders

660 - missing_placeholders: List[str] = []

661 + missing_placeholders: list[str] = []

662 for placeholder in placeholder_counts:

663

664 # For any existing placeholder in the text prompt, we leave it as is

@@ -684,10 +685,10 @@ _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)

685 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)

686 _VideoParser = partial(cast, ChatCompletionContentPartVideoParam)

687

687 -_ContentPart: TypeAlias = Union[str, Dict[str, str], InputAudio]

688 +_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio]

689

690 # Define a mapping from part types to their corresponding parsing functions.

690 -MM_PARSER_MAP: Dict[

691 +MM_PARSER_MAP: dict[

692 str,

693 Callable[[ChatCompletionContentPartParam], _ContentPart],

694 ] = {

@@ -749,7 +750,7 @@ def _parse_chat_message_content_mm_part(

750 part)

751 return "audio_url", audio_params.get("audio_url", "")

752 if part.get("input_audio") is not None:

752 - input_audio_params = cast(Dict[str, str], part)

753 + input_audio_params = cast(dict[str, str], part)

754 return "input_audio", input_audio_params

755 if part.get("video_url") is not None:

756 video_params = cast(CustomChatCompletionContentSimpleVideoParam,

@@ -773,7 +774,7 @@ def _parse_chat_message_content_parts(

774 mm_tracker: BaseMultiModalItemTracker,

775 *,

776 wrap_dicts: bool,

776 -) -> List[ConversationMessage]:

777 +) -> list[ConversationMessage]:

778 content = list[_ContentPart]()

779

780 mm_parser = mm_tracker.create_parser()

@@ -791,7 +792,7 @@ def _parse_chat_message_content_parts(

792 # Parsing wraps images and texts as interleaved dictionaries

793 return [ConversationMessage(role=role,

794 content=content)] # type: ignore

794 - texts = cast(List[str], content)

795 + texts = cast(list[str], content)

796 text_prompt = "\n".join(texts)

797 mm_placeholder_counts = mm_parser.mm_placeholder_counts()

798 if mm_placeholder_counts:

@@ -866,7 +867,7 @@ def _parse_chat_message_content(

867 message: ChatCompletionMessageParam,

868 mm_tracker: BaseMultiModalItemTracker,

869 content_format: _ChatTemplateContentFormat,

869 -) -> List[ConversationMessage]:

870 +) -> list[ConversationMessage]:

871 role = message["role"]

872 content = message.get("content")

873

@@ -900,7 +901,7 @@ def _parse_chat_message_content(

901 return result

902

903

903 -def _postprocess_messages(messages: List[ConversationMessage]) -> None:

904 +def _postprocess_messages(messages: list[ConversationMessage]) -> None:

905 # per the Transformers docs & maintainers, tool call arguments in

906 # assistant-role messages with tool_calls need to be dicts not JSON str -

907 # this is how tool-use chat templates will expect them moving forwards

@@ -916,12 +917,12 @@ def _postprocess_messages(messages: List[ConversationMessage]) -> None:

917

918

919 def parse_chat_messages(

919 - messages: List[ChatCompletionMessageParam],

920 + messages: list[ChatCompletionMessageParam],

921 model_config: ModelConfig,

922 tokenizer: AnyTokenizer,

923 content_format: _ChatTemplateContentFormat,

923 -) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:

924 - conversation: List[ConversationMessage] = []

924 +) -> tuple[list[ConversationMessage], Optional[MultiModalDataDict]]:

925 + conversation: list[ConversationMessage] = []

926 mm_tracker = MultiModalItemTracker(model_config, tokenizer)

927

928 for msg in messages:

@@ -939,12 +940,12 @@ def parse_chat_messages(

940

941

942 def parse_chat_messages_futures(

942 - messages: List[ChatCompletionMessageParam],

943 + messages: list[ChatCompletionMessageParam],

944 model_config: ModelConfig,

945 tokenizer: AnyTokenizer,

946 content_format: _ChatTemplateContentFormat,

946 -) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:

947 - conversation: List[ConversationMessage] = []

947 +) -> tuple[list[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:

948 + conversation: list[ConversationMessage] = []

949 mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)

950

951 for msg in messages:

@@ -963,7 +964,7 @@ def parse_chat_messages_futures(

964

965 def apply_hf_chat_template(

966 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],

966 - conversation: List[ConversationMessage],

967 + conversation: list[ConversationMessage],

968 chat_template: Optional[str],

969 *,

970 tokenize: bool = False, # Different from HF's default

@@ -985,10 +986,10 @@ def apply_hf_chat_template(

986

987 def apply_mistral_chat_template(

988 tokenizer: MistralTokenizer,

988 - messages: List[ChatCompletionMessageParam],

989 + messages: list[ChatCompletionMessageParam],

990 chat_template: Optional[str] = None,

991 **kwargs: Any,

991 -) -> List[int]:

992 +) -> list[int]:

993 if chat_template is not None:

994 logger.warning_once(

995 "'chat_template' cannot be overridden for mistral tokenizer.")

vllm/entrypoints/cli/openai.py

+5 -5

@@ -5,7 +5,7 @@ import argparse

5 import os

6 import signal

7 import sys

8 -from typing import List, Optional, Tuple

8 +from typing import Optional

9

10 from openai import OpenAI

11 from openai.types.chat import ChatCompletionMessageParam

@@ -23,7 +23,7 @@ def _register_signal_handlers():

23 signal.signal(signal.SIGTSTP, signal_handler)

24

25

26 -def _interactive_cli(args: argparse.Namespace) -> Tuple[str, OpenAI]:

26 +def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:

27 _register_signal_handlers()

28

29 base_url = args.url

@@ -43,7 +43,7 @@ def _interactive_cli(args: argparse.Namespace) -> Tuple[str, OpenAI]:

43

44 def chat(system_prompt: Optional[str], model_name: str,

45 client: OpenAI) -> None:

46 - conversation: List[ChatCompletionMessageParam] = []

46 + conversation: list[ChatCompletionMessageParam] = []

47 if system_prompt is not None:

48 conversation.append({"role": "system", "content": system_prompt})

49

@@ -100,7 +100,7 @@ class ChatCommand(CLISubcommand):

100 def cmd(args: argparse.Namespace) -> None:

101 model_name, client = _interactive_cli(args)

102 system_prompt = args.system_prompt

103 - conversation: List[ChatCompletionMessageParam] = []

103 + conversation: list[ChatCompletionMessageParam] = []

104 if system_prompt is not None:

105 conversation.append({"role": "system", "content": system_prompt})

106

@@ -168,5 +168,5 @@ class CompleteCommand(CLISubcommand):

168 return complete_parser

169

170

171 -def cmd_init() -> List[CLISubcommand]:

171 +def cmd_init() -> list[CLISubcommand]:

172 return [ChatCommand(), CompleteCommand()]

vllm/entrypoints/cli/serve.py

+1 -2

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import argparse

4 -from typing import List

4

5 import uvloop

6

@@ -59,5 +58,5 @@ class ServeSubcommand(CLISubcommand):

58 return make_arg_parser(serve_parser)

59

60

62 -def cmd_init() -> List[CLISubcommand]:

61 +def cmd_init() -> list[CLISubcommand]:

62 return [ServeSubcommand()]

vllm/entrypoints/llm.py

+105 -105

@@ -2,9 +2,9 @@

2

3 import itertools

4 import warnings

5 +from collections.abc import Sequence

6 from contextlib import contextmanager

6 -from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,

7 - Tuple, Type, Union, cast, overload)

7 +from typing import Any, Callable, ClassVar, Optional, Union, cast, overload

8

9 import cloudpickle

10 import torch.nn as nn

@@ -177,11 +177,11 @@ class LLM:

177 disable_custom_all_reduce: bool = False,

178 disable_async_output_proc: bool = False,

179 hf_overrides: Optional[HfOverrides] = None,

180 - mm_processor_kwargs: Optional[Dict[str, Any]] = None,

180 + mm_processor_kwargs: Optional[dict[str, Any]] = None,

181 # After positional args are removed, move this right below `model`

182 task: TaskOption = "auto",

183 override_pooler_config: Optional[PoolerConfig] = None,

184 - compilation_config: Optional[Union[int, Dict[str, Any]]] = None,

184 + compilation_config: Optional[Union[int, dict[str, Any]]] = None,

185 **kwargs,

186 ) -> None:

187 '''

@@ -246,7 +246,7 @@ class LLM:

246 self.request_counter = Counter()

247

248 @staticmethod

249 - def get_engine_class() -> Type[LLMEngine]:

249 + def get_engine_class() -> type[LLMEngine]:

250 if envs.VLLM_USE_V1:

251 # Lazy import: the v1 package isn't distributed

252 from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine

@@ -283,11 +283,11 @@ class LLM:

283 Sequence[SamplingParams]]] = None,

284 *,

285 use_tqdm: bool = True,

286 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

286 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

287 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

288 guided_options_request: Optional[Union[LLMGuidedOptions,

289 GuidedDecodingRequest]] = None,

290 - ) -> List[RequestOutput]:

290 + ) -> list[RequestOutput]:

291 ...

292

293 @overload # LEGACY: single (prompt + optional token ids)

@@ -296,30 +296,30 @@ class LLM:

296 self,

297 prompts: str,

298 sampling_params: Optional[Union[SamplingParams,

299 - List[SamplingParams]]] = None,

300 - prompt_token_ids: Optional[List[int]] = None,

299 + list[SamplingParams]]] = None,

300 + prompt_token_ids: Optional[list[int]] = None,

301 use_tqdm: bool = True,

302 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

302 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

303 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

304 guided_options_request: Optional[Union[LLMGuidedOptions,

305 GuidedDecodingRequest]] = None,

306 - ) -> List[RequestOutput]:

306 + ) -> list[RequestOutput]:

307 ...

308

309 @overload # LEGACY: multi (prompt + optional token ids)

310 @deprecated("'prompt_token_ids' will become part of 'prompts'")

311 def generate(

312 self,

313 - prompts: List[str],

313 + prompts: list[str],

314 sampling_params: Optional[Union[SamplingParams,

315 - List[SamplingParams]]] = None,

316 - prompt_token_ids: Optional[List[List[int]]] = None,

315 + list[SamplingParams]]] = None,

316 + prompt_token_ids: Optional[list[list[int]]] = None,

317 use_tqdm: bool = True,

318 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

318 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

319 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

320 guided_options_request: Optional[Union[LLMGuidedOptions,

321 GuidedDecodingRequest]] = None,

322 - ) -> List[RequestOutput]:

322 + ) -> list[RequestOutput]:

323 ...

324

325 @overload # LEGACY: single (token ids + optional prompt)

@@ -328,32 +328,32 @@ class LLM:

328 self,

329 prompts: Optional[str] = None,

330 sampling_params: Optional[Union[SamplingParams,

331 - List[SamplingParams]]] = None,

331 + list[SamplingParams]]] = None,

332 *,

333 - prompt_token_ids: List[int],

333 + prompt_token_ids: list[int],

334 use_tqdm: bool = True,

335 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

335 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

336 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

337 guided_options_request: Optional[Union[LLMGuidedOptions,

338 GuidedDecodingRequest]] = None,

339 - ) -> List[RequestOutput]:

339 + ) -> list[RequestOutput]:

340 ...

341

342 @overload # LEGACY: multi (token ids + optional prompt)

343 @deprecated("'prompt_token_ids' will become part of 'prompts'")

344 def generate(

345 self,

346 - prompts: Optional[List[str]] = None,

346 + prompts: Optional[list[str]] = None,

347 sampling_params: Optional[Union[SamplingParams,

348 - List[SamplingParams]]] = None,

348 + list[SamplingParams]]] = None,

349 *,

350 - prompt_token_ids: List[List[int]],

350 + prompt_token_ids: list[list[int]],

351 use_tqdm: bool = True,

352 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

352 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

353 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

354 guided_options_request: Optional[Union[LLMGuidedOptions,

355 GuidedDecodingRequest]] = None,

356 - ) -> List[RequestOutput]:

356 + ) -> list[RequestOutput]:

357 ...

358

359 @overload # LEGACY: single or multi token ids [pos-only]

@@ -362,13 +362,13 @@ class LLM:

362 self,

363 prompts: None,

364 sampling_params: None,

365 - prompt_token_ids: Union[List[int], List[List[int]]],

365 + prompt_token_ids: Union[list[int], list[list[int]]],

366 use_tqdm: bool = True,

367 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

367 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

368 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

369 guided_options_request: Optional[Union[LLMGuidedOptions,

370 GuidedDecodingRequest]] = None,

371 - ) -> List[RequestOutput]:

371 + ) -> list[RequestOutput]:

372 ...

373

374 @deprecate_kwargs(

@@ -379,17 +379,17 @@ class LLM:

379 def generate(

380 self,

381 prompts: Union[Union[PromptType, Sequence[PromptType]],

382 - Optional[Union[str, List[str]]]] = None,

382 + Optional[Union[str, list[str]]]] = None,

383 sampling_params: Optional[Union[SamplingParams,

384 Sequence[SamplingParams]]] = None,

385 - prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,

385 + prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,

386 use_tqdm: bool = True,

387 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

387 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

388 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

389 guided_options_request: Optional[Union[LLMGuidedOptions,

390 GuidedDecodingRequest]] = None,

391 - priority: Optional[List[int]] = None,

392 - ) -> List[RequestOutput]:

391 + priority: Optional[list[int]] = None,

392 + ) -> list[RequestOutput]:

393 """Generates the completions for the input prompts.

394

395 This class automatically batches the given prompts, considering

@@ -440,7 +440,7 @@ class LLM:

440

441 if prompt_token_ids is not None:

442 parsed_prompts = self._convert_v1_inputs(

443 - prompts=cast(Optional[Union[str, List[str]]], prompts),

443 + prompts=cast(Optional[Union[str, list[str]]], prompts),

444 prompt_token_ids=prompt_token_ids,

445 )

446 else:

@@ -473,8 +473,8 @@ class LLM:

473 def collective_rpc(self,

474 method: Union[str, Callable[..., _R]],

475 timeout: Optional[float] = None,

476 - args: Tuple = (),

477 - kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:

476 + args: tuple = (),

477 + kwargs: Optional[dict[str, Any]] = None) -> list[_R]:

478 """

479 Execute an RPC call on all workers.

480

@@ -510,9 +510,9 @@ class LLM:

510

511 def beam_search(

512 self,

513 - prompts: List[Union[TokensPrompt, TextPrompt]],

513 + prompts: list[Union[TokensPrompt, TextPrompt]],

514 params: BeamSearchParams,

515 - ) -> List[BeamSearchOutput]:

515 + ) -> list[BeamSearchOutput]:

516 """

517 Generate sequences using beam search.

518

@@ -543,7 +543,7 @@ class LLM:

543 beam_search_params = SamplingParams(logprobs=2 * beam_width,

544 max_tokens=1,

545 temperature=temperature)

546 - instances: List[BeamSearchInstance] = []

546 + instances: list[BeamSearchInstance] = []

547

548 for prompt in prompts:

549 if is_token_prompt(prompt):

@@ -553,12 +553,12 @@ class LLM:

553 instances.append(BeamSearchInstance(prompt_tokens))

554

555 for _ in range(max_tokens):

556 - all_beams: List[BeamSearchSequence] = list(

556 + all_beams: list[BeamSearchSequence] = list(

557 sum((instance.beams for instance in instances), []))

558 pos = [0] + list(

559 itertools.accumulate(

560 len(instance.beams) for instance in instances))

561 - instance_start_and_end: List[Tuple[int, int]] = list(

561 + instance_start_and_end: list[tuple[int, int]] = list(

562 zip(pos[:-1], pos[1:]))

563

564 if len(all_beams) == 0:

@@ -620,19 +620,19 @@ class LLM:

620

621 def chat(

622 self,

623 - messages: Union[List[ChatCompletionMessageParam],

624 - List[List[ChatCompletionMessageParam]]],

623 + messages: Union[list[ChatCompletionMessageParam],

624 + list[list[ChatCompletionMessageParam]]],

625 sampling_params: Optional[Union[SamplingParams,

626 - List[SamplingParams]]] = None,

626 + list[SamplingParams]]] = None,

627 use_tqdm: bool = True,

628 lora_request: Optional[LoRARequest] = None,

629 chat_template: Optional[str] = None,

630 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

631 add_generation_prompt: bool = True,

632 continue_final_message: bool = False,

633 - tools: Optional[List[Dict[str, Any]]] = None,

634 - mm_processor_kwargs: Optional[Dict[str, Any]] = None,

635 - ) -> List[RequestOutput]:

633 + tools: Optional[list[dict[str, Any]]] = None,

634 + mm_processor_kwargs: Optional[dict[str, Any]] = None,

635 + ) -> list[RequestOutput]:

636 """

637 Generate responses for a chat conversation.

638

@@ -678,17 +678,17 @@ class LLM:

678 A list of ``RequestOutput`` objects containing the generated

679 responses in the same order as the input messages.

680 """

681 - list_of_messages: List[List[ChatCompletionMessageParam]]

681 + list_of_messages: list[list[ChatCompletionMessageParam]]

682

683 # Handle multi and single conversations

684 if is_list_of(messages, list):

685 - # messages is List[List[...]]

686 - list_of_messages = cast(List[List[ChatCompletionMessageParam]],

685 + # messages is list[list[...]]

686 + list_of_messages = cast(list[list[ChatCompletionMessageParam]],

687 messages)

688 else:

689 - # messages is List[...]

689 + # messages is list[...]

690 list_of_messages = [

691 - cast(List[ChatCompletionMessageParam], messages)

691 + cast(list[ChatCompletionMessageParam], messages)

692 ]

693

694 tokenizer = self.get_tokenizer()

@@ -699,7 +699,7 @@ class LLM:

699 tokenizer,

700 )

701

702 - prompts: List[Union[TokensPrompt, TextPrompt]] = []

702 + prompts: list[Union[TokensPrompt, TextPrompt]] = []

703

704 for msgs in list_of_messages:

705 # NOTE: _parse_chat_message_content_parts() currently doesn't

@@ -712,7 +712,7 @@ class LLM:

712 content_format=resolved_content_format,

713 )

714

715 - prompt_data: Union[str, List[int]]

715 + prompt_data: Union[str, list[int]]

716 if isinstance(tokenizer, MistralTokenizer):

717 prompt_data = apply_mistral_chat_template(

718 tokenizer,

@@ -762,9 +762,9 @@ class LLM:

762 Sequence[PoolingParams]]] = None,

763 *,

764 use_tqdm: bool = True,

765 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

765 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

766 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

767 - ) -> List[PoolingRequestOutput]:

767 + ) -> list[PoolingRequestOutput]:

768 ...

769

770 @overload # LEGACY: single (prompt + optional token ids)

@@ -774,25 +774,25 @@ class LLM:

774 prompts: str,

775 pooling_params: Optional[Union[PoolingParams,

776 Sequence[PoolingParams]]] = None,

777 - prompt_token_ids: Optional[List[int]] = None,

777 + prompt_token_ids: Optional[list[int]] = None,

778 use_tqdm: bool = True,

779 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

779 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

780 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

781 - ) -> List[PoolingRequestOutput]:

781 + ) -> list[PoolingRequestOutput]:

782 ...

783

784 @overload # LEGACY: multi (prompt + optional token ids)

785 @deprecated("'prompt_token_ids' will become part of 'prompts'")

786 def encode(

787 self,

788 - prompts: List[str],

788 + prompts: list[str],

789 pooling_params: Optional[Union[PoolingParams,

790 Sequence[PoolingParams]]] = None,

791 - prompt_token_ids: Optional[List[List[int]]] = None,

791 + prompt_token_ids: Optional[list[list[int]]] = None,

792 use_tqdm: bool = True,

793 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

793 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

794 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

795 - ) -> List[PoolingRequestOutput]:

795 + ) -> list[PoolingRequestOutput]:

796 ...

797

798 @overload # LEGACY: single (token ids + optional prompt)

@@ -803,26 +803,26 @@ class LLM:

803 pooling_params: Optional[Union[PoolingParams,

804 Sequence[PoolingParams]]] = None,

805 *,

806 - prompt_token_ids: List[int],

806 + prompt_token_ids: list[int],

807 use_tqdm: bool = True,

808 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

808 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

809 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

810 - ) -> List[PoolingRequestOutput]:

810 + ) -> list[PoolingRequestOutput]:

811 ...

812

813 @overload # LEGACY: multi (token ids + optional prompt)

814 @deprecated("'prompt_token_ids' will become part of 'prompts'")

815 def encode(

816 self,

817 - prompts: Optional[List[str]] = None,

817 + prompts: Optional[list[str]] = None,

818 pooling_params: Optional[Union[PoolingParams,

819 Sequence[PoolingParams]]] = None,

820 *,

821 - prompt_token_ids: List[List[int]],

821 + prompt_token_ids: list[list[int]],

822 use_tqdm: bool = True,

823 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

823 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

824 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

825 - ) -> List[PoolingRequestOutput]:

825 + ) -> list[PoolingRequestOutput]:

826 ...

827

828 @overload # LEGACY: single or multi token ids [pos-only]

@@ -831,11 +831,11 @@ class LLM:

831 self,

832 prompts: None,

833 pooling_params: None,

834 - prompt_token_ids: Union[List[int], List[List[int]]],

834 + prompt_token_ids: Union[list[int], list[list[int]]],

835 use_tqdm: bool = True,

836 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

836 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

837 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

838 - ) -> List[PoolingRequestOutput]:

838 + ) -> list[PoolingRequestOutput]:

839 ...

840

841 @deprecate_kwargs(

@@ -846,14 +846,14 @@ class LLM:

846 def encode(

847 self,

848 prompts: Union[Union[PromptType, Sequence[PromptType]],

849 - Optional[Union[str, List[str]]]] = None,

849 + Optional[Union[str, list[str]]]] = None,

850 pooling_params: Optional[Union[PoolingParams,

851 Sequence[PoolingParams]]] = None,

852 - prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,

852 + prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,

853 use_tqdm: bool = True,

854 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

854 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

855 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

856 - ) -> List[PoolingRequestOutput]:

856 + ) -> list[PoolingRequestOutput]:

857 """Apply pooling to the hidden states corresponding to the input

858 prompts.

859

@@ -898,7 +898,7 @@ class LLM:

898

899 if prompt_token_ids is not None:

900 parsed_prompts = self._convert_v1_inputs(

901 - prompts=cast(Optional[Union[str, List[str]]], prompts),

901 + prompts=cast(Optional[Union[str, list[str]]], prompts),

902 prompt_token_ids=prompt_token_ids,

903 )

904 else:

@@ -926,9 +926,9 @@ class LLM:

926 /,

927 *,

928 use_tqdm: bool = True,

929 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

929 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

930 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

931 - ) -> List[EmbeddingRequestOutput]:

931 + ) -> list[EmbeddingRequestOutput]:

932 """

933 Generate an embedding vector for each prompt.

934

@@ -966,9 +966,9 @@ class LLM:

966 /,

967 *,

968 use_tqdm: bool = True,

969 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

969 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

970 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

971 - ) -> List[ClassificationRequestOutput]:

971 + ) -> list[ClassificationRequestOutput]:

972 """

973 Generate class logits for each prompt.

974

@@ -1003,29 +1003,29 @@ class LLM:

1003 def _embedding_score(

1004 self,

1005 tokenizer: AnyTokenizer,

1006 - text_1: List[Union[str, TextPrompt, TokensPrompt]],

1007 - text_2: List[Union[str, TextPrompt, TokensPrompt]],

1006 + text_1: list[Union[str, TextPrompt, TokensPrompt]],

1007 + text_2: list[Union[str, TextPrompt, TokensPrompt]],

1008 truncate_prompt_tokens: Optional[int] = None,

1009 use_tqdm: bool = True,

1010 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

1010 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

1011 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

1012 - ) -> List[ScoringRequestOutput]:

1012 + ) -> list[ScoringRequestOutput]:

1013

1014 - encoded_output: List[PoolingRequestOutput] = self.encode(

1014 + encoded_output: list[PoolingRequestOutput] = self.encode(

1015 text_1 + text_2,

1016 use_tqdm=use_tqdm,

1017 lora_request=lora_request,

1018 prompt_adapter_request=prompt_adapter_request)

1019

1020 - encoded_output_1: List[PoolingRequestOutput] = encoded_output[

1020 + encoded_output_1: list[PoolingRequestOutput] = encoded_output[

1021 0:len(text_1)]

1022 - encoded_output_2: List[PoolingRequestOutput] = encoded_output[

1022 + encoded_output_2: list[PoolingRequestOutput] = encoded_output[

1023 len(text_1):]

1024

1025 if len(encoded_output_1) == 1:

1026 encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1027

1028 - scores: List[PoolingRequestOutput] = []

1028 + scores: list[PoolingRequestOutput] = []

1029

1030 scores = _cosine_similarity(tokenizer=tokenizer,

1031 embed_1=encoded_output_1,

@@ -1038,13 +1038,13 @@ class LLM:

1038 def _cross_encoding_score(

1039 self,

1040 tokenizer: AnyTokenizer,

1041 - text_1: List[str],

1042 - text_2: List[str],

1041 + text_1: list[str],

1042 + text_2: list[str],

1043 truncate_prompt_tokens: Optional[int] = None,

1044 use_tqdm: bool = True,

1045 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

1045 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

1046 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

1047 - ) -> List[ScoringRequestOutput]:

1047 + ) -> list[ScoringRequestOutput]:

1048

1049 if isinstance(tokenizer, MistralTokenizer):

1050 raise ValueError(

@@ -1057,7 +1057,7 @@ class LLM:

1057

1058 pooling_params = PoolingParams()

1059

1060 - tokenization_kwargs: Dict[str, Any] = {}

1060 + tokenization_kwargs: dict[str, Any] = {}

1061 if truncate_prompt_tokens is not None:

1062 tokenization_kwargs["truncation"] = True

1063 tokenization_kwargs["max_length"] = truncate_prompt_tokens

@@ -1094,9 +1094,9 @@ class LLM:

1094 *,

1095 truncate_prompt_tokens: Optional[int] = None,

1096 use_tqdm: bool = True,

1097 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,

1097 + lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

1098 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

1099 - ) -> List[ScoringRequestOutput]:

1099 + ) -> list[ScoringRequestOutput]:

1100 """Generate similarity scores for all pairs ``<text,text_pair>``.

1101

1102 The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.

@@ -1162,12 +1162,12 @@ class LLM:

1162 if isinstance(text_1, (str, dict)):

1163 # Convert a single prompt to a list.

1164 text_1 = [text_1]

1165 - input_text_1: List[str] = [ensure_str(t) for t in text_1]

1165 + input_text_1: list[str] = [ensure_str(t) for t in text_1]

1166

1167 if isinstance(text_2, (str, dict)):

1168 # Convert a single prompt to a list.

1169 text_2 = [text_2]

1170 - input_text_2: List[str] = [ensure_str(t) for t in text_2]

1170 + input_text_2: list[str] = [ensure_str(t) for t in text_2]

1171

1172 _validate_score_input_lens(input_text_1, input_text_2)

1173

@@ -1226,8 +1226,8 @@ class LLM:

1226 # LEGACY

1227 def _convert_v1_inputs(

1228 self,

1229 - prompts: Optional[Union[str, List[str]]],

1230 - prompt_token_ids: Optional[Union[List[int], List[List[int]]]],

1229 + prompts: Optional[Union[str, list[str]]],

1230 + prompt_token_ids: Optional[Union[list[int], list[list[int]]]],

1231 ):

1232 # skip_tokenizer_init is now checked in engine

1233

@@ -1252,7 +1252,7 @@ class LLM:

1252 raise ValueError("Either prompts or prompt_token_ids must be "

1253 "provided.")

1254

1255 - parsed_prompts: List[PromptType] = []

1255 + parsed_prompts: list[PromptType] = []

1256 for i in range(num_requests):

1257 item: PromptType

1258

@@ -1275,7 +1275,7 @@ class LLM:

1275 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],

1276 prompt_adapter_request: Optional[PromptAdapterRequest],

1277 guided_options: Optional[GuidedDecodingRequest] = None,

1278 - priority: Optional[List[int]] = None,

1278 + priority: Optional[list[int]] = None,

1279 ) -> None:

1280 if guided_options is not None:

1281 warnings.warn(

@@ -1357,7 +1357,7 @@ class LLM:

1357

1358 def _run_engine(

1359 self, *, use_tqdm: bool

1360 - ) -> List[Union[RequestOutput, PoolingRequestOutput]]:

1360 + ) -> list[Union[RequestOutput, PoolingRequestOutput]]:

1361 # Initialize tqdm.

1362 if use_tqdm:

1363 num_requests = self.llm_engine.get_num_unfinished_requests()

@@ -1370,7 +1370,7 @@ class LLM:

1370 )

1371

1372 # Run the engine.

1373 - outputs: List[Union[RequestOutput, PoolingRequestOutput]] = []

1373 + outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []

1374 total_in_toks = 0

1375 total_out_toks = 0

1376 while self.llm_engine.has_unfinished_requests():

vllm/entrypoints/logger.py

+2 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List, Optional, Union

3 +from typing import Optional, Union

4

5 from vllm.logger import init_logger

6 from vllm.lora.request import LoRARequest

@@ -22,7 +22,7 @@ class RequestLogger:

22 self,

23 request_id: str,

24 prompt: Optional[str],

25 - prompt_token_ids: Optional[List[int]],

25 + prompt_token_ids: Optional[list[int]],

26 params: Optional[Union[SamplingParams, PoolingParams,

27 BeamSearchParams]],

28 lora_request: Optional[LoRARequest],

vllm/entrypoints/openai/api_server.py

+5 -4

@@ -13,10 +13,11 @@ import socket

13 import tempfile

14 import uuid

15 from argparse import Namespace

16 +from collections.abc import AsyncIterator

17 from contextlib import asynccontextmanager

18 from functools import partial

19 from http import HTTPStatus

19 -from typing import Annotated, AsyncIterator, Dict, Optional, Set, Tuple, Union

20 +from typing import Annotated, Optional, Union

21

22 import uvloop

23 from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request

@@ -93,7 +94,7 @@ prometheus_multiproc_dir: tempfile.TemporaryDirectory

94 # Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)

95 logger = init_logger('vllm.entrypoints.openai.api_server')

96

96 -_running_tasks: Set[asyncio.Task] = set()

97 +_running_tasks: set[asyncio.Task] = set()

98

99

100 @asynccontextmanager

@@ -587,7 +588,7 @@ async def do_rerank_v2(request: RerankRequest, raw_request: Request):

588 return await do_rerank(request, raw_request)

589

590

590 -TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {

591 +TASK_HANDLERS: dict[str, dict[str, tuple]] = {

592 "generate": {

593 "messages": (ChatCompletionRequest, create_chat_completion),

594 "default": (CompletionRequest, create_completion),

@@ -894,7 +895,7 @@ async def init_app_state(

895 state.task = model_config.task

896

897

897 -def create_server_socket(addr: Tuple[str, int]) -> socket.socket:

898 +def create_server_socket(addr: tuple[str, int]) -> socket.socket:

899 family = socket.AF_INET

900 if is_valid_ipv6_address(addr[0]):

901 family = socket.AF_INET6

vllm/entrypoints/openai/cli_args.py

+4 -3

@@ -8,7 +8,8 @@ purposes.

8 import argparse

9 import json

10 import ssl

11 -from typing import List, Optional, Sequence, Union, get_args

11 +from collections.abc import Sequence

12 +from typing import Optional, Union, get_args

13

14 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str

15 from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,

@@ -33,7 +34,7 @@ class LoRAParserAction(argparse.Action):

34 if isinstance(values, str):

35 raise TypeError("Expected values to be a list")

36

36 - lora_list: List[LoRAModulePath] = []

37 + lora_list: list[LoRAModulePath] = []

38 for item in values:

39 if item in [None, '']: # Skip if item is None or empty string

40 continue

@@ -69,7 +70,7 @@ class PromptAdapterParserAction(argparse.Action):

70 if isinstance(values, str):

71 raise TypeError("Expected values to be a list")

72

72 - adapter_list: List[PromptAdapterPath] = []

73 + adapter_list: list[PromptAdapterPath] = []

74 for item in values:

75 name, path = item.split('=')

76 adapter_list.append(PromptAdapterPath(name, path))

vllm/entrypoints/openai/logits_processors.py

+12 -11

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import Iterable

4 from functools import lru_cache, partial

4 -from typing import Dict, FrozenSet, Iterable, List, Optional, Union

5 +from typing import Optional, Union

6

7 import torch

8

@@ -14,10 +15,10 @@ class AllowedTokenIdsLogitsProcessor:

15 specific set of token ids."""

16

17 def __init__(self, allowed_ids: Iterable[int]):

17 - self.allowed_ids: Optional[List[int]] = list(allowed_ids)

18 + self.allowed_ids: Optional[list[int]] = list(allowed_ids)

19 self.mask: Optional[torch.Tensor] = None

20

20 - def __call__(self, token_ids: List[int],

21 + def __call__(self, token_ids: list[int],

22 logits: torch.Tensor) -> torch.Tensor:

23 if self.mask is None:

24 self.mask = torch.ones((logits.shape[-1], ),

@@ -31,7 +32,7 @@ class AllowedTokenIdsLogitsProcessor:

32

33 @lru_cache(maxsize=32)

34 def _get_allowed_token_ids_logits_processor(

34 - allowed_token_ids: FrozenSet[int],

35 + allowed_token_ids: frozenset[int],

36 vocab_size: int,

37 ) -> LogitsProcessor:

38 if not allowed_token_ids:

@@ -43,8 +44,8 @@ def _get_allowed_token_ids_logits_processor(

44

45

46 def logit_bias_logits_processor(

46 - logit_bias: Dict[int, float],

47 - token_ids: List[int],

47 + logit_bias: dict[int, float],

48 + token_ids: list[int],

49 logits: torch.Tensor,

50 ) -> torch.Tensor:

51 for token_id, bias in logit_bias.items():

@@ -53,16 +54,16 @@ def logit_bias_logits_processor(

54

55

56 def get_logits_processors(

56 - logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],

57 - allowed_token_ids: Optional[List[int]],

57 + logit_bias: Optional[Union[dict[int, float], dict[str, float]]],

58 + allowed_token_ids: Optional[list[int]],

59 tokenizer: AnyTokenizer,

59 -) -> List[LogitsProcessor]:

60 - logits_processors: List[LogitsProcessor] = []

60 +) -> list[LogitsProcessor]:

61 + logits_processors: list[LogitsProcessor] = []

62 if logit_bias:

63 try:

64 # Convert token_id to integer

65 # Clamp the bias between -100 and 100 per OpenAI API spec

65 - clamped_logit_bias: Dict[int, float] = {

66 + clamped_logit_bias: dict[int, float] = {

67 int(token_id): min(100.0, max(-100.0, bias))

68 for token_id, bias in logit_bias.items()

69 }

vllm/entrypoints/openai/protocol.py

+64 -64

@@ -5,13 +5,13 @@

5 import re

6 import time

7 from argparse import Namespace

8 -from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union

8 +from typing import Annotated, Any, ClassVar, Literal, Optional, Union

9

10 import torch

11 from fastapi import UploadFile

12 from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,

13 ValidationInfo, field_validator, model_validator)

14 -from typing_extensions import Annotated, TypeAlias

14 +from typing_extensions import TypeAlias

15

16 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

17 from vllm.logger import init_logger

@@ -47,7 +47,7 @@ class OpenAIBaseModel(BaseModel):

47 model_config = ConfigDict(extra="allow")

48

49 # Cache class field names

50 - field_names: ClassVar[Optional[Set[str]]] = None

50 + field_names: ClassVar[Optional[set[str]]] = None

51

52 @model_validator(mode="wrap")

53 @classmethod

@@ -105,12 +105,12 @@ class ModelCard(OpenAIBaseModel):

105 root: Optional[str] = None

106 parent: Optional[str] = None

107 max_model_len: Optional[int] = None

108 - permission: List[ModelPermission] = Field(default_factory=list)

108 + permission: list[ModelPermission] = Field(default_factory=list)

109

110

111 class ModelList(OpenAIBaseModel):

112 object: str = "list"

113 - data: List[ModelCard] = Field(default_factory=list)

113 + data: list[ModelCard] = Field(default_factory=list)

114

115

116 class PromptTokenUsageInfo(OpenAIBaseModel):

@@ -134,7 +134,7 @@ class JsonSchemaResponseFormat(OpenAIBaseModel):

134 description: Optional[str] = None

135 # schema is the field in openai but that causes conflicts with pydantic so

136 # instead use json_schema with an alias

137 - json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')

137 + json_schema: Optional[dict[str, Any]] = Field(default=None, alias='schema')

138 strict: Optional[bool] = None

139

140

@@ -152,7 +152,7 @@ class StreamOptions(OpenAIBaseModel):

152 class FunctionDefinition(OpenAIBaseModel):

153 name: str

154 description: Optional[str] = None

155 - parameters: Optional[Dict[str, Any]] = None

155 + parameters: Optional[dict[str, Any]] = None

156

157

158 class ChatCompletionToolsParam(OpenAIBaseModel):

@@ -171,15 +171,15 @@ class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):

171

172 class LogitsProcessorConstructor(BaseModel):

173 qualname: str

174 - args: Optional[List[Any]] = None

175 - kwargs: Optional[Dict[str, Any]] = None

174 + args: Optional[list[Any]] = None

175 + kwargs: Optional[dict[str, Any]] = None

176

177

178 -LogitsProcessors = List[Union[str, LogitsProcessorConstructor]]

178 +LogitsProcessors = list[Union[str, LogitsProcessorConstructor]]

179

180

181 def get_logits_processors(processors: Optional[LogitsProcessors],

182 - pattern: Optional[str]) -> Optional[List[Any]]:

182 + pattern: Optional[str]) -> Optional[list[Any]]:

183 if processors and pattern:

184 logits_processors = []

185 for processor in processors:

@@ -212,10 +212,10 @@ def get_logits_processors(processors: Optional[LogitsProcessors],

212 class ChatCompletionRequest(OpenAIBaseModel):

213 # Ordered by official OpenAI API documentation

214 # https://platform.openai.com/docs/api-reference/chat/create

215 - messages: List[ChatCompletionMessageParam]

215 + messages: list[ChatCompletionMessageParam]

216 model: Optional[str] = None

217 frequency_penalty: Optional[float] = 0.0

218 - logit_bias: Optional[Dict[str, float]] = None

218 + logit_bias: Optional[dict[str, float]] = None

219 logprobs: Optional[bool] = False

220 top_logprobs: Optional[int] = 0

221 # TODO(#9845): remove max_tokens when field is removed from OpenAI API

@@ -228,12 +228,12 @@ class ChatCompletionRequest(OpenAIBaseModel):

228 presence_penalty: Optional[float] = 0.0

229 response_format: Optional[ResponseFormat] = None

230 seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

231 - stop: Optional[Union[str, List[str]]] = Field(default_factory=list)

231 + stop: Optional[Union[str, list[str]]] = Field(default_factory=list)

232 stream: Optional[bool] = False

233 stream_options: Optional[StreamOptions] = None

234 temperature: Optional[float] = None

235 top_p: Optional[float] = None

236 - tools: Optional[List[ChatCompletionToolsParam]] = None

236 + tools: Optional[list[ChatCompletionToolsParam]] = None

237 tool_choice: Optional[Union[Literal["none"], Literal["auto"],

238 ChatCompletionNamedToolChoiceParam]] = "none"

239

@@ -248,7 +248,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

248 min_p: Optional[float] = None

249 repetition_penalty: Optional[float] = None

250 length_penalty: float = 1.0

251 - stop_token_ids: Optional[List[int]] = Field(default_factory=list)

251 + stop_token_ids: Optional[list[int]] = Field(default_factory=list)

252 include_stop_str_in_output: bool = False

253 ignore_eos: bool = False

254 min_tokens: int = 0

@@ -290,7 +290,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

290 "special tokens so this should be set to false (as is the "

291 "default)."),

292 )

293 - documents: Optional[List[Dict[str, str]]] = Field(

293 + documents: Optional[list[dict[str, str]]] = Field(

294 default=None,

295 description=

296 ("A list of dicts representing documents that will be accessible to "

@@ -307,12 +307,12 @@ class ChatCompletionRequest(OpenAIBaseModel):

307 "allowed, so you must provide a chat template if the tokenizer "

308 "does not define one."),

309 )

310 - chat_template_kwargs: Optional[Dict[str, Any]] = Field(

310 + chat_template_kwargs: Optional[dict[str, Any]] = Field(

311 default=None,

312 description=("Additional kwargs to pass to the template renderer. "

313 "Will be accessible by the chat template."),

314 )

315 - mm_processor_kwargs: Optional[Dict[str, Any]] = Field(

315 + mm_processor_kwargs: Optional[dict[str, Any]] = Field(

316 default=None,

317 description=("Additional kwargs to pass to the HF processor."),

318 )

@@ -325,7 +325,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

325 description=(

326 "If specified, the output will follow the regex pattern."),

327 )

328 - guided_choice: Optional[List[str]] = Field(

328 + guided_choice: Optional[list[str]] = Field(

329 default=None,

330 description=(

331 "If specified, the output will be exactly one of the choices."),

@@ -643,17 +643,17 @@ class CompletionRequest(OpenAIBaseModel):

643 # Ordered by official OpenAI API documentation

644 # https://platform.openai.com/docs/api-reference/completions/create

645 model: Optional[str] = None

646 - prompt: Union[List[int], List[List[int]], str, List[str]]

646 + prompt: Union[list[int], list[list[int]], str, list[str]]

647 best_of: Optional[int] = None

648 echo: Optional[bool] = False

649 frequency_penalty: Optional[float] = 0.0

650 - logit_bias: Optional[Dict[str, float]] = None

650 + logit_bias: Optional[dict[str, float]] = None

651 logprobs: Optional[int] = None

652 max_tokens: Optional[int] = 16

653 n: int = 1

654 presence_penalty: Optional[float] = 0.0

655 seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

656 - stop: Optional[Union[str, List[str]]] = Field(default_factory=list)

656 + stop: Optional[Union[str, list[str]]] = Field(default_factory=list)

657 stream: Optional[bool] = False

658 stream_options: Optional[StreamOptions] = None

659 suffix: Optional[str] = None

@@ -667,14 +667,14 @@ class CompletionRequest(OpenAIBaseModel):

667 min_p: Optional[float] = None

668 repetition_penalty: Optional[float] = None

669 length_penalty: float = 1.0

670 - stop_token_ids: Optional[List[int]] = Field(default_factory=list)

670 + stop_token_ids: Optional[list[int]] = Field(default_factory=list)

671 include_stop_str_in_output: bool = False

672 ignore_eos: bool = False

673 min_tokens: int = 0

674 skip_special_tokens: bool = True

675 spaces_between_special_tokens: bool = True

676 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

677 - allowed_token_ids: Optional[List[int]] = None

677 + allowed_token_ids: Optional[list[int]] = None

678 prompt_logprobs: Optional[int] = None

679 # doc: end-completion-sampling-params

680

@@ -701,7 +701,7 @@ class CompletionRequest(OpenAIBaseModel):

701 description=(

702 "If specified, the output will follow the regex pattern."),

703 )

704 - guided_choice: Optional[List[str]] = Field(

704 + guided_choice: Optional[list[str]] = Field(

705 default=None,

706 description=(

707 "If specified, the output will be exactly one of the choices."),

@@ -908,7 +908,7 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):

908 # Ordered by official OpenAI API documentation

909 # https://platform.openai.com/docs/api-reference/embeddings

910 model: Optional[str] = None

911 - input: Union[List[int], List[List[int]], str, List[str]]

911 + input: Union[list[int], list[list[int]], str, list[str]]

912 encoding_format: Literal["float", "base64"] = "float"

913 dimensions: Optional[int] = None

914 user: Optional[str] = None

@@ -940,7 +940,7 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):

940

941 class EmbeddingChatRequest(OpenAIBaseModel):

942 model: Optional[str] = None

943 - messages: List[ChatCompletionMessageParam]

943 + messages: list[ChatCompletionMessageParam]

944

945 encoding_format: Literal["float", "base64"] = "float"

946 dimensions: Optional[int] = None

@@ -969,12 +969,12 @@ class EmbeddingChatRequest(OpenAIBaseModel):

969 "allowed, so you must provide a chat template if the tokenizer "

970 "does not define one."),

971 )

972 - chat_template_kwargs: Optional[Dict[str, Any]] = Field(

972 + chat_template_kwargs: Optional[dict[str, Any]] = Field(

973 default=None,

974 description=("Additional kwargs to pass to the template renderer. "

975 "Will be accessible by the chat template."),

976 )

977 - mm_processor_kwargs: Optional[Dict[str, Any]] = Field(

977 + mm_processor_kwargs: Optional[dict[str, Any]] = Field(

978 default=None,

979 description=("Additional kwargs to pass to the HF processor."),

980 )

@@ -1008,8 +1008,8 @@ PoolingRequest = Union[PoolingCompletionRequest, PoolingChatRequest]

1008

1009 class ScoreRequest(OpenAIBaseModel):

1010 model: Optional[str] = None

1011 - text_1: Union[List[str], str]

1012 - text_2: Union[List[str], str]

1011 + text_1: Union[list[str], str]

1012 + text_2: Union[list[str], str]

1013 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

1014

1015 # doc: begin-score-pooling-params

@@ -1033,7 +1033,7 @@ class ScoreRequest(OpenAIBaseModel):

1033 class RerankRequest(OpenAIBaseModel):

1034 model: Optional[str] = None

1035 query: str

1036 - documents: List[str]

1036 + documents: list[str]

1037 top_n: int = Field(default_factory=lambda: 0)

1038 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

1039

@@ -1073,14 +1073,14 @@ class RerankResponse(OpenAIBaseModel):

1073 id: str

1074 model: str

1075 usage: RerankUsage

1076 - results: List[RerankResult]

1076 + results: list[RerankResult]

1077

1078

1079 class CompletionLogProbs(OpenAIBaseModel):

1080 - text_offset: List[int] = Field(default_factory=list)

1081 - token_logprobs: List[Optional[float]] = Field(default_factory=list)

1082 - tokens: List[str] = Field(default_factory=list)

1083 - top_logprobs: List[Optional[Dict[str,

1080 + text_offset: list[int] = Field(default_factory=list)

1081 + token_logprobs: list[Optional[float]] = Field(default_factory=list)

1082 + tokens: list[str] = Field(default_factory=list)

1083 + top_logprobs: list[Optional[dict[str,

1084 float]]] = Field(default_factory=list)

1085

1086

@@ -1096,7 +1096,7 @@ class CompletionResponseChoice(OpenAIBaseModel):

1096 "to stop, None if the completion finished for some other reason "

1097 "including encountering the EOS token"),

1098 )

1099 - prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None

1099 + prompt_logprobs: Optional[list[Optional[dict[int, Logprob]]]] = None

1100

1101

1102 class CompletionResponse(OpenAIBaseModel):

@@ -1104,7 +1104,7 @@ class CompletionResponse(OpenAIBaseModel):

1104 object: str = "text_completion"

1105 created: int = Field(default_factory=lambda: int(time.time()))

1106 model: str

1107 - choices: List[CompletionResponseChoice]

1107 + choices: list[CompletionResponseChoice]

1108 usage: UsageInfo

1109

1110

@@ -1127,14 +1127,14 @@ class CompletionStreamResponse(OpenAIBaseModel):

1127 object: str = "text_completion"

1128 created: int = Field(default_factory=lambda: int(time.time()))

1129 model: str

1130 - choices: List[CompletionResponseStreamChoice]

1130 + choices: list[CompletionResponseStreamChoice]

1131 usage: Optional[UsageInfo] = Field(default=None)

1132

1133

1134 class EmbeddingResponseData(OpenAIBaseModel):

1135 index: int

1136 object: str = "embedding"

1137 - embedding: Union[List[float], str]

1137 + embedding: Union[list[float], str]

1138

1139

1140 class EmbeddingResponse(OpenAIBaseModel):

@@ -1142,14 +1142,14 @@ class EmbeddingResponse(OpenAIBaseModel):

1142 object: str = "list"

1143 created: int = Field(default_factory=lambda: int(time.time()))

1144 model: str

1145 - data: List[EmbeddingResponseData]

1145 + data: list[EmbeddingResponseData]

1146 usage: UsageInfo

1147

1148

1149 class PoolingResponseData(OpenAIBaseModel):

1150 index: int

1151 object: str = "pooling"

1152 - data: Union[List[List[float]], List[float], str]

1152 + data: Union[list[list[float]], list[float], str]

1153

1154

1155 class PoolingResponse(OpenAIBaseModel):

@@ -1157,7 +1157,7 @@ class PoolingResponse(OpenAIBaseModel):

1157 object: str = "list"

1158 created: int = Field(default_factory=lambda: int(time.time()))

1159 model: str

1160 - data: List[PoolingResponseData]

1160 + data: list[PoolingResponseData]

1161 usage: UsageInfo

1162

1163

@@ -1172,7 +1172,7 @@ class ScoreResponse(OpenAIBaseModel):

1172 object: str = "list"

1173 created: int = Field(default_factory=lambda: int(time.time()))

1174 model: str

1175 - data: List[ScoreResponseData]

1175 + data: list[ScoreResponseData]

1176 usage: UsageInfo

1177

1178

@@ -1205,7 +1205,7 @@ class ExtractedToolCallInformation(BaseModel):

1205 tools_called: bool

1206

1207 # extracted tool calls

1208 - tool_calls: List[ToolCall]

1208 + tool_calls: list[ToolCall]

1209

1210 # content - per OpenAI spec, content AND tool calls can be returned rarely

1211 # But some models will do this intentionally

@@ -1216,21 +1216,21 @@ class ChatMessage(OpenAIBaseModel):

1216 role: str

1217 reasoning_content: Optional[str] = None

1218 content: Optional[str] = None

1219 - tool_calls: List[ToolCall] = Field(default_factory=list)

1219 + tool_calls: list[ToolCall] = Field(default_factory=list)

1220

1221

1222 class ChatCompletionLogProb(OpenAIBaseModel):

1223 token: str

1224 logprob: float = -9999.0

1225 - bytes: Optional[List[int]] = None

1225 + bytes: Optional[list[int]] = None

1226

1227

1228 class ChatCompletionLogProbsContent(ChatCompletionLogProb):

1229 - top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)

1229 + top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)

1230

1231

1232 class ChatCompletionLogProbs(OpenAIBaseModel):

1233 - content: Optional[List[ChatCompletionLogProbsContent]] = None

1233 + content: Optional[list[ChatCompletionLogProbsContent]] = None

1234

1235

1236 class ChatCompletionResponseChoice(OpenAIBaseModel):

@@ -1248,16 +1248,16 @@ class ChatCompletionResponse(OpenAIBaseModel):

1248 object: Literal["chat.completion"] = "chat.completion"

1249 created: int = Field(default_factory=lambda: int(time.time()))

1250 model: str

1251 - choices: List[ChatCompletionResponseChoice]

1251 + choices: list[ChatCompletionResponseChoice]

1252 usage: UsageInfo

1253 - prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None

1253 + prompt_logprobs: Optional[list[Optional[dict[int, Logprob]]]] = None

1254

1255

1256 class DeltaMessage(OpenAIBaseModel):

1257 role: Optional[str] = None

1258 content: Optional[str] = None

1259 reasoning_content: Optional[str] = None

1260 - tool_calls: List[DeltaToolCall] = Field(default_factory=list)

1260 + tool_calls: list[DeltaToolCall] = Field(default_factory=list)

1261

1262

1263 class ChatCompletionResponseStreamChoice(OpenAIBaseModel):

@@ -1273,7 +1273,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):

1273 object: Literal["chat.completion.chunk"] = "chat.completion.chunk"

1274 created: int = Field(default_factory=lambda: int(time.time()))

1275 model: str

1276 - choices: List[ChatCompletionResponseStreamChoice]

1276 + choices: list[ChatCompletionResponseStreamChoice]

1277 usage: Optional[UsageInfo] = Field(default=None)

1278

1279

@@ -1358,7 +1358,7 @@ class TokenizeCompletionRequest(OpenAIBaseModel):

1358

1359 class TokenizeChatRequest(OpenAIBaseModel):

1360 model: Optional[str] = None

1361 - messages: List[ChatCompletionMessageParam]

1361 + messages: list[ChatCompletionMessageParam]

1362

1363 add_generation_prompt: bool = Field(

1364 default=True,

@@ -1393,12 +1393,12 @@ class TokenizeChatRequest(OpenAIBaseModel):

1393 "allowed, so you must provide a chat template if the tokenizer "

1394 "does not define one."),

1395 )

1396 - chat_template_kwargs: Optional[Dict[str, Any]] = Field(

1396 + chat_template_kwargs: Optional[dict[str, Any]] = Field(

1397 default=None,

1398 description=("Additional kwargs to pass to the template renderer. "

1399 "Will be accessible by the chat template."),

1400 )

1401 - mm_processor_kwargs: Optional[Dict[str, Any]] = Field(

1401 + mm_processor_kwargs: Optional[dict[str, Any]] = Field(

1402 default=None,

1403 description=("Additional kwargs to pass to the HF processor."),

1404 )

@@ -1419,12 +1419,12 @@ TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]

1419 class TokenizeResponse(OpenAIBaseModel):

1420 count: int

1421 max_model_len: int

1422 - tokens: List[int]

1422 + tokens: list[int]

1423

1424

1425 class DetokenizeRequest(OpenAIBaseModel):

1426 model: Optional[str] = None

1427 - tokens: List[int]

1427 + tokens: list[int]

1428

1429

1430 class DetokenizeResponse(OpenAIBaseModel):

@@ -1492,7 +1492,7 @@ class TranscriptionRequest(OpenAIBaseModel):

1492 to automatically increase the temperature until certain thresholds are hit.

1493 """

1494

1495 - timestamp_granularities: List[Literal["word", "segment"]] = Field(

1495 + timestamp_granularities: list[Literal["word", "segment"]] = Field(

1496 alias="timestamp_granularities[]", default=[])

1497 """The timestamp granularities to populate for this transcription.

1498

@@ -1580,7 +1580,7 @@ class TranscriptionSegment(OpenAIBaseModel):

1580 text: str

1581 """Text content of the segment."""

1582

1583 - tokens: List[int]

1583 + tokens: list[int]

1584 """Array of token IDs for the text content."""

1585

1586

@@ -1594,8 +1594,8 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):

1594 text: str

1595 """The transcribed text."""

1596

1597 - segments: Optional[List[TranscriptionSegment]] = None

1597 + segments: Optional[list[TranscriptionSegment]] = None

1598 """Segments of the transcribed text and their corresponding details."""

1599

1600 - words: Optional[List[TranscriptionWord]] = None

1600 + words: Optional[list[TranscriptionWord]] = None

1601 """Extracted words and their corresponding timestamps."""

vllm/entrypoints/openai/reasoning_parsers/abs_reasoning_parsers.py

+11 -10

@@ -1,8 +1,9 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import os

4 +from collections.abc import Sequence

5 from functools import cached_property

5 -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union

6 +from typing import Callable, Optional, Union

7

8 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

9 DeltaMessage)

@@ -25,14 +26,14 @@ class ReasoningParser:

26 self.model_tokenizer = tokenizer

27

28 @cached_property

28 - def vocab(self) -> Dict[str, int]:

29 + def vocab(self) -> dict[str, int]:

30 # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab

31 # whereas all tokenizers have .get_vocab()

32 return self.model_tokenizer.get_vocab()

33

34 def extract_reasoning_content(

35 self, model_output: str, request: ChatCompletionRequest

35 - ) -> Tuple[Optional[str], Optional[str]]:

36 + ) -> tuple[Optional[str], Optional[str]]:

37 """

38 Extract reasoning content from a complete model-generated string.

39

@@ -47,7 +48,7 @@ class ReasoningParser:

48 The request object that was used to generate the model_output.

49

50 Returns:

50 - Tuple[Optional[str], Optional[str]]

51 + tuple[Optional[str], Optional[str]]

52 A tuple containing the reasoning content and the content.

53 """

54

@@ -77,10 +78,10 @@ class ReasoningParser:

78

79

80 class ReasoningParserManager:

80 - reasoning_parsers: Dict[str, Type] = {}

81 + reasoning_parsers: dict[str, type] = {}

82

83 @classmethod

83 - def get_reasoning_parser(cls, name) -> Type:

84 + def get_reasoning_parser(cls, name) -> type:

85 """

86 Get reasoning parser by name which is registered by `register_module`.

87

@@ -94,8 +95,8 @@ class ReasoningParserManager:

95

96 @classmethod

97 def _register_module(cls,

97 - module: Type,

98 - module_name: Optional[Union[str, List[str]]] = None,

98 + module: type,

99 + module_name: Optional[Union[str, list[str]]] = None,

100 force: bool = True) -> None:

101 if not issubclass(module, ReasoningParser):

102 raise TypeError("module must be subclass of ReasoningParser, "

@@ -114,9 +115,9 @@ class ReasoningParserManager:

115 @classmethod

116 def register_module(

117 cls,

117 - name: Optional[Union[str, List[str]]] = None,

118 + name: Optional[Union[str, list[str]]] = None,

119 force: bool = True,

119 - module: Union[Type, None] = None) -> Union[type, Callable]:

120 + module: Union[type, None] = None) -> Union[type, Callable]:

121 """

122 Register module with the given name or name list. it can be used as a

123 decoder(with module as None) or normal function(with module as not

vllm/entrypoints/openai/reasoning_parsers/deepseek_r1_reasoning_parser.py

+3 -2

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import re

4 -from typing import Optional, Sequence, Tuple, Union

4 +from collections.abc import Sequence

5 +from typing import Optional, Union

6

7 from transformers import PreTrainedTokenizerBase

8

@@ -122,7 +123,7 @@ class DeepSeekR1ReasoningParser(ReasoningParser):

123

124 def extract_reasoning_content(

125 self, model_output: str, request: ChatCompletionRequest

125 - ) -> Tuple[Optional[str], Optional[str]]:

126 + ) -> tuple[Optional[str], Optional[str]]:

127

128 # DeepSeek R1 doesn't generate <think> now.

129 # Thus we assume the reasoning content is always at the start.

vllm/entrypoints/openai/run_batch.py

+5 -4

@@ -2,9 +2,10 @@

2

3 import asyncio

4 import tempfile

5 +from collections.abc import Awaitable

6 from http import HTTPStatus

7 from io import StringIO

7 -from typing import Awaitable, Callable, List, Optional

8 +from typing import Callable, Optional

9

10 import aiohttp

11 import torch

@@ -143,7 +144,7 @@ async def read_file(path_or_url: str) -> str:

144

145

146 async def write_local_file(output_path: str,

146 - batch_outputs: List[BatchRequestOutput]) -> None:

147 + batch_outputs: list[BatchRequestOutput]) -> None:

148 """

149 Write the responses to a local file.

150 output_path: The path to write the responses to.

@@ -204,7 +205,7 @@ async def upload_data(output_url: str, data_or_file: str,

205 f"Error message: {str(e)}.") from e

206

207

207 -async def write_file(path_or_url: str, batch_outputs: List[BatchRequestOutput],

208 +async def write_file(path_or_url: str, batch_outputs: list[BatchRequestOutput],

209 output_tmp_dir: str) -> None:

210 """

211 Write batch_outputs to a file or upload to a URL.

@@ -353,7 +354,7 @@ async def main(args):

354 logger.info("Reading batch from %s...", args.input_file)

355

356 # Submit all requests in the file to the engine "concurrently".

356 - response_futures: List[Awaitable[BatchRequestOutput]] = []

357 + response_futures: list[Awaitable[BatchRequestOutput]] = []

358 for request_json in (await read_file(args.input_file)).strip().split("\n"):

359 # Skip empty lines.

360 request_json = request_json.strip()

vllm/entrypoints/openai/serving_chat.py

+15 -16

@@ -3,10 +3,9 @@

3 import asyncio

4 import json

5 import time

6 -from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,

7 - Optional)

8 -from typing import Sequence as GenericSequence

9 -from typing import Union

6 +from collections.abc import AsyncGenerator, AsyncIterator

7 +from collections.abc import Sequence as GenericSequence

8 +from typing import Callable, Final, Optional, Union

9

10 from fastapi import Request

11

@@ -205,7 +204,7 @@ class OpenAIServingChat(OpenAIServing):

204 raw_request.state.request_metadata = request_metadata

205

206 # Schedule the request and get the result generator.

208 - generators: List[AsyncGenerator[RequestOutput, None]] = []

207 + generators: list[AsyncGenerator[RequestOutput, None]] = []

208 try:

209 for i, engine_prompt in enumerate(engine_prompts):

210 sampling_params: Union[SamplingParams, BeamSearchParams]

@@ -282,7 +281,7 @@ class OpenAIServingChat(OpenAIServing):

281 result_generator: AsyncIterator[RequestOutput],

282 request_id: str,

283 model_name: str,

285 - conversation: List[ConversationMessage],

284 + conversation: list[ConversationMessage],

285 tokenizer: AnyTokenizer,

286 request_metadata: RequestResponseMetadata,

287 ) -> AsyncGenerator[str, None]:

@@ -310,7 +309,7 @@ class OpenAIServingChat(OpenAIServing):

309 should_stream_with_reasoning_parsing = (

310 self._should_stream_with_reasoning_parsing(request))

311

313 - all_previous_token_ids: Optional[List[List[int]]]

312 + all_previous_token_ids: Optional[list[list[int]]]

313

314 # Only one of these will be used, thus previous_texts and

315 # all_previous_token_ids will not be used twice in the same iteration.

@@ -339,7 +338,7 @@ class OpenAIServingChat(OpenAIServing):

338 # Prepare the tool parser if it's needed

339 try:

340 if tool_choice_auto and self.tool_parser:

342 - tool_parsers: List[Optional[ToolParser]] = [

341 + tool_parsers: list[Optional[ToolParser]] = [

342 self.tool_parser(tokenizer)

343 ] * num_choices

344 else:

@@ -406,7 +405,7 @@ class OpenAIServingChat(OpenAIServing):

405 # Send response to echo the input portion of the

406 # last message

407 if request.echo:

409 - last_msg_content: Union[str, List[Dict[str, str]]] = ""

408 + last_msg_content: Union[str, list[dict[str, str]]] = ""

409 if conversation and "content" in conversation[

410 -1] and conversation[-1].get("role") == role:

411 last_msg_content = conversation[-1]["content"] or ""

@@ -674,7 +673,7 @@ class OpenAIServingChat(OpenAIServing):

673 result_generator: AsyncIterator[RequestOutput],

674 request_id: str,

675 model_name: str,

677 - conversation: List[ConversationMessage],

676 + conversation: list[ConversationMessage],

677 tokenizer: AnyTokenizer,

678 request_metadata: RequestResponseMetadata,

679 ) -> Union[ErrorResponse, ChatCompletionResponse]:

@@ -693,7 +692,7 @@ class OpenAIServingChat(OpenAIServing):

692

693 assert final_res is not None

694

696 - choices: List[ChatCompletionResponseChoice] = []

695 + choices: list[ChatCompletionResponseChoice] = []

696

697 role = self.get_chat_request_role(request)

698 for output in final_res.outputs:

@@ -812,7 +811,7 @@ class OpenAIServingChat(OpenAIServing):

811 choices.append(choice_data)

812

813 if request.echo:

815 - last_msg_content: Union[str, List[Dict[str, str]]] = ""

814 + last_msg_content: Union[str, list[dict[str, str]]] = ""

815 if conversation and "content" in conversation[-1] and conversation[

816 -1].get("role") == role:

817 last_msg_content = conversation[-1]["content"] or ""

@@ -853,8 +852,8 @@ class OpenAIServingChat(OpenAIServing):

852 return response

853

854 def _get_top_logprobs(

856 - self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],

857 - tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:

855 + self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],

856 + tokenizer: AnyTokenizer) -> list[ChatCompletionLogProb]:

857 return [

858 ChatCompletionLogProb(token=(token := self._get_decoded_token(

859 p[1],

@@ -871,12 +870,12 @@ class OpenAIServingChat(OpenAIServing):

870 def _create_chat_logprobs(

871 self,

872 token_ids: GenericSequence[int],

874 - top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],

873 + top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],

874 tokenizer: AnyTokenizer,

875 num_output_top_logprobs: Optional[int] = None,

876 ) -> ChatCompletionLogProbs:

877 """Create OpenAI-style logprobs."""

879 - logprobs_content: List[ChatCompletionLogProbsContent] = []

878 + logprobs_content: list[ChatCompletionLogProbsContent] = []

879

880 for i, token_id in enumerate(token_ids):

881 step_top_logprobs = top_logprobs[i]

vllm/entrypoints/openai/serving_completion.py

+16 -16

@@ -2,9 +2,9 @@

2

3 import asyncio

4 import time

5 -from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional

6 -from typing import Sequence as GenericSequence

7 -from typing import Tuple, Union, cast

5 +from collections.abc import AsyncGenerator, AsyncIterator

6 +from collections.abc import Sequence as GenericSequence

7 +from typing import Optional, Union, cast

8

9 from fastapi import Request

10

@@ -113,7 +113,7 @@ class OpenAIServingCompletion(OpenAIServing):

113 return self.create_error_response(str(e))

114

115 # Schedule the request and get the result generator.

116 - generators: List[AsyncGenerator[RequestOutput, None]] = []

116 + generators: list[AsyncGenerator[RequestOutput, None]] = []

117 try:

118 for i, engine_prompt in enumerate(engine_prompts):

119 sampling_params: Union[SamplingParams, BeamSearchParams]

@@ -189,7 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):

189 request_metadata=request_metadata)

190

191 # Non-streaming response

192 - final_res_batch: List[Optional[RequestOutput]] = [None] * num_prompts

192 + final_res_batch: list[Optional[RequestOutput]] = [None] * num_prompts

193 try:

194 async for i, res in result_generator:

195 final_res_batch[i] = res

@@ -203,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):

203 if final_res.prompt is None:

204 final_res.prompt = request_prompts[i]["prompt"]

205

206 - final_res_batch_checked = cast(List[RequestOutput],

206 + final_res_batch_checked = cast(list[RequestOutput],

207 final_res_batch)

208

209 response = self.request_output_to_completion_response(

@@ -237,7 +237,7 @@ class OpenAIServingCompletion(OpenAIServing):

237 async def completion_stream_generator(

238 self,

239 request: CompletionRequest,

240 - result_generator: AsyncIterator[Tuple[int, RequestOutput]],

240 + result_generator: AsyncIterator[tuple[int, RequestOutput]],

241 request_id: str,

242 created_time: int,

243 model_name: str,

@@ -270,7 +270,7 @@ class OpenAIServingCompletion(OpenAIServing):

270 num_prompt_tokens[prompt_idx] = len(res.prompt_token_ids)

271

272 delta_token_ids: GenericSequence[int]

273 - out_logprobs: Optional[GenericSequence[Optional[Dict[

273 + out_logprobs: Optional[GenericSequence[Optional[dict[

274 int, Logprob]]]]

275

276 for output in res.outputs:

@@ -381,7 +381,7 @@ class OpenAIServingCompletion(OpenAIServing):

381

382 def request_output_to_completion_response(

383 self,

384 - final_res_batch: List[RequestOutput],

384 + final_res_batch: list[RequestOutput],

385 request: CompletionRequest,

386 request_id: str,

387 created_time: int,

@@ -389,7 +389,7 @@ class OpenAIServingCompletion(OpenAIServing):

389 tokenizer: AnyTokenizer,

390 request_metadata: RequestResponseMetadata,

391 ) -> CompletionResponse:

392 - choices: List[CompletionResponseChoice] = []

392 + choices: list[CompletionResponseChoice] = []

393 num_prompt_tokens = 0

394 num_generated_tokens = 0

395

@@ -406,7 +406,7 @@ class OpenAIServingCompletion(OpenAIServing):

406 prompt_text = final_res.prompt

407

408 token_ids: GenericSequence[int]

409 - out_logprobs: Optional[GenericSequence[Optional[Dict[int,

409 + out_logprobs: Optional[GenericSequence[Optional[dict[int,

410 Logprob]]]]

411

412 for output in final_res.outputs:

@@ -480,16 +480,16 @@ class OpenAIServingCompletion(OpenAIServing):

480 def _create_completion_logprobs(

481 self,

482 token_ids: GenericSequence[int],

483 - top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],

483 + top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],

484 num_output_top_logprobs: int,

485 tokenizer: AnyTokenizer,

486 initial_text_offset: int = 0,

487 ) -> CompletionLogProbs:

488 """Create logprobs for OpenAI Completion API."""

489 - out_text_offset: List[int] = []

490 - out_token_logprobs: List[Optional[float]] = []

491 - out_tokens: List[str] = []

492 - out_top_logprobs: List[Optional[Dict[str, float]]] = []

489 + out_text_offset: list[int] = []

490 + out_token_logprobs: list[Optional[float]] = []

491 + out_tokens: list[str] = []

492 + out_top_logprobs: list[Optional[dict[str, float]]] = []

493

494 last_token_len = 0

495

vllm/entrypoints/openai/serving_embedding.py

+8 -7

@@ -3,7 +3,8 @@

3 import asyncio

4 import base64

5 import time

6 -from typing import AsyncGenerator, Final, List, Literal, Optional, Union, cast

6 +from collections.abc import AsyncGenerator

7 +from typing import Final, Literal, Optional, Union, cast

8

9 import numpy as np

10 from fastapi import Request

@@ -31,7 +32,7 @@ logger = init_logger(__name__)

32 def _get_embedding(

33 output: EmbeddingOutput,

34 encoding_format: Literal["float", "base64"],

34 -) -> Union[List[float], str]:

35 +) -> Union[list[float], str]:

36 if encoding_format == "float":

37 return output.embedding

38 elif encoding_format == "base64":

@@ -143,7 +144,7 @@ class OpenAIServingEmbedding(OpenAIServing):

144 return self.create_error_response(str(e))

145

146 # Schedule the request and get the result generator.

146 - generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []

147 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

148 try:

149 pooling_params = request.to_pooling_params()

150

@@ -178,7 +179,7 @@ class OpenAIServingEmbedding(OpenAIServing):

179 num_prompts = len(engine_prompts)

180

181 # Non-streaming response

181 - final_res_batch: List[Optional[PoolingRequestOutput]]

182 + final_res_batch: list[Optional[PoolingRequestOutput]]

183 final_res_batch = [None] * num_prompts

184 try:

185 async for i, res in result_generator:

@@ -186,7 +187,7 @@ class OpenAIServingEmbedding(OpenAIServing):

187

188 assert all(final_res is not None for final_res in final_res_batch)

189

189 - final_res_batch_checked = cast(List[PoolingRequestOutput],

190 + final_res_batch_checked = cast(list[PoolingRequestOutput],

191 final_res_batch)

192

193 response = self.request_output_to_embedding_response(

@@ -206,13 +207,13 @@ class OpenAIServingEmbedding(OpenAIServing):

207

208 def request_output_to_embedding_response(

209 self,

209 - final_res_batch: List[PoolingRequestOutput],

210 + final_res_batch: list[PoolingRequestOutput],

211 request_id: str,

212 created_time: int,

213 model_name: str,

214 encoding_format: Literal["float", "base64"],

215 ) -> EmbeddingResponse:

215 - items: List[EmbeddingResponseData] = []

216 + items: list[EmbeddingResponseData] = []

217 num_prompt_tokens = 0

218

219 for idx, final_res in enumerate(final_res_batch):

vllm/entrypoints/openai/serving_engine.py

+21 -22

@@ -1,15 +1,14 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 +from collections.abc import Iterable, Iterator, Mapping, Sequence

5 from concurrent.futures.thread import ThreadPoolExecutor

6 from http import HTTPStatus

6 -from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,

7 - Optional, Sequence, Tuple, TypedDict, Union)

7 +from typing import Annotated, Any, Callable, Optional, TypedDict, Union

8

9 from fastapi import Request

10 from pydantic import Field

11 from starlette.datastructures import Headers

12 -from typing_extensions import Annotated

12

13 from vllm.config import ModelConfig

14 from vllm.engine.protocol import EngineClient

@@ -64,10 +63,10 @@ AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest,

63

64 class TextTokensPrompt(TypedDict):

65 prompt: str

67 - prompt_token_ids: List[int]

66 + prompt_token_ids: list[int]

67

68

70 -RequestPrompt = Union[List[int], str, TextTokensPrompt]

69 +RequestPrompt = Union[list[int], str, TextTokensPrompt]

70

71

72 class OpenAIServing:

@@ -144,7 +143,7 @@ class OpenAIServing:

143

144 def _maybe_get_adapters(

145 self, request: AnyRequest

147 - ) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[

146 + ) -> Union[tuple[None, None], tuple[LoRARequest, None], tuple[

147 None, PromptAdapterRequest]]:

148 if self._is_model_supported(request.model):

149 return None, None

@@ -188,7 +187,7 @@ class OpenAIServing:

187 self,

188 request: AnyRequest,

189 tokenizer: AnyTokenizer,

191 - prompt_ids: List[int],

190 + prompt_ids: list[int],

191 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],

192 ) -> TextTokensPrompt:

193 if truncate_prompt_tokens is None:

@@ -203,7 +202,7 @@ class OpenAIServing:

202 def _validate_input(

203 self,

204 request: AnyRequest,

206 - input_ids: List[int],

205 + input_ids: list[int],

206 input_text: str,

207 ) -> TextTokensPrompt:

208 token_num = len(input_ids)

@@ -259,7 +258,7 @@ class OpenAIServing:

258 self,

259 request: AnyRequest,

260 tokenizer: AnyTokenizer,

262 - prompt_input: Union[str, List[int]],

261 + prompt_input: Union[str, list[int]],

262 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

263 add_special_tokens: bool = True,

264 ) -> TextTokensPrompt:

@@ -280,7 +279,7 @@ class OpenAIServing:

279 self,

280 request: AnyRequest,

281 tokenizer: AnyTokenizer,

283 - prompt_inputs: Iterable[Union[str, List[int]]],

282 + prompt_inputs: Iterable[Union[str, list[int]]],

283 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

284 add_special_tokens: bool = True,

285 ) -> Iterator[TextTokensPrompt]:

@@ -309,10 +308,10 @@ class OpenAIServing:

308 self,

309 request: AnyRequest,

310 tokenizer: AnyTokenizer,

312 - input_or_inputs: Union[str, List[str], List[int], List[List[int]]],

311 + input_or_inputs: Union[str, list[str], list[int], list[list[int]]],

312 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

313 add_special_tokens: bool = True,

315 - ) -> List[TextTokensPrompt]:

314 + ) -> list[TextTokensPrompt]:

315 """

316 Tokenize/detokenize depending on the input format.

317

@@ -344,10 +343,10 @@ class OpenAIServing:

343 self,

344 request: CompletionLikeRequest,

345 tokenizer: AnyTokenizer,

347 - input_or_inputs: Union[str, List[str], List[int], List[List[int]]],

346 + input_or_inputs: Union[str, list[str], list[int], list[list[int]]],

347 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

348 add_special_tokens: bool = True,

350 - ) -> Tuple[List[TextTokensPrompt], List[TokensPrompt]]:

349 + ) -> tuple[list[TextTokensPrompt], list[TokensPrompt]]:

350 request_prompts = await self._tokenize_prompt_input_or_inputs_async(

351 request,

352 tokenizer,

@@ -367,19 +366,19 @@ class OpenAIServing:

366 self,

367 request: ChatLikeRequest,

368 tokenizer: AnyTokenizer,

370 - messages: List[ChatCompletionMessageParam],

369 + messages: list[ChatCompletionMessageParam],

370 chat_template: Optional[str],

371 chat_template_content_format: ChatTemplateContentFormatOption,

372 add_generation_prompt: bool = True,

373 continue_final_message: bool = False,

375 - tool_dicts: Optional[List[Dict[str, Any]]] = None,

376 - documents: Optional[List[Dict[str, str]]] = None,

377 - chat_template_kwargs: Optional[Dict[str, Any]] = None,

374 + tool_dicts: Optional[list[dict[str, Any]]] = None,

375 + documents: Optional[list[dict[str, str]]] = None,

376 + chat_template_kwargs: Optional[dict[str, Any]] = None,

377 tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,

378 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

379 add_special_tokens: bool = False,

381 - ) -> Tuple[List[ConversationMessage], Sequence[RequestPrompt],

382 - List[TokensPrompt]]:

380 + ) -> tuple[list[ConversationMessage], Sequence[RequestPrompt],

381 + list[TokensPrompt]]:

382 resolved_content_format = resolve_chat_template_content_format(

383 chat_template,

384 chat_template_content_format,

@@ -392,7 +391,7 @@ class OpenAIServing:

391 content_format=resolved_content_format,

392 )

393

395 - _chat_template_kwargs: Dict[str, Any] = dict(

394 + _chat_template_kwargs: dict[str, Any] = dict(

395 chat_template=chat_template,

396 add_generation_prompt=add_generation_prompt,

397 continue_final_message=continue_final_message,

@@ -401,7 +400,7 @@ class OpenAIServing:

400 )

401 _chat_template_kwargs.update(chat_template_kwargs or {})

402

404 - request_prompt: Union[str, List[int]]

403 + request_prompt: Union[str, list[int]]

404 if isinstance(tokenizer, MistralTokenizer):

405 request_prompt = apply_mistral_chat_template(

406 tokenizer,

vllm/entrypoints/openai/serving_models.py

+5 -5

@@ -4,7 +4,7 @@ import json

4 import pathlib

5 from dataclasses import dataclass

6 from http import HTTPStatus

7 -from typing import List, Optional, Union

7 +from typing import Optional, Union

8

9 from vllm.config import ModelConfig

10 from vllm.engine.protocol import EngineClient

@@ -53,10 +53,10 @@ class OpenAIServingModels:

53 self,

54 engine_client: EngineClient,

55 model_config: ModelConfig,

56 - base_model_paths: List[BaseModelPath],

56 + base_model_paths: list[BaseModelPath],

57 *,

58 - lora_modules: Optional[List[LoRAModulePath]] = None,

59 - prompt_adapters: Optional[List[PromptAdapterPath]] = None,

58 + lora_modules: Optional[list[LoRAModulePath]] = None,

59 + prompt_adapters: Optional[list[PromptAdapterPath]] = None,

60 ):

61 super().__init__()

62

@@ -65,7 +65,7 @@ class OpenAIServingModels:

65 self.engine_client = engine_client

66

67 self.static_lora_modules = lora_modules

68 - self.lora_requests: List[LoRARequest] = []

68 + self.lora_requests: list[LoRARequest] = []

69 self.lora_id_counter = AtomicCounter(0)

70

71 self.prompt_adapter_requests = []

vllm/entrypoints/openai/serving_pooling.py

+8 -7

@@ -3,7 +3,8 @@

3 import asyncio

4 import base64

5 import time

6 -from typing import AsyncGenerator, Final, List, Literal, Optional, Union, cast

6 +from collections.abc import AsyncGenerator

7 +from typing import Final, Literal, Optional, Union, cast

8

9 import numpy as np

10 from fastapi import Request

@@ -29,7 +30,7 @@ logger = init_logger(__name__)

30 def _get_data(

31 output: PoolingOutput,

32 encoding_format: Literal["float", "base64"],

32 -) -> Union[List[float], str]:

33 +) -> Union[list[float], str]:

34 if encoding_format == "float":

35 return output.data.tolist()

36 elif encoding_format == "base64":

@@ -139,7 +140,7 @@ class OpenAIServingPooling(OpenAIServing):

140 return self.create_error_response(str(e))

141

142 # Schedule the request and get the result generator.

142 - generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []

143 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

144 try:

145 pooling_params = request.to_pooling_params()

146

@@ -174,7 +175,7 @@ class OpenAIServingPooling(OpenAIServing):

175 num_prompts = len(engine_prompts)

176

177 # Non-streaming response

177 - final_res_batch: List[Optional[PoolingRequestOutput]]

178 + final_res_batch: list[Optional[PoolingRequestOutput]]

179 final_res_batch = [None] * num_prompts

180 try:

181 async for i, res in result_generator:

@@ -182,7 +183,7 @@ class OpenAIServingPooling(OpenAIServing):

183

184 assert all(final_res is not None for final_res in final_res_batch)

185

185 - final_res_batch_checked = cast(List[PoolingRequestOutput],

186 + final_res_batch_checked = cast(list[PoolingRequestOutput],

187 final_res_batch)

188

189 response = self.request_output_to_pooling_response(

@@ -202,13 +203,13 @@ class OpenAIServingPooling(OpenAIServing):

203

204 def request_output_to_pooling_response(

205 self,

205 - final_res_batch: List[PoolingRequestOutput],

206 + final_res_batch: list[PoolingRequestOutput],

207 request_id: str,

208 created_time: int,

209 model_name: str,

210 encoding_format: Literal["float", "base64"],

211 ) -> PoolingResponse:

211 - items: List[PoolingResponseData] = []

212 + items: list[PoolingResponseData] = []

213 num_prompt_tokens = 0

214

215 for idx, final_res in enumerate(final_res_batch):

vllm/entrypoints/openai/serving_score.py

+25 -24

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 import asyncio

3 import time

4 -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Union

4 +from collections.abc import AsyncGenerator, Mapping

5 +from typing import Any, Optional, Union

6

7 from fastapi import Request

8

@@ -48,8 +49,8 @@ class ServingScores(OpenAIServing):

49 async def _embedding_score(

50 self,

51 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],

51 - texts_1: List[str],

52 - texts_2: List[str],

52 + texts_1: list[str],

53 + texts_2: list[str],

54 request: Union[RerankRequest, ScoreRequest],

55 request_id=str,

56 tokenization_kwargs: Optional[dict[str, Any]] = None,

@@ -57,11 +58,11 @@ class ServingScores(OpenAIServing):

58 prompt_adapter_request: Optional[Union[PromptAdapterRequest,

59 None]] = None,

60 trace_headers: Optional[Mapping[str, str]] = None,

60 - ) -> List[PoolingRequestOutput]:

61 + ) -> list[PoolingRequestOutput]:

62

63 input_texts = texts_1 + texts_2

64

64 - engine_prompts: List[TokensPrompt] = []

65 + engine_prompts: list[TokensPrompt] = []

66 tokenize_async = make_async(tokenizer.__call__,

67 executor=self._tokenizer_executor)

68

@@ -82,7 +83,7 @@ class ServingScores(OpenAIServing):

83 prompt_token_ids=text_token_prompt["prompt_token_ids"]))

84

85 # Schedule the request and get the result generator.

85 - generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []

86 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

87 pooling_params = request.to_pooling_params()

88

89 for i, engine_prompt in enumerate(engine_prompts):

@@ -108,16 +109,16 @@ class ServingScores(OpenAIServing):

109 result_generator = merge_async_iterators(*generators)

110

111 # Non-streaming response

111 - final_res_batch: List[PoolingRequestOutput] = []

112 + final_res_batch: list[PoolingRequestOutput] = []

113

113 - embeddings: List[Optional[PoolingRequestOutput]] =\

114 + embeddings: list[Optional[PoolingRequestOutput]] =\

115 [None] * len(engine_prompts)

116

117 async for i, res in result_generator:

118 embeddings[i] = res

119

119 - emb_texts_1: List[PoolingRequestOutput] = []

120 - emb_texts_2: List[PoolingRequestOutput] = []

120 + emb_texts_1: list[PoolingRequestOutput] = []

121 + emb_texts_2: list[PoolingRequestOutput] = []

122

123 for i in range(0, len(texts_1)):

124 assert (emb := embeddings[i]) is not None

@@ -139,8 +140,8 @@ class ServingScores(OpenAIServing):

140 async def _cross_encoding_score(

141 self,

142 tokenizer: Union[AnyTokenizer],

142 - texts_1: List[str],

143 - texts_2: List[str],

143 + texts_1: list[str],

144 + texts_2: list[str],

145 request: Union[RerankRequest, ScoreRequest],

146 request_id=str,

147 tokenization_kwargs: Optional[dict[str, Any]] = None,

@@ -148,10 +149,10 @@ class ServingScores(OpenAIServing):

149 prompt_adapter_request: Optional[Union[PromptAdapterRequest,

150 None]] = None,

151 trace_headers: Optional[Mapping[str, str]] = None,

151 - ) -> List[PoolingRequestOutput]:

152 + ) -> list[PoolingRequestOutput]:

153

153 - request_prompts: List[str] = []

154 - engine_prompts: List[TokensPrompt] = []

154 + request_prompts: list[str] = []

155 + engine_prompts: list[TokensPrompt] = []

156

157 if len(texts_1) == 1:

158 texts_1 = texts_1 * len(texts_2)

@@ -185,7 +186,7 @@ class ServingScores(OpenAIServing):

186 engine_prompts.append(engine_prompt)

187

188 # Schedule the request and get the result generator.

188 - generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []

189 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []

190

191 pooling_params = request.to_pooling_params()

192

@@ -212,7 +213,7 @@ class ServingScores(OpenAIServing):

213 result_generator = merge_async_iterators(*generators)

214

215 # Non-streaming response

215 - final_res_batch: List[

216 + final_res_batch: list[

217 Optional[PoolingRequestOutput]] = [None] * len(engine_prompts)

218

219 async for i, res in result_generator:

@@ -228,9 +229,9 @@ class ServingScores(OpenAIServing):

229 request_id: str,

230 raw_request: Optional[Request] = None,

231 truncate_prompt_tokens: Optional[int] = None,

231 - ) -> List[PoolingRequestOutput]:

232 + ) -> list[PoolingRequestOutput]:

233

233 - tokenization_kwargs: Dict[str, Any] = {}

234 + tokenization_kwargs: dict[str, Any] = {}

235 if truncate_prompt_tokens is not None:

236 tokenization_kwargs["truncation"] = True

237 tokenization_kwargs["max_length"] = truncate_prompt_tokens

@@ -372,12 +373,12 @@ class ServingScores(OpenAIServing):

373

374 def request_output_to_score_response(

375 self,

375 - final_res_batch: List[PoolingRequestOutput],

376 + final_res_batch: list[PoolingRequestOutput],

377 request_id: str,

378 created_time: int,

379 model_name: str,

380 ) -> ScoreResponse:

380 - items: List[ScoreResponseData] = []

381 + items: list[ScoreResponseData] = []

382 num_prompt_tokens = 0

383

384 for idx, final_res in enumerate(final_res_batch):

@@ -406,13 +407,13 @@ class ServingScores(OpenAIServing):

407 )

408

409 def request_output_to_rerank_response(

409 - self, final_res_batch: List[PoolingRequestOutput], request_id: str,

410 - model_name: str, documents: List[str],

410 + self, final_res_batch: list[PoolingRequestOutput], request_id: str,

411 + model_name: str, documents: list[str],

412 top_n: int) -> RerankResponse:

413 """

414 Convert the output of do_rank to a RerankResponse

415 """

415 - results: List[RerankResult] = []

416 + results: list[RerankResult] = []

417 num_prompt_tokens = 0

418 for idx, final_res in enumerate(final_res_batch):

419 classify_res = ScoringRequestOutput.from_base(final_res)

vllm/entrypoints/openai/serving_tokenization.py

+2 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Final, List, Optional, Union

3 +from typing import Final, Optional, Union

4

5 from fastapi import Request

6

@@ -92,7 +92,7 @@ class OpenAIServingTokenization(OpenAIServing):

92 logger.exception("Error in preprocessing prompt inputs")

93 return self.create_error_response(str(e))

94

95 - input_ids: List[int] = []

95 + input_ids: list[int] = []

96 for i, engine_prompt in enumerate(engine_prompts):

97 self._log_inputs(request_id,

98 request_prompts[i],

vllm/entrypoints/openai/serving_transcription.py

+2 -1

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 import asyncio

3 import io

4 -from typing import AsyncGenerator, Optional, Union, cast

4 +from collections.abc import AsyncGenerator

5 +from typing import Optional, Union, cast

6

7 from fastapi import Request

8

vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py

+11 -10

@@ -1,8 +1,9 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import os

4 +from collections.abc import Sequence

5 from functools import cached_property

5 -from typing import Callable, Dict, List, Optional, Sequence, Type, Union

6 +from typing import Callable, Optional, Union

7

8 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

9 DeltaMessage,

@@ -22,16 +23,16 @@ class ToolParser:

23 """

24

25 def __init__(self, tokenizer: AnyTokenizer):

25 - self.prev_tool_call_arr: List[Dict] = []

26 + self.prev_tool_call_arr: list[dict] = []

27 # the index of the tool call that is currently being parsed

28 self.current_tool_id: int = -1

29 self.current_tool_name_sent: bool = False

29 - self.streamed_args_for_tool: List[str] = []

30 + self.streamed_args_for_tool: list[str] = []

31

32 self.model_tokenizer = tokenizer

33

34 @cached_property

34 - def vocab(self) -> Dict[str, int]:

35 + def vocab(self) -> dict[str, int]:

36 # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab

37 # whereas all tokenizers have .get_vocab()

38 return self.model_tokenizer.get_vocab()

@@ -79,10 +80,10 @@ class ToolParser:

80

81

82 class ToolParserManager:

82 - tool_parsers: Dict[str, Type] = {}

83 + tool_parsers: dict[str, type] = {}

84

85 @classmethod

85 - def get_tool_parser(cls, name) -> Type:

86 + def get_tool_parser(cls, name) -> type:

87 """

88 Get tool parser by name which is registered by `register_module`.

89

@@ -95,8 +96,8 @@ class ToolParserManager:

96

97 @classmethod

98 def _register_module(cls,

98 - module: Type,

99 - module_name: Optional[Union[str, List[str]]] = None,

99 + module: type,

100 + module_name: Optional[Union[str, list[str]]] = None,

101 force: bool = True) -> None:

102 if not issubclass(module, ToolParser):

103 raise TypeError(

@@ -116,9 +117,9 @@ class ToolParserManager:

117 @classmethod

118 def register_module(

119 cls,

119 - name: Optional[Union[str, List[str]]] = None,

120 + name: Optional[Union[str, list[str]]] = None,

121 force: bool = True,

121 - module: Union[Type, None] = None) -> Union[type, Callable]:

122 + module: Union[type, None] = None) -> Union[type, Callable]:

123 """

124 Register module with the given name or name list. it can be used as a

125 decoder(with module as None) or normal function(with module as not

vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py

+3 -2

@@ -2,8 +2,9 @@

2

3 import json

4 import re

5 +from collections.abc import Sequence

6 from json import JSONDecoder

6 -from typing import Dict, Sequence, Union

7 +from typing import Union

8

9 import partial_json_parser

10 from partial_json_parser.core.options import Allow

@@ -145,7 +146,7 @@ class Granite20bFCToolParser(ToolParser):

146 return None

147

148 # select as the current tool call the one we're on the state at

148 - current_tool_call: Dict = tool_call_arr[self.current_tool_id] \

149 + current_tool_call: dict = tool_call_arr[self.current_tool_id] \

150 if len(tool_call_arr) > 0 else {}

151

152 # case -- if no tokens have been streamed for the tool, e.g.

vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py

+3 -2

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 -from typing import Dict, Sequence, Union

4 +from collections.abc import Sequence

5 +from typing import Union

6

7 import partial_json_parser

8 from partial_json_parser.core.options import Allow

@@ -136,7 +137,7 @@ class GraniteToolParser(ToolParser):

137 return None

138

139 # select as the current tool call the one we're on the state at

139 - current_tool_call: Dict = tool_call_arr[self.current_tool_id]

140 + current_tool_call: dict = tool_call_arr[self.current_tool_id]

141

142 delta = None

143 # case: we are starting a new tool in the array

vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py

+4 -3

@@ -2,7 +2,8 @@

2

3 import json

4 import re

5 -from typing import Dict, List, Sequence, Union

5 +from collections.abc import Sequence

6 +from typing import Union

7

8 import partial_json_parser

9 from partial_json_parser.core.options import Allow

@@ -33,9 +34,9 @@ class Hermes2ProToolParser(ToolParser):

34 self.model_tokenizer = self.model_tokenizer.tokenizer

35

36 self.current_tool_name_sent: bool = False

36 - self.prev_tool_call_arr: List[Dict] = []

37 + self.prev_tool_call_arr: list[dict] = []

38 self.current_tool_id: int = -1

38 - self.streamed_args_for_tool: List[str] = [

39 + self.streamed_args_for_tool: list[str] = [

40 ] # map what has been streamed for each tool so far to a list

41

42 self.tool_call_start_token: str = "<tool_call>"

vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py

+3 -2

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import json

4 -from typing import Dict, Sequence, Union

4 +from collections.abc import Sequence

5 +from typing import Union

6

7 import partial_json_parser

8 from partial_json_parser.core.options import Allow

@@ -90,7 +91,7 @@ class Internlm2ToolParser(ToolParser):

91 # tool calls are generated in an object in inernlm2

92 # it's not support parallel tool calls

93 try:

93 - tool_call_arr: Dict = partial_json_parser.loads(

94 + tool_call_arr: dict = partial_json_parser.loads(

95 parsable_arr, flags)

96 except partial_json_parser.core.exceptions.MalformedJSON:

97 logger.debug('not enough tokens to parse into JSON yet')

vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py

+6 -5

@@ -2,7 +2,8 @@

2

3 import json

4 import re

5 -from typing import Dict, List, Sequence, Union

5 +from collections.abc import Sequence

6 +from typing import Union

7

8 import partial_json_parser

9 from partial_json_parser.core.options import Allow

@@ -35,9 +36,9 @@ class JambaToolParser(ToolParser):

36 )

37

38 self.current_tool_name_sent: bool = False

38 - self.prev_tool_call_arr: List[Dict] = []

39 + self.prev_tool_call_arr: list[dict] = []

40 self.current_tool_id: int = -1

40 - self.streamed_args_for_tool: List[str] = [

41 + self.streamed_args_for_tool: list[str] = [

42 ] # map what has been streamed for each tool so far to a list

43

44 self.tool_calls_start_token: str = "<tool_calls>"

@@ -157,7 +158,7 @@ class JambaToolParser(ToolParser):

158 # tool calls are generated in an array, so do partial JSON

159 # parsing on the entire array

160 try:

160 - tool_call_arr: List[Dict] = partial_json_parser.loads(

161 + tool_call_arr: list[dict] = partial_json_parser.loads(

162 parsable_arr, flags)

163 except partial_json_parser.core.exceptions.MalformedJSON:

164 logger.debug('not enough tokens to parse into JSON yet')

@@ -165,7 +166,7 @@ class JambaToolParser(ToolParser):

166

167 # select as the current tool call the one we're on the state at

168

168 - current_tool_call: Dict = tool_call_arr[self.current_tool_id] \

169 + current_tool_call: dict = tool_call_arr[self.current_tool_id] \

170 if len(tool_call_arr) > 0 else {}

171

172 # case -- if no tokens have been streamed for the tool, e.g.

vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py

+6 -5

@@ -2,8 +2,9 @@

2

3 import json

4 import re

5 +from collections.abc import Sequence

6 from json import JSONDecoder

6 -from typing import Dict, List, Sequence, Union

7 +from typing import Union

8

9 import partial_json_parser

10 from partial_json_parser.core.options import Allow

@@ -40,10 +41,10 @@ class Llama3JsonToolParser(ToolParser):

41

42 # initialize properties used for state when parsing tool calls in

43 # streaming mode

43 - self.prev_tool_call_arr: List[Dict] = []

44 + self.prev_tool_call_arr: list[dict] = []

45 self.current_tool_id: int = -1

46 self.current_tool_name_sent: bool = False

46 - self.streamed_args_for_tool: List[str] = [

47 + self.streamed_args_for_tool: list[str] = [

48 ] # map what has been streamed for each tool so far to a list

49 self.bot_token = "<|python_tag|>"

50 self.bot_token_id = tokenizer.encode(self.bot_token,

@@ -78,7 +79,7 @@ class Llama3JsonToolParser(ToolParser):

79 start_idx += end_idx + len('; ')

80 function_call_arr.append(obj)

81

81 - tool_calls: List[ToolCall] = [

82 + tool_calls: list[ToolCall] = [

83 ToolCall(

84 type="function",

85 function=FunctionCall(

@@ -152,7 +153,7 @@ class Llama3JsonToolParser(ToolParser):

153 return None

154

155 # select as the current tool call the one we're on the state at

155 - current_tool_call: Dict = tool_call_arr[self.current_tool_id] \

156 + current_tool_call: dict = tool_call_arr[self.current_tool_id] \

157 if len(tool_call_arr) > 0 else {}

158

159 # case -- if no tokens have been streamed for the tool, e.g.

vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py

+7 -6

@@ -2,9 +2,10 @@

2

3 import json

4 import re

5 +from collections.abc import Sequence

6 from random import choices

7 from string import ascii_letters, digits

7 -from typing import Dict, List, Sequence, Union

8 +from typing import Union

9

10 import partial_json_parser

11 from partial_json_parser.core.options import Allow

@@ -56,10 +57,10 @@ class MistralToolParser(ToolParser):

57

58 # initialize properties used for state when parsing tool calls in

59 # streaming mode

59 - self.prev_tool_call_arr: List[Dict] = []

60 + self.prev_tool_call_arr: list[dict] = []

61 self.current_tool_id: int = -1

62 self.current_tool_name_sent: bool = False

62 - self.streamed_args_for_tool: List[str] = [

63 + self.streamed_args_for_tool: list[str] = [

64 ] # map what has been streamed for each tool so far to a list

65 self.bot_token = "[TOOL_CALLS]"

66 self.bot_token_id = self.vocab.get(self.bot_token)

@@ -104,7 +105,7 @@ class MistralToolParser(ToolParser):

105 function_call_arr = json.loads(raw_tool_call)

106

107 # Tool Call

107 - tool_calls: List[MistralToolCall] = [

108 + tool_calls: list[MistralToolCall] = [

109 MistralToolCall(

110 type="function",

111 function=FunctionCall(

@@ -172,7 +173,7 @@ class MistralToolParser(ToolParser):

173 # tool calls are generated in an array, so do partial JSON

174 # parsing on the entire array

175 try:

175 - tool_call_arr: List[Dict] = partial_json_parser.loads(

176 + tool_call_arr: list[dict] = partial_json_parser.loads(

177 parsable_arr, flags)

178 except partial_json_parser.core.exceptions.MalformedJSON:

179 logger.debug('not enough tokens to parse into JSON yet')

@@ -180,7 +181,7 @@ class MistralToolParser(ToolParser):

181

182 # select as the current tool call the one we're on the state at

183

183 - current_tool_call: Dict = tool_call_arr[self.current_tool_id] \

184 + current_tool_call: dict = tool_call_arr[self.current_tool_id] \

185 if len(tool_call_arr) > 0 else {}

186

187 # case -- if no tokens have been streamed for the tool, e.g.

vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py

+3 -2

@@ -3,7 +3,8 @@

3 import ast

4 import json

5 import re

6 -from typing import Any, Sequence, Tuple, Union

6 +from collections.abc import Sequence

7 +from typing import Any, Union

8

9 from transformers import PreTrainedTokenizerBase

10

@@ -204,7 +205,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:

205 arguments=json.dumps(arguments)))

206

207

207 -def _make_valid_python(text: str) -> Union[Tuple[str, str], None]:

208 +def _make_valid_python(text: str) -> Union[tuple[str, str], None]:

209 bracket_stack = []

210 for index, char in enumerate(text):

211 if char in {"[", "(", "{"}:

vllm/entrypoints/openai/tool_parsers/utils.py

+3 -3

@@ -2,7 +2,7 @@

2

3 import json

4 from json import JSONDecodeError, JSONDecoder

5 -from typing import Any, List, Tuple

5 +from typing import Any

6

7 import partial_json_parser

8 from partial_json_parser.core.options import Allow

@@ -82,7 +82,7 @@ def extract_intermediate_diff(curr: str, old: str) -> str:

82 return diff

83

84

85 -def find_all_indices(string: str, substring: str) -> List[int]:

85 +def find_all_indices(string: str, substring: str) -> list[int]:

86 """

87 Find all (starting) indices of a substring in a given string. Useful for

88 tool call extraction

@@ -99,7 +99,7 @@ def find_all_indices(string: str, substring: str) -> List[int]:

99

100 # partial_json_parser doesn't support extra data and

101 # JSONDecorder.raw_decode doesn't support partial JSON

102 -def partial_json_loads(input_str: str, flags: Allow) -> Tuple[Any, int]:

102 +def partial_json_loads(input_str: str, flags: Allow) -> tuple[Any, int]:

103 try:

104 return (partial_json_parser.loads(input_str, flags), len(input_str))

105 except JSONDecodeError as e:

vllm/entrypoints/score_utils.py

+7 -7

@@ -1,5 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 -from typing import List, Union

2 +from typing import Union

3

4 from torch.nn import CosineSimilarity

5

@@ -10,12 +10,12 @@ from vllm.transformers_utils.tokenizer import (PreTrainedTokenizer,

10

11 def _cosine_similarity(

12 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],

13 - embed_1: List[PoolingRequestOutput],

14 - embed_2: List[PoolingRequestOutput],

15 -) -> List[PoolingRequestOutput]:

13 + embed_1: list[PoolingRequestOutput],

14 + embed_2: list[PoolingRequestOutput],

15 +) -> list[PoolingRequestOutput]:

16

17 scorer = CosineSimilarity(0)

18 - scores: Union[List[PoolingRequestOutput]] = []

18 + scores: Union[list[PoolingRequestOutput]] = []

19

20 for emb_1, emb_2 in zip(embed_1, embed_2):

21 pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data)

@@ -38,8 +38,8 @@ def _cosine_similarity(

38

39

40 def _validate_score_input_lens(

41 - texts_1: Union[List[str], List[dict]],

42 - texts_2: Union[List[str], List[dict]],

41 + texts_1: Union[list[str], list[dict]],

42 + texts_2: Union[list[str], list[dict]],

43 ):

44 if len(texts_1) > 1 and len(texts_1) != len(texts_2):

45 raise ValueError("Input lengths must be either 1:1, 1:N or N:N")

vllm/envs.py

+4 -4

@@ -2,7 +2,7 @@

2

3 import os

4 import tempfile

5 -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional

5 +from typing import TYPE_CHECKING, Any, Callable, Optional

6

7 if TYPE_CHECKING:

8 VLLM_HOST_IP: str = ""

@@ -67,12 +67,12 @@ if TYPE_CHECKING:

67 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False

68 VLLM_TEST_FORCE_FP8_MARLIN: bool = False

69 VLLM_RPC_TIMEOUT: int = 10000 # ms

70 - VLLM_PLUGINS: Optional[List[str]] = None

70 + VLLM_PLUGINS: Optional[list[str]] = None

71 VLLM_TORCH_PROFILER_DIR: Optional[str] = None

72 VLLM_USE_TRITON_AWQ: bool = False

73 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False

74 VLLM_SKIP_P2P_CHECK: bool = False

75 - VLLM_DISABLED_KERNELS: List[str] = []

75 + VLLM_DISABLED_KERNELS: list[str] = []

76 VLLM_USE_V1: bool = False

77 VLLM_ROCM_FP8_PADDING: bool = True

78 VLLM_ENABLE_V1_MULTIPROCESSING: bool = True

@@ -123,7 +123,7 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]:

123

124 # begin-env-vars-definition

125

126 -environment_variables: Dict[str, Callable[[], Any]] = {

126 +environment_variables: dict[str, Callable[[], Any]] = {

127

128 # ================== Installation Time Env Vars ==================

129

vllm/forward_context.py

+3 -3

@@ -4,7 +4,7 @@ import time

4 from collections import defaultdict

5 from contextlib import contextmanager

6 from dataclasses import dataclass

7 -from typing import TYPE_CHECKING, Any, Dict, List, Optional

7 +from typing import TYPE_CHECKING, Any, Optional

8

9 import torch

10 import torch.distributed as dist

@@ -28,13 +28,13 @@ batchsize_forward_time: defaultdict = defaultdict(list)

28 @dataclass

29 class ForwardContext:

30 # copy from vllm_config.compilation_config.static_forward_context

31 - attn_layers: Dict[str, Any]

31 + attn_layers: dict[str, Any]

32 # TODO: extend to support per-layer dynamic forward context

33 attn_metadata: "AttentionMetadata" # set dynamically for each forward pass

34 # TODO: remove after making all virtual_engines share the same kv cache

35 virtual_engine: int # set dynamically for each forward pass

36 num_tokens_across_dp: Optional[

37 - List[int]] = None # set dynamically for each forward pass

37 + list[int]] = None # set dynamically for each forward pass

38

39

40 _forward_context: Optional[ForwardContext] = None

vllm/logger.py

+1 -1

@@ -109,7 +109,7 @@ def _configure_vllm_root_logger() -> None:

109 custom_config = json.loads(file.read())

110

111 if not isinstance(custom_config, dict):

112 - raise ValueError("Invalid logging config. Expected Dict, got %s.",

112 + raise ValueError("Invalid logging config. Expected dict, got %s.",

113 type(custom_config).__name__)

114 logging_config = custom_config

115

vllm/logits_process.py

+8 -8

@@ -1,13 +1,13 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Callable, List, Tuple, Union

3 +from typing import Callable, Union

4

5 import torch

6

7 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

8

9 -LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],

10 - Callable[[List[int], List[int], torch.Tensor],

9 +LogitsProcessor = Union[Callable[[list[int], torch.Tensor], torch.Tensor],

10 + Callable[[list[int], list[int], torch.Tensor],

11 torch.Tensor]]

12 """LogitsProcessor is a function that takes a list

13 of previously generated tokens, the logits tensor

@@ -17,9 +17,9 @@ to sample from."""

17

18

19 def get_bad_words_logits_processors(

20 - bad_words: List[str],

21 - tokenizer: AnyTokenizer) -> List[LogitsProcessor]:

22 - bad_words_ids: List[List[int]] = list()

20 + bad_words: list[str],

21 + tokenizer: AnyTokenizer) -> list[LogitsProcessor]:

22 + bad_words_ids: list[list[int]] = list()

23

24 for bad_word in bad_words:

25 # To prohibit words both at the beginning

@@ -51,13 +51,13 @@ class NoBadWordsLogitsProcessor:

51 _SMALLEST_LOGIT = float("-inf")

52 _NEUTRAL_LOGIT = 0.0

53

54 - def __init__(self, bad_words_ids: List[List[int]]):

54 + def __init__(self, bad_words_ids: list[list[int]]):

55 self.bad_words_ids = bad_words_ids

56 self.word_bias: torch.FloatTensor = None

57

58 def __call__(

59 self,

60 - past_tokens_ids: Union[List[int], Tuple[int]],

60 + past_tokens_ids: Union[list[int], tuple[int]],

61 logits: torch.FloatTensor,

62 ) -> torch.Tensor:

63 if self.word_bias is None:

vllm/outputs.py

+12 -12

@@ -1,10 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import time

4 +from collections.abc import MutableSequence

5 +from collections.abc import Sequence as GenericSequence

6 from dataclasses import dataclass

5 -from typing import Dict, Generic, List, MutableSequence, Optional

6 -from typing import Sequence as GenericSequence

7 -from typing import Union

7 +from typing import Generic, Optional, Union

8

9 import torch

10 from typing_extensions import TypeVar, deprecated

@@ -109,14 +109,14 @@ class RequestOutput:

109 self,

110 request_id: str,

111 prompt: Optional[str],

112 - prompt_token_ids: Optional[List[int]],

112 + prompt_token_ids: Optional[list[int]],

113 prompt_logprobs: Optional[PromptLogprobs],

114 - outputs: List[CompletionOutput],

114 + outputs: list[CompletionOutput],

115 finished: bool,

116 metrics: Optional[RequestMetrics] = None,

117 lora_request: Optional[LoRARequest] = None,

118 encoder_prompt: Optional[str] = None,

119 - encoder_prompt_token_ids: Optional[List[int]] = None,

119 + encoder_prompt_token_ids: Optional[list[int]] = None,

120 num_cached_tokens: Optional[int] = None,

121 *,

122 multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None,

@@ -139,9 +139,9 @@ class RequestOutput:

139 cls,

140 request_id: str,

141 prompt: Optional[str],

142 - prompt_token_ids: Optional[List[int]],

142 + prompt_token_ids: Optional[list[int]],

143 text: str,

144 - token_ids: List[int],

144 + token_ids: list[int],

145 logprobs: Optional[SampleLogprobs],

146 prompt_logprobs: Optional[PromptLogprobs],

147 cumulative_logprob: Optional[float],

@@ -189,7 +189,7 @@ class RequestOutput:

189 @classmethod

190 def from_seq_group(

191 cls, seq_group: SequenceGroup, use_cache: bool,

192 - seq_id_to_seq_group: Dict[str, SequenceGroupBase]

192 + seq_id_to_seq_group: dict[str, SequenceGroupBase]

193 ) -> Optional["RequestOutput"]:

194 finished = seq_group.is_finished()

195

@@ -363,12 +363,12 @@ class PoolingRequestOutput(Generic[_O]):

363 Args:

364 request_id (str): A unique identifier for the pooling request.

365 outputs (PoolingOutput): The pooling results for the given input.

366 - prompt_token_ids (List[int]): A list of token IDs used in the prompt.

366 + prompt_token_ids (list[int]): A list of token IDs used in the prompt.

367 finished (bool): A flag indicating whether the pooling is completed.

368 """

369

370 def __init__(self, request_id: str, outputs: _O,

371 - prompt_token_ids: List[int], finished: bool):

371 + prompt_token_ids: list[int], finished: bool):

372 self.request_id = request_id

373 self.prompt_token_ids = prompt_token_ids

374 self.finished = finished

@@ -407,7 +407,7 @@ class RequestOutputFactory:

407

408 @staticmethod

409 def create(seq_group: SequenceGroup,

410 - seq_id_to_seq_group: Dict[str, SequenceGroupBase],

410 + seq_id_to_seq_group: dict[str, SequenceGroupBase],

411 use_cache: bool = False):

412 if seq_group.pooled_data is not None:

413 return PoolingRequestOutput.from_seq_group(seq_group)

vllm/sampling_params.py

+26 -27

@@ -4,11 +4,10 @@ import copy

4 from dataclasses import dataclass

5 from enum import Enum, IntEnum

6 from functools import cached_property

7 -from typing import Any, Dict, List, Optional, Set, Union

7 +from typing import Annotated, Any, Optional, Union

8

9 import msgspec

10 from pydantic import BaseModel

11 -from typing_extensions import Annotated

11

12 from vllm.logger import init_logger

13 from vllm.logits_process import LogitsProcessor

@@ -29,9 +28,9 @@ class SamplingType(IntEnum):

28 @dataclass

29 class GuidedDecodingParams:

30 """One of these fields will be used to build a logit processor."""

32 - json: Optional[Union[str, Dict]] = None

31 + json: Optional[Union[str, dict]] = None

32 regex: Optional[str] = None

34 - choice: Optional[List[str]] = None

33 + choice: Optional[list[str]] = None

34 grammar: Optional[str] = None

35 json_object: Optional[bool] = None

36 """These are other options that can be set"""

@@ -40,9 +39,9 @@ class GuidedDecodingParams:

39

40 @staticmethod

41 def from_optional(

43 - json: Optional[Union[Dict, BaseModel, str]] = None,

42 + json: Optional[Union[dict, BaseModel, str]] = None,

43 regex: Optional[str] = None,

45 - choice: Optional[List[str]] = None,

44 + choice: Optional[list[str]] = None,

45 grammar: Optional[str] = None,

46 json_object: Optional[bool] = None,

47 backend: Optional[str] = None,

@@ -72,7 +71,7 @@ class GuidedDecodingParams:

71 """

72 return (self.backend or "").split(":")[0]

73

75 - def backend_options(self) -> List[str]:

74 + def backend_options(self) -> list[str]:

75 """Return the backend options as a list of strings."""

76 if not self.backend or ":" not in self.backend:

77 return []

@@ -144,12 +143,12 @@ class SamplingParams(

143 considered, relative to the probability of the most likely token.

144 Must be in [0, 1]. Set to 0 to disable this.

145 seed: Random seed to use for the generation.

147 - stop: List of strings that stop the generation when they are generated.

146 + stop: list of strings that stop the generation when they are generated.

147 The returned output will not contain the stop strings.

149 - stop_token_ids: List of tokens that stop the generation when they are

148 + stop_token_ids: list of tokens that stop the generation when they are

149 generated. The returned output will contain the stop tokens unless

150 the stop tokens are special tokens.

152 - bad_words: List of words that are not allowed to be generated.

151 + bad_words: list of words that are not allowed to be generated.

152 More precisely, only the last token of a corresponding

153 token sequence is not allowed when the next generated token

154 can complete the sequence.

@@ -172,7 +171,7 @@ class SamplingParams(

171 skip_special_tokens: Whether to skip special tokens in the output.

172 spaces_between_special_tokens: Whether to add spaces between special

173 tokens in the output. Defaults to True.

175 - logits_processors: List of functions that modify logits based on

174 + logits_processors: list of functions that modify logits based on

175 previously generated tokens, and optionally prompt tokens as

176 a first argument.

177 truncate_prompt_tokens: If set to an integer k, will use only the last k

@@ -198,9 +197,9 @@ class SamplingParams(

197 top_k: int = -1

198 min_p: float = 0.0

199 seed: Optional[int] = None

201 - stop: Optional[Union[str, List[str]]] = None

202 - stop_token_ids: Optional[List[int]] = None

203 - bad_words: Optional[List[str]] = None

200 + stop: Optional[Union[str, list[str]]] = None

201 + stop_token_ids: Optional[list[int]] = None

202 + bad_words: Optional[list[str]] = None

203 ignore_eos: bool = False

204 max_tokens: Optional[int] = 16

205 min_tokens: int = 0

@@ -212,8 +211,8 @@ class SamplingParams(

211 detokenize: bool = True

212 skip_special_tokens: bool = True

213 spaces_between_special_tokens: bool = True

215 - # Optional[List[LogitsProcessor]] type. We use Any here because

216 - # Optional[List[LogitsProcessor]] type is not supported by msgspec.

214 + # Optional[list[LogitsProcessor]] type. We use Any here because

215 + # Optional[list[LogitsProcessor]] type is not supported by msgspec.

216 logits_processors: Optional[Any] = None

217 include_stop_str_in_output: bool = False

218 truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None

@@ -222,12 +221,12 @@ class SamplingParams(

221 # The below fields are not supposed to be used as an input.

222 # They are set in post_init.

223 output_text_buffer_length: int = 0

225 - _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)

224 + _all_stop_token_ids: set[int] = msgspec.field(default_factory=set)

225

226 # Fields used to construct logits processors

227 guided_decoding: Optional[GuidedDecodingParams] = None

229 - logit_bias: Optional[Dict[int, float]] = None

230 - allowed_token_ids: Optional[List[int]] = None

228 + logit_bias: Optional[dict[int, float]] = None

229 + allowed_token_ids: Optional[list[int]] = None

230

231 @staticmethod

232 def from_optional(

@@ -241,9 +240,9 @@ class SamplingParams(

240 top_k: int = -1,

241 min_p: float = 0.0,

242 seed: Optional[int] = None,

244 - stop: Optional[Union[str, List[str]]] = None,

245 - stop_token_ids: Optional[List[int]] = None,

246 - bad_words: Optional[List[str]] = None,

243 + stop: Optional[Union[str, list[str]]] = None,

244 + stop_token_ids: Optional[list[int]] = None,

245 + bad_words: Optional[list[str]] = None,

246 include_stop_str_in_output: bool = False,

247 ignore_eos: bool = False,

248 max_tokens: Optional[int] = 16,

@@ -253,13 +252,13 @@ class SamplingParams(

252 detokenize: bool = True,

253 skip_special_tokens: bool = True,

254 spaces_between_special_tokens: bool = True,

256 - logits_processors: Optional[List[LogitsProcessor]] = None,

255 + logits_processors: Optional[list[LogitsProcessor]] = None,

256 truncate_prompt_tokens: Optional[Annotated[int,

257 msgspec.Meta(ge=1)]] = None,

258 output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,

259 guided_decoding: Optional[GuidedDecodingParams] = None,

261 - logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None,

262 - allowed_token_ids: Optional[List[int]] = None,

260 + logit_bias: Optional[Union[dict[int, float], dict[str, float]]] = None,

261 + allowed_token_ids: Optional[list[int]] = None,

262 ) -> "SamplingParams":

263 if logit_bias is not None:

264 # Convert token_id to integer

@@ -435,7 +434,7 @@ class SamplingParams(

434

435 def update_from_generation_config(

436 self,

438 - generation_config: Dict[str, Any],

437 + generation_config: dict[str, Any],

438 model_eos_token_id: Optional[int] = None) -> None:

439 """Update if there are non-default values from generation_config"""

440

@@ -468,7 +467,7 @@ class SamplingParams(

467 return SamplingType.RANDOM

468

469 @property

471 - def all_stop_token_ids(self) -> Set[int]:

470 + def all_stop_token_ids(self) -> set[int]:

471 return self._all_stop_token_ids

472

473 def clone(self) -> "SamplingParams":

vllm/sequence.py

+66 -66

@@ -5,11 +5,11 @@ import enum

5 from abc import ABC, abstractmethod

6 from array import array

7 from collections import defaultdict

8 +from collections.abc import Mapping

9 +from collections.abc import Sequence as GenericSequence

10 from dataclasses import dataclass, field

11 from functools import reduce

10 -from typing import Any, Callable, DefaultDict, Dict, List, Mapping, Optional

11 -from typing import Sequence as GenericSequence

12 -from typing import Set, Tuple, Union

12 +from typing import Any, Callable, Optional, Union

13

14 import msgspec

15 import torch

@@ -50,9 +50,9 @@ class Logprob:

50

51 # {token_id -> logprob} per each sequence group. None if the corresponding

52 # sequence group doesn't require prompt logprob.

53 -PromptLogprobs = List[Optional[Dict[int, Logprob]]]

53 +PromptLogprobs = list[Optional[dict[int, Logprob]]]

54 # {token_id -> logprob} for each sequence group.

55 -SampleLogprobs = List[Dict[int, Logprob]]

55 +SampleLogprobs = list[dict[int, Logprob]]

56

57

58 class SequenceStatus(enum.IntEnum):

@@ -129,7 +129,7 @@ class SequenceDataDelta(

129 omit_defaults=True): # type: ignore[call-arg]

130 """Delta SequenceData to send to workers per step."""

131 # A new token to be appended to existing SequenceData.

132 - new_output_token_ids: List[int]

132 + new_output_token_ids: list[int]

133 # Overwriting existing `cumulative_logprob`

134 new_cumulative_logprob: float

135 # Overwriting existing `num_computed_tokens`.

@@ -152,7 +152,7 @@ class SequenceData(msgspec.Struct,

152 output_token_ids: The token IDs of the output.

153 cumulative_logprob: The cumulative log probability of the output.

154 """

155 - # NOTE: we cannot use Union[List, array] because msgspec cannot support

155 + # NOTE: we cannot use Union[list, array] because msgspec cannot support

156 # union of 2 list types.

157 _prompt_token_ids: array

158 _output_token_ids: array = msgspec.field(

@@ -160,25 +160,25 @@ class SequenceData(msgspec.Struct,

160

161 ### The below fields should not be passed as an argument ###

162 _cumulative_logprob: float = 0.0

163 - _prompt_token_ids_tuple: Tuple[int,

163 + _prompt_token_ids_tuple: tuple[int,

164 ...] = msgspec.field(default_factory=tuple)

165 # The number of tokens that are computed (that run against the model).

166 _num_computed_tokens: int = 0

167 # The number of tokens with prefix cache hit.

168 _num_cached_tokens: int = 0

169 _stage: SequenceStage = SequenceStage.PREFILL

170 - _cached_all_token_ids: List[int] = msgspec.field(default_factory=list)

170 + _cached_all_token_ids: list[int] = msgspec.field(default_factory=list)

171

172 # It is used to get delta input. It is reset when `get_delta_and_reset`

173 # is called.

174 - _new_appended_tokens: List[int] = msgspec.field(default_factory=list)

174 + _new_appended_tokens: list[int] = msgspec.field(default_factory=list)

175

176 # It is used to compute mrope_position_ids.

177 _mrope_position_delta: Optional[int] = None

178

179 @staticmethod

180 def from_prompt_token_counts(

181 - *token_counts: Tuple[int, int]) -> "SequenceData":

181 + *token_counts: tuple[int, int]) -> "SequenceData":

182 """

183 Construct a :class:`SequenceData` instance by concatenating

184 prompt token sequences.

@@ -220,14 +220,14 @@ class SequenceData(msgspec.Struct,

220 def __post_init__(self) -> None:

221 assert self._prompt_token_ids.typecode == "l"

222 assert self._output_token_ids.typecode == "l"

223 - self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(

223 + self._prompt_token_ids_tuple: tuple[int, ...] = tuple(

224 self._prompt_token_ids)

225 self._update_cached_all_tokens()

226

227 def _update_cached_all_tokens(self):

228 assert isinstance(self._prompt_token_ids, array)

229 assert isinstance(self._output_token_ids, array)

230 - self._cached_all_token_ids: List[int] = list(self._prompt_token_ids +

230 + self._cached_all_token_ids: list[int] = list(self._prompt_token_ids +

231 self._output_token_ids)

232

233 @property

@@ -235,7 +235,7 @@ class SequenceData(msgspec.Struct,

235 return self._cumulative_logprob

236

237 @property

238 - def prompt_token_ids(self) -> Tuple[int, ...]:

238 + def prompt_token_ids(self) -> tuple[int, ...]:

239 return self._prompt_token_ids_tuple

240

241 @prompt_token_ids.setter

@@ -252,7 +252,7 @@ class SequenceData(msgspec.Struct,

252 return self._prompt_token_ids

253

254 @property

255 - def output_token_ids(self) -> Tuple[int, ...]:

255 + def output_token_ids(self) -> tuple[int, ...]:

256 return tuple(self._output_token_ids)

257

258 @output_token_ids.setter

@@ -295,12 +295,12 @@ class SequenceData(msgspec.Struct,

295 def get_output_len(self) -> int:

296 return len(self._output_token_ids)

297

298 - def get_token_ids(self) -> List[int]:

298 + def get_token_ids(self) -> list[int]:

299 return self._cached_all_token_ids

300

301 def get_prefix_token_ids(

302 self, num_tokens: int

303 - ) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]:

303 + ) -> tuple[tuple[int, ...], Optional[tuple[int, ...]]]:

304 """Get prefix tokens, and make the return value hashable"""

305 prompt_length = self.get_prompt_len()

306 if num_tokens > prompt_length:

@@ -351,10 +351,10 @@ class SequenceData(msgspec.Struct,

351 return self._prompt_token_ids[-1]

352 return self._output_token_ids[-1]

353

354 - def get_prompt_token_ids(self) -> Tuple[int, ...]:

354 + def get_prompt_token_ids(self) -> tuple[int, ...]:

355 return self.prompt_token_ids

356

357 - def get_output_token_ids(self) -> Tuple[int, ...]:

357 + def get_output_token_ids(self) -> tuple[int, ...]:

358 return self.output_token_ids

359

360 def get_delta_and_reset(self) -> SequenceDataDelta:

@@ -432,7 +432,7 @@ class Sequence:

432 self.prefix_offset = 0

433 self.read_offset = 0

434 # Input + output tokens

435 - self.tokens: Optional[List[str]] = None

435 + self.tokens: Optional[list[str]] = None

436

437 @property

438 def n_blocks(self) -> int:

@@ -443,7 +443,7 @@ class Sequence:

443 return self.inputs.prompt

444

445 @property

446 - def prompt_token_ids(self) -> List[int]:

446 + def prompt_token_ids(self) -> list[int]:

447 return self.inputs.prompt_token_ids

448

449 @property

@@ -451,7 +451,7 @@ class Sequence:

451 return self.inputs.prompt_embeds

452

453 @property

454 - def token_type_ids(self) -> List[int]:

454 + def token_type_ids(self) -> list[int]:

455 return self.inputs.token_type_ids

456

457 @property

@@ -463,7 +463,7 @@ class Sequence:

463 return self.inputs.multi_modal_placeholders

464

465 @property

466 - def mm_processor_kwargs(self) -> Dict[str, Any]:

466 + def mm_processor_kwargs(self) -> dict[str, Any]:

467 return self.inputs.mm_processor_kwargs

468

469 @property

@@ -548,7 +548,7 @@ class Sequence:

548 """Reset the sequence states for recomputation."""

549 self.data.reset_state_for_recompute()

550

551 - def append_token_id(self, token_id: int, logprobs: Dict[int,

551 + def append_token_id(self, token_id: int, logprobs: dict[int,

552 Logprob]) -> None:

553 assert token_id in logprobs

554 self.output_logprobs.append(logprobs)

@@ -563,16 +563,16 @@ class Sequence:

563 def get_output_len(self) -> int:

564 return self.data.get_output_len()

565

566 - def get_token_ids(self) -> List[int]:

566 + def get_token_ids(self) -> list[int]:

567 return self.data.get_token_ids()

568

569 - def get_prompt_token_ids(self) -> Tuple[int, ...]:

569 + def get_prompt_token_ids(self) -> tuple[int, ...]:

570 return self.data.get_prompt_token_ids()

571

572 def get_last_token_id(self) -> int:

573 return self.data.get_last_token_id()

574

575 - def get_output_token_ids(self) -> Tuple[int, ...]:

575 + def get_output_token_ids(self) -> tuple[int, ...]:

576 return self.data.get_output_token_ids()

577

578 def get_cumulative_logprob(self) -> float:

@@ -644,7 +644,7 @@ class SequenceGroup:

644 def __init__(

645 self,

646 request_id: str,

647 - seqs: List[Sequence],

647 + seqs: list[Sequence],

648 arrival_time: float,

649 sampling_params: Optional[SamplingParams] = None,

650 lora_request: Optional[LoRARequest] = None,

@@ -686,7 +686,7 @@ class SequenceGroup:

686 return self.first_seq.prompt

687

688 @property

689 - def prompt_token_ids(self) -> List[int]:

689 + def prompt_token_ids(self) -> list[int]:

690 return self.first_seq.prompt_token_ids

691

692 @property

@@ -698,7 +698,7 @@ class SequenceGroup:

698 if self.encoder_seq is not None else None)

699

700 @property

701 - def encoder_prompt_token_ids(self) -> Optional[List[int]]:

701 + def encoder_prompt_token_ids(self) -> Optional[list[int]]:

702 # There are either 0 or 1 encoder sequences

703 # If one is present, its prompt token ids are

704 # distinct from the decoder's.

@@ -706,7 +706,7 @@ class SequenceGroup:

706 if self.encoder_seq is not None else None)

707

708 @property

709 - def token_type_ids(self) -> Optional[List[int]]:

709 + def token_type_ids(self) -> Optional[list[int]]:

710 return self.first_seq.token_type_ids

711

712 @property

@@ -726,7 +726,7 @@ class SequenceGroup:

726 return {}

727

728 @property

729 - def mm_processor_kwargs(self) -> Dict[str, Any]:

729 + def mm_processor_kwargs(self) -> dict[str, Any]:

730 if self.first_seq.multi_modal_data:

731 return self.first_seq.mm_processor_kwargs

732 elif self.encoder_seq is not None:

@@ -823,7 +823,7 @@ class SequenceGroup:

823 def get_seqs(

824 self,

825 status: Optional[SequenceStatus] = None,

826 - ) -> List[Sequence]:

826 + ) -> list[Sequence]:

827 if status is None:

828 return self.seqs

829

@@ -838,7 +838,7 @@ class SequenceGroup:

838 def get_encoder_seq(self) -> Optional[Sequence]:

839 return self.encoder_seq

840

841 - def get_finished_seqs(self) -> List[Sequence]:

841 + def get_finished_seqs(self) -> list[Sequence]:

842 if self.is_single_seq:

843 return self.seqs if self.first_seq.is_finished() else []

844

@@ -897,13 +897,13 @@ class SequenceGroupMetadataDelta(

897 After sending the first SequenceGroupMetadata, vLLM scheduler

898 only sends delta to reduce the data payload size.

899 """

900 - seq_data_delta: Dict[int, SequenceDataDelta]

900 + seq_data_delta: dict[int, SequenceDataDelta]

901 request_id: str

902 - block_tables: Dict[int, List[int]]

902 + block_tables: dict[int, list[int]]

903 is_prompt: bool

904 do_sample: bool = True

905 token_chunk_size: Optional[int] = None

906 - computed_block_nums: Optional[List[int]] = None

906 + computed_block_nums: Optional[list[int]] = None

907 state: Optional[SequenceGroupState] = msgspec.field(

908 default_factory=lambda: SequenceGroupState())

909

@@ -947,23 +947,23 @@ class SequenceGroupMetadata(

947

948 request_id: str

949 is_prompt: bool

950 - seq_data: Dict[int, SequenceData]

950 + seq_data: dict[int, SequenceData]

951 sampling_params: Optional[SamplingParams]

952 - block_tables: Dict[int, List[int]]

952 + block_tables: dict[int, list[int]]

953 do_sample: bool = True

954 pooling_params: Optional[PoolingParams] = None

955 lora_request: Optional[LoRARequest] = None

956 - computed_block_nums: Optional[List[int]] = None

956 + computed_block_nums: Optional[list[int]] = None

957 state: Optional[SequenceGroupState] = msgspec.field(

958 default_factory=lambda: SequenceGroupState())

959 # "MultiModalDataDict" types. We have to use Any due to msgspec

960 # doesn't allow to have union of 2 different dicts.

961 - token_type_ids: Optional[List[int]] = None

961 + token_type_ids: Optional[list[int]] = None

962 multi_modal_data: Optional[Any] = None

963 multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None

964 - mm_processor_kwargs: Optional[Dict[str, Any]] = None

964 + mm_processor_kwargs: Optional[dict[str, Any]] = None

965 encoder_seq_data: Optional[SequenceData] = None

966 - cross_block_table: Optional[List[int]] = None

966 + cross_block_table: Optional[list[int]] = None

967 prompt_adapter_request: Optional[PromptAdapterRequest] = None

968 token_chunk_size: Optional[int] = None

969

@@ -1042,7 +1042,7 @@ class SequenceOutput(

1042 """

1043 parent_seq_id: int

1044 output_token: int

1045 - logprobs: Dict[int, Logprob]

1045 + logprobs: dict[int, Logprob]

1046

1047 def __repr__(self) -> str:

1048 return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "

@@ -1076,7 +1076,7 @@ class CompletionSequenceGroupOutput(

1076 array_like=True): # type: ignore[call-arg]

1077 """The model output associated with a completion sequence group."""

1078 __metaclass__ = SequenceGroupOutput

1079 - samples: List[SequenceOutput]

1079 + samples: list[SequenceOutput]

1080 # Prompt logprob for each prompt query token.

1081 prompt_logprobs: Optional[PromptLogprobs]

1082

@@ -1119,7 +1119,7 @@ class IntermediateTensors:

1119 contains the hidden states and residuals for a request.

1120 """

1121

1122 - tensors: Dict[str, torch.Tensor]

1122 + tensors: dict[str, torch.Tensor]

1123

1124 def __init__(self, tensors):

1125 # manually define this function, so that

@@ -1155,7 +1155,7 @@ class PoolerOutput(

1155 omit_defaults=True, # type: ignore[call-arg]

1156 array_like=True): # type: ignore[call-arg]

1157 """The output from a pooling operation in the pooling model."""

1158 - outputs: List[PoolingSequenceGroupOutput]

1158 + outputs: list[PoolingSequenceGroupOutput]

1159

1160 def __getitem__(self, idx: int) -> PoolingSequenceGroupOutput:

1161 return self.outputs[idx]

@@ -1172,7 +1172,7 @@ class PoolerOutput(

1172

1173

1174 def get_all_seq_ids(

1175 - seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:

1175 + seq_group_metadata_list: list[SequenceGroupMetadata]) -> list[int]:

1176 """Given a list of SequenceGroupMetadata, create a list of all

1177 sequence ids.

1178 """

@@ -1180,13 +1180,13 @@ def get_all_seq_ids(

1180

1181

1182 def get_all_seq_ids_and_request_ids(

1183 - seq_group_metadata_list: List[SequenceGroupMetadata]

1184 -) -> Tuple[List[int], Dict[str, Set[int]]]:

1183 + seq_group_metadata_list: list[SequenceGroupMetadata]

1184 +) -> tuple[list[int], dict[str, set[int]]]:

1185 """Given a list of SequenceGroupMetadata, create a list of all

1186 sequence ids.

1187 """

1188 - seq_ids: List[int] = []

1189 - request_id_seq_ids_mapping: DefaultDict[str, Set[int]] = defaultdict(set)

1188 + seq_ids: list[int] = []

1189 + request_id_seq_ids_mapping: defaultdict[str, set[int]] = defaultdict(set)

1190 for sg in seq_group_metadata_list:

1191 for seq_id in sg.seq_data:

1192 seq_ids.append(seq_id)

@@ -1206,14 +1206,14 @@ class HiddenStates(msgspec.Struct, array_like=True,

1206 # all tokens, whereas for decode step, it use used for last accepted tokens.

1207 hidden_states: torch.Tensor

1208 # The sequence group metadata list. Only needed for decode step.

1209 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None

1209 + seq_group_metadata_list: Optional[list[SequenceGroupMetadata]] = None

1210 # Scorer hidden states of the 2nd last token proposed by the proposer (

1211 # irrespective of whether it was accepted or not). Only used for cases when

1212 # last proposed token is accepted (i.e., in case of bonus tokens). For the

1213 # case of no bonus tokens, these are ignored.

1214 second_last_token_hidden_states: Optional[torch.Tensor] = None

1215

1216 - _seq_ids: List[int] = msgspec.field(default_factory=list)

1216 + _seq_ids: list[int] = msgspec.field(default_factory=list)

1217

1218 def __post_init__(self):

1219 if self.seq_group_metadata_list is not None:

@@ -1221,12 +1221,12 @@ class HiddenStates(msgspec.Struct, array_like=True,

1221 self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)

1222

1223 @property

1224 - def seq_ids(self) -> List[int]:

1224 + def seq_ids(self) -> list[int]:

1225 return self._seq_ids

1226

1227 def update(self,

1228 hidden_states: torch.Tensor,

1229 - seq_group_metadata_list: List[SequenceGroupMetadata],

1229 + seq_group_metadata_list: list[SequenceGroupMetadata],

1230 second_last_token_hidden_states: Optional[torch.Tensor] = None):

1231 """Update hidden states from target model invocation. Only used for

1232 decode steps"""

@@ -1244,7 +1244,7 @@ class HiddenStates(msgspec.Struct, array_like=True,

1244 ])

1245

1246 def prune(self,

1247 - seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:

1247 + seq_group_metadata_list: list[SequenceGroupMetadata]) -> None:

1248 """Prune to provided list of sequence ids. Only used for decode steps.

1249 """

1250 # Currently this prunes all seq_ids not present in

@@ -1287,16 +1287,16 @@ class ExecuteModelRequest(

1287 """The model execution request, containing CPU metadata only. The LLM

1288 engine should create an instance of this class for each request batch."""

1289 # The sequence group metadata list.

1290 - seq_group_metadata_list: List[Union[SequenceGroupMetadata,

1290 + seq_group_metadata_list: list[Union[SequenceGroupMetadata,

1291 SequenceGroupMetadataDelta]]

1292 # Blocks to swap in. List of CPU -> GPU block number.

1293 - blocks_to_swap_in: List[Tuple[int,

1293 + blocks_to_swap_in: list[tuple[int,

1294 int]] = msgspec.field(default_factory=list)

1295 # Blocks to swap out. List of GPU -> CPU block number.

1296 - blocks_to_swap_out: List[Tuple[int,

1296 + blocks_to_swap_out: list[tuple[int,

1297 int]] = msgspec.field(default_factory=list)

1298 # Blocks to copy. Source to dest block.

1299 - blocks_to_copy: List[Tuple[int, int]] = msgspec.field(default_factory=list)

1299 + blocks_to_copy: list[tuple[int, int]] = msgspec.field(default_factory=list)

1300 # Virtual engine ID for pipeline parallel.

1301 virtual_engine: int = 0

1302 # The number of slots for lookahead decoding.

@@ -1310,7 +1310,7 @@ class ExecuteModelRequest(

1310 # The step index for spec model input.

1311 spec_step_idx: Optional[int] = None

1312 # Finished request ids since last step.

1313 - finished_requests_ids: List[str] = msgspec.field(default_factory=list)

1313 + finished_requests_ids: list[str] = msgspec.field(default_factory=list)

1314 # The last sampled token ids for multi step decoding.

1315 last_sampled_token_ids: Optional[torch.Tensor] = None

1316 # Async callback

@@ -1344,7 +1344,7 @@ class ExecuteModelRequest(

1344 return state.current_step

1345

1346 def clone(

1347 - self, seq_group_metadata_list: List[Union[SequenceGroupMetadata,

1347 + self, seq_group_metadata_list: list[Union[SequenceGroupMetadata,

1348 SequenceGroupMetadataDelta]]

1349 ) -> "ExecuteModelRequest":

1350 """Clone the request with a new sequence group metadata list."""

@@ -1371,13 +1371,13 @@ class SequenceGroupBase:

1371 assembled_seq_group: Optional[SequenceGroup] = None

1372

1373 # seq id to a unique index inside this group

1374 - seq_id_to_index: Dict[str, int] = field(default_factory=dict)

1374 + seq_id_to_index: dict[str, int] = field(default_factory=dict)

1375

1376 # seq ids to be finished

1377 - to_be_finished: Dict[str, SequenceGroup] = field(default_factory=dict)

1377 + to_be_finished: dict[str, SequenceGroup] = field(default_factory=dict)

1378

1379 # seq id to finished sequences

1380 - finished_reqs: Dict[str, SequenceGroup] = field(default_factory=dict)

1380 + finished_reqs: dict[str, SequenceGroup] = field(default_factory=dict)

1381

1382 streaming: bool = False

1383

vllm/tracing.py

+2 -1

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import os

4 -from typing import Mapping, Optional

4 +from collections.abc import Mapping

5 +from typing import Optional

6

7 from vllm.logger import init_logger

8 from vllm.utils import run_once

vllm/utils.py

+38 -38

@@ -28,12 +28,12 @@ import warnings

28 import weakref

29 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task

30 from collections import OrderedDict, UserDict, defaultdict

31 -from collections.abc import Hashable, Iterable, Mapping

31 +from collections.abc import (AsyncGenerator, Awaitable, Generator, Hashable,

32 + Iterable, Iterator, Mapping)

33 from dataclasses import dataclass, field

34 from functools import cache, lru_cache, partial, wraps

34 -from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,

35 - Dict, Generator, Generic, Iterator, List, Literal,

36 - NamedTuple, Optional, Tuple, Type, TypeVar, Union)

35 +from typing import (TYPE_CHECKING, Any, Callable, Generic, Literal, NamedTuple,

36 + Optional, TypeVar, Union)

37 from uuid import uuid4

38

39 import cloudpickle

@@ -400,7 +400,7 @@ def _next_task(iterator: AsyncGenerator[T, None],

400

401 async def merge_async_iterators(

402 *iterators: AsyncGenerator[T,

403 - None], ) -> AsyncGenerator[Tuple[int, T], None]:

403 + None], ) -> AsyncGenerator[tuple[int, T], None]:

404 """Merge multiple asynchronous iterators into a single iterator.

405

406 This method handle the case where some iterators finish before others.

@@ -433,7 +433,7 @@ async def merge_async_iterators(

433

434

435 async def collect_from_async_generator(

436 - iterator: AsyncGenerator[T, None]) -> List[T]:

436 + iterator: AsyncGenerator[T, None]) -> list[T]:

437 """Collect all items from an async generator into a list."""

438 items = []

439 async for item in iterator:

@@ -560,7 +560,7 @@ def find_process_using_port(port: int) -> Optional[psutil.Process]:

560 return None

561

562

563 -def update_environment_variables(envs: Dict[str, str]):

563 +def update_environment_variables(envs: dict[str, str]):

564 for k, v in envs.items():

565 if k in os.environ and os.environ[k] != v:

566 logger.warning(

@@ -569,7 +569,7 @@ def update_environment_variables(envs: Dict[str, str]):

569 os.environ[k] = v

570

571

572 -def chunk_list(lst: List[T], chunk_size: int):

572 +def chunk_list(lst: list[T], chunk_size: int):

573 """Yield successive chunk_size chunks from lst."""

574 for i in range(0, len(lst), chunk_size):

575 yield lst[i:i + chunk_size]

@@ -642,7 +642,7 @@ def create_kv_caches_with_random_flash(

642 model_dtype: Optional[Union[str, torch.dtype]] = None,

643 seed: int = 0,

644 device: Optional[str] = "cuda",

645 -) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:

645 +) -> tuple[list[torch.Tensor], list[torch.Tensor]]:

646 from vllm.platforms import current_platform

647 current_platform.seed_everything(seed)

648

@@ -650,8 +650,8 @@ def create_kv_caches_with_random_flash(

650 key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)

651 scale = head_size**-0.5

652

653 - key_caches: List[torch.Tensor] = []

654 - value_caches: List[torch.Tensor] = []

653 + key_caches: list[torch.Tensor] = []

654 + value_caches: list[torch.Tensor] = []

655

656 for _ in range(num_layers):

657 key_value_cache = torch.empty(size=key_value_cache_shape,

@@ -679,7 +679,7 @@ def create_kv_caches_with_random(

679 model_dtype: Optional[Union[str, torch.dtype]] = None,

680 seed: int = 0,

681 device: Optional[str] = "cuda",

682 -) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:

682 +) -> tuple[list[torch.Tensor], list[torch.Tensor]]:

683

684 if cache_dtype == "fp8" and head_size % 16:

685 raise ValueError(

@@ -693,7 +693,7 @@ def create_kv_caches_with_random(

693 scale = head_size**-0.5

694 x = 16 // torch.tensor([], dtype=torch_dtype).element_size()

695 key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)

696 - key_caches: List[torch.Tensor] = []

696 + key_caches: list[torch.Tensor] = []

697 for _ in range(num_layers):

698 key_cache = torch.empty(size=key_cache_shape,

699 dtype=torch_dtype,

@@ -708,7 +708,7 @@ def create_kv_caches_with_random(

708 key_caches.append(key_cache)

709

710 value_cache_shape = (num_blocks, num_heads, head_size, block_size)

711 - value_caches: List[torch.Tensor] = []

711 + value_caches: list[torch.Tensor] = []

712 for _ in range(num_layers):

713 value_cache = torch.empty(size=value_cache_shape,

714 dtype=torch_dtype,

@@ -754,7 +754,7 @@ class DeviceMemoryProfiler:

754

755

756 def make_ndarray_with_pad(

757 - x: List[List[T]],

757 + x: list[list[T]],

758 pad: T,

759 dtype: npt.DTypeLike,

760 *,

@@ -779,7 +779,7 @@ def make_ndarray_with_pad(

779

780

781 def make_tensor_with_pad(

782 - x: List[List[T]],

782 + x: list[list[T]],

783 pad: T,

784 dtype: torch.dtype,

785 *,

@@ -831,7 +831,7 @@ def is_list_of(

831 typ: Union[type[T], tuple[type[T], ...]],

832 *,

833 check: Literal["first", "all"] = "first",

834 -) -> TypeIs[List[T]]:

834 +) -> TypeIs[list[T]]:

835 if not isinstance(value, list):

836 return False

837

@@ -843,8 +843,8 @@ def is_list_of(

843 assert_never(check)

844

845

846 -JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],

847 - Tuple["JSONTree[T]", ...], T]

846 +JSONTree = Union[dict[str, "JSONTree[T]"], list["JSONTree[T]"],

847 + tuple["JSONTree[T]", ...], T]

848 """A nested JSON structure where the leaves need not be JSON-serializable."""

849

850

@@ -859,7 +859,7 @@ def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]:

859 return func(value)

860

861

862 -def flatten_2d_lists(lists: List[List[T]]) -> List[T]:

862 +def flatten_2d_lists(lists: list[list[T]]) -> list[T]:

863 """Flatten a list of lists to a single list."""

864 return [item for sublist in lists for item in sublist]

865

@@ -1226,7 +1226,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):

1226

1227 return value

1228

1229 - def _pull_args_from_config(self, args: List[str]) -> List[str]:

1229 + def _pull_args_from_config(self, args: list[str]) -> list[str]:

1230 """Method to pull arguments specified in the config file

1231 into the command-line args variable.

1232

@@ -1291,7 +1291,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):

1291

1292 return args

1293

1294 - def _load_config_file(self, file_path: str) -> List[str]:

1294 + def _load_config_file(self, file_path: str) -> list[str]:

1295 """Loads a yaml file and returns the key value pairs as a

1296 flattened list with argparse like pattern

1297 ```yaml

@@ -1313,9 +1313,9 @@ class FlexibleArgumentParser(argparse.ArgumentParser):

1313 %s supplied", extension)

1314

1315 # only expecting a flat dictionary of atomic types

1316 - processed_args: List[str] = []

1316 + processed_args: list[str] = []

1317

1318 - config: Dict[str, Union[int, str]] = {}

1318 + config: dict[str, Union[int, str]] = {}

1319 try:

1320 with open(file_path) as config_file:

1321 config = yaml.safe_load(config_file)

@@ -1399,7 +1399,7 @@ def resolve_mm_processor_kwargs(

1399 *,

1400 requires_kw_only: bool = True,

1401 allow_var_kwargs: bool = False,

1402 -) -> Dict[str, Any]:

1402 +) -> dict[str, Any]:

1403 """Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,

1404 those who are not explicit keywords to the given callable (of one is

1405 given; otherwise no filtering is done), then merges the kwarg dicts,

@@ -1440,7 +1440,7 @@ def get_allowed_kwarg_only_overrides(

1440 *,

1441 requires_kw_only: bool = True,

1442 allow_var_kwargs: bool = False,

1443 -) -> Dict[str, Any]:

1443 +) -> dict[str, Any]:

1444 """

1445 Given a callable which has one or more keyword only params and a dict

1446 mapping param names to values, drop values that can be not be kwarg

@@ -1531,9 +1531,9 @@ class AtomicCounter:

1531 # Adapted from: https://stackoverflow.com/a/47212782/5082708

1532 class LazyDict(Mapping[str, T], Generic[T]):

1533

1534 - def __init__(self, factory: Dict[str, Callable[[], T]]):

1534 + def __init__(self, factory: dict[str, Callable[[], T]]):

1535 self._factory = factory

1536 - self._dict: Dict[str, T] = {}

1536 + self._dict: dict[str, T] = {}

1537

1538 def __getitem__(self, key: str) -> T:

1539 if key not in self._dict:

@@ -1552,9 +1552,9 @@ class LazyDict(Mapping[str, T], Generic[T]):

1552 return len(self._factory)

1553

1554

1555 -class ClassRegistry(UserDict[Type[T], _V]):

1555 +class ClassRegistry(UserDict[type[T], _V]):

1556

1557 - def __getitem__(self, key: Type[T]) -> _V:

1557 + def __getitem__(self, key: type[T]) -> _V:

1558 for cls in key.mro():

1559 if cls in self.data:

1560 return self.data[cls]

@@ -1584,8 +1584,8 @@ def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:

1584

1585

1586 def weak_ref_tensors(

1587 - tensors: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]

1588 -) -> Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]:

1587 + tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]]

1588 +) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]]:

1589 """

1590 Convenience function to create weak references to tensors,

1591 for single tensor, list of tensors or tuple of tensors.

@@ -1857,7 +1857,7 @@ vllm_lib = Library("vllm", "FRAGMENT") # noqa

1857 def direct_register_custom_op(

1858 op_name: str,

1859 op_func: Callable,

1860 - mutates_args: List[str],

1860 + mutates_args: list[str],

1861 fake_impl: Optional[Callable] = None,

1862 target_lib: Optional[Library] = None,

1863 dispatch_key: str = "CUDA",

@@ -2177,8 +2177,8 @@ def get_mp_context():

2177

2178

2179 def bind_kv_cache(

2180 - ctx: Dict[str, Any],

2181 - kv_cache: List[List[torch.Tensor]], # [virtual_engine][layer_index]

2180 + ctx: dict[str, Any],

2181 + kv_cache: list[list[torch.Tensor]], # [virtual_engine][layer_index]

2182 ) -> None:

2183 # Bind the kv_cache tensor to Attention modules, similar to

2184 # ctx[layer_name].kv_cache[ve]=kv_cache[ve][extract_layer_index(layer_name)]

@@ -2210,8 +2210,8 @@ def bind_kv_cache(

2210 forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]

2211

2212

2213 -def run_method(obj: Any, method: Union[str, bytes, Callable], args: Tuple[Any],

2214 - kwargs: Dict[str, Any]) -> Any:

2213 +def run_method(obj: Any, method: Union[str, bytes, Callable], args: tuple[Any],

2214 + kwargs: dict[str, Any]) -> Any:

2215 """

2216 Run a method of an object with the given arguments and keyword arguments.

2217 If the method is string, it will be converted to a method using getattr.

@@ -2263,7 +2263,7 @@ def import_pynvml():

2263 return pynvml

2264

2265

2266 -def warn_for_unimplemented_methods(cls: Type[T]) -> Type[T]:

2266 +def warn_for_unimplemented_methods(cls: type[T]) -> type[T]:

2267 """

2268 A replacement for `abc.ABC`.

2269 When we use `abc.ABC`, subclasses will fail to instantiate

vllm/v1/attention/backends/flash_attn.py

+9 -9

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Attention layer with FlashAttention."""

3 from dataclasses import dataclass

4 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type

4 +from typing import TYPE_CHECKING, Any, Optional

5

6 import numpy as np

7 import torch

@@ -30,7 +30,7 @@ class FlashAttentionBackend(AttentionBackend):

30 accept_output_buffer: bool = True

31

32 @staticmethod

33 - def get_supported_head_sizes() -> List[int]:

33 + def get_supported_head_sizes() -> list[int]:

34 return [32, 64, 96, 128, 160, 192, 224, 256]

35

36 @staticmethod

@@ -38,15 +38,15 @@ class FlashAttentionBackend(AttentionBackend):

38 return "FLASH_ATTN_VLLM_V1"

39

40 @staticmethod

41 - def get_impl_cls() -> Type["FlashAttentionImpl"]:

41 + def get_impl_cls() -> type["FlashAttentionImpl"]:

42 return FlashAttentionImpl

43

44 @staticmethod

45 - def get_metadata_cls() -> Type["AttentionMetadata"]:

45 + def get_metadata_cls() -> type["AttentionMetadata"]:

46 return FlashAttentionMetadata

47

48 @staticmethod

49 - def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]:

49 + def get_builder_cls() -> type["FlashAttentionMetadataBuilder"]:

50 return FlashAttentionMetadataBuilder

51

52 @staticmethod

@@ -55,7 +55,7 @@ class FlashAttentionBackend(AttentionBackend):

55 block_size: int,

56 num_kv_heads: int,

57 head_size: int,

58 - ) -> Tuple[int, ...]:

58 + ) -> tuple[int, ...]:

59 if block_size % 16 != 0:

60 raise ValueError("Block size must be a multiple of 16.")

61 return (2, num_blocks, block_size, num_kv_heads, head_size)

@@ -158,10 +158,10 @@ class FlashAttentionImpl(AttentionImpl):

158 head_size: int,

159 scale: float,

160 num_kv_heads: int,

161 - alibi_slopes: Optional[List[float]],

161 + alibi_slopes: Optional[list[float]],

162 sliding_window: Optional[int],

163 kv_cache_dtype: str,

164 - blocksparse_params: Optional[Dict[str, Any]] = None,

164 + blocksparse_params: Optional[dict[str, Any]] = None,

165 logits_soft_cap: Optional[float] = None,

166 attn_type: AttentionType = AttentionType.DECODER,

167 ) -> None:

@@ -381,7 +381,7 @@ def cascade_attention(

381 max_kv_len: int,

382 softmax_scale: float,

383 alibi_slopes: Optional[torch.Tensor],

384 - sliding_window: Tuple[int, int],

384 + sliding_window: tuple[int, int],

385 logits_soft_cap: float,

386 block_table: torch.Tensor,

387 common_prefix_len: int,

vllm/v1/attention/backends/mla/common.py

+10 -11

@@ -195,8 +195,7 @@ return curr_o @ W_O

195 import functools

196 from abc import abstractmethod

197 from dataclasses import dataclass

198 -from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Tuple,

199 - Type, TypeVar)

198 +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar

199

200 import torch

201 from compressed_tensors.quantization import QuantizationStrategy

@@ -250,11 +249,11 @@ class MLACommonBackend(AttentionBackend):

249 return "TRITON_MLA_VLLM_V1"

250

251 @staticmethod

253 - def get_metadata_cls() -> Type["AttentionMetadata"]:

252 + def get_metadata_cls() -> type["AttentionMetadata"]:

253 return MLACommonMetadata

254

255 @staticmethod

257 - def get_builder_cls() -> Type["MLACommonMetadataBuilder"]:

256 + def get_builder_cls() -> type["MLACommonMetadataBuilder"]:

257 return MLACommonMetadataBuilder

258

259 @staticmethod

@@ -263,11 +262,11 @@ class MLACommonBackend(AttentionBackend):

262 block_size: int,

263 num_kv_heads: int, # assumed to be 1 for MLA

264 head_size: int,

266 - ) -> Tuple[int, ...]:

265 + ) -> tuple[int, ...]:

266 return (num_blocks, block_size, head_size)

267

268 @staticmethod

270 - def get_supported_head_sizes() -> List[int]:

269 + def get_supported_head_sizes() -> list[int]:

270 return [576]

271

272 @staticmethod

@@ -317,8 +316,8 @@ class MLACommonMetadata:

316 has_context: bool = False

317 context_chunk_cu_seq_lens: Optional[torch.Tensor] = None

318 context_chunk_starts: Optional[torch.Tensor] = None

320 - context_chunk_seq_tot: Optional[List[int]] = None

321 - context_chunk_max_seq_lens: Optional[List[int]] = None

319 + context_chunk_seq_tot: Optional[list[int]] = None

320 + context_chunk_max_seq_lens: Optional[list[int]] = None

321 chunked_prefill_workspace: Optional[torch.Tensor] = None

322

323 def __post_init__(self):

@@ -538,10 +537,10 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):

537 head_size: int,

538 scale: float,

539 num_kv_heads: int,

541 - alibi_slopes: Optional[List[float]],

540 + alibi_slopes: Optional[list[float]],

541 sliding_window: Optional[int],

542 kv_cache_dtype: str,

544 - blocksparse_params: Optional[Dict[str, Any]],

543 + blocksparse_params: Optional[dict[str, Any]],

544 logits_soft_cap: Optional[float],

545 attn_type: str,

546 # MLA Specific Arguments

@@ -634,7 +633,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):

633 #

634 # returns input_group_shape, weight_group_shape

635 def get_scale_group_shapes_for_fp8(layer: LinearBase) -> \

637 - Tuple[Tuple[int, int], Tuple[int, int]]:

636 + tuple[tuple[int, int], tuple[int, int]]:

637 if isinstance(layer.quant_method, Fp8LinearMethod):

638 if layer.quant_method.block_quant:

639 weight_block_size = \

vllm/v1/attention/backends/mla/flashmla.py

+7 -7

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Any, Dict, List, Optional, Tuple, Type

4 +from typing import Any, Optional

5

6 import torch

7

@@ -25,21 +25,21 @@ class FlashMLABackend(MLACommonBackend):

25 return "FLASHMLA_VLLM_V1"

26

27 @staticmethod

28 - def get_metadata_cls() -> Type["FlashMLAMetadata"]:

28 + def get_metadata_cls() -> type["FlashMLAMetadata"]:

29 return FlashMLAMetadata

30

31 @staticmethod

32 - def get_builder_cls() -> Type["FlashMLAMetadataBuilder"]:

32 + def get_builder_cls() -> type["FlashMLAMetadataBuilder"]:

33 return FlashMLAMetadataBuilder

34

35 @staticmethod

36 - def get_impl_cls() -> Type["FlashMLAImpl"]:

36 + def get_impl_cls() -> type["FlashMLAImpl"]:

37 return FlashMLAImpl

38

39

40 @dataclass

41 class FlashMLAMetadata(MLACommonMetadata):

42 - decode_tile_scheduler_metadata: Optional[Tuple[torch.Tensor,

42 + decode_tile_scheduler_metadata: Optional[tuple[torch.Tensor,

43 torch.Tensor]] = None

44 decode_num_splits: Optional[torch.Tensor] = None

45

@@ -76,10 +76,10 @@ class FlashMLAImpl(MLACommonImpl[FlashMLAMetadata]):

76 head_size: int,

77 scale: float,

78 num_kv_heads: int,

79 - alibi_slopes: Optional[List[float]],

79 + alibi_slopes: Optional[list[float]],

80 sliding_window: Optional[int],

81 kv_cache_dtype: str,

82 - blocksparse_params: Optional[Dict[str, Any]],

82 + blocksparse_params: Optional[dict[str, Any]],

83 logits_soft_cap: Optional[float],

84 attn_type: str,

85 # MLA Specific Arguments

vllm/v1/attention/backends/mla/triton_mla.py

+4 -4

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Any, Dict, List, Optional, Type

3 +from typing import Any, Optional

4

5 import torch

6

@@ -21,7 +21,7 @@ class TritonMLABackend(MLACommonBackend):

21 return "TRITON_MLA_VLLM_V1"

22

23 @staticmethod

24 - def get_impl_cls() -> Type["TritonMLAImpl"]:

24 + def get_impl_cls() -> type["TritonMLAImpl"]:

25 return TritonMLAImpl

26

27

@@ -33,10 +33,10 @@ class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]):

33 head_size: int,

34 scale: float,

35 num_kv_heads: int,

36 - alibi_slopes: Optional[List[float]],

36 + alibi_slopes: Optional[list[float]],

37 sliding_window: Optional[int],

38 kv_cache_dtype: str,

39 - blocksparse_params: Optional[Dict[str, Any]],

39 + blocksparse_params: Optional[dict[str, Any]],

40 logits_soft_cap: Optional[float],

41 attn_type: str,

42 # MLA Specific Arguments

vllm/v1/attention/backends/pallas.py

+8 -8

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Any, Dict, List, Optional, Tuple, Type

4 +from typing import Any, Optional

5

6 import torch

7 # Required to register custom ops.

@@ -22,15 +22,15 @@ class PallasAttentionBackend(AttentionBackend):

22 return "PALLAS_VLLM_V1"

23

24 @staticmethod

25 - def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:

25 + def get_impl_cls() -> type["PallasAttentionBackendImpl"]:

26 return PallasAttentionBackendImpl

27

28 @staticmethod

29 - def get_metadata_cls() -> Type["PallasMetadata"]:

29 + def get_metadata_cls() -> type["PallasMetadata"]:

30 return PallasMetadata

31

32 @staticmethod

33 - def get_state_cls() -> Type["CommonAttentionState"]:

33 + def get_state_cls() -> type["CommonAttentionState"]:

34 return CommonAttentionState

35

36 @staticmethod

@@ -39,7 +39,7 @@ class PallasAttentionBackend(AttentionBackend):

39 block_size: int,

40 num_kv_heads: int,

41 head_size: int,

42 - ) -> Tuple[int, ...]:

42 + ) -> tuple[int, ...]:

43 return (num_kv_heads, num_blocks, block_size, head_size)

44

45 @staticmethod

@@ -77,10 +77,10 @@ class PallasAttentionBackendImpl(AttentionImpl):

77 head_size: int,

78 scale: float,

79 num_kv_heads: int,

80 - alibi_slopes: Optional[List[float]],

80 + alibi_slopes: Optional[list[float]],

81 sliding_window: Optional[int],

82 kv_cache_dtype: str,

83 - blocksparse_params: Optional[Dict[str, Any]] = None,

83 + blocksparse_params: Optional[dict[str, Any]] = None,

84 logits_soft_cap: Optional[float] = None,

85 attn_type: str = AttentionType.DECODER,

86 ) -> None:

@@ -120,7 +120,7 @@ class PallasAttentionBackendImpl(AttentionImpl):

120 query: torch.Tensor,

121 key: torch.Tensor,

122 value: torch.Tensor,

123 - kv_cache: Tuple[torch.Tensor, torch.Tensor],

123 + kv_cache: tuple[torch.Tensor, torch.Tensor],

124 attn_metadata: PallasMetadata,

125 output: Optional[torch.Tensor] = None,

126 ) -> torch.Tensor:

vllm/v1/attention/backends/rocm_attn.py

+8 -8

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """Attention layer with PagedAttention on rocm"""

3 -from typing import Any, Dict, List, Optional, Tuple, Type

3 +from typing import Any, Optional

4

5 import torch

6

@@ -20,7 +20,7 @@ class ROCmAttentionBackend(AttentionBackend):

20 accept_output_buffer: bool = True

21

22 @staticmethod

23 - def get_supported_head_sizes() -> List[int]:

23 + def get_supported_head_sizes() -> list[int]:

24 return [32, 64, 96, 128, 160, 192, 224, 256]

25

26 @staticmethod

@@ -28,11 +28,11 @@ class ROCmAttentionBackend(AttentionBackend):

28 return "ROCM_ATTN_VLLM_V1"

29

30 @staticmethod

31 - def get_impl_cls() -> Type["ROCmAttentionImpl"]:

31 + def get_impl_cls() -> type["ROCmAttentionImpl"]:

32 return ROCmAttentionImpl

33

34 @staticmethod

35 - def get_metadata_cls() -> Type["AttentionMetadata"]:

35 + def get_metadata_cls() -> type["AttentionMetadata"]:

36 return FlashAttentionMetadata

37

38 @staticmethod

@@ -41,7 +41,7 @@ class ROCmAttentionBackend(AttentionBackend):

41 block_size: int,

42 num_kv_heads: int,

43 head_size: int,

44 - ) -> Tuple[int, ...]:

44 + ) -> tuple[int, ...]:

45 if block_size % 16 != 0:

46 raise ValueError("Block size must be a multiple of 16.")

47 return (2, num_blocks, block_size, num_kv_heads, head_size)

@@ -51,7 +51,7 @@ class ROCmAttentionBackend(AttentionBackend):

51 return False

52

53 @staticmethod

54 - def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]:

54 + def get_builder_cls() -> type["FlashAttentionMetadataBuilder"]:

55 return FlashAttentionMetadataBuilder

56

57

@@ -63,10 +63,10 @@ class ROCmAttentionImpl(AttentionImpl):

63 head_size: int,

64 scale: float,

65 num_kv_heads: int,

66 - alibi_slopes: Optional[List[float]],

66 + alibi_slopes: Optional[list[float]],

67 sliding_window: Optional[int],

68 kv_cache_dtype: str,

69 - blocksparse_params: Optional[Dict[str, Any]] = None,

69 + blocksparse_params: Optional[dict[str, Any]] = None,

70 logits_soft_cap: Optional[float] = None,

71 attn_type: AttentionType = AttentionType.DECODER,

72 ) -> None:

vllm/v1/core/block_pool.py

+9 -8

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 from collections import defaultdict

3 -from typing import Dict, Iterable, List, Optional

3 +from collections.abc import Iterable

4 +from typing import Optional

5

6 from vllm.logger import init_logger

7 from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,

@@ -29,7 +30,7 @@ class BlockPool:

30 self.num_gpu_blocks = num_gpu_blocks

31 self.enable_caching = enable_caching

32 # All kv-cache blocks.

32 - self.blocks: List[KVCacheBlock] = [

33 + self.blocks: list[KVCacheBlock] = [

34 KVCacheBlock(idx) for idx in range(num_gpu_blocks)

35 ]

36 # Free block queue that constructs and manipulates a doubly linked

@@ -46,7 +47,7 @@ class BlockPool:

47 # if there is already an identical block in the cache. This is because

48 # we want to make sure the allocated block IDs won't change so that

49 # block tables are append-only.

49 - self.cached_block_hash_to_block: Dict[BlockHashType, Dict[

50 + self.cached_block_hash_to_block: dict[BlockHashType, dict[

51 int, KVCacheBlock]] = defaultdict(dict)

52

53 def get_cached_block(self,

@@ -69,8 +70,8 @@ class BlockPool:

70 def cache_full_blocks(

71 self,

72 request: Request,

72 - blocks: List[KVCacheBlock],

73 - block_hashes: List[BlockHashType],

73 + blocks: list[KVCacheBlock],

74 + block_hashes: list[BlockHashType],

75 num_cached_blocks: int,

76 num_full_blocks: int,

77 block_size: int,

@@ -146,7 +147,7 @@ class BlockPool:

147 self.cached_block_hash_to_block[block_hash][blk.block_id] = blk

148 prev_block_hash_value = block_hash.hash_value

149

149 - def get_new_blocks(self, num_blocks: int) -> List[KVCacheBlock]:

150 + def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:

151 """Get new blocks from the free block pool.

152

153 Note that we do not check block cache in this function.

@@ -161,7 +162,7 @@ class BlockPool:

162 raise ValueError(

163 f"Cannot get {num_blocks} free blocks from the pool")

164

164 - ret: List[KVCacheBlock] = []

165 + ret: list[KVCacheBlock] = []

166 idx = 0

167 while idx < num_blocks:

168 # First allocate blocks.

@@ -200,7 +201,7 @@ class BlockPool:

201 return True

202 return False

203

203 - def touch(self, blocks: List[KVCacheBlock]) -> None:

204 + def touch(self, blocks: list[KVCacheBlock]) -> None:

205 """Touch a block increases its reference count by 1, and may remove

206 the block from the free queue. This is used when a block is hit by

207 another request with the same prefix.

vllm/v1/core/encoder_cache_manager.py

+8 -8

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import TYPE_CHECKING, Dict, List, Set, Tuple

3 +from typing import TYPE_CHECKING

4

5 from vllm.logger import init_logger

6 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -18,9 +18,9 @@ class EncoderCacheManager:

18 self.cache_size = cache_size

19 self.num_free_slots = cache_size

20 # req_id -> cached input ids

21 - self.cached: Dict[str, Set[int]] = {}

22 - # List of [req_id, input_id]

23 - self.freed: List[Tuple[str, int]] = []

21 + self.cached: dict[str, set[int]] = {}

22 + # list of [req_id, input_id]

23 + self.freed: list[tuple[str, int]] = []

24

25 def has_cache(self, request: Request, input_id: int) -> bool:

26 req_id = request.request_id

@@ -37,7 +37,7 @@ class EncoderCacheManager:

37 self.cached[req_id].add(input_id)

38 self.num_free_slots -= request.get_num_encoder_tokens(input_id)

39

40 - def get_cached_input_ids(self, request: Request) -> Set[int]:

40 + def get_cached_input_ids(self, request: Request) -> set[int]:

41 return self.cached.get(request.request_id, set())

42

43 def free_encoder_input(self, request: Request, input_id: int) -> None:

@@ -58,7 +58,7 @@ class EncoderCacheManager:

58 for input_id in input_ids:

59 self.free_encoder_input(request, input_id)

60

61 - def get_freed_ids(self) -> List[Tuple[str, int]]:

61 + def get_freed_ids(self) -> list[tuple[str, int]]:

62 freed = self.freed

63 self.freed = []

64 return freed

@@ -67,7 +67,7 @@ class EncoderCacheManager:

67 def compute_encoder_budget(

68 model_config: "ModelConfig",

69 scheduler_config: "SchedulerConfig",

70 -) -> Tuple[int, int]:

70 +) -> tuple[int, int]:

71 """Compute the encoder cache budget based on the model and scheduler

72 configurations.

73

@@ -97,7 +97,7 @@ def compute_encoder_budget(

97 def _compute_encoder_budget_multimodal(

98 model_config: "ModelConfig",

99 scheduler_config: "SchedulerConfig",

100 -) -> Tuple[int, int]:

100 +) -> tuple[int, int]:

101 """Compute the encoder cache budget based on the model and scheduler

102 configurations for a multimodal model.

103

vllm/v1/core/kv_cache_manager.py

+10 -9

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from collections import defaultdict

4 -from typing import DefaultDict, Dict, Iterable, List, Optional, Tuple

4 +from collections.abc import Iterable

5 +from typing import Optional

6

7 from vllm.logger import init_logger

8 from vllm.utils import cdiv

@@ -52,20 +53,20 @@ class KVCacheManager:

53 # Mapping from request ID to blocks to track the blocks allocated

54 # for each request, so that we can free the blocks when the request

55 # is finished.

55 - self.req_to_blocks: DefaultDict[str,

56 - List[KVCacheBlock]] = defaultdict(list)

56 + self.req_to_blocks: defaultdict[str,

57 + list[KVCacheBlock]] = defaultdict(list)

58

59 # Mapping from request ID to kv block hashes.

60 # This is to avoid recomputing the block hashes for each call of

61 # `get_computed_blocks` or `allocate_slots`.

61 - self.req_to_block_hashes: DefaultDict[

62 - str, List[BlockHashType]] = defaultdict(list)

62 + self.req_to_block_hashes: defaultdict[

63 + str, list[BlockHashType]] = defaultdict(list)

64

65 # {req_id: The number of cached blocks for this given request}

66 # This is used to track the number of cached blocks for each request.

67 # This is only used to track the RUNNING requests, we do not track the

68 # data for reempted ones.

68 - self.num_cached_block: Dict[str, int] = {}

69 + self.num_cached_block: dict[str, int] = {}

70 self.prefix_cache_stats = PrefixCacheStats()

71

72 @property

@@ -88,7 +89,7 @@ class KVCacheManager:

89 return stats

90

91 def get_computed_blocks(

91 - self, request: Request) -> Tuple[List[KVCacheBlock], int]:

92 + self, request: Request) -> tuple[list[KVCacheBlock], int]:

93 """Get the computed (cached) blocks for the request.

94 Note that the computed blocks must be full.

95

@@ -136,8 +137,8 @@ class KVCacheManager:

137 self,

138 request: Request,

139 num_tokens: int,

139 - new_computed_blocks: Optional[List[KVCacheBlock]] = None

140 - ) -> Optional[List[KVCacheBlock]]:

140 + new_computed_blocks: Optional[list[KVCacheBlock]] = None

141 + ) -> Optional[list[KVCacheBlock]]:

142 """Add slots for a request with new tokens to append.

143

144 Args:

vllm/v1/core/kv_cache_utils.py

+16 -16

@@ -3,7 +3,7 @@

3 from collections import deque

4 from collections.abc import Sequence

5 from dataclasses import dataclass

6 -from typing import Any, List, NamedTuple, Optional, Tuple

6 +from typing import Any, NamedTuple, Optional

7

8 from vllm.config import VllmConfig

9 from vllm.logger import init_logger

@@ -25,7 +25,7 @@ class BlockHashType(NamedTuple):

25 # Hash value of the block in an integer.

26 hash_value: int

27 # Token IDs in the block.

28 - token_ids: Tuple[int, ...]

28 + token_ids: tuple[int, ...]

29 # Extra keys for the block.

30 extra_keys: Optional[Any] = None

31

@@ -45,7 +45,7 @@ class PrefixCachingMetrics:

45 self.aggregated_query_total = 0

46 self.aggregated_query_hit = 0

47 # A deque of (requests, queries, hits) for the most recent requests.

48 - self.query_queue: deque[Tuple[int, int, int]] = deque()

48 + self.query_queue: deque[tuple[int, int, int]] = deque()

49

50 def observe(self, stats: PrefixCacheStats):

51 """Observe the prefix caching for a set of requests.

@@ -164,7 +164,7 @@ class FreeKVCacheBlockQueue:

164 blocks: A list of KVCacheBlock objects.

165 """

166

167 - def __init__(self, blocks: List[KVCacheBlock]) -> None:

167 + def __init__(self, blocks: list[KVCacheBlock]) -> None:

168 self.num_free_blocks = len(blocks)

169

170 # Initialize the doubly linked list of free blocks.

@@ -233,7 +233,7 @@ class FreeKVCacheBlockQueue:

233 block.next_free_block = None

234 self.num_free_blocks += 1

235

236 - def get_all_free_blocks(self) -> List[KVCacheBlock]:

236 + def get_all_free_blocks(self) -> list[KVCacheBlock]:

237 """Get all free blocks in the free list. Mainly used for testing.

238

239 Returns:

@@ -264,7 +264,7 @@ def need_extra_keys(request: Request) -> bool:

264

265 def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

266 end_token_idx: int,

267 - start_mm_idx: int) -> Tuple[List[Any], int]:

267 + start_mm_idx: int) -> tuple[list[Any], int]:

268 """Generate extra keys related to MultiModal request for block hash

269 computation. For multi-modal inputs, the extra keys are

270 (mm_hash, start_offset) that indicate a mm input contained in the

@@ -279,7 +279,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

279 Returns:

280 A tuple of extra keys and the next multi-modal index.

281 """

282 - extra_keys: List[Any] = []

282 + extra_keys: list[Any] = []

283

284 mm_positions, mm_hashes = request.mm_positions, request.mm_hashes

285 if not mm_positions:

@@ -331,7 +331,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

331 return extra_keys, curr_mm_idx

332

333

334 -def _gen_lora_extra_hash_keys(request: Request) -> List[int]:

334 +def _gen_lora_extra_hash_keys(request: Request) -> list[int]:

335 """Generate extra keys related to LoRA for block hash computation.

336

337 Args:

@@ -348,7 +348,7 @@ def _gen_lora_extra_hash_keys(request: Request) -> List[int]:

348

349 def generate_block_hash_extra_keys(

350 request: Request, start_token_idx: int, end_token_idx: int,

351 - start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:

351 + start_mm_idx: int) -> tuple[Optional[tuple[Any, ...]], int]:

352 """Generate extra keys for the block hash. The extra keys can come from

353 the multi-modal inputs and request specific metadata (e.g., LoRA ID).

354

@@ -361,12 +361,12 @@ def generate_block_hash_extra_keys(

361 Returns:

362 A tuple of extra keys and the next multi-modal index.

363 """

364 - mm_extra_keys: List[Any]

364 + mm_extra_keys: list[Any]

365 mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(

366 request, start_token_idx, end_token_idx, start_mm_idx)

367 - lora_extra_keys: List[int] = _gen_lora_extra_hash_keys(request)

367 + lora_extra_keys: list[int] = _gen_lora_extra_hash_keys(request)

368

369 - extra_keys: List[Any] = lora_extra_keys + mm_extra_keys

369 + extra_keys: list[Any] = lora_extra_keys + mm_extra_keys

370

371 if not extra_keys:

372 return None, new_start_mm_idx

@@ -377,7 +377,7 @@ def generate_block_hash_extra_keys(

377 def hash_block_tokens(

378 parent_block_hash: Optional[int],

379 curr_block_token_ids: Sequence[int],

380 - extra_keys: Optional[Tuple[Any, ...]] = None) -> BlockHashType:

380 + extra_keys: Optional[tuple[Any, ...]] = None) -> BlockHashType:

381 """Computes a hash value corresponding to the contents of a block and

382 the contents of the preceding block(s). The hash value is used for

383 prefix caching. We use LRU cache for this function to avoid recomputing

@@ -410,7 +410,7 @@ def hash_block_tokens(

410

411

412 def hash_request_tokens(block_size: int,

413 - request: Request) -> List[BlockHashType]:

413 + request: Request) -> list[BlockHashType]:

414 """Computes hash values of a chain of blocks given a sequence of

415 token IDs. The hash value is used for prefix caching.

416

@@ -554,8 +554,8 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,

554

555

556 def get_kv_cache_configs(vllm_config: VllmConfig,

557 - kv_cache_specs: List[KVCacheSpec],

558 - available_memory: int) -> List[KVCacheConfig]:

557 + kv_cache_specs: list[KVCacheSpec],

558 + available_memory: int) -> list[KVCacheConfig]:

559 """

560 Generates the KV cache configuration for a model

561 TODO: support hybrid models with more than one type of KV cache.

vllm/v1/core/scheduler.py

+23 -22

@@ -2,7 +2,8 @@

2

3 import time

4 from collections import deque

5 -from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union

5 +from collections.abc import Iterable

6 +from typing import Optional, Union

7

8 from vllm.config import (CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig,

9 SpeculativeConfig)

@@ -57,24 +58,24 @@ class Scheduler:

58 self.block_size = self.cache_config.block_size

59

60 # req_id -> Request

60 - self.requests: Dict[str, Request] = {}

61 + self.requests: dict[str, Request] = {}

62 # Priority queues for requests.

62 - self.waiting: Deque[Request] = deque()

63 - self.running: List[Request] = []

63 + self.waiting: deque[Request] = deque()

64 + self.running: list[Request] = []

65 # The requests that have been scheduled and are being executed

66 # by the executor.

66 - self.scheduled_req_ids: Set[str] = set()

67 + self.scheduled_req_ids: set[str] = set()

68

69 # The request IDs that are finished in between the previous and the

70 # current steps. This is used to notify the workers about the finished

71 # requests so that they can free the cached states for those requests.

72 # This is flushed at the end of each scheduling step.

72 - self.finished_req_ids: Set[str] = set()

73 + self.finished_req_ids: set[str] = set()

74

75 # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating

76 # them at each scheduling step.

77 # Request id -> CachedRequestData

77 - self._cached_reqs_data: Dict[str, CachedRequestData] = {}

78 + self._cached_reqs_data: dict[str, CachedRequestData] = {}

79

80 # Encoder-related.

81 # Calculate encoder cache size if applicable

@@ -108,19 +109,19 @@ class Scheduler:

109 # chunked prefills, prefix caching, speculative decoding,

110 # and the "jump decoding" optimization in the future.

111

111 - scheduled_new_reqs: List[Request] = []

112 - scheduled_resumed_reqs: List[Request] = []

113 - scheduled_running_reqs: List[Request] = []

114 - preempted_reqs: List[Request] = []

112 + scheduled_new_reqs: list[Request] = []

113 + scheduled_resumed_reqs: list[Request] = []

114 + scheduled_running_reqs: list[Request] = []

115 + preempted_reqs: list[Request] = []

116

116 - req_to_new_block_ids: Dict[str, List[int]] = {}

117 - num_scheduled_tokens: Dict[str, int] = {}

117 + req_to_new_block_ids: dict[str, list[int]] = {}

118 + num_scheduled_tokens: dict[str, int] = {}

119 token_budget = self.max_num_scheduled_tokens

120 # Encoder-related.

120 - scheduled_encoder_inputs: Dict[str, List[int]] = {}

121 + scheduled_encoder_inputs: dict[str, list[int]] = {}

122 encoder_budget = self.max_num_encoder_input_tokens

123 # Spec decode-related.

123 - scheduled_spec_decode_tokens: Dict[str, List[int]] = {}

124 + scheduled_spec_decode_tokens: dict[str, list[int]] = {}

125

126 # For logging.

127 scheduled_timestamp = time.monotonic()

@@ -211,7 +212,7 @@ class Scheduler:

212 encoder_budget = new_encoder_budget

213

214 # Record the LoRAs in scheduled_running_reqs

214 - requested_loras: Set[int] = set()

215 + requested_loras: set[int] = set()

216 if self.lora_config:

217 requested_loras = set(

218 req.lora_request.lora_int_id for req in scheduled_running_reqs

@@ -378,7 +379,7 @@ class Scheduler:

379 request: Request,

380 num_scheduled_tokens: int,

381 num_scheduled_spec_tokens: int,

381 - new_block_ids: List[int],

382 + new_block_ids: list[int],

383 resumed_from_preemption: bool,

384 ) -> "CachedRequestData":

385 # OPTIMIZATION: Cache the CachedRequestData objects to avoid creating

@@ -407,7 +408,7 @@ class Scheduler:

408 num_computed_tokens: int,

409 num_new_tokens: int,

410 encoder_budget: int,

410 - ) -> Tuple[List[int], int, int]:

411 + ) -> tuple[list[int], int, int]:

412 """

413 Determine which encoder inputs need to be scheduled in the current step,

414 and update `num_new_tokens` and encoder token budget accordingly.

@@ -427,7 +428,7 @@ class Scheduler:

428 if not request.has_encoder_inputs():

429 return [], num_new_tokens, encoder_budget

430

430 - encoder_inputs_to_schedule: List[int] = []

431 + encoder_inputs_to_schedule: list[int] = []

432 mm_positions = request.mm_positions

433 assert mm_positions is not None

434 assert len(mm_positions) > 0

@@ -482,8 +483,8 @@ class Scheduler:

483 prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict

484 num_scheduled_tokens = scheduler_output.num_scheduled_tokens

485

485 - new_running: List[Request] = []

486 - outputs: List[EngineCoreOutput] = []

486 + new_running: list[Request] = []

487 + outputs: list[EngineCoreOutput] = []

488

489 # NOTE(woosuk): As len(self.running) can be up to 1K or more, the below

490 # loop can be a performance bottleneck. We should do our best to avoid

@@ -543,7 +544,7 @@ class Scheduler:

544

545 stopped = False

546 new_logprobs = None

546 - new_token_ids: List[int] = []

547 + new_token_ids: list[int] = []

548

549 if request.num_computed_tokens >= request.num_tokens:

550 for output_token_id in generated_token_ids:

vllm/v1/core/scheduler_output.py

+21 -21

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple

4 +from typing import TYPE_CHECKING, Optional

5

6 if TYPE_CHECKING:

7 from vllm.lora.request import LoRARequest

@@ -15,13 +15,13 @@ if TYPE_CHECKING:

15 class NewRequestData:

16

17 req_id: str

18 - prompt_token_ids: List[int]

18 + prompt_token_ids: list[int]

19 prompt: Optional[str]

20 - mm_inputs: List["MultiModalKwargs"]

21 - mm_hashes: List[str]

22 - mm_positions: List["PlaceholderRange"]

20 + mm_inputs: list["MultiModalKwargs"]

21 + mm_hashes: list[str]

22 + mm_positions: list["PlaceholderRange"]

23 sampling_params: "SamplingParams"

24 - block_ids: List[int]

24 + block_ids: list[int]

25 num_computed_tokens: int

26 lora_request: Optional["LoRARequest"]

27

@@ -29,7 +29,7 @@ class NewRequestData:

29 def from_request(

30 cls,

31 request: "Request",

32 - block_ids: List[int],

32 + block_ids: list[int],

33 ) -> "NewRequestData":

34 return cls(

35 req_id=request.request_id,

@@ -53,8 +53,8 @@ class CachedRequestData:

53 # the request's block IDs. If True, new_block_ids will be used as the

54 # request's block IDs instead of appending to the existing block IDs.

55 resumed_from_preemption: bool

56 - new_token_ids: List[int]

57 - new_block_ids: List[int]

56 + new_token_ids: list[int]

57 + new_block_ids: list[int]

58 num_computed_tokens: int

59

60 @classmethod

@@ -62,8 +62,8 @@ class CachedRequestData:

62 cls,

63 request: "Request",

64 resumed_from_preemption: bool,

65 - new_token_ids: List[int],

66 - new_block_ids: List[int],

65 + new_token_ids: list[int],

66 + new_block_ids: list[int],

67 ) -> "CachedRequestData":

68 return cls(

69 req_id=request.request_id,

@@ -77,29 +77,29 @@ class CachedRequestData:

77 @dataclass

78 class SchedulerOutput:

79

80 - # List of the requests that are scheduled for the first time.

80 + # list of the requests that are scheduled for the first time.

81 # We cache the request's data in each worker process, so that we don't

82 # need to re-send it every scheduling step.

83 - scheduled_new_reqs: List[NewRequestData]

84 - # List of the requests that have been scheduled before.

83 + scheduled_new_reqs: list[NewRequestData]

84 + # list of the requests that have been scheduled before.

85 # Since the request's data is already cached in the worker processes,

86 # we only send the diff to minimize the communication cost.

87 - scheduled_cached_reqs: List[CachedRequestData]

87 + scheduled_cached_reqs: list[CachedRequestData]

88

89 # req_id -> num_scheduled_tokens

90 # Number of tokens scheduled for each request.

91 - num_scheduled_tokens: Dict[str, int]

91 + num_scheduled_tokens: dict[str, int]

92 # Total number of tokens scheduled for all requests.

93 # Equal to sum(num_scheduled_tokens.values())

94 total_num_scheduled_tokens: int

95 # req_id -> spec_token_ids

96 # If a request does not have any spec decode tokens, it will not be

97 # included in the dictionary.

98 - scheduled_spec_decode_tokens: Dict[str, List[int]]

98 + scheduled_spec_decode_tokens: dict[str, list[int]]

99 # req_id -> encoder input indices that need processing.

100 # E.g., if a request has [0, 1], it could mean the vision encoder needs

101 # to process that the request's 0-th and 1-th images in the current step.

102 - scheduled_encoder_inputs: Dict[str, List[int]]

102 + scheduled_encoder_inputs: dict[str, list[int]]

103 # Number of common prefix blocks for all requests.

104 # This can be used for cascade attention.

105 num_common_prefix_blocks: int

@@ -107,7 +107,7 @@ class SchedulerOutput:

107 # Request IDs that are finished in between the previous and the current

108 # steps. This is used to notify the workers about the finished requests

109 # so that they can free the cached states for those requests.

110 - finished_req_ids: Set[str]

111 - # List of (req_id, encoder_input_index) tuples.

110 + finished_req_ids: set[str]

111 + # list of (req_id, encoder_input_index) tuples.

112 # Used to free the encoder cache.

113 - free_encoder_input_ids: List[Tuple[str, int]]

113 + free_encoder_input_ids: list[tuple[str, int]]

vllm/v1/engine/__init__.py

+8 -8

@@ -2,7 +2,7 @@

2

3 import enum

4 import time

5 -from typing import Any, List, Optional, Union

5 +from typing import Any, Optional, Union

6

7 import msgspec

8

@@ -51,10 +51,10 @@ class EngineCoreRequest(

51 # NOTE(ywang96): original text prompt is needed when a request is added to

52 # Detokenizer, but set to None when it is added to EngineCoreClient.

53 prompt: Optional[str]

54 - prompt_token_ids: List[int]

55 - mm_inputs: Optional[List[Optional[MultiModalKwargs]]]

56 - mm_hashes: Optional[List[str]]

57 - mm_placeholders: Optional[List[PlaceholderRange]]

54 + prompt_token_ids: list[int]

55 + mm_inputs: Optional[list[Optional[MultiModalKwargs]]]

56 + mm_hashes: Optional[list[str]]

57 + mm_placeholders: Optional[list[PlaceholderRange]]

58 sampling_params: SamplingParams

59 eos_token_id: Optional[int]

60 arrival_time: float

@@ -93,14 +93,14 @@ class EngineCoreOutput(

93 gc=False): # type: ignore[call-arg]

94

95 request_id: str

96 - new_token_ids: List[int]

96 + new_token_ids: list[int]

97

98 new_logprobs: Optional[LogprobsLists] = None

99 new_prompt_logprobs_tensors: Optional[LogprobsTensors] = None

100

101 finish_reason: Optional[FinishReason] = None

102 stop_reason: Union[int, str, None] = None

103 - events: Optional[List[EngineCoreEvent]] = None

103 + events: Optional[list[EngineCoreEvent]] = None

104

105 @property

106 def finished(self) -> bool:

@@ -129,7 +129,7 @@ class EngineCoreOutputs(

129 # e.g. columnwise layout

130

131 # [num_reqs]

132 - outputs: List[EngineCoreOutput] = []

132 + outputs: list[EngineCoreOutput] = []

133 scheduler_stats: Optional[SchedulerStats] = None

134 timestamp: float = 0.0

135

vllm/v1/engine/async_llm.py

+5 -4

@@ -2,7 +2,8 @@

2

3 import asyncio

4 import os

5 -from typing import AsyncGenerator, List, Mapping, Optional, Set, Type, Union

5 +from collections.abc import AsyncGenerator, Mapping

6 +from typing import Optional, Union

7

8 import numpy as np

9

@@ -39,7 +40,7 @@ class AsyncLLM(EngineClient):

40 def __init__(

41 self,

42 vllm_config: VllmConfig,

42 - executor_class: Type[Executor],

43 + executor_class: type[Executor],

44 log_stats: bool,

45 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,

46 input_registry: InputRegistry = INPUT_REGISTRY,

@@ -54,7 +55,7 @@ class AsyncLLM(EngineClient):

55

56 self.log_requests = log_requests

57 self.log_stats = log_stats

57 - self.stat_loggers: List[StatLoggerBase] = []

58 + self.stat_loggers: list[StatLoggerBase] = []

59 if self.log_stats:

60 self.stat_loggers.extend([

61 LoggingStatLogger(),

@@ -400,7 +401,7 @@ class AsyncLLM(EngineClient):

401 """Remove an already loaded LoRA adapter."""

402 return await self.engine_core.remove_lora_async(lora_id)

403

403 - async def list_loras(self) -> Set[int]:

404 + async def list_loras(self) -> set[int]:

405 """List all registered adapters."""

406 return await self.engine_core.list_loras_async()

407

vllm/v1/engine/core.py

+8 -8

@@ -7,7 +7,7 @@ import time

7 from concurrent.futures import Future

8 from inspect import isclass, signature

9 from multiprocessing.connection import Connection

10 -from typing import Any, List, Optional, Set, Tuple, Type

10 +from typing import Any, Optional

11

12 import msgspec

13 import psutil

@@ -42,7 +42,7 @@ class EngineCore:

42 def __init__(

43 self,

44 vllm_config: VllmConfig,

45 - executor_class: Type[Executor],

45 + executor_class: type[Executor],

46 log_stats: bool,

47 ):

48 assert vllm_config.model_config.runner_type != "pooling"

@@ -80,7 +80,7 @@ class EngineCore:

80 # schedule and execute batches, and is required by pipeline parallelism

81 # to eliminate pipeline bubbles.

82 self.batch_queue_size = self.model_executor.max_concurrent_batches

83 - self.batch_queue: Optional[queue.Queue[Tuple[Future[ModelRunnerOutput],

83 + self.batch_queue: Optional[queue.Queue[tuple[Future[ModelRunnerOutput],

84 SchedulerOutput]]] = None

85 if self.batch_queue_size > 1:

86 logger.info("Batch queue is enabled with size %d",

@@ -88,7 +88,7 @@ class EngineCore:

88 self.batch_queue = queue.Queue(self.batch_queue_size)

89

90 def _initialize_kv_caches(self,

91 - vllm_config: VllmConfig) -> Tuple[int, int]:

91 + vllm_config: VllmConfig) -> tuple[int, int]:

92 start = time.time()

93

94 # Get all kv cache needed by the model

@@ -134,7 +134,7 @@ class EngineCore:

134

135 self.scheduler.add_request(req)

136

137 - def abort_requests(self, request_ids: List[str]):

137 + def abort_requests(self, request_ids: list[str]):

138 """Abort requests from the scheduler."""

139

140 # TODO: The scheduler doesn't really need to know the

@@ -228,7 +228,7 @@ class EngineCore:

228 def remove_lora(self, lora_id: int) -> bool:

229 return self.model_executor.remove_lora(lora_id)

230

231 - def list_loras(self) -> Set[int]:

231 + def list_loras(self) -> set[int]:

232 return self.model_executor.list_loras()

233

234 def pin_lora(self, lora_id: int) -> bool:

@@ -244,7 +244,7 @@ class EngineCoreProc(EngineCore):

244 output_path: str,

245 ready_pipe: Connection,

246 vllm_config: VllmConfig,

247 - executor_class: Type[Executor],

247 + executor_class: type[Executor],

248 log_stats: bool,

249 ):

250 super().__init__(vllm_config, executor_class, log_stats)

@@ -254,7 +254,7 @@ class EngineCoreProc(EngineCore):

254 # and to overlap some serialization/deserialization with the

255 # model forward pass.

256 # Threads handle Socket <-> Queues and core_busy_loop uses Queue.

257 - self.input_queue: queue.Queue[Tuple[EngineCoreRequestType,

257 + self.input_queue: queue.Queue[tuple[EngineCoreRequestType,

258 Any]] = queue.Queue()

259 self.output_queue: queue.Queue[EngineCoreOutputs] = queue.Queue()

260 threading.Thread(target=self.process_input_socket,

vllm/v1/engine/core_client.py

+17 -17

@@ -10,7 +10,7 @@ from abc import ABC, abstractmethod

10 from concurrent.futures import Future

11 from dataclasses import dataclass

12 from threading import Thread

13 -from typing import Any, Dict, List, Optional, Set, Type, Union

13 +from typing import Any, Optional, Union

14

15 import zmq

16 import zmq.asyncio

@@ -48,7 +48,7 @@ class EngineCoreClient(ABC):

48 multiprocess_mode: bool,

49 asyncio_mode: bool,

50 vllm_config: VllmConfig,

51 - executor_class: Type[Executor],

51 + executor_class: type[Executor],

52 log_stats: bool,

53 ) -> "EngineCoreClient":

54

@@ -94,7 +94,7 @@ class EngineCoreClient(ABC):

94 async def execute_dummy_batch_async(self) -> None:

95 raise NotImplementedError

96

97 - def abort_requests(self, request_ids: List[str]) -> None:

97 + def abort_requests(self, request_ids: list[str]) -> None:

98 raise NotImplementedError

99

100 def add_lora(self, lora_request: LoRARequest) -> bool:

@@ -103,7 +103,7 @@ class EngineCoreClient(ABC):

103 def remove_lora(self, lora_id: int) -> bool:

104 raise NotImplementedError

105

106 - def list_loras(self) -> Set[int]:

106 + def list_loras(self) -> set[int]:

107 raise NotImplementedError

108

109 def pin_lora(self, lora_id: int) -> bool:

@@ -127,7 +127,7 @@ class EngineCoreClient(ABC):

127 async def wake_up_async(self) -> None:

128 raise NotImplementedError

129

130 - async def abort_requests_async(self, request_ids: List[str]) -> None:

130 + async def abort_requests_async(self, request_ids: list[str]) -> None:

131 raise NotImplementedError

132

133 async def add_lora_async(self, lora_request: LoRARequest) -> bool:

@@ -136,7 +136,7 @@ class EngineCoreClient(ABC):

136 async def remove_lora_async(self, lora_id: int) -> bool:

137 raise NotImplementedError

138

139 - async def list_loras_async(self) -> Set[int]:

139 + async def list_loras_async(self) -> set[int]:

140 raise NotImplementedError

141

142 async def pin_lora_async(self, lora_id: int) -> bool:

@@ -162,7 +162,7 @@ class InprocClient(EngineCoreClient):

162 def add_request(self, request: EngineCoreRequest) -> None:

163 self.engine_core.add_request(request)

164

165 - def abort_requests(self, request_ids: List[str]) -> None:

165 + def abort_requests(self, request_ids: list[str]) -> None:

166 if len(request_ids) > 0:

167 self.engine_core.abort_requests(request_ids)

168

@@ -190,7 +190,7 @@ class InprocClient(EngineCoreClient):

190 def remove_lora(self, lora_id: int) -> bool:

191 return self.engine_core.remove_lora(lora_id)

192

193 - def list_loras(self) -> Set[int]:

193 + def list_loras(self) -> set[int]:

194 return self.engine_core.list_loras()

195

196 def pin_lora(self, lora_id: int) -> bool:

@@ -239,7 +239,7 @@ class MPClient(EngineCoreClient):

239 self,

240 asyncio_mode: bool,

241 vllm_config: VllmConfig,

242 - executor_class: Type[Executor],

242 + executor_class: type[Executor],

243 log_stats: bool,

244 ):

245 # The child processes will send SIGUSR1 when unrecoverable

@@ -293,14 +293,14 @@ class MPClient(EngineCoreClient):

293

294 self.output_socket = resources.output_socket

295 self.input_socket = resources.input_socket

296 - self.utility_results: Dict[int, AnyFuture] = {}

296 + self.utility_results: dict[int, AnyFuture] = {}

297

298 def shutdown(self):

299 self._finalizer()

300

301

302 def _process_utility_output(output: UtilityOutput,

303 - utility_results: Dict[int, AnyFuture]):

303 + utility_results: dict[int, AnyFuture]):

304 """Set the result from a utility method in the waiting future"""

305 future = utility_results.pop(output.call_id)

306 if output.failure_message is not None:

@@ -312,7 +312,7 @@ def _process_utility_output(output: UtilityOutput,

312 class SyncMPClient(MPClient):

313 """Synchronous client for multi-proc EngineCore."""

314

315 - def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],

315 + def __init__(self, vllm_config: VllmConfig, executor_class: type[Executor],

316 log_stats: bool):

317 super().__init__(

318 asyncio_mode=False,

@@ -373,7 +373,7 @@ class SyncMPClient(MPClient):

373 request.prompt = None

374 self._send_input(EngineCoreRequestType.ADD, request)

375

376 - def abort_requests(self, request_ids: List[str]) -> None:

376 + def abort_requests(self, request_ids: list[str]) -> None:

377 if len(request_ids) > 0:

378 self._send_input(EngineCoreRequestType.ABORT, request_ids)

379

@@ -389,7 +389,7 @@ class SyncMPClient(MPClient):

389 def remove_lora(self, lora_id: int) -> bool:

390 return self._call_utility("remove_lora", lora_id)

391

392 - def list_loras(self) -> Set[int]:

392 + def list_loras(self) -> set[int]:

393 return self._call_utility("list_loras")

394

395 def pin_lora(self, lora_id: int) -> bool:

@@ -408,7 +408,7 @@ class SyncMPClient(MPClient):

408 class AsyncMPClient(MPClient):

409 """Asyncio-compatible client for multi-proc EngineCore."""

410

411 - def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],

411 + def __init__(self, vllm_config: VllmConfig, executor_class: type[Executor],

412 log_stats: bool):

413 super().__init__(

414 asyncio_mode=True,

@@ -471,7 +471,7 @@ class AsyncMPClient(MPClient):

471 request.prompt = None

472 await self._send_input(EngineCoreRequestType.ADD, request)

473

474 - async def abort_requests_async(self, request_ids: List[str]) -> None:

474 + async def abort_requests_async(self, request_ids: list[str]) -> None:

475 if len(request_ids) > 0:

476 await self._send_input(EngineCoreRequestType.ABORT, request_ids)

477

@@ -496,7 +496,7 @@ class AsyncMPClient(MPClient):

496 async def remove_lora_async(self, lora_id: int) -> bool:

497 return await self._call_utility_async("remove_lora", lora_id)

498

499 - async def list_loras_async(self) -> Set[int]:

499 + async def list_loras_async(self) -> set[int]:

500 return await self._call_utility_async("list_loras")

501

502 async def pin_lora_async(self, lora_id: int) -> bool:

vllm/v1/engine/detokenizer.py

+6 -6

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import List, Optional

4 +from typing import Optional

5

6 from vllm.engine.output_processor.stop_checker import StopChecker

7 from vllm.logger import init_logger

@@ -17,12 +17,12 @@ class IncrementalDetokenizer:

17

18 # Generation data

19 output_text: str

20 - tokens: List[str]

21 - token_ids: List[int]

20 + tokens: list[str]

21 + token_ids: list[int]

22 prompt_len: int

23

24 # Stop strings

25 - stop: List[str]

25 + stop: list[str]

26 include_stop_str_in_output: bool

27

28 # Metadata for incremental detokenization

@@ -41,7 +41,7 @@ class IncrementalDetokenizer:

41 _last_output_text_offset: int = 0

42

43 @property

44 - def output_token_ids(self) -> List[int]:

44 + def output_token_ids(self) -> list[int]:

45 return self.token_ids[self.prompt_len:]

46

47 @classmethod

@@ -84,7 +84,7 @@ class IncrementalDetokenizer:

84 stop_buffer_length=stop_buffer_length,

85 )

86

87 - def update(self, new_token_ids: List[int]) -> Optional[str]:

87 + def update(self, new_token_ids: list[int]) -> Optional[str]:

88 """

89 Update RequestState for the request_id by:

90 1) Detokenize the new token ids incrementally.

vllm/v1/engine/llm_engine.py

+9 -8

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List, Mapping, Optional, Set, Type, Union

3 +from collections.abc import Mapping

4 +from typing import Optional, Union

5

6 from typing_extensions import TypeVar

7

@@ -36,10 +37,10 @@ class LLMEngine:

37 def __init__(

38 self,

39 vllm_config: VllmConfig,

39 - executor_class: Type[Executor],

40 + executor_class: type[Executor],

41 log_stats: bool,

42 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,

42 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,

43 + stat_loggers: Optional[dict[str, StatLoggerBase]] = None,

44 input_registry: InputRegistry = INPUT_REGISTRY,

45 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

46 use_cached_outputs: bool = False,

@@ -97,7 +98,7 @@ class LLMEngine:

98 cls,

99 engine_args: EngineArgs,

100 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,

100 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,

101 + stat_loggers: Optional[dict[str, StatLoggerBase]] = None,

102 enable_multiprocessing: bool = False,

103 ) -> "LLMEngine":

104 """Creates an LLM engine from the engine arguments."""

@@ -139,7 +140,7 @@ class LLMEngine:

140 def validate_outputs(cls, outputs, output_type):

141 return outputs

142

142 - def abort_request(self, request_ids: List[str]) -> None:

143 + def abort_request(self, request_ids: list[str]) -> None:

144 """Remove request_ids from EngineCore and Detokenizer."""

145

146 self.engine_core.abort_requests(request_ids)

@@ -199,7 +200,7 @@ class LLMEngine:

200 # 3) Add the request to EngineCore.

201 self.engine_core.add_request(request)

202

202 - def step(self) -> List[RequestOutput]:

203 + def step(self) -> list[RequestOutput]:

204

205 if self.should_execute_dummy_batch:

206 self.should_execute_dummy_batch = False

@@ -241,7 +242,7 @@ class LLMEngine:

242

243 def get_tokenizer_group(

244 self,

244 - group_type: Type[_G] = BaseTokenizerGroup,

245 + group_type: type[_G] = BaseTokenizerGroup,

246 ) -> _G:

247 tokenizer_group = self.tokenizer

248

@@ -263,7 +264,7 @@ class LLMEngine:

264 """Remove an already loaded LoRA adapter."""

265 return self.engine_core.remove_lora(lora_id)

266

266 - def list_loras(self) -> Set[int]:

267 + def list_loras(self) -> set[int]:

268 """List all registered adapters."""

269 return self.engine_core.list_loras()

270

vllm/v1/engine/logprobs.py

+6 -6

@@ -2,7 +2,7 @@

2

3 import itertools

4 from dataclasses import dataclass

5 -from typing import Dict, List, Optional

5 +from typing import Optional

6

7 from vllm.logger import init_logger

8 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs

@@ -151,12 +151,12 @@ class LogprobsProcessor:

151

152 @staticmethod

153 def _make_logprob_dict(

154 - logprobs: List[float],

155 - logprob_token_ids: List[int],

156 - decoded_tokens: List[str],

154 + logprobs: list[float],

155 + logprob_token_ids: list[int],

156 + decoded_tokens: list[str],

157 rank: int,

158 num_logprobs: int,

159 - ) -> Dict[int, Logprob]:

159 + ) -> dict[int, Logprob]:

160 """Make a Logprob dictionary for a position.

161

162 Args:

@@ -168,7 +168,7 @@ class LogprobsProcessor:

168 by the user (in addition to sampled logprob)

169

170 Returns:

171 - Dict[token id, Logprob]

171 + dict[token id, Logprob]

172 """

173

174 # We do not need a special case for the sampled token

vllm/v1/engine/mm_input_cache.py

+9 -9

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Any, Dict, List, Optional

3 +from typing import Any, Optional

4

5 from vllm.config import ModelConfig

6 from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE

@@ -68,10 +68,10 @@ class MMInputCacheClient:

68 def process_inputs(

69 self,

70 mm_data: MultiModalDataDict,

71 - mm_hashes: Optional[List[str]],

72 - mm_processor_kwargs: Optional[Dict[str, Any]],

73 - precomputed_mm_inputs: Optional[List[MultiModalKwargs]],

74 - ) -> List[MultiModalKwargs]:

71 + mm_hashes: Optional[list[str]],

72 + mm_processor_kwargs: Optional[dict[str, Any]],

73 + precomputed_mm_inputs: Optional[list[MultiModalKwargs]],

74 + ) -> list[MultiModalKwargs]:

75 if precomputed_mm_inputs is None:

76 image_inputs = mm_data["image"]

77 if not isinstance(image_inputs, list):

@@ -88,7 +88,7 @@ class MMInputCacheClient:

88 # Process each image input separately, so that later we can schedule

89 # them in a fine-grained manner.

90 # Apply caching (if enabled) and reuse precomputed inputs (if provided)

91 - ret_inputs: List[MultiModalKwargs] = []

91 + ret_inputs: list[MultiModalKwargs] = []

92 for input_id in range(num_inputs):

93 if self.mm_debug_cache_hit_ratio_steps is not None:

94 self.cache_hit_ratio(self.mm_debug_cache_hit_ratio_steps)

@@ -133,9 +133,9 @@ class MMInputCacheServer:

133

134 def get_and_update(

135 self,

136 - mm_inputs: List[Optional[MultiModalKwargs]],

137 - mm_hashes: List[str],

138 - ) -> List[MultiModalKwargs]:

136 + mm_inputs: list[Optional[MultiModalKwargs]],

137 + mm_hashes: list[str],

138 + ) -> list[MultiModalKwargs]:

139 assert len(mm_inputs) == len(mm_hashes)

140

141 if not self.use_cache:

vllm/v1/engine/output_processor.py

+10 -10

@@ -2,7 +2,7 @@

2

3 import asyncio

4 from dataclasses import dataclass

5 -from typing import Dict, List, Optional, Union

5 +from typing import Optional, Union

6

7 from vllm.outputs import RequestOutput

8 from vllm.sampling_params import RequestOutputKind

@@ -18,8 +18,8 @@ from vllm.v1.metrics.stats import (IterationStats, LoRARequestStates,

18 @dataclass

19 class OutputProcessorOutput:

20

21 - request_outputs: List[RequestOutput]

22 - reqs_to_abort: List[str]

21 + request_outputs: list[RequestOutput]

22 + reqs_to_abort: list[str]

23

24

25 class RequestState:

@@ -30,7 +30,7 @@ class RequestState:

30 lora_name: Optional[str],

31 output_kind: RequestOutputKind,

32 prompt: Optional[str],

33 - prompt_token_ids: List[int],

33 + prompt_token_ids: list[int],

34 logprobs_processor: LogprobsProcessor,

35 detokenizer: IncrementalDetokenizer,

36 arrival_time: float,

@@ -90,7 +90,7 @@ class OutputProcessor:

90 ):

91 self.log_stats = log_stats

92 self.tokenizer = tokenizer

93 - self.request_states: Dict[str, RequestState] = {}

93 + self.request_states: dict[str, RequestState] = {}

94 self.lora_states = LoRARequestStates()

95

96 def is_request_active(self, request_id: str) -> bool:

@@ -104,7 +104,7 @@ class OutputProcessor:

104

105 def abort_requests(

106 self,

107 - request_ids: List[str],

107 + request_ids: list[str],

108 ) -> None:

109 for request_id in request_ids:

110 req_state = self.request_states.pop(request_id, None)

@@ -130,7 +130,7 @@ class OutputProcessor:

130

131 def process_outputs(

132 self,

133 - engine_core_outputs: List[EngineCoreOutput],

133 + engine_core_outputs: list[EngineCoreOutput],

134 engine_core_timestamp: Optional[float] = None,

135 iteration_stats: Optional[IterationStats] = None,

136 ) -> OutputProcessorOutput:

@@ -158,8 +158,8 @@ class OutputProcessor:

158 **********************************************************

159 """

160

161 - request_outputs: List[RequestOutput] = []

162 - reqs_to_abort: List[str] = []

161 + request_outputs: list[RequestOutput] = []

162 + reqs_to_abort: list[str] = []

163 for engine_core_output in engine_core_outputs:

164 req_id = engine_core_output.request_id

165 req_state = self.request_states.get(req_id)

@@ -265,7 +265,7 @@ class OutputProcessor:

265 @staticmethod

266 def _make_request_output(

267 request_state: RequestState,

268 - new_token_ids: List[int],

268 + new_token_ids: list[int],

269 finish_reason: Optional[FinishReason],

270 stop_reason: Union[int, str, None],

271 ) -> Optional[RequestOutput]:

vllm/v1/engine/parallel_sampling.py

+8 -8

@@ -1,8 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 +from collections.abc import AsyncGenerator, Mapping

4 from copy import copy

4 -from typing import (AsyncGenerator, Dict, List, Mapping, Optional, Protocol,

5 - Tuple, Union)

5 +from typing import Optional, Protocol, Union

6

7 from vllm.inputs import PromptType

8 from vllm.lora.request import LoRARequest

@@ -137,7 +137,7 @@ class ParallelSamplingRequest:

137 key=lambda x: x.index)

138 return self.request_output

139

140 - def get_child_info(self, index: int) -> Tuple[str, SamplingParams]:

140 + def get_child_info(self, index: int) -> tuple[str, SamplingParams]:

141 """Get child request ID and sampling params.

142

143 Args:

@@ -237,9 +237,9 @@ class SyncParallelSamplingManager:

237

238 def __init__(self):

239 # Parent req ID -> parent request manager

240 - self.parent_reqs: Dict[str, ParallelSamplingRequest] = {}

240 + self.parent_reqs: dict[str, ParallelSamplingRequest] = {}

241 # Child req ID -> (child req index, parent req ID)

242 - self.child_reqs: Dict[str, Tuple[int, str]] = {}

242 + self.child_reqs: dict[str, tuple[int, str]] = {}

243

244 def _register_parent_request(self, req: ParallelSamplingRequest) -> None:

245 """Register parallel sampling parent request."""

@@ -299,8 +299,8 @@ class SyncParallelSamplingManager:

299

300 def step(

301 self,

302 - outputs: List[RequestOutput],

303 - ) -> List[RequestOutput]:

302 + outputs: list[RequestOutput],

303 + ) -> list[RequestOutput]:

304 """Build parallel sampling request outputs.

305

306 Extract child request outputs, aggregate them

@@ -355,7 +355,7 @@ async def generate_parallel_sampling_async(

355 parent_req = ParallelSamplingRequest(request_id, sampling_params)

356

357 # Aggregate generators for n child requests

358 - gens: List[AsyncGenerator[RequestOutput, None]] = []

358 + gens: list[AsyncGenerator[RequestOutput, None]] = []

359 for idx in range(parent_req.n):

360 child_req_id, child_params = parent_req.get_child_info(idx)

361 child_gen = generate(

vllm/v1/engine/processor.py

+2 -1

@@ -1,7 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import time

4 -from typing import Mapping, Optional, Union

4 +from collections.abc import Mapping

5 +from typing import Optional, Union

6

7 from vllm.config import CacheConfig, LoRAConfig, ModelConfig

8 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,

vllm/v1/executor/abstract.py

+5 -5

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from concurrent.futures import Future

4 -from typing import List, Type, Union

4 +from typing import Union

5

6 import torch

7 import torch.distributed as dist

@@ -22,8 +22,8 @@ class Executor(ExecutorBase):

22 For methods shared by v0 and v1, define them in ExecutorBase"""

23

24 @staticmethod

25 - def get_class(vllm_config: VllmConfig) -> Type["Executor"]:

26 - executor_class: Type[Executor]

25 + def get_class(vllm_config: VllmConfig) -> type["Executor"]:

26 + executor_class: type[Executor]

27 parallel_config = vllm_config.parallel_config

28 distributed_executor_backend = (

29 parallel_config.distributed_executor_backend)

@@ -53,7 +53,7 @@ class Executor(ExecutorBase):

53 return executor_class

54

55 def initialize_from_config(self,

56 - kv_cache_configs: List[KVCacheConfig]) -> None:

56 + kv_cache_configs: list[KVCacheConfig]) -> None:

57 """

58 Initialize the KV caches and begin the model execution loop of the

59 underlying workers.

@@ -69,7 +69,7 @@ class Executor(ExecutorBase):

69 # operators can be applied to all workers.

70 return min(output)

71

72 - def get_kv_cache_specs(self) -> List[KVCacheSpec]:

72 + def get_kv_cache_specs(self) -> list[KVCacheSpec]:

73 output = self.collective_rpc("get_kv_cache_spec")

74 return output

75

vllm/v1/executor/multiproc_executor.py

+5 -5

@@ -10,7 +10,7 @@ from dataclasses import dataclass

10 from enum import Enum, auto

11 from functools import partial

12 from multiprocessing.process import BaseProcess

13 -from typing import Any, Callable, Dict, List, Optional, Tuple, Union

13 +from typing import Any, Callable, Optional, Union

14

15 import cloudpickle

16 import psutil

@@ -77,7 +77,7 @@ class MultiprocExecutor(Executor):

77 scheduler_output_handle = self.rpc_broadcast_mq.export_handle()

78

79 # Create workers

80 - self.workers: List[WorkerProcHandle] = []

80 + self.workers: list[WorkerProcHandle] = []

81 for rank in range(self.world_size):

82 worker = WorkerProc.make_worker_process(self.vllm_config, rank,

83 rank,

@@ -94,8 +94,8 @@ class MultiprocExecutor(Executor):

94 def collective_rpc(self,

95 method: Union[str, Callable],

96 timeout: Optional[float] = None,

97 - args: Tuple = (),

98 - kwargs: Optional[Dict] = None) -> List[Any]:

97 + args: tuple = (),

98 + kwargs: Optional[dict] = None) -> list[Any]:

99 start_time = time.monotonic()

100 kwargs = kwargs or {}

101

@@ -208,7 +208,7 @@ class WorkerProc:

208 self.rank = rank

209 wrapper = WorkerWrapperBase(vllm_config=vllm_config, rpc_rank=rank)

210 # TODO: move `init_worker` to executor level as a collective rpc call

211 - all_kwargs: List[Dict] = [

211 + all_kwargs: list[dict] = [

212 {} for _ in range(vllm_config.parallel_config.world_size)

213 ]

214 all_kwargs[rank] = {

vllm/v1/kv_cache_interface.py

+3 -4

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Dict, List

4

5 import torch

6

@@ -74,7 +73,7 @@ class FullAttentionSpec(KVCacheSpecBase):

73 return cdiv(num_tokens, self.block_size) * self.page_size_bytes

74

75

77 -KVCacheSpec = Dict[str, KVCacheSpecBase]

76 +KVCacheSpec = dict[str, KVCacheSpecBase]

77

78

79 @dataclass

@@ -95,7 +94,7 @@ class KVCacheConfig:

94 """The number of KV cache blocks"""

95 num_blocks: int

96 """layer_name -> how to initialize KV cache for that layer"""

98 - tensors: Dict[str, KVCacheTensor]

97 + tensors: dict[str, KVCacheTensor]

98 """

99 A list of kv-cache groups. Each group includes a set of layers with

100 the same kv-cache spec, and the total page_size of layers inside a group

@@ -108,6 +107,6 @@ class KVCacheConfig:

107 3. (not implemented yet) A model with 2 full attention layers and 4 sliding

108 window attention layers: three groups, (full * 2), (sw * 2), (sw * 2).

109 """

111 - groups: List[List[str]]

110 + groups: list[list[str]]

111 """the KVCacheSpec of the model"""

112 kv_cache_spec: KVCacheSpec

vllm/v1/metrics/loggers.py

+9 -9

@@ -2,7 +2,7 @@

2

3 import time

4 from abc import ABC, abstractmethod

5 -from typing import Dict, List, Optional

5 +from typing import Optional

6

7 import numpy as np

8 import prometheus_client

@@ -35,8 +35,8 @@ class LoggingStatLogger(StatLoggerBase):

35 self.last_log_time = now

36

37 # Tracked stats over current local logging interval.

38 - self.num_prompt_tokens: List[int] = []

39 - self.num_generation_tokens: List[int] = []

38 + self.num_prompt_tokens: list[int] = []

39 + self.num_generation_tokens: list[int] = []

40

41 # Prefix cache metrics. TODO: Make the interval configurable.

42 self.prefix_caching_metrics = PrefixCachingMetrics()

@@ -52,7 +52,7 @@ class LoggingStatLogger(StatLoggerBase):

52 self.num_generation_tokens.append(

53 iteration_stats.num_generation_tokens)

54

55 - def _get_throughput(self, tracked_stats: List[int], now: float) -> float:

55 + def _get_throughput(self, tracked_stats: list[int], now: float) -> float:

56 # Compute summary metrics for tracked stats

57 return float(np.sum(tracked_stats) / (now - self.last_log_time))

58

@@ -147,7 +147,7 @@ class PrometheusStatLogger(StatLoggerBase):

147 documentation="Number of generation tokens processed.",

148 labelnames=labelnames).labels(*labelvalues)

149

150 - self.counter_request_success: Dict[FinishReason,

150 + self.counter_request_success: dict[FinishReason,

151 prometheus_client.Counter] = {}

152 counter_request_success_base = prometheus_client.Counter(

153 name="vllm:request_success_total",

@@ -338,14 +338,14 @@ class PrometheusStatLogger(StatLoggerBase):

338 prometheus_client.REGISTRY.unregister(collector)

339

340

341 -def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:

341 +def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:

342 """

343 Builds a list of buckets with increasing powers of 10 multiplied by

344 mantissa values until the value exceeds the specified maximum.

345

346 """

347 exponent = 0

348 - buckets: List[int] = []

348 + buckets: list[int] = []

349 while True:

350 for m in mantissa_lst:

351 value = m * 10**exponent

@@ -356,7 +356,7 @@ def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:

356 exponent += 1

357

358

359 -def build_1_2_5_buckets(max_value: int) -> List[int]:

359 +def build_1_2_5_buckets(max_value: int) -> list[int]:

360 """

361 Example:

362 >>> build_1_2_5_buckets(100)

@@ -365,7 +365,7 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:

365 return build_buckets([1, 2, 5], max_value)

366

367

368 -def build_cudagraph_buckets(vllm_config: VllmConfig) -> List[int]:

368 +def build_cudagraph_buckets(vllm_config: VllmConfig) -> list[int]:

369 if not vllm_config.model_config.enforce_eager:

370 buckets = vllm_config.compilation_config.\

371 cudagraph_capture_sizes.copy()

vllm/v1/metrics/stats.py

+10 -10

@@ -2,7 +2,7 @@

2

3 import time

4 from dataclasses import dataclass, field

5 -from typing import TYPE_CHECKING, Dict, List, Optional, Set

5 +from typing import TYPE_CHECKING, Optional

6

7 if TYPE_CHECKING:

8 from vllm.outputs import RequestOutput

@@ -39,8 +39,8 @@ class SchedulerStats:

39

40 @dataclass

41 class LoRAStats:

42 - waiting_requests: Set[str] = field(default_factory=set)

43 - running_requests: Set[str] = field(default_factory=set)

42 + waiting_requests: set[str] = field(default_factory=set)

43 + running_requests: set[str] = field(default_factory=set)

44

45

46 @dataclass

@@ -81,11 +81,11 @@ class IterationStats:

81 self.num_generation_tokens = 0

82 self.num_prompt_tokens = 0

83 self.num_preempted_reqs = 0

84 - self.finished_requests: List[FinishedRequestStats] = []

85 - self.time_to_first_tokens_iter: List[float] = []

86 - self.time_per_output_tokens_iter: List[float] = []

87 - self.waiting_lora_adapters: Dict[str, int] = {}

88 - self.running_lora_adapters: Dict[str, int] = {}

84 + self.finished_requests: list[FinishedRequestStats] = []

85 + self.time_to_first_tokens_iter: list[float] = []

86 + self.time_per_output_tokens_iter: list[float] = []

87 + self.waiting_lora_adapters: dict[str, int] = {}

88 + self.running_lora_adapters: dict[str, int] = {}

89

90 def _time_since(self, start: float) -> float:

91 """Calculate an interval relative to this iteration's timestamp."""

@@ -132,7 +132,7 @@ class IterationStats:

132 if num_new_generation_tokens > 0:

133 req_stats.last_token_ts = engine_core_timestamp

134

135 - def update_from_events(self, req_id: str, events: List["EngineCoreEvent"],

135 + def update_from_events(self, req_id: str, events: list["EngineCoreEvent"],

136 is_prefilling: bool, req_stats: RequestStateStats,

137 lora_stats: Optional[LoRAStats]):

138 # Avoid circular dependency

@@ -185,7 +185,7 @@ class LoRARequestStates:

185 """Per-LoRA request state stats."""

186

187 def __init__(self):

188 - self.lora_name_to_stats: Dict[str, LoRAStats] = {}

188 + self.lora_name_to_stats: dict[str, LoRAStats] = {}

189

190 def get_stats(self, req_state: 'RequestState') -> Optional[LoRAStats]:

191 if req_state.lora_name is None:

vllm/v1/outputs.py

+10 -10

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Dict, List, NamedTuple, Optional

4 +from typing import NamedTuple, Optional

5

6 import torch

7

@@ -9,11 +9,11 @@ import torch

9 class LogprobsLists(NamedTuple):

10

11 # [num_reqs, max_num_logprobs + 1]

12 - logprob_token_ids: List[List[int]]

12 + logprob_token_ids: list[list[int]]

13 # [num_reqs, max_num_logprobs + 1]

14 - logprobs: List[List[float]]

14 + logprobs: list[list[float]]

15 # [num_reqs]

16 - sampled_token_ranks: List[int]

16 + sampled_token_ranks: list[int]

17

18 def slice(self, start: int, end: int):

19 return LogprobsLists(

@@ -52,23 +52,23 @@ class SamplerOutput:

52

53

54 # ModelRunnerOutput is serialized and sent to the scheduler process.

55 -# This is expensive for torch.Tensor so prefer to use List instead.

55 +# This is expensive for torch.Tensor so prefer to use list instead.

56 @dataclass

57 class ModelRunnerOutput:

58

59 # [num_reqs]

60 - req_ids: List[str]

60 + req_ids: list[str]

61 # req_id -> index

62 - req_id_to_index: Dict[str, int]

62 + req_id_to_index: dict[str, int]

63

64 # num_reqs x num_generated_tokens

65 # num_generated_tokens is the number of tokens

66 # generated in the current step. It can be different for

67 # each request due to speculative/jump decoding.

68 - sampled_token_ids: List[List[int]]

68 + sampled_token_ids: list[list[int]]

69

70 # num_reqs x num_spec_tokens

71 - spec_token_ids: Optional[List[List[int]]]

71 + spec_token_ids: Optional[list[list[int]]]

72

73 # [num_reqs, max_num_logprobs + 1]

74 # [num_reqs, max_num_logprobs + 1]

@@ -79,4 +79,4 @@ class ModelRunnerOutput:

79 # [prompt_len, num_prompt_logprobs]

80 # [prompt_len, num_prompt_logprobs]

81 # [prompt_len]

82 - prompt_logprobs_dict: Dict[str, Optional[LogprobsTensors]]

82 + prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]]

vllm/v1/request.py

+12 -12

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 import enum

4 -from typing import TYPE_CHECKING, List, Optional, Union

4 +from typing import TYPE_CHECKING, Optional, Union

5

6 from vllm.lora.request import LoRARequest

7 from vllm.sampling_params import SamplingParams

@@ -20,10 +20,10 @@ class Request:

20 self,

21 request_id: str,

22 prompt: Optional[str],

23 - prompt_token_ids: List[int],

24 - multi_modal_inputs: Optional[List["MultiModalKwargs"]],

25 - multi_modal_hashes: Optional[List[str]],

26 - multi_modal_placeholders: Optional[List["PlaceholderRange"]],

23 + prompt_token_ids: list[int],

24 + multi_modal_inputs: Optional[list["MultiModalKwargs"]],

25 + multi_modal_hashes: Optional[list[str]],

26 + multi_modal_placeholders: Optional[list["PlaceholderRange"]],

27 sampling_params: SamplingParams,

28 eos_token_id: Optional[int],

29 arrival_time: float,

@@ -36,7 +36,7 @@ class Request:

36 self.lora_request = lora_request

37

38 self.status = RequestStatus.WAITING

39 - self.events: List[EngineCoreEvent] = []

39 + self.events: list[EngineCoreEvent] = []

40 self.stop_reason: Union[int, str, None] = None

41 assert sampling_params.max_tokens is not None

42 self.max_tokens = sampling_params.max_tokens

@@ -44,15 +44,15 @@ class Request:

44 self.prompt = prompt

45 self.prompt_token_ids = prompt_token_ids

46 self.num_prompt_tokens = len(self.prompt_token_ids)

47 - self._output_token_ids: List[int] = []

48 - self._all_token_ids: List[int] = self.prompt_token_ids.copy()

49 - self.spec_token_ids: List[int] = []

47 + self._output_token_ids: list[int] = []

48 + self._all_token_ids: list[int] = self.prompt_token_ids.copy()

49 + self.spec_token_ids: list[int] = []

50 self.num_computed_tokens = 0

51

52 # Multi-modal related

53 self.mm_positions = multi_modal_placeholders or []

54 self.mm_inputs = multi_modal_inputs or []

55 - self.mm_hashes: List[str] = multi_modal_hashes or []

55 + self.mm_hashes: list[str] = multi_modal_hashes or []

56

57 # Sanity check

58 assert len(self.mm_inputs) == len(self.mm_positions)

@@ -89,7 +89,7 @@ class Request:

89 EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,

90 timestamp))

91

92 - def take_events(self) -> Optional[List[EngineCoreEvent]]:

92 + def take_events(self) -> Optional[list[EngineCoreEvent]]:

93 if not self.events:

94 return None

95 events, self.events = self.events, []

@@ -97,7 +97,7 @@ class Request:

97

98 def append_output_token_ids(

99 self,

100 - token_ids: Union[int, List[int]],

100 + token_ids: Union[int, list[int]],

101 ) -> None:

102 if isinstance(token_ids, int):

103 token_ids = [token_ids]

vllm/v1/sample/metadata.py

+5 -5

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from dataclasses import dataclass

4 -from typing import Dict, List, Optional, Set, Tuple

4 +from typing import Optional

5

6 import torch

7

@@ -17,7 +17,7 @@ class SamplingMetadata:

17 top_k: Optional[torch.Tensor]

18 min_p: Optional[torch.Tensor]

19

20 - generators: Dict[int, torch.Generator]

20 + generators: dict[int, torch.Generator]

21

22 # None means no logprobs, 0 means sampled token logprobs only

23 max_num_logprobs: Optional[int]

@@ -28,12 +28,12 @@ class SamplingMetadata:

28 presence_penalties: torch.Tensor

29 repetition_penalties: torch.Tensor

30

31 - output_token_ids: List[List[int]]

31 + output_token_ids: list[list[int]]

32

33 # req_index -> (min_tokens, stop_token_ids)

34 - min_tokens: Dict[int, Tuple[int, Set[int]]]

34 + min_tokens: dict[int, tuple[int, set[int]]]

35

36 - logit_bias: List[Optional[Dict[int, float]]]

36 + logit_bias: list[Optional[dict[int, float]]]

37

38 # `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size,

39 # vocab size).

vllm/v1/sample/ops/penalties.py

+5 -7

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, List, Set, Tuple

4 -

3 import torch

4

5 from vllm.model_executor.layers.utils import apply_penalties

@@ -9,13 +7,13 @@ from vllm.utils import is_pin_memory_available, make_tensor_with_pad

7

8

9 def apply_min_token_penalties(

12 - logits: torch.Tensor, output_token_ids: List[List[int]],

13 - min_tokens: Dict[int, Tuple[int, Set[int]]]) -> None:

10 + logits: torch.Tensor, output_token_ids: list[list[int]],

11 + min_tokens: dict[int, tuple[int, set[int]]]) -> None:

12 """

13 Applies minimum token penalty by setting the logits of the stop tokens

14 to -inf.

15 """

18 - min_tokens_logits_to_penalize: List[Tuple[int, int]] = []

16 + min_tokens_logits_to_penalize: list[tuple[int, int]] = []

17 for index, (min_token, stop_token_ids) in min_tokens.items():

18 if len(output_token_ids[index]) < min_token:

19 for stop_token_id in stop_token_ids:

@@ -30,7 +28,7 @@ def apply_all_penalties(

28 presence_penalties: torch.Tensor,

29 frequency_penalties: torch.Tensor,

30 repetition_penalties: torch.Tensor,

33 - output_token_ids: List[List[int]],

31 + output_token_ids: list[list[int]],

32 ) -> torch.Tensor:

33 """

34 Applies presence, frequency and repetition penalties to the logits.

@@ -43,7 +41,7 @@ def apply_all_penalties(

41 repetition_penalties)

42

43

46 -def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,

44 +def _convert_to_tensors(output_token_ids: list[list[int]], vocab_size: int,

45 device: torch.device) -> torch.Tensor:

46 """

47 Convert the different list data structures to tensors.

vllm/v1/sample/ops/topk_topp_sampler.py

+5 -5

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import Dict, Optional

3 +from typing import Optional

4

5 import torch

6 import torch.nn as nn

@@ -54,7 +54,7 @@ class TopKTopPSampler(nn.Module):

54 def forward_native(

55 self,

56 logits: torch.Tensor,

57 - generators: Dict[int, torch.Generator],

57 + generators: dict[int, torch.Generator],

58 k: Optional[torch.Tensor],

59 p: Optional[torch.Tensor],

60 ) -> torch.Tensor:

@@ -66,7 +66,7 @@ class TopKTopPSampler(nn.Module):

66 def forward_cuda(

67 self,

68 logits: torch.Tensor,

69 - generators: Dict[int, torch.Generator],

69 + generators: dict[int, torch.Generator],

70 k: Optional[torch.Tensor],

71 p: Optional[torch.Tensor],

72 ) -> torch.Tensor:

@@ -117,7 +117,7 @@ def apply_top_k_top_p(

117

118 def random_sample(

119 probs: torch.Tensor,

120 - generators: Dict[int, torch.Generator],

120 + generators: dict[int, torch.Generator],

121 ) -> torch.Tensor:

122 """Randomly sample from the probabilities.

123

@@ -143,7 +143,7 @@ def flashinfer_sample(

143 probs: torch.Tensor,

144 k: Optional[torch.Tensor],

145 p: Optional[torch.Tensor],

146 - generators: Dict[int, torch.Generator],

146 + generators: dict[int, torch.Generator],

147 ) -> torch.Tensor:

148 """Sample from the probabilities using FlashInfer.

149

vllm/v1/sample/rejection_sampler.py

+3 -4

@@ -1,5 +1,4 @@

1 # SPDX-License-Identifier: Apache-2.0

2 -from typing import List

2

3 import torch

4 import torch.nn as nn

@@ -54,7 +53,7 @@ class RejectionSampler(nn.Module):

53 else:

54 self.forward_method = self.forward_native

55

57 - def forward(self, draft_token_ids: List[List[int]],

56 + def forward(self, draft_token_ids: list[list[int]],

57 target_probs: torch.Tensor,

58 sampling_metadata: SamplingMetadata) -> SamplerOutput:

59 if not sampling_metadata.all_greedy:

@@ -66,7 +65,7 @@ class RejectionSampler(nn.Module):

65

66 def flashinfer_sample(

67 self,

69 - draft_token_ids: List[List[int]],

68 + draft_token_ids: list[list[int]],

69 target_probs: torch.Tensor,

70 sampling_metadata: SamplingMetadata,

71 ) -> SamplerOutput:

@@ -119,7 +118,7 @@ class RejectionSampler(nn.Module):

118 # TODO: The following method can be optimized for better performance.

119 def forward_native(

120 self,

122 - draft_token_ids: List[List[int]],

121 + draft_token_ids: list[list[int]],

122 target_probs: torch.Tensor,

123 sampling_metadata: SamplingMetadata,

124 ) -> SamplerOutput:

vllm/v1/stats/common.py

+9 -9

@@ -4,7 +4,7 @@ import time

4 from dataclasses import dataclass

5 from dataclasses import field as dataclass_field

6 from enum import IntEnum

7 -from typing import ClassVar, Dict, List, Optional, Set

7 +from typing import ClassVar, Optional

8

9 import msgspec

10 from msgspec import field as msgspec_field

@@ -78,7 +78,7 @@ class RequestStatsUpdate(

78 ▼

79 FINISHED (All could go to FINISHED)

80 """

81 - _VALID_TRANSITIONS: ClassVar[Dict[Type, Set[Type]]] = {

81 + _VALID_TRANSITIONS: ClassVar[dict[Type, set[Type]]] = {

82 Type.ARRIVED: {

83 Type.INPUT_PROCESSED,

84 Type.FINISHED,

@@ -140,7 +140,7 @@ class RequestStatsUpdate(

140 finish_reason: Optional[str] = None

141

142 # Non-optional fields for each update type.

143 - _REQUIRED_FIELDS: ClassVar[Dict[Type, List[str]]] = {

143 + _REQUIRED_FIELDS: ClassVar[dict[Type, list[str]]] = {

144 Type.INPUT_PROCESSED: ["num_prompt_tokens", "sampling_params"],

145 Type.PREFILLING: ["num_computed_tokens", "num_cached_tokens"],

146 Type.DETOKENIZED: ["num_new_tokens"],

@@ -218,13 +218,13 @@ class RequestStats:

218 # 2. the request was preempted and resumed. It is equivalent to running

219 # a prefill of the original prefill tokens + generated output tokens

220 # before preemption.

221 - prefill_start_ts_s_lst: List[float] = dataclass_field(default_factory=list)

221 + prefill_start_ts_s_lst: list[float] = dataclass_field(default_factory=list)

222

223 # A list of timestamps when a token is decoded by the engine core.

224 - decoding_ts_s_lst: List[float] = dataclass_field(default_factory=list)

224 + decoding_ts_s_lst: list[float] = dataclass_field(default_factory=list)

225

226 # A sorted list of timestamps for each output token.

227 - output_token_ts_s_lst: List[float] = dataclass_field(default_factory=list)

227 + output_token_ts_s_lst: list[float] = dataclass_field(default_factory=list)

228

229 # First token's timestamp.

230 first_token_ts_s: Optional[float] = None

@@ -241,7 +241,7 @@ class RequestStats:

241 # metric to measure the impact of preemption other than observation of

242 # large P99 TPOT. Ideally we could quantify the impact of preemption by

243 # measuring the number of tokens re-computed due to preemption.

244 - preempted_ts_s_lst: List[float] = dataclass_field(default_factory=list)

244 + preempted_ts_s_lst: list[float] = dataclass_field(default_factory=list)

245

246 # Timestamp when the request was finished at the engine core.

247 finished_ts_s: Optional[float] = None

@@ -308,7 +308,7 @@ class RequestStats:

308 return self.e2e_latency_s - self.first_token_latency_s

309

310 @property

311 - def output_token_latency_s_lst(self) -> List[float]:

311 + def output_token_latency_s_lst(self) -> list[float]:

312 if len(self.output_token_ts_s_lst) == 0:

313 return []

314 latency_s_lst = []

@@ -442,7 +442,7 @@ class EngineCoreStatsSnapshot(

442 default_factory=SchedulerStats)

443

444 # Per request stats updates.

445 - requests_stats_updates: List[RequestStatsUpdate] = msgspec_field(

445 + requests_stats_updates: list[RequestStatsUpdate] = msgspec_field(

446 default_factory=list)

447

448 # Engine core's queue stats.

vllm/v1/utils.py

+10 -10

@@ -5,8 +5,8 @@ import os

5 import weakref

6 from collections import defaultdict

7 from collections.abc import Sequence

8 -from typing import (TYPE_CHECKING, Any, Callable, Dict, Generic, List,

9 - Optional, TypeVar, Union, overload)

8 +from typing import (TYPE_CHECKING, Any, Callable, Generic, Optional, TypeVar,

9 + Union, overload)

10

11 import torch

12

@@ -24,7 +24,7 @@ T = TypeVar("T")

24

25 class ConstantList(Generic[T], Sequence):

26

27 - def __init__(self, x: List[T]) -> None:

27 + def __init__(self, x: list[T]) -> None:

28 self._x = x

29

30 def append(self, item):

@@ -57,10 +57,10 @@ class ConstantList(Generic[T], Sequence):

57 ...

58

59 @overload

60 - def __getitem__(self, s: slice, /) -> List[T]:

60 + def __getitem__(self, s: slice, /) -> list[T]:

61 ...

62

63 - def __getitem__(self, item: Union[int, slice]) -> Union[T, List[T]]:

63 + def __getitem__(self, item: Union[int, slice]) -> Union[T, list[T]]:

64 return self._x[item]

65

66 @overload

@@ -71,7 +71,7 @@ class ConstantList(Generic[T], Sequence):

71 def __setitem__(self, s: slice, value: T, /):

72 ...

73

74 - def __setitem__(self, item: Union[int, slice], value: Union[T, List[T]]):

74 + def __setitem__(self, item: Union[int, slice], value: Union[T, list[T]]):

75 raise Exception("Cannot set item in a constant list")

76

77 def __delitem__(self, item):

@@ -99,7 +99,7 @@ class BackgroundProcHandle:

99 output_path: str,

100 process_name: str,

101 target_fn: Callable,

102 - process_kwargs: Dict[Any, Any],

102 + process_kwargs: dict[Any, Any],

103 ):

104 context = get_mp_context()

105 reader, writer = context.Pipe(duplex=False)

@@ -146,9 +146,9 @@ def shutdown(proc: multiprocessing.Process, input_path: str, output_path: str):

146

147

148 def bind_kv_cache(

149 - kv_caches: Dict[str, torch.Tensor],

150 - forward_context: Dict[str, "Attention"],

151 - runner_kv_caches: List[torch.Tensor],

149 + kv_caches: dict[str, torch.Tensor],

150 + forward_context: dict[str, "Attention"],

151 + runner_kv_caches: list[torch.Tensor],

152 ) -> None:

153 """

154 Bind the allocated KV cache to both ModelRunner and forward context so

vllm/v1/worker/block_table.py

+2 -4

@@ -1,7 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 -from typing import List

4 -

3 import numpy as np

4 import torch

5

@@ -40,7 +38,7 @@ class BlockTable:

38

39 def append_row(

40 self,

43 - block_ids: List[int],

41 + block_ids: list[int],

42 row_idx: int,

43 ) -> None:

44 if not block_ids:

@@ -50,7 +48,7 @@ class BlockTable:

48 self.num_blocks_per_row[row_idx] += num_blocks

49 self.block_table_np[row_idx, start:start + num_blocks] = block_ids

50

53 - def add_row(self, block_ids: List[int], row_idx: int) -> None:

51 + def add_row(self, block_ids: list[int], row_idx: int) -> None:

52 self.num_blocks_per_row[row_idx] = 0

53 self.append_row(block_ids, row_idx)

54

vllm/v1/worker/gpu_input_batch.py

+31 -31

@@ -2,7 +2,7 @@

2 # Datastructures defining an input batch

3

4 from dataclasses import dataclass

5 -from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast

5 +from typing import TYPE_CHECKING, Optional, cast

6

7 import numpy as np

8 import torch

@@ -24,16 +24,16 @@ if TYPE_CHECKING:

24 class CachedRequestState:

25

26 req_id: str

27 - prompt_token_ids: List[int]

27 + prompt_token_ids: list[int]

28 prompt: Optional[str]

29 - mm_inputs: List[MultiModalKwargs]

30 - mm_positions: List["PlaceholderRange"]

29 + mm_inputs: list[MultiModalKwargs]

30 + mm_positions: list["PlaceholderRange"]

31 sampling_params: SamplingParams

32 generator: Optional[torch.Generator]

33

34 - block_ids: List[int]

34 + block_ids: list[int]

35 num_computed_tokens: int

36 - output_token_ids: List[int]

36 + output_token_ids: list[int]

37

38 mrope_positions: Optional[torch.Tensor] = None

39 mrope_position_delta: Optional[int] = None

@@ -63,8 +63,8 @@ class InputBatch:

63 self.pin_memory = pin_memory

64 self.vocab_size = vocab_size

65

66 - self._req_ids: List[Optional[str]] = []

67 - self.req_id_to_index: Dict[str, int] = {}

66 + self._req_ids: list[Optional[str]] = []

67 + self.req_id_to_index: dict[str, int] = {}

68

69 # TODO(woosuk): This buffer could be too large if max_model_len is big.

70 # Find a way to reduce the CPU memory usage.

@@ -106,8 +106,8 @@ class InputBatch:

106 device="cpu",

107 pin_memory=pin_memory)

108 self.temperature_cpu = self.temperature_cpu_tensor.numpy()

109 - self.greedy_reqs: Set[str] = set()

110 - self.random_reqs: Set[str] = set()

109 + self.greedy_reqs: set[str] = set()

110 + self.random_reqs: set[str] = set()

111

112 self.top_p = torch.empty((max_num_reqs, ),

113 dtype=torch.float32,

@@ -117,7 +117,7 @@ class InputBatch:

117 device="cpu",

118 pin_memory=pin_memory)

119 self.top_p_cpu = self.top_p_cpu_tensor.numpy()

120 - self.top_p_reqs: Set[str] = set()

120 + self.top_p_reqs: set[str] = set()

121

122 self.top_k = torch.empty((max_num_reqs, ),

123 dtype=torch.int32,

@@ -127,7 +127,7 @@ class InputBatch:

127 device="cpu",

128 pin_memory=pin_memory)

129 self.top_k_cpu = self.top_k_cpu_tensor.numpy()

130 - self.top_k_reqs: Set[str] = set()

130 + self.top_k_reqs: set[str] = set()

131

132 self.min_p = torch.empty((max_num_reqs, ),

133 dtype=torch.float32,

@@ -137,7 +137,7 @@ class InputBatch:

137 device="cpu",

138 pin_memory=pin_memory)

139 self.min_p_cpu = self.min_p_cpu_tensor.numpy()

140 - self.min_p_reqs: Set[str] = set()

140 + self.min_p_reqs: set[str] = set()

141

142 # Frequency penalty related data structures

143 self.frequency_penalties = torch.empty((max_num_reqs, ),

@@ -150,7 +150,7 @@ class InputBatch:

150 pin_memory=pin_memory)

151 self.frequency_penalties_cpu = \

152 self.frequency_penalties_cpu_tensor.numpy()

153 - self.frequency_penalties_reqs: Set[str] = set()

153 + self.frequency_penalties_reqs: set[str] = set()

154

155 # Presence penalty related data structures

156 self.presence_penalties = torch.empty((max_num_reqs, ),

@@ -162,7 +162,7 @@ class InputBatch:

162 pin_memory=pin_memory)

163 self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy(

164 )

165 - self.presence_penalties_reqs: Set[str] = set()

165 + self.presence_penalties_reqs: set[str] = set()

166

167 # Repetition penalty related data structures

168 self.repetition_penalties = torch.empty((max_num_reqs, ),

@@ -175,43 +175,43 @@ class InputBatch:

175 pin_memory=pin_memory)

176 self.repetition_penalties_cpu = \

177 self.repetition_penalties_cpu_tensor.numpy()

178 - self.repetition_penalties_reqs: Set[str] = set()

178 + self.repetition_penalties_reqs: set[str] = set()

179

180 # req_index -> (min_tokens, stop_token_ids)

181 - self.min_tokens: Dict[int, Tuple[int, Set[int]]] = {}

181 + self.min_tokens: dict[int, tuple[int, set[int]]] = {}

182

183 # lora related

184 self.request_lora_mapping = np.zeros((self.max_num_reqs, ),

185 dtype=np.int32)

186 - self.lora_id_to_request_ids: Dict[int, Set[str]] = {}

187 - self.lora_id_to_lora_request: Dict[int, LoRARequest] = {}

186 + self.lora_id_to_request_ids: dict[int, set[str]] = {}

187 + self.lora_id_to_lora_request: dict[int, LoRARequest] = {}

188

189 # req_index -> generator

190 # NOTE(woosuk): The indices of the requests that do not have their own

191 # generator should not be included in the dictionary.

192 - self.generators: Dict[int, torch.Generator] = {}

192 + self.generators: dict[int, torch.Generator] = {}

193

194 - self.num_logprobs: Dict[str, int] = {}

194 + self.num_logprobs: dict[str, int] = {}

195 # NOTE(rob): num_prompt_logprobs only includes reqs

196 # that are currently in the prefill phase.

197 - self.num_prompt_logprobs: Dict[str, int] = {}

197 + self.num_prompt_logprobs: dict[str, int] = {}

198

199 - self.logit_bias: List[Optional[Dict[int,

199 + self.logit_bias: list[Optional[dict[int,

200 float]]] = [None] * max_num_reqs

201 - self.has_allowed_token_ids: Set[str] = set()

201 + self.has_allowed_token_ids: set[str] = set()

202 self.allowed_token_ids_mask: Optional[torch.Tensor] = None

203 self.allowed_token_ids_mask_cpu_tensor: Optional[torch.Tensor] = None

204

205 - self.req_output_token_ids: List[Optional[List[int]]] = []

205 + self.req_output_token_ids: list[Optional[list[int]]] = []

206

207 # This is updated each time the batch constituents change.

208 self.sampling_metadata = self._make_sampling_metadata()

209

210 @property

211 - def req_ids(self) -> List[str]:

211 + def req_ids(self) -> list[str]:

212 # None elements should only be present transiently

213 # while performing state updates to the batch.

214 - return cast(List[str], self._req_ids)

214 + return cast(list[str], self._req_ids)

215

216 def add_request(

217 self,

@@ -417,7 +417,7 @@ class InputBatch:

417 self.logit_bias[i2], self.logit_bias[i1]

418 self.block_table.swap_row(i1, i2)

419

420 - def condense(self, empty_req_indices: List[int]) -> None:

420 + def condense(self, empty_req_indices: list[int]) -> None:

421 num_reqs = self.num_reqs

422 if num_reqs == 0:

423 # The batched states are empty.

@@ -550,7 +550,7 @@ class InputBatch:

550 frequency_penalties=self.frequency_penalties[:num_reqs],

551 presence_penalties=self.presence_penalties[:num_reqs],

552 repetition_penalties=self.repetition_penalties[:num_reqs],

553 - output_token_ids=cast(List[List[int]], self.req_output_token_ids),

553 + output_token_ids=cast(list[list[int]], self.req_output_token_ids),

554 min_tokens=self.min_tokens,

555 no_penalties=self.no_penalties,

556 logit_bias=self.logit_bias[:num_reqs],

@@ -577,7 +577,7 @@ class InputBatch:

577

578 def make_lora_inputs(

579 self, num_scheduled_tokens: np.ndarray

580 - ) -> Tuple[Tuple[int, ...], Tuple[int, ...], Set[LoRARequest]]:

580 + ) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]:

581 """

582 Given the num_scheduled_tokens for each request in the batch, return

583 datastructures used to activate the current LoRAs.

@@ -593,7 +593,7 @@ class InputBatch:

593 prompt_lora_mapping = tuple(req_lora_mapping)

594 token_lora_mapping = tuple(

595 req_lora_mapping.repeat(num_scheduled_tokens))

596 - active_lora_requests: Set[LoRARequest] = set(

596 + active_lora_requests: set[LoRARequest] = set(

597 self.lora_id_to_lora_request.values())

598

599 return prompt_lora_mapping, token_lora_mapping, active_lora_requests

vllm/v1/worker/gpu_model_runner.py

+17 -17

@@ -3,7 +3,7 @@

3 import gc

4 import time

5 import weakref

6 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

6 +from typing import TYPE_CHECKING, Optional, Union

7

8 import numpy as np

9 import torch

@@ -135,9 +135,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):

135

136 # Lazy initialization

137 # self.model: nn.Module # Set after load_model

138 - self.kv_caches: List[torch.Tensor] = []

138 + self.kv_caches: list[torch.Tensor] = []

139 # req_id -> (input_id -> encoder_output)

140 - self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}

140 + self.encoder_cache: dict[str, dict[int, torch.Tensor]] = {}

141

142 # Set up speculative decoding.

143 self.use_spec_decode = False

@@ -158,7 +158,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

158 )

159

160 # Request states.

161 - self.requests: Dict[str, CachedRequestState] = {}

161 + self.requests: dict[str, CachedRequestState] = {}

162 # Persistent batch.

163 self.input_batch = InputBatch(

164 max_num_reqs=self.max_num_reqs,

@@ -274,7 +274,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

274 # then resubmitted with the same ID. In this case, we treat them as two

275 # distinct requests - clearing the cached states for the first request

276 # and handling the second as a new request.

277 - removed_req_indices: List[int] = []

277 + removed_req_indices: list[int] = []

278 for req_id in scheduler_output.finished_req_ids:

279 req_index = self.input_batch.remove_request(req_id)

280 if req_index is not None:

@@ -305,7 +305,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

305 assert req_index is not None

306 removed_req_indices.append(req_index)

307

308 - req_ids_to_add: List[str] = []

308 + req_ids_to_add: list[str] = []

309 # Add new requests to the cached states.

310 for new_req_data in scheduler_output.scheduled_new_reqs:

311 req_id = new_req_data.req_id

@@ -446,7 +446,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

446 def _prepare_inputs(

447 self,

448 scheduler_output: "SchedulerOutput",

449 - ) -> Tuple[FlashAttentionMetadata, torch.Tensor]:

449 + ) -> tuple[FlashAttentionMetadata, torch.Tensor]:

450 total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens

451 assert total_num_scheduled_tokens > 0

452 num_reqs = self.input_batch.num_reqs

@@ -774,8 +774,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):

774 return

775

776 # Batch the multi-modal inputs.

777 - mm_inputs: List[MultiModalKwargs] = []

778 - req_input_ids: List[Tuple[str, int]] = []

777 + mm_inputs: list[MultiModalKwargs] = []

778 + req_input_ids: list[tuple[str, int]] = []

779 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():

780 req_state = self.requests[req_id]

781 for input_id in encoder_input_ids:

@@ -819,8 +819,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):

819 def _gather_encoder_outputs(

820 self,

821 scheduler_output: "SchedulerOutput",

822 - ) -> List[torch.Tensor]:

823 - encoder_outputs: List[torch.Tensor] = []

822 + ) -> list[torch.Tensor]:

823 + encoder_outputs: list[torch.Tensor] = []

824 for req_id in self.input_batch.req_ids:

825 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[

826 req_id]

@@ -1022,10 +1022,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1022

1023 def generate_draft_token_ids(

1024 self,

1025 - sampled_token_ids: List[List[int]],

1026 - ) -> List[List[int]]:

1025 + sampled_token_ids: list[list[int]],

1026 + ) -> list[list[int]]:

1027 # TODO(woosuk): Optimize.

1028 - draft_token_ids: List[List[int]] = []

1028 + draft_token_ids: list[list[int]] = []

1029 for i, sampled_ids in enumerate(sampled_token_ids):

1030 num_sampled_ids = len(sampled_ids)

1031 if not num_sampled_ids:

@@ -1069,12 +1069,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1069 self,

1070 hidden_states: torch.Tensor,

1071 scheduler_output: "SchedulerOutput",

1072 - ) -> Dict[str, Optional[LogprobsTensors]]:

1072 + ) -> dict[str, Optional[LogprobsTensors]]:

1073 num_prompt_logprobs_dict = self.input_batch.num_prompt_logprobs

1074 if not num_prompt_logprobs_dict:

1075 return {}

1076

1077 - prompt_logprobs_dict: Dict[str, Optional[LogprobsTensors]] = {}

1077 + prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]] = {}

1078

1079 # Since prompt logprobs are a rare feature, prioritize simple,

1080 # maintainable loop over optimal performance.

@@ -1365,7 +1365,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1365 "Hybrid models with more than one KV cache type are not "

1366 "supported yet.")

1367

1368 - kv_caches: Dict[str, torch.Tensor] = {}

1368 + kv_caches: dict[str, torch.Tensor] = {}

1369

1370 for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():

1371 tensor_config = kv_cache_config.tensors[layer_name]

vllm/v1/worker/gpu_worker.py

+2 -2

@@ -2,7 +2,7 @@

2 """A GPU worker class."""

3 import gc

4 import os

5 -from typing import TYPE_CHECKING, Optional, Set

5 +from typing import TYPE_CHECKING, Optional

6

7 import torch

8 import torch.distributed

@@ -243,7 +243,7 @@ class Worker(WorkerBase):

243 def remove_lora(self, lora_id: int) -> bool:

244 return self.model_runner.remove_lora(lora_id)

245

246 - def list_loras(self) -> Set[int]:

246 + def list_loras(self) -> set[int]:

247 return self.model_runner.list_loras()

248

249 def pin_lora(self, lora_id: int) -> bool:

vllm/v1/worker/lora_model_runner_mixin.py

+8 -9

@@ -4,7 +4,6 @@ Define LoRA functionality mixin for model runners.

4 """

5

6 from contextlib import contextmanager

7 -from typing import Set, Tuple

7

8 import numpy as np

9 import torch.nn as nn

@@ -57,9 +56,9 @@ class LoRAModelRunnerMixin:

56 )

57 return self.lora_manager.create_lora_manager(model)

58

60 - def _set_active_loras(self, prompt_lora_mapping: Tuple[int, ...],

61 - token_lora_mapping: Tuple[int, ...],

62 - lora_requests: Set[LoRARequest]) -> None:

59 + def _set_active_loras(self, prompt_lora_mapping: tuple[int, ...],

60 + token_lora_mapping: tuple[int, ...],

61 + lora_requests: set[LoRARequest]) -> None:

62 if not self.lora_manager:

63 raise RuntimeError("LoRA is not enabled.")

64

@@ -74,10 +73,10 @@ class LoRAModelRunnerMixin:

73 def set_active_loras(self, input_batch: InputBatch,

74 num_scheduled_tokens: np.ndarray) -> None:

75

77 - prompt_lora_mapping: Tuple[int, ...] # of size input_batch.num_reqs

78 - token_lora_mapping: Tuple[int,

76 + prompt_lora_mapping: tuple[int, ...] # of size input_batch.num_reqs

77 + token_lora_mapping: tuple[int,

78 ...] # of size np.sum(num_scheduled_tokens)

80 - lora_requests: Set[LoRARequest]

79 + lora_requests: set[LoRARequest]

80 prompt_lora_mapping, token_lora_mapping, lora_requests = \

81 input_batch.make_lora_inputs(num_scheduled_tokens)

82 return self._set_active_loras(prompt_lora_mapping, token_lora_mapping,

@@ -105,7 +104,7 @@ class LoRAModelRunnerMixin:

104 num_scheduled_tokens)

105

106 # Make dummy lora requests

108 - lora_requests: Set[LoRARequest] = {

107 + lora_requests: set[LoRARequest] = {

108 LoRARequest(lora_name=f"warmup_{lora_id}",

109 lora_int_id=lora_id,

110 lora_path="/not/a/real/path")

@@ -143,7 +142,7 @@ class LoRAModelRunnerMixin:

142 raise RuntimeError("LoRA is not enabled.")

143 return self.lora_manager.pin_adapter(lora_id)

144

146 - def list_loras(self) -> Set[int]:

145 + def list_loras(self) -> set[int]:

146 if not self.lora_manager:

147 raise RuntimeError("LoRA is not enabled.")

148 return self.lora_manager.list_adapters()

\ No newline at end of file

vllm/v1/worker/tpu_model_runner.py

+12 -12

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 import time

3 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast

3 +from typing import TYPE_CHECKING, Optional, cast

4 from unittest.mock import patch

5

6 import numpy as np

@@ -95,13 +95,13 @@ class TPUModelRunner:

95 )

96

97 # Request states.

98 - self.requests: Dict[str, CachedRequestState] = {}

98 + self.requests: dict[str, CachedRequestState] = {}

99

100 # req_id -> (input_id -> encoder_output)

101 - self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}

101 + self.encoder_cache: dict[str, dict[int, torch.Tensor]] = {}

102

103 # KV caches for forward pass

104 - self.kv_caches: List[Tuple[torch.Tensor, torch.Tensor]] = []

104 + self.kv_caches: list[tuple[torch.Tensor, torch.Tensor]] = []

105

106 # Cached torch/numpy tensor

107 # The pytorch tensor and numpy array share the same buffer.

@@ -171,7 +171,7 @@ class TPUModelRunner:

171 # then resubmitted with the same ID. In this case, we treat them as two

172 # distinct requests - clearing the cached states for the first request

173 # and handling the second as a new request.

174 - removed_req_indices: List[int] = []

174 + removed_req_indices: list[int] = []

175 for req_id in scheduler_output.finished_req_ids:

176 req_index = self.input_batch.remove_request(req_id)

177 if req_index is not None:

@@ -194,7 +194,7 @@ class TPUModelRunner:

194 assert req_index is not None

195 removed_req_indices.append(req_index)

196

197 - req_ids_to_add: List[str] = []

197 + req_ids_to_add: list[str] = []

198 # Add new requests to the cached states.

199 for new_req_data in scheduler_output.scheduled_new_reqs:

200 req_id = new_req_data.req_id

@@ -453,7 +453,7 @@ class TPUModelRunner:

453 selected_token_ids = torch.argmax(logits, dim=-1, keepdim=True)

454

455 # Then, let's update the cache state.

456 - request_seq_lens: List[Tuple[int, CachedRequestState, int]] = []

456 + request_seq_lens: list[tuple[int, CachedRequestState, int]] = []

457 for i, req_id in zip(range(num_reqs), self.input_batch.req_ids):

458 assert req_id is not None

459 req_state = self.requests[req_id]

@@ -473,9 +473,9 @@ class TPUModelRunner:

473 assert all(

474 req_id is not None for req_id in

475 self.input_batch.req_ids[:num_reqs]), "req_ids contains None"

476 - req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])

476 + req_ids = cast(list[str], self.input_batch.req_ids[:num_reqs])

477

478 - prompt_logprobs_dict: Dict[str, Optional[LogprobsTensors]] = {}

478 + prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]] = {}

479 for req_id in self.input_batch.req_ids[:num_reqs]:

480 prompt_logprobs_dict[req_id] = None

481

@@ -612,7 +612,7 @@ class TPUModelRunner:

612 "Hybrid models with more than one KV cache type are not "

613 "supported yet.")

614

615 - kv_caches: Dict[str, torch.Tensor] = {}

615 + kv_caches: dict[str, torch.Tensor] = {}

616

617 for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():

618 tensor_config = kv_cache_config.tensors[layer_name]

@@ -649,7 +649,7 @@ class ModelWrapperV1(nn.Module):

649 self,

650 token_ids: torch.Tensor,

651 position_ids: torch.Tensor,

652 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],

652 + kv_caches: list[tuple[torch.Tensor, torch.Tensor]],

653 ) -> torch.Tensor:

654 """Executes the forward pass of the model and samples the next token.

655

@@ -667,7 +667,7 @@ class ModelWrapperV1(nn.Module):

667 # [num_kv_heads, num_blocks, block_size, head_size]. To make it

668 # work, we need to flatten the first three dimensions and modify

669 # the slot_mapping accordingly.

670 - # kv_caches: List[Tuple[torch.Tensor, torch.Tensor]]

670 + # kv_caches: list[tuple[torch.Tensor, torch.Tensor]]

671 num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape

672 slot_mapping = attn_metadata.slot_mapping

673 slot_mapping = slot_mapping.flatten()

vllm/v1/worker/tpu_worker.py

+3 -3

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 """A TPU worker class."""

3 import os

4 -from typing import Dict, List, Optional

4 +from typing import Optional

5

6 import torch

7 import torch.distributed

@@ -103,7 +103,7 @@ class TPUWorker:

103 self.model_runner = TPUModelRunner(self.vllm_config, self.device)

104

105 def determine_available_memory(self) -> int:

106 - kv_caches: Dict[str, torch.Tensor] = {}

106 + kv_caches: dict[str, torch.Tensor] = {}

107 kv_cache_spec = self.model_runner.get_kv_cache_spec()

108 for layer_name, layer_spec in kv_cache_spec.items():

109 if isinstance(layer_spec, FullAttentionSpec):

@@ -118,7 +118,7 @@ class TPUWorker:

118 else:

119 raise NotImplementedError

120

121 - runner_kv_caches: List[torch.Tensor] = []

121 + runner_kv_caches: list[torch.Tensor] = []

122 bind_kv_cache(

123 kv_caches,

124 self.vllm_config.compilation_config.static_forward_context,

Update deprecated Python 3.8 typing (#13971)