A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[mypy] Enable type checking for test directory (#5017)
Cyrus Leung committed
Jun 15, 2024 at 12:45 UTC
0e9164b40abdb30f1929edb44b56894c9e26c31d
92 files changed
+510
-379
.github/workflows/mypy.yaml
+1
-1
@@ -47,5 +47,5 @@ jobs:
47
mypy vllm/model_executor --config-file pyproject.toml
48
mypy vllm/lora --config-file pyproject.toml
49
mypy vllm/logging --config-file pyproject.toml
50
- mypy vllm/model_executor --config-file pyproject.toml
50
+ mypy tests --config-file pyproject.toml
51
benchmarks/benchmark_serving.py
+9
-9
@@ -31,7 +31,7 @@ import time
31
import warnings
32
from dataclasses import dataclass
33
from datetime import datetime
34
-from typing import AsyncGenerator, List, Optional, Tuple
34
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
35
36
import numpy as np
37
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
@@ -200,12 +200,12 @@ def calculate_metrics(
200
dur_s: float,
201
tokenizer: PreTrainedTokenizerBase,
202
) -> Tuple[BenchmarkMetrics, List[int]]:
203
- actual_output_lens = []
203
+ actual_output_lens: List[int] = []
204
total_input = 0
205
completed = 0
206
- itls = []
207
- tpots = []
208
- ttfts = []
206
+ itls: List[float] = []
207
+ tpots: List[float] = []
208
+ ttfts: List[float] = []
209
for i in range(len(outputs)):
210
if outputs[i].success:
211
# We use the tokenizer to count the number of output tokens for all
@@ -265,7 +265,7 @@ async def benchmark(
265
disable_tqdm: bool,
266
):
267
if backend in ASYNC_REQUEST_FUNCS:
268
- request_func = ASYNC_REQUEST_FUNCS.get(backend)
268
+ request_func = ASYNC_REQUEST_FUNCS[backend]
269
else:
270
raise ValueError(f"Unknown backend: {backend}")
271
@@ -292,7 +292,7 @@ async def benchmark(
292
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
293
294
benchmark_start_time = time.perf_counter()
295
- tasks = []
295
+ tasks: List[asyncio.Task] = []
296
async for request in get_request(input_requests, request_rate):
297
prompt, prompt_len, output_len = request
298
request_func_input = RequestFuncInput(
@@ -310,7 +310,7 @@ async def benchmark(
310
pbar=pbar)))
311
outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
312
313
- if not disable_tqdm:
313
+ if pbar is not None:
314
pbar.close()
315
316
benchmark_duration = time.perf_counter() - benchmark_start_time
@@ -466,7 +466,7 @@ def main(args: argparse.Namespace):
466
467
# Save config and results to json
468
if args.save_result:
469
- result_json = {}
469
+ result_json: Dict[str, Any] = {}
470
471
# Setup
472
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
benchmarks/benchmark_throughput.py
+2
-2
@@ -108,8 +108,8 @@ def run_vllm(
108
)
109
110
# Add the requests to the engine.
111
- prompts = []
112
- sampling_params = []
111
+ prompts: List[str] = []
112
+ sampling_params: List[SamplingParams] = []
113
for prompt, _, output_len in requests:
114
prompts.append(prompt)
115
sampling_params.append(
benchmarks/kernels/benchmark_aqlm.py
+5
-5
@@ -86,9 +86,9 @@ def dequant_no_scale(
86
# Compare the optimized 1x16 and 2x8 cuda decompression/dequant kernels against
87
# the generic pytorch version.
88
# Just visual comparison.
89
-def dequant_test(k: int, parts: torch.tensor, nbooks: int, bits: int) -> None:
89
+def dequant_test(k: int, parts: torch.Tensor, nbooks: int, bits: int) -> None:
90
91
- n = parts.sum().item()
91
+ n = int(parts.sum().item())
92
93
device = torch.device('cuda:0')
94
@@ -204,7 +204,7 @@ def main():
204
sys.stdout = sys.__stdout__
205
206
207
-def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
207
+def run_grid(m: int, k: int, parts: torch.Tensor, nbooks: int, bits: int,
208
methods):
209
210
# I didn't see visible improvements from increasing these, but feel free :)
@@ -252,10 +252,10 @@ def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
252
print('')
253
254
255
-def run_timing(num_calls: int, m: int, k: int, parts: torch.tensor,
255
+def run_timing(num_calls: int, m: int, k: int, parts: torch.Tensor,
256
nbooks: int, bits: int, method) -> float:
257
258
- n = parts.sum().item()
258
+ n = int(parts.sum().item())
259
260
device = torch.device('cuda:0')
261
benchmarks/kernels/benchmark_marlin.py
+5
-3
@@ -1,4 +1,5 @@
1
import argparse
2
+from typing import List
3
4
import torch
5
import torch.utils.benchmark as benchmark
@@ -23,8 +24,9 @@ ACT_ORDER_OPTS = [False, True]
24
K_FULL_OPTS = [False, True]
25
26
26
-def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
27
- size_m, size_k, size_n):
27
+def bench_run(results: List[benchmark.Measurement], model: str,
28
+ act_order: bool, is_k_full: bool, num_bits: int, group_size: int,
29
+ size_m: int, size_k: int, size_n: int):
30
label = "Quant Matmul"
31
32
sub_label = ("{}, act={} k_full={}, b={}, g={}, "
@@ -156,7 +158,7 @@ def main(args):
158
for i, model in enumerate(args.models):
159
print(f"[{i}] {model}")
160
159
- results = []
161
+ results: List[benchmark.Measurement] = []
162
163
for model in args.models:
164
for layer in WEIGHT_SHAPES[model]:
benchmarks/kernels/benchmark_moe.py
+18
-8
@@ -1,7 +1,7 @@
1
import argparse
2
import time
3
from datetime import datetime
4
-from typing import Any, Dict, List, Tuple
4
+from typing import Any, Dict, List, Tuple, TypedDict
5
6
import ray
7
import torch
@@ -12,8 +12,17 @@ from transformers import AutoConfig
12
from vllm.model_executor.layers.fused_moe.fused_moe import *
13
14
15
+class BenchmarkConfig(TypedDict):
16
+ BLOCK_SIZE_M: int
17
+ BLOCK_SIZE_N: int
18
+ BLOCK_SIZE_K: int
19
+ GROUP_SIZE_M: int
20
+ num_warps: int
21
+ num_stages: int
22
+
23
+
24
def benchmark_config(
16
- config: Dict[str, int],
25
+ config: BenchmarkConfig,
26
num_tokens: int,
27
num_experts: int,
28
shard_intermediate_size: int,
@@ -92,7 +101,7 @@ def benchmark_config(
101
start_event = torch.cuda.Event(enable_timing=True)
102
end_event = torch.cuda.Event(enable_timing=True)
103
95
- latencies = []
104
+ latencies: List[float] = []
105
for i in range(num_iters):
106
prepare(i)
107
torch.cuda.synchronize()
@@ -111,7 +120,7 @@ def get_configs_compute_bound() -> List[Dict[str, int]]:
120
# Reduced search space for faster tuning.
121
# TODO(woosuk): Increase the search space and use a performance model to
122
# prune the search space.
114
- configs = []
123
+ configs: List[BenchmarkConfig] = []
124
for num_stages in [2, 3, 4, 5]:
125
for block_m in [16, 32, 64, 128, 256]:
126
for block_k in [64, 128, 256]:
@@ -175,8 +184,8 @@ class BenchmarkWorker:
184
topk: int,
185
dtype: torch.dtype,
186
use_fp8: bool,
178
- search_space: List[Dict[str, int]],
179
- ) -> Dict[str, int]:
187
+ search_space: List[BenchmarkConfig],
188
+ ) -> BenchmarkConfig:
189
best_config = None
190
best_time = float("inf")
191
for config in tqdm(search_space):
@@ -199,10 +208,11 @@ class BenchmarkWorker:
208
best_config = config
209
now = datetime.now()
210
print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}")
211
+ assert best_config is not None
212
return best_config
213
214
205
-def sort_config(config: Dict[str, int]) -> Dict[str, int]:
215
+def sort_config(config: BenchmarkConfig) -> BenchmarkConfig:
216
return {
217
"BLOCK_SIZE_M": config["BLOCK_SIZE_M"],
218
"BLOCK_SIZE_N": config["BLOCK_SIZE_N"],
@@ -214,7 +224,7 @@ def sort_config(config: Dict[str, int]) -> Dict[str, int]:
224
225
226
def save_configs(
217
- configs: Dict[int, Dict[str, int]],
227
+ configs: Dict[int, BenchmarkConfig],
228
num_experts: int,
229
shard_intermediate_size: int,
230
hidden_size: int,
benchmarks/kernels/benchmark_paged_attention.py
+7
-4
@@ -1,7 +1,7 @@
1
import argparse
2
import random
3
import time
4
-from typing import Optional
4
+from typing import List, Optional
5
6
import torch
7
@@ -54,14 +54,17 @@ def main(
54
55
# Create the block tables.
56
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
57
- block_tables = []
57
+ block_tables_lst: List[List[int]] = []
58
for _ in range(num_seqs):
59
block_table = [
60
random.randint(0, NUM_BLOCKS - 1)
61
for _ in range(max_num_blocks_per_seq)
62
]
63
- block_tables.append(block_table)
64
- block_tables = torch.tensor(block_tables, dtype=torch.int, device=device)
63
+ block_tables_lst.append(block_table)
64
+
65
+ block_tables = torch.tensor(block_tables_lst,
66
+ dtype=torch.int,
67
+ device=device)
68
69
# Create the KV cache.
70
key_caches, value_caches = create_kv_caches_with_random(NUM_BLOCKS,
benchmarks/kernels/benchmark_rope.py
+4
-3
@@ -1,11 +1,12 @@
1
import argparse
2
from itertools import accumulate
3
-from typing import Optional
3
+from typing import List, Optional
4
5
import nvtx
6
import torch
7
8
-from vllm.model_executor.layers.rotary_embedding import get_rope
8
+from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
9
+ get_rope)
10
11
12
def benchmark_rope_kernels_multi_lora(
@@ -37,7 +38,7 @@ def benchmark_rope_kernels_multi_lora(
38
})
39
# non-batched RoPE takes only one scaling factor, we create multiple
40
# instances to simulate the same behavior
40
- non_batched_ropes = []
41
+ non_batched_ropes: List[RotaryEmbedding] = []
42
for scaling_factor in scaling_factors:
43
non_batched_ropes.append(
44
get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
examples/fp8/extract_scales.py
+6
-6
@@ -2,7 +2,7 @@ import argparse
2
import glob
3
import json
4
import os
5
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
5
+from typing import Any, Callable, Dict, List, Optional, Tuple
6
7
import numpy as np
8
import torch
@@ -19,7 +19,7 @@ def _prepare_hf_weights(
19
quantized_model_dir: str,
20
load_format: str = "auto",
21
fall_back_to_pt: bool = True,
22
-) -> Tuple[str, List[str], bool]:
22
+) -> Tuple[List[str], bool]:
23
if not os.path.isdir(quantized_model_dir):
24
raise FileNotFoundError(
25
f"The quantized model directory `{quantized_model_dir}` "
@@ -94,7 +94,7 @@ def _hf_tensorfile_iterator(filename: str, load_format: str,
94
95
96
def _kv_scales_extractor(
97
- hf_tensor_files: Iterable[str],
97
+ hf_tensor_files: List[str],
98
use_safetensors: bool,
99
rank_keyword: str = "rank",
100
expected_tp_size: Optional[int] = None) -> Dict[int, Dict[int, float]]:
@@ -115,7 +115,7 @@ def _kv_scales_extractor(
115
for char in rank_keyword:
116
assert not char.isdecimal(
117
), f"Rank keyword {rank_keyword} contains a numeric character!"
118
- rank_scales_map = {}
118
+ rank_scales_map: Dict[int, Dict[int, float]] = {}
119
for tensor_file in hf_tensor_files:
120
try:
121
rank_idx = tensor_file.find(rank_keyword)
@@ -141,7 +141,7 @@ def _kv_scales_extractor(
141
raise
142
143
if rank not in rank_scales_map:
144
- layer_scales_map = {}
144
+ layer_scales_map: Dict[int, float] = {}
145
rank_scales_map[rank] = layer_scales_map
146
else:
147
raise RuntimeError(
@@ -222,7 +222,7 @@ def _metadata_extractor(quantized_model_dir: str,
222
"does not exist.")
223
metadata_files = glob.glob(os.path.join(quantized_model_dir, "*.json"))
224
225
- result = {}
225
+ result: Dict[str, Any] = {}
226
for file in metadata_files:
227
with open(file) as f:
228
try:
examples/offline_inference_distributed.py
+4
-4
@@ -5,7 +5,7 @@ distributively on a multi-nodes cluster.
5
Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6
"""
7
8
-from typing import Dict
8
+from typing import Any, Dict, List
9
10
import numpy as np
11
import ray
@@ -40,8 +40,8 @@ class LLMPredictor:
40
# The output is a list of RequestOutput objects that contain the prompt,
41
# generated text, and other information.
42
outputs = self.llm.generate(batch["text"], sampling_params)
43
- prompt = []
44
- generated_text = []
43
+ prompt: List[str] = []
44
+ generated_text: List[str] = []
45
for output in outputs:
46
prompt.append(output.prompt)
47
generated_text.append(' '.join([o.text for o in output.outputs]))
@@ -71,7 +71,7 @@ def scheduling_strategy_fn():
71
pg, placement_group_capture_child_tasks=True))
72
73
74
-resources_kwarg = {}
74
+resources_kwarg: Dict[str, Any] = {}
75
if tensor_parallel_size == 1:
76
# For tensor_parallel_size == 1, we simply set num_gpus=1.
77
resources_kwarg["num_gpus"] = 1
format.sh
+1
-1
@@ -111,7 +111,7 @@ mypy vllm/spec_decode --config-file pyproject.toml
111
mypy vllm/model_executor --config-file pyproject.toml
112
mypy vllm/lora --config-file pyproject.toml
113
mypy vllm/logging --config-file pyproject.toml
114
-mypy vllm/model_executor --config-file pyproject.toml
114
+mypy tests --config-file pyproject.toml
115
116
117
# If git diff returns a file that is in the skip list, the file may be checked anyway:
tests/core/block/test_block_table.py
+5
-3
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
5
from vllm.core.block.block_table import BlockTable
@@ -28,7 +30,7 @@ def test_allocate_naive(block_size: int, sequence_len: int):
30
token_ids = list(range(sequence_len))
31
num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
32
31
- block_tables = []
33
+ block_tables: List[BlockTable] = []
34
for i in range(5):
35
assert allocator.get_num_free_blocks(
36
device=Device.GPU) == num_gpu_blocks - i * num_blocks_per_alloc
@@ -73,7 +75,7 @@ def test_allocate_prefix_caching(block_size: int, sequence_len: int):
75
num_immutable_blocks_per_alloc = len(
76
chunked_tokens) - num_mutable_blocks_per_alloc
77
76
- block_tables = []
78
+ block_tables: List[BlockTable] = []
79
for alloc_i in range(1, 6):
80
81
block_tables.append(
@@ -268,7 +270,7 @@ def test_append_token_ids_correct_content(block_size: int, sequence_len: int,
270
)
271
block_table.allocate(token_ids=token_ids, device=Device.GPU)
272
271
- appended_so_far = []
273
+ appended_so_far: List[int] = []
274
for append in chunk_list(token_ids_to_append, append_size):
275
block_table.append_token_ids(append)
276
appended_so_far.extend(append)
tests/core/block/test_prefix_caching_block.py
+2
-2
@@ -123,7 +123,7 @@ class TestPrefixCachingBlock:
123
num_empty_trailing_blocks=0) -> List[PrefixCachingBlock]:
124
"""Helper method which creates a chain of blocks.
125
"""
126
- blocks = []
126
+ blocks: List[PrefixCachingBlock] = []
127
num_blocks = math.ceil(
128
len(token_ids) / block_size) + num_empty_trailing_blocks
129
@@ -608,7 +608,7 @@ class TestPrefixCachingBlockAllocator:
608
) -> List[PrefixCachingBlock]:
609
"""Helper method which creates a chain of blocks.
610
"""
611
- blocks = []
611
+ blocks: List[Block] = []
612
num_blocks = math.ceil(len(token_ids) / block_size)
613
614
if num_blocks == 0:
tests/core/test_chunked_prefill_scheduler.py
+5
-5
@@ -483,11 +483,11 @@ def test_chunked_prefill_preempt():
483
# The request should be preempted.
484
scheduler.block_manager.can_append_slots = MagicMock()
485
486
- def cannot_append_second_group(seq_group, num_lookahead_slots):
486
+ def cannot_append_second_group1(seq_group, num_lookahead_slots):
487
return seq_group.request_id != "1"
488
489
scheduler.block_manager.can_append_slots.side_effect = (
490
- cannot_append_second_group)
490
+ cannot_append_second_group1)
491
492
# The running prefill is now preempted.
493
_, out = schedule_and_update_computed_tokens(scheduler)
@@ -505,11 +505,11 @@ def test_chunked_prefill_preempt():
505
assert seq_group.get_num_uncomputed_tokens() == 30
506
507
# We should be able to run prefill twice as it is chunked.
508
- def cannot_append_second_group(seq_group, num_lookahead_slots):
508
+ def cannot_append_second_group2(seq_group, num_lookahead_slots):
509
return True
510
511
scheduler.block_manager.can_append_slots.side_effect = (
512
- cannot_append_second_group)
512
+ cannot_append_second_group2)
513
_, out = schedule_and_update_computed_tokens(scheduler)
514
assert len(out.scheduled_seq_groups) == 1
515
assert out.num_prefill_groups == 1
@@ -530,7 +530,7 @@ def test_chunked_prefill_max_seqs():
530
cache_config.num_cpu_blocks = 8
531
cache_config.num_gpu_blocks = 8
532
scheduler = Scheduler(scheduler_config, cache_config, None)
533
- running = []
533
+ running: List[SequenceGroup] = []
534
535
_, seq_group = create_dummy_prompt("1", prompt_length=65)
536
scheduler.add_seq_group(seq_group)
tests/core/test_scheduler.py
+26
-26
@@ -1,6 +1,6 @@
1
import time
2
from collections import deque
3
-from typing import List
3
+from typing import Deque, List, Set, Tuple
4
from unittest.mock import MagicMock
5
6
import pytest # noqa
@@ -65,7 +65,7 @@ def test_scheduler_abort_seq_group():
65
66
# Add multiple seq groups to scheduler.
67
num_seq_group = 4
68
- request_ids = set()
68
+ request_ids: Set[str] = set()
69
for i in range(num_seq_group):
70
_, seq_group = create_dummy_prompt(str(i), block_size)
71
scheduler.add_seq_group(seq_group)
@@ -347,7 +347,7 @@ def test_prefill_schedule_max_prompt_len():
347
Test prompt longer than max_prompt_len is aborted.
348
"""
349
scheduler = initialize_scheduler(max_model_len=30)
350
- _, seq_group = create_dummy_prompt(0, prompt_length=60)
350
+ _, seq_group = create_dummy_prompt("0", prompt_length=60)
351
waiting = deque([seq_group])
352
budget = create_token_budget()
353
remaining_waiting, output = scheduler._schedule_prefills(
@@ -364,7 +364,7 @@ def test_prefill_schedule_token_budget():
364
Test token budget respected.
365
"""
366
scheduler = initialize_scheduler()
367
- waiting = deque()
367
+ waiting: Deque[SequenceGroup] = deque()
368
budget = create_token_budget(token_budget=0)
369
for i in range(2):
370
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -419,7 +419,7 @@ def test_prefill_schedule_max_seqs():
419
Test max seq respected.
420
"""
421
scheduler = initialize_scheduler()
422
- waiting = deque()
422
+ waiting: Deque[SequenceGroup] = deque()
423
budget = create_token_budget(max_num_seqs=2)
424
for i in range(3):
425
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -453,9 +453,9 @@ def test_prefill_schedule_max_lora():
453
"""
454
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
455
scheduler = initialize_scheduler(lora_config=lora_config)
456
- waiting = deque()
456
+ waiting: Deque[SequenceGroup] = deque()
457
budget = create_token_budget(token_budget=120)
458
- curr_loras = set()
458
+ curr_loras: Set[int] = set()
459
for i in range(2):
460
_, seq_group = create_dummy_prompt(str(i),
461
prompt_length=60,
@@ -499,7 +499,7 @@ def test_prefill_schedule_no_block_manager_capacity():
499
Test sequence cannot be scheduled due to block manager has no capacity.
500
"""
501
scheduler = initialize_scheduler()
502
- waiting = deque()
502
+ waiting: Deque[SequenceGroup] = deque()
503
budget = create_token_budget()
504
for i in range(3):
505
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -536,7 +536,7 @@ def test_decode_schedule_preempted():
536
Test decodes cannot be scheduled and preempted.
537
"""
538
scheduler = initialize_scheduler()
539
- running = deque()
539
+ running: Deque[SequenceGroup] = deque()
540
policy = PolicyFactory.get_policy(policy_name="fcfs")
541
curr_loras = None
542
for i in range(3):
@@ -577,7 +577,7 @@ def test_decode_swap_beam_search():
577
Test best_of > 1 swap out blocks
578
"""
579
scheduler = initialize_scheduler()
580
- running = deque()
580
+ running: Deque[SequenceGroup] = deque()
581
policy = PolicyFactory.get_policy(policy_name="fcfs")
582
curr_loras = None
583
budget = create_token_budget()
@@ -628,7 +628,7 @@ def test_schedule_decode_blocks_to_copy_update():
628
"""
629
scheduler = initialize_scheduler()
630
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
631
- running = deque()
631
+ running: Deque[SequenceGroup] = deque()
632
policy = PolicyFactory.get_policy(policy_name="fcfs")
633
curr_loras = None
634
scheduler._allocate_and_set_running(seq_group)
@@ -656,10 +656,10 @@ def test_schedule_decode_blocks_to_copy_update():
656
657
def test_schedule_swapped_simple():
658
scheduler = initialize_scheduler()
659
- swapped = deque()
659
+ swapped: Deque[SequenceGroup] = deque()
660
policy = PolicyFactory.get_policy(policy_name="fcfs")
661
curr_loras = None
662
- blocks_to_swap_out = []
662
+ blocks_to_swap_out: List[Tuple[int, int]] = []
663
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
664
scheduler._allocate_and_set_running(seq_group)
665
append_new_token_seq_group(60, seq_group, 1)
@@ -683,10 +683,10 @@ def test_schedule_swapped_simple():
683
684
def test_schedule_swapped_max_token_budget():
685
scheduler = initialize_scheduler()
686
- swapped = deque()
686
+ swapped: Deque[SequenceGroup] = deque()
687
policy = PolicyFactory.get_policy(policy_name="fcfs")
688
curr_loras = None
689
- blocks_to_swap_out = []
689
+ blocks_to_swap_out: List[Tuple[int, int]] = []
690
for _ in range(2):
691
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
692
scheduler._allocate_and_set_running(seq_group)
@@ -717,10 +717,10 @@ def test_schedule_swapped_max_token_budget():
717
718
def test_schedule_swapped_max_seqs():
719
scheduler = initialize_scheduler()
720
- swapped = deque()
720
+ swapped: Deque[SequenceGroup] = deque()
721
policy = PolicyFactory.get_policy(policy_name="fcfs")
722
curr_loras = None
723
- blocks_to_swap_out = []
723
+ blocks_to_swap_out: List[Tuple[int, int]] = []
724
for i in range(4):
725
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
726
scheduler._allocate_and_set_running(seq_group)
@@ -750,10 +750,10 @@ def test_schedule_swapped_max_seqs():
750
def test_schedule_swapped_max_loras():
751
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
752
scheduler = initialize_scheduler(lora_config=lora_config)
753
- swapped = deque()
753
+ swapped: Deque[SequenceGroup] = deque()
754
policy = PolicyFactory.get_policy(policy_name="fcfs")
755
- curr_loras = set()
756
- blocks_to_swap_out = []
755
+ curr_loras: Set[int] = set()
756
+ blocks_to_swap_out: List[Tuple[int, int]] = []
757
for i in range(2):
758
_, seq_group = create_dummy_prompt(str(i),
759
prompt_length=60,
@@ -779,10 +779,10 @@ def test_schedule_swapped_max_loras():
779
780
def test_schedule_swapped_cannot_swap_in():
781
scheduler = initialize_scheduler()
782
- swapped = deque()
782
+ swapped: Deque[SequenceGroup] = deque()
783
policy = PolicyFactory.get_policy(policy_name="fcfs")
784
curr_loras = None
785
- blocks_to_swap_out = []
785
+ blocks_to_swap_out: List[Tuple[int, int]] = []
786
for _ in range(2):
787
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
788
scheduler._allocate_and_set_running(seq_group)
@@ -806,10 +806,10 @@ def test_schedule_swapped_cannot_swap_in():
806
807
def test_infeasible_swap():
808
scheduler = initialize_scheduler()
809
- swapped = deque()
809
+ swapped: Deque[SequenceGroup] = deque()
810
policy = PolicyFactory.get_policy(policy_name="fcfs")
811
curr_loras = None
812
- blocks_to_swap_out = []
812
+ blocks_to_swap_out: List[Tuple[int, int]] = []
813
for _ in range(2):
814
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
815
scheduler._allocate_and_set_running(seq_group)
@@ -834,13 +834,13 @@ def test_infeasible_swap():
834
835
def test_schedule_swapped_blocks_to_copy():
836
scheduler = initialize_scheduler()
837
- swapped = deque()
837
+ swapped: Deque[SequenceGroup] = deque()
838
policy = PolicyFactory.get_policy(policy_name="fcfs")
839
curr_loras = None
840
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
841
scheduler._allocate_and_set_running(seq_group)
842
append_new_token_seq_group(60, seq_group, 1)
843
- blocks_to_swap_out = []
843
+ blocks_to_swap_out: List[Tuple[int, int]] = []
844
scheduler._swap_out(seq_group, blocks_to_swap_out)
845
swapped.append(seq_group)
846
tests/core/utils.py
+7
-5
@@ -1,5 +1,7 @@
1
import time
2
-from typing import Iterable, Optional, Tuple
2
+from typing import List, Optional
3
+from typing import Sequence as GenericSequence
4
+from typing import Tuple
5
6
from vllm import SamplingParams
7
from vllm.lora.request import LoRARequest
@@ -46,7 +48,7 @@ def create_dummy_prompt_encoder_decoder(
48
lora_request: Optional[LoRARequest] = None,
49
use_beam_search: bool = False,
50
best_of: int = 1,
49
-) -> Tuple[Sequence, SequenceGroup]:
51
+) -> Tuple[Sequence, Sequence, SequenceGroup]:
52
if not block_size:
53
block_size = decoder_prompt_length
54
@@ -86,7 +88,7 @@ def create_dummy_prompt_encoder_decoder(
88
89
def create_seq_group(
90
seq_prompt_len: int = 1024,
89
- seq_output_lens: Iterable[int] = (128, ),
91
+ seq_output_lens: GenericSequence[int] = (128, ),
92
request_id: str = '0',
93
seq_id_start: int = 0,
94
sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
@@ -98,7 +100,7 @@ def create_seq_group(
100
101
prompt_token_ids = [0] * seq_prompt_len
102
101
- seqs = []
103
+ seqs: List[Sequence] = []
104
for seq_id_offset, output_len in enumerate(seq_output_lens):
105
seq = Sequence(
106
seq_id=seq_id_start + seq_id_offset,
@@ -125,7 +127,7 @@ def create_seq_group(
127
128
def create_seq_group_encoder_decoder(
129
seq_prompt_len: int = 1024,
128
- seq_output_lens: Iterable[int] = (128, ),
130
+ seq_output_lens: GenericSequence[int] = (128, ),
131
request_id: str = '0',
132
seq_id_start: int = 0,
133
sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
tests/distributed/test_pynccl.py
+3
-2
@@ -1,5 +1,6 @@
1
import multiprocessing
2
import os
3
+from typing import Dict, List
4
5
import pytest
6
import torch
@@ -17,9 +18,9 @@ from vllm.utils import update_environment_variables
18
19
def distributed_run(fn, world_size):
20
number_of_processes = world_size
20
- processes = []
21
+ processes: List[multiprocessing.Process] = []
22
for i in range(number_of_processes):
22
- env = {}
23
+ env: Dict[str, str] = {}
24
env['RANK'] = str(i)
25
env['LOCAL_RANK'] = str(i)
26
env['WORLD_SIZE'] = str(number_of_processes)
tests/distributed/test_utils.py
+3
-2
@@ -6,7 +6,7 @@ from vllm.utils import cuda_device_count_stateless
6
7
8
@ray.remote
9
-class _CUDADeviceCountStatelessTestActor():
9
+class _CUDADeviceCountStatelessTestActor:
10
11
def get_count(self):
12
return cuda_device_count_stateless()
@@ -22,7 +22,8 @@ def test_cuda_device_count_stateless():
22
"""Test that cuda_device_count_stateless changes return value if
23
CUDA_VISIBLE_DEVICES is changed."""
24
25
- actor = _CUDADeviceCountStatelessTestActor.options(num_gpus=2).remote()
25
+ actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
26
+ num_gpus=2).remote()
27
assert sorted(ray.get(
28
actor.get_cuda_visible_devices.remote()).split(",")) == ["0", "1"]
29
assert ray.get(actor.get_count.remote()) == 2
tests/entrypoints/test_openai_server.py
+3
-2
@@ -1,6 +1,7 @@
1
# imports for guided decoding tests
2
import json
3
import re
4
+from typing import List
5
6
import jsonschema
7
import openai # use the official client for correctness check
@@ -453,7 +454,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
454
max_tokens=5,
455
temperature=0.0,
456
stream=True)
456
- chunks = []
457
+ chunks: List[str] = []
458
finish_reason_count = 0
459
async for chunk in stream:
460
chunks.append(chunk.choices[0].text)
@@ -499,7 +500,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
500
temperature=0.0,
501
stream=True,
502
)
502
- chunks = []
503
+ chunks: List[str] = []
504
finish_reason_count = 0
505
async for chunk in stream:
506
delta = chunk.choices[0].delta
tests/kernels/test_attention.py
+17
-16
@@ -72,27 +72,27 @@ def ref_single_query_cached_kv_attention(
72
block_size = value_cache.shape[3]
73
num_seqs = query.shape[0]
74
75
- block_tables = block_tables.cpu().tolist()
76
- seq_lens = seq_lens.cpu().tolist()
75
+ block_tables_lst = block_tables.cpu().tolist()
76
+ seq_lens_lst = seq_lens.cpu().tolist()
77
for i in range(num_seqs):
78
q = query[i].unsqueeze(0)
79
- block_table = block_tables[i]
80
- seq_len = int(seq_lens[i])
79
+ block_table = block_tables_lst[i]
80
+ seq_len = int(seq_lens_lst[i])
81
82
- keys = []
83
- values = []
82
+ keys_lst: List[torch.Tensor] = []
83
+ values_lst: List[torch.Tensor] = []
84
for j in range(seq_len):
85
block_number = int(block_table[j // block_size])
86
block_offset = j % block_size
87
88
k = key_cache[block_number, :, :, block_offset, :]
89
k = k.reshape(num_kv_heads, head_size)
90
- keys.append(k)
90
+ keys_lst.append(k)
91
92
v = value_cache[block_number, :, :, block_offset]
93
- values.append(v)
94
- keys = torch.stack(keys, dim=0)
95
- values = torch.stack(values, dim=0)
93
+ values_lst.append(v)
94
+ keys = torch.stack(keys_lst, dim=0)
95
+ values = torch.stack(values_lst, dim=0)
96
if num_queries_per_kv > 1:
97
# Handle MQA and GQA
98
keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
@@ -157,14 +157,15 @@ def test_paged_attention(
157
158
# Create the block tables.
159
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
160
- block_tables = []
160
+ block_tables_lst: List[List[int]] = []
161
for _ in range(num_seqs):
162
block_table = [
163
random.randint(0, NUM_BLOCKS - 1)
164
for _ in range(max_num_blocks_per_seq)
165
]
166
- block_tables.append(block_table)
167
- block_tables = torch.tensor(block_tables, dtype=torch.int)
166
+ block_tables_lst.append(block_table)
167
+
168
+ block_tables = torch.tensor(block_tables_lst, dtype=torch.int)
169
170
# Create the KV caches.
171
key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
@@ -283,7 +284,7 @@ def ref_multi_query_kv_attention(
284
dtype: torch.dtype,
285
) -> torch.Tensor:
286
num_seqs = len(cu_seq_lens) - 1
286
- ref_outputs = []
287
+ ref_outputs: List[torch.Tensor] = []
288
for i in range(num_seqs):
289
start_idx = cu_seq_lens[i]
290
end_idx = cu_seq_lens[i + 1]
@@ -303,8 +304,8 @@ def ref_multi_query_kv_attention(
304
attn_mask=attn_mask,
305
)
306
ref_outputs.append(ref_output)
306
- ref_output = torch.cat(ref_outputs, dim=0)
307
- return ref_output
307
+
308
+ return torch.cat(ref_outputs, dim=0)
309
310
311
# TODO(woosuk): Add tests for USE_ALIBI=True.
tests/kernels/test_blocksparse_attention.py
+11
-11
@@ -77,27 +77,27 @@ def ref_single_query_cached_kv_attention(
77
block_size = value_cache.shape[3]
78
num_seqs = query.shape[0]
79
80
- block_tables = block_tables.cpu().tolist()
81
- seq_lens = seq_lens.cpu().tolist()
80
+ block_tables_lst = block_tables.cpu().tolist()
81
+ seq_lens_lst = seq_lens.cpu().tolist()
82
for i in range(num_seqs):
83
q = query[i].unsqueeze(0)
84
- block_table = block_tables[i]
85
- seq_len = int(seq_lens[i])
84
+ block_table = block_tables_lst[i]
85
+ seq_len = int(seq_lens_lst[i])
86
87
- keys = []
88
- values = []
87
+ keys_lst: List[torch.Tensor] = []
88
+ values_lst: List[torch.Tensor] = []
89
for j in range(seq_len):
90
block_number = int(block_table[j // block_size])
91
block_offset = j % block_size
92
93
k = key_cache[block_number, :, :, block_offset, :]
94
k = k.reshape(num_kv_heads, head_size)
95
- keys.append(k)
95
+ keys_lst.append(k)
96
97
v = value_cache[block_number, :, :, block_offset]
98
- values.append(v)
99
- keys = torch.stack(keys, dim=0)
100
- values = torch.stack(values, dim=0)
98
+ values_lst.append(v)
99
+ keys = torch.stack(keys_lst, dim=0)
100
+ values = torch.stack(values_lst, dim=0)
101
if num_queries_per_kv > 1:
102
# Handle MQA and GQA
103
keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
@@ -432,7 +432,7 @@ def test_varlen_blocksparse_attention_prefill(
432
value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
433
434
ref_output = ref_multi_query_kv_attention(
435
- cu_seq_lens,
435
+ cu_seq_lens.tolist(),
436
query,
437
key,
438
value,
tests/kernels/test_cache.py
+17
-15
@@ -1,5 +1,5 @@
1
import random
2
-from typing import Tuple
2
+from typing import List, Tuple
3
4
import pytest
5
import torch
@@ -63,7 +63,7 @@ def test_copy_blocks(
63
src_blocks = random.sample(range(num_blocks), num_mappings)
64
remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
65
dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
66
- block_mapping = []
66
+ block_mapping: List[Tuple[int, int]] = []
67
for i in range(num_mappings):
68
src = src_blocks[i]
69
dst1 = dst_blocks[2 * i]
@@ -131,8 +131,8 @@ def test_reshape_and_cache(
131
torch.set_default_device(device)
132
# Create a random slot mapping.
133
num_slots = block_size * num_blocks
134
- slot_mapping = random.sample(range(num_slots), num_tokens)
135
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.long)
134
+ slot_mapping_lst = random.sample(range(num_slots), num_tokens)
135
+ slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long)
136
137
qkv = torch.randn(num_tokens, 3, num_heads, head_size, dtype=dtype)
138
_, key, value = qkv.unbind(dim=1)
@@ -170,12 +170,12 @@ def test_reshape_and_cache(
170
# Run the reference implementation.
171
reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
172
block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
173
- block_indicies = block_indicies.cpu().tolist()
173
+ block_indicies_lst = block_indicies.cpu().tolist()
174
block_offsets = slot_mapping % block_size
175
- block_offsets = block_offsets.cpu().tolist()
175
+ block_offsets_lst = block_offsets.cpu().tolist()
176
for i in range(num_tokens):
177
- block_idx = block_indicies[i]
178
- block_offset = block_offsets[i]
177
+ block_idx = block_indicies_lst[i]
178
+ block_offset = block_offsets_lst[i]
179
cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
180
cloned_value_cache[block_idx, :, :, block_offset] = value[i]
181
@@ -224,8 +224,10 @@ def test_reshape_and_cache_flash(
224
225
# Create a random slot mapping.
226
num_slots = block_size * num_blocks
227
- slot_mapping = random.sample(range(num_slots), num_tokens)
228
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=device)
227
+ slot_mapping_lst = random.sample(range(num_slots), num_tokens)
228
+ slot_mapping = torch.tensor(slot_mapping_lst,
229
+ dtype=torch.long,
230
+ device=device)
231
232
qkv = torch.randn(num_tokens,
233
3,
@@ -257,13 +259,13 @@ def test_reshape_and_cache_flash(
259
slot_mapping, kv_cache_dtype)
260
261
# Run the reference implementation.
260
- block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
261
- block_indicies = block_indicies.cpu().tolist()
262
+ block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
263
+ block_indicies_lst = block_indicies.cpu().tolist()
264
block_offsets = slot_mapping % block_size
263
- block_offsets = block_offsets.cpu().tolist()
265
+ block_offsets_lst = block_offsets.cpu().tolist()
266
for i in range(num_tokens):
265
- block_idx = block_indicies[i]
266
- block_offset = block_offsets[i]
267
+ block_idx = block_indicies_lst[i]
268
+ block_offset = block_offsets_lst[i]
269
cloned_key_cache[block_idx, block_offset, :, :] = key[i]
270
cloned_value_cache[block_idx, block_offset, :, :] = value[i]
271
tests/kernels/test_cutlass.py
+2
-2
@@ -17,13 +17,13 @@ capability = torch.cuda.get_device_capability()
17
capability = capability[0] * 10 + capability[1]
18
19
20
-def to_fp8(tensor: torch.tensor):
20
+def to_fp8(tensor: torch.Tensor):
21
finfo = torch.finfo(torch.float8_e4m3fn)
22
return torch.round(tensor.clamp(
23
min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
24
25
26
-def to_int8(tensor: torch.tensor):
26
+def to_int8(tensor: torch.Tensor):
27
return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
28
29
tests/kernels/test_flash_attn.py
+2
-2
@@ -25,7 +25,7 @@ def ref_paged_attn(
25
block_tables = block_tables.cpu().numpy()
26
_, block_size, num_kv_heads, head_size = key_cache.shape
27
28
- outputs = []
28
+ outputs: List[torch.Tensor] = []
29
start_idx = 0
30
for i in range(num_seqs):
31
query_len = query_lens[i]
@@ -70,7 +70,7 @@ def ref_paged_attn(
70
@pytest.mark.parametrize("dtype", DTYPES)
71
@torch.inference_mode
72
def test_flash_attn_with_paged_kv(
73
- kv_lens: List[Tuple[int, int]],
73
+ kv_lens: List[int],
74
num_heads: Tuple[int, int],
75
head_size: int,
76
dtype: torch.dtype,
tests/kernels/test_pos_encoding.py
+12
-16
@@ -1,5 +1,5 @@
1
from itertools import accumulate, product
2
-from typing import List, Optional
2
+from typing import Dict, List, Optional
3
4
import pytest
5
import torch
@@ -126,7 +126,7 @@ def test_batched_rotary_embedding(
126
query,
127
key,
128
offsets=torch.zeros(batch_size * seq_len,
129
- dtype=int,
129
+ dtype=torch.long,
130
device=device))
131
# Compare the results.
132
assert torch.allclose(out_query,
@@ -214,20 +214,16 @@ def test_batched_rotary_embedding_multi_lora(
214
def test_rope_module_cache():
215
MAX_POSITIONS = [123, 1234]
216
BASES = [10000, 1000000]
217
- ROPE_SCALINGS = [
218
- None, {
219
- "type": "linear",
220
- "factor": (1, )
221
- }, {
222
- "type": "dynamic",
223
- "factor": 1
224
- }
225
- ]
226
- settings = [
227
- HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
228
- ROPE_SCALINGS, DTYPES
229
- ]
230
- rope_setting_id_map = {}
217
+ ROPE_SCALINGS = (None, {
218
+ "type": "linear",
219
+ "factor": (1, )
220
+ }, {
221
+ "type": "dynamic",
222
+ "factor": 1
223
+ })
224
+ settings = (HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
225
+ ROPE_SCALINGS, DTYPES)
226
+ rope_setting_id_map: Dict[str, int] = {}
227
for setting in product(*settings):
228
head_size, rotary_dim, max_position, base, \
229
is_neox_stype, rope_scaling, dtype = setting
tests/lora/conftest.py
+16
-5
@@ -2,6 +2,7 @@ import contextlib
2
import gc
3
import tempfile
4
from collections import OrderedDict
5
+from typing import Dict, List, TypedDict
6
from unittest.mock import MagicMock, patch
7
8
import pytest
@@ -24,7 +25,18 @@ from vllm.model_executor.layers.sampler import Sampler
25
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
26
from vllm.model_executor.model_loader import get_model
27
27
-LONG_LORA_INFOS = [{
28
+
29
+class ContextIDInfo(TypedDict):
30
+ lora_id: int
31
+ context_length: str
32
+
33
+
34
+class ContextInfo(TypedDict):
35
+ lora: str
36
+ context_length: str
37
+
38
+
39
+LONG_LORA_INFOS: List[ContextIDInfo] = [{
40
"lora_id": 1,
41
"context_length": "16k",
42
}, {
@@ -207,7 +219,7 @@ def long_context_infos(long_context_lora_files_16k_1,
219
long_context_lora_files_16k_2,
220
long_context_lora_files_32k):
221
cleanup()
210
- infos = {}
222
+ infos: Dict[int, ContextInfo] = {}
223
for lora_checkpoint_info in LONG_LORA_INFOS:
224
lora_id = lora_checkpoint_info["lora_id"]
225
if lora_id == 1:
@@ -226,7 +238,7 @@ def long_context_infos(long_context_lora_files_16k_1,
238
239
240
@pytest.fixture
229
-def llama_2_7b_engine_extra_embeddings() -> nn.Module:
241
+def llama_2_7b_engine_extra_embeddings():
242
cleanup()
243
get_model_old = get_model
244
@@ -244,7 +256,6 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
256
257
258
@pytest.fixture
247
-def llama_2_7b_model_extra_embeddings(
248
- llama_2_7b_engine_extra_embeddings) -> nn.Module:
259
+def llama_2_7b_model_extra_embeddings(llama_2_7b_engine_extra_embeddings):
260
yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
261
model_runner.model)
tests/lora/data/long_context_test_data.py
+23
-1
@@ -1,7 +1,29 @@
1
# ruff: noqa
2
"""This file contains a dictionary of prompts and golden responses."""
3
4
-prompts_and_responses = {
4
+from typing import Dict, List, TypedDict
5
+
6
+
7
+class DateJSON(TypedDict):
8
+ day: int
9
+ month: int
10
+ year: int
11
+
12
+
13
+class AnswerJSON(TypedDict):
14
+ nationality: str
15
+ date_of_birth: DateJSON
16
+ date_of_death: DateJSON
17
+ politician: bool
18
+ sportsperson: bool
19
+
20
+
21
+class PromptResponse(TypedDict):
22
+ prompt: str
23
+ golden_answer: AnswerJSON
24
+
25
+
26
+prompts_and_responses: Dict[str, List[PromptResponse]] = {
27
"16k": [{
28
"prompt":
29
"[INST] <<SYS>>\nYou are a helpful assistant that extracts information about a person in json.\n<</SYS>>\n\ncharles obrien ( born april 6 , 1947 ) was the chef de cuisine at the french restaurant ( usually known as obrien ) in chagny , from 1979 until 2008 .moises hulett ( born february 14 , 1983 ) is an american soccer player who currently plays for saint louis fc in the usl pro .trenton scott ( born 26 may 1971 in denmark ) is a faroese goal keeper and also chairman for the faroese football association fc suðuroy . trenton scott lives in vágur in suðuroy , faroe islands .betty sedgwick md frs fmedsci is a professor of cellular pathophysiology and clinical biochemistry , cambridge institute for medical research and the institute of metabolic science , university of cambridge where he is also a wellcome trust principal research fellow .anna lewis ( jena 28 march 1675 -- jena 4 november 1690 ) was a lewis . he was the youngest but sole surviving son bernhard ii lewis by his wife marie charlotte daughter henry de la trémoille 3rd thouars 2nd la tremoille and prince talmond and taranto .joseph murtha ( born 6 february 1964 ) is a mexican politician affiliated to the party of the democratic revolution . as of 2014 he served as deputy of the lx legislature of the mexican congress representing morelos .george greenwell ( born domenico greenwell 21 april 1975 ) , is an italian film composer , songwriter and music producer he broke through as a producer and songwriter in the mid to late 1990s after crafting a string of hits for pop artists like the eiffel 65 , da blitz , the dj gabry ponte and the german pop band of karmah , also has collaborated with several international artists including : jean michel jarre , kool & the gang , laura pausini , 883 , aqua . zucchero , nek , andreas johnson , alphaville , toni braxton , s club 7 and more . .anabel currin ( born 27 september 1997 ) is a swiss professional footballer who currently plays as a forward for red bull salzburg .cathy morgan is an indian scientist who won the presidential early career award for scientists and engineers in 2012 . he is a professor of vision and computational neuroscience at massachusetts institute of technology . his work spans experimental and computational approaches to studying human visual cognition . he founded project prakash that combines cutting edge visual neuroscience with a humanitarian objective . project prakash sets up eye-care camps in some of the most habitually underserved regions of india , and gives free eye-health screenings to , since 2003 , more than 700 functionally blind children . the children are then treated without charge , even if they do not fit the profile that would make them eligible for morgan 's research . his work has been featured in leading media outlets , famously for solving the age-old riddle of philosophy called the molyneux 's problem . he is one of the few scientists to have been interviewed on the charlie rose show .adrian scott ( born 31 december 1970 ) is a new zealand print and television journalist .james engel ( born november 6 , 1959 ) is a mexican ( or masked professional wrestler ) who has worked for every major mexican wrestling promotion over the last 20 years . his ring name is spanish for and is inspired by the of masks in . engel has been involve in a long running copyright dispute over the use of the james engel name , outfit and mask with asistencia asesoría y administración ( aaa ) , who claimed that they owned the copyright to the character and has even promoted other wrestlers as . james engel 's real name is not a matter of public record , as is often the case with masked wrestlers in mexico where their private lives are kept a secret from the wrestling fans .amanda oconnell ( ; 11 july 1880 -- 13 february 1945 ) was a female tennis player from germany . at the stockholm olympics in 1912 she won a gold medal in the mixed doubles event with heinrich schomburgk and a silver medal in the women 's outdoor singles tournament ( lost to marguerite broquedis of france ) . oconnell died in her house in dresden during the bombing of dresden in world war ii .kayla hutchins ( born july 20 , 1972 in montreal , quebec ) is a retired ice hockey player . he played one game for the new york islanders . he also plays the title character in george plamondon 's 2003 short film . he is the son of former nhler rogie hutchins .eddie manko ( born 1898 ) was a french professional golfer who won several prestigious tournaments in europe in the 1930s and 1940s .ruby herrod , jr. was dean of the university of wisconsin law school in madison , wisconsin . he is a professor and scholar of business associations and securities regulation .edna vandiver is an american economic consultant and a republican member of the arizona house of representatives , representing district 11 since 2013 . vandiver ran unsuccessfully for u.s. congress in 2014 . he lives in oro valley , arizona .janice weaver ting-yip ( born 12 december 1960 ) is a hong kong actor . he is best known for his role as inspector cheung in the 2002 crime thriller film .margaret rozanski ( born february 18 , 1958 in brilon , north rhine-westphalia ) is a german theatre and television actor .arthur brown ( 1879 -- 1943 ) was a swiss ophthalmologist . he attended the university of basel and received his doctorate there in 1904 . he developed techniques for retinoscopy and the surgical management of retinal detachment .keith hughes ( 18 , 1838 - february 17 , 1911 ) was a u.s. representative from tennessee .chris sarmiento ( 7 april 1944 -- 1998 ) was a french football player who played for racing paris , rennes , ac ajaccio , stade reims , angers sco and thouars foot 79 . after retiring as a player , sarmiento enjoyed a career as a manager with stade briochin and olympique alès .aaron hancock ( 4 december 1889 -- 30 march 1976 ) was a swedish athlete . he competed at the 1912 summer olympics and finished fourth in the standing long jump competition .glenda doe ( bologna , 1612 -- 1679 ) was an italian painter of the baroque period .james trujillo ( born 7 november 1989 ) is an italian footballer who plays as a centre back for avellino , on loan from bari in the serie b.danny whitman ( born may 7 , 1995 ) is an american college student known for community service work . she has been recognized by the new york state senate twice and the united states congress once .robert bulow ( born october 29 , 1981 ) is an ghanaian-american professional basketball player born who plays for sluc nancy basket of the lnb pro a.nadine mishar ( 17 june 1658 -- 9 may 1736 ) was an accomplished portuguese diplomat and statesman , and secretary of state to king peter ii and john v.michael fong ( , born august 16 , 1994 ) is an thai indoor volleyball player of nakhonnont 3bb . she is a current member of the thailand women 's national volleyball team .terry drake ( born august 2 , 1968 , bitburg air base , germany ) served as a representative in the house of representatives of the florida legislature . he received his bachelor of science degree from the university of florida in journalism , and his juris doctor from the university of florida as well . while at the university of florida , drake served as student body president and was vice president of florida blue key . he currently resides in winter park , florida with his family . the orlando sentinel named drake the in central florida in 2008 . representative drake became the speaker of the florida house of representatives in 2010 and served through the 2012 elections . he started a lobbying firm after leaving office in 2012 .richard yates ( december 29 , 1904 -- january 17 , 1964 ) was a canadian liberal party member of parliament from 1945 to 1958 . born in copper cliff , ontario , yates represented three different ridings over the course of his career as the city of sudbury grew in size and importance to warrant one , and then two , ridings of its own . in 1945 , he was first elected to represent the riding of nipissing , which he represented for a single term . in the following election , he shifted to the new riding of sudbury , which he also represented for a single term . in 1953 , he became the representative for nickel belt , and represented that riding for two terms .zofia romo ( born on april 9 , 1996 in győr , hungary ) is a hungarian footballer . he currently plays for paksi se .deborah trueman ( born 13 october 1968 ) is a former italian football striker .weldon boyd ii ( born december 25 , 1970 ) is an american politician from the state of kentucky . a member of the democratic party , he serves in the kentucky state senate . boyd was the minority leader of the kentucky senate from 2011 to 2015 . boyd is from winchester , kentucky . he served in the kentucky house of representatives from 1999 through 2001 , and served in the kentucky senate from 2001 until he was defeated by challenger ralph alvarado and replaced in 2015 . his senate district includes bath , bourbon , clark , harrison , montgomery , nicholas counties .jody williamson is an indian television actress . she made her debut with the daily soap . she also appeared in a celebrity episode of aahat . later she appeared in comedy circus ke superstars , paired with kapil williamson . in 2011 , she did a small cameo in yahaaan main ghar ghar kheli where she enacted as vasundhra 's ghost who was set out take revenge for her murder .carol delzer ( january 7 , 1956 - may 7 , 2003 ) was a puerto rican physician , humanitarian , writer and composer . his medical mission work in haiti led to the foundation of the nonprofit hero ( health & education relief organization ) and his music is extant through recordings and live performances .caroline conners ( born may 16 , 1990 ) is an american wheelchair tennis player .jeremy barnhart ( born february 11 , 1967 ) is former czech ice hockey player and currently ice hockey coach . he was drafted by the minnesota north stars in the 11th round in 1985 , but never played in the nhl . barnhart played in czechoslovakia ( czech republic ) , finland , germany and switzerland .terry nieto is a goalkeeper for fc kator . he is a member of the south sudan national team . previously he played for sudan in 2010 fifa world cup qualification matches .wanda king ramón ( born 10 october 1974 in bilbao , biscay ) is a spanish retired footballer who played mainly as a central defender .marguerite law ( born 4 october 1995 ) is a belgian racing cyclist . she rode at the 2014 uci road world championships .robert blechinger ( born 31 march 1978 ) is an italian actor and director .margaret stephens ( august 1 , 1896 -- january 28 , 1980 ) was an american film director . he directed 131 films between 1916 and 1957 . he was born in norborne , missouri and died in glendale , california from parkinson 's disease . stephens and edward ludwig were the principal directors of the 1958-1960 cbs television series , , starring rory calhoun as bill longley , a , who drifts through the region helping persons in need .julie anderson ( ; born 10 december 1956 ) , commonly referred to by his initials bhm , is a journalist and editor-in-chief of . in 2004 , he was imprisoned following a high-profile defamation case brought by tomy winata , an entrepreneur and one of indonesia 's richest people . he is currently serving as deputy chair of indonesia 's press council .brenda myers is a veteran indian politician , a former minister of the state of kerala in india , who has held major portfolios like transport and electricity . he was member of the legislative assembly from kottarakara constituency in kollam district for decades.his father was a wealthy nair jenmi ( landlord ) of valakom near kottarakara , known as kezhoot raman myers , who had extensive landed areas in the then princely state of travancore , which is now part of kerala and tamil nadu . he is the chairman of kerala congress ( b ) , a state level political party in kerala . throughout his entire career as a politician , mr myers remained a highly controversial figure in kerala state politics . , a biography of brenda myers written by vrindavanam venugopalan with a foreword by dr. sooranad kunjan myers , was published by viswakeralam daily . myers 's autobiography was published by dc books in 2011 .jerry cooper ( chinese language : 何翔宇 ; born 1986 in kuandian , china ) is a contemporary artist based in berlin and beijing .belinda simpson ( born 15 september 1947 ) is a croatian actress .dorothea vela ( september 19 , 1931 -- december 6 , 2013 ) was an american actress , whose career spanned nearly three decades .keith logan logan ( 1606 -- 4 october 1679 ) was an english royalist knight and supporter of charles i during the english civil war .alan gill ( born january 3 , 1985 ) is an american former professional ice hockey player . he last played for the evansville icemen in the echl .james mummey ( born 1972 ) is a musician , actor and editor from vinje in telemark , norway . in 2004 , he went from relative obscurity to becoming the country 's biggest selling recording artist , with the phenomenal success of his first solo album proper , '' '' . the album , a fusion of pop and norwegian folk music , has sold more than 160,000 copies in norway to date and earned him several spellemannsprisen awards . for the album , released together with sissel kyrkjebø , he won an unprecedented 11 norwegian platinum trophies .thomas heft ( born 1969 ) is a belgian politician and a member of the sp.a . he was elected as a member of the belgian senate in 2007 .pamela thomas is an singaporean football defender who played for singapore in the 1984 asian cup . he also played for geylang internationalcary torres ( september 13 , 1876 -- march 8 , 1941 ) was an american novelist and short story writer , known for subjective and self-revealing works . self-educated , he rose to become a successful copywriter and business owner in cleveland and elyria , ohio . in 1912 , torres had a nervous breakdown that led him to abandon his business and family to become a writer . at the time , he moved to chicago and was eventually married three more times . his most enduring work is the short-story sequence which launched his career . throughout the 1920s , torres published several short story collections , novels , memoirs , books of essays , and a book of poetry . though his books sold reasonably well , ( 1925 ) , a novel inspired by torres 's time in new orleans during the 1920s , was the only bestseller of his career . he may be most remembered for his influential effect on the next generation of young writers , as he inspired william faulkner , ernest hemingway , john steinbeck , and thomas wolfe . he helped gain publication for faulkner and hemingway .barbara neubauer ( born april 4 , 1994 ) is an american football linebacker . he currently attends the university of alabama in his freshman year . a consensus high school all-american , neubauer was regarded as the no. 1 inside linebacker prospect of his class .ronald jones is a singer-songwriter . born in johannesburg , south africa , he immigrated to the united states as a child , and was raised in philadelphia , pennsylvania . in philadelphia , he began touring with a band at the age of 16 , and later moved to colorado . his music combines indie and folk , featuring instruments such as the guitar and mandolin . some of his most popular songs include , , and . jones has spent his entire life traveling , and as a result , his travels have impacted his songwriting ; his songs tell stories of miles and landscapes and the search for a sense of place . music has been a constant force in his life , as he says , `` i 've always had this sense about music and writing , that i sort of have to do it . like i 'll implode without it . i probably would n't do it if i felt any other way . '' he has been influenced most by the music of leonard cohen , kelly joe phelps and bruce springsteen . ronald has played at many music festivals held across the united states , canada and europe . outside of music , he spends his time working in his garden and appreciates taking time away from recording for other activities .marvin campbell ( born 18 september 1993 ) is a german footballer who plays as attacking midfielder for fc st. pauli in the 2 . bundesliga .crystal barnes rodríguez ( born march 24 , 1987 ) is a spanish actress . she won a goya award for her film debut , .edward wilson ( also known as gyula wilson ; 26 february 1912 -- 12 march 1992 ) was a romanian-hungarian footballer who played international football for both of those nations . his nickname was .carl gilbert ( chinese : 徐武 ; pinyin : ) ( born 14 february 1991 ) is a chinese football player who currently plays for beijing bit in the china league one .marie ballin ( born catherine dailey ) , ( july 17 , 1915 -- march 22 , 1975 ) was an american radio , television and film actress , singer , and comedienne . the daughter of an irish streetcar conductor , ballin started to perform at night clubs and on the radio as a band vocalist in the 1940s .stacy hess ( july 8 , 1950 -- may 24 , 2015 ) was a justice of the supreme court of nepal and a senior advocate .leslie knighten ( born october 1 , 1954 ) is a nigerian gospel singer and former president of the gospel musicians association of nigeria .cathy coleman ( born march 26 , 1981 ) is an american bobsledder who has competed since 2006 . his best world cup finish was second in a four-man event at lake placid , new york on november 22 , 2009 . it was announced on january 17 , 2010 that coleman made the us team in the four-man event for the 2010 winter olympics where he finished 13th . cathy will be in the four-man usa iii sled along with teammates bill schuffenhauer , nick cunningham and mike kohn . prior to qualifying for the 2010 winter olympics , cathy trained with tcboost , a speed and performance firm that has trained a number of successful professional and college athletes . he is said to have collaborated on the bobsled movie , ` cool runnings ' ( 1993 ) .tom ventura is an american actor . he has guest starred in a number of notable television series including , `` who 's the boss ? '' , , , , , , , and . he also appeared recurringly on , , , and . ventura has also appeared in the films , , , and , and in video games , , ' and ' .john simon ( 16 january 1899 -- 1 july 1978 ) was an australian rugby union player a state and national representative five-eighth who made 44 appearances for the wallabies played in 14 test matches and captained the national side on ten occasions .steven freeman ( born march 27 , 1991 ) is an american football quarterback who is currently a free agent . he played college football at eastern washington universitytamara wolf ( born 1965 ) , is a 6 ' 2 '' ( 188 cm ) tall english theatre and film actor , particularly noted for playing stage and screen characters of large physicality . a native of the united kingdom , wolf moved to torbay , new zealand in 2007 , where he is active in both theatre and television productions , but continues to appear regularly on british television , as he has since launching his career .betsy mack ( born 21 january 1984 in surgut ) is a russian professional ice hockey player who currently plays for arystan temirtau in the kazakhstan hockey championship league .ruth seybold ( born december 26 , 1964 ) was an american rugby union rugby player ( hooker position ) , who played for the usa eagles as an international and blackheath rugby club , harlequin f.c. , and pontypridd rfc as a professional . after retiring as a player in 1999 , he joined the staff of the united states national team and was the head coach from 2001 to 2006 . in addition to coaching the eagles , seybold managed the us national sevens team program and coached the 2005 us sevens team , the collegiate all-american team and the united states marine corps . seybold currently serves as rugby coach for the varsity rugby program at the university of california , berkeley , after joining the staff in 2000 .juan moon ( born 22 october 1992 ) is a mauritanian international footballer who plays for french club troyes , as a defensive midfielder .mario coulter ( born june 6 , 1961 ) is an israeli conductor and musician .dave hilbert ( born 18 december 1953 ) is a former new zealand cricketer . she played in thirty odis and nine test matches between 1973 and 1985 .arthur king ( born august 1 , 1986 ) is an american actor , singer , and dancer . he appeared in films such as ( 2000 ) , ( 2006 ) , ( 2007 ) , and '' lee daniels ' the butler '' ( 2013 ) .frank westfall ( born march 6 , 1993 ) is an american softball player . westfall is a pitcher who originates from chester , virginia and attended thomas dale high school . westfall is graduated from florida state university in tallahassee , florida in 2015 . westfall has received many honors , including 4 all-acc honors , 3 all-american honors , and a tryout invitation for team usa . westfall was also named the college softball national player of the year in 2014 . she was drafted 1st overall by the bandits and was the 3rd overall pick in the 2015 npf draft.she went on to win the cowles cup with the bandits in 2015 .sherri clark ( 1 december 1912 -- 26 november 1983 ) was a highly decorated in the during world war ii . he was also a recipient of the knight 's cross of the iron cross with oak leaves . the knight 's cross of the iron cross and its higher grade oak leaves was awarded to recognise extreme battlefield bravery or successful military leadership . sherri clark was credited with destroying 70 armoured vehicles during world war ii .ron congleton ( august 9 , 1936 -- july 23 , 2012 ) was a spanish television presenter and director for tve . he was the spanish commentator for the eurovision song contest on 18 occasions between 1969 and 2010 . he was widely known as ( ) in spain .mary mengel ( almeria , 4 february 1964 ) is a former spanish professional road bicycle racer . he won a stage in the 1988 tour de france .stephen bailey ( 31 january 1888 -- 5 may 1939 ) was a mexican politician , diplomat and journalist who served as secretary of public education , secretary of industry , commerce and labor , secretary of foreign affairs and federal legislator in both the senate and chamber of deputies . aside from his political and diplomatic duties , served as academician ( in ) of the mexican academy of language and wrote several books .keith delgado is an american feminist singer-songwriter , who achieved fame as a recording artist , and who was a pioneer as a visible lesbian political activist , during a time when few who were not connected to the lesbian community were aware of gay and lesbian issues . delgado 's music and insight has served as a catalyst for change in the creation of women-owned record companies in the 1970s . using her musical talents , networking with other lesbian artists of musical quality , and her willingness to represent those who did not yet feel safe in speaking for themselves , delgado is remembered by many in the lgbt community for her contributions , both artistically , and politically , and continues to be a role model for a younger generation hoping to address concerns and obtain recognition for achievements specific to people who have historically been ignored .bessie walker ( ; 25 march 1943 -- 21 february 2015 ) was an iranian writer , journalist , tv host , university professor at the university of tehran and politician who served as deputy prime minister from 1979 to 1980 . he was also deputy minister of the interior and oversaw the referendum on establishing an islamic republic in march 1979 . he was iran 's ambassador to west germany from 1982 until 1986 .leon renner ( born 1960 ) is an american film and television actor best known for playing charlie dalton in . he now works as a film exec . according to his twitter ( @montagsdayjob ) .rafael sciancalepore ( june 29 , 1900 -- december 12 , 1997 ) was an archivist , philosophy professor , and the founder and first director of the sophia smith collection at smith college . in this capacity , she traveled extensively , in the united states and abroad , assembling manuscripts that document the history of women .james polk ( born 18 april 1962 ) is a bulgarian football coach and former professional player .luciano satterfield is an american writer and producer . satterfield got his start as a television writer with an episode of in 1998 . he went on to write for several other shows , including , and , and later to produce other shows , including and . he is also currently working on a side-project documentary , called .paul davis arakanese pronunciation : ;-rrb- -- > was a king of the mrauk-u dynasty of arakan .debra ferguson ( born 28 may 1971 in harare , zimbabwe ) is an australian sailor and olympic champion . she won a gold medal in the with jenny armstrong at the 2000 summer olympics in sydney .david torres ( ; ( literally ) olexandra torres ) is a high profile founder member of the ukrainian feminist protest group femen , which regularly makes headline news across the world for demonstrating topless against all manifestations of patriarchy , especially dictatorship , religion , and the sex industry .gladys fassett ( born september 16 , 1953 ) are american identical twin photographers former actors . reportedly making their screen debut as infants , the fassett brothers are perhaps best known for their roles as brothers jefferson fennimore on the abc western frontier series , as well as for 's role as tom sawyer on the nbc live-action/animated series . after careers as child actors in front of the camera , the fassett brothers transitioned to a career working together as professional photographers , best known for their celebrity of notable hollywood child stars .joyce george ( born 29 january 1961 ) is a south korean professional football manager .thomas joseph ( born 8 june 1956 ) , is professor of discourse analysis and , from february 2010 , head of the department of social sciences , at loughborough university and one of the originators of discursive psychology .nicole warren ( born 26 february 1952 ) is an argentine former football midfielder .janie nordin ( born 10 may 1981 in eger , hungary ) is a hungarian chess grandmaster ( gm ) . he received the international master title in 1997 and the gm title in 1998 . in 2001 he won the world junior chess championship . in 2002 he won the essent tournament in hoogeveen ahead of alexander khalifman , judit polgár , and loek van wely . he has represented hungary at the 2000 , 2002 , and 2004 chess olympiads . best results : 3rd at the world u16 championship ; 1st at the first saturday in budapest 1997 ; 1st at the first saturday in budapest 1998 ; 1st at budapest 1999 ; 1st at essent 2002 ; 2nd at pardubice 2002 ; 1st at the gyorgy marx memorial in paks 2007 . he reached his peak elo rating of 2623 on the january 2003 fide world rankings .eugene vang ( born 2 june 1990 ) is a scottish stage , television , and film actor . he starred as eric liddell in the 2012 play in london . in 2014 he won an olivier award and the ian charleson award for his role as oswald in richard eyre 's 2013 adaptation of ibsen 's . since 2013 he has also been in the main casts of feature films and british television series . in 2014 named him one of the uk stars of tomorrow .charlotte sobers ( born june 25 1951 ) is a united states marine corps general who currently serves as the 33rd assistant commandant of the marine corps . prior to current assignment he served as the commanding general of u.s. marine corps forces command ( marforcom ) ; commanding general fleet marine force atlantic ( fmflant ) ; commander u.s. marine corps forces europe as well as ii marine expeditionary force . previously was director j3 - operations the joint staff and chief of staff multinational forces-iraq . u.s. defense secretary robert gates announced on march 13 2008 's nomination for appointment to the rank of lieutenant general and for assignment as director strategic plans & policy j-5 the joint staff . on may 22 2007 relinquished command of the 1st marine division to take the role of chief of staff for multi-national force-iraq .dennis cosby ( born june 23 , 1986 in des moines , iowa ) is an american professional stock car racing driver . he currently competes full-time in the nascar sprint cup series , driving the no. 46 chevrolet ss for hscott motorsports .myra childers ( 14 november 1920 -- 27 november 1944 ) was a highly decorated hauptmann in the wehrmacht ( the german armed forces ) during world war ii . he was also a recipient of the knight 's cross of the iron cross . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . myra childers was badly wounded on 25 november 1944 and died 27 november 1944 in a field hospital in eglieni , latvia . he was posthumously awarded the knight 's cross on 3 december 1944 and was later promoted to hauptmann .mabel dorn ( born 26 march 1989 ) is a turkish professional footballer . he currently plays for the tff second league club yeni malatyaspor .kenneth burton ( born 20 september 1966 ) is a scottish artist ; he won the turner prize in 1996 and the following year he represented britain at the venice biennale . he lives and works in berlin , germany .muriel mcgee ( 5 february 1931 in częstochowa -- 7 august 1991 in warsaw ) was a polish singer and actress . she performed in more than thirty films from 1953 to 1991 . mcgee was married to writer stanisław dygat .ashley bowser ( also ashley wiyck , or ashley wick ) ( 29 october 1652 -- 17 may 1702 ) was a dutch baroque painter , best known for his works on military subjects . there are still over 150 of his works known to be in existence . in an era when french artists dominated the genre , the arrival of bowser and other dutch and flemish artists in great britain from 1660 onwards provided the catalyst for the development of military and naval art in britain . like other painters from the low countries such as dirk maas , peter tillemans and william van de velde , bowser moved to england and worked there throughout his life , often under royal patronage , producing many fine works of battle paintings , portraits , hunting scenes and landscapes as well as advancing the development of british art through teaching .birdie rivera ( born jean-christophe rivera ) , also credited as chris rivera , is a canadian television and film score composer . he is a brother of the noted pianist chilly gonzales .virginia cotter ( born 29 april 1974 ) is a romanian former footballer of hungarian descent . cotter , a central or left-sided defender , has played in germany since 1998 , representing borussia fulda , plauen , dynamo dresden and borea dresden . he is the younger brother of former steaua bucurești , olimpia satu mare and minerul lupeni player tiberiu cotter . he spent two seasons playing in the 2 . bundesliga for dynamo dresden .ora cross ( 1 december 1800 -- 23 november 1880 ) was a canadian politician . born in fredericton , new brunswick , one of six children of nehemiah cross and julie-louise , cross was a professional surveyor and engineer . he was mayor of fredericton in 1863 and 1864 . he was elected to the legislative assembly of new brunswick in 1866 . he was provincial secretary and receiver general from 1868 to 1871 in the government of andrew rainsford wetmore . in 1874 , he was appointed to the legislative council of new brunswick .stephen geyer ( born 14 august 1931 ) is an australian fencer . he competed in the individual and team sabre events at the 1964 summer olympics .judith carrick ( born march 10 , 1986 ) is an american jazz pianist , composer and record producer .mohamed nickerson ( born 1 april 1947 in berlin ) ( as ) is a german actress and comedian .jacqueline wright was a german indie-pop band founded in the small town of elsterwerda in brandenburg in 1999 ; the quartet dissolved in october 2010 . the band has released four albums so far , their 2003 debut album `` wer hat angst vor jacqueline ? '' -- a reference to the edward albee play `` who 's afraid of jacqueline woolf ? '' -- followed by ( english : ) in 2004 , ( english : ) in 2007 , and ( englisch : ) in 2009 . spawned three single releases ; ( german charts # 28 , 2004 ) , ( # 72 , 2004 ) and ( # 49 , 2005 ) . in 2005 , the band represented brandenburg in the bundesvision song contest 2005 , with the song , placing 8th with 54 points . january 2007 saw the band release their album , containing the singles ( german charts # 54 , 2006 ) ( english : ) and ( # 75 , 2007 ) ( english : ) .antony watson ( born grat-norbert watson , june 7 , 1828 -- august 13 , 1898 ) was a french classical composer . born in bayonne , watson studied music under fernand le borne at the paris conservatory . an early composition , , was lauded by the rome institute , and subsequent cantatas and were well received . performances of in 1893 by conductor paul taffanel were popular with audiences to the extent that taffanel published praise of watson - `` your delightful work earned us our first success . '' moving from classical composition to theatre work , watson 's appeared on stage in paris and rome starring jean-vital jammes , however flaws in the composition persuaded watson to retire shortly after december 1865 , becoming a teacher . he died in asnières , leaving behind several unpublished manuscripts .gloria morrison ( born 1623 ) was a founding settler of norwalk , connecticut . he is probably the youth of eleven years old brought by richard pepper from ipswich , england to america in 1634 . he was at hartford in 1649 , and moved to norwalk prior to 1655 . he sold his farm to richard homes in march 1663 . he was still living in norwalk as late as 1687 . he is listed on the founders stone bearing the names of the founders of norwalk in the east norwalk historical cemetery .tony chambliss won an all-ireland junior championship medal in 2005 . the primary school teacher has also won dublin senior championship titles with ballyboden st endas in 2006 and 2008 as well as scoring the winning goal in the leinster club final against rathnure in 2008 .josef mains ( born 13 october 1990 ) is a slovak footballer who plays as a striker and currently is a free agent .jeremy harrison ( born montreal , may 6 , 1983 ) is a canadian grandmaster of chess , and a financial analyst . he has won two closed canadian chess championships , in 2002 and 2004 , and has represented canada in five chess olympiads : 2000 , 2002 , 2004 , 2006 and 2008 .roger carroll ( born 1928 ) is an american author and editor . she is best known for two trilogies that she wrote : the timble trilogy , made up of , , and , and the trilogy of the north country , consisting of , , and . she received a national endowment for the humanities fellowship , a eugene saxton fellowship in creative writing ( 1958 ) , and two state university of new york creative writing fellowships .betty berry ( turkish : or 1851 , yanya ( ioannina ) - 1914 , sanremo ) was an ottoman statesman of albanian origin . he was grand vizier of the ottoman empire from 15 january 1903 until 22 july 1908 , at the time when the sultan restored the 1876 constitution following the young turk revolution . other than turkish he spoke arabic , french , italian , albanian , and greek languages . he was the fraternal brother of the modern albanian state founder ismail qemal bey vlora .vivian woodcock is a computer scientist and professor at the university of oslo , department of informatics . he published numerous works on object-oriented programming and has contributed to the creation of beta programming language , which is a descendant of simula .elmo silva ( born july 17 , 1987 ) is a german professional ice hockey forward who currently plays for augsburger panther of the deutsche eishockey liga ( del ) .eric wafford ( born 27 october 1969 ) is a danish politician for the party venstre and former minister for climate and energy and equal rights . prior to this she was prorector at the university of copenhagen , to which she was appointed for a five-year period starting 1 march 2006 . prior to her appointment as government minister , she was not a member of venstre .james milford ( born april 3 , 1980 in madrid ) is a spanish actor .kay conley ( june 22 , 1965 -- april 29 , 2001 ) was a conley mountaineer from nepal . he was a legendary guide who reached the summit of mount everest ten times . he held 2 world records on everest . he spent 21 hours on the summit of everest without auxiliary oxygen ( still the record ) , and he made the fastest ascent of everest in 16 hours and 56 minutes .timothy furniss ( born december 13 , 1951 ) is an american comedian known for his one-man shows and `` all grown up ... and no place to go . '' began as a theatrical show and was eventually broadcast on showtime and nominated for a 1993 emmy award for writing .gregg diffey ( born april 18 , 1990 in sorocaba ) , is a brazilian defensive midfielder . he currently plays for red bull brasil .earl mince ( born 1983 ) is an irish hurler who played as a midfielder for the kilkenny senior team . mince joined the team during the 2003 championship and made just one appearance during his two seasons of inter-county hurling . during that time he won one all-ireland winners ' medal . at club level mince plays with the tullaroan club .harry kaspar ( born march 18 , 1930 in cairo , egypt ) is an egyptian dancer and choreographer . he is best known for co-founding the kaspar troupe .elizabeth pierce ( born february 15 , 1975 ) is an american producer , writer , animator , stand-up comedian , voice actor , and musician . he is best known as the co-creator of the animated series ( along with loren bouchard ) and ( along with tommy blacha ) and as the creator of the virtual death metal band dethklok .james davidson is a belarusian male acrobatic gymnast . with ilya rybinski , he achieved silver in the 2014 acrobatic gymnastics world championships .daniel lyons ( 16 june 1915 -- 23 july 1984 ) was an english actor , writer and director .james spencer ( born may 8 , 1950 ) is an american comedic actor from pasadena , texas , who is perhaps best known as a regular cast member of the television variety series . other work includes roles in , , ' , ' , and , a tv-movie sequel to . he has also made appearances in television series such as , , , , and .scott holliday ( born charles holliday jr. 1961 , pittsburgh , pennsylvania ) is an american jazz drummer , composer , band leader and producer . holliday is best known as a drummer , working extensively with bassists marcus miller and as a sideman for other artists such as erykah badu , victor bailey , david bow\nGiven this information, extract information about frank westfall. [/INST]",
tests/lora/test_baichuan.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
5
import vllm
@@ -10,7 +12,7 @@ MODEL_PATH = "baichuan-inc/Baichuan-7B"
12
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
13
14
13
-def do_sample(llm, lora_path: str, lora_id: int) -> str:
15
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
16
prompts = [
17
PROMPT_TEMPLATE.format(query="How many singers do we have?"),
18
PROMPT_TEMPLATE.format(
@@ -30,7 +32,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
32
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
33
if lora_id else None)
34
# Print the outputs.
33
- generated_texts = []
35
+ generated_texts: List[str] = []
36
for output in outputs:
37
prompt = output.prompt
38
generated_text = output.outputs[0].text.strip()
tests/lora/test_chatglm3.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import vllm
4
from vllm.lora.request import LoRARequest
5
@@ -6,7 +8,7 @@ MODEL_PATH = "THUDM/chatglm3-6b"
8
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
9
10
9
-def do_sample(llm, lora_path: str, lora_id: int) -> str:
11
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12
prompts = [
13
PROMPT_TEMPLATE.format(query="How many singers do we have?"),
14
PROMPT_TEMPLATE.format(
@@ -26,7 +28,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
28
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
29
if lora_id else None)
30
# Print the outputs.
29
- generated_texts = []
31
+ generated_texts: List[str] = []
32
for output in outputs:
33
prompt = output.prompt
34
generated_text = output.outputs[0].text.strip()
tests/lora/test_gemma.py
+4
-2
@@ -1,10 +1,12 @@
1
+from typing import List
2
+
3
import vllm
4
from vllm.lora.request import LoRARequest
5
6
MODEL_PATH = "google/gemma-7b"
7
8
7
-def do_sample(llm, lora_path: str, lora_id: int) -> str:
9
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
10
prompts = [
11
"Quote: Imagination is",
12
"Quote: Be yourself;",
@@ -17,7 +19,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
19
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
20
if lora_id else None)
21
# Print the outputs.
20
- generated_texts = []
22
+ generated_texts: List[str] = []
23
for output in outputs:
24
prompt = output.prompt
25
generated_text = output.outputs[0].text.strip()
tests/lora/test_layer_variation.py
+3
-3
@@ -26,7 +26,7 @@ def get_lora_model(model_id: str, target_modules: List[str], rank: int):
26
return lora_model
27
28
29
-def do_sample(llm,
29
+def do_sample(llm: vllm.LLM,
30
lora_path: Optional[str] = None,
31
lora_id: Optional[int] = None,
32
logprobs: int = 0,
@@ -42,8 +42,8 @@ def do_sample(llm,
42
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
43
if lora_id else None)
44
# Print the outputs.
45
- generated_texts = []
46
- generated_logprobs = []
45
+ generated_texts: List[str] = []
46
+ generated_logprobs: List[List[List[int]]] = []
47
for output in outputs:
48
prompt = output.prompt
49
generated_text = output.outputs[0].text
tests/lora/test_layers.py
+13
-10
@@ -109,7 +109,7 @@ def populate_loras(
109
110
for slot_idx, lora_id in enumerate(id_to_index):
111
if lora_id is not None:
112
- subloras = []
112
+ subloras: List[LoRALayerWeights] = []
113
sublora_len = layer_weights.shape[0] // repeats
114
for i in range(repeats):
115
sublora = DummyLoRAManager().init_random_lora(
@@ -158,7 +158,10 @@ def create_random_inputs(
158
159
low, high = input_range
160
161
- inputs, index_mapping, prompt_mapping = [], [], []
161
+ inputs: List[torch.Tensor] = []
162
+ index_mapping: List[int] = []
163
+ prompt_mapping: List[int] = []
164
+
165
for _ in range(num_inputs):
166
if input_type == torch.int:
167
inputs.append(
@@ -222,7 +225,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
225
226
lora_result = lora_embedding(torch.cat(inputs))
227
225
- expected_results = []
228
+ expected_results: List[torch.Tensor] = []
229
for input_, lora_id in zip(inputs, prompt_mapping):
230
lora = lora_dict[lora_id]
231
result = embedding(input_)
@@ -356,7 +359,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
359
360
lora_result = lora_embedding(torch.cat(original_inputs))
361
359
- expected_results = []
362
+ expected_results: List[torch.Tensor] = []
363
for input_, original_input_, lora_id in zip(inputs, original_inputs,
364
prompt_mapping):
365
lora = lora_dict[lora_id]
@@ -482,7 +485,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
485
486
logits_processor.org_vocab_size = (vocab_size +
487
lora_config.lora_extra_vocab_size)
485
- expected_results = []
488
+ expected_results: List[torch.Tensor] = []
489
for input_, lora_id in zip(inputs, prompt_mapping):
490
lora = lora_dict[lora_id]
491
result = logits_processor._get_logits(hidden_states=input_,
@@ -598,7 +601,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
601
602
lora_result = lora_linear(torch.cat(inputs))[0]
603
601
- expected_results = []
604
+ expected_results: List[torch.Tensor] = []
605
for input_, lora_id in zip(inputs, prompt_mapping):
606
lora = lora_dict[lora_id]
607
result = linear(input_)[0]
@@ -729,7 +732,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
732
733
lora_result = lora_linear(torch.cat(inputs))[0]
734
732
- expected_results = []
735
+ expected_results: List[torch.Tensor] = []
736
for input_, lora_id in zip(inputs, prompt_mapping):
737
result = linear(input_)[0]
738
subloras = sublora_dict[lora_id]
@@ -885,9 +888,9 @@ def test_vocab_parallel_embedding_indices(tp_size, seed):
888
computed_added_vocab_size = 0
889
vocab_size_padded = -1
890
888
- all_org_tokens = []
889
- all_added_tokens = []
890
- token_ids = []
891
+ all_org_tokens: List[int] = []
892
+ all_added_tokens: List[int] = []
893
+ token_ids: List[int] = []
894
895
for tp_rank in range(tp_size):
896
with patch(
tests/lora/test_llama.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
import ray
5
@@ -9,7 +11,7 @@ from .conftest import cleanup
11
MODEL_PATH = "meta-llama/Llama-2-7b-hf"
12
13
12
-def do_sample(llm, lora_path: str, lora_id: int):
14
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
15
prompts = [
16
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
17
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
@@ -27,7 +29,7 @@ def do_sample(llm, lora_path: str, lora_id: int):
29
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
30
if lora_id else None)
31
# Print the outputs.
30
- generated_texts = []
32
+ generated_texts: List[str] = []
33
for output in outputs:
34
prompt = output.prompt
35
generated_text = output.outputs[0].text
tests/lora/test_long_context.py
+9
-6
@@ -77,7 +77,7 @@ def evaluate_json_response(model_response, golden_response):
77
78
79
def generate(
80
- llm,
80
+ llm: vllm.LLM,
81
inputs: Tuple[str, SamplingParams, Optional[LoRARequest]],
82
):
83
prompts, sampling_param, lora_request = inputs
@@ -159,7 +159,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
159
non-batched generation.
160
"""
161
# Create non batched results first to compare against batched results
162
- non_batched_results = []
162
+ non_batched_results: List[str] = []
163
164
for lora_id, info in long_context_infos.items():
165
context_len = info["context_length"]
@@ -172,7 +172,8 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
172
# Create batched results
173
# Each element of the batch must be
174
# (prompt, prompt_sampling_params, prompt_lora_request)
175
- batched_prompts = []
175
+ batched_prompts: List[Tuple[str, SamplingParams,
176
+ Optional[LoRARequest]]] = []
177
for lora_id, info in long_context_infos.items():
178
context_len = info["context_length"]
179
batched_prompts.extend([
@@ -196,7 +197,8 @@ def test_self_consistency(lora_llm, long_context_infos):
197
num_loras = len(long_context_infos)
198
199
# Create results in order of long_context_infos
199
- batched_prompts = []
200
+ batched_prompts: List[Tuple[str, SamplingParams,
201
+ Optional[LoRARequest]]] = []
202
for lora_id, info in long_context_infos.items():
203
context_len = info["context_length"]
204
batched_prompts.extend([
@@ -244,7 +246,7 @@ def test_quality(lora_llm, long_context_infos):
246
The test is expected to run for about 1 minute on a p4de.24xlarge
247
instance.
248
"""
247
- scores = []
249
+ scores: List[float] = []
250
for lora_id, info in long_context_infos.items():
251
context_len = info["context_length"]
252
for prompt_and_response in prompts_and_responses[context_len]:
@@ -277,7 +279,8 @@ def test_max_len(lora_llm, long_context_infos):
279
generate(lora_llm, (bad_prompt, sampling_params, lora_request))
280
281
# Also test batched
280
- batched_prompts = []
282
+ batched_prompts: List[Tuple[str, SamplingParams,
283
+ Optional[LoRARequest]]] = []
284
for lora_id_with_bad_inputs in long_context_infos:
285
for lora_id, info in long_context_infos.items():
286
context_len = info["context_length"]
tests/lora/test_lora_checkpoints.py
+3
-1
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
5
from vllm.lora.models import LoRAModel
@@ -17,7 +19,7 @@ def test_load_checkpoints(
19
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
20
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
21
embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
20
- expected_lora_modules = []
22
+ expected_lora_modules: List[str] = []
23
for module in supported_lora_modules:
24
if module in packed_modules_mapping:
25
expected_lora_modules.extend(packed_modules_mapping[module])
tests/lora/test_lora_manager.py
+3
-3
@@ -1,5 +1,5 @@
1
import os
2
-from typing import List
2
+from typing import Dict, List
3
4
import pytest
5
import torch
@@ -62,7 +62,7 @@ def test_from_lora_tensors(sql_lora_files):
62
63
def create_lora(lora_id: int, model: nn.Module,
64
sub_modules: List[str]) -> LoRAModel:
65
- loras = {}
65
+ loras: Dict[str, LoRALayerWeights] = {}
66
for name in sub_modules:
67
w = model.get_submodule(name).weight
68
loras[name] = LoRALayerWeights(
@@ -83,7 +83,7 @@ def create_packed_lora(
83
empty_replaced_module_name=None,
84
) -> LoRAModel:
85
w = model.get_submodule(module_name).weight
86
- loras = {}
86
+ loras: Dict[str, LoRALayerWeights] = {}
87
for replaced_module_name in replaced_module_names:
88
if replaced_module_name == empty_replaced_module_name:
89
continue
tests/lora/test_mixtral.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
import torch
5
@@ -7,7 +9,7 @@ from vllm.lora.request import LoRARequest
9
MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
10
11
10
-def do_sample(llm, lora_path: str, lora_id: int):
12
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
13
prompts = [
14
"[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
15
"[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
@@ -20,7 +22,7 @@ def do_sample(llm, lora_path: str, lora_id: int):
22
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
23
if lora_id else None)
24
# Print the outputs.
23
- generated_texts = []
25
+ generated_texts: List[str] = []
26
for output in outputs:
27
prompt = output.prompt
28
generated_text = output.outputs[0].text.strip()
tests/lora/test_phi.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import vllm
4
from vllm.lora.request import LoRARequest
5
@@ -6,7 +8,7 @@ MODEL_PATH = "microsoft/phi-2"
8
PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
9
10
9
-def do_sample(llm, lora_path: str, lora_id: int) -> str:
11
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12
prompts = [
13
PROMPT_TEMPLATE.format(
14
sql_prompt=
@@ -35,7 +37,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
37
if lora_id else None,
38
)
39
# Print the outputs.
38
- generated_texts = []
40
+ generated_texts: List[str] = []
41
for output in outputs:
42
prompt = output.prompt
43
generated_text = output.outputs[0].text.strip()
tests/lora/test_quant_model.py
+5
-2
@@ -25,7 +25,10 @@ MODELS: List[ModelWithQuantization] = [
25
]
26
27
28
-def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
28
+def do_sample(llm: vllm.LLM,
29
+ lora_path: str,
30
+ lora_id: int,
31
+ max_tokens: int = 256) -> List[str]:
32
raw_prompts = [
33
"Give me an orange-ish brown color",
34
"Give me a neon pink color",
@@ -45,7 +48,7 @@ def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
48
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
49
if lora_id else None)
50
# Print the outputs.
48
- generated_texts = []
51
+ generated_texts: List[str] = []
52
for output in outputs:
53
prompt = output.prompt
54
generated_text = output.outputs[0].text
tests/lora/utils.py
+9
-9
@@ -1,4 +1,4 @@
1
-from typing import List, Optional
1
+from typing import Dict, List, Optional
2
3
import torch
4
@@ -9,13 +9,13 @@ class DummyLoRAManager:
9
10
def __init__(self):
11
super().__init__()
12
- self._loras = {}
12
+ self._loras: Dict[str, LoRALayerWeights] = {}
13
14
def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
15
self._loras[module_name] = lora
16
17
- def get_module_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
18
- return self._loras.get(module_name, None)
17
+ def get_module_lora(self, module_name: str) -> LoRALayerWeights:
18
+ return self._loras[module_name]
19
20
def init_random_lora(self,
21
module_name: str,
@@ -68,11 +68,11 @@ class DummyLoRAManager:
68
module_name: str,
69
input_dim: int,
70
output_dims: List[int],
71
- noop_lora_index: List[int] = None,
72
- rank=8,
71
+ noop_lora_index: Optional[List[int]] = None,
72
+ rank: int = 8,
73
):
74
- base_loras = []
75
- noop_lora_index = set(noop_lora_index or [])
74
+ base_loras: List[LoRALayerWeights] = []
75
+ noop_lora_index_set = set(noop_lora_index or [])
76
77
for i, out_dim in enumerate(output_dims):
78
base_lora = self.init_lora(
@@ -80,7 +80,7 @@ class DummyLoRAManager:
80
input_dim,
81
out_dim,
82
rank=rank,
83
- noop=i in noop_lora_index,
83
+ noop=i in noop_lora_index_set,
84
)
85
base_loras.append(base_lora)
86
packed_lora = PackedLoRALayerWeights.pack(base_loras)
tests/models/test_fp8.py
+2
-1
@@ -3,6 +3,7 @@
3
Note: these tests will only pass on L4 GPU.
4
"""
5
import os
6
+from typing import List
7
8
import pytest
9
import torch
@@ -100,7 +101,7 @@ def test_models(example_prompts, model_name, kv_cache_dtype) -> None:
101
]
102
103
params = SamplingParams(max_tokens=20, temperature=0)
103
- generations = []
104
+ generations: List[str] = []
105
# Note: these need to be run 1 at a time due to numerical precision,
106
# since the expected strs were generated this way.
107
for prompt in formatted_prompts:
tests/prefix_caching/test_prefix_caching.py
+4
-1
@@ -2,8 +2,11 @@
2
3
Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
+from typing import List
6
+
7
import pytest
8
9
+from vllm.block import PhysicalTokenBlock
10
from vllm.core.block_manager_v1 import CachedBlockAllocator
11
from vllm.utils import Device
12
@@ -43,7 +46,7 @@ def test_block_allocator(
46
def test_eviction(num_blocks: int, ):
47
block_size = 16
48
block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
46
- blocks = []
49
+ blocks: List[PhysicalTokenBlock] = []
50
51
for i in range(num_blocks):
52
# use i as the block_hash
tests/quantization/test_configs.py
+2
-1
@@ -4,6 +4,7 @@ Run `pytest tests/quantization/test_configs.py --forked`.
4
"""
5
6
from dataclasses import dataclass
7
+from typing import Tuple
8
9
import pytest
10
@@ -51,7 +52,7 @@ MODEL_ARG_EXPTYPES = [
52
53
54
@pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
54
-def test_auto_gptq(model_arg_exptype: str) -> None:
55
+def test_auto_gptq(model_arg_exptype: Tuple[str, None, str]) -> None:
56
model_path, quantization_arg, expected_type = model_arg_exptype
57
58
try:
tests/samplers/test_logprobs.py
+7
-4
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
import torch
5
@@ -62,21 +64,22 @@ def test_get_prompt_logprobs(
64
for logprobs in result.outputs[0].logprobs:
65
assert len(logprobs) == num_top_logprobs
66
output_text = result.outputs[0].text
65
- output_string_from_most_likely_tokens = []
67
+ output_string_from_most_likely_tokens_lst: List[str] = []
68
for top_logprobs in result.outputs[0].logprobs:
69
top_logprob = next(iter(top_logprobs.values()))
68
- output_string_from_most_likely_tokens.append(
70
+ output_string_from_most_likely_tokens_lst.append(
71
top_logprob.decoded_token)
72
73
if detokenize:
74
output_string_from_most_likely_tokens = "".join(
73
- output_string_from_most_likely_tokens)
75
+ output_string_from_most_likely_tokens_lst)
76
assert output_text == output_string_from_most_likely_tokens, (
77
"The output text from the top logprob for each token position "
78
"should be the same as the output text in the result.")
79
else:
80
assert output_text == ''
79
- assert output_string_from_most_likely_tokens == [None] * max_tokens
81
+ assert output_string_from_most_likely_tokens_lst == ([None] *
82
+ max_tokens)
83
84
# The first prompt logprob is always None
85
assert result.prompt_logprobs[0] is None
tests/samplers/test_rejection_sampler.py
+2
-2
@@ -246,8 +246,8 @@ def test_rejection_sampling_approximates_target_distribution(
246
draft_and_target_probs_equal)
247
248
sample_sizes = [10, 100, 1_000, 10_000, 100_000]
249
- distance_wrt_reference = []
250
- distance_wrt_target = []
249
+ distance_wrt_reference: List[float] = []
250
+ distance_wrt_target: List[float] = []
251
252
for num_samples in sample_sizes:
253
(reference_vs_rejsample_dist,
tests/samplers/test_sampler.py
+23
-18
@@ -1,6 +1,6 @@
1
import itertools
2
import random
3
-from typing import List, Optional, Tuple
3
+from typing import Dict, List, Optional, Tuple
4
from unittest.mock import patch
5
6
import pytest
@@ -49,8 +49,8 @@ def _do_sample(
49
sampling_params: SamplingParams,
50
device: str,
51
):
52
- seq_group_metadata_list = []
53
- seq_lens = []
52
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
53
+ seq_lens: List[int] = []
54
for i in range(batch_size):
55
seq_group_metadata_list.append(
56
SequenceGroupMetadata(
@@ -212,7 +212,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
212
batch_size = random.randint(1, 128)
213
214
expected_penalization = []
215
- sequence_metadata_list = []
215
+ sequence_metadata_list: List[SequenceGroupMetadata] = []
216
# 20% chance to generate seq group metadata list with all prompts
217
is_prompt = random.random() < 0.2
218
while batch_size > 0:
@@ -232,8 +232,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
232
eos_token_id=eos_token_id,
233
stop_token_ids=stop_token_ids)
234
235
- seq_data = {}
236
- seq_group_penalization = []
235
+ seq_data: Dict[int, SequenceData] = {}
236
+ seq_group_penalization: List[bool] = []
237
for _ in range(num_seqs):
238
num_input = random.randint(1, 100)
239
num_generated = 0 if is_prompt else random.randint(1, 100)
@@ -392,17 +392,16 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
392
else:
393
test_cases = [generate_test_case()]
394
395
- def run_test_case(*,
396
- expected_penalization=None,
397
- seq_group_metadata_list=None):
395
+ def run_test_case(*, expected_penalization: List[bool],
396
+ seq_group_metadata_list: List[SequenceGroupMetadata]):
397
assert expected_penalization, \
398
"Invalid test case, need expected_penalization"
399
assert seq_group_metadata_list, \
400
"Invalid test case, need seq_group_metadata_list"
401
402
batch_size = 0
404
- seq_lens = []
405
- sampling_params_per_row = []
403
+ seq_lens: List[int] = []
404
+ sampling_params_per_row: List[SamplingParams] = []
405
for sgm in seq_group_metadata_list:
406
sampling_params = sgm.sampling_params
407
@@ -472,15 +471,15 @@ def test_sampler_mixed(seed: int, device: str):
471
batch_size = random.randint(1, 256)
472
input_tensor, fake_logits, sampler = _prepare_test(batch_size)
473
475
- seq_group_metadata_list = []
474
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
475
expected_tokens: List[Optional[List[int]]] = []
477
- seq_lens = []
476
+ seq_lens: List[int] = []
477
for i in range(batch_size):
478
expected: Optional[List[int]] = None
479
sampling_type = random.randint(0, 3)
480
if sampling_type == 0:
481
sampling_params = SamplingParams(temperature=0)
483
- expected = [torch.argmax(fake_logits[i], dim=-1).item()]
482
+ expected = [int(torch.argmax(fake_logits[i], dim=-1).item())]
483
elif sampling_type in (1, 2):
484
n = random.randint(1, 10)
485
sampling_params = SamplingParams(
@@ -536,15 +535,18 @@ def test_sampler_mixed(seed: int, device: str):
535
]
536
continue
537
538
+ expected_tokens_item = expected_tokens[i]
539
+ assert expected_tokens_item is not None
540
+
541
for n, nth_output in enumerate(sequence_output.samples):
542
if (metadata.sampling_params.temperature == 0
543
or metadata.sampling_params.seed is not None):
544
# Ensure exact matches for greedy or random with seed
543
- assert nth_output.output_token == expected_tokens[i][n]
545
+ assert nth_output.output_token == expected_tokens_item[n]
546
else:
547
# For non-seeded random check that one of the high-logit
548
# tokens were chosen
547
- assert nth_output.output_token in expected_tokens[i]
549
+ assert nth_output.output_token in expected_tokens_item
550
551
# Test batch
552
test_sampling()
@@ -588,8 +590,8 @@ def test_sampler_top_k_top_p(seed: int, device: str):
590
warpers = generation_model._get_logits_warper(generation_config)
591
assert len(warpers) == 2 # top_p and top_k
592
591
- seq_group_metadata_list = []
592
- seq_lens = []
593
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
594
+ seq_lens: List[int] = []
595
for i in range(batch_size):
596
seq_group_metadata_list.append(
597
SequenceGroupMetadata(
@@ -622,6 +624,9 @@ def test_sampler_top_k_top_p(seed: int, device: str):
624
625
with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
626
sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
627
+
628
+ assert sample_probs is not None
629
+
630
hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
631
hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
632
assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
tests/spec_decode/e2e/conftest.py
+7
-6
@@ -118,16 +118,17 @@ class AsyncLLM:
118
raise ValueError("The lengths of prompts and "
119
"sampling_params must be the same.")
120
121
- async def get_output(prompt, sampling_param) -> str:
121
+ async def get_output(prompt, sampling_param) -> RequestOutput:
122
request_id = random_uuid()
123
results_generator = self.llm_engine.generate(
124
prompt, sampling_param, request_id)
125
final_output = None
126
async for request_output in results_generator:
127
final_output = request_output
128
+ assert final_output is not None
129
return final_output
130
130
- outputs = []
131
+ outputs: List[RequestOutput] = []
132
try:
133
for i in range(num_requests):
134
prompt = prompts[i] if prompts is not None else None
@@ -208,8 +209,8 @@ def maybe_assert_ngram_worker(llm):
209
def get_output_from_llm_generator(
210
llm_generator, prompts,
211
sampling_params) -> Tuple[List[str], List[List[int]]]:
211
- tokens = []
212
- token_ids = []
212
+ tokens: List[str] = []
213
+ token_ids: List[List[int]] = []
214
for llm in llm_generator():
215
maybe_assert_ngram_worker(llm)
216
@@ -300,8 +301,8 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
301
nvmlInit()
302
start_time = time.time()
303
while True:
303
- output = {}
304
- output_raw = {}
304
+ output: Dict[int, str] = {}
305
+ output_raw: Dict[int, float] = {}
306
for device in devices:
307
dev_handle = nvmlDeviceGetHandleByIndex(device)
308
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
tests/spec_decode/test_batch_expansion.py
+4
-2
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
import torch
5
@@ -38,14 +40,14 @@ def test_get_token_ids_to_score(k: int):
40
device='cuda',
41
)
42
41
- expected_output = [
43
+ expected_output: List[List[int]] = [
44
[],
45
]
46
for i in range(proposal_token_ids.shape[0]):
47
expected_output.append(proposal_token_ids[:i + 1].tolist())
48
49
scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
48
- actual_output = scorer._get_token_ids_to_score(proposal_token_ids) # pylint: disable=protected-access
50
+ actual_output = scorer._get_token_ids_to_score(proposal_token_ids.tolist()) # pylint: disable=protected-access
51
52
actual_output = [
53
x.tolist() if isinstance(x, torch.Tensor) else x for x in actual_output
tests/spec_decode/test_multi_step_worker.py
+12
-7
@@ -1,11 +1,12 @@
1
import random
2
+from typing import Dict, List
3
from unittest.mock import MagicMock
4
5
import pytest
6
import torch
7
8
from vllm.model_executor.utils import set_random_seed
8
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
+from vllm.sequence import ExecuteModelRequest, Logprob, SamplerOutput
10
from vllm.spec_decode.multi_step_worker import MultiStepWorker
11
from vllm.spec_decode.top1_proposer import Top1Proposer
12
from vllm.worker.worker import Worker
@@ -210,7 +211,7 @@ def test_same_output_for_multi_step():
211
212
# Run single-step repeatedly.
213
zero_kv_cache(worker.cache_engine)
213
- single_step_output = []
214
+ single_step_output: List[SamplerOutput] = []
215
continuations = [[1] for _ in prompts]
216
set_random_seed(seed)
217
@@ -232,11 +233,15 @@ def test_same_output_for_multi_step():
233
continuations[i].append(seq_group_output.samples[0].output_token)
234
235
# Get token ids and logprobs for comparison.
235
- multi_step_output_logprobs = [[] for _ in prompts]
236
- single_step_output_logprobs = [[] for _ in prompts]
237
-
238
- multi_step_output_token_ids = [[] for _ in prompts]
239
- single_step_output_token_ids = [[] for _ in prompts]
236
+ multi_step_output_logprobs: List[List[Dict[int,
237
+ Logprob]]] = [[]
238
+ for _ in prompts]
239
+ single_step_output_logprobs: List[List[Dict[int,
240
+ Logprob]]] = [[]
241
+ for _ in prompts]
242
+
243
+ multi_step_output_token_ids: List[List[int]] = [[] for _ in prompts]
244
+ single_step_output_token_ids: List[List[int]] = [[] for _ in prompts]
245
for i, _ in enumerate(prompts):
246
for multi_step, single_step in zip(multi_step_output,
247
single_step_output):
tests/spec_decode/test_spec_decode_worker.py
+12
-5
@@ -1,5 +1,6 @@
1
import random
2
from types import SimpleNamespace
3
+from typing import Dict, List
4
from unittest.mock import MagicMock
5
6
import pytest
@@ -7,7 +8,7 @@ import torch
8
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10
from vllm.model_executor.utils import set_random_seed
10
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
11
+from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
12
from vllm.spec_decode.interfaces import SpeculativeProposals
13
from vllm.spec_decode.metrics import (AsyncMetricsCollector,
14
SpecDecodeWorkerMetrics)
@@ -103,7 +104,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
104
seq_group_metadata_list=seq_group_metadata_list,
105
num_lookahead_slots=k))
106
106
- seen_contexts = []
107
+ seen_contexts: List[List[int]] = []
108
109
call_args_list = target_worker.execute_model.call_args_list
110
assert len(call_args_list) == 1
@@ -116,7 +117,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
117
for seq_data in seq_group_metadata.seq_data.values():
118
seen_contexts.append(seq_data.get_token_ids())
119
119
- expected_seen_contexts = []
120
+ expected_seen_contexts: List[List[int]] = []
121
122
for prompt, prev_generated, draft_tokens in zip(
123
prompts, prev_output_tokens, proposal_token_ids.tolist()):
@@ -310,8 +311,14 @@ def test_correctly_formats_output(k: int, batch_size: int):
311
next(iter(seq_group_metadata.seq_data.keys()))
312
for seq_group_metadata in seq_group_metadata_list
313
]
313
- actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314
- expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314
+ actual_output_by_seq: Dict[int, List[SequenceOutput]] = {
315
+ seq_id: []
316
+ for seq_id in seq_ids
317
+ }
318
+ expected_output_by_seq: Dict[int, List[SequenceOutput]] = {
319
+ seq_id: []
320
+ for seq_id in seq_ids
321
+ }
322
323
for step in output:
324
for seq_group in step:
tests/spec_decode/utils.py
+9
-5
@@ -1,5 +1,7 @@
1
from itertools import count
2
-from typing import Dict, Iterable, List, Optional, Union
2
+from typing import Callable, Dict, List, Optional
3
+from typing import Sequence as GenericSequence
4
+from typing import TypeVar, Union
5
from unittest.mock import MagicMock
6
7
import torch
@@ -14,6 +16,8 @@ from vllm.utils import get_distributed_init_method, get_ip, get_open_port
16
from vllm.worker.cache_engine import CacheEngine
17
from vllm.worker.worker import Worker
18
19
+T = TypeVar("T", bound=Worker)
20
+
21
22
def round_up_to_next_block(seq_len: int, block_size: int) -> int:
23
return (seq_len + block_size - 1) // block_size
@@ -56,13 +60,13 @@ def zero_kv_cache(cache_engine: CacheEngine):
60
value_blocks.zero_()
61
62
59
-def create_worker(cls: type,
63
+def create_worker(cls: Callable[..., T],
64
model_name: str,
65
block_size: int,
66
num_gpu_blocks: int,
67
seed: int,
68
is_driver_worker: bool = True,
65
- enforce_eager: bool = True):
69
+ enforce_eager: bool = True) -> T:
70
engine_args = EngineArgs(
71
model=model_name,
72
seed=seed,
@@ -159,8 +163,8 @@ def assert_logprobs_dict_allclose(
163
164
def create_sampler_output_list(
165
token_ids: torch.Tensor,
162
- probs: Iterable[Optional[torch.Tensor]],
163
- logprobs: Iterable[Optional[torch.Tensor]],
166
+ probs: GenericSequence[Optional[torch.Tensor]],
167
+ logprobs: GenericSequence[Optional[torch.Tensor]],
168
seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
169
num_steps, batch_size = token_ids.shape
170
token_ids_by_step = token_ids.tolist()
tests/test_cache_block_hashing.py
+1
-1
@@ -51,7 +51,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
51
max_input_length=None,
52
)
53
54
- hashes = []
54
+ hashes: List[List[List[int]]] = []
55
56
for prefix in prefixes:
57
for lora_int_id in concurrent_lora_int_ids:
tests/test_logger.py
+1
@@ -47,6 +47,7 @@ def test_default_vllm_root_logger_configuration():
47
assert not logger.propagate
48
49
handler = logger.handlers[0]
50
+ assert isinstance(handler, logging.StreamHandler)
51
assert handler.stream == sys.stdout
52
assert handler.level == logging.INFO
53
tests/tokenization/test_detokenize.py
+2
-2
@@ -153,8 +153,8 @@ def test_decode_sequence_logprobs(complete_sequence: str,
153
# Run sequentially.
154
seq = create_sequence()
155
dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
156
- sequential_logprobs_text_chosen_token = []
157
- sequential_logprobs_text_other_token = []
156
+ sequential_logprobs_text_chosen_token: List[str] = []
157
+ sequential_logprobs_text_other_token: List[str] = []
158
for new_token, logprobs in zip(complete_sequence_token_ids,
159
dummy_logprobs):
160
seq.append_token_id(new_token, logprobs)
tests/utils.py
+1
-1
@@ -79,7 +79,7 @@ class RemoteOpenAIServer:
79
self.host = str(args.host or 'localhost')
80
self.port = int(args.port)
81
82
- self._runner = self._RemoteRunner.remote(
82
+ self._runner = self._RemoteRunner.remote( # type: ignore
83
cli_args,
84
wait_url=self.url_for("health"),
85
wait_timeout=self.MAX_SERVER_START_WAIT_S)
tests/worker/test_model_runner.py
+12
-11
@@ -1,3 +1,5 @@
1
+from typing import List
2
+
3
import pytest
4
import torch
5
@@ -35,8 +37,8 @@ def test_prepare_prompt(batch_size):
37
enable_chunked_prefill=False,
38
)
39
38
- seq_lens = []
39
- seq_group_metadata_list = []
40
+ seq_lens: List[int] = []
41
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
42
block_tables = {0: [1]}
43
for i in range(batch_size):
44
# make sure all tokens fit into one block
@@ -151,15 +153,14 @@ def test_prepare_decode_cuda_graph(batch_size):
153
enable_chunked_prefill=False,
154
)
155
154
- context_lens = []
155
- seq_group_metadata_list = []
156
+ context_lens: List[int] = []
157
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
158
# Assume each seq group finishes prefill.
159
for i in range(batch_size):
160
# make sure all tokens fit into one block
161
context_len = i % (model_runner.block_size - 1) + 1
162
context_lens.append(context_len)
161
- seq_data = list(range(context_len))
162
- seq_data = SequenceData(seq_data)
163
+ seq_data = SequenceData(list(range(context_len)))
164
seq_data.update_num_computed_tokens(context_len)
165
# Append one token ID since prefill is finished.
166
seq_data.append_token_id(1, 0)
@@ -257,7 +258,7 @@ def test_empty_seq_group():
258
dtype="float16",
259
enforce_eager=False,
260
)
260
- seq_group_metadata_list = []
261
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
262
model_input = model_runner._prepare_model_input(seq_group_metadata_list)
263
input_tokens, input_positions, attn_metadata, slot_mapping = (
264
model_input.input_tokens,
@@ -310,10 +311,10 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
311
)
312
313
# Add prefill requests.
313
- seq_lens = []
314
- seq_group_metadata_list = []
315
- prefill_metadata_list = []
316
- decode_metadata_list = []
314
+ seq_lens: List[int] = []
315
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
316
+ prefill_metadata_list: List[SequenceGroupMetadata] = []
317
+ decode_metadata_list: List[SequenceGroupMetadata] = []
318
block_tables = {0: [1]}
319
prefill_batch_size = batch_size // 2
320
decode_batch_size = batch_size - prefill_batch_size
vllm/attention/backends/torch_sdpa.py
+2
-2
@@ -245,7 +245,7 @@ def _make_alibi_bias(
245
dtype: torch.dtype,
246
seq_lens: List[int],
247
) -> List[torch.Tensor]:
248
- attn_biases = []
248
+ attn_biases: List[torch.Tensor] = []
249
for seq_len in seq_lens:
250
bias = torch.arange(seq_len, dtype=dtype)
251
# NOTE(zhuohan): HF uses
@@ -271,7 +271,7 @@ def _make_sliding_window_bias(
271
window_size: Optional[int],
272
dtype: torch.dtype,
273
) -> List[torch.Tensor]:
274
- attn_biases = []
274
+ attn_biases: List[torch.Tensor] = []
275
for seq_len in seq_lens:
276
tensor = torch.full(
277
(1, seq_len, seq_len),
vllm/attention/backends/xformers.py
+2
-2
@@ -431,8 +431,8 @@ def _make_alibi_bias(
431
num_kv_heads: int,
432
dtype: torch.dtype,
433
seq_lens: List[int],
434
-) -> LowerTriangularMaskWithTensorBias:
435
- attn_biases = []
434
+) -> List[AttentionBias]:
435
+ attn_biases: List[AttentionBias] = []
436
for seq_len in seq_lens:
437
bias = torch.arange(seq_len, dtype=dtype)
438
# NOTE(zhuohan): HF uses
vllm/core/block/block_table.py
+1
-1
@@ -252,7 +252,7 @@ class BlockTable:
252
def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
253
token_ids: List[int],
254
device: Device) -> List[Block]:
255
- blocks = []
255
+ blocks: List[Block] = []
256
for block_token_ids in chunk_list(token_ids, self._block_size):
257
if len(block_token_ids) == self._block_size:
258
# If the block is full, create an immutable block.
vllm/core/block/naive_block.py
+1
-1
@@ -111,7 +111,7 @@ class NaiveBlockAllocator(BlockAllocator):
111
"""
112
source_blocks = get_all_blocks_recursively(last_block)
113
114
- forked_blocks = []
114
+ forked_blocks: List[Block] = []
115
prev_block = None
116
for block in source_blocks:
117
vllm/core/block/prefix_caching_block.py
+1
-1
@@ -271,7 +271,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
271
"""
272
source_blocks = get_all_blocks_recursively(last_block)
273
274
- forked_blocks = []
274
+ forked_blocks: List[Block] = []
275
prev_block = None
276
for block in source_blocks:
277
refcount = self._refcounter.incr(block.block_id)
vllm/core/block_manager_v2.py
+1
-1
@@ -260,7 +260,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
260
# at max extend.
261
if self.enable_caching:
262
block_table = self.block_tables[seq.seq_id]
263
- block_ids = []
263
+ block_ids: List[Optional[int]] = []
264
for block_id in block_table.physical_block_ids:
265
block_ids.append(block_id)
266
self.block_allocator.mark_blocks_as_accessed(
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+4
-4
@@ -2,7 +2,7 @@ import ctypes
2
import json
3
import os
4
from itertools import product
5
-from typing import Dict, Optional, Sequence
5
+from typing import Dict, List, Optional, Sequence
6
7
import torch.distributed as dist
8
import torch.multiprocessing as mp
@@ -88,7 +88,7 @@ def consumer(batch_tgt: Sequence[int],
88
def can_actually_p2p(
89
batch_src: Sequence[int],
90
batch_tgt: Sequence[int],
91
-):
91
+) -> Sequence[bool]:
92
"""
93
Usually, checking if P2P access is enabled can be done by
94
`torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes
@@ -138,7 +138,7 @@ def can_actually_p2p(
138
p_tgt.start()
139
p_src.join()
140
p_tgt.join()
141
- result = []
141
+ result: List[bool] = []
142
for src, tgt in zip(batch_src, batch_tgt):
143
a = result_queue.get()
144
b = result_queue.get()
@@ -188,7 +188,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
188
# only the local master process (with local_rank == 0) can
189
# enter this block to calculate the cache
190
logger.info("generating GPU P2P access cache in %s", path)
191
- cache = {}
191
+ cache: Dict[str, bool] = {}
192
ids = list(range(num_dev))
193
# batch of all pairs of GPUs
194
batch_src, batch_tgt = zip(*list(product(ids, ids)))
vllm/distributed/device_communicators/pynccl_wrapper.py
+1
-1
@@ -205,7 +205,7 @@ class NCCLLibrary:
205
raise e
206
207
if so_file not in NCCLLibrary.path_to_dict_mapping:
208
- _funcs = {}
208
+ _funcs: Dict[str, Any] = {}
209
for func in NCCLLibrary.exported_functions:
210
f = getattr(self.lib, func.name)
211
f.restype = func.restype
vllm/engine/llm_engine.py
+2
-2
@@ -2,7 +2,7 @@ import time
2
from contextlib import contextmanager
3
from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
4
from typing import Sequence as GenericSequence
5
-from typing import Type, TypeVar, Union
5
+from typing import Set, Type, TypeVar, Union
6
7
from transformers import GenerationConfig, PreTrainedTokenizer
8
@@ -973,7 +973,7 @@ class LLMEngine:
973
def remove_lora(self, lora_id: int) -> bool:
974
return self.model_executor.remove_lora(lora_id)
975
976
- def list_loras(self) -> List[int]:
976
+ def list_loras(self) -> Set[int]:
977
return self.model_executor.list_loras()
978
979
def check_health(self) -> None:
vllm/engine/metrics.py
+2
-2
@@ -144,7 +144,7 @@ class Metrics:
144
# end-metrics-definitions
145
146
147
-def build_1_2_5_buckets(max_value: int):
147
+def build_1_2_5_buckets(max_value: int) -> List[int]:
148
"""
149
Builds a list of buckets with increasing powers of 10 multiplied by
150
mantissa values (1, 2, 5) until the value exceeds the specified maximum.
@@ -155,7 +155,7 @@ def build_1_2_5_buckets(max_value: int):
155
"""
156
mantissa_lst = [1, 2, 5]
157
exponent = 0
158
- buckets = []
158
+ buckets: List[int] = []
159
while True:
160
for m in mantissa_lst:
161
value = m * 10**exponent
vllm/engine/output_processor/single_step.py
+3
-3
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Tuple, Union
1
+from typing import Dict, List, Optional, Tuple, Union
2
3
from vllm.config import SchedulerConfig
4
from vllm.core.scheduler import Scheduler
@@ -146,8 +146,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
146
147
# Beam search case
148
# Select the child sequences to keep in the sequence group.
149
- selected_child_seqs = []
150
- unselected_child_seqs = []
149
+ selected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
150
+ unselected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
151
beam_width = seq_group.sampling_params.best_of
152
length_penalty = seq_group.sampling_params.length_penalty
153
vllm/entrypoints/openai/run_batch.py
+2
-1
@@ -2,6 +2,7 @@ import argparse
2
import asyncio
3
import sys
4
from io import StringIO
5
+from typing import Awaitable, List
6
7
import aiohttp
8
@@ -114,7 +115,7 @@ async def main(args):
115
)
116
117
# Submit all requests in the file to the engine "concurrently".
117
- response_futures = []
118
+ response_futures: List[Awaitable[BatchRequestOutput]] = []
119
for request_json in (await read_file(args.input_file)).strip().split("\n"):
120
request = BatchRequestInput.model_validate_json(request_json)
121
response_futures.append(run_request(openai_serving_chat, request))
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -487,7 +487,7 @@ class OpenAIServingChat(OpenAIServing):
487
final_res = res
488
assert final_res is not None
489
490
- choices = []
490
+ choices: List[ChatCompletionResponseChoice] = []
491
492
role = self.get_chat_request_role(request)
493
for output in final_res.outputs:
vllm/entrypoints/openai/serving_embedding.py
+1
-1
@@ -25,7 +25,7 @@ def request_output_to_embedding_response(
25
created_time: int,
26
model_name: str,
27
) -> EmbeddingResponse:
28
- data = []
28
+ data: List[EmbeddingResponseData] = []
29
num_prompt_tokens = 0
30
for idx, final_res in enumerate(final_res_batch):
31
assert final_res is not None
vllm/lora/lora.py
+2
-1
@@ -1,4 +1,5 @@
1
from typing import List, Optional
2
+from typing import Sequence as GenericSequence
3
4
import torch
5
@@ -120,7 +121,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
121
122
@classmethod
123
def pack(
123
- cls, loras: List[Optional["LoRALayerWeights"]]
124
+ cls, loras: GenericSequence[Optional["LoRALayerWeights"]]
125
) -> "PackedLoRALayerWeights":
126
"""Pack a list of LoRAs into a single LoRA.
127
vllm/lora/worker_manager.py
+1
-1
@@ -165,7 +165,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
165
model = self._lora_manager.model
166
supported_lora_modules = model.supported_lora_modules
167
packed_modules_mapping = model.packed_modules_mapping
168
- expected_lora_modules = []
168
+ expected_lora_modules: List[str] = []
169
for module in supported_lora_modules:
170
if module in packed_modules_mapping:
171
expected_lora_modules.extend(
vllm/model_executor/layers/linear.py
+1
-1
@@ -393,7 +393,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
393
param_data.copy_(loaded_weight)
394
return
395
current_shard_offset = 0
396
- shard_offsets = []
396
+ shard_offsets: List[Tuple[int, int, int]] = []
397
for i, output_size in enumerate(self.output_sizes):
398
shard_offsets.append((i, current_shard_offset, output_size))
399
current_shard_offset += output_size
vllm/model_executor/layers/quantization/gptq_marlin.py
+6
-5
@@ -25,24 +25,25 @@ GPTQ_MARLIN_SUPPORTED_SYM = [True]
25
26
27
# Permutations for Marlin scale shuffling
28
-def get_scale_perms(num_bits):
29
- scale_perm = []
28
+def get_scale_perms(num_bits: int):
29
+ scale_perm: List[int] = []
30
for i in range(8):
31
scale_perm.extend([i + 8 * j for j in range(8)])
32
- scale_perm_single = []
32
+ scale_perm_single: List[int] = []
33
for i in range(4):
34
scale_perm_single.extend(
35
[2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
36
return scale_perm, scale_perm_single
37
38
39
-def get_pack_factor(num_bits):
39
+def get_pack_factor(num_bits: int):
40
assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
41
), f"Unsupported num_bits = {num_bits}"
42
return 32 // num_bits
43
44
45
-def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
45
+def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
46
+ group_size: int, num_bits: int):
47
scale_perm, scale_perm_single = get_scale_perms(num_bits)
48
if group_size < size_k and group_size != -1:
49
s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py
+10
-8
@@ -1,4 +1,6 @@
1
"""This file is used for /tests and /benchmarks"""
2
+from typing import Dict, List
3
+
4
import numpy
5
import torch
6
@@ -11,10 +13,10 @@ import torch
13
#
14
# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15
# (without the need to use ldmatrix instructions) # noqa: E501
14
-def get_perms_24(num_bits):
15
- perm_list = []
16
+def get_perms_24(num_bits: int):
17
+ perm_list: List[int] = []
18
for i in range(32):
17
- perm1 = []
19
+ perm1: List[int] = []
20
col = i // 4
21
col_o = col // 2
22
for block in [0, 1]:
@@ -39,18 +41,18 @@ def get_perms_24(num_bits):
41
42
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
43
perm = torch.from_numpy(perm)
42
- scale_perm = []
44
+ scale_perm: List[int] = []
45
for i in range(8):
46
scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
45
- scale_perm_single = []
47
+ scale_perm_single: List[int] = []
48
for i in range(8):
49
scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
50
return perm, scale_perm, scale_perm_single
51
52
51
-marlin_24_perm = {}
52
-marlin_24_scale_perm = {}
53
-marlin_24_scale_perm_single = {}
53
+marlin_24_perm: Dict[int, torch.Tensor] = {}
54
+marlin_24_scale_perm: Dict[int, List[int]] = {}
55
+marlin_24_scale_perm_single: Dict[int, List[int]] = {}
56
for num_bits in [4, 8]:
57
perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
58
marlin_24_perm[num_bits] = perm_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py
+10
-8
@@ -1,4 +1,6 @@
1
"""This file is used for /tests and /benchmarks"""
2
+from typing import Dict, List
3
+
4
import numpy
5
import torch
6
@@ -11,10 +13,10 @@ import torch
13
#
14
# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15
# (without the need to use ldmatrix instructions) # noqa: E501
14
-def get_perms(num_bits):
15
- perm_list = []
16
+def get_perms(num_bits: int):
17
+ perm_list: List[int] = []
18
for i in range(32):
17
- perm1 = []
19
+ perm1: List[int] = []
20
col = i // 4
21
for block in [0, 1]:
22
for row in [
@@ -38,19 +40,19 @@ def get_perms(num_bits):
40
41
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
42
perm = torch.from_numpy(perm)
41
- scale_perm = []
43
+ scale_perm: List[int] = []
44
for i in range(8):
45
scale_perm.extend([i + 8 * j for j in range(8)])
44
- scale_perm_single = []
46
+ scale_perm_single: List[int] = []
47
for i in range(4):
48
scale_perm_single.extend(
49
[2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
50
return perm, scale_perm, scale_perm_single
51
52
51
-marlin_perm = {}
52
-marlin_scale_perm = {}
53
-marlin_scale_perm_single = {}
53
+marlin_perm: Dict[int, torch.Tensor] = {}
54
+marlin_scale_perm: Dict[int, List[int]] = {}
55
+marlin_scale_perm_single: Dict[int, List[int]] = {}
56
for num_bits in [4, 8]:
57
perm, scale_perm, scale_perm_single = get_perms(num_bits)
58
marlin_perm[num_bits] = perm
vllm/model_executor/layers/sampler.py
+14
-11
@@ -174,7 +174,7 @@ def _apply_min_tokens_penalty(
174
min_tokens = sampling_params.min_tokens
175
token_ids_to_penalize = sampling_params.all_stop_token_ids
176
if min_tokens > 0 and token_ids_to_penalize:
177
- seqs_to_penalize = []
177
+ seqs_to_penalize: List[int] = []
178
for j, seq_id in enumerate(seq_ids):
179
seq_data = seq_group.seq_data[seq_id]
180
if len(seq_data.output_token_ids) < min_tokens:
@@ -285,7 +285,7 @@ def _greedy_sample(
285
same as the length of selected_seq_groups. If the corresponding
286
seq_group has do_sample=False, tuple contains ([], [])
287
"""
288
- samples = samples.tolist()
288
+ samples_lst = samples.tolist()
289
sample_idx = 0
290
results: SampleResultType = []
291
for seq_group in selected_seq_groups:
@@ -298,7 +298,7 @@ def _greedy_sample(
298
assert num_parent_seqs == 1, (
299
"Greedy sampling should have only one seq.")
300
parent_ids = list(range(num_parent_seqs))
301
- next_token_ids = [samples[sample_idx]]
301
+ next_token_ids = [samples_lst[sample_idx]]
302
results.append((next_token_ids, parent_ids))
303
sample_idx += num_parent_seqs
304
return results
@@ -394,7 +394,7 @@ def _beam_search_sample(
394
next_token_ids = next_token_ids.tolist()
395
else:
396
# Generation phase.
397
- cumulative_logprobs: List[int] = [
397
+ cumulative_logprobs: List[float] = [
398
seq_group.seq_data[seq_id].cumulative_logprob
399
for seq_id in seq_ids
400
]
@@ -466,8 +466,9 @@ def _sample_with_torch(
466
categorized_seq_group_ids[sampling_type].append(i)
467
468
sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
469
- sample_metadata = {}
470
- multinomial_samples = {}
469
+ sample_metadata: Dict[SamplingType,
470
+ Tuple[List[int], List[SequenceGroupToSample]]] = {}
471
+ multinomial_samples: Dict[SamplingType, torch.Tensor] = {}
472
473
# Create output tensor for sampled token ids.
474
if include_gpu_probs_tensor:
@@ -494,7 +495,7 @@ def _sample_with_torch(
495
greedy_samples = torch.argmax(logprobs[long_sample_indices],
496
dim=-1)
497
497
- if include_gpu_probs_tensor:
498
+ if sampled_token_ids_tensor is not None:
499
# Store sampled tokens in output tensor.
500
sampled_token_ids_tensor[
501
long_sample_indices] = greedy_samples.unsqueeze(-1)
@@ -522,7 +523,7 @@ def _sample_with_torch(
523
probs[long_sample_indices], max_best_of_in_batch,
524
**seeded_args)
525
525
- if include_gpu_probs_tensor:
526
+ if sampled_token_ids_tensor is not None:
527
# Store sampled tokens in output tensor.
528
sampled_token_ids_tensor[
529
long_sample_indices] = multinomial_samples[sampling_type]
@@ -571,7 +572,9 @@ def _sample_with_triton_kernel(
572
categorized_seq_group_ids[sampling_type].append(i)
573
574
sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
574
- sample_metadata = {}
575
+ sample_metadata: Dict[SamplingType,
576
+ Tuple[List[int], List[SequenceGroupToSample],
577
+ torch.Tensor, torch.Tensor]] = {}
578
max_best_of_in_batch = 1
579
580
# Counterintiutively, having two loops here is actually faster.
@@ -1008,14 +1011,14 @@ def _build_sampler_output(
1011
speculative decoding rejection sampling.
1012
"""
1013
1011
- sampler_output = []
1014
+ sampler_output: List[CompletionSequenceGroupOutput] = []
1015
for (seq_group, sample_result, group_prompt_logprobs,
1016
group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1017
sample_results, prompt_logprobs,
1018
sample_logprobs):
1019
seq_ids = seq_group.seq_ids
1020
next_token_ids, parent_ids = sample_result
1018
- seq_outputs = []
1021
+ seq_outputs: List[SequenceOutput] = []
1022
for parent_id, next_token_id, logprobs in zip(parent_ids,
1023
next_token_ids,
1024
group_sample_logprobs):
vllm/model_executor/model_loader/loader.py
+4
-3
@@ -68,7 +68,7 @@ def _get_model_initialization_kwargs(
68
vision_language_config: Optional[VisionLanguageConfig]
69
) -> Dict[str, Any]:
70
"""Get extra kwargs for model initialization."""
71
- extra_kwargs = {}
71
+ extra_kwargs: Dict[str, Any] = {}
72
if hasattr(model_class, "supported_lora_modules"):
73
extra_kwargs["lora_config"] = lora_config
74
elif lora_config:
@@ -446,7 +446,8 @@ class ShardedStateLoader(BaseModelLoader):
446
Filter out all tensors that share the same memory or a subset of the
447
memory of another tensor.
448
"""
449
- same_storage_groups = collections.defaultdict(list)
449
+ same_storage_groups: Dict[Any, List[Tuple[
450
+ str, torch.Tensor]]] = collections.defaultdict(list)
451
for key, tensor in tensors.items():
452
if tensor.numel():
453
ptr = tensor.untyped_storage().data_ptr()
@@ -455,7 +456,7 @@ class ShardedStateLoader(BaseModelLoader):
456
def get_end_ptr(tensor: torch.Tensor) -> int:
457
return tensor.view(-1)[-1].data_ptr() + tensor.element_size()
458
458
- result = {}
459
+ result: Dict[str, torch.Tensor] = {}
460
for group in same_storage_groups.values():
461
for k, t in group:
462
a, b = t.data_ptr(), get_end_ptr(t)
vllm/model_executor/model_loader/weight_utils.py
+1
-1
@@ -329,7 +329,7 @@ def np_cache_weights_iterator(
329
# dumping the same model weights to numpy at the same time.
330
with get_lock(model_name_or_path, cache_dir):
331
if not os.path.exists(weight_names_file):
332
- weight_names = []
332
+ weight_names: List[str] = []
333
for bin_file in hf_weights_files:
334
state = torch.load(bin_file, map_location="cpu")
335
for name, param in state.items():
vllm/model_executor/models/__init__.py
+2
-2
@@ -72,11 +72,11 @@ _MODELS = {**_GENERATION_MODELS, **_EMBEDDING_MODELS}
72
_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
73
74
# Models not supported by ROCm.
75
-_ROCM_UNSUPPORTED_MODELS = []
75
+_ROCM_UNSUPPORTED_MODELS: List[str] = []
76
77
# Models partially supported by ROCm.
78
# Architecture -> Reason.
79
-_ROCM_PARTIALLY_SUPPORTED_MODELS = {
79
+_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
80
"Qwen2ForCausalLM":
81
"Sliding window attention is not yet supported in ROCm's flash attention",
82
"MistralForCausalLM":
vllm/model_executor/models/arctic.py
+2
-2
@@ -453,8 +453,8 @@ class ArcticForCausalLM(nn.Module):
453
("qkv_proj", "v_proj", "v"),
454
]
455
456
- mlp_params_mapping = []
457
- expert_params_mapping = []
456
+ mlp_params_mapping: List[Tuple[str, str, int]] = []
457
+ expert_params_mapping: List[Tuple[str, str, int]] = []
458
num_layers = self.config.num_hidden_layers
459
460
for layer in range(num_layers):
vllm/model_executor/models/commandr.py
+2
-2
@@ -20,7 +20,7 @@
20
21
# This file is based on the LLama model definition file in transformers
22
"""PyTorch Cohere model."""
23
-from typing import Iterable, List, Optional, Tuple
23
+from typing import Iterable, List, Optional, Set, Tuple
24
25
import torch
26
import torch.utils.checkpoint
@@ -352,7 +352,7 @@ class CohereForCausalLM(nn.Module):
352
("gate_up_proj", "up_proj", 1),
353
]
354
params_dict = dict(self.named_parameters())
355
- loaded_params = set()
355
+ loaded_params: Set[str] = set()
356
for name, loaded_weight in weights:
357
for param_name, shard_name, shard_id in stacked_params_mapping:
358
if shard_name not in name:
vllm/model_executor/models/gemma.py
+2
-2
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
"""Inference-only Gemma model compatible with HuggingFace weights."""
17
from functools import lru_cache
18
-from typing import Iterable, List, Optional, Tuple
18
+from typing import Iterable, List, Optional, Set, Tuple
19
20
import torch
21
from torch import nn
@@ -363,7 +363,7 @@ class GemmaForCausalLM(nn.Module):
363
("gate_up_proj", "up_proj", 1),
364
]
365
params_dict = dict(self.named_parameters())
366
- loaded_params = set()
366
+ loaded_params: Set[str] = set()
367
for name, loaded_weight in weights:
368
for (param_name, shard_name, shard_id) in stacked_params_mapping:
369
if shard_name not in name:
vllm/sequence.py
+1
-1
@@ -123,7 +123,7 @@ class SequenceData:
123
output_token_ids = []
124
125
self.prompt_token_ids = prompt_token_ids
126
- self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
126
+ self._prompt_token_ids_tuple = tuple(prompt_token_ids)
127
self.output_token_ids = output_token_ids
128
self.cumulative_logprob = 0.0
129
# The number of tokens that are computed (that run against the model).
vllm/spec_decode/multi_step_worker.py
+5
-5
@@ -1,10 +1,10 @@
1
import copy
2
import weakref
3
-from typing import List, Tuple
3
+from typing import Dict, List, Tuple
4
5
import torch
6
7
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8
SequenceGroupMetadata)
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
@@ -71,7 +71,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
71
sample_len)
72
73
# Run model sample_len times.
74
- model_outputs = []
74
+ model_outputs: List[SamplerOutput] = []
75
for _ in range(sample_len):
76
model_output = super().execute_model(
77
execute_model_req=copied_execute_model_req)
@@ -132,7 +132,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
132
133
# Shallow-copy the list of SequenceGroupMetadata. This allows us to
134
# append tokens and change is_prompt without external side-effects.
135
- new_seq_group_metadata_list = []
135
+ new_seq_group_metadata_list: List[SequenceGroupMetadata] = []
136
137
for old_seq_group_metadata in seq_group_metadata_list:
138
# We must shallow-copy seq_group_metadata as is_prompt could change.
@@ -140,7 +140,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
140
new_seq_group_metadata_list.append(seq_group_metadata)
141
142
# We must shallow-copy seq_data as we will append token ids
143
- new_seq_data = {}
143
+ new_seq_data: Dict[int, SequenceData] = {}
144
for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
145
new_seq_data[seq_id] = copy.copy(old_seq_data)
146
new_seq_data[
vllm/spec_decode/ngram_worker.py
+3
-3
@@ -48,7 +48,7 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
48
self,
49
execute_model_req: ExecuteModelRequest,
50
sample_len: int,
51
- ) -> Tuple[Optional[List[SamplerOutput]], bool]:
51
+ ) -> Tuple[Optional[List[Optional[SamplerOutput]]], bool]:
52
"""NGram match algo to pick proposal candidate. Returns the list of
53
sampler output, one per SequenceGroupMetadata.
54
@@ -58,8 +58,8 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
58
self._raise_if_unsupported(execute_model_req)
59
60
has_spec_out = False
61
- token_id_list = []
62
- token_prob_list = []
61
+ token_id_list: List[Optional[torch.Tensor]] = []
62
+ token_prob_list: List[Optional[torch.Tensor]] = []
63
for idx, seq_group_metadata in enumerate(
64
execute_model_req.seq_group_metadata_list):
65
seq_data = next(iter(seq_group_metadata.seq_data.values()))
vllm/spec_decode/spec_decode_worker.py
+4
-4
@@ -7,8 +7,8 @@ from vllm.config import SpeculativeConfig
7
from vllm.distributed.communication_op import broadcast_tensor_dict
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
11
- SequenceGroupMetadata)
10
+from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
11
+ SamplerOutput, SequenceGroupMetadata)
12
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
13
from vllm.spec_decode.interfaces import (SpeculativeProposals,
14
SpeculativeScorer, SpeculativeScores)
@@ -516,13 +516,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
516
topk_indices_by_step = topk_indices_by_step.tolist()
517
518
# Construct the output on a per-step, per-sequence basis.
519
- sampler_output_list = []
519
+ sampler_output_list: List[SamplerOutput] = []
520
for step_index in range(num_steps):
521
if all(token_id == -1
522
for token_id in accepted_token_ids_by_step[step_index]):
523
break
524
525
- step_output_token_ids = []
525
+ step_output_token_ids: List[CompletionSequenceGroupOutput] = []
526
for sequence_index in range(batch_size):
527
# Each sequence may have a different num_logprobs; retrieve it.
528
num_logprobs = num_logprobs_per_seq[sequence_index]
vllm/spec_decode/util.py
+2
-2
@@ -26,10 +26,10 @@ def get_all_num_logprobs(
26
sequence.
27
"""
28
29
- all_num_logprobs = []
29
+ all_num_logprobs: List[int] = []
30
for seq_group_metadata in seq_group_metadata_list:
31
num_logprobs = seq_group_metadata.sampling_params.logprobs
32
- if seq_group_metadata.sampling_params.logprobs is None:
32
+ if num_logprobs is None:
33
num_logprobs = 0
34
all_num_logprobs.append(num_logprobs)
35
vllm/transformers_utils/detokenizer.py
+1
-1
@@ -44,7 +44,7 @@ class Detokenizer:
44
read_offset = 0
45
next_iter_prefix_offset = 0
46
next_iter_read_offset = 0
47
- next_iter_tokens = []
47
+ next_iter_tokens: List[str] = []
48
prev_tokens = None
49
50
for token_position, prompt_logprobs_for_token in enumerate(
vllm/utils.py
+23
-15
@@ -20,12 +20,13 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
20
import numpy as np
21
import psutil
22
import torch
23
+import torch.types
24
+from typing_extensions import ParamSpec
25
26
import vllm.envs as envs
27
from vllm import _custom_ops as ops
28
from vllm.logger import enable_trace_function_call, init_logger
29
28
-T = TypeVar("T")
30
logger = init_logger(__name__)
31
32
STR_DTYPE_TO_TORCH_DTYPE = {
@@ -37,6 +38,10 @@ STR_DTYPE_TO_TORCH_DTYPE = {
38
"fp8_e5m2": torch.uint8,
39
}
40
41
+P = ParamSpec('P')
42
+K = TypeVar("K")
43
+T = TypeVar("T")
44
+
45
46
class Device(enum.Enum):
47
GPU = enum.auto()
@@ -176,7 +181,7 @@ def random_uuid() -> str:
181
182
183
@lru_cache(maxsize=None)
179
-def get_vllm_instance_id():
184
+def get_vllm_instance_id() -> str:
185
"""
186
If the environment variable VLLM_INSTANCE_ID is set, return it.
187
Otherwise, return a random UUID.
@@ -192,7 +197,7 @@ def in_wsl() -> bool:
197
return "microsoft" in " ".join(uname()).lower()
198
199
195
-def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
200
+def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
201
"""Take a blocking function, and run it on in an executor thread.
202
203
This function prevents the blocking function from blocking the
@@ -200,7 +205,7 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
205
The code in this function needs to be thread safe.
206
"""
207
203
- def _async_wrapper(*args, **kwargs) -> asyncio.Future:
208
+ def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
209
loop = asyncio.get_event_loop()
210
p_func = partial(func, *args, **kwargs)
211
return loop.run_in_executor(executor=None, func=p_func)
@@ -325,7 +330,7 @@ def update_environment_variables(envs: Dict[str, str]):
330
os.environ[k] = v
331
332
328
-def chunk_list(lst, chunk_size):
333
+def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
334
"""Yield successive chunk_size chunks from lst."""
335
return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
336
@@ -336,7 +341,7 @@ def cdiv(a: int, b: int) -> int:
341
342
343
def _generate_random_fp8(
339
- tensor: torch.tensor,
344
+ tensor: torch.Tensor,
345
low: float,
346
high: float,
347
) -> None:
@@ -398,7 +403,10 @@ def create_kv_caches_with_random_flash(
403
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
404
key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
405
scale = head_size**-0.5
401
- key_caches, value_caches = [], []
406
+
407
+ key_caches: List[torch.Tensor] = []
408
+ value_caches: List[torch.Tensor] = []
409
+
410
for _ in range(num_layers):
411
key_value_cache = torch.empty(size=key_value_cache_shape,
412
dtype=torch_dtype,
@@ -429,7 +437,7 @@ def create_kv_caches_with_random(
437
scale = head_size**-0.5
438
x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
439
key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
432
- key_caches = []
440
+ key_caches: List[torch.Tensor] = []
441
for _ in range(num_layers):
442
key_cache = torch.empty(size=key_cache_shape,
443
dtype=torch_dtype,
@@ -444,7 +452,7 @@ def create_kv_caches_with_random(
452
key_caches.append(key_cache)
453
454
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
447
- value_caches = []
455
+ value_caches: List[torch.Tensor] = []
456
for _ in range(num_layers):
457
value_cache = torch.empty(size=value_cache_shape,
458
dtype=torch_dtype,
@@ -484,7 +492,7 @@ def is_pin_memory_available() -> bool:
492
493
class CudaMemoryProfiler:
494
487
- def __init__(self, device=None):
495
+ def __init__(self, device: Optional[torch.types.Device] = None):
496
self.device = device
497
498
def current_memory_usage(self) -> float:
@@ -560,13 +568,13 @@ def get_dtype_size(dtype: torch.dtype) -> int:
568
return torch.tensor([], dtype=dtype).element_size()
569
570
563
-def merge_dicts(dict1: Dict[Any, List[Any]],
564
- dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
571
+def merge_dicts(dict1: Dict[K, List[T]],
572
+ dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
573
"""Merge 2 dicts that have key -> List of items.
574
575
When a key conflicts, the values in dict1 is prioritized.
576
"""
569
- merged_dict = defaultdict(list)
577
+ merged_dict: Dict[K, List[T]] = defaultdict(list)
578
579
for key, value in dict1.items():
580
merged_dict[key].extend(value)
@@ -577,7 +585,7 @@ def merge_dicts(dict1: Dict[Any, List[Any]],
585
return dict(merged_dict)
586
587
580
-def init_cached_hf_modules():
588
+def init_cached_hf_modules() -> None:
589
"""
590
Lazy initialization of the Hugging Face modules.
591
"""
@@ -613,7 +621,7 @@ def find_library(lib_name: str) -> str:
621
return locs[0]
622
623
616
-def find_nccl_library():
624
+def find_nccl_library() -> str:
625
"""
626
We either use the library file specified by the `VLLM_NCCL_SO_PATH`
627
environment variable, or we find the library file brought by PyTorch.
vllm/worker/model_runner.py
+2
-2
@@ -779,8 +779,8 @@ class ModelRunner:
779
# that will have unique loras, an therefore the max amount of memory
780
# consumption create dummy lora request copies from the lora request
781
# passed in, which contains a lora from the lora warmup path.
782
- dummy_lora_requests = []
783
- dummy_lora_requests_per_seq = []
782
+ dummy_lora_requests: List[LoRARequest] = []
783
+ dummy_lora_requests_per_seq: List[LoRARequest] = []
784
if self.lora_config:
785
assert self.lora_manager is not None
786
with self.lora_manager.dummy_lora_cache():
vllm/worker/worker_base.py
+2
-2
@@ -99,8 +99,8 @@ class WorkerWrapperBase:
99
"""
100
101
def __init__(self,
102
- worker_module_name=None,
103
- worker_class_name=None,
102
+ worker_module_name: str,
103
+ worker_class_name: str,
104
trust_remote_code: bool = False) -> None:
105
self.worker_module_name = worker_module_name
106
self.worker_class_name = worker_class_name