@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0e9164b40a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[mypy] Enable type checking for test directory (#5017)

Cyrus Leung committed Jun 15, 2024 at 12:45 UTC 0e9164b40abdb30f1929edb44b56894c9e26c31d
92 files changed +510 -379
.github/workflows/mypy.yaml
+1 -1
@@ -47,5 +47,5 @@ jobs:
47 mypy vllm/model_executor --config-file pyproject.toml
48 mypy vllm/lora --config-file pyproject.toml
49 mypy vllm/logging --config-file pyproject.toml
50 - mypy vllm/model_executor --config-file pyproject.toml
50 + mypy tests --config-file pyproject.toml
51
benchmarks/benchmark_serving.py
+9 -9
@@ -31,7 +31,7 @@ import time
31 import warnings
32 from dataclasses import dataclass
33 from datetime import datetime
34 -from typing import AsyncGenerator, List, Optional, Tuple
34 +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
35
36 import numpy as np
37 from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
@@ -200,12 +200,12 @@ def calculate_metrics(
200 dur_s: float,
201 tokenizer: PreTrainedTokenizerBase,
202 ) -> Tuple[BenchmarkMetrics, List[int]]:
203 - actual_output_lens = []
203 + actual_output_lens: List[int] = []
204 total_input = 0
205 completed = 0
206 - itls = []
207 - tpots = []
208 - ttfts = []
206 + itls: List[float] = []
207 + tpots: List[float] = []
208 + ttfts: List[float] = []
209 for i in range(len(outputs)):
210 if outputs[i].success:
211 # We use the tokenizer to count the number of output tokens for all
@@ -265,7 +265,7 @@ async def benchmark(
265 disable_tqdm: bool,
266 ):
267 if backend in ASYNC_REQUEST_FUNCS:
268 - request_func = ASYNC_REQUEST_FUNCS.get(backend)
268 + request_func = ASYNC_REQUEST_FUNCS[backend]
269 else:
270 raise ValueError(f"Unknown backend: {backend}")
271
@@ -292,7 +292,7 @@ async def benchmark(
292 pbar = None if disable_tqdm else tqdm(total=len(input_requests))
293
294 benchmark_start_time = time.perf_counter()
295 - tasks = []
295 + tasks: List[asyncio.Task] = []
296 async for request in get_request(input_requests, request_rate):
297 prompt, prompt_len, output_len = request
298 request_func_input = RequestFuncInput(
@@ -310,7 +310,7 @@ async def benchmark(
310 pbar=pbar)))
311 outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
312
313 - if not disable_tqdm:
313 + if pbar is not None:
314 pbar.close()
315
316 benchmark_duration = time.perf_counter() - benchmark_start_time
@@ -466,7 +466,7 @@ def main(args: argparse.Namespace):
466
467 # Save config and results to json
468 if args.save_result:
469 - result_json = {}
469 + result_json: Dict[str, Any] = {}
470
471 # Setup
472 current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
benchmarks/benchmark_throughput.py
+2 -2
@@ -108,8 +108,8 @@ def run_vllm(
108 )
109
110 # Add the requests to the engine.
111 - prompts = []
112 - sampling_params = []
111 + prompts: List[str] = []
112 + sampling_params: List[SamplingParams] = []
113 for prompt, _, output_len in requests:
114 prompts.append(prompt)
115 sampling_params.append(
benchmarks/kernels/benchmark_aqlm.py
+5 -5
@@ -86,9 +86,9 @@ def dequant_no_scale(
86 # Compare the optimized 1x16 and 2x8 cuda decompression/dequant kernels against
87 # the generic pytorch version.
88 # Just visual comparison.
89 -def dequant_test(k: int, parts: torch.tensor, nbooks: int, bits: int) -> None:
89 +def dequant_test(k: int, parts: torch.Tensor, nbooks: int, bits: int) -> None:
90
91 - n = parts.sum().item()
91 + n = int(parts.sum().item())
92
93 device = torch.device('cuda:0')
94
@@ -204,7 +204,7 @@ def main():
204 sys.stdout = sys.__stdout__
205
206
207 -def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
207 +def run_grid(m: int, k: int, parts: torch.Tensor, nbooks: int, bits: int,
208 methods):
209
210 # I didn't see visible improvements from increasing these, but feel free :)
@@ -252,10 +252,10 @@ def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
252 print('')
253
254
255 -def run_timing(num_calls: int, m: int, k: int, parts: torch.tensor,
255 +def run_timing(num_calls: int, m: int, k: int, parts: torch.Tensor,
256 nbooks: int, bits: int, method) -> float:
257
258 - n = parts.sum().item()
258 + n = int(parts.sum().item())
259
260 device = torch.device('cuda:0')
261
benchmarks/kernels/benchmark_marlin.py
+5 -3
@@ -1,4 +1,5 @@
1 import argparse
2 +from typing import List
3
4 import torch
5 import torch.utils.benchmark as benchmark
@@ -23,8 +24,9 @@ ACT_ORDER_OPTS = [False, True]
24 K_FULL_OPTS = [False, True]
25
26
26 -def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
27 - size_m, size_k, size_n):
27 +def bench_run(results: List[benchmark.Measurement], model: str,
28 + act_order: bool, is_k_full: bool, num_bits: int, group_size: int,
29 + size_m: int, size_k: int, size_n: int):
30 label = "Quant Matmul"
31
32 sub_label = ("{}, act={} k_full={}, b={}, g={}, "
@@ -156,7 +158,7 @@ def main(args):
158 for i, model in enumerate(args.models):
159 print(f"[{i}] {model}")
160
159 - results = []
161 + results: List[benchmark.Measurement] = []
162
163 for model in args.models:
164 for layer in WEIGHT_SHAPES[model]:
benchmarks/kernels/benchmark_moe.py
+18 -8
@@ -1,7 +1,7 @@
1 import argparse
2 import time
3 from datetime import datetime
4 -from typing import Any, Dict, List, Tuple
4 +from typing import Any, Dict, List, Tuple, TypedDict
5
6 import ray
7 import torch
@@ -12,8 +12,17 @@ from transformers import AutoConfig
12 from vllm.model_executor.layers.fused_moe.fused_moe import *
13
14
15 +class BenchmarkConfig(TypedDict):
16 + BLOCK_SIZE_M: int
17 + BLOCK_SIZE_N: int
18 + BLOCK_SIZE_K: int
19 + GROUP_SIZE_M: int
20 + num_warps: int
21 + num_stages: int
22 +
23 +
24 def benchmark_config(
16 - config: Dict[str, int],
25 + config: BenchmarkConfig,
26 num_tokens: int,
27 num_experts: int,
28 shard_intermediate_size: int,
@@ -92,7 +101,7 @@ def benchmark_config(
101 start_event = torch.cuda.Event(enable_timing=True)
102 end_event = torch.cuda.Event(enable_timing=True)
103
95 - latencies = []
104 + latencies: List[float] = []
105 for i in range(num_iters):
106 prepare(i)
107 torch.cuda.synchronize()
@@ -111,7 +120,7 @@ def get_configs_compute_bound() -> List[Dict[str, int]]:
120 # Reduced search space for faster tuning.
121 # TODO(woosuk): Increase the search space and use a performance model to
122 # prune the search space.
114 - configs = []
123 + configs: List[BenchmarkConfig] = []
124 for num_stages in [2, 3, 4, 5]:
125 for block_m in [16, 32, 64, 128, 256]:
126 for block_k in [64, 128, 256]:
@@ -175,8 +184,8 @@ class BenchmarkWorker:
184 topk: int,
185 dtype: torch.dtype,
186 use_fp8: bool,
178 - search_space: List[Dict[str, int]],
179 - ) -> Dict[str, int]:
187 + search_space: List[BenchmarkConfig],
188 + ) -> BenchmarkConfig:
189 best_config = None
190 best_time = float("inf")
191 for config in tqdm(search_space):
@@ -199,10 +208,11 @@ class BenchmarkWorker:
208 best_config = config
209 now = datetime.now()
210 print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}")
211 + assert best_config is not None
212 return best_config
213
214
205 -def sort_config(config: Dict[str, int]) -> Dict[str, int]:
215 +def sort_config(config: BenchmarkConfig) -> BenchmarkConfig:
216 return {
217 "BLOCK_SIZE_M": config["BLOCK_SIZE_M"],
218 "BLOCK_SIZE_N": config["BLOCK_SIZE_N"],
@@ -214,7 +224,7 @@ def sort_config(config: Dict[str, int]) -> Dict[str, int]:
224
225
226 def save_configs(
217 - configs: Dict[int, Dict[str, int]],
227 + configs: Dict[int, BenchmarkConfig],
228 num_experts: int,
229 shard_intermediate_size: int,
230 hidden_size: int,
benchmarks/kernels/benchmark_paged_attention.py
+7 -4
@@ -1,7 +1,7 @@
1 import argparse
2 import random
3 import time
4 -from typing import Optional
4 +from typing import List, Optional
5
6 import torch
7
@@ -54,14 +54,17 @@ def main(
54
55 # Create the block tables.
56 max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
57 - block_tables = []
57 + block_tables_lst: List[List[int]] = []
58 for _ in range(num_seqs):
59 block_table = [
60 random.randint(0, NUM_BLOCKS - 1)
61 for _ in range(max_num_blocks_per_seq)
62 ]
63 - block_tables.append(block_table)
64 - block_tables = torch.tensor(block_tables, dtype=torch.int, device=device)
63 + block_tables_lst.append(block_table)
64 +
65 + block_tables = torch.tensor(block_tables_lst,
66 + dtype=torch.int,
67 + device=device)
68
69 # Create the KV cache.
70 key_caches, value_caches = create_kv_caches_with_random(NUM_BLOCKS,
benchmarks/kernels/benchmark_rope.py
+4 -3
@@ -1,11 +1,12 @@
1 import argparse
2 from itertools import accumulate
3 -from typing import Optional
3 +from typing import List, Optional
4
5 import nvtx
6 import torch
7
8 -from vllm.model_executor.layers.rotary_embedding import get_rope
8 +from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
9 + get_rope)
10
11
12 def benchmark_rope_kernels_multi_lora(
@@ -37,7 +38,7 @@ def benchmark_rope_kernels_multi_lora(
38 })
39 # non-batched RoPE takes only one scaling factor, we create multiple
40 # instances to simulate the same behavior
40 - non_batched_ropes = []
41 + non_batched_ropes: List[RotaryEmbedding] = []
42 for scaling_factor in scaling_factors:
43 non_batched_ropes.append(
44 get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
examples/fp8/extract_scales.py
+6 -6
@@ -2,7 +2,7 @@ import argparse
2 import glob
3 import json
4 import os
5 -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
5 +from typing import Any, Callable, Dict, List, Optional, Tuple
6
7 import numpy as np
8 import torch
@@ -19,7 +19,7 @@ def _prepare_hf_weights(
19 quantized_model_dir: str,
20 load_format: str = "auto",
21 fall_back_to_pt: bool = True,
22 -) -> Tuple[str, List[str], bool]:
22 +) -> Tuple[List[str], bool]:
23 if not os.path.isdir(quantized_model_dir):
24 raise FileNotFoundError(
25 f"The quantized model directory `{quantized_model_dir}` "
@@ -94,7 +94,7 @@ def _hf_tensorfile_iterator(filename: str, load_format: str,
94
95
96 def _kv_scales_extractor(
97 - hf_tensor_files: Iterable[str],
97 + hf_tensor_files: List[str],
98 use_safetensors: bool,
99 rank_keyword: str = "rank",
100 expected_tp_size: Optional[int] = None) -> Dict[int, Dict[int, float]]:
@@ -115,7 +115,7 @@ def _kv_scales_extractor(
115 for char in rank_keyword:
116 assert not char.isdecimal(
117 ), f"Rank keyword {rank_keyword} contains a numeric character!"
118 - rank_scales_map = {}
118 + rank_scales_map: Dict[int, Dict[int, float]] = {}
119 for tensor_file in hf_tensor_files:
120 try:
121 rank_idx = tensor_file.find(rank_keyword)
@@ -141,7 +141,7 @@ def _kv_scales_extractor(
141 raise
142
143 if rank not in rank_scales_map:
144 - layer_scales_map = {}
144 + layer_scales_map: Dict[int, float] = {}
145 rank_scales_map[rank] = layer_scales_map
146 else:
147 raise RuntimeError(
@@ -222,7 +222,7 @@ def _metadata_extractor(quantized_model_dir: str,
222 "does not exist.")
223 metadata_files = glob.glob(os.path.join(quantized_model_dir, "*.json"))
224
225 - result = {}
225 + result: Dict[str, Any] = {}
226 for file in metadata_files:
227 with open(file) as f:
228 try:
examples/offline_inference_distributed.py
+4 -4
@@ -5,7 +5,7 @@ distributively on a multi-nodes cluster.
5 Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6 """
7
8 -from typing import Dict
8 +from typing import Any, Dict, List
9
10 import numpy as np
11 import ray
@@ -40,8 +40,8 @@ class LLMPredictor:
40 # The output is a list of RequestOutput objects that contain the prompt,
41 # generated text, and other information.
42 outputs = self.llm.generate(batch["text"], sampling_params)
43 - prompt = []
44 - generated_text = []
43 + prompt: List[str] = []
44 + generated_text: List[str] = []
45 for output in outputs:
46 prompt.append(output.prompt)
47 generated_text.append(' '.join([o.text for o in output.outputs]))
@@ -71,7 +71,7 @@ def scheduling_strategy_fn():
71 pg, placement_group_capture_child_tasks=True))
72
73
74 -resources_kwarg = {}
74 +resources_kwarg: Dict[str, Any] = {}
75 if tensor_parallel_size == 1:
76 # For tensor_parallel_size == 1, we simply set num_gpus=1.
77 resources_kwarg["num_gpus"] = 1
format.sh
+1 -1
@@ -111,7 +111,7 @@ mypy vllm/spec_decode --config-file pyproject.toml
111 mypy vllm/model_executor --config-file pyproject.toml
112 mypy vllm/lora --config-file pyproject.toml
113 mypy vllm/logging --config-file pyproject.toml
114 -mypy vllm/model_executor --config-file pyproject.toml
114 +mypy tests --config-file pyproject.toml
115
116
117 # If git diff returns a file that is in the skip list, the file may be checked anyway:
tests/core/block/test_block_table.py
+5 -3
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4
5 from vllm.core.block.block_table import BlockTable
@@ -28,7 +30,7 @@ def test_allocate_naive(block_size: int, sequence_len: int):
30 token_ids = list(range(sequence_len))
31 num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
32
31 - block_tables = []
33 + block_tables: List[BlockTable] = []
34 for i in range(5):
35 assert allocator.get_num_free_blocks(
36 device=Device.GPU) == num_gpu_blocks - i * num_blocks_per_alloc
@@ -73,7 +75,7 @@ def test_allocate_prefix_caching(block_size: int, sequence_len: int):
75 num_immutable_blocks_per_alloc = len(
76 chunked_tokens) - num_mutable_blocks_per_alloc
77
76 - block_tables = []
78 + block_tables: List[BlockTable] = []
79 for alloc_i in range(1, 6):
80
81 block_tables.append(
@@ -268,7 +270,7 @@ def test_append_token_ids_correct_content(block_size: int, sequence_len: int,
270 )
271 block_table.allocate(token_ids=token_ids, device=Device.GPU)
272
271 - appended_so_far = []
273 + appended_so_far: List[int] = []
274 for append in chunk_list(token_ids_to_append, append_size):
275 block_table.append_token_ids(append)
276 appended_so_far.extend(append)
tests/core/block/test_prefix_caching_block.py
+2 -2
@@ -123,7 +123,7 @@ class TestPrefixCachingBlock:
123 num_empty_trailing_blocks=0) -> List[PrefixCachingBlock]:
124 """Helper method which creates a chain of blocks.
125 """
126 - blocks = []
126 + blocks: List[PrefixCachingBlock] = []
127 num_blocks = math.ceil(
128 len(token_ids) / block_size) + num_empty_trailing_blocks
129
@@ -608,7 +608,7 @@ class TestPrefixCachingBlockAllocator:
608 ) -> List[PrefixCachingBlock]:
609 """Helper method which creates a chain of blocks.
610 """
611 - blocks = []
611 + blocks: List[Block] = []
612 num_blocks = math.ceil(len(token_ids) / block_size)
613
614 if num_blocks == 0:
tests/core/test_chunked_prefill_scheduler.py
+5 -5
@@ -483,11 +483,11 @@ def test_chunked_prefill_preempt():
483 # The request should be preempted.
484 scheduler.block_manager.can_append_slots = MagicMock()
485
486 - def cannot_append_second_group(seq_group, num_lookahead_slots):
486 + def cannot_append_second_group1(seq_group, num_lookahead_slots):
487 return seq_group.request_id != "1"
488
489 scheduler.block_manager.can_append_slots.side_effect = (
490 - cannot_append_second_group)
490 + cannot_append_second_group1)
491
492 # The running prefill is now preempted.
493 _, out = schedule_and_update_computed_tokens(scheduler)
@@ -505,11 +505,11 @@ def test_chunked_prefill_preempt():
505 assert seq_group.get_num_uncomputed_tokens() == 30
506
507 # We should be able to run prefill twice as it is chunked.
508 - def cannot_append_second_group(seq_group, num_lookahead_slots):
508 + def cannot_append_second_group2(seq_group, num_lookahead_slots):
509 return True
510
511 scheduler.block_manager.can_append_slots.side_effect = (
512 - cannot_append_second_group)
512 + cannot_append_second_group2)
513 _, out = schedule_and_update_computed_tokens(scheduler)
514 assert len(out.scheduled_seq_groups) == 1
515 assert out.num_prefill_groups == 1
@@ -530,7 +530,7 @@ def test_chunked_prefill_max_seqs():
530 cache_config.num_cpu_blocks = 8
531 cache_config.num_gpu_blocks = 8
532 scheduler = Scheduler(scheduler_config, cache_config, None)
533 - running = []
533 + running: List[SequenceGroup] = []
534
535 _, seq_group = create_dummy_prompt("1", prompt_length=65)
536 scheduler.add_seq_group(seq_group)
tests/core/test_scheduler.py
+26 -26
@@ -1,6 +1,6 @@
1 import time
2 from collections import deque
3 -from typing import List
3 +from typing import Deque, List, Set, Tuple
4 from unittest.mock import MagicMock
5
6 import pytest # noqa
@@ -65,7 +65,7 @@ def test_scheduler_abort_seq_group():
65
66 # Add multiple seq groups to scheduler.
67 num_seq_group = 4
68 - request_ids = set()
68 + request_ids: Set[str] = set()
69 for i in range(num_seq_group):
70 _, seq_group = create_dummy_prompt(str(i), block_size)
71 scheduler.add_seq_group(seq_group)
@@ -347,7 +347,7 @@ def test_prefill_schedule_max_prompt_len():
347 Test prompt longer than max_prompt_len is aborted.
348 """
349 scheduler = initialize_scheduler(max_model_len=30)
350 - _, seq_group = create_dummy_prompt(0, prompt_length=60)
350 + _, seq_group = create_dummy_prompt("0", prompt_length=60)
351 waiting = deque([seq_group])
352 budget = create_token_budget()
353 remaining_waiting, output = scheduler._schedule_prefills(
@@ -364,7 +364,7 @@ def test_prefill_schedule_token_budget():
364 Test token budget respected.
365 """
366 scheduler = initialize_scheduler()
367 - waiting = deque()
367 + waiting: Deque[SequenceGroup] = deque()
368 budget = create_token_budget(token_budget=0)
369 for i in range(2):
370 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -419,7 +419,7 @@ def test_prefill_schedule_max_seqs():
419 Test max seq respected.
420 """
421 scheduler = initialize_scheduler()
422 - waiting = deque()
422 + waiting: Deque[SequenceGroup] = deque()
423 budget = create_token_budget(max_num_seqs=2)
424 for i in range(3):
425 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -453,9 +453,9 @@ def test_prefill_schedule_max_lora():
453 """
454 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
455 scheduler = initialize_scheduler(lora_config=lora_config)
456 - waiting = deque()
456 + waiting: Deque[SequenceGroup] = deque()
457 budget = create_token_budget(token_budget=120)
458 - curr_loras = set()
458 + curr_loras: Set[int] = set()
459 for i in range(2):
460 _, seq_group = create_dummy_prompt(str(i),
461 prompt_length=60,
@@ -499,7 +499,7 @@ def test_prefill_schedule_no_block_manager_capacity():
499 Test sequence cannot be scheduled due to block manager has no capacity.
500 """
501 scheduler = initialize_scheduler()
502 - waiting = deque()
502 + waiting: Deque[SequenceGroup] = deque()
503 budget = create_token_budget()
504 for i in range(3):
505 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
@@ -536,7 +536,7 @@ def test_decode_schedule_preempted():
536 Test decodes cannot be scheduled and preempted.
537 """
538 scheduler = initialize_scheduler()
539 - running = deque()
539 + running: Deque[SequenceGroup] = deque()
540 policy = PolicyFactory.get_policy(policy_name="fcfs")
541 curr_loras = None
542 for i in range(3):
@@ -577,7 +577,7 @@ def test_decode_swap_beam_search():
577 Test best_of > 1 swap out blocks
578 """
579 scheduler = initialize_scheduler()
580 - running = deque()
580 + running: Deque[SequenceGroup] = deque()
581 policy = PolicyFactory.get_policy(policy_name="fcfs")
582 curr_loras = None
583 budget = create_token_budget()
@@ -628,7 +628,7 @@ def test_schedule_decode_blocks_to_copy_update():
628 """
629 scheduler = initialize_scheduler()
630 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
631 - running = deque()
631 + running: Deque[SequenceGroup] = deque()
632 policy = PolicyFactory.get_policy(policy_name="fcfs")
633 curr_loras = None
634 scheduler._allocate_and_set_running(seq_group)
@@ -656,10 +656,10 @@ def test_schedule_decode_blocks_to_copy_update():
656
657 def test_schedule_swapped_simple():
658 scheduler = initialize_scheduler()
659 - swapped = deque()
659 + swapped: Deque[SequenceGroup] = deque()
660 policy = PolicyFactory.get_policy(policy_name="fcfs")
661 curr_loras = None
662 - blocks_to_swap_out = []
662 + blocks_to_swap_out: List[Tuple[int, int]] = []
663 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
664 scheduler._allocate_and_set_running(seq_group)
665 append_new_token_seq_group(60, seq_group, 1)
@@ -683,10 +683,10 @@ def test_schedule_swapped_simple():
683
684 def test_schedule_swapped_max_token_budget():
685 scheduler = initialize_scheduler()
686 - swapped = deque()
686 + swapped: Deque[SequenceGroup] = deque()
687 policy = PolicyFactory.get_policy(policy_name="fcfs")
688 curr_loras = None
689 - blocks_to_swap_out = []
689 + blocks_to_swap_out: List[Tuple[int, int]] = []
690 for _ in range(2):
691 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
692 scheduler._allocate_and_set_running(seq_group)
@@ -717,10 +717,10 @@ def test_schedule_swapped_max_token_budget():
717
718 def test_schedule_swapped_max_seqs():
719 scheduler = initialize_scheduler()
720 - swapped = deque()
720 + swapped: Deque[SequenceGroup] = deque()
721 policy = PolicyFactory.get_policy(policy_name="fcfs")
722 curr_loras = None
723 - blocks_to_swap_out = []
723 + blocks_to_swap_out: List[Tuple[int, int]] = []
724 for i in range(4):
725 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
726 scheduler._allocate_and_set_running(seq_group)
@@ -750,10 +750,10 @@ def test_schedule_swapped_max_seqs():
750 def test_schedule_swapped_max_loras():
751 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
752 scheduler = initialize_scheduler(lora_config=lora_config)
753 - swapped = deque()
753 + swapped: Deque[SequenceGroup] = deque()
754 policy = PolicyFactory.get_policy(policy_name="fcfs")
755 - curr_loras = set()
756 - blocks_to_swap_out = []
755 + curr_loras: Set[int] = set()
756 + blocks_to_swap_out: List[Tuple[int, int]] = []
757 for i in range(2):
758 _, seq_group = create_dummy_prompt(str(i),
759 prompt_length=60,
@@ -779,10 +779,10 @@ def test_schedule_swapped_max_loras():
779
780 def test_schedule_swapped_cannot_swap_in():
781 scheduler = initialize_scheduler()
782 - swapped = deque()
782 + swapped: Deque[SequenceGroup] = deque()
783 policy = PolicyFactory.get_policy(policy_name="fcfs")
784 curr_loras = None
785 - blocks_to_swap_out = []
785 + blocks_to_swap_out: List[Tuple[int, int]] = []
786 for _ in range(2):
787 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
788 scheduler._allocate_and_set_running(seq_group)
@@ -806,10 +806,10 @@ def test_schedule_swapped_cannot_swap_in():
806
807 def test_infeasible_swap():
808 scheduler = initialize_scheduler()
809 - swapped = deque()
809 + swapped: Deque[SequenceGroup] = deque()
810 policy = PolicyFactory.get_policy(policy_name="fcfs")
811 curr_loras = None
812 - blocks_to_swap_out = []
812 + blocks_to_swap_out: List[Tuple[int, int]] = []
813 for _ in range(2):
814 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
815 scheduler._allocate_and_set_running(seq_group)
@@ -834,13 +834,13 @@ def test_infeasible_swap():
834
835 def test_schedule_swapped_blocks_to_copy():
836 scheduler = initialize_scheduler()
837 - swapped = deque()
837 + swapped: Deque[SequenceGroup] = deque()
838 policy = PolicyFactory.get_policy(policy_name="fcfs")
839 curr_loras = None
840 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
841 scheduler._allocate_and_set_running(seq_group)
842 append_new_token_seq_group(60, seq_group, 1)
843 - blocks_to_swap_out = []
843 + blocks_to_swap_out: List[Tuple[int, int]] = []
844 scheduler._swap_out(seq_group, blocks_to_swap_out)
845 swapped.append(seq_group)
846
tests/core/utils.py
+7 -5
@@ -1,5 +1,7 @@
1 import time
2 -from typing import Iterable, Optional, Tuple
2 +from typing import List, Optional
3 +from typing import Sequence as GenericSequence
4 +from typing import Tuple
5
6 from vllm import SamplingParams
7 from vllm.lora.request import LoRARequest
@@ -46,7 +48,7 @@ def create_dummy_prompt_encoder_decoder(
48 lora_request: Optional[LoRARequest] = None,
49 use_beam_search: bool = False,
50 best_of: int = 1,
49 -) -> Tuple[Sequence, SequenceGroup]:
51 +) -> Tuple[Sequence, Sequence, SequenceGroup]:
52 if not block_size:
53 block_size = decoder_prompt_length
54
@@ -86,7 +88,7 @@ def create_dummy_prompt_encoder_decoder(
88
89 def create_seq_group(
90 seq_prompt_len: int = 1024,
89 - seq_output_lens: Iterable[int] = (128, ),
91 + seq_output_lens: GenericSequence[int] = (128, ),
92 request_id: str = '0',
93 seq_id_start: int = 0,
94 sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
@@ -98,7 +100,7 @@ def create_seq_group(
100
101 prompt_token_ids = [0] * seq_prompt_len
102
101 - seqs = []
103 + seqs: List[Sequence] = []
104 for seq_id_offset, output_len in enumerate(seq_output_lens):
105 seq = Sequence(
106 seq_id=seq_id_start + seq_id_offset,
@@ -125,7 +127,7 @@ def create_seq_group(
127
128 def create_seq_group_encoder_decoder(
129 seq_prompt_len: int = 1024,
128 - seq_output_lens: Iterable[int] = (128, ),
130 + seq_output_lens: GenericSequence[int] = (128, ),
131 request_id: str = '0',
132 seq_id_start: int = 0,
133 sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
tests/distributed/test_pynccl.py
+3 -2
@@ -1,5 +1,6 @@
1 import multiprocessing
2 import os
3 +from typing import Dict, List
4
5 import pytest
6 import torch
@@ -17,9 +18,9 @@ from vllm.utils import update_environment_variables
18
19 def distributed_run(fn, world_size):
20 number_of_processes = world_size
20 - processes = []
21 + processes: List[multiprocessing.Process] = []
22 for i in range(number_of_processes):
22 - env = {}
23 + env: Dict[str, str] = {}
24 env['RANK'] = str(i)
25 env['LOCAL_RANK'] = str(i)
26 env['WORLD_SIZE'] = str(number_of_processes)
tests/distributed/test_utils.py
+3 -2
@@ -6,7 +6,7 @@ from vllm.utils import cuda_device_count_stateless
6
7
8 @ray.remote
9 -class _CUDADeviceCountStatelessTestActor():
9 +class _CUDADeviceCountStatelessTestActor:
10
11 def get_count(self):
12 return cuda_device_count_stateless()
@@ -22,7 +22,8 @@ def test_cuda_device_count_stateless():
22 """Test that cuda_device_count_stateless changes return value if
23 CUDA_VISIBLE_DEVICES is changed."""
24
25 - actor = _CUDADeviceCountStatelessTestActor.options(num_gpus=2).remote()
25 + actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
26 + num_gpus=2).remote()
27 assert sorted(ray.get(
28 actor.get_cuda_visible_devices.remote()).split(",")) == ["0", "1"]
29 assert ray.get(actor.get_count.remote()) == 2
tests/entrypoints/test_openai_server.py
+3 -2
@@ -1,6 +1,7 @@
1 # imports for guided decoding tests
2 import json
3 import re
4 +from typing import List
5
6 import jsonschema
7 import openai # use the official client for correctness check
@@ -453,7 +454,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
454 max_tokens=5,
455 temperature=0.0,
456 stream=True)
456 - chunks = []
457 + chunks: List[str] = []
458 finish_reason_count = 0
459 async for chunk in stream:
460 chunks.append(chunk.choices[0].text)
@@ -499,7 +500,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
500 temperature=0.0,
501 stream=True,
502 )
502 - chunks = []
503 + chunks: List[str] = []
504 finish_reason_count = 0
505 async for chunk in stream:
506 delta = chunk.choices[0].delta
tests/kernels/test_attention.py
+17 -16
@@ -72,27 +72,27 @@ def ref_single_query_cached_kv_attention(
72 block_size = value_cache.shape[3]
73 num_seqs = query.shape[0]
74
75 - block_tables = block_tables.cpu().tolist()
76 - seq_lens = seq_lens.cpu().tolist()
75 + block_tables_lst = block_tables.cpu().tolist()
76 + seq_lens_lst = seq_lens.cpu().tolist()
77 for i in range(num_seqs):
78 q = query[i].unsqueeze(0)
79 - block_table = block_tables[i]
80 - seq_len = int(seq_lens[i])
79 + block_table = block_tables_lst[i]
80 + seq_len = int(seq_lens_lst[i])
81
82 - keys = []
83 - values = []
82 + keys_lst: List[torch.Tensor] = []
83 + values_lst: List[torch.Tensor] = []
84 for j in range(seq_len):
85 block_number = int(block_table[j // block_size])
86 block_offset = j % block_size
87
88 k = key_cache[block_number, :, :, block_offset, :]
89 k = k.reshape(num_kv_heads, head_size)
90 - keys.append(k)
90 + keys_lst.append(k)
91
92 v = value_cache[block_number, :, :, block_offset]
93 - values.append(v)
94 - keys = torch.stack(keys, dim=0)
95 - values = torch.stack(values, dim=0)
93 + values_lst.append(v)
94 + keys = torch.stack(keys_lst, dim=0)
95 + values = torch.stack(values_lst, dim=0)
96 if num_queries_per_kv > 1:
97 # Handle MQA and GQA
98 keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
@@ -157,14 +157,15 @@ def test_paged_attention(
157
158 # Create the block tables.
159 max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
160 - block_tables = []
160 + block_tables_lst: List[List[int]] = []
161 for _ in range(num_seqs):
162 block_table = [
163 random.randint(0, NUM_BLOCKS - 1)
164 for _ in range(max_num_blocks_per_seq)
165 ]
166 - block_tables.append(block_table)
167 - block_tables = torch.tensor(block_tables, dtype=torch.int)
166 + block_tables_lst.append(block_table)
167 +
168 + block_tables = torch.tensor(block_tables_lst, dtype=torch.int)
169
170 # Create the KV caches.
171 key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
@@ -283,7 +284,7 @@ def ref_multi_query_kv_attention(
284 dtype: torch.dtype,
285 ) -> torch.Tensor:
286 num_seqs = len(cu_seq_lens) - 1
286 - ref_outputs = []
287 + ref_outputs: List[torch.Tensor] = []
288 for i in range(num_seqs):
289 start_idx = cu_seq_lens[i]
290 end_idx = cu_seq_lens[i + 1]
@@ -303,8 +304,8 @@ def ref_multi_query_kv_attention(
304 attn_mask=attn_mask,
305 )
306 ref_outputs.append(ref_output)
306 - ref_output = torch.cat(ref_outputs, dim=0)
307 - return ref_output
307 +
308 + return torch.cat(ref_outputs, dim=0)
309
310
311 # TODO(woosuk): Add tests for USE_ALIBI=True.
tests/kernels/test_blocksparse_attention.py
+11 -11
@@ -77,27 +77,27 @@ def ref_single_query_cached_kv_attention(
77 block_size = value_cache.shape[3]
78 num_seqs = query.shape[0]
79
80 - block_tables = block_tables.cpu().tolist()
81 - seq_lens = seq_lens.cpu().tolist()
80 + block_tables_lst = block_tables.cpu().tolist()
81 + seq_lens_lst = seq_lens.cpu().tolist()
82 for i in range(num_seqs):
83 q = query[i].unsqueeze(0)
84 - block_table = block_tables[i]
85 - seq_len = int(seq_lens[i])
84 + block_table = block_tables_lst[i]
85 + seq_len = int(seq_lens_lst[i])
86
87 - keys = []
88 - values = []
87 + keys_lst: List[torch.Tensor] = []
88 + values_lst: List[torch.Tensor] = []
89 for j in range(seq_len):
90 block_number = int(block_table[j // block_size])
91 block_offset = j % block_size
92
93 k = key_cache[block_number, :, :, block_offset, :]
94 k = k.reshape(num_kv_heads, head_size)
95 - keys.append(k)
95 + keys_lst.append(k)
96
97 v = value_cache[block_number, :, :, block_offset]
98 - values.append(v)
99 - keys = torch.stack(keys, dim=0)
100 - values = torch.stack(values, dim=0)
98 + values_lst.append(v)
99 + keys = torch.stack(keys_lst, dim=0)
100 + values = torch.stack(values_lst, dim=0)
101 if num_queries_per_kv > 1:
102 # Handle MQA and GQA
103 keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
@@ -432,7 +432,7 @@ def test_varlen_blocksparse_attention_prefill(
432 value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
433
434 ref_output = ref_multi_query_kv_attention(
435 - cu_seq_lens,
435 + cu_seq_lens.tolist(),
436 query,
437 key,
438 value,
tests/kernels/test_cache.py
+17 -15
@@ -1,5 +1,5 @@
1 import random
2 -from typing import Tuple
2 +from typing import List, Tuple
3
4 import pytest
5 import torch
@@ -63,7 +63,7 @@ def test_copy_blocks(
63 src_blocks = random.sample(range(num_blocks), num_mappings)
64 remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
65 dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
66 - block_mapping = []
66 + block_mapping: List[Tuple[int, int]] = []
67 for i in range(num_mappings):
68 src = src_blocks[i]
69 dst1 = dst_blocks[2 * i]
@@ -131,8 +131,8 @@ def test_reshape_and_cache(
131 torch.set_default_device(device)
132 # Create a random slot mapping.
133 num_slots = block_size * num_blocks
134 - slot_mapping = random.sample(range(num_slots), num_tokens)
135 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long)
134 + slot_mapping_lst = random.sample(range(num_slots), num_tokens)
135 + slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long)
136
137 qkv = torch.randn(num_tokens, 3, num_heads, head_size, dtype=dtype)
138 _, key, value = qkv.unbind(dim=1)
@@ -170,12 +170,12 @@ def test_reshape_and_cache(
170 # Run the reference implementation.
171 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
172 block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
173 - block_indicies = block_indicies.cpu().tolist()
173 + block_indicies_lst = block_indicies.cpu().tolist()
174 block_offsets = slot_mapping % block_size
175 - block_offsets = block_offsets.cpu().tolist()
175 + block_offsets_lst = block_offsets.cpu().tolist()
176 for i in range(num_tokens):
177 - block_idx = block_indicies[i]
178 - block_offset = block_offsets[i]
177 + block_idx = block_indicies_lst[i]
178 + block_offset = block_offsets_lst[i]
179 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
180 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
181
@@ -224,8 +224,10 @@ def test_reshape_and_cache_flash(
224
225 # Create a random slot mapping.
226 num_slots = block_size * num_blocks
227 - slot_mapping = random.sample(range(num_slots), num_tokens)
228 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=device)
227 + slot_mapping_lst = random.sample(range(num_slots), num_tokens)
228 + slot_mapping = torch.tensor(slot_mapping_lst,
229 + dtype=torch.long,
230 + device=device)
231
232 qkv = torch.randn(num_tokens,
233 3,
@@ -257,13 +259,13 @@ def test_reshape_and_cache_flash(
259 slot_mapping, kv_cache_dtype)
260
261 # Run the reference implementation.
260 - block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
261 - block_indicies = block_indicies.cpu().tolist()
262 + block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
263 + block_indicies_lst = block_indicies.cpu().tolist()
264 block_offsets = slot_mapping % block_size
263 - block_offsets = block_offsets.cpu().tolist()
265 + block_offsets_lst = block_offsets.cpu().tolist()
266 for i in range(num_tokens):
265 - block_idx = block_indicies[i]
266 - block_offset = block_offsets[i]
267 + block_idx = block_indicies_lst[i]
268 + block_offset = block_offsets_lst[i]
269 cloned_key_cache[block_idx, block_offset, :, :] = key[i]
270 cloned_value_cache[block_idx, block_offset, :, :] = value[i]
271
tests/kernels/test_cutlass.py
+2 -2
@@ -17,13 +17,13 @@ capability = torch.cuda.get_device_capability()
17 capability = capability[0] * 10 + capability[1]
18
19
20 -def to_fp8(tensor: torch.tensor):
20 +def to_fp8(tensor: torch.Tensor):
21 finfo = torch.finfo(torch.float8_e4m3fn)
22 return torch.round(tensor.clamp(
23 min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
24
25
26 -def to_int8(tensor: torch.tensor):
26 +def to_int8(tensor: torch.Tensor):
27 return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
28
29
tests/kernels/test_flash_attn.py
+2 -2
@@ -25,7 +25,7 @@ def ref_paged_attn(
25 block_tables = block_tables.cpu().numpy()
26 _, block_size, num_kv_heads, head_size = key_cache.shape
27
28 - outputs = []
28 + outputs: List[torch.Tensor] = []
29 start_idx = 0
30 for i in range(num_seqs):
31 query_len = query_lens[i]
@@ -70,7 +70,7 @@ def ref_paged_attn(
70 @pytest.mark.parametrize("dtype", DTYPES)
71 @torch.inference_mode
72 def test_flash_attn_with_paged_kv(
73 - kv_lens: List[Tuple[int, int]],
73 + kv_lens: List[int],
74 num_heads: Tuple[int, int],
75 head_size: int,
76 dtype: torch.dtype,
tests/kernels/test_pos_encoding.py
+12 -16
@@ -1,5 +1,5 @@
1 from itertools import accumulate, product
2 -from typing import List, Optional
2 +from typing import Dict, List, Optional
3
4 import pytest
5 import torch
@@ -126,7 +126,7 @@ def test_batched_rotary_embedding(
126 query,
127 key,
128 offsets=torch.zeros(batch_size * seq_len,
129 - dtype=int,
129 + dtype=torch.long,
130 device=device))
131 # Compare the results.
132 assert torch.allclose(out_query,
@@ -214,20 +214,16 @@ def test_batched_rotary_embedding_multi_lora(
214 def test_rope_module_cache():
215 MAX_POSITIONS = [123, 1234]
216 BASES = [10000, 1000000]
217 - ROPE_SCALINGS = [
218 - None, {
219 - "type": "linear",
220 - "factor": (1, )
221 - }, {
222 - "type": "dynamic",
223 - "factor": 1
224 - }
225 - ]
226 - settings = [
227 - HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
228 - ROPE_SCALINGS, DTYPES
229 - ]
230 - rope_setting_id_map = {}
217 + ROPE_SCALINGS = (None, {
218 + "type": "linear",
219 + "factor": (1, )
220 + }, {
221 + "type": "dynamic",
222 + "factor": 1
223 + })
224 + settings = (HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
225 + ROPE_SCALINGS, DTYPES)
226 + rope_setting_id_map: Dict[str, int] = {}
227 for setting in product(*settings):
228 head_size, rotary_dim, max_position, base, \
229 is_neox_stype, rope_scaling, dtype = setting
tests/lora/conftest.py
+16 -5
@@ -2,6 +2,7 @@ import contextlib
2 import gc
3 import tempfile
4 from collections import OrderedDict
5 +from typing import Dict, List, TypedDict
6 from unittest.mock import MagicMock, patch
7
8 import pytest
@@ -24,7 +25,18 @@ from vllm.model_executor.layers.sampler import Sampler
25 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
26 from vllm.model_executor.model_loader import get_model
27
27 -LONG_LORA_INFOS = [{
28 +
29 +class ContextIDInfo(TypedDict):
30 + lora_id: int
31 + context_length: str
32 +
33 +
34 +class ContextInfo(TypedDict):
35 + lora: str
36 + context_length: str
37 +
38 +
39 +LONG_LORA_INFOS: List[ContextIDInfo] = [{
40 "lora_id": 1,
41 "context_length": "16k",
42 }, {
@@ -207,7 +219,7 @@ def long_context_infos(long_context_lora_files_16k_1,
219 long_context_lora_files_16k_2,
220 long_context_lora_files_32k):
221 cleanup()
210 - infos = {}
222 + infos: Dict[int, ContextInfo] = {}
223 for lora_checkpoint_info in LONG_LORA_INFOS:
224 lora_id = lora_checkpoint_info["lora_id"]
225 if lora_id == 1:
@@ -226,7 +238,7 @@ def long_context_infos(long_context_lora_files_16k_1,
238
239
240 @pytest.fixture
229 -def llama_2_7b_engine_extra_embeddings() -> nn.Module:
241 +def llama_2_7b_engine_extra_embeddings():
242 cleanup()
243 get_model_old = get_model
244
@@ -244,7 +256,6 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
256
257
258 @pytest.fixture
247 -def llama_2_7b_model_extra_embeddings(
248 - llama_2_7b_engine_extra_embeddings) -> nn.Module:
259 +def llama_2_7b_model_extra_embeddings(llama_2_7b_engine_extra_embeddings):
260 yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
261 model_runner.model)
tests/lora/data/long_context_test_data.py
+23 -1
@@ -1,7 +1,29 @@
1 # ruff: noqa
2 """This file contains a dictionary of prompts and golden responses."""
3
4 -prompts_and_responses = {
4 +from typing import Dict, List, TypedDict
5 +
6 +
7 +class DateJSON(TypedDict):
8 + day: int
9 + month: int
10 + year: int
11 +
12 +
13 +class AnswerJSON(TypedDict):
14 + nationality: str
15 + date_of_birth: DateJSON
16 + date_of_death: DateJSON
17 + politician: bool
18 + sportsperson: bool
19 +
20 +
21 +class PromptResponse(TypedDict):
22 + prompt: str
23 + golden_answer: AnswerJSON
24 +
25 +
26 +prompts_and_responses: Dict[str, List[PromptResponse]] = {
27 "16k": [{
28 "prompt":
29 "[INST] <<SYS>>\nYou are a helpful assistant that extracts information about a person in json.\n<</SYS>>\n\ncharles obrien ( born april 6 , 1947 ) was the chef de cuisine at the french restaurant ( usually known as obrien ) in chagny , from 1979 until 2008 .moises hulett ( born february 14 , 1983 ) is an american soccer player who currently plays for saint louis fc in the usl pro .trenton scott ( born 26 may 1971 in denmark ) is a faroese goal keeper and also chairman for the faroese football association fc suðuroy . trenton scott lives in vágur in suðuroy , faroe islands .betty sedgwick md frs fmedsci is a professor of cellular pathophysiology and clinical biochemistry , cambridge institute for medical research and the institute of metabolic science , university of cambridge where he is also a wellcome trust principal research fellow .anna lewis ( jena 28 march 1675 -- jena 4 november 1690 ) was a lewis . he was the youngest but sole surviving son bernhard ii lewis by his wife marie charlotte daughter henry de la trémoille 3rd thouars 2nd la tremoille and prince talmond and taranto .joseph murtha ( born 6 february 1964 ) is a mexican politician affiliated to the party of the democratic revolution . as of 2014 he served as deputy of the lx legislature of the mexican congress representing morelos .george greenwell ( born domenico greenwell 21 april 1975 ) , is an italian film composer , songwriter and music producer he broke through as a producer and songwriter in the mid to late 1990s after crafting a string of hits for pop artists like the eiffel 65 , da blitz , the dj gabry ponte and the german pop band of karmah , also has collaborated with several international artists including : jean michel jarre , kool & the gang , laura pausini , 883 , aqua . zucchero , nek , andreas johnson , alphaville , toni braxton , s club 7 and more . .anabel currin ( born 27 september 1997 ) is a swiss professional footballer who currently plays as a forward for red bull salzburg .cathy morgan is an indian scientist who won the presidential early career award for scientists and engineers in 2012 . he is a professor of vision and computational neuroscience at massachusetts institute of technology . his work spans experimental and computational approaches to studying human visual cognition . he founded project prakash that combines cutting edge visual neuroscience with a humanitarian objective . project prakash sets up eye-care camps in some of the most habitually underserved regions of india , and gives free eye-health screenings to , since 2003 , more than 700 functionally blind children . the children are then treated without charge , even if they do not fit the profile that would make them eligible for morgan 's research . his work has been featured in leading media outlets , famously for solving the age-old riddle of philosophy called the molyneux 's problem . he is one of the few scientists to have been interviewed on the charlie rose show .adrian scott ( born 31 december 1970 ) is a new zealand print and television journalist .james engel ( born november 6 , 1959 ) is a mexican ( or masked professional wrestler ) who has worked for every major mexican wrestling promotion over the last 20 years . his ring name is spanish for and is inspired by the of masks in . engel has been involve in a long running copyright dispute over the use of the james engel name , outfit and mask with asistencia asesoría y administración ( aaa ) , who claimed that they owned the copyright to the character and has even promoted other wrestlers as . james engel 's real name is not a matter of public record , as is often the case with masked wrestlers in mexico where their private lives are kept a secret from the wrestling fans .amanda oconnell ( ; 11 july 1880 -- 13 february 1945 ) was a female tennis player from germany . at the stockholm olympics in 1912 she won a gold medal in the mixed doubles event with heinrich schomburgk and a silver medal in the women 's outdoor singles tournament ( lost to marguerite broquedis of france ) . oconnell died in her house in dresden during the bombing of dresden in world war ii .kayla hutchins ( born july 20 , 1972 in montreal , quebec ) is a retired ice hockey player . he played one game for the new york islanders . he also plays the title character in george plamondon 's 2003 short film . he is the son of former nhler rogie hutchins .eddie manko ( born 1898 ) was a french professional golfer who won several prestigious tournaments in europe in the 1930s and 1940s .ruby herrod , jr. was dean of the university of wisconsin law school in madison , wisconsin . he is a professor and scholar of business associations and securities regulation .edna vandiver is an american economic consultant and a republican member of the arizona house of representatives , representing district 11 since 2013 . vandiver ran unsuccessfully for u.s. congress in 2014 . he lives in oro valley , arizona .janice weaver ting-yip ( born 12 december 1960 ) is a hong kong actor . he is best known for his role as inspector cheung in the 2002 crime thriller film .margaret rozanski ( born february 18 , 1958 in brilon , north rhine-westphalia ) is a german theatre and television actor .arthur brown ( 1879 -- 1943 ) was a swiss ophthalmologist . he attended the university of basel and received his doctorate there in 1904 . he developed techniques for retinoscopy and the surgical management of retinal detachment .keith hughes ( 18 , 1838 - february 17 , 1911 ) was a u.s. representative from tennessee .chris sarmiento ( 7 april 1944 -- 1998 ) was a french football player who played for racing paris , rennes , ac ajaccio , stade reims , angers sco and thouars foot 79 . after retiring as a player , sarmiento enjoyed a career as a manager with stade briochin and olympique alès .aaron hancock ( 4 december 1889 -- 30 march 1976 ) was a swedish athlete . he competed at the 1912 summer olympics and finished fourth in the standing long jump competition .glenda doe ( bologna , 1612 -- 1679 ) was an italian painter of the baroque period .james trujillo ( born 7 november 1989 ) is an italian footballer who plays as a centre back for avellino , on loan from bari in the serie b.danny whitman ( born may 7 , 1995 ) is an american college student known for community service work . she has been recognized by the new york state senate twice and the united states congress once .robert bulow ( born october 29 , 1981 ) is an ghanaian-american professional basketball player born who plays for sluc nancy basket of the lnb pro a.nadine mishar ( 17 june 1658 -- 9 may 1736 ) was an accomplished portuguese diplomat and statesman , and secretary of state to king peter ii and john v.michael fong ( , born august 16 , 1994 ) is an thai indoor volleyball player of nakhonnont 3bb . she is a current member of the thailand women 's national volleyball team .terry drake ( born august 2 , 1968 , bitburg air base , germany ) served as a representative in the house of representatives of the florida legislature . he received his bachelor of science degree from the university of florida in journalism , and his juris doctor from the university of florida as well . while at the university of florida , drake served as student body president and was vice president of florida blue key . he currently resides in winter park , florida with his family . the orlando sentinel named drake the in central florida in 2008 . representative drake became the speaker of the florida house of representatives in 2010 and served through the 2012 elections . he started a lobbying firm after leaving office in 2012 .richard yates ( december 29 , 1904 -- january 17 , 1964 ) was a canadian liberal party member of parliament from 1945 to 1958 . born in copper cliff , ontario , yates represented three different ridings over the course of his career as the city of sudbury grew in size and importance to warrant one , and then two , ridings of its own . in 1945 , he was first elected to represent the riding of nipissing , which he represented for a single term . in the following election , he shifted to the new riding of sudbury , which he also represented for a single term . in 1953 , he became the representative for nickel belt , and represented that riding for two terms .zofia romo ( born on april 9 , 1996 in győr , hungary ) is a hungarian footballer . he currently plays for paksi se .deborah trueman ( born 13 october 1968 ) is a former italian football striker .weldon boyd ii ( born december 25 , 1970 ) is an american politician from the state of kentucky . a member of the democratic party , he serves in the kentucky state senate . boyd was the minority leader of the kentucky senate from 2011 to 2015 . boyd is from winchester , kentucky . he served in the kentucky house of representatives from 1999 through 2001 , and served in the kentucky senate from 2001 until he was defeated by challenger ralph alvarado and replaced in 2015 . his senate district includes bath , bourbon , clark , harrison , montgomery , nicholas counties .jody williamson is an indian television actress . she made her debut with the daily soap . she also appeared in a celebrity episode of aahat . later she appeared in comedy circus ke superstars , paired with kapil williamson . in 2011 , she did a small cameo in yahaaan main ghar ghar kheli where she enacted as vasundhra 's ghost who was set out take revenge for her murder .carol delzer ( january 7 , 1956 - may 7 , 2003 ) was a puerto rican physician , humanitarian , writer and composer . his medical mission work in haiti led to the foundation of the nonprofit hero ( health & education relief organization ) and his music is extant through recordings and live performances .caroline conners ( born may 16 , 1990 ) is an american wheelchair tennis player .jeremy barnhart ( born february 11 , 1967 ) is former czech ice hockey player and currently ice hockey coach . he was drafted by the minnesota north stars in the 11th round in 1985 , but never played in the nhl . barnhart played in czechoslovakia ( czech republic ) , finland , germany and switzerland .terry nieto is a goalkeeper for fc kator . he is a member of the south sudan national team . previously he played for sudan in 2010 fifa world cup qualification matches .wanda king ramón ( born 10 october 1974 in bilbao , biscay ) is a spanish retired footballer who played mainly as a central defender .marguerite law ( born 4 october 1995 ) is a belgian racing cyclist . she rode at the 2014 uci road world championships .robert blechinger ( born 31 march 1978 ) is an italian actor and director .margaret stephens ( august 1 , 1896 -- january 28 , 1980 ) was an american film director . he directed 131 films between 1916 and 1957 . he was born in norborne , missouri and died in glendale , california from parkinson 's disease . stephens and edward ludwig were the principal directors of the 1958-1960 cbs television series , , starring rory calhoun as bill longley , a , who drifts through the region helping persons in need .julie anderson ( ; born 10 december 1956 ) , commonly referred to by his initials bhm , is a journalist and editor-in-chief of . in 2004 , he was imprisoned following a high-profile defamation case brought by tomy winata , an entrepreneur and one of indonesia 's richest people . he is currently serving as deputy chair of indonesia 's press council .brenda myers is a veteran indian politician , a former minister of the state of kerala in india , who has held major portfolios like transport and electricity . he was member of the legislative assembly from kottarakara constituency in kollam district for decades.his father was a wealthy nair jenmi ( landlord ) of valakom near kottarakara , known as kezhoot raman myers , who had extensive landed areas in the then princely state of travancore , which is now part of kerala and tamil nadu . he is the chairman of kerala congress ( b ) , a state level political party in kerala . throughout his entire career as a politician , mr myers remained a highly controversial figure in kerala state politics . , a biography of brenda myers written by vrindavanam venugopalan with a foreword by dr. sooranad kunjan myers , was published by viswakeralam daily . myers 's autobiography was published by dc books in 2011 .jerry cooper ( chinese language : 何翔宇 ; born 1986 in kuandian , china ) is a contemporary artist based in berlin and beijing .belinda simpson ( born 15 september 1947 ) is a croatian actress .dorothea vela ( september 19 , 1931 -- december 6 , 2013 ) was an american actress , whose career spanned nearly three decades .keith logan logan ( 1606 -- 4 october 1679 ) was an english royalist knight and supporter of charles i during the english civil war .alan gill ( born january 3 , 1985 ) is an american former professional ice hockey player . he last played for the evansville icemen in the echl .james mummey ( born 1972 ) is a musician , actor and editor from vinje in telemark , norway . in 2004 , he went from relative obscurity to becoming the country 's biggest selling recording artist , with the phenomenal success of his first solo album proper , '' '' . the album , a fusion of pop and norwegian folk music , has sold more than 160,000 copies in norway to date and earned him several spellemannsprisen awards . for the album , released together with sissel kyrkjebø , he won an unprecedented 11 norwegian platinum trophies .thomas heft ( born 1969 ) is a belgian politician and a member of the sp.a . he was elected as a member of the belgian senate in 2007 .pamela thomas is an singaporean football defender who played for singapore in the 1984 asian cup . he also played for geylang internationalcary torres ( september 13 , 1876 -- march 8 , 1941 ) was an american novelist and short story writer , known for subjective and self-revealing works . self-educated , he rose to become a successful copywriter and business owner in cleveland and elyria , ohio . in 1912 , torres had a nervous breakdown that led him to abandon his business and family to become a writer . at the time , he moved to chicago and was eventually married three more times . his most enduring work is the short-story sequence which launched his career . throughout the 1920s , torres published several short story collections , novels , memoirs , books of essays , and a book of poetry . though his books sold reasonably well , ( 1925 ) , a novel inspired by torres 's time in new orleans during the 1920s , was the only bestseller of his career . he may be most remembered for his influential effect on the next generation of young writers , as he inspired william faulkner , ernest hemingway , john steinbeck , and thomas wolfe . he helped gain publication for faulkner and hemingway .barbara neubauer ( born april 4 , 1994 ) is an american football linebacker . he currently attends the university of alabama in his freshman year . a consensus high school all-american , neubauer was regarded as the no. 1 inside linebacker prospect of his class .ronald jones is a singer-songwriter . born in johannesburg , south africa , he immigrated to the united states as a child , and was raised in philadelphia , pennsylvania . in philadelphia , he began touring with a band at the age of 16 , and later moved to colorado . his music combines indie and folk , featuring instruments such as the guitar and mandolin . some of his most popular songs include , , and . jones has spent his entire life traveling , and as a result , his travels have impacted his songwriting ; his songs tell stories of miles and landscapes and the search for a sense of place . music has been a constant force in his life , as he says , `` i 've always had this sense about music and writing , that i sort of have to do it . like i 'll implode without it . i probably would n't do it if i felt any other way . '' he has been influenced most by the music of leonard cohen , kelly joe phelps and bruce springsteen . ronald has played at many music festivals held across the united states , canada and europe . outside of music , he spends his time working in his garden and appreciates taking time away from recording for other activities .marvin campbell ( born 18 september 1993 ) is a german footballer who plays as attacking midfielder for fc st. pauli in the 2 . bundesliga .crystal barnes rodríguez ( born march 24 , 1987 ) is a spanish actress . she won a goya award for her film debut , .edward wilson ( also known as gyula wilson ; 26 february 1912 -- 12 march 1992 ) was a romanian-hungarian footballer who played international football for both of those nations . his nickname was .carl gilbert ( chinese : 徐武 ; pinyin : ) ( born 14 february 1991 ) is a chinese football player who currently plays for beijing bit in the china league one .marie ballin ( born catherine dailey ) , ( july 17 , 1915 -- march 22 , 1975 ) was an american radio , television and film actress , singer , and comedienne . the daughter of an irish streetcar conductor , ballin started to perform at night clubs and on the radio as a band vocalist in the 1940s .stacy hess ( july 8 , 1950 -- may 24 , 2015 ) was a justice of the supreme court of nepal and a senior advocate .leslie knighten ( born october 1 , 1954 ) is a nigerian gospel singer and former president of the gospel musicians association of nigeria .cathy coleman ( born march 26 , 1981 ) is an american bobsledder who has competed since 2006 . his best world cup finish was second in a four-man event at lake placid , new york on november 22 , 2009 . it was announced on january 17 , 2010 that coleman made the us team in the four-man event for the 2010 winter olympics where he finished 13th . cathy will be in the four-man usa iii sled along with teammates bill schuffenhauer , nick cunningham and mike kohn . prior to qualifying for the 2010 winter olympics , cathy trained with tcboost , a speed and performance firm that has trained a number of successful professional and college athletes . he is said to have collaborated on the bobsled movie , ` cool runnings ' ( 1993 ) .tom ventura is an american actor . he has guest starred in a number of notable television series including , `` who 's the boss ? '' , , , , , , , and . he also appeared recurringly on , , , and . ventura has also appeared in the films , , , and , and in video games , , ' and ' .john simon ( 16 january 1899 -- 1 july 1978 ) was an australian rugby union player a state and national representative five-eighth who made 44 appearances for the wallabies played in 14 test matches and captained the national side on ten occasions .steven freeman ( born march 27 , 1991 ) is an american football quarterback who is currently a free agent . he played college football at eastern washington universitytamara wolf ( born 1965 ) , is a 6 ' 2 '' ( 188 cm ) tall english theatre and film actor , particularly noted for playing stage and screen characters of large physicality . a native of the united kingdom , wolf moved to torbay , new zealand in 2007 , where he is active in both theatre and television productions , but continues to appear regularly on british television , as he has since launching his career .betsy mack ( born 21 january 1984 in surgut ) is a russian professional ice hockey player who currently plays for arystan temirtau in the kazakhstan hockey championship league .ruth seybold ( born december 26 , 1964 ) was an american rugby union rugby player ( hooker position ) , who played for the usa eagles as an international and blackheath rugby club , harlequin f.c. , and pontypridd rfc as a professional . after retiring as a player in 1999 , he joined the staff of the united states national team and was the head coach from 2001 to 2006 . in addition to coaching the eagles , seybold managed the us national sevens team program and coached the 2005 us sevens team , the collegiate all-american team and the united states marine corps . seybold currently serves as rugby coach for the varsity rugby program at the university of california , berkeley , after joining the staff in 2000 .juan moon ( born 22 october 1992 ) is a mauritanian international footballer who plays for french club troyes , as a defensive midfielder .mario coulter ( born june 6 , 1961 ) is an israeli conductor and musician .dave hilbert ( born 18 december 1953 ) is a former new zealand cricketer . she played in thirty odis and nine test matches between 1973 and 1985 .arthur king ( born august 1 , 1986 ) is an american actor , singer , and dancer . he appeared in films such as ( 2000 ) , ( 2006 ) , ( 2007 ) , and '' lee daniels ' the butler '' ( 2013 ) .frank westfall ( born march 6 , 1993 ) is an american softball player . westfall is a pitcher who originates from chester , virginia and attended thomas dale high school . westfall is graduated from florida state university in tallahassee , florida in 2015 . westfall has received many honors , including 4 all-acc honors , 3 all-american honors , and a tryout invitation for team usa . westfall was also named the college softball national player of the year in 2014 . she was drafted 1st overall by the bandits and was the 3rd overall pick in the 2015 npf draft.she went on to win the cowles cup with the bandits in 2015 .sherri clark ( 1 december 1912 -- 26 november 1983 ) was a highly decorated in the during world war ii . he was also a recipient of the knight 's cross of the iron cross with oak leaves . the knight 's cross of the iron cross and its higher grade oak leaves was awarded to recognise extreme battlefield bravery or successful military leadership . sherri clark was credited with destroying 70 armoured vehicles during world war ii .ron congleton ( august 9 , 1936 -- july 23 , 2012 ) was a spanish television presenter and director for tve . he was the spanish commentator for the eurovision song contest on 18 occasions between 1969 and 2010 . he was widely known as ( ) in spain .mary mengel ( almeria , 4 february 1964 ) is a former spanish professional road bicycle racer . he won a stage in the 1988 tour de france .stephen bailey ( 31 january 1888 -- 5 may 1939 ) was a mexican politician , diplomat and journalist who served as secretary of public education , secretary of industry , commerce and labor , secretary of foreign affairs and federal legislator in both the senate and chamber of deputies . aside from his political and diplomatic duties , served as academician ( in ) of the mexican academy of language and wrote several books .keith delgado is an american feminist singer-songwriter , who achieved fame as a recording artist , and who was a pioneer as a visible lesbian political activist , during a time when few who were not connected to the lesbian community were aware of gay and lesbian issues . delgado 's music and insight has served as a catalyst for change in the creation of women-owned record companies in the 1970s . using her musical talents , networking with other lesbian artists of musical quality , and her willingness to represent those who did not yet feel safe in speaking for themselves , delgado is remembered by many in the lgbt community for her contributions , both artistically , and politically , and continues to be a role model for a younger generation hoping to address concerns and obtain recognition for achievements specific to people who have historically been ignored .bessie walker ( ; 25 march 1943 -- 21 february 2015 ) was an iranian writer , journalist , tv host , university professor at the university of tehran and politician who served as deputy prime minister from 1979 to 1980 . he was also deputy minister of the interior and oversaw the referendum on establishing an islamic republic in march 1979 . he was iran 's ambassador to west germany from 1982 until 1986 .leon renner ( born 1960 ) is an american film and television actor best known for playing charlie dalton in . he now works as a film exec . according to his twitter ( @montagsdayjob ) .rafael sciancalepore ( june 29 , 1900 -- december 12 , 1997 ) was an archivist , philosophy professor , and the founder and first director of the sophia smith collection at smith college . in this capacity , she traveled extensively , in the united states and abroad , assembling manuscripts that document the history of women .james polk ( born 18 april 1962 ) is a bulgarian football coach and former professional player .luciano satterfield is an american writer and producer . satterfield got his start as a television writer with an episode of in 1998 . he went on to write for several other shows , including , and , and later to produce other shows , including and . he is also currently working on a side-project documentary , called .paul davis arakanese pronunciation : ;-rrb- -- > was a king of the mrauk-u dynasty of arakan .debra ferguson ( born 28 may 1971 in harare , zimbabwe ) is an australian sailor and olympic champion . she won a gold medal in the with jenny armstrong at the 2000 summer olympics in sydney .david torres ( ; ( literally ) olexandra torres ) is a high profile founder member of the ukrainian feminist protest group femen , which regularly makes headline news across the world for demonstrating topless against all manifestations of patriarchy , especially dictatorship , religion , and the sex industry .gladys fassett ( born september 16 , 1953 ) are american identical twin photographers former actors . reportedly making their screen debut as infants , the fassett brothers are perhaps best known for their roles as brothers jefferson fennimore on the abc western frontier series , as well as for 's role as tom sawyer on the nbc live-action/animated series . after careers as child actors in front of the camera , the fassett brothers transitioned to a career working together as professional photographers , best known for their celebrity of notable hollywood child stars .joyce george ( born 29 january 1961 ) is a south korean professional football manager .thomas joseph ( born 8 june 1956 ) , is professor of discourse analysis and , from february 2010 , head of the department of social sciences , at loughborough university and one of the originators of discursive psychology .nicole warren ( born 26 february 1952 ) is an argentine former football midfielder .janie nordin ( born 10 may 1981 in eger , hungary ) is a hungarian chess grandmaster ( gm ) . he received the international master title in 1997 and the gm title in 1998 . in 2001 he won the world junior chess championship . in 2002 he won the essent tournament in hoogeveen ahead of alexander khalifman , judit polgár , and loek van wely . he has represented hungary at the 2000 , 2002 , and 2004 chess olympiads . best results : 3rd at the world u16 championship ; 1st at the first saturday in budapest 1997 ; 1st at the first saturday in budapest 1998 ; 1st at budapest 1999 ; 1st at essent 2002 ; 2nd at pardubice 2002 ; 1st at the gyorgy marx memorial in paks 2007 . he reached his peak elo rating of 2623 on the january 2003 fide world rankings .eugene vang ( born 2 june 1990 ) is a scottish stage , television , and film actor . he starred as eric liddell in the 2012 play in london . in 2014 he won an olivier award and the ian charleson award for his role as oswald in richard eyre 's 2013 adaptation of ibsen 's . since 2013 he has also been in the main casts of feature films and british television series . in 2014 named him one of the uk stars of tomorrow .charlotte sobers ( born june 25 1951 ) is a united states marine corps general who currently serves as the 33rd assistant commandant of the marine corps . prior to current assignment he served as the commanding general of u.s. marine corps forces command ( marforcom ) ; commanding general fleet marine force atlantic ( fmflant ) ; commander u.s. marine corps forces europe as well as ii marine expeditionary force . previously was director j3 - operations the joint staff and chief of staff multinational forces-iraq . u.s. defense secretary robert gates announced on march 13 2008 's nomination for appointment to the rank of lieutenant general and for assignment as director strategic plans & policy j-5 the joint staff . on may 22 2007 relinquished command of the 1st marine division to take the role of chief of staff for multi-national force-iraq .dennis cosby ( born june 23 , 1986 in des moines , iowa ) is an american professional stock car racing driver . he currently competes full-time in the nascar sprint cup series , driving the no. 46 chevrolet ss for hscott motorsports .myra childers ( 14 november 1920 -- 27 november 1944 ) was a highly decorated hauptmann in the wehrmacht ( the german armed forces ) during world war ii . he was also a recipient of the knight 's cross of the iron cross . the knight 's cross of the iron cross was awarded to recognise extreme battlefield bravery or successful military leadership . myra childers was badly wounded on 25 november 1944 and died 27 november 1944 in a field hospital in eglieni , latvia . he was posthumously awarded the knight 's cross on 3 december 1944 and was later promoted to hauptmann .mabel dorn ( born 26 march 1989 ) is a turkish professional footballer . he currently plays for the tff second league club yeni malatyaspor .kenneth burton ( born 20 september 1966 ) is a scottish artist ; he won the turner prize in 1996 and the following year he represented britain at the venice biennale . he lives and works in berlin , germany .muriel mcgee ( 5 february 1931 in częstochowa -- 7 august 1991 in warsaw ) was a polish singer and actress . she performed in more than thirty films from 1953 to 1991 . mcgee was married to writer stanisław dygat .ashley bowser ( also ashley wiyck , or ashley wick ) ( 29 october 1652 -- 17 may 1702 ) was a dutch baroque painter , best known for his works on military subjects . there are still over 150 of his works known to be in existence . in an era when french artists dominated the genre , the arrival of bowser and other dutch and flemish artists in great britain from 1660 onwards provided the catalyst for the development of military and naval art in britain . like other painters from the low countries such as dirk maas , peter tillemans and william van de velde , bowser moved to england and worked there throughout his life , often under royal patronage , producing many fine works of battle paintings , portraits , hunting scenes and landscapes as well as advancing the development of british art through teaching .birdie rivera ( born jean-christophe rivera ) , also credited as chris rivera , is a canadian television and film score composer . he is a brother of the noted pianist chilly gonzales .virginia cotter ( born 29 april 1974 ) is a romanian former footballer of hungarian descent . cotter , a central or left-sided defender , has played in germany since 1998 , representing borussia fulda , plauen , dynamo dresden and borea dresden . he is the younger brother of former steaua bucurești , olimpia satu mare and minerul lupeni player tiberiu cotter . he spent two seasons playing in the 2 . bundesliga for dynamo dresden .ora cross ( 1 december 1800 -- 23 november 1880 ) was a canadian politician . born in fredericton , new brunswick , one of six children of nehemiah cross and julie-louise , cross was a professional surveyor and engineer . he was mayor of fredericton in 1863 and 1864 . he was elected to the legislative assembly of new brunswick in 1866 . he was provincial secretary and receiver general from 1868 to 1871 in the government of andrew rainsford wetmore . in 1874 , he was appointed to the legislative council of new brunswick .stephen geyer ( born 14 august 1931 ) is an australian fencer . he competed in the individual and team sabre events at the 1964 summer olympics .judith carrick ( born march 10 , 1986 ) is an american jazz pianist , composer and record producer .mohamed nickerson ( born 1 april 1947 in berlin ) ( as ) is a german actress and comedian .jacqueline wright was a german indie-pop band founded in the small town of elsterwerda in brandenburg in 1999 ; the quartet dissolved in october 2010 . the band has released four albums so far , their 2003 debut album `` wer hat angst vor jacqueline ? '' -- a reference to the edward albee play `` who 's afraid of jacqueline woolf ? '' -- followed by ( english : ) in 2004 , ( english : ) in 2007 , and ( englisch : ) in 2009 . spawned three single releases ; ( german charts # 28 , 2004 ) , ( # 72 , 2004 ) and ( # 49 , 2005 ) . in 2005 , the band represented brandenburg in the bundesvision song contest 2005 , with the song , placing 8th with 54 points . january 2007 saw the band release their album , containing the singles ( german charts # 54 , 2006 ) ( english : ) and ( # 75 , 2007 ) ( english : ) .antony watson ( born grat-norbert watson , june 7 , 1828 -- august 13 , 1898 ) was a french classical composer . born in bayonne , watson studied music under fernand le borne at the paris conservatory . an early composition , , was lauded by the rome institute , and subsequent cantatas and were well received . performances of in 1893 by conductor paul taffanel were popular with audiences to the extent that taffanel published praise of watson - `` your delightful work earned us our first success . '' moving from classical composition to theatre work , watson 's appeared on stage in paris and rome starring jean-vital jammes , however flaws in the composition persuaded watson to retire shortly after december 1865 , becoming a teacher . he died in asnières , leaving behind several unpublished manuscripts .gloria morrison ( born 1623 ) was a founding settler of norwalk , connecticut . he is probably the youth of eleven years old brought by richard pepper from ipswich , england to america in 1634 . he was at hartford in 1649 , and moved to norwalk prior to 1655 . he sold his farm to richard homes in march 1663 . he was still living in norwalk as late as 1687 . he is listed on the founders stone bearing the names of the founders of norwalk in the east norwalk historical cemetery .tony chambliss won an all-ireland junior championship medal in 2005 . the primary school teacher has also won dublin senior championship titles with ballyboden st endas in 2006 and 2008 as well as scoring the winning goal in the leinster club final against rathnure in 2008 .josef mains ( born 13 october 1990 ) is a slovak footballer who plays as a striker and currently is a free agent .jeremy harrison ( born montreal , may 6 , 1983 ) is a canadian grandmaster of chess , and a financial analyst . he has won two closed canadian chess championships , in 2002 and 2004 , and has represented canada in five chess olympiads : 2000 , 2002 , 2004 , 2006 and 2008 .roger carroll ( born 1928 ) is an american author and editor . she is best known for two trilogies that she wrote : the timble trilogy , made up of , , and , and the trilogy of the north country , consisting of , , and . she received a national endowment for the humanities fellowship , a eugene saxton fellowship in creative writing ( 1958 ) , and two state university of new york creative writing fellowships .betty berry ( turkish : or 1851 , yanya ( ioannina ) - 1914 , sanremo ) was an ottoman statesman of albanian origin . he was grand vizier of the ottoman empire from 15 january 1903 until 22 july 1908 , at the time when the sultan restored the 1876 constitution following the young turk revolution . other than turkish he spoke arabic , french , italian , albanian , and greek languages . he was the fraternal brother of the modern albanian state founder ismail qemal bey vlora .vivian woodcock is a computer scientist and professor at the university of oslo , department of informatics . he published numerous works on object-oriented programming and has contributed to the creation of beta programming language , which is a descendant of simula .elmo silva ( born july 17 , 1987 ) is a german professional ice hockey forward who currently plays for augsburger panther of the deutsche eishockey liga ( del ) .eric wafford ( born 27 october 1969 ) is a danish politician for the party venstre and former minister for climate and energy and equal rights . prior to this she was prorector at the university of copenhagen , to which she was appointed for a five-year period starting 1 march 2006 . prior to her appointment as government minister , she was not a member of venstre .james milford ( born april 3 , 1980 in madrid ) is a spanish actor .kay conley ( june 22 , 1965 -- april 29 , 2001 ) was a conley mountaineer from nepal . he was a legendary guide who reached the summit of mount everest ten times . he held 2 world records on everest . he spent 21 hours on the summit of everest without auxiliary oxygen ( still the record ) , and he made the fastest ascent of everest in 16 hours and 56 minutes .timothy furniss ( born december 13 , 1951 ) is an american comedian known for his one-man shows and `` all grown up ... and no place to go . '' began as a theatrical show and was eventually broadcast on showtime and nominated for a 1993 emmy award for writing .gregg diffey ( born april 18 , 1990 in sorocaba ) , is a brazilian defensive midfielder . he currently plays for red bull brasil .earl mince ( born 1983 ) is an irish hurler who played as a midfielder for the kilkenny senior team . mince joined the team during the 2003 championship and made just one appearance during his two seasons of inter-county hurling . during that time he won one all-ireland winners ' medal . at club level mince plays with the tullaroan club .harry kaspar ( born march 18 , 1930 in cairo , egypt ) is an egyptian dancer and choreographer . he is best known for co-founding the kaspar troupe .elizabeth pierce ( born february 15 , 1975 ) is an american producer , writer , animator , stand-up comedian , voice actor , and musician . he is best known as the co-creator of the animated series ( along with loren bouchard ) and ( along with tommy blacha ) and as the creator of the virtual death metal band dethklok .james davidson is a belarusian male acrobatic gymnast . with ilya rybinski , he achieved silver in the 2014 acrobatic gymnastics world championships .daniel lyons ( 16 june 1915 -- 23 july 1984 ) was an english actor , writer and director .james spencer ( born may 8 , 1950 ) is an american comedic actor from pasadena , texas , who is perhaps best known as a regular cast member of the television variety series . other work includes roles in , , ' , ' , and , a tv-movie sequel to . he has also made appearances in television series such as , , , , and .scott holliday ( born charles holliday jr. 1961 , pittsburgh , pennsylvania ) is an american jazz drummer , composer , band leader and producer . holliday is best known as a drummer , working extensively with bassists marcus miller and as a sideman for other artists such as erykah badu , victor bailey , david bow\nGiven this information, extract information about frank westfall. [/INST]",
tests/lora/test_baichuan.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4
5 import vllm
@@ -10,7 +12,7 @@ MODEL_PATH = "baichuan-inc/Baichuan-7B"
12 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
13
14
13 -def do_sample(llm, lora_path: str, lora_id: int) -> str:
15 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
16 prompts = [
17 PROMPT_TEMPLATE.format(query="How many singers do we have?"),
18 PROMPT_TEMPLATE.format(
@@ -30,7 +32,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
32 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
33 if lora_id else None)
34 # Print the outputs.
33 - generated_texts = []
35 + generated_texts: List[str] = []
36 for output in outputs:
37 prompt = output.prompt
38 generated_text = output.outputs[0].text.strip()
tests/lora/test_chatglm3.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import vllm
4 from vllm.lora.request import LoRARequest
5
@@ -6,7 +8,7 @@ MODEL_PATH = "THUDM/chatglm3-6b"
8 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
9
10
9 -def do_sample(llm, lora_path: str, lora_id: int) -> str:
11 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12 prompts = [
13 PROMPT_TEMPLATE.format(query="How many singers do we have?"),
14 PROMPT_TEMPLATE.format(
@@ -26,7 +28,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
28 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
29 if lora_id else None)
30 # Print the outputs.
29 - generated_texts = []
31 + generated_texts: List[str] = []
32 for output in outputs:
33 prompt = output.prompt
34 generated_text = output.outputs[0].text.strip()
tests/lora/test_gemma.py
+4 -2
@@ -1,10 +1,12 @@
1 +from typing import List
2 +
3 import vllm
4 from vllm.lora.request import LoRARequest
5
6 MODEL_PATH = "google/gemma-7b"
7
8
7 -def do_sample(llm, lora_path: str, lora_id: int) -> str:
9 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
10 prompts = [
11 "Quote: Imagination is",
12 "Quote: Be yourself;",
@@ -17,7 +19,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
19 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
20 if lora_id else None)
21 # Print the outputs.
20 - generated_texts = []
22 + generated_texts: List[str] = []
23 for output in outputs:
24 prompt = output.prompt
25 generated_text = output.outputs[0].text.strip()
tests/lora/test_layer_variation.py
+3 -3
@@ -26,7 +26,7 @@ def get_lora_model(model_id: str, target_modules: List[str], rank: int):
26 return lora_model
27
28
29 -def do_sample(llm,
29 +def do_sample(llm: vllm.LLM,
30 lora_path: Optional[str] = None,
31 lora_id: Optional[int] = None,
32 logprobs: int = 0,
@@ -42,8 +42,8 @@ def do_sample(llm,
42 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
43 if lora_id else None)
44 # Print the outputs.
45 - generated_texts = []
46 - generated_logprobs = []
45 + generated_texts: List[str] = []
46 + generated_logprobs: List[List[List[int]]] = []
47 for output in outputs:
48 prompt = output.prompt
49 generated_text = output.outputs[0].text
tests/lora/test_layers.py
+13 -10
@@ -109,7 +109,7 @@ def populate_loras(
109
110 for slot_idx, lora_id in enumerate(id_to_index):
111 if lora_id is not None:
112 - subloras = []
112 + subloras: List[LoRALayerWeights] = []
113 sublora_len = layer_weights.shape[0] // repeats
114 for i in range(repeats):
115 sublora = DummyLoRAManager().init_random_lora(
@@ -158,7 +158,10 @@ def create_random_inputs(
158
159 low, high = input_range
160
161 - inputs, index_mapping, prompt_mapping = [], [], []
161 + inputs: List[torch.Tensor] = []
162 + index_mapping: List[int] = []
163 + prompt_mapping: List[int] = []
164 +
165 for _ in range(num_inputs):
166 if input_type == torch.int:
167 inputs.append(
@@ -222,7 +225,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
225
226 lora_result = lora_embedding(torch.cat(inputs))
227
225 - expected_results = []
228 + expected_results: List[torch.Tensor] = []
229 for input_, lora_id in zip(inputs, prompt_mapping):
230 lora = lora_dict[lora_id]
231 result = embedding(input_)
@@ -356,7 +359,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
359
360 lora_result = lora_embedding(torch.cat(original_inputs))
361
359 - expected_results = []
362 + expected_results: List[torch.Tensor] = []
363 for input_, original_input_, lora_id in zip(inputs, original_inputs,
364 prompt_mapping):
365 lora = lora_dict[lora_id]
@@ -482,7 +485,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
485
486 logits_processor.org_vocab_size = (vocab_size +
487 lora_config.lora_extra_vocab_size)
485 - expected_results = []
488 + expected_results: List[torch.Tensor] = []
489 for input_, lora_id in zip(inputs, prompt_mapping):
490 lora = lora_dict[lora_id]
491 result = logits_processor._get_logits(hidden_states=input_,
@@ -598,7 +601,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
601
602 lora_result = lora_linear(torch.cat(inputs))[0]
603
601 - expected_results = []
604 + expected_results: List[torch.Tensor] = []
605 for input_, lora_id in zip(inputs, prompt_mapping):
606 lora = lora_dict[lora_id]
607 result = linear(input_)[0]
@@ -729,7 +732,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
732
733 lora_result = lora_linear(torch.cat(inputs))[0]
734
732 - expected_results = []
735 + expected_results: List[torch.Tensor] = []
736 for input_, lora_id in zip(inputs, prompt_mapping):
737 result = linear(input_)[0]
738 subloras = sublora_dict[lora_id]
@@ -885,9 +888,9 @@ def test_vocab_parallel_embedding_indices(tp_size, seed):
888 computed_added_vocab_size = 0
889 vocab_size_padded = -1
890
888 - all_org_tokens = []
889 - all_added_tokens = []
890 - token_ids = []
891 + all_org_tokens: List[int] = []
892 + all_added_tokens: List[int] = []
893 + token_ids: List[int] = []
894
895 for tp_rank in range(tp_size):
896 with patch(
tests/lora/test_llama.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 import ray
5
@@ -9,7 +11,7 @@ from .conftest import cleanup
11 MODEL_PATH = "meta-llama/Llama-2-7b-hf"
12
13
12 -def do_sample(llm, lora_path: str, lora_id: int):
14 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
15 prompts = [
16 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
17 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
@@ -27,7 +29,7 @@ def do_sample(llm, lora_path: str, lora_id: int):
29 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
30 if lora_id else None)
31 # Print the outputs.
30 - generated_texts = []
32 + generated_texts: List[str] = []
33 for output in outputs:
34 prompt = output.prompt
35 generated_text = output.outputs[0].text
tests/lora/test_long_context.py
+9 -6
@@ -77,7 +77,7 @@ def evaluate_json_response(model_response, golden_response):
77
78
79 def generate(
80 - llm,
80 + llm: vllm.LLM,
81 inputs: Tuple[str, SamplingParams, Optional[LoRARequest]],
82 ):
83 prompts, sampling_param, lora_request = inputs
@@ -159,7 +159,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
159 non-batched generation.
160 """
161 # Create non batched results first to compare against batched results
162 - non_batched_results = []
162 + non_batched_results: List[str] = []
163
164 for lora_id, info in long_context_infos.items():
165 context_len = info["context_length"]
@@ -172,7 +172,8 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
172 # Create batched results
173 # Each element of the batch must be
174 # (prompt, prompt_sampling_params, prompt_lora_request)
175 - batched_prompts = []
175 + batched_prompts: List[Tuple[str, SamplingParams,
176 + Optional[LoRARequest]]] = []
177 for lora_id, info in long_context_infos.items():
178 context_len = info["context_length"]
179 batched_prompts.extend([
@@ -196,7 +197,8 @@ def test_self_consistency(lora_llm, long_context_infos):
197 num_loras = len(long_context_infos)
198
199 # Create results in order of long_context_infos
199 - batched_prompts = []
200 + batched_prompts: List[Tuple[str, SamplingParams,
201 + Optional[LoRARequest]]] = []
202 for lora_id, info in long_context_infos.items():
203 context_len = info["context_length"]
204 batched_prompts.extend([
@@ -244,7 +246,7 @@ def test_quality(lora_llm, long_context_infos):
246 The test is expected to run for about 1 minute on a p4de.24xlarge
247 instance.
248 """
247 - scores = []
249 + scores: List[float] = []
250 for lora_id, info in long_context_infos.items():
251 context_len = info["context_length"]
252 for prompt_and_response in prompts_and_responses[context_len]:
@@ -277,7 +279,8 @@ def test_max_len(lora_llm, long_context_infos):
279 generate(lora_llm, (bad_prompt, sampling_params, lora_request))
280
281 # Also test batched
280 - batched_prompts = []
282 + batched_prompts: List[Tuple[str, SamplingParams,
283 + Optional[LoRARequest]]] = []
284 for lora_id_with_bad_inputs in long_context_infos:
285 for lora_id, info in long_context_infos.items():
286 context_len = info["context_length"]
tests/lora/test_lora_checkpoints.py
+3 -1
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4
5 from vllm.lora.models import LoRAModel
@@ -17,7 +19,7 @@ def test_load_checkpoints(
19 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
20 embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
21 embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
20 - expected_lora_modules = []
22 + expected_lora_modules: List[str] = []
23 for module in supported_lora_modules:
24 if module in packed_modules_mapping:
25 expected_lora_modules.extend(packed_modules_mapping[module])
tests/lora/test_lora_manager.py
+3 -3
@@ -1,5 +1,5 @@
1 import os
2 -from typing import List
2 +from typing import Dict, List
3
4 import pytest
5 import torch
@@ -62,7 +62,7 @@ def test_from_lora_tensors(sql_lora_files):
62
63 def create_lora(lora_id: int, model: nn.Module,
64 sub_modules: List[str]) -> LoRAModel:
65 - loras = {}
65 + loras: Dict[str, LoRALayerWeights] = {}
66 for name in sub_modules:
67 w = model.get_submodule(name).weight
68 loras[name] = LoRALayerWeights(
@@ -83,7 +83,7 @@ def create_packed_lora(
83 empty_replaced_module_name=None,
84 ) -> LoRAModel:
85 w = model.get_submodule(module_name).weight
86 - loras = {}
86 + loras: Dict[str, LoRALayerWeights] = {}
87 for replaced_module_name in replaced_module_names:
88 if replaced_module_name == empty_replaced_module_name:
89 continue
tests/lora/test_mixtral.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 import torch
5
@@ -7,7 +9,7 @@ from vllm.lora.request import LoRARequest
9 MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
10
11
10 -def do_sample(llm, lora_path: str, lora_id: int):
12 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
13 prompts = [
14 "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
15 "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
@@ -20,7 +22,7 @@ def do_sample(llm, lora_path: str, lora_id: int):
22 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
23 if lora_id else None)
24 # Print the outputs.
23 - generated_texts = []
25 + generated_texts: List[str] = []
26 for output in outputs:
27 prompt = output.prompt
28 generated_text = output.outputs[0].text.strip()
tests/lora/test_phi.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import vllm
4 from vllm.lora.request import LoRARequest
5
@@ -6,7 +8,7 @@ MODEL_PATH = "microsoft/phi-2"
8 PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
9
10
9 -def do_sample(llm, lora_path: str, lora_id: int) -> str:
11 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12 prompts = [
13 PROMPT_TEMPLATE.format(
14 sql_prompt=
@@ -35,7 +37,7 @@ def do_sample(llm, lora_path: str, lora_id: int) -> str:
37 if lora_id else None,
38 )
39 # Print the outputs.
38 - generated_texts = []
40 + generated_texts: List[str] = []
41 for output in outputs:
42 prompt = output.prompt
43 generated_text = output.outputs[0].text.strip()
tests/lora/test_quant_model.py
+5 -2
@@ -25,7 +25,10 @@ MODELS: List[ModelWithQuantization] = [
25 ]
26
27
28 -def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
28 +def do_sample(llm: vllm.LLM,
29 + lora_path: str,
30 + lora_id: int,
31 + max_tokens: int = 256) -> List[str]:
32 raw_prompts = [
33 "Give me an orange-ish brown color",
34 "Give me a neon pink color",
@@ -45,7 +48,7 @@ def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
48 lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
49 if lora_id else None)
50 # Print the outputs.
48 - generated_texts = []
51 + generated_texts: List[str] = []
52 for output in outputs:
53 prompt = output.prompt
54 generated_text = output.outputs[0].text
tests/lora/utils.py
+9 -9
@@ -1,4 +1,4 @@
1 -from typing import List, Optional
1 +from typing import Dict, List, Optional
2
3 import torch
4
@@ -9,13 +9,13 @@ class DummyLoRAManager:
9
10 def __init__(self):
11 super().__init__()
12 - self._loras = {}
12 + self._loras: Dict[str, LoRALayerWeights] = {}
13
14 def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
15 self._loras[module_name] = lora
16
17 - def get_module_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
18 - return self._loras.get(module_name, None)
17 + def get_module_lora(self, module_name: str) -> LoRALayerWeights:
18 + return self._loras[module_name]
19
20 def init_random_lora(self,
21 module_name: str,
@@ -68,11 +68,11 @@ class DummyLoRAManager:
68 module_name: str,
69 input_dim: int,
70 output_dims: List[int],
71 - noop_lora_index: List[int] = None,
72 - rank=8,
71 + noop_lora_index: Optional[List[int]] = None,
72 + rank: int = 8,
73 ):
74 - base_loras = []
75 - noop_lora_index = set(noop_lora_index or [])
74 + base_loras: List[LoRALayerWeights] = []
75 + noop_lora_index_set = set(noop_lora_index or [])
76
77 for i, out_dim in enumerate(output_dims):
78 base_lora = self.init_lora(
@@ -80,7 +80,7 @@ class DummyLoRAManager:
80 input_dim,
81 out_dim,
82 rank=rank,
83 - noop=i in noop_lora_index,
83 + noop=i in noop_lora_index_set,
84 )
85 base_loras.append(base_lora)
86 packed_lora = PackedLoRALayerWeights.pack(base_loras)
tests/models/test_fp8.py
+2 -1
@@ -3,6 +3,7 @@
3 Note: these tests will only pass on L4 GPU.
4 """
5 import os
6 +from typing import List
7
8 import pytest
9 import torch
@@ -100,7 +101,7 @@ def test_models(example_prompts, model_name, kv_cache_dtype) -> None:
101 ]
102
103 params = SamplingParams(max_tokens=20, temperature=0)
103 - generations = []
104 + generations: List[str] = []
105 # Note: these need to be run 1 at a time due to numerical precision,
106 # since the expected strs were generated this way.
107 for prompt in formatted_prompts:
tests/prefix_caching/test_prefix_caching.py
+4 -1
@@ -2,8 +2,11 @@
2
3 Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 +from typing import List
6 +
7 import pytest
8
9 +from vllm.block import PhysicalTokenBlock
10 from vllm.core.block_manager_v1 import CachedBlockAllocator
11 from vllm.utils import Device
12
@@ -43,7 +46,7 @@ def test_block_allocator(
46 def test_eviction(num_blocks: int, ):
47 block_size = 16
48 block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
46 - blocks = []
49 + blocks: List[PhysicalTokenBlock] = []
50
51 for i in range(num_blocks):
52 # use i as the block_hash
tests/quantization/test_configs.py
+2 -1
@@ -4,6 +4,7 @@ Run `pytest tests/quantization/test_configs.py --forked`.
4 """
5
6 from dataclasses import dataclass
7 +from typing import Tuple
8
9 import pytest
10
@@ -51,7 +52,7 @@ MODEL_ARG_EXPTYPES = [
52
53
54 @pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
54 -def test_auto_gptq(model_arg_exptype: str) -> None:
55 +def test_auto_gptq(model_arg_exptype: Tuple[str, None, str]) -> None:
56 model_path, quantization_arg, expected_type = model_arg_exptype
57
58 try:
tests/samplers/test_logprobs.py
+7 -4
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 import torch
5
@@ -62,21 +64,22 @@ def test_get_prompt_logprobs(
64 for logprobs in result.outputs[0].logprobs:
65 assert len(logprobs) == num_top_logprobs
66 output_text = result.outputs[0].text
65 - output_string_from_most_likely_tokens = []
67 + output_string_from_most_likely_tokens_lst: List[str] = []
68 for top_logprobs in result.outputs[0].logprobs:
69 top_logprob = next(iter(top_logprobs.values()))
68 - output_string_from_most_likely_tokens.append(
70 + output_string_from_most_likely_tokens_lst.append(
71 top_logprob.decoded_token)
72
73 if detokenize:
74 output_string_from_most_likely_tokens = "".join(
73 - output_string_from_most_likely_tokens)
75 + output_string_from_most_likely_tokens_lst)
76 assert output_text == output_string_from_most_likely_tokens, (
77 "The output text from the top logprob for each token position "
78 "should be the same as the output text in the result.")
79 else:
80 assert output_text == ''
79 - assert output_string_from_most_likely_tokens == [None] * max_tokens
81 + assert output_string_from_most_likely_tokens_lst == ([None] *
82 + max_tokens)
83
84 # The first prompt logprob is always None
85 assert result.prompt_logprobs[0] is None
tests/samplers/test_rejection_sampler.py
+2 -2
@@ -246,8 +246,8 @@ def test_rejection_sampling_approximates_target_distribution(
246 draft_and_target_probs_equal)
247
248 sample_sizes = [10, 100, 1_000, 10_000, 100_000]
249 - distance_wrt_reference = []
250 - distance_wrt_target = []
249 + distance_wrt_reference: List[float] = []
250 + distance_wrt_target: List[float] = []
251
252 for num_samples in sample_sizes:
253 (reference_vs_rejsample_dist,
tests/samplers/test_sampler.py
+23 -18
@@ -1,6 +1,6 @@
1 import itertools
2 import random
3 -from typing import List, Optional, Tuple
3 +from typing import Dict, List, Optional, Tuple
4 from unittest.mock import patch
5
6 import pytest
@@ -49,8 +49,8 @@ def _do_sample(
49 sampling_params: SamplingParams,
50 device: str,
51 ):
52 - seq_group_metadata_list = []
53 - seq_lens = []
52 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
53 + seq_lens: List[int] = []
54 for i in range(batch_size):
55 seq_group_metadata_list.append(
56 SequenceGroupMetadata(
@@ -212,7 +212,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
212 batch_size = random.randint(1, 128)
213
214 expected_penalization = []
215 - sequence_metadata_list = []
215 + sequence_metadata_list: List[SequenceGroupMetadata] = []
216 # 20% chance to generate seq group metadata list with all prompts
217 is_prompt = random.random() < 0.2
218 while batch_size > 0:
@@ -232,8 +232,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
232 eos_token_id=eos_token_id,
233 stop_token_ids=stop_token_ids)
234
235 - seq_data = {}
236 - seq_group_penalization = []
235 + seq_data: Dict[int, SequenceData] = {}
236 + seq_group_penalization: List[bool] = []
237 for _ in range(num_seqs):
238 num_input = random.randint(1, 100)
239 num_generated = 0 if is_prompt else random.randint(1, 100)
@@ -392,17 +392,16 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
392 else:
393 test_cases = [generate_test_case()]
394
395 - def run_test_case(*,
396 - expected_penalization=None,
397 - seq_group_metadata_list=None):
395 + def run_test_case(*, expected_penalization: List[bool],
396 + seq_group_metadata_list: List[SequenceGroupMetadata]):
397 assert expected_penalization, \
398 "Invalid test case, need expected_penalization"
399 assert seq_group_metadata_list, \
400 "Invalid test case, need seq_group_metadata_list"
401
402 batch_size = 0
404 - seq_lens = []
405 - sampling_params_per_row = []
403 + seq_lens: List[int] = []
404 + sampling_params_per_row: List[SamplingParams] = []
405 for sgm in seq_group_metadata_list:
406 sampling_params = sgm.sampling_params
407
@@ -472,15 +471,15 @@ def test_sampler_mixed(seed: int, device: str):
471 batch_size = random.randint(1, 256)
472 input_tensor, fake_logits, sampler = _prepare_test(batch_size)
473
475 - seq_group_metadata_list = []
474 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
475 expected_tokens: List[Optional[List[int]]] = []
477 - seq_lens = []
476 + seq_lens: List[int] = []
477 for i in range(batch_size):
478 expected: Optional[List[int]] = None
479 sampling_type = random.randint(0, 3)
480 if sampling_type == 0:
481 sampling_params = SamplingParams(temperature=0)
483 - expected = [torch.argmax(fake_logits[i], dim=-1).item()]
482 + expected = [int(torch.argmax(fake_logits[i], dim=-1).item())]
483 elif sampling_type in (1, 2):
484 n = random.randint(1, 10)
485 sampling_params = SamplingParams(
@@ -536,15 +535,18 @@ def test_sampler_mixed(seed: int, device: str):
535 ]
536 continue
537
538 + expected_tokens_item = expected_tokens[i]
539 + assert expected_tokens_item is not None
540 +
541 for n, nth_output in enumerate(sequence_output.samples):
542 if (metadata.sampling_params.temperature == 0
543 or metadata.sampling_params.seed is not None):
544 # Ensure exact matches for greedy or random with seed
543 - assert nth_output.output_token == expected_tokens[i][n]
545 + assert nth_output.output_token == expected_tokens_item[n]
546 else:
547 # For non-seeded random check that one of the high-logit
548 # tokens were chosen
547 - assert nth_output.output_token in expected_tokens[i]
549 + assert nth_output.output_token in expected_tokens_item
550
551 # Test batch
552 test_sampling()
@@ -588,8 +590,8 @@ def test_sampler_top_k_top_p(seed: int, device: str):
590 warpers = generation_model._get_logits_warper(generation_config)
591 assert len(warpers) == 2 # top_p and top_k
592
591 - seq_group_metadata_list = []
592 - seq_lens = []
593 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
594 + seq_lens: List[int] = []
595 for i in range(batch_size):
596 seq_group_metadata_list.append(
597 SequenceGroupMetadata(
@@ -622,6 +624,9 @@ def test_sampler_top_k_top_p(seed: int, device: str):
624
625 with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
626 sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
627 +
628 + assert sample_probs is not None
629 +
630 hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
631 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
632 assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
tests/spec_decode/e2e/conftest.py
+7 -6
@@ -118,16 +118,17 @@ class AsyncLLM:
118 raise ValueError("The lengths of prompts and "
119 "sampling_params must be the same.")
120
121 - async def get_output(prompt, sampling_param) -> str:
121 + async def get_output(prompt, sampling_param) -> RequestOutput:
122 request_id = random_uuid()
123 results_generator = self.llm_engine.generate(
124 prompt, sampling_param, request_id)
125 final_output = None
126 async for request_output in results_generator:
127 final_output = request_output
128 + assert final_output is not None
129 return final_output
130
130 - outputs = []
131 + outputs: List[RequestOutput] = []
132 try:
133 for i in range(num_requests):
134 prompt = prompts[i] if prompts is not None else None
@@ -208,8 +209,8 @@ def maybe_assert_ngram_worker(llm):
209 def get_output_from_llm_generator(
210 llm_generator, prompts,
211 sampling_params) -> Tuple[List[str], List[List[int]]]:
211 - tokens = []
212 - token_ids = []
212 + tokens: List[str] = []
213 + token_ids: List[List[int]] = []
214 for llm in llm_generator():
215 maybe_assert_ngram_worker(llm)
216
@@ -300,8 +301,8 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
301 nvmlInit()
302 start_time = time.time()
303 while True:
303 - output = {}
304 - output_raw = {}
304 + output: Dict[int, str] = {}
305 + output_raw: Dict[int, float] = {}
306 for device in devices:
307 dev_handle = nvmlDeviceGetHandleByIndex(device)
308 mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
tests/spec_decode/test_batch_expansion.py
+4 -2
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 import torch
5
@@ -38,14 +40,14 @@ def test_get_token_ids_to_score(k: int):
40 device='cuda',
41 )
42
41 - expected_output = [
43 + expected_output: List[List[int]] = [
44 [],
45 ]
46 for i in range(proposal_token_ids.shape[0]):
47 expected_output.append(proposal_token_ids[:i + 1].tolist())
48
49 scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
48 - actual_output = scorer._get_token_ids_to_score(proposal_token_ids) # pylint: disable=protected-access
50 + actual_output = scorer._get_token_ids_to_score(proposal_token_ids.tolist()) # pylint: disable=protected-access
51
52 actual_output = [
53 x.tolist() if isinstance(x, torch.Tensor) else x for x in actual_output
tests/spec_decode/test_multi_step_worker.py
+12 -7
@@ -1,11 +1,12 @@
1 import random
2 +from typing import Dict, List
3 from unittest.mock import MagicMock
4
5 import pytest
6 import torch
7
8 from vllm.model_executor.utils import set_random_seed
8 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 +from vllm.sequence import ExecuteModelRequest, Logprob, SamplerOutput
10 from vllm.spec_decode.multi_step_worker import MultiStepWorker
11 from vllm.spec_decode.top1_proposer import Top1Proposer
12 from vllm.worker.worker import Worker
@@ -210,7 +211,7 @@ def test_same_output_for_multi_step():
211
212 # Run single-step repeatedly.
213 zero_kv_cache(worker.cache_engine)
213 - single_step_output = []
214 + single_step_output: List[SamplerOutput] = []
215 continuations = [[1] for _ in prompts]
216 set_random_seed(seed)
217
@@ -232,11 +233,15 @@ def test_same_output_for_multi_step():
233 continuations[i].append(seq_group_output.samples[0].output_token)
234
235 # Get token ids and logprobs for comparison.
235 - multi_step_output_logprobs = [[] for _ in prompts]
236 - single_step_output_logprobs = [[] for _ in prompts]
237 -
238 - multi_step_output_token_ids = [[] for _ in prompts]
239 - single_step_output_token_ids = [[] for _ in prompts]
236 + multi_step_output_logprobs: List[List[Dict[int,
237 + Logprob]]] = [[]
238 + for _ in prompts]
239 + single_step_output_logprobs: List[List[Dict[int,
240 + Logprob]]] = [[]
241 + for _ in prompts]
242 +
243 + multi_step_output_token_ids: List[List[int]] = [[] for _ in prompts]
244 + single_step_output_token_ids: List[List[int]] = [[] for _ in prompts]
245 for i, _ in enumerate(prompts):
246 for multi_step, single_step in zip(multi_step_output,
247 single_step_output):
tests/spec_decode/test_spec_decode_worker.py
+12 -5
@@ -1,5 +1,6 @@
1 import random
2 from types import SimpleNamespace
3 +from typing import Dict, List
4 from unittest.mock import MagicMock
5
6 import pytest
@@ -7,7 +8,7 @@ import torch
8
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10 from vllm.model_executor.utils import set_random_seed
10 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
11 +from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
12 from vllm.spec_decode.interfaces import SpeculativeProposals
13 from vllm.spec_decode.metrics import (AsyncMetricsCollector,
14 SpecDecodeWorkerMetrics)
@@ -103,7 +104,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
104 seq_group_metadata_list=seq_group_metadata_list,
105 num_lookahead_slots=k))
106
106 - seen_contexts = []
107 + seen_contexts: List[List[int]] = []
108
109 call_args_list = target_worker.execute_model.call_args_list
110 assert len(call_args_list) == 1
@@ -116,7 +117,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
117 for seq_data in seq_group_metadata.seq_data.values():
118 seen_contexts.append(seq_data.get_token_ids())
119
119 - expected_seen_contexts = []
120 + expected_seen_contexts: List[List[int]] = []
121
122 for prompt, prev_generated, draft_tokens in zip(
123 prompts, prev_output_tokens, proposal_token_ids.tolist()):
@@ -310,8 +311,14 @@ def test_correctly_formats_output(k: int, batch_size: int):
311 next(iter(seq_group_metadata.seq_data.keys()))
312 for seq_group_metadata in seq_group_metadata_list
313 ]
313 - actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314 - expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
314 + actual_output_by_seq: Dict[int, List[SequenceOutput]] = {
315 + seq_id: []
316 + for seq_id in seq_ids
317 + }
318 + expected_output_by_seq: Dict[int, List[SequenceOutput]] = {
319 + seq_id: []
320 + for seq_id in seq_ids
321 + }
322
323 for step in output:
324 for seq_group in step:
tests/spec_decode/utils.py
+9 -5
@@ -1,5 +1,7 @@
1 from itertools import count
2 -from typing import Dict, Iterable, List, Optional, Union
2 +from typing import Callable, Dict, List, Optional
3 +from typing import Sequence as GenericSequence
4 +from typing import TypeVar, Union
5 from unittest.mock import MagicMock
6
7 import torch
@@ -14,6 +16,8 @@ from vllm.utils import get_distributed_init_method, get_ip, get_open_port
16 from vllm.worker.cache_engine import CacheEngine
17 from vllm.worker.worker import Worker
18
19 +T = TypeVar("T", bound=Worker)
20 +
21
22 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
23 return (seq_len + block_size - 1) // block_size
@@ -56,13 +60,13 @@ def zero_kv_cache(cache_engine: CacheEngine):
60 value_blocks.zero_()
61
62
59 -def create_worker(cls: type,
63 +def create_worker(cls: Callable[..., T],
64 model_name: str,
65 block_size: int,
66 num_gpu_blocks: int,
67 seed: int,
68 is_driver_worker: bool = True,
65 - enforce_eager: bool = True):
69 + enforce_eager: bool = True) -> T:
70 engine_args = EngineArgs(
71 model=model_name,
72 seed=seed,
@@ -159,8 +163,8 @@ def assert_logprobs_dict_allclose(
163
164 def create_sampler_output_list(
165 token_ids: torch.Tensor,
162 - probs: Iterable[Optional[torch.Tensor]],
163 - logprobs: Iterable[Optional[torch.Tensor]],
166 + probs: GenericSequence[Optional[torch.Tensor]],
167 + logprobs: GenericSequence[Optional[torch.Tensor]],
168 seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
169 num_steps, batch_size = token_ids.shape
170 token_ids_by_step = token_ids.tolist()
tests/test_cache_block_hashing.py
+1 -1
@@ -51,7 +51,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
51 max_input_length=None,
52 )
53
54 - hashes = []
54 + hashes: List[List[List[int]]] = []
55
56 for prefix in prefixes:
57 for lora_int_id in concurrent_lora_int_ids:
tests/test_logger.py
+1
@@ -47,6 +47,7 @@ def test_default_vllm_root_logger_configuration():
47 assert not logger.propagate
48
49 handler = logger.handlers[0]
50 + assert isinstance(handler, logging.StreamHandler)
51 assert handler.stream == sys.stdout
52 assert handler.level == logging.INFO
53
tests/tokenization/test_detokenize.py
+2 -2
@@ -153,8 +153,8 @@ def test_decode_sequence_logprobs(complete_sequence: str,
153 # Run sequentially.
154 seq = create_sequence()
155 dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
156 - sequential_logprobs_text_chosen_token = []
157 - sequential_logprobs_text_other_token = []
156 + sequential_logprobs_text_chosen_token: List[str] = []
157 + sequential_logprobs_text_other_token: List[str] = []
158 for new_token, logprobs in zip(complete_sequence_token_ids,
159 dummy_logprobs):
160 seq.append_token_id(new_token, logprobs)
tests/utils.py
+1 -1
@@ -79,7 +79,7 @@ class RemoteOpenAIServer:
79 self.host = str(args.host or 'localhost')
80 self.port = int(args.port)
81
82 - self._runner = self._RemoteRunner.remote(
82 + self._runner = self._RemoteRunner.remote( # type: ignore
83 cli_args,
84 wait_url=self.url_for("health"),
85 wait_timeout=self.MAX_SERVER_START_WAIT_S)
tests/worker/test_model_runner.py
+12 -11
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 import torch
5
@@ -35,8 +37,8 @@ def test_prepare_prompt(batch_size):
37 enable_chunked_prefill=False,
38 )
39
38 - seq_lens = []
39 - seq_group_metadata_list = []
40 + seq_lens: List[int] = []
41 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
42 block_tables = {0: [1]}
43 for i in range(batch_size):
44 # make sure all tokens fit into one block
@@ -151,15 +153,14 @@ def test_prepare_decode_cuda_graph(batch_size):
153 enable_chunked_prefill=False,
154 )
155
154 - context_lens = []
155 - seq_group_metadata_list = []
156 + context_lens: List[int] = []
157 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
158 # Assume each seq group finishes prefill.
159 for i in range(batch_size):
160 # make sure all tokens fit into one block
161 context_len = i % (model_runner.block_size - 1) + 1
162 context_lens.append(context_len)
161 - seq_data = list(range(context_len))
162 - seq_data = SequenceData(seq_data)
163 + seq_data = SequenceData(list(range(context_len)))
164 seq_data.update_num_computed_tokens(context_len)
165 # Append one token ID since prefill is finished.
166 seq_data.append_token_id(1, 0)
@@ -257,7 +258,7 @@ def test_empty_seq_group():
258 dtype="float16",
259 enforce_eager=False,
260 )
260 - seq_group_metadata_list = []
261 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
262 model_input = model_runner._prepare_model_input(seq_group_metadata_list)
263 input_tokens, input_positions, attn_metadata, slot_mapping = (
264 model_input.input_tokens,
@@ -310,10 +311,10 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
311 )
312
313 # Add prefill requests.
313 - seq_lens = []
314 - seq_group_metadata_list = []
315 - prefill_metadata_list = []
316 - decode_metadata_list = []
314 + seq_lens: List[int] = []
315 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
316 + prefill_metadata_list: List[SequenceGroupMetadata] = []
317 + decode_metadata_list: List[SequenceGroupMetadata] = []
318 block_tables = {0: [1]}
319 prefill_batch_size = batch_size // 2
320 decode_batch_size = batch_size - prefill_batch_size
vllm/attention/backends/torch_sdpa.py
+2 -2
@@ -245,7 +245,7 @@ def _make_alibi_bias(
245 dtype: torch.dtype,
246 seq_lens: List[int],
247 ) -> List[torch.Tensor]:
248 - attn_biases = []
248 + attn_biases: List[torch.Tensor] = []
249 for seq_len in seq_lens:
250 bias = torch.arange(seq_len, dtype=dtype)
251 # NOTE(zhuohan): HF uses
@@ -271,7 +271,7 @@ def _make_sliding_window_bias(
271 window_size: Optional[int],
272 dtype: torch.dtype,
273 ) -> List[torch.Tensor]:
274 - attn_biases = []
274 + attn_biases: List[torch.Tensor] = []
275 for seq_len in seq_lens:
276 tensor = torch.full(
277 (1, seq_len, seq_len),
vllm/attention/backends/xformers.py
+2 -2
@@ -431,8 +431,8 @@ def _make_alibi_bias(
431 num_kv_heads: int,
432 dtype: torch.dtype,
433 seq_lens: List[int],
434 -) -> LowerTriangularMaskWithTensorBias:
435 - attn_biases = []
434 +) -> List[AttentionBias]:
435 + attn_biases: List[AttentionBias] = []
436 for seq_len in seq_lens:
437 bias = torch.arange(seq_len, dtype=dtype)
438 # NOTE(zhuohan): HF uses
vllm/core/block/block_table.py
+1 -1
@@ -252,7 +252,7 @@ class BlockTable:
252 def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
253 token_ids: List[int],
254 device: Device) -> List[Block]:
255 - blocks = []
255 + blocks: List[Block] = []
256 for block_token_ids in chunk_list(token_ids, self._block_size):
257 if len(block_token_ids) == self._block_size:
258 # If the block is full, create an immutable block.
vllm/core/block/naive_block.py
+1 -1
@@ -111,7 +111,7 @@ class NaiveBlockAllocator(BlockAllocator):
111 """
112 source_blocks = get_all_blocks_recursively(last_block)
113
114 - forked_blocks = []
114 + forked_blocks: List[Block] = []
115 prev_block = None
116 for block in source_blocks:
117
vllm/core/block/prefix_caching_block.py
+1 -1
@@ -271,7 +271,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
271 """
272 source_blocks = get_all_blocks_recursively(last_block)
273
274 - forked_blocks = []
274 + forked_blocks: List[Block] = []
275 prev_block = None
276 for block in source_blocks:
277 refcount = self._refcounter.incr(block.block_id)
vllm/core/block_manager_v2.py
+1 -1
@@ -260,7 +260,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
260 # at max extend.
261 if self.enable_caching:
262 block_table = self.block_tables[seq.seq_id]
263 - block_ids = []
263 + block_ids: List[Optional[int]] = []
264 for block_id in block_table.physical_block_ids:
265 block_ids.append(block_id)
266 self.block_allocator.mark_blocks_as_accessed(
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+4 -4
@@ -2,7 +2,7 @@ import ctypes
2 import json
3 import os
4 from itertools import product
5 -from typing import Dict, Optional, Sequence
5 +from typing import Dict, List, Optional, Sequence
6
7 import torch.distributed as dist
8 import torch.multiprocessing as mp
@@ -88,7 +88,7 @@ def consumer(batch_tgt: Sequence[int],
88 def can_actually_p2p(
89 batch_src: Sequence[int],
90 batch_tgt: Sequence[int],
91 -):
91 +) -> Sequence[bool]:
92 """
93 Usually, checking if P2P access is enabled can be done by
94 `torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes
@@ -138,7 +138,7 @@ def can_actually_p2p(
138 p_tgt.start()
139 p_src.join()
140 p_tgt.join()
141 - result = []
141 + result: List[bool] = []
142 for src, tgt in zip(batch_src, batch_tgt):
143 a = result_queue.get()
144 b = result_queue.get()
@@ -188,7 +188,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
188 # only the local master process (with local_rank == 0) can
189 # enter this block to calculate the cache
190 logger.info("generating GPU P2P access cache in %s", path)
191 - cache = {}
191 + cache: Dict[str, bool] = {}
192 ids = list(range(num_dev))
193 # batch of all pairs of GPUs
194 batch_src, batch_tgt = zip(*list(product(ids, ids)))
vllm/distributed/device_communicators/pynccl_wrapper.py
+1 -1
@@ -205,7 +205,7 @@ class NCCLLibrary:
205 raise e
206
207 if so_file not in NCCLLibrary.path_to_dict_mapping:
208 - _funcs = {}
208 + _funcs: Dict[str, Any] = {}
209 for func in NCCLLibrary.exported_functions:
210 f = getattr(self.lib, func.name)
211 f.restype = func.restype
vllm/engine/llm_engine.py
+2 -2
@@ -2,7 +2,7 @@ import time
2 from contextlib import contextmanager
3 from typing import TYPE_CHECKING, ClassVar, Iterable, List, Optional
4 from typing import Sequence as GenericSequence
5 -from typing import Type, TypeVar, Union
5 +from typing import Set, Type, TypeVar, Union
6
7 from transformers import GenerationConfig, PreTrainedTokenizer
8
@@ -973,7 +973,7 @@ class LLMEngine:
973 def remove_lora(self, lora_id: int) -> bool:
974 return self.model_executor.remove_lora(lora_id)
975
976 - def list_loras(self) -> List[int]:
976 + def list_loras(self) -> Set[int]:
977 return self.model_executor.list_loras()
978
979 def check_health(self) -> None:
vllm/engine/metrics.py
+2 -2
@@ -144,7 +144,7 @@ class Metrics:
144 # end-metrics-definitions
145
146
147 -def build_1_2_5_buckets(max_value: int):
147 +def build_1_2_5_buckets(max_value: int) -> List[int]:
148 """
149 Builds a list of buckets with increasing powers of 10 multiplied by
150 mantissa values (1, 2, 5) until the value exceeds the specified maximum.
@@ -155,7 +155,7 @@ def build_1_2_5_buckets(max_value: int):
155 """
156 mantissa_lst = [1, 2, 5]
157 exponent = 0
158 - buckets = []
158 + buckets: List[int] = []
159 while True:
160 for m in mantissa_lst:
161 value = m * 10**exponent
vllm/engine/output_processor/single_step.py
+3 -3
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Tuple, Union
1 +from typing import Dict, List, Optional, Tuple, Union
2
3 from vllm.config import SchedulerConfig
4 from vllm.core.scheduler import Scheduler
@@ -146,8 +146,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
146
147 # Beam search case
148 # Select the child sequences to keep in the sequence group.
149 - selected_child_seqs = []
150 - unselected_child_seqs = []
149 + selected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
150 + unselected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
151 beam_width = seq_group.sampling_params.best_of
152 length_penalty = seq_group.sampling_params.length_penalty
153
vllm/entrypoints/openai/run_batch.py
+2 -1
@@ -2,6 +2,7 @@ import argparse
2 import asyncio
3 import sys
4 from io import StringIO
5 +from typing import Awaitable, List
6
7 import aiohttp
8
@@ -114,7 +115,7 @@ async def main(args):
115 )
116
117 # Submit all requests in the file to the engine "concurrently".
117 - response_futures = []
118 + response_futures: List[Awaitable[BatchRequestOutput]] = []
119 for request_json in (await read_file(args.input_file)).strip().split("\n"):
120 request = BatchRequestInput.model_validate_json(request_json)
121 response_futures.append(run_request(openai_serving_chat, request))
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -487,7 +487,7 @@ class OpenAIServingChat(OpenAIServing):
487 final_res = res
488 assert final_res is not None
489
490 - choices = []
490 + choices: List[ChatCompletionResponseChoice] = []
491
492 role = self.get_chat_request_role(request)
493 for output in final_res.outputs:
vllm/entrypoints/openai/serving_embedding.py
+1 -1
@@ -25,7 +25,7 @@ def request_output_to_embedding_response(
25 created_time: int,
26 model_name: str,
27 ) -> EmbeddingResponse:
28 - data = []
28 + data: List[EmbeddingResponseData] = []
29 num_prompt_tokens = 0
30 for idx, final_res in enumerate(final_res_batch):
31 assert final_res is not None
vllm/lora/lora.py
+2 -1
@@ -1,4 +1,5 @@
1 from typing import List, Optional
2 +from typing import Sequence as GenericSequence
3
4 import torch
5
@@ -120,7 +121,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
121
122 @classmethod
123 def pack(
123 - cls, loras: List[Optional["LoRALayerWeights"]]
124 + cls, loras: GenericSequence[Optional["LoRALayerWeights"]]
125 ) -> "PackedLoRALayerWeights":
126 """Pack a list of LoRAs into a single LoRA.
127
vllm/lora/worker_manager.py
+1 -1
@@ -165,7 +165,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
165 model = self._lora_manager.model
166 supported_lora_modules = model.supported_lora_modules
167 packed_modules_mapping = model.packed_modules_mapping
168 - expected_lora_modules = []
168 + expected_lora_modules: List[str] = []
169 for module in supported_lora_modules:
170 if module in packed_modules_mapping:
171 expected_lora_modules.extend(
vllm/model_executor/layers/linear.py
+1 -1
@@ -393,7 +393,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
393 param_data.copy_(loaded_weight)
394 return
395 current_shard_offset = 0
396 - shard_offsets = []
396 + shard_offsets: List[Tuple[int, int, int]] = []
397 for i, output_size in enumerate(self.output_sizes):
398 shard_offsets.append((i, current_shard_offset, output_size))
399 current_shard_offset += output_size
vllm/model_executor/layers/quantization/gptq_marlin.py
+6 -5
@@ -25,24 +25,25 @@ GPTQ_MARLIN_SUPPORTED_SYM = [True]
25
26
27 # Permutations for Marlin scale shuffling
28 -def get_scale_perms(num_bits):
29 - scale_perm = []
28 +def get_scale_perms(num_bits: int):
29 + scale_perm: List[int] = []
30 for i in range(8):
31 scale_perm.extend([i + 8 * j for j in range(8)])
32 - scale_perm_single = []
32 + scale_perm_single: List[int] = []
33 for i in range(4):
34 scale_perm_single.extend(
35 [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
36 return scale_perm, scale_perm_single
37
38
39 -def get_pack_factor(num_bits):
39 +def get_pack_factor(num_bits: int):
40 assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
41 ), f"Unsupported num_bits = {num_bits}"
42 return 32 // num_bits
43
44
45 -def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
45 +def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
46 + group_size: int, num_bits: int):
47 scale_perm, scale_perm_single = get_scale_perms(num_bits)
48 if group_size < size_k and group_size != -1:
49 s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py
+10 -8
@@ -1,4 +1,6 @@
1 """This file is used for /tests and /benchmarks"""
2 +from typing import Dict, List
3 +
4 import numpy
5 import torch
6
@@ -11,10 +13,10 @@ import torch
13 #
14 # As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15 # (without the need to use ldmatrix instructions) # noqa: E501
14 -def get_perms_24(num_bits):
15 - perm_list = []
16 +def get_perms_24(num_bits: int):
17 + perm_list: List[int] = []
18 for i in range(32):
17 - perm1 = []
19 + perm1: List[int] = []
20 col = i // 4
21 col_o = col // 2
22 for block in [0, 1]:
@@ -39,18 +41,18 @@ def get_perms_24(num_bits):
41
42 perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
43 perm = torch.from_numpy(perm)
42 - scale_perm = []
44 + scale_perm: List[int] = []
45 for i in range(8):
46 scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
45 - scale_perm_single = []
47 + scale_perm_single: List[int] = []
48 for i in range(8):
49 scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
50 return perm, scale_perm, scale_perm_single
51
52
51 -marlin_24_perm = {}
52 -marlin_24_scale_perm = {}
53 -marlin_24_scale_perm_single = {}
53 +marlin_24_perm: Dict[int, torch.Tensor] = {}
54 +marlin_24_scale_perm: Dict[int, List[int]] = {}
55 +marlin_24_scale_perm_single: Dict[int, List[int]] = {}
56 for num_bits in [4, 8]:
57 perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
58 marlin_24_perm[num_bits] = perm_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py
+10 -8
@@ -1,4 +1,6 @@
1 """This file is used for /tests and /benchmarks"""
2 +from typing import Dict, List
3 +
4 import numpy
5 import torch
6
@@ -11,10 +13,10 @@ import torch
13 #
14 # As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15 # (without the need to use ldmatrix instructions) # noqa: E501
14 -def get_perms(num_bits):
15 - perm_list = []
16 +def get_perms(num_bits: int):
17 + perm_list: List[int] = []
18 for i in range(32):
17 - perm1 = []
19 + perm1: List[int] = []
20 col = i // 4
21 for block in [0, 1]:
22 for row in [
@@ -38,19 +40,19 @@ def get_perms(num_bits):
40
41 perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
42 perm = torch.from_numpy(perm)
41 - scale_perm = []
43 + scale_perm: List[int] = []
44 for i in range(8):
45 scale_perm.extend([i + 8 * j for j in range(8)])
44 - scale_perm_single = []
46 + scale_perm_single: List[int] = []
47 for i in range(4):
48 scale_perm_single.extend(
49 [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
50 return perm, scale_perm, scale_perm_single
51
52
51 -marlin_perm = {}
52 -marlin_scale_perm = {}
53 -marlin_scale_perm_single = {}
53 +marlin_perm: Dict[int, torch.Tensor] = {}
54 +marlin_scale_perm: Dict[int, List[int]] = {}
55 +marlin_scale_perm_single: Dict[int, List[int]] = {}
56 for num_bits in [4, 8]:
57 perm, scale_perm, scale_perm_single = get_perms(num_bits)
58 marlin_perm[num_bits] = perm
vllm/model_executor/layers/sampler.py
+14 -11
@@ -174,7 +174,7 @@ def _apply_min_tokens_penalty(
174 min_tokens = sampling_params.min_tokens
175 token_ids_to_penalize = sampling_params.all_stop_token_ids
176 if min_tokens > 0 and token_ids_to_penalize:
177 - seqs_to_penalize = []
177 + seqs_to_penalize: List[int] = []
178 for j, seq_id in enumerate(seq_ids):
179 seq_data = seq_group.seq_data[seq_id]
180 if len(seq_data.output_token_ids) < min_tokens:
@@ -285,7 +285,7 @@ def _greedy_sample(
285 same as the length of selected_seq_groups. If the corresponding
286 seq_group has do_sample=False, tuple contains ([], [])
287 """
288 - samples = samples.tolist()
288 + samples_lst = samples.tolist()
289 sample_idx = 0
290 results: SampleResultType = []
291 for seq_group in selected_seq_groups:
@@ -298,7 +298,7 @@ def _greedy_sample(
298 assert num_parent_seqs == 1, (
299 "Greedy sampling should have only one seq.")
300 parent_ids = list(range(num_parent_seqs))
301 - next_token_ids = [samples[sample_idx]]
301 + next_token_ids = [samples_lst[sample_idx]]
302 results.append((next_token_ids, parent_ids))
303 sample_idx += num_parent_seqs
304 return results
@@ -394,7 +394,7 @@ def _beam_search_sample(
394 next_token_ids = next_token_ids.tolist()
395 else:
396 # Generation phase.
397 - cumulative_logprobs: List[int] = [
397 + cumulative_logprobs: List[float] = [
398 seq_group.seq_data[seq_id].cumulative_logprob
399 for seq_id in seq_ids
400 ]
@@ -466,8 +466,9 @@ def _sample_with_torch(
466 categorized_seq_group_ids[sampling_type].append(i)
467
468 sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
469 - sample_metadata = {}
470 - multinomial_samples = {}
469 + sample_metadata: Dict[SamplingType,
470 + Tuple[List[int], List[SequenceGroupToSample]]] = {}
471 + multinomial_samples: Dict[SamplingType, torch.Tensor] = {}
472
473 # Create output tensor for sampled token ids.
474 if include_gpu_probs_tensor:
@@ -494,7 +495,7 @@ def _sample_with_torch(
495 greedy_samples = torch.argmax(logprobs[long_sample_indices],
496 dim=-1)
497
497 - if include_gpu_probs_tensor:
498 + if sampled_token_ids_tensor is not None:
499 # Store sampled tokens in output tensor.
500 sampled_token_ids_tensor[
501 long_sample_indices] = greedy_samples.unsqueeze(-1)
@@ -522,7 +523,7 @@ def _sample_with_torch(
523 probs[long_sample_indices], max_best_of_in_batch,
524 **seeded_args)
525
525 - if include_gpu_probs_tensor:
526 + if sampled_token_ids_tensor is not None:
527 # Store sampled tokens in output tensor.
528 sampled_token_ids_tensor[
529 long_sample_indices] = multinomial_samples[sampling_type]
@@ -571,7 +572,9 @@ def _sample_with_triton_kernel(
572 categorized_seq_group_ids[sampling_type].append(i)
573
574 sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
574 - sample_metadata = {}
575 + sample_metadata: Dict[SamplingType,
576 + Tuple[List[int], List[SequenceGroupToSample],
577 + torch.Tensor, torch.Tensor]] = {}
578 max_best_of_in_batch = 1
579
580 # Counterintiutively, having two loops here is actually faster.
@@ -1008,14 +1011,14 @@ def _build_sampler_output(
1011 speculative decoding rejection sampling.
1012 """
1013
1011 - sampler_output = []
1014 + sampler_output: List[CompletionSequenceGroupOutput] = []
1015 for (seq_group, sample_result, group_prompt_logprobs,
1016 group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1017 sample_results, prompt_logprobs,
1018 sample_logprobs):
1019 seq_ids = seq_group.seq_ids
1020 next_token_ids, parent_ids = sample_result
1018 - seq_outputs = []
1021 + seq_outputs: List[SequenceOutput] = []
1022 for parent_id, next_token_id, logprobs in zip(parent_ids,
1023 next_token_ids,
1024 group_sample_logprobs):
vllm/model_executor/model_loader/loader.py
+4 -3
@@ -68,7 +68,7 @@ def _get_model_initialization_kwargs(
68 vision_language_config: Optional[VisionLanguageConfig]
69 ) -> Dict[str, Any]:
70 """Get extra kwargs for model initialization."""
71 - extra_kwargs = {}
71 + extra_kwargs: Dict[str, Any] = {}
72 if hasattr(model_class, "supported_lora_modules"):
73 extra_kwargs["lora_config"] = lora_config
74 elif lora_config:
@@ -446,7 +446,8 @@ class ShardedStateLoader(BaseModelLoader):
446 Filter out all tensors that share the same memory or a subset of the
447 memory of another tensor.
448 """
449 - same_storage_groups = collections.defaultdict(list)
449 + same_storage_groups: Dict[Any, List[Tuple[
450 + str, torch.Tensor]]] = collections.defaultdict(list)
451 for key, tensor in tensors.items():
452 if tensor.numel():
453 ptr = tensor.untyped_storage().data_ptr()
@@ -455,7 +456,7 @@ class ShardedStateLoader(BaseModelLoader):
456 def get_end_ptr(tensor: torch.Tensor) -> int:
457 return tensor.view(-1)[-1].data_ptr() + tensor.element_size()
458
458 - result = {}
459 + result: Dict[str, torch.Tensor] = {}
460 for group in same_storage_groups.values():
461 for k, t in group:
462 a, b = t.data_ptr(), get_end_ptr(t)
vllm/model_executor/model_loader/weight_utils.py
+1 -1
@@ -329,7 +329,7 @@ def np_cache_weights_iterator(
329 # dumping the same model weights to numpy at the same time.
330 with get_lock(model_name_or_path, cache_dir):
331 if not os.path.exists(weight_names_file):
332 - weight_names = []
332 + weight_names: List[str] = []
333 for bin_file in hf_weights_files:
334 state = torch.load(bin_file, map_location="cpu")
335 for name, param in state.items():
vllm/model_executor/models/__init__.py
+2 -2
@@ -72,11 +72,11 @@ _MODELS = {**_GENERATION_MODELS, **_EMBEDDING_MODELS}
72 _OOT_MODELS: Dict[str, Type[nn.Module]] = {}
73
74 # Models not supported by ROCm.
75 -_ROCM_UNSUPPORTED_MODELS = []
75 +_ROCM_UNSUPPORTED_MODELS: List[str] = []
76
77 # Models partially supported by ROCm.
78 # Architecture -> Reason.
79 -_ROCM_PARTIALLY_SUPPORTED_MODELS = {
79 +_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
80 "Qwen2ForCausalLM":
81 "Sliding window attention is not yet supported in ROCm's flash attention",
82 "MistralForCausalLM":
vllm/model_executor/models/arctic.py
+2 -2
@@ -453,8 +453,8 @@ class ArcticForCausalLM(nn.Module):
453 ("qkv_proj", "v_proj", "v"),
454 ]
455
456 - mlp_params_mapping = []
457 - expert_params_mapping = []
456 + mlp_params_mapping: List[Tuple[str, str, int]] = []
457 + expert_params_mapping: List[Tuple[str, str, int]] = []
458 num_layers = self.config.num_hidden_layers
459
460 for layer in range(num_layers):
vllm/model_executor/models/commandr.py
+2 -2
@@ -20,7 +20,7 @@
20
21 # This file is based on the LLama model definition file in transformers
22 """PyTorch Cohere model."""
23 -from typing import Iterable, List, Optional, Tuple
23 +from typing import Iterable, List, Optional, Set, Tuple
24
25 import torch
26 import torch.utils.checkpoint
@@ -352,7 +352,7 @@ class CohereForCausalLM(nn.Module):
352 ("gate_up_proj", "up_proj", 1),
353 ]
354 params_dict = dict(self.named_parameters())
355 - loaded_params = set()
355 + loaded_params: Set[str] = set()
356 for name, loaded_weight in weights:
357 for param_name, shard_name, shard_id in stacked_params_mapping:
358 if shard_name not in name:
vllm/model_executor/models/gemma.py
+2 -2
@@ -15,7 +15,7 @@
15 # limitations under the License.
16 """Inference-only Gemma model compatible with HuggingFace weights."""
17 from functools import lru_cache
18 -from typing import Iterable, List, Optional, Tuple
18 +from typing import Iterable, List, Optional, Set, Tuple
19
20 import torch
21 from torch import nn
@@ -363,7 +363,7 @@ class GemmaForCausalLM(nn.Module):
363 ("gate_up_proj", "up_proj", 1),
364 ]
365 params_dict = dict(self.named_parameters())
366 - loaded_params = set()
366 + loaded_params: Set[str] = set()
367 for name, loaded_weight in weights:
368 for (param_name, shard_name, shard_id) in stacked_params_mapping:
369 if shard_name not in name:
vllm/sequence.py
+1 -1
@@ -123,7 +123,7 @@ class SequenceData:
123 output_token_ids = []
124
125 self.prompt_token_ids = prompt_token_ids
126 - self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
126 + self._prompt_token_ids_tuple = tuple(prompt_token_ids)
127 self.output_token_ids = output_token_ids
128 self.cumulative_logprob = 0.0
129 # The number of tokens that are computed (that run against the model).
vllm/spec_decode/multi_step_worker.py
+5 -5
@@ -1,10 +1,10 @@
1 import copy
2 import weakref
3 -from typing import List, Tuple
3 +from typing import Dict, List, Tuple
4
5 import torch
6
7 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8 SequenceGroupMetadata)
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
@@ -71,7 +71,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
71 sample_len)
72
73 # Run model sample_len times.
74 - model_outputs = []
74 + model_outputs: List[SamplerOutput] = []
75 for _ in range(sample_len):
76 model_output = super().execute_model(
77 execute_model_req=copied_execute_model_req)
@@ -132,7 +132,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
132
133 # Shallow-copy the list of SequenceGroupMetadata. This allows us to
134 # append tokens and change is_prompt without external side-effects.
135 - new_seq_group_metadata_list = []
135 + new_seq_group_metadata_list: List[SequenceGroupMetadata] = []
136
137 for old_seq_group_metadata in seq_group_metadata_list:
138 # We must shallow-copy seq_group_metadata as is_prompt could change.
@@ -140,7 +140,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
140 new_seq_group_metadata_list.append(seq_group_metadata)
141
142 # We must shallow-copy seq_data as we will append token ids
143 - new_seq_data = {}
143 + new_seq_data: Dict[int, SequenceData] = {}
144 for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
145 new_seq_data[seq_id] = copy.copy(old_seq_data)
146 new_seq_data[
vllm/spec_decode/ngram_worker.py
+3 -3
@@ -48,7 +48,7 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
48 self,
49 execute_model_req: ExecuteModelRequest,
50 sample_len: int,
51 - ) -> Tuple[Optional[List[SamplerOutput]], bool]:
51 + ) -> Tuple[Optional[List[Optional[SamplerOutput]]], bool]:
52 """NGram match algo to pick proposal candidate. Returns the list of
53 sampler output, one per SequenceGroupMetadata.
54
@@ -58,8 +58,8 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
58 self._raise_if_unsupported(execute_model_req)
59
60 has_spec_out = False
61 - token_id_list = []
62 - token_prob_list = []
61 + token_id_list: List[Optional[torch.Tensor]] = []
62 + token_prob_list: List[Optional[torch.Tensor]] = []
63 for idx, seq_group_metadata in enumerate(
64 execute_model_req.seq_group_metadata_list):
65 seq_data = next(iter(seq_group_metadata.seq_data.values()))
vllm/spec_decode/spec_decode_worker.py
+4 -4
@@ -7,8 +7,8 @@ from vllm.config import SpeculativeConfig
7 from vllm.distributed.communication_op import broadcast_tensor_dict
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
11 - SequenceGroupMetadata)
10 +from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
11 + SamplerOutput, SequenceGroupMetadata)
12 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
13 from vllm.spec_decode.interfaces import (SpeculativeProposals,
14 SpeculativeScorer, SpeculativeScores)
@@ -516,13 +516,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
516 topk_indices_by_step = topk_indices_by_step.tolist()
517
518 # Construct the output on a per-step, per-sequence basis.
519 - sampler_output_list = []
519 + sampler_output_list: List[SamplerOutput] = []
520 for step_index in range(num_steps):
521 if all(token_id == -1
522 for token_id in accepted_token_ids_by_step[step_index]):
523 break
524
525 - step_output_token_ids = []
525 + step_output_token_ids: List[CompletionSequenceGroupOutput] = []
526 for sequence_index in range(batch_size):
527 # Each sequence may have a different num_logprobs; retrieve it.
528 num_logprobs = num_logprobs_per_seq[sequence_index]
vllm/spec_decode/util.py
+2 -2
@@ -26,10 +26,10 @@ def get_all_num_logprobs(
26 sequence.
27 """
28
29 - all_num_logprobs = []
29 + all_num_logprobs: List[int] = []
30 for seq_group_metadata in seq_group_metadata_list:
31 num_logprobs = seq_group_metadata.sampling_params.logprobs
32 - if seq_group_metadata.sampling_params.logprobs is None:
32 + if num_logprobs is None:
33 num_logprobs = 0
34 all_num_logprobs.append(num_logprobs)
35
vllm/transformers_utils/detokenizer.py
+1 -1
@@ -44,7 +44,7 @@ class Detokenizer:
44 read_offset = 0
45 next_iter_prefix_offset = 0
46 next_iter_read_offset = 0
47 - next_iter_tokens = []
47 + next_iter_tokens: List[str] = []
48 prev_tokens = None
49
50 for token_position, prompt_logprobs_for_token in enumerate(
vllm/utils.py
+23 -15
@@ -20,12 +20,13 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
20 import numpy as np
21 import psutil
22 import torch
23 +import torch.types
24 +from typing_extensions import ParamSpec
25
26 import vllm.envs as envs
27 from vllm import _custom_ops as ops
28 from vllm.logger import enable_trace_function_call, init_logger
29
28 -T = TypeVar("T")
30 logger = init_logger(__name__)
31
32 STR_DTYPE_TO_TORCH_DTYPE = {
@@ -37,6 +38,10 @@ STR_DTYPE_TO_TORCH_DTYPE = {
38 "fp8_e5m2": torch.uint8,
39 }
40
41 +P = ParamSpec('P')
42 +K = TypeVar("K")
43 +T = TypeVar("T")
44 +
45
46 class Device(enum.Enum):
47 GPU = enum.auto()
@@ -176,7 +181,7 @@ def random_uuid() -> str:
181
182
183 @lru_cache(maxsize=None)
179 -def get_vllm_instance_id():
184 +def get_vllm_instance_id() -> str:
185 """
186 If the environment variable VLLM_INSTANCE_ID is set, return it.
187 Otherwise, return a random UUID.
@@ -192,7 +197,7 @@ def in_wsl() -> bool:
197 return "microsoft" in " ".join(uname()).lower()
198
199
195 -def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
200 +def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
201 """Take a blocking function, and run it on in an executor thread.
202
203 This function prevents the blocking function from blocking the
@@ -200,7 +205,7 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
205 The code in this function needs to be thread safe.
206 """
207
203 - def _async_wrapper(*args, **kwargs) -> asyncio.Future:
208 + def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
209 loop = asyncio.get_event_loop()
210 p_func = partial(func, *args, **kwargs)
211 return loop.run_in_executor(executor=None, func=p_func)
@@ -325,7 +330,7 @@ def update_environment_variables(envs: Dict[str, str]):
330 os.environ[k] = v
331
332
328 -def chunk_list(lst, chunk_size):
333 +def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
334 """Yield successive chunk_size chunks from lst."""
335 return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
336
@@ -336,7 +341,7 @@ def cdiv(a: int, b: int) -> int:
341
342
343 def _generate_random_fp8(
339 - tensor: torch.tensor,
344 + tensor: torch.Tensor,
345 low: float,
346 high: float,
347 ) -> None:
@@ -398,7 +403,10 @@ def create_kv_caches_with_random_flash(
403 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
404 key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
405 scale = head_size**-0.5
401 - key_caches, value_caches = [], []
406 +
407 + key_caches: List[torch.Tensor] = []
408 + value_caches: List[torch.Tensor] = []
409 +
410 for _ in range(num_layers):
411 key_value_cache = torch.empty(size=key_value_cache_shape,
412 dtype=torch_dtype,
@@ -429,7 +437,7 @@ def create_kv_caches_with_random(
437 scale = head_size**-0.5
438 x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
439 key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
432 - key_caches = []
440 + key_caches: List[torch.Tensor] = []
441 for _ in range(num_layers):
442 key_cache = torch.empty(size=key_cache_shape,
443 dtype=torch_dtype,
@@ -444,7 +452,7 @@ def create_kv_caches_with_random(
452 key_caches.append(key_cache)
453
454 value_cache_shape = (num_blocks, num_heads, head_size, block_size)
447 - value_caches = []
455 + value_caches: List[torch.Tensor] = []
456 for _ in range(num_layers):
457 value_cache = torch.empty(size=value_cache_shape,
458 dtype=torch_dtype,
@@ -484,7 +492,7 @@ def is_pin_memory_available() -> bool:
492
493 class CudaMemoryProfiler:
494
487 - def __init__(self, device=None):
495 + def __init__(self, device: Optional[torch.types.Device] = None):
496 self.device = device
497
498 def current_memory_usage(self) -> float:
@@ -560,13 +568,13 @@ def get_dtype_size(dtype: torch.dtype) -> int:
568 return torch.tensor([], dtype=dtype).element_size()
569
570
563 -def merge_dicts(dict1: Dict[Any, List[Any]],
564 - dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
571 +def merge_dicts(dict1: Dict[K, List[T]],
572 + dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
573 """Merge 2 dicts that have key -> List of items.
574
575 When a key conflicts, the values in dict1 is prioritized.
576 """
569 - merged_dict = defaultdict(list)
577 + merged_dict: Dict[K, List[T]] = defaultdict(list)
578
579 for key, value in dict1.items():
580 merged_dict[key].extend(value)
@@ -577,7 +585,7 @@ def merge_dicts(dict1: Dict[Any, List[Any]],
585 return dict(merged_dict)
586
587
580 -def init_cached_hf_modules():
588 +def init_cached_hf_modules() -> None:
589 """
590 Lazy initialization of the Hugging Face modules.
591 """
@@ -613,7 +621,7 @@ def find_library(lib_name: str) -> str:
621 return locs[0]
622
623
616 -def find_nccl_library():
624 +def find_nccl_library() -> str:
625 """
626 We either use the library file specified by the `VLLM_NCCL_SO_PATH`
627 environment variable, or we find the library file brought by PyTorch.
vllm/worker/model_runner.py
+2 -2
@@ -779,8 +779,8 @@ class ModelRunner:
779 # that will have unique loras, an therefore the max amount of memory
780 # consumption create dummy lora request copies from the lora request
781 # passed in, which contains a lora from the lora warmup path.
782 - dummy_lora_requests = []
783 - dummy_lora_requests_per_seq = []
782 + dummy_lora_requests: List[LoRARequest] = []
783 + dummy_lora_requests_per_seq: List[LoRARequest] = []
784 if self.lora_config:
785 assert self.lora_manager is not None
786 with self.lora_manager.dummy_lora_cache():
vllm/worker/worker_base.py
+2 -2
@@ -99,8 +99,8 @@ class WorkerWrapperBase:
99 """
100
101 def __init__(self,
102 - worker_module_name=None,
103 - worker_class_name=None,
102 + worker_module_name: str,
103 + worker_class_name: str,
104 trust_remote_code: bool = False) -> None:
105 self.worker_module_name = worker_module_name
106 self.worker_class_name = worker_class_name