@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 823ab79633
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Update `pre-commit` hooks (#12475)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jan 28, 2025 at 00:23 UTC 823ab796330825f4052d771e2c462ad3b55236eb
64 files changed +322 -288
.pre-commit-config.yaml
+5 -5
@@ -3,18 +3,18 @@ default_stages:
3 - manual # Run in CI
4 repos:
5 - repo: https://github.com/google/yapf
6 - rev: v0.32.0
6 + rev: v0.43.0
7 hooks:
8 - id: yapf
9 args: [--in-place, --verbose]
10 additional_dependencies: [toml] # TODO: Remove when yapf is upgraded
11 - repo: https://github.com/astral-sh/ruff-pre-commit
12 - rev: v0.6.5
12 + rev: v0.9.3
13 hooks:
14 - id: ruff
15 args: [--output-format, github]
16 - repo: https://github.com/codespell-project/codespell
17 - rev: v2.3.0
17 + rev: v2.4.0
18 hooks:
19 - id: codespell
20 exclude: 'benchmarks/sonnet.txt|(build|tests/(lora/data|models/fixtures|prompts))/.*'
@@ -23,7 +23,7 @@ repos:
23 hooks:
24 - id: isort
25 - repo: https://github.com/pre-commit/mirrors-clang-format
26 - rev: v18.1.5
26 + rev: v19.1.7
27 hooks:
28 - id: clang-format
29 exclude: 'csrc/(moe/topk_softmax_kernels.cu|quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))'
@@ -35,7 +35,7 @@ repos:
35 - id: pymarkdown
36 files: docs/.*
37 - repo: https://github.com/rhysd/actionlint
38 - rev: v1.7.6
38 + rev: v1.7.7
39 hooks:
40 - id: actionlint
41 - repo: local
benchmarks/benchmark_serving.py
+2 -2
@@ -926,8 +926,8 @@ def main(args: argparse.Namespace):
926 )
927
928 # Traffic
929 - result_json["request_rate"] = (
930 - args.request_rate if args.request_rate < float("inf") else "inf")
929 + result_json["request_rate"] = (args.request_rate if args.request_rate
930 + < float("inf") else "inf")
931 result_json["burstiness"] = args.burstiness
932 result_json["max_concurrency"] = args.max_concurrency
933
csrc/custom_all_reduce.cuh
+6 -2
@@ -38,9 +38,13 @@ struct Signal {
38 alignas(128) FlagType peer_counter[2][kMaxBlocks][8];
39 };
40
41 -struct __align__(16) RankData { const void* __restrict__ ptrs[8]; };
41 +struct __align__(16) RankData {
42 + const void* __restrict__ ptrs[8];
43 +};
44
43 -struct __align__(16) RankSignals { Signal* signals[8]; };
45 +struct __align__(16) RankSignals {
46 + Signal* signals[8];
47 +};
48
49 // like std::array, but aligned
50 template <typename T, int sz>
csrc/moe/marlin_kernels/marlin_moe_kernel.h
+4 -4
@@ -138,8 +138,8 @@ __device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
138 const int HI = 0x00f000f0;
139 const int EX = 0x64006400;
140 // Guarantee that the `(a & b) | c` operations are LOP3s.
141 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
142 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
141 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
142 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
143 // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
144 // directly into `SUB` and `ADD`.
145 const int SUB = 0x64086408;
@@ -182,8 +182,8 @@ __device__ inline FragB dequant<vllm::kU4.id()>(int q) {
182 const int HI = 0x00f000f0;
183 const int EX = 0x64006400;
184 // Guarantee that the `(a & b) | c` operations are LOP3s.
185 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
186 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
185 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
186 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
187
188 const int SUB = 0x64006400;
189 const int MUL = 0x2c002c00;
csrc/quantization/gptq_marlin/gptq_marlin.cu
+8 -8
@@ -173,8 +173,8 @@ dequant<half, vllm::kU4B8.id()>(int q) {
173 const int HI = 0x00f000f0;
174 const int EX = 0x64006400;
175 // Guarantee that the `(a & b) | c` operations are LOP3s.
176 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
177 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
176 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
177 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
178 // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
179 // directly into `SUB` and `ADD`.
180 const int SUB = 0x64086408;
@@ -197,9 +197,9 @@ dequant<nv_bfloat16, vllm::kU4B8.id()>(int q) {
197
198 // Guarantee that the `(a & b) | c` operations are LOP3s.
199
200 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
200 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, MASK, EX);
201 q >>= 4;
202 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
202 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, MASK, EX);
203
204 typename ScalarType<nv_bfloat16>::FragB frag_b;
205 static constexpr uint32_t MUL = 0x3F803F80;
@@ -221,8 +221,8 @@ dequant<half, vllm::kU4.id()>(int q) {
221 const int HI = 0x00f000f0;
222 const int EX = 0x64006400;
223 // Guarantee that the `(a & b) | c` operations are LOP3s.
224 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
225 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
224 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
225 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
226
227 const int SUB = 0x64006400;
228 const int MUL = 0x2c002c00;
@@ -244,9 +244,9 @@ dequant<nv_bfloat16, vllm::kU4.id()>(int q) {
244
245 // Guarantee that the `(a & b) | c` operations are LOP3s.
246
247 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
247 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, MASK, EX);
248 q >>= 4;
249 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
249 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, MASK, EX);
250
251 typename ScalarType<nv_bfloat16>::FragB frag_b;
252 static constexpr uint32_t MUL = 0x3F803F80;
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+2 -2
@@ -96,8 +96,8 @@ __device__ inline FragB dequant(int q) {
96 const int HI = 0x00f000f0;
97 const int EX = 0x64006400;
98 // Guarantee that the `(a & b) | c` operations are LOP3s.
99 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
100 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
99 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
100 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
101 // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
102 // directly into `SUB` and `ADD`.
103 const int SUB = 0x64086408;
csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu
+2 -2
@@ -141,8 +141,8 @@ __device__ inline FragB dequant_per_group(int q, FragS_GROUP& frag_s, int i) {
141 static constexpr uint32_t HI = 0x00f000f0;
142 static constexpr uint32_t EX = 0x64006400;
143 // Guarantee that the `(a & b) | c` operations are LOP3s.
144 - uint32_t t0 = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
145 - uint32_t t1 = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
144 + uint32_t t0 = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
145 + uint32_t t1 = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
146 // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
147 // directly into `SUB` and `ADD`.
148 static constexpr uint32_t SUB = 0x64086408;
csrc/quantization/marlin/sparse/common/mma.h
+2 -2
@@ -127,8 +127,8 @@ __device__ inline FragB dequant_4bit(int q) {
127 const int HI = 0x00f000f0;
128 const int EX = 0x64006400;
129 // Guarantee that the `(a & b) | c` operations are LOP3s.
130 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
131 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
130 + int lo = lop3 < (0xf0 & 0xcc) | 0xaa > (q, LO, EX);
131 + int hi = lop3 < (0xf0 & 0xcc) | 0xaa > (q, HI, EX);
132 // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
133 // directly into `SUB` and `ADD`.
134 const int SUB = 0x64086408;
csrc/rocm/attention.cu
+3 -1
@@ -907,7 +907,9 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
907 const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
908 // max_num_partitions, head_size]
909 const int* __restrict__ context_lens, // [num_seqs]
910 - const int max_num_partitions){UNREACHABLE_CODE}
910 + const int max_num_partitions) {
911 + UNREACHABLE_CODE
912 +}
913
914 #endif // defined(__HIP__MI300_MI250__) TODO: Add NAVI support
915
setup.py
+1 -1
@@ -417,7 +417,7 @@ def get_rocm_version():
417
418 if (get_rocm_core_version(ctypes.byref(major), ctypes.byref(minor),
419 ctypes.byref(patch)) == 0):
420 - return "%d.%d.%d" % (major.value, minor.value, patch.value)
420 + return f"{major.value}.{minor.value}.{patch.value}"
421 return None
422 except Exception:
423 return None
tests/kernels/test_block_fp8.py
+14 -11
@@ -92,8 +92,10 @@ def native_w8a8_block_fp8_matmul(A,
92 A[:, i * block_k:min((i + 1) * block_k, K)] for i in range(k_tiles)
93 ]
94 B_tiles = [[
95 - B[j * block_n:min((j + 1) * block_n, N),
96 - i * block_k:min((i + 1) * block_k, K), ] for i in range(k_tiles)
95 + B[
96 + j * block_n:min((j + 1) * block_n, N),
97 + i * block_k:min((i + 1) * block_k, K),
98 + ] for i in range(k_tiles)
99 ] for j in range(n_tiles)]
100 C_tiles = [
101 C[:, j * block_n:min((j + 1) * block_n, N)] for j in range(n_tiles)
@@ -157,9 +159,9 @@ def setup_cuda():
159 torch.set_default_device("cuda")
160
161
160 -@pytest.mark.parametrize("num_tokens,d,dtype,group_size,seed",
161 - itertools.product(NUM_TOKENS, D, DTYPES, GROUP_SIZE,
162 - SEEDS))
162 +@pytest.mark.parametrize(
163 + "num_tokens,d,dtype,group_size,seed",
164 + itertools.product(NUM_TOKENS, D, DTYPES, GROUP_SIZE, SEEDS))
165 @torch.inference_mode()
166 def test_per_token_group_quant_fp8(num_tokens, d, dtype, group_size, seed):
167 torch.manual_seed(seed)
@@ -174,9 +176,9 @@ def test_per_token_group_quant_fp8(num_tokens, d, dtype, group_size, seed):
176 assert torch.allclose(scale, ref_scale)
177
178
177 -@pytest.mark.parametrize("M,N,K,block_size,out_dtype,seed",
178 - itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES,
179 - SEEDS))
179 +@pytest.mark.parametrize(
180 + "M,N,K,block_size,out_dtype,seed",
181 + itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS))
182 @torch.inference_mode()
183 def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
184 torch.manual_seed(seed)
@@ -207,9 +209,10 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
209 assert rel_diff < 0.001
210
211
210 -@pytest.mark.parametrize("M,N,K,E,topk,block_size,dtype,seed",
211 - itertools.product(M_moe, N_moe, K_moe, E, TOP_KS,
212 - BLOCK_SIZE, DTYPES, SEEDS))
212 +@pytest.mark.parametrize(
213 + "M,N,K,E,topk,block_size,dtype,seed",
214 + itertools.product(M_moe, N_moe, K_moe, E, TOP_KS, BLOCK_SIZE, DTYPES,
215 + SEEDS))
216 @torch.inference_mode()
217 def test_w8a8_block_fp8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
218 torch.manual_seed(seed)
tests/kv_transfer/test_lookup_buffer.py
+5 -5
@@ -20,7 +20,7 @@ def test_run(my_rank, buffer, device):
20 assert buffer.buffer_size == 0
21 assert len(buffer.buffer) == 0
22
23 - print("My rank: %d, device: %s" % (my_rank, device))
23 + print(f"My rank: {my_rank}, device: {device}")
24
25 # insert
26 tokens = torch.tensor([1, 2, 3]).to(device)
@@ -48,7 +48,7 @@ def test_run(my_rank, buffer, device):
48 assert buffer.buffer_size == 0
49 assert len(buffer.buffer) == 0
50
51 - print("My rank: %d, Test run passed!" % (my_rank))
51 + print(f"My rank: {my_rank}, Test run passed!")
52
53
54 def stress_test(my_rank, buf, device):
@@ -94,7 +94,7 @@ def stress_test(my_rank, buf, device):
94 assert torch.allclose(k, k_)
95 assert torch.allclose(v, v_)
96 assert torch.allclose(h, h_)
97 - print('Rank %d done' % my_rank)
97 + print(f"Rank {my_rank} done")
98 torch.distributed.barrier()
99
100 if my_rank == 0:
@@ -108,7 +108,7 @@ def stress_test(my_rank, buf, device):
108 else:
109 torch.distributed.send(torch.tensor([n]), 0)
110
111 - print("My rank: %d, Passed stress test!" % (my_rank))
111 + print(f"My rank: {my_rank}, Passed stress test!")
112
113
114 if __name__ == "__main__":
@@ -122,7 +122,7 @@ if __name__ == "__main__":
122 rank=my_rank,
123 )
124
125 - print("initialized! My rank is %d" % my_rank)
125 + print(f"initialized! My rank is {my_rank}")
126
127 config = KVTransferConfig(
128 kv_connector='PyNcclConnector',
tests/lora/test_qwen2vl.py
+3 -3
@@ -55,9 +55,9 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
55 return generated_texts
56
57
58 -@pytest.mark.xfail(current_platform.is_rocm(),
59 - reason="Qwen2-VL dependency xformers incompatible with ROCm"
60 - )
58 +@pytest.mark.xfail(
59 + current_platform.is_rocm(),
60 + reason="Qwen2-VL dependency xformers incompatible with ROCm")
61 def test_qwen2vl_lora(qwen2vl_lora_files):
62 llm = vllm.LLM(
63 MODEL_PATH,
tests/models/decoder_only/vision_language/test_models.py
+70 -60
@@ -521,12 +521,13 @@ VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)
521 # - image embeddings
522 # - video
523 # - custom inputs
524 -@pytest.mark.parametrize("model_type,test_case",
525 - get_parametrized_options(
526 - VLM_TEST_SETTINGS,
527 - test_type=VLMTestType.IMAGE,
528 - fork_new_process_for_each_test=False,
529 - ))
524 +@pytest.mark.parametrize(
525 + "model_type,test_case",
526 + get_parametrized_options(
527 + VLM_TEST_SETTINGS,
528 + test_type=VLMTestType.IMAGE,
529 + fork_new_process_for_each_test=False,
530 + ))
531 def test_single_image_models(tmp_path: PosixPath, model_type: str,
532 test_case: ExpandableVLMTestArgs,
533 hf_runner: Type[HfRunner],
@@ -543,12 +544,13 @@ def test_single_image_models(tmp_path: PosixPath, model_type: str,
544 )
545
546
546 -@pytest.mark.parametrize("model_type,test_case",
547 - get_parametrized_options(
548 - VLM_TEST_SETTINGS,
549 - test_type=VLMTestType.MULTI_IMAGE,
550 - fork_new_process_for_each_test=False,
551 - ))
547 +@pytest.mark.parametrize(
548 + "model_type,test_case",
549 + get_parametrized_options(
550 + VLM_TEST_SETTINGS,
551 + test_type=VLMTestType.MULTI_IMAGE,
552 + fork_new_process_for_each_test=False,
553 + ))
554 def test_multi_image_models(tmp_path: PosixPath, model_type: str,
555 test_case: ExpandableVLMTestArgs,
556 hf_runner: Type[HfRunner],
@@ -565,12 +567,13 @@ def test_multi_image_models(tmp_path: PosixPath, model_type: str,
567 )
568
569
568 -@pytest.mark.parametrize("model_type,test_case",
569 - get_parametrized_options(
570 - VLM_TEST_SETTINGS,
571 - test_type=VLMTestType.EMBEDDING,
572 - fork_new_process_for_each_test=False,
573 - ))
570 +@pytest.mark.parametrize(
571 + "model_type,test_case",
572 + get_parametrized_options(
573 + VLM_TEST_SETTINGS,
574 + test_type=VLMTestType.EMBEDDING,
575 + fork_new_process_for_each_test=False,
576 + ))
577 def test_image_embedding_models(model_type: str,
578 test_case: ExpandableVLMTestArgs,
579 hf_runner: Type[HfRunner],
@@ -586,12 +589,13 @@ def test_image_embedding_models(model_type: str,
589 )
590
591
589 -@pytest.mark.parametrize("model_type,test_case",
590 - get_parametrized_options(
591 - VLM_TEST_SETTINGS,
592 - test_type=VLMTestType.VIDEO,
593 - fork_new_process_for_each_test=False,
594 - ))
592 +@pytest.mark.parametrize(
593 + "model_type,test_case",
594 + get_parametrized_options(
595 + VLM_TEST_SETTINGS,
596 + test_type=VLMTestType.VIDEO,
597 + fork_new_process_for_each_test=False,
598 + ))
599 def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,
600 hf_runner: Type[HfRunner], vllm_runner: Type[VllmRunner],
601 video_assets: _VideoAssets):
@@ -605,12 +609,13 @@ def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,
609 )
610
611
608 -@pytest.mark.parametrize("model_type,test_case",
609 - get_parametrized_options(
610 - VLM_TEST_SETTINGS,
611 - test_type=VLMTestType.CUSTOM_INPUTS,
612 - fork_new_process_for_each_test=False,
613 - ))
612 +@pytest.mark.parametrize(
613 + "model_type,test_case",
614 + get_parametrized_options(
615 + VLM_TEST_SETTINGS,
616 + test_type=VLMTestType.CUSTOM_INPUTS,
617 + fork_new_process_for_each_test=False,
618 + ))
619 def test_custom_inputs_models(
620 model_type: str,
621 test_case: ExpandableVLMTestArgs,
@@ -627,12 +632,13 @@ def test_custom_inputs_models(
632
633
634 #### Tests filtering for things running each test as a new process
630 -@pytest.mark.parametrize("model_type,test_case",
631 - get_parametrized_options(
632 - VLM_TEST_SETTINGS,
633 - test_type=VLMTestType.IMAGE,
634 - fork_new_process_for_each_test=True,
635 - ))
635 +@pytest.mark.parametrize(
636 + "model_type,test_case",
637 + get_parametrized_options(
638 + VLM_TEST_SETTINGS,
639 + test_type=VLMTestType.IMAGE,
640 + fork_new_process_for_each_test=True,
641 + ))
642 @fork_new_process_for_each_test
643 def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,
644 test_case: ExpandableVLMTestArgs,
@@ -650,12 +656,13 @@ def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,
656 )
657
658
653 -@pytest.mark.parametrize("model_type,test_case",
654 - get_parametrized_options(
655 - VLM_TEST_SETTINGS,
656 - test_type=VLMTestType.MULTI_IMAGE,
657 - fork_new_process_for_each_test=True,
658 - ))
659 +@pytest.mark.parametrize(
660 + "model_type,test_case",
661 + get_parametrized_options(
662 + VLM_TEST_SETTINGS,
663 + test_type=VLMTestType.MULTI_IMAGE,
664 + fork_new_process_for_each_test=True,
665 + ))
666 @fork_new_process_for_each_test
667 def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,
668 test_case: ExpandableVLMTestArgs,
@@ -673,12 +680,13 @@ def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,
680 )
681
682
676 -@pytest.mark.parametrize("model_type,test_case",
677 - get_parametrized_options(
678 - VLM_TEST_SETTINGS,
679 - test_type=VLMTestType.EMBEDDING,
680 - fork_new_process_for_each_test=True,
681 - ))
683 +@pytest.mark.parametrize(
684 + "model_type,test_case",
685 + get_parametrized_options(
686 + VLM_TEST_SETTINGS,
687 + test_type=VLMTestType.EMBEDDING,
688 + fork_new_process_for_each_test=True,
689 + ))
690 @fork_new_process_for_each_test
691 def test_image_embedding_models_heavy(model_type: str,
692 test_case: ExpandableVLMTestArgs,
@@ -695,12 +703,13 @@ def test_image_embedding_models_heavy(model_type: str,
703 )
704
705
698 -@pytest.mark.parametrize("model_type,test_case",
699 - get_parametrized_options(
700 - VLM_TEST_SETTINGS,
701 - test_type=VLMTestType.VIDEO,
702 - fork_new_process_for_each_test=True,
703 - ))
706 +@pytest.mark.parametrize(
707 + "model_type,test_case",
708 + get_parametrized_options(
709 + VLM_TEST_SETTINGS,
710 + test_type=VLMTestType.VIDEO,
711 + fork_new_process_for_each_test=True,
712 + ))
713 def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,
714 hf_runner: Type[HfRunner],
715 vllm_runner: Type[VllmRunner],
@@ -715,12 +724,13 @@ def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,
724 )
725
726
718 -@pytest.mark.parametrize("model_type,test_case",
719 - get_parametrized_options(
720 - VLM_TEST_SETTINGS,
721 - test_type=VLMTestType.CUSTOM_INPUTS,
722 - fork_new_process_for_each_test=True,
723 - ))
727 +@pytest.mark.parametrize(
728 + "model_type,test_case",
729 + get_parametrized_options(
730 + VLM_TEST_SETTINGS,
731 + test_type=VLMTestType.CUSTOM_INPUTS,
732 + fork_new_process_for_each_test=True,
733 + ))
734 @fork_new_process_for_each_test
735 def test_custom_inputs_models_heavy(
736 model_type: str,
tests/models/decoder_only/vision_language/test_pixtral.py
+8 -9
@@ -135,10 +135,10 @@ def _dump_outputs_w_logprobs(
135 outputs: OutputsLogprobs,
136 filename: "StrPath",
137 ) -> None:
138 - json_data = [(tokens, text,
139 - [{k: asdict(v)
140 - for k, v in token_logprobs.items()}
141 - for token_logprobs in (logprobs or [])])
138 + json_data = [(tokens, text, [{
139 + k: asdict(v)
140 + for k, v in token_logprobs.items()
141 + } for token_logprobs in (logprobs or [])])
142 for tokens, text, logprobs in outputs]
143
144 with open(filename, "w") as f:
@@ -149,11 +149,10 @@ def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
149 with open(filename, "rb") as f:
150 json_data = json.load(f)
151
152 - return [(tokens, text,
153 - [{int(k): Logprob(**v)
154 - for k, v in token_logprobs.items()}
155 - for token_logprobs in logprobs])
156 - for tokens, text, logprobs in json_data]
152 + return [(tokens, text, [{
153 + int(k): Logprob(**v)
154 + for k, v in token_logprobs.items()
155 + } for token_logprobs in logprobs]) for tokens, text, logprobs in json_data]
156
157
158 @large_gpu_test(min_gb=80)
tests/quantization/test_compressed_tensors.py
+3 -3
@@ -314,9 +314,9 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
314
315
316 @pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
317 -@pytest.mark.skipif(not sparse_cutlass_supported(),
318 - reason="2of4 Sparse is not yet supported on this GPU type."
319 - )
317 +@pytest.mark.skipif(
318 + not sparse_cutlass_supported(),
319 + reason="2of4 Sparse is not yet supported on this GPU type.")
320 @pytest.mark.parametrize(
321 "args_2of4",
322 [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
tests/samplers/test_rejection_sampler.py
+8 -7
@@ -23,16 +23,17 @@ def mock_causal_accepted_tensor(
23 """
24 batch_size = last_accepted_indices.shape[0]
25
26 - accepted = (torch.arange(k).expand(batch_size, k) <=
27 - last_accepted_indices.unsqueeze(-1).broadcast_to(
26 + accepted = (torch.arange(k).expand(batch_size, k)
27 + <= last_accepted_indices.unsqueeze(-1).broadcast_to(
28 batch_size, k))
29
30 # Sprinkle accepted values after the contiguous initial accepted values.
31 # This replicates the behavior of rejection sampling, which may "accept"
32 # a token that cannot be accepted because of causality.
33 - sprinkle_candidates = (
34 - torch.arange(k).expand(batch_size, k) >
35 - last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) + 1)
33 + sprinkle_candidates = (torch.arange(k).expand(
34 + batch_size,
35 + k) > last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) +
36 + 1)
37 sprinkle = torch.rand(batch_size, k) > 0.5
38 accepted[sprinkle_candidates] = sprinkle[sprinkle_candidates]
39 return accepted
@@ -445,8 +446,8 @@ def test_rejection_sampling_approximates_target_distribution(
446 distance_wrt_reference)
447
448 expected_improvement_multiplier = 20
448 - assert (relative_change_in_distance_wrt_target >
449 - relative_change_in_distance_wrt_reference *
449 + assert (relative_change_in_distance_wrt_target
450 + > relative_change_in_distance_wrt_reference *
451 expected_improvement_multiplier)
452
453
tools/report_build_time_ninja.py
+3 -2
@@ -274,8 +274,9 @@ def SummarizeEntries(entries, extra_step_types):
274 print(' {:.1f} s weighted time ({:.1f} s elapsed time sum, {:1.1f}x '
275 'parallelism)'.format(length, total_cpu_time,
276 total_cpu_time * 1.0 / length))
277 - print(' %d build steps completed, average of %1.2f/s' %
278 - (len(entries), len(entries) / (length)))
277 + print(' {} build steps completed, average of {:1.2f}/s'.format(
278 + len(entries),
279 + len(entries) / (length)))
280
281
282 def main():
vllm/_custom_ops.py
+2 -2
@@ -820,8 +820,8 @@ def scaled_int8_quant(
820 if scale is not None:
821 # static-per-tensor quantization.
822 assert symmetric == (
823 - azp is
824 - None), "azp must only be provided for asymmetric quantization."
823 + azp
824 + is None), "azp must only be provided for asymmetric quantization."
825 torch.ops._C.static_scaled_int8_quant(output, input, scale, azp)
826 return output, scale, azp
827
vllm/attention/ops/prefix_prefill.py
+14 -14
@@ -219,8 +219,8 @@ if triton.__version__ >= "2.1.0":
219 float("-inf"))
220 if SLIDING_WINDOW > 0:
221 qk = tl.where(
222 - offs_m[:, None] -
223 - (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, -10000)
222 + offs_m[:, None] - (start_n + offs_n[None, :])
223 + < SLIDING_WINDOW, qk, -10000)
224
225 # -- compute m_ij, p, l_ij
226 m_ij = tl.max(qk, 1)
@@ -324,10 +324,10 @@ if triton.__version__ >= "2.1.0":
324 (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
325 cur_head * stride_qh + offs_d[None, :] * stride_qd)
326
327 - q = tl.load(
328 - Q + off_q,
329 - mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
330 - other=0.0)
327 + q = tl.load(Q + off_q,
328 + mask=offs_m[:, None]
329 + < cur_batch_seq_len - cur_batch_ctx_len,
330 + other=0.0)
331
332 # # initialize pointer to m and l
333 m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
@@ -402,8 +402,8 @@ if triton.__version__ >= "2.1.0":
402 # -- compute qk ----
403 k = tl.load(k_ptrs +
404 (cur_batch_in_all_start_index + start_n) * stride_kbs,
405 - mask=(start_n + offs_n[None, :]) <
406 - cur_batch_seq_len - cur_batch_ctx_len,
405 + mask=(start_n + offs_n[None, :])
406 + < cur_batch_seq_len - cur_batch_ctx_len,
407 other=0.0)
408
409 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
@@ -430,8 +430,8 @@ if triton.__version__ >= "2.1.0":
430 # update acc
431 v = tl.load(v_ptrs +
432 (cur_batch_in_all_start_index + start_n) * stride_vbs,
433 - mask=(start_n + offs_n[:, None]) <
434 - cur_batch_seq_len - cur_batch_ctx_len,
433 + mask=(start_n + offs_n[:, None])
434 + < cur_batch_seq_len - cur_batch_ctx_len,
435 other=0.0)
436
437 p = p.to(v.dtype)
@@ -639,8 +639,8 @@ if triton.__version__ >= "2.1.0":
639 k = tl.load(k_ptrs +
640 (cur_batch_in_all_start_index + start_n) * stride_kbs,
641 mask=dim_mask[:, None] &
642 - ((start_n + offs_n[None, :]) <
643 - cur_batch_seq_len - cur_batch_ctx_len),
642 + ((start_n + offs_n[None, :])
643 + < cur_batch_seq_len - cur_batch_ctx_len),
644 other=0.0)
645
646 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
@@ -677,8 +677,8 @@ if triton.__version__ >= "2.1.0":
677 v = tl.load(v_ptrs +
678 (cur_batch_in_all_start_index + start_n) * stride_vbs,
679 mask=dim_mask[None, :] &
680 - ((start_n + offs_n[:, None]) <
681 - cur_batch_seq_len - cur_batch_ctx_len),
680 + ((start_n + offs_n[:, None])
681 + < cur_batch_seq_len - cur_batch_ctx_len),
682 other=0.0)
683 p = p.to(v.dtype)
684
vllm/attention/ops/triton_flash_attention.py
+2 -2
@@ -627,8 +627,8 @@ def attn_fwd(
627 causal_start_idx,
628 dtype=tl.int32)
629 mask_m_offsets = start_m_idx + tl.arange(0, BLOCK_M)
630 - out_ptrs_mask = (mask_m_offsets[:, None] >=
631 - out_mask_boundary[None, :])
630 + out_ptrs_mask = (mask_m_offsets[:, None]
631 + >= out_mask_boundary[None, :])
632 z = 0.0
633 acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty))
634 # write back LSE
vllm/attention/selector.py
+2 -2
@@ -1,6 +1,6 @@
1 import os
2 from contextlib import contextmanager
3 -from functools import lru_cache
3 +from functools import cache
4 from typing import Generator, Optional, Type
5
6 import torch
@@ -100,7 +100,7 @@ def get_attn_backend(
100 )
101
102
103 -@lru_cache(maxsize=None)
103 +@cache
104 def _cached_get_attn_backend(
105 head_size: int,
106 dtype: torch.dtype,
vllm/config.py
+4 -3
@@ -67,7 +67,8 @@ _RUNNER_TASKS: Dict[RunnerType, List[_ResolvedTask]] = {
67
68 _TASK_RUNNER: Dict[_ResolvedTask, RunnerType] = {
69 task: runner
70 - for runner, tasks in _RUNNER_TASKS.items() for task in tasks
70 + for runner, tasks in _RUNNER_TASKS.items()
71 + for task in tasks
72 }
73
74 HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
@@ -1976,8 +1977,8 @@ class SpeculativeConfig:
1977 "typical_acceptance_sampler.")
1978
1979 if (self.draft_token_acceptance_method != 'rejection_sampler'
1979 - and self.draft_token_acceptance_method !=
1980 - 'typical_acceptance_sampler'):
1980 + and self.draft_token_acceptance_method
1981 + != 'typical_acceptance_sampler'):
1982 raise ValueError(
1983 "Expected draft_token_acceptance_method to be either "
1984 "rejection_sampler or typical_acceptance_sampler. Instead it "
vllm/core/block/common.py
+4 -3
@@ -34,9 +34,10 @@ class RefCounter(RefCounterProtocol):
34
35 def __init__(self, all_block_indices: Iterable[BlockId]):
36 deduped = set(all_block_indices)
37 - self._refcounts: Dict[BlockId,
38 - RefCount] = {index: 0
39 - for index in deduped}
37 + self._refcounts: Dict[BlockId, RefCount] = {
38 + index: 0
39 + for index in deduped
40 + }
41
42 def incr(self, block_id: BlockId) -> RefCount:
43 assert block_id in self._refcounts
vllm/core/block_manager.py
+2 -2
@@ -136,8 +136,8 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
136 device=Device.GPU)
137
138 # Use watermark to avoid frequent cache eviction.
139 - if (self.num_total_gpu_blocks - num_required_blocks <
140 - self.watermark_blocks):
139 + if (self.num_total_gpu_blocks - num_required_blocks
140 + < self.watermark_blocks):
141 return AllocStatus.NEVER
142 if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
143 return AllocStatus.OK
vllm/core/scheduler.py
+11 -12
@@ -988,8 +988,8 @@ class Scheduler:
988 waiting_queue.popleft()
989 continue
990
991 - if (budget.num_batched_tokens >=
992 - self.scheduler_config.max_num_batched_tokens):
991 + if (budget.num_batched_tokens
992 + >= self.scheduler_config.max_num_batched_tokens):
993 # We've reached the budget limit - since there might be
994 # continuous prefills in the running queue, we should break
995 # to avoid scheduling any new prefills.
@@ -1096,8 +1096,8 @@ class Scheduler:
1096 running_scheduled.swapped_out) == 0:
1097 swapped_in = self._schedule_swapped(budget, curr_loras)
1098
1099 - assert (budget.num_batched_tokens <=
1100 - self.scheduler_config.max_num_batched_tokens)
1099 + assert (budget.num_batched_tokens
1100 + <= self.scheduler_config.max_num_batched_tokens)
1101 assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
1102
1103 # Update waiting requests.
@@ -1189,8 +1189,8 @@ class Scheduler:
1189 curr_loras,
1190 enable_chunking=True)
1191
1192 - assert (budget.num_batched_tokens <=
1193 - self.scheduler_config.max_num_batched_tokens)
1192 + assert (budget.num_batched_tokens
1193 + <= self.scheduler_config.max_num_batched_tokens)
1194 assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
1195
1196 # Update waiting requests.
@@ -1358,8 +1358,8 @@ class Scheduler:
1358 # NOTE: We use get_len instead of get_prompt_len because when
1359 # a sequence is preempted, prefill includes previous generated
1360 # output tokens.
1361 - if (token_chunk_size + num_computed_tokens <
1362 - seqs[0].data.get_len()):
1361 + if (token_chunk_size + num_computed_tokens
1362 + < seqs[0].data.get_len()):
1363 do_sample = False
1364
1365 # It assumes the scheduled_seq_groups is ordered by
@@ -1625,10 +1625,9 @@ class Scheduler:
1625 if self.scheduler_config.delay_factor > 0 and self.waiting:
1626 earliest_arrival_time = min(
1627 [e.metrics.arrival_time for e in self.waiting])
1628 - passed_delay = (
1629 - (now - earliest_arrival_time) >
1630 - (self.scheduler_config.delay_factor * self.last_prompt_latency)
1631 - or not self.running)
1628 + passed_delay = ((now - earliest_arrival_time)
1629 + > (self.scheduler_config.delay_factor *
1630 + self.last_prompt_latency) or not self.running)
1631 else:
1632 passed_delay = True
1633 return passed_delay
vllm/distributed/device_communicators/shm_broadcast.py
+4 -4
@@ -352,8 +352,8 @@ class MessageQueue:
352 sched_yield()
353
354 # if we wait for a long time, log a message
355 - if (time.monotonic() - start_time >
356 - VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
355 + if (time.monotonic() - start_time
356 + > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
357 logger.debug("No available block found in %s second. ",
358 VLLM_RINGBUFFER_WARNING_INTERVAL)
359 n_warning += 1
@@ -410,8 +410,8 @@ class MessageQueue:
410 sched_yield()
411
412 # if we wait for a long time, log a message
413 - if (time.monotonic() - start_time >
414 - VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
413 + if (time.monotonic() - start_time
414 + > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
415 logger.debug("No available block found in %s second. ",
416 VLLM_RINGBUFFER_WARNING_INTERVAL)
417 n_warning += 1
vllm/distributed/parallel_state.py
+4 -4
@@ -1014,8 +1014,8 @@ def initialize_model_parallel(
1014 backend = backend or torch.distributed.get_backend(
1015 get_world_group().device_group)
1016
1017 - if (world_size !=
1018 - tensor_model_parallel_size * pipeline_model_parallel_size):
1017 + if (world_size
1018 + != tensor_model_parallel_size * pipeline_model_parallel_size):
1019 raise RuntimeError(
1020 f"world_size ({world_size}) is not equal to "
1021 f"tensor_model_parallel_size ({tensor_model_parallel_size}) x "
@@ -1069,8 +1069,8 @@ def ensure_kv_transfer_initialized(vllm_config: "VllmConfig") -> None:
1069 return
1070
1071 if all([
1072 - vllm_config.kv_transfer_config.need_kv_parallel_group,
1073 - _KV_TRANSFER is None
1072 + vllm_config.kv_transfer_config.need_kv_parallel_group, _KV_TRANSFER
1073 + is None
1074 ]):
1075 _KV_TRANSFER = kv_transfer.KVTransferAgent(
1076 rank=get_world_group().rank,
vllm/entrypoints/chat_utils.py
+2 -2
@@ -3,7 +3,7 @@ import codecs
3 import json
4 from abc import ABC, abstractmethod
5 from collections import defaultdict, deque
6 -from functools import lru_cache, partial
6 +from functools import cache, lru_cache, partial
7 from pathlib import Path
8 from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9 Literal, Optional, Tuple, TypeVar, Union, cast)
@@ -377,7 +377,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
377 return self._model_config.allowed_local_media_path
378
379 @staticmethod
380 - @lru_cache(maxsize=None)
380 + @cache
381 def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
382 return tokenizer.decode(token_index)
383
vllm/entrypoints/openai/serving_completion.py
+4 -5
@@ -522,11 +522,10 @@ class OpenAIServingCompletion(OpenAIServing):
522 out_top_logprobs.append({
523 # Convert float("-inf") to the
524 # JSON-serializable float that OpenAI uses
525 - self._get_decoded_token(
526 - top_lp[1],
527 - top_lp[0],
528 - tokenizer,
529 - return_as_token_id=self.return_tokens_as_token_ids):
525 + self._get_decoded_token(top_lp[1],
526 + top_lp[0],
527 + tokenizer,
528 + return_as_token_id=self.return_tokens_as_token_ids):
529 max(top_lp[1].logprob, -9999.0)
530 for i, top_lp in enumerate(step_top_logprobs.items())
531 if num_output_top_logprobs >= i
vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py
+2 -2
@@ -62,8 +62,8 @@ class Granite20bFCToolParser(ToolParser):
62 start_of_json = match.end()
63 # end_index == the start of the next function call
64 # (if exists)
65 - next_function_call_start = (matches[i + 1].start()
66 - if i + 1 < len(matches) else None)
65 + next_function_call_start = (matches[i + 1].start() if i +
66 + 1 < len(matches) else None)
67
68 raw_function_calls.append(
69 dec.raw_decode(
vllm/lora/layers.py
+8 -4
@@ -220,8 +220,10 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
220 lora_b.T, non_blocking=True)
221 if embeddings_tensor is not None:
222 self.embeddings_tensors[
223 - index, :embeddings_tensor.shape[0], :embeddings_tensor.
224 - shape[1], ].copy_(embeddings_tensor, non_blocking=True)
223 + index,
224 + :embeddings_tensor.shape[0],
225 + :embeddings_tensor.shape[1],
226 + ].copy_(embeddings_tensor, non_blocking=True)
227 if self.embeddings_slice is not None:
228 # TODO(yard1): Optimize this copy, we don't need to copy
229 # everything, just the modified part
@@ -1024,8 +1026,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1026 lora_b.T, non_blocking=True)
1027 if embeddings_tensor is not None:
1028 self.embeddings_tensors[
1027 - index, :embeddings_tensor.shape[0], :embeddings_tensor.
1028 - shape[1], ] = embeddings_tensor
1029 + index,
1030 + :embeddings_tensor.shape[0],
1031 + :embeddings_tensor.shape[1],
1032 + ] = embeddings_tensor
1033
1034 def _get_logits(
1035 self,
vllm/lora/models.py
+3 -2
@@ -75,8 +75,9 @@ class LoRAModel(AdapterModel):
75 # Scaling factor for long context lora model. None if it is not
76 # fine tuned for the long context.
77 self.scaling_factor = scaling_factor
78 - assert (lora_model_id >
79 - 0), f"a valid lora id should be greater than 0, got {self.id}"
78 + assert (
79 + lora_model_id
80 + > 0), f"a valid lora id should be greater than 0, got {self.id}"
81 self.rank = rank
82 self.loras: Dict[str, LoRALayerWeights] = loras
83
vllm/lora/ops/triton_ops/sgmv_expand.py
+2 -3
@@ -136,9 +136,8 @@ def _sgmv_expand_kernel(
136 c_ptr = (out_ptr + offset_cm[:, None] * output_d0_stride +
137 offset_cn[None, :] * output_d1_stride)
138 M = tl.load(seq_lens + cur_batch)
139 - c_mask = (offset_cm[:, None] <
140 - (cur_seq_start + M)) & (offset_cn[None, :] <
141 - (cur_slice_start + curr_N))
139 + c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (
140 + offset_cn[None, :] < (cur_slice_start + curr_N))
141 if ADD_INPUTS:
142 tiled_out = tl.load(c_ptr, mask=c_mask)
143 tiled_c += tiled_out
vllm/lora/ops/triton_ops/sgmv_shrink.py
+2 -2
@@ -114,8 +114,8 @@ def _sgmv_shrink_kernel(
114 slice_id * output_d0_stride)
115 c_ptr = cur_out_ptr + offset_cm[:, None] * output_d1_stride + offset_cn[
116 None, :] * output_d2_stride
117 - c_mask = (offset_cm[:, None] <
118 - (cur_seq_start + M)) & (offset_cn[None, :] < N)
117 + c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :]
118 + < N)
119 accumulator *= scaling
120 # handles write-back with reduction-splitting
121 if SPLIT_K == 1:
vllm/model_executor/layers/quantization/kernels/mixed_precision/MPLinearKernel.py
+6 -6
@@ -73,12 +73,12 @@ class MPLinearKernel(ABC):
73 torch.nn.Parameter(new_param.data, requires_grad=False))
74
75 def _get_weight_params(
76 - self, layer: torch.nn.Module
77 - ) -> Tuple[torch.Tensor, # w_q
78 - torch.Tensor, # w_s
79 - Optional[torch.Tensor], # w_zp,
80 - Optional[torch.Tensor] # w_gidx
81 - ]:
76 + self, layer: torch.nn.Module) -> Tuple[
77 + torch.Tensor, # w_q
78 + torch.Tensor, # w_s
79 + Optional[torch.Tensor], # w_zp,
80 + Optional[torch.Tensor] # w_gidx
81 + ]:
82 return (
83 getattr(layer, self.w_q_name),
84 getattr(layer, self.w_s_name),
vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py
+7 -7
@@ -48,13 +48,13 @@ class ScaledMMLinearKernel(ABC):
48 raise NotImplementedError
49
50 def _get_weight_params(
51 - self, layer: torch.nn.Module
52 - ) -> Tuple[torch.Tensor, # weight
53 - torch.Tensor, # weight_scale
54 - Optional[torch.Tensor], # input_scale,
55 - Optional[torch.Tensor], # input_zp
56 - Optional[torch.Tensor], # azp_adj
57 - ]:
51 + self, layer: torch.nn.Module) -> Tuple[
52 + torch.Tensor, # weight
53 + torch.Tensor, # weight_scale
54 + Optional[torch.Tensor], # input_scale,
55 + Optional[torch.Tensor], # input_zp
56 + Optional[torch.Tensor], # azp_adj
57 + ]:
58 return (
59 getattr(layer, self.w_q_name),
60 getattr(layer, self.w_s_name),
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+4 -3
@@ -72,9 +72,10 @@ def block_quant_to_tensor_quant(
72 x_dq_block = x_q_block.to(torch.float32)
73
74 x_dq_block_tiles = [[
75 - x_dq_block[j * block_n:min((j + 1) * block_n, n),
76 - i * block_k:min((i + 1) * block_k, k), ]
77 - for i in range(k_tiles)
75 + x_dq_block[
76 + j * block_n:min((j + 1) * block_n, n),
77 + i * block_k:min((i + 1) * block_k, k),
78 + ] for i in range(k_tiles)
79 ] for j in range(n_tiles)]
80
81 for i in range(k_tiles):
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2 -2
@@ -73,8 +73,8 @@ def requantize_with_max_scale(
73 # from disk in this case. Skip requantization in this case (since)
74 # we already are quantized with the single scale.
75 # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8
76 - unfused_module_in_checkpoint = (weight_scale[-1] > torch.finfo(
77 - torch.float8_e4m3fn).min)
76 + unfused_module_in_checkpoint = (weight_scale[-1]
77 + > torch.finfo(torch.float8_e4m3fn).min)
78
79 # If unfused checkpoint, need requanize with the single scale.
80 if unfused_module_in_checkpoint:
vllm/model_executor/layers/sampler.py
+4 -3
@@ -716,9 +716,10 @@ def _sample_with_torch(
716 tensors required for Pythonization
717 '''
718
719 - categorized_seq_group_ids: Dict[SamplingType,
720 - List[int]] = {t: []
721 - for t in SamplingType}
719 + categorized_seq_group_ids: Dict[SamplingType, List[int]] = {
720 + t: []
721 + for t in SamplingType
722 + }
723 categorized_sample_indices = sampling_metadata.categorized_sample_indices
724 for i, seq_group in enumerate(sampling_metadata.seq_groups):
725 sampling_params = seq_group.sampling_params
vllm/model_executor/layers/vocab_parallel_embedding.py
+8 -8
@@ -115,17 +115,17 @@ class VocabParallelEmbeddingShardIndices:
115
116 def __post_init__(self):
117 # sanity checks
118 - assert (self.padded_org_vocab_start_index <=
119 - self.padded_org_vocab_end_index)
120 - assert (self.padded_added_vocab_start_index <=
121 - self.padded_added_vocab_end_index)
118 + assert (self.padded_org_vocab_start_index
119 + <= self.padded_org_vocab_end_index)
120 + assert (self.padded_added_vocab_start_index
121 + <= self.padded_added_vocab_end_index)
122
123 assert self.org_vocab_start_index <= self.org_vocab_end_index
124 assert self.added_vocab_start_index <= self.added_vocab_end_index
125
126 assert self.org_vocab_start_index <= self.padded_org_vocab_start_index
127 - assert (self.added_vocab_start_index <=
128 - self.padded_added_vocab_start_index)
127 + assert (self.added_vocab_start_index
128 + <= self.padded_added_vocab_start_index)
129 assert self.org_vocab_end_index <= self.padded_org_vocab_end_index
130 assert self.added_vocab_end_index <= self.padded_added_vocab_end_index
131
@@ -141,8 +141,8 @@ def get_masked_input_and_mask(
141 added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]:
142 # torch.compile will fuse all of the pointwise ops below
143 # into a single kernel, making it very fast
144 - org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ <
145 - org_vocab_end_index)
144 + org_vocab_mask = (input_ >= org_vocab_start_index) & (
145 + input_ < org_vocab_end_index)
146 added_vocab_mask = (input_ >= added_vocab_start_index) & (
147 input_ < added_vocab_end_index)
148 added_offset = added_vocab_start_index - (
vllm/model_executor/model_loader/loader.py
+3 -2
@@ -1121,8 +1121,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1121 # from being incorrectly identified as being present in
1122 # 'vpm.encoder.layers.0.self_attn.qkv_proj.weight
1123 shard_pos = quant_param_name.find(shard_name)
1124 - can_correct_rename = (shard_pos > 0) and (
1125 - quant_param_name[shard_pos - 1] == ".")
1124 + can_correct_rename = (shard_pos
1125 + > 0) and (quant_param_name[shard_pos - 1]
1126 + == ".")
1127 # If the quant_param_name is packed, it won't occur in the
1128 # param_dict before renaming.
1129 new_quant_param_name = quant_param_name.replace(
vllm/model_executor/model_loader/tensorizer.py
+2 -2
@@ -298,8 +298,8 @@ class TensorizerAgent:
298 to allow for adapter added tokens."""
299 for child in self.model.modules():
300 if (isinstance(child, VocabParallelEmbedding)
301 - and child.weight.shape[0] <
302 - child.num_embeddings_per_partition):
301 + and child.weight.shape[0]
302 + < child.num_embeddings_per_partition):
303 new_weight = torch.empty(child.num_embeddings_per_partition,
304 child.embedding_dim,
305 dtype=child.weight.dtype,
vllm/model_executor/models/gemma.py
+2 -2
@@ -13,7 +13,7 @@
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """Inference-only Gemma model compatible with HuggingFace weights."""
16 -from functools import lru_cache
16 +from functools import cache
17 from typing import Iterable, List, Optional, Set, Tuple, Union
18
19 import torch
@@ -48,7 +48,7 @@ from .utils import (is_pp_missing_parameter,
48 logger = init_logger(__name__)
49
50
51 -@lru_cache(maxsize=None)
51 +@cache
52 def _get_gemma_act_fn(
53 hidden_act: Optional[str],
54 hidden_activation: Optional[str],
vllm/model_executor/models/granitemoe.py
+3 -3
@@ -429,10 +429,10 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
429 for e in range(p.size(0)):
430 w1_name = n.replace(
431 '.block_sparse_moe.input_linear.weight',
432 - ".block_sparse_moe.experts.%d.w1.weight" % e)
432 + f".block_sparse_moe.experts.{e}.w1.weight")
433 w3_name = n.replace(
434 '.block_sparse_moe.input_linear.weight',
435 - ".block_sparse_moe.experts.%d.w3.weight" % e)
435 + f".block_sparse_moe.experts.{e}.w3.weight")
436 w1_param, w3_param = p[e].chunk(2, dim=0)
437 assert w1_name not in new_weights
438 assert w3_name not in new_weights
@@ -442,7 +442,7 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
442 for e in range(p.size(0)):
443 w2_name = n.replace(
444 '.block_sparse_moe.output_linear.weight',
445 - ".block_sparse_moe.experts.%d.w2.weight" % e)
445 + f".block_sparse_moe.experts.{e}.w2.weight")
446 w2_param = p[e]
447 assert w2_name not in new_weights
448 new_weights[w2_name] = w2_param
vllm/model_executor/models/mllama.py
+2 -2
@@ -1365,8 +1365,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1365 # For 1) text-only prefill and decode, 2) image-present decode.
1366 if image_inputs is None:
1367 full_text_row_masked_out_mask = (
1368 - attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to(
1369 - input_ids.device)
1368 + attn_metadata.encoder_seq_lens_tensor
1369 + != 0).reshape(-1, 1).to(input_ids.device)
1370 skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1371
1372 # For image-present prefill.
vllm/model_executor/models/mlp_speculator.py
+2 -2
@@ -81,8 +81,8 @@ class MLPSpeculator(nn.Module):
81
82 if self.tie_weights:
83 assert (
84 - self.n_predict >
85 - 1), "You cannot tie weights between stages when only 1 exists"
84 + self.n_predict > 1
85 + ), "You cannot tie weights between stages when only 1 exists"
86 embedding = VocabParallelEmbedding(
87 config.vocab_size,
88 self.inner_dim,
vllm/model_executor/models/phimoe.py
+4 -4
@@ -167,8 +167,8 @@ def sparsemixer(scores, jitter_eps=0.01):
167 # compute mask for sparsity
168 mask_logits_threshold, max_ind = scores.max(dim=-1, keepdim=True)
169 factor = scores.abs().clamp(min=mask_logits_threshold)
170 - mask_logits_threshold = (
171 - (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
170 + mask_logits_threshold = ((mask_logits_threshold - scores) /
171 + factor) > (2 * jitter_eps)
172
173 # apply mask
174 masked_gates = scores.masked_fill(mask_logits_threshold, float("-inf"))
@@ -192,8 +192,8 @@ def sparsemixer(scores, jitter_eps=0.01):
192 mask_logits_threshold, max_ind = masked_scores.max(dim=-1,
193 keepdim=True)
194 factor = scores.abs().clamp(min=mask_logits_threshold)
195 - mask_logits_threshold = (
196 - (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
195 + mask_logits_threshold = ((mask_logits_threshold - scores) /
196 + factor) > (2 * jitter_eps)
197
198 # apply mask
199 masked_gates_top2 = masked_scores.masked_fill(mask_logits_threshold,
vllm/model_executor/models/registry.py
+2 -1
@@ -462,7 +462,8 @@ class _ModelRegistry:
462
463
464 ModelRegistry = _ModelRegistry({
465 - model_arch: _LazyRegisteredModel(
465 + model_arch:
466 + _LazyRegisteredModel(
467 module_name=f"vllm.model_executor.models.{mod_relname}",
468 class_name=cls_name,
469 )
vllm/model_executor/models/ultravox.py
+4 -4
@@ -333,10 +333,10 @@ class ModifiedWhisperEncoder(WhisperEncoder):
333 return hidden_states
334
335
336 -@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor,
337 - info=UltravoxProcessingInfo,
338 - dummy_inputs=UltravoxDummyInputsBuilder
339 - )
336 +@MULTIMODAL_REGISTRY.register_processor(
337 + UltravoxMultiModalProcessor,
338 + info=UltravoxProcessingInfo,
339 + dummy_inputs=UltravoxDummyInputsBuilder)
340 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
341
342 hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/utils.py
+2 -3
@@ -599,9 +599,8 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
599 device: torch.device,
600 ) -> IntermediateTensors:
601 return IntermediateTensors({
602 - key: torch.zeros((batch_size, hidden_size),
603 - dtype=dtype,
604 - device=device)
602 + key:
603 + torch.zeros((batch_size, hidden_size), dtype=dtype, device=device)
604 for key in keys
605 })
606
vllm/model_executor/sampling_metadata.py
+8 -3
@@ -166,7 +166,8 @@ class SamplingMetadata:
166 pin_memory=pin_memory,
167 )
168 categorized_sample_indices = {
169 - t: async_tensor_h2d(
169 + t:
170 + async_tensor_h2d(
171 seq_ids,
172 dtype=torch.int,
173 target_device=device,
@@ -198,8 +199,12 @@ def _prepare_seq_groups(
199 device: str,
200 generators: Optional[Dict[str, torch.Generator]] = None,
201 cache: Optional[SamplingMetadataCache] = None,
201 -) -> Tuple[List[SequenceGroupToSample], List[int], Dict[SamplingType,
202 - List[int]], int, ]:
202 +) -> Tuple[
203 + List[SequenceGroupToSample],
204 + List[int],
205 + Dict[SamplingType, List[int]],
206 + int,
207 +]:
208 """Prepare sequence groups and indices for sampling.
209
210 Args:
vllm/platforms/neuron.py
+2 -2
@@ -38,8 +38,8 @@ class NeuronPlatform(Platform):
38 if parallel_config.world_size > 1:
39 parallel_config.distributed_executor_backend = "uni"
40
41 - assert (vllm_config.lora_config is
42 - None), "LoRA is not supported for Neuron backend."
41 + assert (vllm_config.lora_config
42 + is None), "LoRA is not supported for Neuron backend."
43 assert (not vllm_config.speculative_config
44 ), "Speculative decoding not yet supported for Neuron backend."
45
vllm/scalar_type.py
+2 -2
@@ -121,8 +121,8 @@ class ScalarType:
121 min_raw = max_raw | sign_bit_double
122 return struct.unpack('!d', struct.pack('!Q', min_raw))[0]
123 else:
124 - assert (not self.is_signed() or
125 - self.size_bits <= 64), "Cannot represent min as a int64_t"
124 + assert (not self.is_signed() or self.size_bits
125 + <= 64), "Cannot represent min as a int64_t"
126
127 if self.is_signed():
128 return -(1 << (self.size_bits - 1))
vllm/spec_decode/spec_decode_worker.py
+2 -2
@@ -510,8 +510,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
510 self, execute_model_req: ExecuteModelRequest) -> bool:
511 # When the batch size is too large, disable speculative decoding
512 # to stop trading off throughput for latency.
513 - return (execute_model_req.running_queue_size >=
514 - self.disable_by_batch_size)
513 + return (execute_model_req.running_queue_size
514 + >= self.disable_by_batch_size)
515
516 def _maybe_disable_speculative_tokens(
517 self, disable_all_speculation: bool,
vllm/spec_decode/top1_proposer.py
+5 -5
@@ -104,11 +104,11 @@ class Top1Proposer(SpeculativeProposer):
104 sampler_transposed=transposed,
105 )
106
107 - proposals = SpeculativeProposals(
108 - proposal_token_ids=proposal_tokens,
109 - proposal_probs=proposal_probs,
110 - proposal_lens=proposal_lens,
111 - no_proposals=maybe_sampler_output is None)
107 + proposals = SpeculativeProposals(proposal_token_ids=proposal_tokens,
108 + proposal_probs=proposal_probs,
109 + proposal_lens=proposal_lens,
110 + no_proposals=maybe_sampler_output
111 + is None)
112 return proposals
113
114 def _split_by_proposal_len(
vllm/spec_decode/util.py
+7 -5
@@ -40,13 +40,15 @@ def get_sampled_token_logprobs(
40 """
41 num_steps, batch_size, vocab_size = logprob_tensor.shape
42
43 - selected_logprobs = logprob_tensor[torch.arange(num_steps).unsqueeze(1),
44 - torch.arange(batch_size),
45 - sampled_token_ids, ]
43 + selected_logprobs = logprob_tensor[
44 + torch.arange(num_steps).unsqueeze(1),
45 + torch.arange(batch_size),
46 + sampled_token_ids,
47 + ]
48 expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand(
49 -1, -1, vocab_size)
48 - sampled_token_ids_ranks = (logprob_tensor >
49 - expanded_selected_logprobs).sum(-1).add_(1)
50 + sampled_token_ids_ranks = (logprob_tensor
51 + > expanded_selected_logprobs).sum(-1).add_(1)
52
53 return sampled_token_ids_ranks, selected_logprobs
54
vllm/transformers_utils/configs/nemotron.py
+2 -2
@@ -182,8 +182,8 @@ class NemotronConfig(PretrainedConfig):
182 if self.rope_scaling is None:
183 return
184
185 - if not isinstance(self.rope_scaling,
186 - dict) or len(self.rope_scaling) != 2:
185 + if not isinstance(self.rope_scaling, dict) or len(
186 + self.rope_scaling) != 2:
187 raise ValueError(
188 "`rope_scaling` must be a dictionary with two fields, "
189 f"`type` and `factor`, got {self.rope_scaling}")
vllm/utils.py
+5 -5
@@ -29,7 +29,7 @@ from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
29 from collections import OrderedDict, UserDict, defaultdict
30 from collections.abc import Hashable, Iterable, Mapping
31 from dataclasses import dataclass, field
32 -from functools import lru_cache, partial, wraps
32 +from functools import cache, lru_cache, partial, wraps
33 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
34 Dict, Generator, Generic, Iterator, List, Literal,
35 NamedTuple, Optional, Tuple, Type, TypeVar, Union,
@@ -352,7 +352,7 @@ class PyObjectCache:
352 self._index = 0
353
354
355 -@lru_cache(maxsize=None)
355 +@cache
356 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
357 """Returns the maximum shared memory per thread block in bytes."""
358 from vllm import _custom_ops as ops
@@ -697,7 +697,7 @@ def create_kv_caches_with_random(
697 return key_caches, value_caches
698
699
700 -@lru_cache(maxsize=None)
700 +@cache
701 def is_pin_memory_available() -> bool:
702 from vllm.platforms import current_platform
703 return current_platform.is_pin_memory_available()
@@ -886,7 +886,7 @@ def init_cached_hf_modules() -> None:
886 init_hf_modules()
887
888
889 -@lru_cache(maxsize=None)
889 +@cache
890 def find_library(lib_name: str) -> str:
891 """
892 Find the library file in the system.
@@ -1607,7 +1607,7 @@ def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1607 return module
1608
1609
1610 -@lru_cache(maxsize=None)
1610 +@cache
1611 def get_vllm_optional_dependencies():
1612 metadata = importlib.metadata.metadata("vllm")
1613 requirements = metadata.get_all("Requires-Dist", [])
vllm/v1/core/scheduler.py
+2 -2
@@ -247,8 +247,8 @@ class Scheduler:
247 token_budget -= num_new_tokens
248 request.status = RequestStatus.RUNNING
249 request.num_computed_tokens = num_computed_tokens
250 - has_partial_request = (num_computed_tokens + num_new_tokens <
251 - request.num_tokens)
250 + has_partial_request = (num_computed_tokens + num_new_tokens
251 + < request.num_tokens)
252
253 # Encoder-related.
254 if encoder_inputs_to_schedule:
vllm/v1/stats/common.py
+2 -2
@@ -311,8 +311,8 @@ class RequestStats:
311 return []
312 latency_s_lst = []
313 for i in range(1, len(self.output_token_ts_s_lst)):
314 - assert (self.output_token_ts_s_lst[i] >=
315 - self.output_token_ts_s_lst[i - 1])
314 + assert (self.output_token_ts_s_lst[i]
315 + >= self.output_token_ts_s_lst[i - 1])
316 latency_s = (self.output_token_ts_s_lst[i] -
317 self.output_token_ts_s_lst[i - 1])
318 latency_s_lst.append(latency_s)
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -205,7 +205,7 @@ class GPUModelRunner:
205
206 def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
207 # Remove stopped requests from the cached states.
208 - # Keep the states of the pre-empted requests.
208 + # Keep the states of the preempted requests.
209 for req_id in scheduler_output.finished_req_ids:
210 self.requests.pop(req_id, None)
211 self.encoder_cache.pop(req_id, None)
vllm/worker/hpu_worker.py
+4 -4
@@ -173,13 +173,13 @@ class HPUWorker(LocalOrDistributedWorkerBase):
173 cpu_fallback_ctx as cpu_fallback_local_metric:
174 output = LocalOrDistributedWorkerBase.execute_model(
175 self, execute_model_req)
176 - if (log_graph_compilation and gc_local_metric.stats()[0][1] > 0
177 - ) or log_graph_compilation_all:
176 + if (log_graph_compilation and gc_local_metric.stats()[0][1]
177 + > 0) or log_graph_compilation_all:
178 msg = ("VLLM_HPU_STEP_GRAPH_COMPILATION: "
179 f"{gc_local_metric.stats()}, {input_stats}")
180 logger.warning(msg)
181 - if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1] >
182 - 0) or log_cpu_fallbacks_all:
181 + if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1]
182 + > 0) or log_cpu_fallbacks_all:
183 msg = ("VLLM_HPU_STEP_CPU_FALLBACK: "
184 f"{cpu_fallback_local_metric.stats()}, {input_stats}")
185 logger.warning(msg)
vllm/worker/tpu_model_runner.py
+2 -2
@@ -316,8 +316,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
316 logger.info("batch_size: %d, seq_len: %d", batch_size,
317 seq_len)
318 num_tokens = batch_size * seq_len
319 - if (num_tokens >=
320 - self.scheduler_config.max_num_batched_tokens):
319 + if (num_tokens
320 + >= self.scheduler_config.max_num_batched_tokens):
321 break
322 seq_len = seq_len * 2
323 end = time.time()