@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c04638214
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Fix per file ruff ignores related to line length (#26262)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Oct 6, 2025 at 06:12 UTC 6c04638214d413dfafa2ab1bf3f16069878e60f9
65 files changed +301 -291
benchmarks/benchmark_ngram_proposer.py
+1 -1
@@ -164,7 +164,7 @@ def invoke_main() -> None:
164 )
165 parser.add_argument(
166 "--batched", action="store_true", help="consider time to prepare batch"
167 - ) # noqa: E501
167 + )
168 parser.add_argument(
169 "--num-iteration",
170 type=int,
benchmarks/benchmark_serving_structured_output.py
+2 -2
@@ -909,13 +909,13 @@ def create_argument_parser():
909 parser.add_argument(
910 "--tokenizer",
911 type=str,
912 - help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
912 + help="Name or path of the tokenizer, if not using the default tokenizer.",
913 )
914 parser.add_argument(
915 "--tokenizer-mode",
916 type=str,
917 default="auto",
918 - help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
918 + help="Name or path of the tokenizer, if not using the default tokenizer.",
919 )
920 parser.add_argument(
921 "--num-prompts",
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
+3 -3
@@ -72,8 +72,8 @@ VLLMKernelScheduleTag: dict[
72 ] = {
73 **KernelScheduleTag, # type: ignore
74 **{
75 - MixedInputKernelScheduleType.TmaWarpSpecialized: "cutlass::gemm::KernelTmaWarpSpecialized",
76 - MixedInputKernelScheduleType.TmaWarpSpecializedPingpong: "cutlass::gemm::KernelTmaWarpSpecializedPingpong",
77 - MixedInputKernelScheduleType.TmaWarpSpecializedCooperative: "cutlass::gemm::KernelTmaWarpSpecializedCooperative",
75 + MixedInputKernelScheduleType.TmaWarpSpecialized: "cutlass::gemm::KernelTmaWarpSpecialized", # noqa: E501
76 + MixedInputKernelScheduleType.TmaWarpSpecializedPingpong: "cutlass::gemm::KernelTmaWarpSpecializedPingpong", # noqa: E501
77 + MixedInputKernelScheduleType.TmaWarpSpecializedCooperative: "cutlass::gemm::KernelTmaWarpSpecializedCooperative", # noqa: E501
78 },
79 }
examples/offline_inference/vision_language_pooling.py
+1 -1
@@ -113,7 +113,7 @@ def run_e5_v(query: Query) -> ModelRequestData:
113 def _get_vlm2vec_prompt_image(query: Query, image_token: str):
114 if query["modality"] == "text":
115 text = query["text"]
116 - prompt = f"Find me an everyday image that matches the given caption: {text}" # noqa: E501
116 + prompt = f"Find me an everyday image that matches the given caption: {text}"
117 image = None
118 elif query["modality"] == "image":
119 prompt = f"{image_token} Find a day-to-day image that looks similar to the provided image." # noqa: E501
examples/online_serving/disaggregated_serving/disagg_proxy_demo.py
+2 -2
@@ -203,9 +203,9 @@ class Proxy:
203 async with session.post(
204 url=url, json=data, headers=headers
205 ) as response:
206 - if 200 <= response.status < 300 or 400 <= response.status < 500: # noqa: E501
206 + if 200 <= response.status < 300 or 400 <= response.status < 500:
207 if use_chunked:
208 - async for chunk_bytes in response.content.iter_chunked( # noqa: E501
208 + async for chunk_bytes in response.content.iter_chunked(
209 1024
210 ):
211 yield chunk_bytes
pyproject.toml
-46
@@ -56,52 +56,6 @@ include = ["vllm*"]
56 "vllm/third_party/**" = ["ALL"]
57 "vllm/version.py" = ["F401"]
58 "vllm/_version.py" = ["ALL"]
59 -# TEMPORARY! These ignores will be fixed forward
60 -## Line length violations
61 -"csrc/cutlass_extensions/vllm_cutlass_library_extension.py" = ["E501"]
62 -"tests/compile/piecewise/test_simple.py" = ["E501"]
63 -"tests/compile/piecewise/test_toy_llama.py" = ["E501", "B023"]
64 -"tests/entrypoints/conftest.py" = ["E501"]
65 -"tests/entrypoints/openai/test_audio.py" = ["E501"]
66 -"tests/entrypoints/openai/test_chat.py" = ["E501"]
67 -"tests/entrypoints/openai/test_chat_template.py" = ["E501"]
68 -"tests/entrypoints/openai/test_chat_with_tool_reasoning.py" = ["E501"]
69 -"tests/entrypoints/openai/test_completion_with_function_calling.py" = ["E501"]
70 -"tests/entrypoints/openai/test_video.py" = ["E501"]
71 -"tests/entrypoints/openai/test_vision.py" = ["E501"]
72 -"tests/entrypoints/test_chat_utils.py" = ["E501"]
73 -"tests/kernels/moe/modular_kernel_tools/common.py" = ["E501"]
74 -"tests/models/language/generation/test_gemma.py" = ["E501"]
75 -"tests/models/language/generation/test_mistral.py" = ["E501"]
76 -"tests/models/multimodal/generation/test_ultravox.py" = ["E501"]
77 -"tests/models/multimodal/generation/test_voxtral.py" = ["E501"]
78 -"tests/models/multimodal/generation/vlm_utils/custom_inputs.py" = ["E501"]
79 -"tests/tool_use/test_tool_choice_required.py" = ["E501"]
80 -"tests/v1/attention/utils.py" = ["E501"]
81 -"tests/v1/entrypoints/openai/responses/test_image.py" = ["E501"]
82 -"tests/v1/kv_connector/nixl_integration/test_accuracy.py" = ["E501"]
83 -"tests/v1/kv_connector/unit/test_offloading_connector.py" = ["E501"]
84 -"tests/v1/logits_processors/test_custom_offline.py" = ["E501"]
85 -"vllm/attention/ops/pallas_kv_cache_update.py" = ["E501"]
86 -"vllm/compilation/collective_fusion.py" = ["E501"]
87 -"vllm/compilation/wrapper.py" = ["E501"]
88 -"vllm/config/vllm.py" = ["E501"]
89 -"vllm/distributed/device_communicators/all2all.py" = ["E501"]
90 -"vllm/entrypoints/openai/protocol.py" = ["E501"]
91 -"vllm/lora/layers/vocal_parallel_embedding.py" = ["E501"]
92 -"vllm/model_executor/model_loader/bitsandbytes_loader.py" = ["E501"]
93 -"vllm/model_executor/models/bailing_moe.py" = ["E501"]
94 -"vllm/model_executor/models/hyperclovax_vision.py" = ["E501"]
95 -"vllm/model_executor/models/llama4_eagle.py" = ["E501"]
96 -"vllm/model_executor/models/longcat_flash_mtp.py" = ["E501"]
97 -"vllm/model_executor/models/phi4mm.py" = ["E501"]
98 -"vllm/model_executor/models/qwen3_next.py" = ["E501"]
99 -"vllm/model_executor/layers/quantization/ptpc_fp8.py" = ["E501"]
100 -"vllm/v1/attention/backends/mla/common.py" = ["E501"]
101 -"vllm/v1/engine/utils.py" = ["E501"]
102 -"vllm/v1/utils.py" = ["E501"]
103 -"vllm/v1/worker/gpu_model_runner.py" = ["E501"]
104 -# End of temporary ignores
59
60 [tool.ruff.lint]
61 select = [
tests/compile/piecewise/test_simple.py
+17 -11
@@ -132,10 +132,14 @@ def test_simple_piecewise_compile(use_inductor):
132 splitting_ops=["silly.attention"],
133 use_inductor_graph_partition=False,
134 use_inductor=use_inductor,
135 - expected_num_piecewise_graphs_seen=5, # 2 * num_layers + 1
136 - expected_num_piecewise_capturable_graphs_seen=3, # 1 + num_layers
137 - expected_num_backend_compilations=3, # num_piecewise_capturable_graphs_seen
138 - expected_num_cudagraph_captured=6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
135 + # 2 * num_layers + 1
136 + expected_num_piecewise_graphs_seen=5,
137 + # 1 + num_layers
138 + expected_num_piecewise_capturable_graphs_seen=3,
139 + # num_piecewise_capturable_graphs_seen
140 + expected_num_backend_compilations=3,
141 + # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
142 + expected_num_cudagraph_captured=6,
143 )
144
145
@@ -147,14 +151,16 @@ def test_simple_inductor_graph_partition(splitting_ops):
151 pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
152
153 _run_simple_model(
150 - # inductor graph partition automatically resets splitting_ops
151 - # to be an empty list
154 + # Inductor graph partition automatically resets splitting_ops to an empty list
155 splitting_ops=splitting_ops,
156 use_inductor_graph_partition=True,
157 use_inductor=True,
155 - expected_num_piecewise_graphs_seen=1, # since not splitting at fx graph level
156 - expected_num_piecewise_capturable_graphs_seen=1, # since not splitting at fx graph level
157 - expected_num_backend_compilations=1, # since not splitting at fx graph level
158 - expected_num_cudagraph_captured=6, # inductor graph partition still captures 6
159 - # graph, same as fx graph partition.
158 + # Since not splitting at fx graph level
159 + expected_num_piecewise_graphs_seen=1,
160 + # Since not splitting at fx graph level
161 + expected_num_piecewise_capturable_graphs_seen=1,
162 + # Since not splitting at fx graph level
163 + expected_num_backend_compilations=1,
164 + # Inductor graph partition still captures 6 graph, same as fx graph partition
165 + expected_num_cudagraph_captured=6,
166 )
tests/compile/piecewise/test_toy_llama.py
+9 -5
@@ -367,11 +367,14 @@ def test_toy_llama(use_inductor: bool):
367 kwargs = {"num_eager_compiles": 1, "num_inductor_compiles": 0}
368
369 with compilation_counter.expect(
370 - num_graphs_seen=1, # one graph for the model
370 + # One graph for the model
371 + num_graphs_seen=1,
372 num_piecewise_graphs_seen=1,
373 num_piecewise_capturable_graphs_seen=1,
373 - num_backend_compilations=1, # num_piecewise_capturable_graphs_seen
374 - num_cudagraph_captured=2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
374 + # num_piecewise_capturable_graphs_seen
375 + num_backend_compilations=1,
376 + # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
377 + num_cudagraph_captured=2,
378 **kwargs,
379 ):
380 outputs.append(
@@ -478,9 +481,10 @@ def benchmark():
481 # it is fine here, because we only use the lambda function once.
482 runtime = do_bench(
483 lambda: graphs[b][0]( # noqa
481 - input_ids[:b], positions[:b]
484 + input_ids[:b], # noqa
485 + positions[:b], # noqa
486 )
483 - ) # noqa
487 + )
488 piecewise_cudagraph_time[b] = runtime
489 else:
490 runtime = do_bench(lambda: graphs[b][0].replay()) # noqa
tests/compile/test_functionalization.py
+1 -1
@@ -243,7 +243,7 @@ def test_fix_functionalization(model_class: torch.nn.Module, do_fusion: bool):
243 # check if the functionalization pass is applied
244 for op in model.ops_in_model(do_fusion):
245 find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
246 - assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None # noqa: E501
246 + assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None
247
248 # make sure the ops were all de-functionalized
249 found = dict()
tests/compile/test_fusion_attn.py
+1 -1
@@ -565,7 +565,7 @@ def test_attention_quant_pattern(
565 elif quant_key.dtype == FP4_DTYPE:
566 assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, (
567 "Attention should have output_block_scale after FP4 fusion"
568 - ) # noqa: E501
568 + )
569
570 # Check that results are close
571 torch.testing.assert_close(result_unfused, result_fused_1, atol=1e-2, rtol=1e-2)
tests/compile/test_sequence_parallelism.py
+2 -2
@@ -186,7 +186,7 @@ class TestQuantModel(torch.nn.Module):
186 ):
187 # If fusion happens, the fused op is the one
188 # we check for (de)functionalization
189 - return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default] # noqa: E501
189 + return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
190 else:
191 # If no fusion, the original ops are checked
192 return [
@@ -322,7 +322,7 @@ def sequence_parallelism_pass_on_test_model(
322 # check if the functionalization pass is applied
323 for op in model.ops_in_model():
324 find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
325 - assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None # noqa: E501
325 + assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None
326
327 # make sure the ops were all de-functionalized
328 found = dict()
tests/distributed/test_pipeline_parallel.py
+4 -4
@@ -104,7 +104,7 @@ TEXT_GENERATION_MODELS = {
104 # [Decoder-only]
105 # Uses Llama
106 # "BAAI/AquilaChat-7B": PPTestSettings.fast(),
107 - "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
107 + "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(load_format="dummy"),
108 "baichuan-inc/Baichuan-7B": PPTestSettings.fast(),
109 "baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(),
110 "bigscience/bloomz-1b1": PPTestSettings.fast(),
@@ -138,7 +138,7 @@ TEXT_GENERATION_MODELS = {
138 # Uses Llama
139 # "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
140 "state-spaces/mamba-130m-hf": PPTestSettings.fast(),
141 - "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(load_format="dummy"), # noqa: E501
141 + "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(load_format="dummy"),
142 "mosaicml/mpt-7b": PPTestSettings.fast(),
143 "nvidia/Minitron-8B-Base": PPTestSettings.fast(),
144 "allenai/OLMo-1B-hf": PPTestSettings.fast(),
@@ -151,13 +151,13 @@ TEXT_GENERATION_MODELS = {
151 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(),
152 "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(
153 multi_node_only=True, load_format="dummy"
154 - ), # noqa: E501
154 + ),
155 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(),
156 "Qwen/Qwen2.5-0.5B-Instruct": PPTestSettings.fast(),
157 "Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
158 "stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
159 "bigcode/starcoder2-3b": PPTestSettings.fast(),
160 - "upstage/solar-pro-preview-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
160 + "upstage/solar-pro-preview-instruct": PPTestSettings.fast(load_format="dummy"),
161 # FIXME: Cannot load tokenizer in latest transformers version.
162 # Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
163 # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(),
tests/entrypoints/conftest.py
+2 -1
@@ -83,7 +83,8 @@ def sample_complex_json_schema():
83 "type": "array",
84 "items": {
85 "type": "string",
86 - "pattern": "^[a-z]{1,10}$", # Combining length and pattern restrictions
86 + # Combining length and pattern restrictions
87 + "pattern": "^[a-z]{1,10}$",
88 },
89 },
90 },
tests/entrypoints/openai/test_audio.py
+1 -1
@@ -145,7 +145,7 @@ async def test_single_chat_session_audio_base64encoded(
145 {
146 "type": "audio_url",
147 "audio_url": {
148 - "url": f"data:audio/wav;base64,{base64_encoded_audio[audio_url]}"
148 + "url": f"data:audio/wav;base64,{base64_encoded_audio[audio_url]}" # noqa: E501
149 },
150 },
151 {"type": "text", "text": "What's happening in this audio?"},
tests/entrypoints/openai/test_chat.py
+7 -6
@@ -835,17 +835,18 @@ async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
835
836 @pytest.mark.asyncio
837 async def test_complex_message_content(client: openai.AsyncOpenAI):
838 + content = [
839 + {
840 + "type": "text",
841 + "text": "what is 1+1? please provide the result without any other text.",
842 + }
843 + ]
844 resp = await client.chat.completions.create(
845 model=MODEL_NAME,
846 messages=[
847 {
848 "role": "user",
843 - "content": [
844 - {
845 - "type": "text",
846 - "text": "what is 1+1? please provide the result without any other text.",
847 - }
848 - ],
849 + "content": content,
850 }
851 ],
852 temperature=0,
tests/entrypoints/openai/test_chat_template.py
+2 -2
@@ -76,8 +76,8 @@ def test_load_chat_template():
76 assert (
77 template_content
78 == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
79 -{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}"""
80 - ) # noqa: E501
79 +{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
80 + )
81
82
83 def test_no_load_chat_template_filelike():
tests/entrypoints/openai/test_chat_with_tool_reasoning.py
+6 -4
@@ -45,12 +45,13 @@ TOOLS = [
45 "properties": {
46 "city": {
47 "type": "string",
48 - "description": "The city to find the weather for, e.g. 'San Francisco'",
48 + "description": "The city to find the weather for, e.g. "
49 + "'San Francisco'",
50 },
51 "state": {
52 "type": "string",
52 - "description": "the two-letter abbreviation for the state that the city is"
53 - " in, e.g. 'CA' which would mean 'California'",
53 + "description": "the two-letter abbreviation for the state that "
54 + "the city is in, e.g. 'CA' which would mean 'California'",
55 },
56 "unit": {
57 "type": "string",
@@ -69,7 +70,8 @@ MESSAGES = [
70 {"role": "assistant", "content": "I'm doing well! How can I help you?"},
71 {
72 "role": "user",
72 - "content": "Can you tell me what the temperate will be in Dallas, in fahrenheit?",
73 + "content": "Can you tell me what the temperate will be in Dallas, "
74 + "in fahrenheit?",
75 },
76 ]
77
tests/entrypoints/openai/test_completion_with_function_calling.py
+8 -4
@@ -25,12 +25,14 @@ tools = [
25 "properties": {
26 "city": {
27 "type": "string",
28 - "description": "The city to find the weather for, e.g. 'Vienna'",
28 + "description": "The city to find the weather for, e.g. "
29 + "'Vienna'",
30 "default": "Vienna",
31 },
32 "country": {
33 "type": "string",
33 - "description": "The country that the city is in, e.g. 'Austria'",
34 + "description": "The country that the city is in, e.g. "
35 + "'Austria'",
36 },
37 "unit": {
38 "type": "string",
@@ -85,12 +87,14 @@ tools = [
87 "properties": {
88 "city": {
89 "type": "string",
88 - "description": "The city to get the forecast for, e.g. 'Vienna'",
90 + "description": "The city to get the forecast for, e.g. "
91 + "'Vienna'",
92 "default": "Vienna",
93 },
94 "country": {
95 "type": "string",
93 - "description": "The country that the city is in, e.g. 'Austria'",
96 + "description": "The country that the city is in, e.g. "
97 + "'Austria'",
98 },
99 "days": {
100 "type": "integer",
tests/entrypoints/openai/test_video.py
+2 -2
@@ -179,7 +179,7 @@ async def test_single_chat_session_video_base64encoded(
179 {
180 "type": "video_url",
181 "video_url": {
182 - "url": f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
182 + "url": f"data:video/jpeg;base64,{base64_encoded_video[video_url]}" # noqa: E501
183 },
184 },
185 {"type": "text", "text": "What's in this video?"},
@@ -238,7 +238,7 @@ async def test_single_chat_session_video_base64encoded_beamsearch(
238 {
239 "type": "video_url",
240 "video_url": {
241 - "url": f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
241 + "url": f"data:video/jpeg;base64,{base64_encoded_video[video_url]}" # noqa: E501
242 },
243 },
244 {"type": "text", "text": "What's in this video?"},
tests/entrypoints/openai/test_vision.py
+2 -2
@@ -233,7 +233,7 @@ async def test_single_chat_session_image_base64encoded(
233 {
234 "type": "image_url",
235 "image_url": {
236 - "url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}"
236 + "url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}" # noqa: E501
237 },
238 },
239 {"type": "text", "text": content_text},
@@ -300,7 +300,7 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
300 {
301 "type": "image_url",
302 "image_url": {
303 - "url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}"
303 + "url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}" # noqa: E501
304 },
305 },
306 {"type": "text", "text": "What's in this image?"},
tests/entrypoints/test_chat_utils.py
+26 -23
@@ -947,7 +947,8 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
947 {"type": "image_url", "image_url": {"url": image_url}},
948 {
949 "type": "text",
950 - "text": "What's in <|image_1|> and how does it compare to the other one?", # noqa: E501
950 + "text": "What's in <|image_1|> and how does it compare to "
951 + "the other one?",
952 },
953 ],
954 }
@@ -960,8 +961,8 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
961 assert conversation == [
962 {
963 "role": "user",
963 - "content": "<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "
964 - "other one?",
964 + "content": "<|image_2|>\nWhat's in <|image_1|> and how does it compare to "
965 + "the other one?",
966 }
967 ]
968 _assert_mm_data_is_image_input(mm_data, 2)
@@ -1364,7 +1365,7 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1365 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1366
1367
1367 -def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave( # noqa: E501
1368 +def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave(
1369 phi3v_model_config_mm_interleaved,
1370 phi3v_tokenizer,
1371 image_url,
@@ -1451,14 +1452,14 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1452 assert conversation == [
1453 {
1454 "role": "user",
1454 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1455 - "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1455 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1456 + "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1457 },
1458 {"role": "assistant", "content": "Some stuff."},
1459 {
1460 "role": "user",
1460 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1461 - "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1461 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1462 + "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1463 },
1464 ]
1465
@@ -1468,7 +1469,7 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1469 _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
1470
1471
1471 -def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave( # noqa: E501
1472 +def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
1473 qwen25omni_model_config_mm_interleaved,
1474 qwen25omni_tokenizer,
1475 image_url,
@@ -1521,14 +1522,14 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
1522 assert conversation == [
1523 {
1524 "role": "user",
1524 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1525 - "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1525 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1526 + "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1527 },
1528 {"role": "assistant", "content": "Some stuff."},
1529 {
1530 "role": "user",
1530 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1531 - "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1531 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1532 + "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1533 },
1534 ]
1535
@@ -1593,14 +1594,14 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes
1594 assert conversation == [
1595 {
1596 "role": "user",
1596 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1597 - "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1597 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1598 + "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1599 },
1600 {"role": "assistant", "content": "Some stuff."},
1601 {
1602 "role": "user",
1602 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1603 - "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1603 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1604 + "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1605 },
1606 ]
1607
@@ -1661,14 +1662,14 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message
1662 assert conversation == [
1663 {
1664 "role": "user",
1664 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1665 - "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1665 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1666 + "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1667 },
1668 {"role": "assistant", "content": "Some stuff."},
1669 {
1670 "role": "user",
1670 - "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1671 - "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1671 + "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
1672 + "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1673 },
1674 ]
1675
@@ -2193,7 +2194,8 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2194 assert conversation == [
2195 {
2196 "role": "user",
2196 - "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
2197 + "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
2198 + "audio say?",
2199 }
2200 ]
2201 _assert_mm_data_inputs(mm_data, {"audio": 1})
@@ -2228,7 +2230,8 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2230 assert conversation == [
2231 {
2232 "role": "user",
2231 - "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
2233 + "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
2234 + "audio say?",
2235 }
2236 ]
2237 _assert_mm_data_inputs(await mm_future, {"audio": 1})
tests/kernels/attention/test_attention_selector.py
+1 -1
@@ -165,7 +165,7 @@ def test_env(
165 # FlashMLA only supports block_size == 64
166 pytest.skip("FlashMLA only supports block_size 64")
167 else:
168 - from vllm.v1.attention.backends.mla.flashmla import ( # noqa: E501
168 + from vllm.v1.attention.backends.mla.flashmla import (
169 is_flashmla_supported,
170 )
171
tests/kernels/moe/modular_kernel_tools/common.py
+2 -1
@@ -331,7 +331,8 @@ class WeightTensors:
331 in_dtype=config.dtype,
332 quant_dtype=config.quant_dtype,
333 block_shape=config.quant_block_shape,
334 - per_out_ch_quant=config.is_per_act_token_quant, # or config.is_per_out_ch_quant
334 + # or config.is_per_out_ch_quant
335 + per_out_ch_quant=config.is_per_act_token_quant,
336 )
337 return WeightTensors(
338 w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
tests/kernels/moe/modular_kernel_tools/make_feature_matrix.py
+1 -1
@@ -124,7 +124,7 @@ def make_feature_matrix(csv_file_path: str):
124 results_df: Optional[pd.DataFrame] = None
125 for m, k, n, e, topks, dtype, pf_type, experts_type, quant_config in tqdm(
126 combinations
127 - ): # noqa: E501
127 + ):
128 config = Config(
129 Ms=[m],
130 K=k,
tests/kernels/moe/modular_kernel_tools/mk_objects.py
+4 -4
@@ -10,7 +10,7 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
10 from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
11 BatchedDeepGemmExperts,
12 )
13 -from vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe import ( # noqa: E501
13 +from vllm.model_executor.layers.fused_moe.batched_triton_or_deep_gemm_moe import (
14 BatchedTritonOrDeepGemmExperts,
15 )
16 from vllm.model_executor.layers.fused_moe.config import (
@@ -196,10 +196,10 @@ register_experts(
196
197 # Disable on blackwell for now
198 if has_deep_ep() and not current_platform.has_device_capability(100):
199 - from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import ( # noqa: E501
199 + from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import (
200 DeepEPHTPrepareAndFinalize,
201 )
202 - from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import ( # noqa: E501
202 + from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import (
203 DeepEPLLPrepareAndFinalize,
204 )
205
@@ -233,7 +233,7 @@ if has_pplx():
233 )
234
235 if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100):
236 - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( # noqa: E501
236 + from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
237 FlashInferExperts,
238 )
239 from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
tests/kernels/moe/parallel_utils.py
+2 -2
@@ -17,10 +17,10 @@ from typing_extensions import Concatenate, ParamSpec
17 from vllm.utils import get_open_port, has_deep_ep
18
19 if has_deep_ep():
20 - from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import ( # noqa: E501
20 + from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import (
21 DeepEPHTPrepareAndFinalize,
22 )
23 - from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import ( # noqa: E501
23 + from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import (
24 DeepEPLLPrepareAndFinalize,
25 )
26
tests/kernels/moe/test_deepep_deepgemm_moe.py
+2 -2
@@ -30,10 +30,10 @@ from .parallel_utils import ProcessGroupInfo, parallel_launch
30 from .utils import make_test_weights
31
32 if has_deep_ep():
33 - from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import ( # noqa: E501
33 + from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import (
34 DeepEPHTPrepareAndFinalize,
35 )
36 - from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import ( # noqa: E501
36 + from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import (
37 DeepEPLLPrepareAndFinalize,
38 )
39
tests/kernels/moe/test_deepep_moe.py
+2 -2
@@ -28,10 +28,10 @@ from ...utils import multi_gpu_test
28 from .parallel_utils import ProcessGroupInfo, parallel_launch
29
30 if has_deep_ep():
31 - from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import ( # noqa: E501
31 + from vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize import (
32 DeepEPHTPrepareAndFinalize,
33 )
34 - from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import ( # noqa: E501
34 + from vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize import (
35 DeepEPLLPrepareAndFinalize,
36 )
37
tests/kernels/moe/test_modular_kernel_combinations.py
+1 -1
@@ -271,7 +271,7 @@ if __name__ == "__main__":
271 parser = make_config_arg_parser(
272 description=(
273 "Run single prepare-finalize & fused-experts combination test"
274 - "Example : python3 -m tests.kernels.moe.test_modular_kernel_combinations " # noqa: E501
274 + "Example : python3 -m tests.kernels.moe.test_modular_kernel_combinations "
275 "--pf-type PplxPrepareAndFinalize --experts-type BatchedTritonExperts"
276 )
277 )
tests/kernels/moe/test_moe.py
+2 -2
@@ -483,8 +483,8 @@ def test_mixtral_moe(
483 }
484
485 if use_rocm_aiter:
486 - # The values of rtol and atol are set based on the tests in ROCM AITER package. # noqa: E501
487 - # https://github.com/ROCm/aiter/blob/dfed377f4be7da96ca2d75ac0761f569676f7240/op_tests/test_moe.py#L174 # noqa: E501
486 + # The values of rtol and atol are set based on the tests in ROCM AITER package.
487 + # https://github.com/ROCm/aiter/blob/dfed377f4be7da96ca2d75ac0761f569676f7240/op_tests/test_moe.py#L174
488 torch.testing.assert_close(
489 hf_states.flatten(0, 1), vllm_states, rtol=0.01, atol=100
490 )
tests/kernels/moe/test_mxfp4_moe.py
+2 -2
@@ -10,11 +10,11 @@ import pytest
10 import torch
11 from packaging import version
12
13 -from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
13 +from vllm.model_executor.layers.quantization.quark.quark import (
14 QuarkLinearMethod,
15 QuarkW4A4MXFP4,
16 )
17 -from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
17 +from vllm.model_executor.layers.quantization.quark.quark_moe import (
18 QuarkW4A4MXFp4MoEMethod,
19 )
20 from vllm.platforms import current_platform
tests/lora/test_chatglm3_tp.py
+9 -3
@@ -12,7 +12,7 @@ PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example
12
13 EXPECTED_LORA_OUTPUT = [
14 "SELECT count(*) FROM singer",
15 - "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
15 + "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'",
16 "SELECT name , country , age FROM singer ORDER BY age",
17 ]
18
@@ -21,10 +21,16 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
21 prompts = [
22 PROMPT_TEMPLATE.format(query="How many singers do we have?"),
23 PROMPT_TEMPLATE.format(
24 - query="What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
24 + query=(
25 + "What is the average, minimum, and maximum "
26 + "age of all singers from France?"
27 + )
28 ),
29 PROMPT_TEMPLATE.format(
27 - query="Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
30 + query=(
31 + "Show name, country, age for all singers ordered "
32 + "by age from the oldest to the youngest."
33 + )
34 ),
35 ]
36 sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
tests/lora/test_llama_tp.py
+2 -2
@@ -15,10 +15,10 @@ MODEL_PATH = "meta-llama/Llama-2-7b-hf"
15
16 EXPECTED_LORA_OUTPUT = [
17 " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
18 - " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
18 + " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",
19 " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
20 " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
21 - " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
21 + " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",
22 " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' ", # noqa: E501
23 ]
24
tests/lora/test_llm_with_multi_loras.py
+1 -1
@@ -26,7 +26,7 @@ LORA_RANK = 8
26 LORA_TEST_PROMPTS = ["What is GitHub?", "Hi, tell me about you"]
27 LORA_TEST_EXPECTED = [
28 "GitHub is an open-source platform that provides a way to manage and develop software projects. It allows developers to store and manage code, collaborate on projects, and automate tasks.", # noqa: E501
29 - "I am Alice, an AI assistant developed by GitHub/Charent.", # noqa: E501
29 + "I am Alice, an AI assistant developed by GitHub/Charent.",
30 ]
31
32
tests/models/language/generation/test_gemma.py
+1 -1
@@ -16,7 +16,7 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
16 ) as llm:
17 if model == "google/gemma-3-4b-it":
18 normalizers = llm.llm.collective_rpc(
19 - lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item()
19 + lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501
20 )
21 config = llm.llm.llm_engine.model_config.hf_config.text_config
22 else:
tests/models/language/generation/test_mistral.py
+10 -6
@@ -46,12 +46,13 @@ TOOLS = [
46 "properties": {
47 "city": {
48 "type": "string",
49 - "description": "The city to find the weather for, e.g. 'San Francisco'",
49 + "description": "The city to find the weather for, e.g. "
50 + "'San Francisco'",
51 },
52 "state": {
53 "type": "string",
53 - "description": "the two-letter abbreviation for the state that the city is"
54 - " in, e.g. 'CA' which would mean 'California'",
54 + "description": "the two-letter abbreviation for the state that "
55 + "the city is in, e.g. 'CA' which would mean 'California'",
56 },
57 "unit": {
58 "type": "string",
@@ -85,7 +86,8 @@ MSGS = [
86 {"role": "system", "content": "You are an assistant."},
87 {
88 "role": "user",
88 - "content": "Could you please rewrite the below article? \n\n My English needs improvving, maybe I make errors.", # noqa
89 + "content": "Could you please rewrite the below article? \n\n My English needs "
90 + "improvving, maybe I make errors.",
91 },
92 {
93 "role": "assistant",
@@ -96,14 +98,16 @@ MSGS = [
98 "type": "function",
99 "function": {
100 "name": "rewrite",
99 - "arguments": '{"text":"My English needs improvving, maybe I make errors."}', # noqa
101 + "arguments": '{"text":"My English needs improvving, maybe '
102 + 'I make errors."}',
103 },
104 }
105 ],
106 },
107 {
108 "role": "tool",
106 - "content": '{"action":"rewrite","outcome":"My English needs improving, maybe I make errors."}', # noqa
109 + "content": '{"action":"rewrite","outcome":"My English needs improving, maybe '
110 + 'I make errors."}',
111 "tool_call_id": "bbc5b7ede",
112 "name": "rewrite",
113 },
tests/models/multimodal/generation/test_common.py
+38 -38
@@ -130,14 +130,14 @@ VLM_TEST_SETTINGS = {
130 dtype="bfloat16",
131 marks=[
132 pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")
133 - ], # noqa: E501
133 + ],
134 ),
135 "qwen2_5_vl": VLMTestInfo(
136 models=["Qwen/Qwen2.5-VL-3B-Instruct"],
137 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
138 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
139 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
140 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
139 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
140 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
141 max_model_len=4096,
142 max_num_seqs=2,
143 auto_cls=AutoModelForImageTextToText,
@@ -149,8 +149,8 @@ VLM_TEST_SETTINGS = {
149 models=["Qwen/Qwen2.5-Omni-3B"],
150 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
151 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
152 - img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
153 - video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
152 + img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>",
153 + video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>",
154 max_model_len=4096,
155 max_num_seqs=2,
156 num_logprobs=6 if current_platform.is_cpu() else 5,
@@ -181,7 +181,7 @@ VLM_TEST_SETTINGS = {
181 max_model_len=16384,
182 hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
183 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
184 - ), # noqa: E501
184 + ),
185 auto_cls=AutoModelForImageTextToText,
186 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
187 image_size_factors=[(0.25, 0.5, 1.0)],
@@ -213,7 +213,7 @@ VLM_TEST_SETTINGS = {
213 models=["Qwen/Qwen2.5-VL-3B-Instruct"],
214 test_type=VLMTestType.IMAGE,
215 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
216 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
216 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
217 max_model_len=4096,
218 max_num_seqs=2,
219 auto_cls=AutoModelForImageTextToText,
@@ -237,10 +237,10 @@ VLM_TEST_SETTINGS = {
237 single_image_prompts=IMAGE_ASSETS.prompts(
238 {
239 "stop_sign": "<vlm_image>Please describe the image shortly.",
240 - "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
240 + "cherry_blossom": "<vlm_image>Please infer the season with reason.",
241 }
242 ),
243 - multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
243 + multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.",
244 stop_str=["<|im_end|>"],
245 image_size_factors=[(0.10, 0.15)],
246 max_tokens=64,
@@ -252,11 +252,11 @@ VLM_TEST_SETTINGS = {
252 prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
253 single_image_prompts=IMAGE_ASSETS.prompts(
254 {
255 - "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
256 - "cherry_blossom": "<image>What is the season?", # noqa: E501
255 + "stop_sign": "<image>What's the content in the center of the image?",
256 + "cherry_blossom": "<image>What is the season?",
257 }
258 ),
259 - multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
259 + multi_image_prompt="<image><image>Describe the two images in detail.",
260 max_model_len=4096,
261 max_num_seqs=2,
262 auto_cls=AutoModelForImageTextToText,
@@ -268,11 +268,11 @@ VLM_TEST_SETTINGS = {
268 prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
269 single_image_prompts=IMAGE_ASSETS.prompts(
270 {
271 - "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
272 - "cherry_blossom": "<image>What is the season?", # noqa: E501
271 + "stop_sign": "<image>What's the content in the center of the image?",
272 + "cherry_blossom": "<image>What is the season?",
273 }
274 ),
275 - multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
275 + multi_image_prompt="<image><image>Describe the two images in detail.",
276 max_model_len=4096,
277 max_num_seqs=2,
278 auto_cls=AutoModelForImageTextToText,
@@ -311,14 +311,14 @@ VLM_TEST_SETTINGS = {
311 max_num_seqs=2,
312 single_image_prompts=IMAGE_ASSETS.prompts(
313 {
314 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
314 + "stop_sign": "<image>\nWhat's the content in the center of the image?",
315 "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
316 }
317 ),
318 multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
319 patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
320 hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
321 - stop_str=["<｜end▁of▁sentence｜>", "<｜begin▁of▁sentence｜>"], # noqa: E501
321 + stop_str=["<｜end▁of▁sentence｜>", "<｜begin▁of▁sentence｜>"],
322 image_size_factors=[(), (1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
323 ),
324 "fuyu": VLMTestInfo(
@@ -342,7 +342,7 @@ VLM_TEST_SETTINGS = {
342 single_image_prompts=IMAGE_ASSETS.prompts(
343 {
344 "stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
345 - "cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
345 + "cherry_blossom": "<start_of_image>What is the season?",
346 }
347 ),
348 multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
@@ -356,7 +356,7 @@ VLM_TEST_SETTINGS = {
356 "glm4v": VLMTestInfo(
357 models=["zai-org/glm-4v-9b"],
358 test_type=VLMTestType.IMAGE,
359 - prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
359 + prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>",
360 single_image_prompts=IMAGE_ASSETS.prompts(
361 {
362 "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
@@ -377,9 +377,9 @@ VLM_TEST_SETTINGS = {
377 "glm4_1v": VLMTestInfo(
378 models=["zai-org/GLM-4.1V-9B-Thinking"],
379 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
380 - prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
381 - img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", # noqa: E501
382 - video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", # noqa: E501
380 + prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>",
381 + img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
382 + video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
383 max_model_len=2048,
384 max_num_seqs=2,
385 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
@@ -410,10 +410,10 @@ VLM_TEST_SETTINGS = {
410 "h2oai/h2ovl-mississippi-2b",
411 ],
412 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
413 - prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
413 + prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>",
414 single_image_prompts=IMAGE_ASSETS.prompts(
415 {
416 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
416 + "stop_sign": "<image>\nWhat's the content in the center of the image?",
417 "cherry_blossom": "<image>\nWhat is the season?",
418 }
419 ),
@@ -444,7 +444,7 @@ VLM_TEST_SETTINGS = {
444 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
445 single_image_prompts=IMAGE_ASSETS.prompts(
446 {
447 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
447 + "stop_sign": "<image>\nWhat's the content in the center of the image?",
448 "cherry_blossom": "<image>\nWhat is the season?",
449 }
450 ),
@@ -529,7 +529,7 @@ VLM_TEST_SETTINGS = {
529 max_model_len=16384,
530 hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
531 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
532 - ), # noqa: E501
532 + ),
533 auto_cls=AutoModelForImageTextToText,
534 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
535 custom_test_opts=[
@@ -583,7 +583,7 @@ VLM_TEST_SETTINGS = {
583 max_num_seqs=2,
584 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
585 ["<|im_end|>", "<|endoftext|>"]
586 - ), # noqa: E501
586 + ),
587 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
588 patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
589 # FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
@@ -598,7 +598,7 @@ VLM_TEST_SETTINGS = {
598 max_num_seqs=2,
599 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
600 ["<|im_end|>", "<|endoftext|>"]
601 - ), # noqa: E501
601 + ),
602 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
603 patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
604 ),
@@ -627,7 +627,7 @@ VLM_TEST_SETTINGS = {
627 models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
628 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
629 prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
630 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
630 + img_idx_to_prompt=lambda idx: "<image>\n",
631 max_model_len=4096,
632 max_num_seqs=2,
633 dtype="half",
@@ -640,7 +640,7 @@ VLM_TEST_SETTINGS = {
640 models=["AIDC-AI/Ovis2-1B"],
641 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
642 prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
643 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
643 + img_idx_to_prompt=lambda idx: "<image>\n",
644 max_model_len=4096,
645 max_num_seqs=2,
646 dtype="half",
@@ -652,7 +652,7 @@ VLM_TEST_SETTINGS = {
652 models=["AIDC-AI/Ovis2.5-2B"],
653 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
654 prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
655 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
655 + img_idx_to_prompt=lambda idx: "<image>\n",
656 video_idx_to_prompt=lambda idx: "<video>\n",
657 max_model_len=4096,
658 max_num_seqs=2,
@@ -701,8 +701,8 @@ VLM_TEST_SETTINGS = {
701 models=["Qwen/Qwen2-VL-2B-Instruct"],
702 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
703 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
704 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
705 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
704 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
705 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
706 multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.", # noqa: E501
707 max_model_len=4096,
708 max_num_seqs=2,
@@ -717,11 +717,11 @@ VLM_TEST_SETTINGS = {
717 prompt_formatter=lambda img_prompt: f"<｜begin▁of▁sentence｜><｜User｜>\n{img_prompt}<｜Assistant｜><think>\n", # noqa: E501
718 single_image_prompts=IMAGE_ASSETS.prompts(
719 {
720 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
720 + "stop_sign": "<image>\nWhat's the content in the center of the image?",
721 "cherry_blossom": "<image>\nWhat is the season?",
722 }
723 ),
724 - multi_image_prompt="<image>\n<image>\nDescribe the two images in short.", # noqa: E501
724 + multi_image_prompt="<image>\n<image>\nDescribe the two images in short.",
725 max_model_len=4096,
726 use_tokenizer_eos=True,
727 patch_hf_runner=model_utils.skyworkr1v_patch_hf_runner,
@@ -754,8 +754,8 @@ VLM_TEST_SETTINGS = {
754 VLMTestType.VIDEO,
755 ),
756 prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
757 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
758 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
757 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
758 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
759 max_model_len=4096,
760 max_num_seqs=2,
761 auto_cls=AutoModelForImageTextToText,
@@ -816,7 +816,7 @@ VLM_TEST_SETTINGS = {
816 auto_cls=AutoModelForImageTextToText,
817 hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
818 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
819 - ), # noqa: E501
819 + ),
820 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
821 custom_test_opts=[
822 CustomTestOptions(
tests/models/multimodal/generation/test_ultravox.py
+1 -1
@@ -170,7 +170,7 @@ async def test_online_serving(client, audio_assets: AudioTestAssets):
170 ],
171 {
172 "type": "text",
173 - "text": f"What's happening in these {len(audio_assets)} audio clips?",
173 + "text": f"What's happening in these {len(audio_assets)} audio clips?", # noqa: E501
174 },
175 ],
176 }
tests/models/multimodal/generation/test_voxtral.py
+2 -7
@@ -101,16 +101,11 @@ async def test_online_serving(client, audio_assets: AudioTestAssets):
101 return audio_dict
102
103 audio_chunks = [asset_to_chunk(asset) for asset in audio_assets]
104 + text = f"What's happening in these {len(audio_assets)} audio clips?"
105 messages = [
106 {
107 "role": "user",
107 - "content": [
108 - *audio_chunks,
109 - {
110 - "type": "text",
111 - "text": f"What's happening in these {len(audio_assets)} audio clips?",
112 - },
113 - ],
108 + "content": [*audio_chunks, {"type": "text", "text": text}],
109 }
110 ]
111
tests/models/multimodal/generation/vlm_utils/custom_inputs.py
+2 -2
@@ -102,8 +102,8 @@ def multi_video_multi_aspect_ratio_inputs(
102 def different_patch_input_cases_internvl():
103 images = [asset.pil_image.resize((896, 896)) for asset in IMAGE_ASSETS]
104 formatter = (
105 - lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n"
106 - ) # noqa: E501
105 + lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
106 + )
107 single_img_prompts = [
108 "<image>\nWhat's the content in the center of the image?",
109 "<image>\nWhat is the season?",
tests/tool_use/test_tool_choice_required.py
+2 -1
@@ -47,7 +47,8 @@ EXAMPLE_TOOLS = [
47 "properties": {
48 "city": {
49 "type": "string",
50 - "description": "The city to get the forecast for, e.g. 'New York'",
50 + "description": "The city to get the forecast for, e.g. "
51 + "'New York'",
52 },
53 "days": {
54 "type": "integer",
tests/v1/attention/utils.py
+7 -7
@@ -134,15 +134,15 @@ def get_attention_backend(backend_name: _Backend):
134 else "vllm.v1.attention.backends.rocm_aiter_fa.AiterFlashAttentionBackend"
135 ),
136 _Backend.FLASHINFER: "vllm.v1.attention.backends.flashinfer.FlashInferBackend",
137 - _Backend.FLEX_ATTENTION: "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend",
138 - _Backend.TRITON_ATTN: "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend",
137 + _Backend.FLEX_ATTENTION: "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend", # noqa: E501
138 + _Backend.TRITON_ATTN: "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend", # noqa: E501
139 _Backend.TREE_ATTN: "vllm.v1.attention.backends.tree_attn.TreeAttentionBackend",
140 - _Backend.XFORMERS: "vllm.v1.attention.backends.xformers.XFormersAttentionBackend",
141 - _Backend.CUTLASS_MLA: "vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend",
140 + _Backend.XFORMERS: "vllm.v1.attention.backends.xformers.XFormersAttentionBackend", # noqa: E501
141 + _Backend.CUTLASS_MLA: "vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend", # noqa: E501
142 _Backend.FLASHMLA: "vllm.v1.attention.backends.mla.flashmla.FlashMLABackend",
143 - _Backend.FLASH_ATTN_MLA: "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend",
144 - _Backend.FLASHINFER_MLA: "vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend",
145 - _Backend.TRITON_MLA: "vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend",
143 + _Backend.FLASH_ATTN_MLA: "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend", # noqa: E501
144 + _Backend.FLASHINFER_MLA: "vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend", # noqa: E501
145 + _Backend.TRITON_MLA: "vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend", # noqa: E501
146 }
147
148 if backend_name not in backend_map:
tests/v1/entrypoints/openai/responses/test_image.py
+1 -1
@@ -104,7 +104,7 @@ async def test_single_chat_session_image_base64encoded(
104 "content": [
105 {
106 "type": "input_image",
107 - "image_url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}",
107 + "image_url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}", # noqa: E501
108 "detail": "auto",
109 },
110 {"type": "input_text", "text": content_text},
tests/v1/kv_connector/nixl_integration/test_accuracy.py
+3 -2
@@ -15,8 +15,9 @@ RTOL = 0.03
15 EXPECTED_VALUES = {"Qwen/Qwen3-0.6B": 0.41, "deepseek-ai/deepseek-vl2-small": 0.59}
16
17 SIMPLE_PROMPT = (
18 - "The best part about working on vLLM is that I got to meet so many people across various different organizations like UCB, Google, and Meta which means",
19 -) # noqa: E501
18 + "The best part about working on vLLM is that I got to meet so many people across "
19 + "various different organizations like UCB, Google, and Meta which means",
20 +)
21
22 # Get model name from environment variable
23 MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")
tests/v1/kv_connector/unit/test_offloading_connector.py
+1 -1
@@ -127,7 +127,7 @@ class RequestRunner:
127 kv_role="kv_both",
128 kv_connector_extra_config={
129 "spec_name": "MockOffloadingSpec",
130 - "spec_module_path": "tests.v1.kv_connector.unit.test_offloading_connector",
130 + "spec_module_path": "tests.v1.kv_connector.unit.test_offloading_connector", # noqa: E501
131 "block_size": offloaded_block_size,
132 },
133 )
tests/v1/logits_processors/test_custom_offline.py
+2 -9
@@ -260,15 +260,8 @@ def test_pooling_rejects_custom_logitsprocs(
260 gpu_memory_utilization=0.1,
261 )
262 # Require that no logitsprocs have been loaded
263 - assert (
264 - sum(
265 - [
266 - 1
267 - for _ in llm.llm_engine.model_executor.driver_worker.worker.model_runner.input_batch.logitsprocs.all
268 - ]
269 - )
270 - == 0
271 - )
263 + worker = llm.llm_engine.model_executor.driver_worker.worker
264 + assert sum([1 for _ in worker.model_runner.input_batch.logitsprocs.all]) == 0
265 return
266
267 kwargs: dict[str, list[Union[str, type[LogitsProcessor]]]] = {}
vllm/attention/ops/pallas_kv_cache_update.py
+8 -4
@@ -76,10 +76,14 @@ def _kv_cache_update_kernel(
76 static_argnames=["page_size", "num_slices_per_block"],
77 )
78 def kv_cache_update(
79 - new_kv: jax.Array, # [total_num_token, num_combined_kv_heads, head_dim]
80 - slices: jax.Array, # [3, slices], list of (kv_cache_start, new_kv_start, slice_len)
81 - kv_cache: jax.Array, # [total_num_pages * page_size, num_combined_kv_heads, head_dim]
82 - num_kv_update_slices: jax.Array, # [1]
79 + # [total_num_token, num_combined_kv_heads, head_dim]
80 + new_kv: jax.Array,
81 + # [3, slices], list of (kv_cache_start, new_kv_start, slice_len)
82 + slices: jax.Array,
83 + # [total_num_pages * page_size, num_combined_kv_heads, head_dim]
84 + kv_cache: jax.Array,
85 + # [1]
86 + num_kv_update_slices: jax.Array,
87 *,
88 page_size: int = 32,
89 num_slices_per_block: int = 8,
vllm/compilation/collective_fusion.py
+17 -5
@@ -834,7 +834,10 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
834 scale_out=None,
835 rms_gamma=weight,
836 rms_eps=self.epsilon,
837 - pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant, # we don't use norm_out afterwards
837 + # We don't use norm_out afterwards
838 + pattern_code=(
839 + flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant
840 + ),
841 scale_factor=scale,
842 **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
843 )
@@ -928,11 +931,14 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
931 scale_out=None,
932 rms_gamma=weight,
933 rms_eps=self.epsilon,
931 - pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant, # we don't use norm_out afterwards
934 + # We don't use norm_out afterwards
935 + pattern_code=(
936 + flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant
937 + ),
938 scale_factor=scale,
939 **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
940 )
935 - # # quant_out, rms_norm_residual
941 + # quant_out, rms_norm_residual
942 return allreduce[4], allreduce[2]
943
944 pm.register_replacement(
@@ -1028,7 +1034,10 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
1034 scale_out=output_scale,
1035 rms_gamma=weight,
1036 rms_eps=self.epsilon,
1031 - pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant, # we don't use norm_out afterwards
1037 + # We don't use norm_out afterwards
1038 + pattern_code=(
1039 + flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant
1040 + ),
1041 scale_factor=input_global_scale,
1042 **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
1043 )
@@ -1130,7 +1139,10 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
1139 scale_out=output_scale,
1140 rms_gamma=weight,
1141 rms_eps=self.epsilon,
1133 - pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant, # we don't use norm_out afterwards
1142 + # We don't use norm_out afterwards
1143 + pattern_code=(
1144 + flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant
1145 + ),
1146 scale_factor=input_global_scale,
1147 **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
1148 )
vllm/compilation/wrapper.py
+9 -5
@@ -119,9 +119,12 @@ class TorchCompileWrapperWithCustomDispatcher:
119
120 src = depyf.decompile(new_code)
121 msg = (
122 - "Assigning / modifying buffers of nn.Module during forward pass is not allowed when using cudagraph inside the compiler because it will cause silent errors. Please use eager mode or fix the code. The following code contains clues about which buffer is being modified (please search for the usage of the function `update`):\n"
123 - + src
124 - ) # noqa
122 + "Assigning / modifying buffers of nn.Module during forward pass is not "
123 + "allowed when using cudagraph inside the compiler because it will "
124 + "cause silent errors. Please use eager mode or fix the code. The "
125 + "following code contains clues about which buffer is being modified "
126 + f"(please search for the usage of the function `update`):\n{src}"
127 + )
128 raise RuntimeError(msg)
129
130 @contextmanager
@@ -132,8 +135,9 @@ class TorchCompileWrapperWithCustomDispatcher:
135 variables as the original code. Therefore we can directly switch
136 the code object in the function and call it.
137
135 - See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details.
136 - """ # noqa
138 + See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7
139 + for more details.
140 + """
141 self.__class__.forward.__code__ = self.compiled_codes[index]
142 yield
143 self.__class__.forward.__code__ = self.original_code_object
vllm/config/vllm.py
+1 -1
@@ -472,7 +472,7 @@ class VllmConfig:
472 self.compilation_config.cudagraph_mode.has_full_cudagraphs()
473 and self.model_config is not None
474 and not self.model_config.disable_cascade_attn
475 - and not self.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
475 + and not self.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() # noqa: E501
476 ):
477 logger.warning_once(
478 "No piecewise cudagraph for executing cascade attention."
vllm/distributed/device_communicators/all2all.py
+5 -3
@@ -147,8 +147,9 @@ class PPLXAll2AllManager(All2AllManagerBase):
147
148 def __init__(self, cpu_group):
149 assert has_pplx(), (
150 - "pplx_kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md to install pplx_kernels."
151 - ) # noqa
150 + "pplx_kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md"
151 + " to install pplx_kernels."
152 + )
153 super().__init__(cpu_group)
154
155 if self.internode:
@@ -220,7 +221,8 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
221
222 def __init__(self, cpu_group):
223 assert has_deep_ep(), (
223 - "DeepEP kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md to install DeepEP kernels."
224 + "DeepEP kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md"
225 + " to install DeepEP kernels."
226 ) # noqa
227 super().__init__(cpu_group)
228 self.handle_cache = Cache()
vllm/entrypoints/openai/protocol.py
+2 -1
@@ -471,7 +471,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
471 top_logprobs: Optional[int] = 0
472 max_tokens: Optional[int] = Field(
473 default=None,
474 - deprecated="max_tokens is deprecated in favor of the max_completion_tokens field",
474 + deprecated="max_tokens is deprecated in favor of "
475 + "the max_completion_tokens field",
476 )
477 max_completion_tokens: Optional[int] = None
478 n: Optional[int] = 1
vllm/lora/layers/vocal_parallel_embedding.py
+1 -1
@@ -31,7 +31,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
31 if self.base_layer.num_added_embeddings_per_partition > 0:
32 # We can start adding lora weights
33 self.embeddings_weights = self.base_layer.weight.data[
34 - self.base_layer.num_org_embeddings_per_partition : self.base_layer.num_org_embeddings_per_partition
34 + self.base_layer.num_org_embeddings_per_partition : self.base_layer.num_org_embeddings_per_partition # noqa: E501
35 + self.base_layer.num_added_embeddings_per_partition
36 ]
37 self.embeddings_slice = (
vllm/model_executor/layers/quantization/ptpc_fp8.py
+2 -2
@@ -107,8 +107,8 @@ class PTPCFp8LinearMethod(Fp8LinearMethod):
107 layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
108
109 assert layer.weight.data.dtype == torch.bfloat16, (
110 - f"Currently torch._scaled_mm (hipBLASLt) rowwise gemm only support output dtype of bfloat16. {str(layer.weight.data.dtype)} is specified."
111 - ) # noqa: E501
110 + f"Currently torch._scaled_mm (hipBLASLt) rowwise gemm only support output dtype of bfloat16. {str(layer.weight.data.dtype)} is specified." # noqa: E501
111 + )
112 # Quantize the weights.
113 qweight, weight_scale = ops.scaled_fp8_quant(
114 layer.weight, scale=None, use_per_token_if_dynamic=True
vllm/model_executor/model_loader/bitsandbytes_loader.py
+1 -1
@@ -391,7 +391,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
391 total_shard_sizes = next(
392 (
393 sizes
394 - for module, sizes in self.maybe_fused_weights_modules.items()
394 + for module, sizes in self.maybe_fused_weights_modules.items() # noqa: E501
395 if check_match(mapped_weight_name, module)
396 )
397 )
vllm/model_executor/models/bailing_moe.py
+2 -2
@@ -270,8 +270,8 @@ class BailingMoE(nn.Module):
270 ) or (
271 self.score_function == "sigmoid" and self.correction_bias is not None
272 ), (
273 - "score_function and correction_bias should be in 2 combination (softmax, None) or (sigmoid, not None)"
274 - ) # noqa: E501
273 + "score_function and correction_bias should be in 2 combination (softmax, None) or (sigmoid, not None)" # noqa: E501
274 + )
275 else:
276 # default value for scoring_func
277 self.score_function = "softmax"
vllm/model_executor/models/hyperclovax_vision.py
+4 -4
@@ -825,10 +825,10 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
825 # Run MM-Projector
826 # len(num_grids) == len(num_queries_vis_abstractors) + 1
827 grid_idx = 0
828 - num_grids = [
829 - grid_idx
830 - ] # e.g. [0, 9, 18, 19, 27, 28, 36, 37, 45, 46, 54, 55, 56]
831 - num_queries_vis_abstractors = [] # e.g. [81, 81, 81, 9, 81, 9, 81, 9, 81, 9, 81, 9]
828 + # e.g. [0, 9, 18, 19, 27, 28, 36, 37, 45, 46, 54, 55, 56]
829 + num_grids = [grid_idx]
830 + # e.g. [81, 81, 81, 9, 81, 9, 81, 9, 81, 9, 81, 9]
831 + num_queries_vis_abstractors = []
832 len_total_frames = video_forward_outs.shape[0]
833
834 if self.config.first_last_frames_slow:
vllm/model_executor/models/llama4_eagle.py
+3 -2
@@ -154,9 +154,10 @@ class LlamaModel(nn.Module):
154 str(layer_index), str(layer_index + start_layer_id)
155 )
156
157 - quant_config.torchao_config.module_fqn_to_config = {
157 + torchao_config = quant_config.torchao_config
158 + torchao_config.module_fqn_to_config = {
159 pad_layer_name(layer): quantization
159 - for layer, quantization in quant_config.torchao_config.module_fqn_to_config.items()
160 + for layer, quantization in torchao_config.module_fqn_to_config.items()
161 }
162
163
vllm/model_executor/models/longcat_flash_mtp.py
+20 -20
@@ -186,26 +186,26 @@ class LongCatFlashMTP(nn.Module, SupportsPP):
186 "model.mtp.layers.0.eh_proj.weight_scale_inv": "eh_proj.weight_scale_inv",
187 "model.mtp.layers.0.enorm.m.weight": "enorm.weight",
188 "model.mtp.layers.0.hnorm.m.weight": "hnorm.weight",
189 - "model.mtp.layers.0.input_layernorm.weight": "model.layers.0.input_layernorm.weight",
190 - "model.mtp.layers.0.post_attention_layernorm.weight": "model.layers.0.post_attention_layernorm.weight",
191 - "model.mtp.layers.0.self_attn.kv_a_layernorm.weight": "model.layers.0.self_attn.kv_a_layernorm.weight",
192 - "model.mtp.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model.layers.0.self_attn.kv_a_proj_with_mqa.weight",
193 - "model.mtp.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale_inv": "model.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale_inv",
194 - "model.mtp.layers.0.self_attn.kv_b_proj.weight": "model.layers.0.self_attn.kv_b_proj.weight",
195 - "model.mtp.layers.0.self_attn.kv_b_proj.weight_scale_inv": "model.layers.0.self_attn.kv_b_proj.weight_scale_inv",
196 - "model.mtp.layers.0.self_attn.o_proj.weight": "model.layers.0.self_attn.o_proj.weight",
197 - "model.mtp.layers.0.self_attn.o_proj.weight_scale_inv": "model.layers.0.self_attn.o_proj.weight_scale_inv",
198 - "model.mtp.layers.0.self_attn.q_a_layernorm.weight": "model.layers.0.self_attn.q_a_layernorm.weight",
199 - "model.mtp.layers.0.self_attn.q_a_proj.weight": "model.layers.0.self_attn.q_a_proj.weight",
200 - "model.mtp.layers.0.self_attn.q_a_proj.weight_scale_inv": "model.layers.0.self_attn.q_a_proj.weight_scale_inv",
201 - "model.mtp.layers.0.self_attn.q_b_proj.weight": "model.layers.0.self_attn.q_b_proj.weight",
202 - "model.mtp.layers.0.self_attn.q_b_proj.weight_scale_inv": "model.layers.0.self_attn.q_b_proj.weight_scale_inv",
203 - "model.mtp.layers.0.transformer_layer.mlp.down_proj.weight": "model.layers.0.mlp.down_proj.weight",
204 - "model.mtp.layers.0.transformer_layer.mlp.down_proj.weight_scale_inv": "model.layers.0.mlp.down_proj.weight_scale_inv",
205 - "model.mtp.layers.0.transformer_layer.mlp.gate_proj.weight": "model.layers.0.mlp.gate_proj.weight",
206 - "model.mtp.layers.0.transformer_layer.mlp.gate_proj.weight_scale_inv": "model.layers.0.mlp.gate_proj.weight_scale_inv",
207 - "model.mtp.layers.0.transformer_layer.mlp.up_proj.weight": "model.layers.0.mlp.up_proj.weight",
208 - "model.mtp.layers.0.transformer_layer.mlp.up_proj.weight_scale_inv": "model.layers.0.mlp.up_proj.weight_scale_inv",
189 + "model.mtp.layers.0.input_layernorm.weight": "model.layers.0.input_layernorm.weight", # noqa: E501
190 + "model.mtp.layers.0.post_attention_layernorm.weight": "model.layers.0.post_attention_layernorm.weight", # noqa: E501
191 + "model.mtp.layers.0.self_attn.kv_a_layernorm.weight": "model.layers.0.self_attn.kv_a_layernorm.weight", # noqa: E501
192 + "model.mtp.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model.layers.0.self_attn.kv_a_proj_with_mqa.weight", # noqa: E501
193 + "model.mtp.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale_inv": "model.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale_inv", # noqa: E501
194 + "model.mtp.layers.0.self_attn.kv_b_proj.weight": "model.layers.0.self_attn.kv_b_proj.weight", # noqa: E501
195 + "model.mtp.layers.0.self_attn.kv_b_proj.weight_scale_inv": "model.layers.0.self_attn.kv_b_proj.weight_scale_inv", # noqa: E501
196 + "model.mtp.layers.0.self_attn.o_proj.weight": "model.layers.0.self_attn.o_proj.weight", # noqa: E501
197 + "model.mtp.layers.0.self_attn.o_proj.weight_scale_inv": "model.layers.0.self_attn.o_proj.weight_scale_inv", # noqa: E501
198 + "model.mtp.layers.0.self_attn.q_a_layernorm.weight": "model.layers.0.self_attn.q_a_layernorm.weight", # noqa: E501
199 + "model.mtp.layers.0.self_attn.q_a_proj.weight": "model.layers.0.self_attn.q_a_proj.weight", # noqa: E501
200 + "model.mtp.layers.0.self_attn.q_a_proj.weight_scale_inv": "model.layers.0.self_attn.q_a_proj.weight_scale_inv", # noqa: E501
201 + "model.mtp.layers.0.self_attn.q_b_proj.weight": "model.layers.0.self_attn.q_b_proj.weight", # noqa: E501
202 + "model.mtp.layers.0.self_attn.q_b_proj.weight_scale_inv": "model.layers.0.self_attn.q_b_proj.weight_scale_inv", # noqa: E501
203 + "model.mtp.layers.0.transformer_layer.mlp.down_proj.weight": "model.layers.0.mlp.down_proj.weight", # noqa: E501
204 + "model.mtp.layers.0.transformer_layer.mlp.down_proj.weight_scale_inv": "model.layers.0.mlp.down_proj.weight_scale_inv", # noqa: E501
205 + "model.mtp.layers.0.transformer_layer.mlp.gate_proj.weight": "model.layers.0.mlp.gate_proj.weight", # noqa: E501
206 + "model.mtp.layers.0.transformer_layer.mlp.gate_proj.weight_scale_inv": "model.layers.0.mlp.gate_proj.weight_scale_inv", # noqa: E501
207 + "model.mtp.layers.0.transformer_layer.mlp.up_proj.weight": "model.layers.0.mlp.up_proj.weight", # noqa: E501
208 + "model.mtp.layers.0.transformer_layer.mlp.up_proj.weight_scale_inv": "model.layers.0.mlp.up_proj.weight_scale_inv", # noqa: E501
209 "model.mtp.norm.weight": "final_layernorm.weight",
210 }
211
vllm/model_executor/models/phi4mm.py
+2 -2
@@ -1000,8 +1000,8 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1000 "base_layer.": "",
1001 },
1002 orig_to_new_prefix={
1003 - "model.embed_tokens_extend.audio_embed.audio_projection.vision.": "embed_tokens_extend.audio_projection_for_vision.",
1004 - "model.embed_tokens_extend.audio_embed.audio_projection.speech.": "embed_tokens_extend.audio_projection.",
1003 + "model.embed_tokens_extend.audio_embed.audio_projection.vision.": "embed_tokens_extend.audio_projection_for_vision.", # noqa: E501
1004 + "model.embed_tokens_extend.audio_embed.audio_projection.speech.": "embed_tokens_extend.audio_projection.", # noqa: E501
1005 "model.embed_tokens_extend.audio_embed.": "embed_tokens_extend.",
1006 "model.embed_tokens_extend.image_embed.": "vision_encoder.",
1007 },
vllm/model_executor/models/qwen3_next.py
+3 -2
@@ -916,8 +916,9 @@ class Qwen3NextDecoderLayer(nn.Module):
916 )
917 else:
918 assert len(hidden_states.shape) == len(self.ffn_layer_scale.shape), (
919 - f"shape must be the same {len(hidden_states.shape)}, {len(self.ffn_layer_scale.shape)}"
920 - ) # noqa: E501
919 + f"shape must be the same {len(hidden_states.shape)}, "
920 + f"{len(self.ffn_layer_scale.shape)}"
921 + )
922 hidden_states = hidden_states * (
923 self.ffn_layer_scale.to(hidden_states.dtype) + 1
924 )
vllm/v1/attention/backends/mla/common.py
+8 -5
@@ -255,8 +255,8 @@ def is_rocm_aiter_fp8bmm_enabled() -> bool:
255
256
257 if is_rocm_aiter_fp8bmm_enabled():
258 - from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip
259 - batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as aiter_triton_fp8_bmm,
258 + from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501
259 + batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as aiter_triton_fp8_bmm, # noqa: E501
260 )
261
262 def dynamic_per_batched_tensor_quant(
@@ -1284,8 +1284,10 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
1284 actual_seq_lens_q=prefill.query_seq_lens.view(-1, 1, 1, 1),
1285 actual_seq_lens_kv=prefill.query_seq_lens.view(-1, 1, 1, 1),
1286 causal=True,
1287 - return_lse=True, # do not support False for now
1288 - is_cuda_graph_compatible=True, # Indicates actual_seq_lens are on GPU or CPU.
1287 + # Do not support False for now
1288 + return_lse=True,
1289 + # Indicates actual_seq_lens are on GPU or CPU.
1290 + is_cuda_graph_compatible=True,
1291 )
1292 if return_softmax_lse:
1293 return output, lse
@@ -1342,7 +1344,8 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
1344 ),
1345 causal=False,
1346 return_lse=True,
1345 - is_cuda_graph_compatible=True, # Indicates actual_seq_lens are on GPU or CPU.
1347 + # Indicates actual_seq_lens are on GPU or CPU.
1348 + is_cuda_graph_compatible=True,
1349 )
1350
1351 def process_weights_after_loading(self, act_dtype: torch.dtype):
vllm/v1/engine/utils.py
+7 -4
@@ -872,10 +872,13 @@ def wait_for_engine_startup(
872 EngineHandshakeMetadata(
873 addresses=addresses,
874 parallel_config={
875 - "data_parallel_master_ip": parallel_config.data_parallel_master_ip,
876 - "data_parallel_master_port": parallel_config.data_parallel_master_port,
877 - "_data_parallel_master_port_list": parallel_config._data_parallel_master_port_list,
878 - "data_parallel_size": parallel_config.data_parallel_size,
875 + k: getattr(parallel_config, k)
876 + for k in (
877 + "data_parallel_master_ip",
878 + "data_parallel_master_port",
879 + "_data_parallel_master_port_list",
880 + "data_parallel_size",
881 + )
882 },
883 )
884 )
vllm/v1/utils.py
+4 -2
@@ -345,13 +345,15 @@ def report_usage_stats(
345
346 from vllm.model_executor.model_loader import get_architecture_class_name
347
348 + parallel_config = vllm_config.parallel_config
349 +
350 usage_message.report_usage(
351 get_architecture_class_name(vllm_config.model_config),
352 usage_context,
353 extra_kvs={
354 # Common configuration
355 "dtype": str(vllm_config.model_config.dtype),
354 - "tensor_parallel_size": vllm_config.parallel_config.tensor_parallel_size,
356 + "tensor_parallel_size": parallel_config.tensor_parallel_size,
357 "block_size": vllm_config.cache_config.block_size,
358 "gpu_memory_utilization": vllm_config.cache_config.gpu_memory_utilization,
359 "kv_cache_memory_bytes": vllm_config.cache_config.kv_cache_memory_bytes,
@@ -362,7 +364,7 @@ def report_usage_stats(
364 "enable_lora": bool(vllm_config.lora_config),
365 "enable_prefix_caching": vllm_config.cache_config.enable_prefix_caching,
366 "enforce_eager": vllm_config.model_config.enforce_eager,
365 - "disable_custom_all_reduce": vllm_config.parallel_config.disable_custom_all_reduce,
367 + "disable_custom_all_reduce": parallel_config.disable_custom_all_reduce,
368 },
369 )
370
vllm/v1/worker/gpu_model_runner.py
+2 -1
@@ -3391,7 +3391,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3391 attn_metadata[ubid][layer_name] = attn_metadata_i
3392 else:
3393 assert type(attn_metadata) is dict
3394 - attn_metadata_i = attn_group.get_metadata_builder().build_for_cudagraph_capture(
3394 + metadata_builder = attn_group.get_metadata_builder()
3395 + attn_metadata_i = metadata_builder.build_for_cudagraph_capture(
3396 common_attn_metadata
3397 )
3398 for layer_name in attn_group.layer_names: