A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Upgrade more code to Python 3.10 (#26463)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 9, 2025 at 17:43 UTC
dc7976dd9fd071497f85f44dddbd153ab49a923c
7 files changed
+16
-33
tests/entrypoints/openai/test_chat.py
+1
-1
@@ -369,7 +369,7 @@ async def test_chat_completion_stream_options(
369
assert chunk.usage is None
370
else:
371
assert chunk.usage is None
372
- final_chunk = await stream.__anext__()
372
+ final_chunk = await anext(stream)
373
assert final_chunk.usage is not None
374
assert final_chunk.usage.prompt_tokens > 0
375
assert final_chunk.usage.completion_tokens > 0
tests/entrypoints/test_context.py
+3
-9
@@ -10,12 +10,6 @@ from vllm.entrypoints.context import HarmonyContext, StreamingHarmonyContext
10
from vllm.outputs import CompletionOutput, RequestOutput
11
12
13
-# Helper function for Python < 3.10 compatibility
14
-async def async_next(async_iterator):
15
- """Compatibility function equivalent to Python 3.10's anext()."""
16
- return await async_iterator.__anext__()
17
-
18
-
13
def create_mock_request_output(
14
prompt_token_ids=None,
15
output_token_ids=None,
@@ -129,7 +123,7 @@ async def test_multi_turn_token_counting():
123
)
124
125
# First turn - initial prompt and response
132
- mock_output1 = await async_next(mock_generator)
126
+ mock_output1 = await anext(mock_generator)
127
context.append_output(mock_output1)
128
129
# At this point, we should have 5 prompt tokens and 3 output tokens
@@ -138,7 +132,7 @@ async def test_multi_turn_token_counting():
132
assert context.num_tool_output_tokens == 0
133
134
# Second turn - after tool output
141
- mock_output2 = await async_next(mock_generator)
135
+ mock_output2 = await anext(mock_generator)
136
context.append_output(mock_output2)
137
# Current prompt tokens (15) - last_turn_input_tokens (5) -
138
# last_turn_output_tokens (3) = 7
@@ -150,7 +144,7 @@ async def test_multi_turn_token_counting():
144
assert context.num_cached_tokens == 5
145
146
# Third turn - final response
153
- mock_output3 = await async_next(mock_generator)
147
+ mock_output3 = await anext(mock_generator)
148
context.append_output(mock_output3)
149
# Additional tool output tokens from third turn:
150
# Current prompt (20) - last_turn_input_tokens (15) -
tests/utils_/test_utils.py
+1
-2
@@ -75,8 +75,7 @@ async def test_merge_async_iterators():
75
76
for iterator in iterators:
77
try:
78
- # Can use anext() in python >= 3.10
79
- await asyncio.wait_for(iterator.__anext__(), 1)
78
+ await asyncio.wait_for(anext(iterator), 1)
79
except StopAsyncIteration:
80
# All iterators should be cancelled and print this message.
81
print("Iterator was cancelled normally")
tests/v1/entrypoints/openai/test_completion.py
+2
-2
@@ -420,7 +420,7 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI, model_name:
420
assert chunk.usage is None
421
else:
422
assert chunk.usage is None
423
- final_chunk = await stream.__anext__()
423
+ final_chunk = await anext(stream)
424
assert final_chunk.usage is not None
425
assert final_chunk.usage.prompt_tokens > 0
426
assert final_chunk.usage.completion_tokens > 0
@@ -450,7 +450,7 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI, model_name:
450
chunk.usage.prompt_tokens + chunk.usage.completion_tokens
451
)
452
if chunk.choices[0].finish_reason is not None:
453
- final_chunk = await stream.__anext__()
453
+ final_chunk = await anext(stream)
454
assert final_chunk.usage is not None
455
assert final_chunk.usage.prompt_tokens > 0
456
assert final_chunk.usage.completion_tokens > 0
vllm/benchmarks/serve.py
+6
-9
@@ -18,6 +18,7 @@ On the client side, run:
18
19
import argparse
20
import asyncio
21
+import contextlib
22
import gc
23
import importlib.util
24
import json
@@ -605,17 +606,13 @@ async def benchmark(
606
607
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
608
608
- # This can be used once the minimum Python version is 3.10 or higher,
609
- # and it will simplify the code in limited_request_func.
610
- # semaphore = (asyncio.Semaphore(max_concurrency)
611
- # if max_concurrency else contextlib.nullcontext())
612
- semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None
609
+ semaphore = (
610
+ asyncio.Semaphore(max_concurrency)
611
+ if max_concurrency
612
+ else contextlib.nullcontext()
613
+ )
614
615
async def limited_request_func(request_func_input, session, pbar):
615
- if semaphore is None:
616
- return await request_func(
617
- request_func_input=request_func_input, session=session, pbar=pbar
618
- )
616
async with semaphore:
617
return await request_func(
618
request_func_input=request_func_input, session=session, pbar=pbar
vllm/utils/__init__.py
+2
-7
@@ -469,11 +469,6 @@ def make_async(
469
return _async_wrapper
470
471
472
-def _next_task(iterator: AsyncGenerator[T, None], loop: AbstractEventLoop) -> Task:
473
- # Can use anext() in python >= 3.10
474
- return loop.create_task(iterator.__anext__()) # type: ignore[arg-type]
475
-
476
-
472
async def merge_async_iterators(
473
*iterators: AsyncGenerator[T, None],
474
) -> AsyncGenerator[tuple[int, T], None]:
@@ -491,7 +486,7 @@ async def merge_async_iterators(
486
487
loop = asyncio.get_running_loop()
488
494
- awaits = {_next_task(pair[1], loop): pair for pair in enumerate(iterators)}
489
+ awaits = {loop.create_task(anext(it)): (i, it) for i, it in enumerate(iterators)}
490
try:
491
while awaits:
492
done, _ = await asyncio.wait(awaits.keys(), return_when=FIRST_COMPLETED)
@@ -500,7 +495,7 @@ async def merge_async_iterators(
495
try:
496
item = await d
497
i, it = pair
503
- awaits[_next_task(it, loop)] = pair
498
+ awaits[loop.create_task(anext(it))] = pair
499
yield i, item
500
except StopAsyncIteration:
501
pass
vllm/v1/serial_utils.py
+1
-3
@@ -290,9 +290,7 @@ class MsgpackDecoder:
290
_log_insecure_serialization_warning()
291
292
def decode(self, bufs: Union[bytestr, Sequence[bytestr]]) -> Any:
293
- if isinstance(bufs, (bytes, bytearray, memoryview, zmq.Frame)):
294
- # TODO - This check can become `isinstance(bufs, bytestr)`
295
- # as of Python 3.10.
293
+ if isinstance(bufs, bytestr): # type: ignore
294
return self.decoder.decode(bufs)
295
296
self.aux_buffers = bufs