A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 09:45 UTC
[Misc] Remove `yapf` directives (#29675)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 28, 2025 at 23:07 UTC
0808eb813b6b219324092ab8cc25d3223e5ccb77
2 files changed
+52
-43
vllm/distributed/ec_transfer/ec_connector/factory.py
-3
@@ -5,15 +5,12 @@ import importlib
5
from collections.abc import Callable
6
from typing import TYPE_CHECKING
7
8
-# yapf: disable
8
from vllm.distributed.ec_transfer.ec_connector.base import (
9
ECConnectorBase,
10
ECConnectorRole,
11
)
12
from vllm.logger import init_logger
13
15
-# yapf: enable
16
-
14
if TYPE_CHECKING:
15
from vllm.config import ECTransferConfig, VllmConfig
16
vllm/entrypoints/openai/serving_tokens.py
+52
-40
@@ -7,7 +7,6 @@ from collections.abc import Sequence as GenericSequence
7
8
from fastapi import Request
9
10
-# yapf: disable
10
from vllm.engine.protocol import EngineClient
11
from vllm.entrypoints.logger import RequestLogger
12
from vllm.entrypoints.openai.protocol import (
@@ -49,22 +48,26 @@ class ServingTokens(OpenAIServing):
48
enable_prompt_tokens_details: bool = False,
49
enable_log_outputs: bool = False,
50
):
52
- super().__init__(engine_client=engine_client,
53
- models=models,
54
- request_logger=request_logger,
55
- return_tokens_as_token_ids=return_tokens_as_token_ids,
56
- log_error_stack=log_error_stack)
51
+ super().__init__(
52
+ engine_client=engine_client,
53
+ models=models,
54
+ request_logger=request_logger,
55
+ return_tokens_as_token_ids=return_tokens_as_token_ids,
56
+ log_error_stack=log_error_stack,
57
+ )
58
self.enable_prompt_tokens_details = enable_prompt_tokens_details
59
self.enable_log_outputs = enable_log_outputs
60
self.force_no_detokenize = force_no_detokenize
61
if force_no_detokenize:
61
- logger.info("Tokens-only mode is enabled, skipping detokenization "
62
- "step for incoming requests.")
62
+ logger.info(
63
+ "Tokens-only mode is enabled, skipping detokenization "
64
+ "step for incoming requests."
65
+ )
66
67
async def serve_tokens(
68
self,
69
request: GenerateRequest,
67
- raw_request: Request | None = None
70
+ raw_request: Request | None = None,
71
) -> GenerateResponse | ErrorResponse:
72
error_check_ret = await self._check_model(request)
73
if error_check_ret is not None:
@@ -78,13 +81,13 @@ class ServingTokens(OpenAIServing):
81
raise self.engine_client.dead_error
82
83
lora_request = None
81
- lora_request = self._maybe_get_adapters(request,
82
- supports_default_mm_loras=True)
84
+ lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
85
86
model_name = self.models.model_name(lora_request)
87
86
- request_id = "generate-tokens-" \
87
- f"{self._base_request_id(raw_request, request.request_id)}"
88
+ request_id = (
89
+ f"generate-tokens-{self._base_request_id(raw_request, request.request_id)}"
90
+ )
91
92
request_metadata = RequestResponseMetadata(request_id=request_id)
93
if raw_request:
@@ -106,13 +109,18 @@ class ServingTokens(OpenAIServing):
109
if self.force_no_detokenize:
110
sampling_params.detokenize = False
111
109
- self._log_inputs(request_id,
110
- request.token_ids,
111
- params=sampling_params,
112
- lora_request=lora_request)
112
+ self._log_inputs(
113
+ request_id,
114
+ request.token_ids,
115
+ params=sampling_params,
116
+ lora_request=lora_request,
117
+ )
118
114
- trace_headers = (None if raw_request is None else await
115
- self._get_trace_headers(raw_request.headers))
119
+ trace_headers = (
120
+ None
121
+ if raw_request is None
122
+ else await self._get_trace_headers(raw_request.headers)
123
+ )
124
125
result_generator = self.engine_client.generate(
126
engine_prompt,
@@ -131,8 +139,8 @@ class ServingTokens(OpenAIServing):
139
try:
140
assert result_generator is not None
141
return await self.serve_tokens_full_generator(
134
- request, result_generator, request_id, model_name,
135
- request_metadata)
142
+ request, result_generator, request_id, model_name, request_metadata
143
+ )
144
except ValueError as e:
145
return self.create_error_response(str(e))
146
@@ -144,7 +152,6 @@ class ServingTokens(OpenAIServing):
152
model_name: str,
153
request_metadata: RequestResponseMetadata,
154
) -> ErrorResponse | GenerateResponse:
147
-
155
created_time = int(time.time())
156
final_res: RequestOutput | None = None
157
sampling_params: SamplingParams = request.sampling_params
@@ -179,9 +186,9 @@ class ServingTokens(OpenAIServing):
186
choice_data = GenerateResponseChoice(
187
index=output.index,
188
logprobs=logprobs,
182
- finish_reason=output.finish_reason
183
- if output.finish_reason else "stop",
184
- token_ids=as_list(output.token_ids))
189
+ finish_reason=output.finish_reason if output.finish_reason else "stop",
190
+ token_ids=as_list(output.token_ids),
191
+ )
192
193
choices.append(choice_data)
194
num_generated_tokens += len(output.token_ids)
@@ -191,14 +198,16 @@ class ServingTokens(OpenAIServing):
198
if final_res.encoder_prompt_token_ids is not None:
199
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
200
194
- usage = UsageInfo(prompt_tokens=num_prompt_tokens,
195
- completion_tokens=num_generated_tokens,
196
- total_tokens=num_prompt_tokens +
197
- num_generated_tokens)
201
+ usage = UsageInfo(
202
+ prompt_tokens=num_prompt_tokens,
203
+ completion_tokens=num_generated_tokens,
204
+ total_tokens=num_prompt_tokens + num_generated_tokens,
205
+ )
206
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
207
# This info is not available at the /coordinator level
208
usage.prompt_tokens_details = PromptTokenUsageInfo(
201
- cached_tokens=final_res.num_cached_tokens)
209
+ cached_tokens=final_res.num_cached_tokens
210
+ )
211
212
request_metadata.final_usage_info = usage
213
@@ -218,14 +227,13 @@ class ServingTokens(OpenAIServing):
227
# Get the corresponding output token IDs
228
output_token_ids = None
229
if choice.index < len(final_res.outputs):
221
- output_token_ids = final_res.outputs[
222
- choice.index].token_ids
230
+ output_token_ids = final_res.outputs[choice.index].token_ids
231
232
if output_token_ids:
233
# Log token_ids only.
234
self.request_logger.log_outputs(
235
request_id=request_id,
228
- outputs="",
236
+ outputs="",
237
output_token_ids=output_token_ids,
238
finish_reason=choice.finish_reason,
239
is_streaming=False,
@@ -246,10 +254,12 @@ class ServingTokens(OpenAIServing):
254
for i, token_id in enumerate(token_ids):
255
token = f"token_id:{token_id}"
256
step_top_logprobs = top_logprobs[i]
249
- if step_top_logprobs is None or step_top_logprobs.get(
250
- token_id) is None:
257
+ if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
258
logprobs_content.append(
252
- ChatCompletionLogProbsContent(token=token, ))
259
+ ChatCompletionLogProbsContent(
260
+ token=token,
261
+ )
262
+ )
263
else:
264
step_token = step_top_logprobs[token_id]
265
@@ -261,9 +271,11 @@ class ServingTokens(OpenAIServing):
271
ChatCompletionLogProb(
272
token=token,
273
logprob=max(p[1].logprob, -9999.0),
264
- ) for i, p in enumerate(step_top_logprobs.items())
265
- if num_output_top_logprobs
266
- and i < num_output_top_logprobs
267
- ]))
274
+ )
275
+ for i, p in enumerate(step_top_logprobs.items())
276
+ if num_output_top_logprobs and i < num_output_top_logprobs
277
+ ],
278
+ )
279
+ )
280
281
return ChatCompletionLogProbs(content=logprobs_content)