@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 812b7f54a8

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Renderer] Move Processor out of AsyncLLM (#24138)

Signed-off-by: Yang <lymailforjob@gmail.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Yang Liu committed Oct 3, 2025 at 04:29 UTC 812b7f54a86acf04f9d556821dedb2086b9c0401

7 files changed +217 -127

tests/entrypoints/openai/test_lora_resolvers.py

@@ -122,6 +122,9 @@ def mock_serving_setup():

122 models,

123 request_logger=None)

124

125 + serving_completion._process_inputs = AsyncMock(return_value=(MagicMock(

126 + name="engine_request"), {}))

127 +

128 return mock_engine, serving_completion

129

130

tests/entrypoints/openai/test_serving_chat.py

+34 -64

@@ -7,7 +7,7 @@ import asyncio

7 from contextlib import suppress

8 from dataclasses import dataclass, field

9 from typing import TYPE_CHECKING, Any, Optional

10 -from unittest.mock import MagicMock

10 +from unittest.mock import AsyncMock, MagicMock

12 import pytest

13 import pytest_asyncio

@@ -230,6 +230,7 @@ class MockHFConfig:

230 @dataclass

231 class MockModelConfig:

232 task = "generate"

233 + runner_type = "generate"

234 tokenizer = MODEL_NAME

235 trust_remote_code = False

236 tokenizer_mode = "auto"

@@ -244,11 +245,33 @@ class MockModelConfig:

245 encoder_config = None

246 generation_config: str = "auto"

247 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

248 + skip_tokenizer_init = False

249

250 def get_diff_sampling_param(self):

251 return self.diff_sampling_param or {}

252

253

254 +def _build_serving_chat(engine: AsyncLLM,

255 + model_config: MockModelConfig) -> OpenAIServingChat:

256 + models = OpenAIServingModels(engine_client=engine,

257 + base_model_paths=BASE_MODEL_PATHS,

258 + model_config=model_config)

259 + serving_chat = OpenAIServingChat(engine,

260 + model_config,

261 + models,

262 + response_role="assistant",

263 + chat_template=CHAT_TEMPLATE,

264 + chat_template_content_format="auto",

265 + request_logger=None)

266 +

267 + async def _fake_process_inputs(request_id, engine_prompt, sampling_params,

268 + *, lora_request, trace_headers, priority):

269 + return dict(engine_prompt), {}

270 +

271 + serving_chat._process_inputs = AsyncMock(side_effect=_fake_process_inputs)

272 + return serving_chat

273 +

274 +

275 @dataclass

276 class MockEngine:

277

@@ -282,16 +305,7 @@ async def test_serving_chat_returns_correct_model_name():

305 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

306 mock_engine.errored = False

307

285 - models = OpenAIServingModels(engine_client=mock_engine,

286 - base_model_paths=BASE_MODEL_PATHS,

287 - model_config=MockModelConfig())

288 - serving_chat = OpenAIServingChat(mock_engine,

289 - MockModelConfig(),

290 - models,

291 - response_role="assistant",

292 - chat_template=CHAT_TEMPLATE,

293 - chat_template_content_format="auto",

294 - request_logger=None)

308 + serving_chat = _build_serving_chat(mock_engine, MockModelConfig())

309 messages = [{"role": "user", "content": "what is 1+1?"}]

310

311 async def return_model_name(*args):

@@ -318,16 +332,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

332 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

333 mock_engine.errored = False

334

321 - models = OpenAIServingModels(engine_client=mock_engine,

322 - base_model_paths=BASE_MODEL_PATHS,

323 - model_config=MockModelConfig())

324 - serving_chat = OpenAIServingChat(mock_engine,

325 - MockModelConfig(),

326 - models,

327 - response_role="assistant",

328 - chat_template=CHAT_TEMPLATE,

329 - chat_template_content_format="auto",

330 - request_logger=None)

335 + serving_chat = _build_serving_chat(mock_engine, MockModelConfig())

336

337 req = ChatCompletionRequest(

338 model=MODEL_NAME,

@@ -361,16 +366,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

366 mock_engine.errored = False

367

368 # Initialize the serving chat

364 - models = OpenAIServingModels(engine_client=mock_engine,

365 - base_model_paths=BASE_MODEL_PATHS,

366 - model_config=mock_model_config)

367 - serving_chat = OpenAIServingChat(mock_engine,

368 - mock_model_config,

369 - models,

370 - response_role="assistant",

371 - chat_template=CHAT_TEMPLATE,

372 - chat_template_content_format="auto",

373 - request_logger=None)

369 + serving_chat = _build_serving_chat(mock_engine, mock_model_config)

370

371 # Test Case 1: No max_tokens specified in request

372 req = ChatCompletionRequest(

@@ -415,16 +411,7 @@ async def test_serving_chat_should_set_correct_max_tokens():

411 mock_engine.errored = False

412

413 # Initialize the serving chat

418 - models = OpenAIServingModels(engine_client=mock_engine,

419 - base_model_paths=BASE_MODEL_PATHS,

420 - model_config=mock_model_config)

421 - serving_chat = OpenAIServingChat(mock_engine,

422 - mock_model_config,

423 - models,

424 - response_role="assistant",

425 - chat_template=CHAT_TEMPLATE,

426 - chat_template_content_format="auto",

427 - request_logger=None)

414 + serving_chat = _build_serving_chat(mock_engine, mock_model_config)

415

416 # Test case 1: No max_tokens specified, defaults to context_window

417 req = ChatCompletionRequest(

@@ -471,16 +458,7 @@ async def test_serving_chat_could_load_correct_generation_config():

458 mock_engine.errored = False

459

460 # Initialize the serving chat

474 - models = OpenAIServingModels(engine_client=mock_engine,

475 - base_model_paths=BASE_MODEL_PATHS,

476 - model_config=mock_model_config)

477 - serving_chat = OpenAIServingChat(mock_engine,

478 - mock_model_config,

479 - models,

480 - response_role="assistant",

481 - chat_template=CHAT_TEMPLATE,

482 - chat_template_content_format="auto",

483 - request_logger=None)

461 + serving_chat = _build_serving_chat(mock_engine, mock_model_config)

462

463 req = ChatCompletionRequest(

464 model=MODEL_NAME,

@@ -525,17 +503,7 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

503 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

504 mock_engine.errored = False

505

528 - # Initialize the serving chat

529 - models = OpenAIServingModels(engine_client=mock_engine,

530 - base_model_paths=BASE_MODEL_PATHS,

531 - model_config=mock_model_config)

532 - serving_chat = OpenAIServingChat(mock_engine,

533 - mock_model_config,

534 - models,

535 - response_role="assistant",

536 - chat_template=CHAT_TEMPLATE,

537 - chat_template_content_format="auto",

538 - request_logger=None)

506 + serving_chat = _build_serving_chat(mock_engine, mock_model_config)

507

508 # Test cache_salt

509 req = ChatCompletionRequest(

@@ -549,10 +517,12 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):

517 # By default, cache_salt in the engine prompt is not set

518 with suppress(Exception):

519 await serving_chat.create_chat_completion(req)

552 - assert "cache_salt" not in mock_engine.generate.call_args.args[0]

520 + engine_prompt = serving_chat._process_inputs.await_args_list[0].args[1]

521 + assert "cache_salt" not in engine_prompt

522

523 # Test with certain cache_salt

524 req.cache_salt = "test_salt"

525 with suppress(Exception):

526 await serving_chat.create_chat_completion(req)

558 - assert mock_engine.generate.call_args.args[0]["cache_salt"] == "test_salt"

527 + engine_prompt = serving_chat._process_inputs.await_args_list[1].args[1]

528 + assert engine_prompt.get("cache_salt") == "test_salt"

vllm/engine/protocol.py

+6 -1

@@ -19,6 +19,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams

19 from vllm.tasks import SupportedTask

20 from vllm.transformers_utils.tokenizer import AnyTokenizer

21 from vllm.utils import Device, collect_from_async_generator, random_uuid

22 +from vllm.v1.engine import EngineCoreRequest

24 logger = init_logger(__name__)

@@ -49,12 +50,16 @@ class EngineClient(ABC):

50 @abstractmethod

51 def generate(

52 self,

52 - prompt: PromptType,

53 + prompt: Union[EngineCoreRequest, PromptType],

54 sampling_params: SamplingParams,

55 request_id: str,

56 + *,

57 + prompt_text: Optional[str] = None,

58 lora_request: Optional[LoRARequest] = None,

59 + tokenization_kwargs: Optional[dict[str, Any]] = None,

60 trace_headers: Optional[Mapping[str, str]] = None,

61 priority: int = 0,

62 + data_parallel_rank: Optional[int] = None,

63 ) -> AsyncGenerator[RequestOutput, None]:

64 """Generate outputs for a request."""

65 ...

vllm/entrypoints/openai/serving_chat.py

+16 -2

@@ -274,7 +274,8 @@ class OpenAIServingChat(OpenAIServing):

274 generators: list[AsyncGenerator[RequestOutput, None]] = []

275 try:

276 for i, engine_prompt in enumerate(engine_prompts):

277 - sampling_params: Union[SamplingParams, BeamSearchParams]

277 + prompt_text, _, _ = (self._get_prompt_components(

278 + request_prompts[i]))

279

280 if self.default_sampling_params is None:

281 self.default_sampling_params = {}

@@ -285,6 +286,7 @@ class OpenAIServingChat(OpenAIServing):

286 input_length=len(engine_prompt["prompt_token_ids"]),

287 default_sampling_params=self.default_sampling_params)

288

289 + sampling_params: Union[SamplingParams, BeamSearchParams]

290 if request.use_beam_search:

291 sampling_params = request.to_beam_search_params(

292 max_tokens, self.default_sampling_params)

@@ -309,13 +311,25 @@ class OpenAIServingChat(OpenAIServing):

311 lora_request=lora_request,

312 )

313 else:

314 + engine_request, tokenization_kwargs = (

315 + await self._process_inputs(

316 + request_id,

317 + engine_prompt,

318 + sampling_params,

319 + lora_request=lora_request,

320 + trace_headers=trace_headers,

321 + priority=request.priority,

322 + ))

323 +

324 generator = self.engine_client.generate(

313 - engine_prompt,

325 + engine_request,

326 sampling_params,

327 request_id,

328 lora_request=lora_request,

329 trace_headers=trace_headers,

330 priority=request.priority,

331 + prompt_text=prompt_text,

332 + tokenization_kwargs=tokenization_kwargs,

333 )

334

335 generators.append(generator)

vllm/entrypoints/openai/serving_completion.py

+24 -20

@@ -9,7 +9,6 @@ from typing import Optional, Union, cast

10 import jinja2

11 from fastapi import Request

12 -from typing_extensions import assert_never

13 from vllm.config import ModelConfig

14 from vllm.engine.protocol import EngineClient

@@ -32,8 +31,7 @@ from vllm.entrypoints.openai.serving_engine import (OpenAIServing,

31 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

32 from vllm.entrypoints.renderer import RenderConfig

33 from vllm.entrypoints.utils import get_max_tokens

35 -from vllm.inputs.data import (EmbedsPrompt, TokensPrompt, is_embeds_prompt,

36 - is_tokens_prompt)

34 +from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt

35 from vllm.logger import init_logger

36 from vllm.logprobs import Logprob

37 from vllm.outputs import RequestOutput

@@ -157,23 +155,16 @@ class OpenAIServingCompletion(OpenAIServing):

155 generators: list[AsyncGenerator[RequestOutput, None]] = []

156 try:

157 for i, engine_prompt in enumerate(engine_prompts):

160 - sampling_params: Union[SamplingParams, BeamSearchParams]

161 - # Mypy does not infer that engine_prompt will have only one of

162 - # "prompt_token_ids" or "prompt_embeds" defined, and both of

163 - # these as Union[object, the expected type], where it infers

164 - # object if engine_prompt is a subclass of one of the

165 - # typeddicts that defines both keys. Worse, because of

166 - # https://github.com/python/mypy/issues/8586, mypy does not

167 - # infer the type of engine_prompt correctly because of the

168 - # enumerate. So we need an unnecessary cast here.

169 - engine_prompt = cast(Union[EmbedsPrompt, TokensPrompt],

170 - engine_prompt)

171 - if is_embeds_prompt(engine_prompt):

172 - input_length = len(engine_prompt["prompt_embeds"])

173 - elif is_tokens_prompt(engine_prompt):

174 - input_length = len(engine_prompt["prompt_token_ids"])

158 + prompt_text, prompt_token_ids, prompt_embeds = (

159 + self._get_prompt_components(engine_prompt))

160 +

161 + input_length = None

162 + if prompt_token_ids is not None:

163 + input_length = len(prompt_token_ids)

164 + elif prompt_embeds is not None:

165 + input_length = len(prompt_embeds)

166 else:

176 - assert_never(engine_prompt)

167 + raise NotImplementedError

168

169 if self.default_sampling_params is None:

170 self.default_sampling_params = {}

@@ -185,6 +176,7 @@ class OpenAIServingCompletion(OpenAIServing):

176 default_sampling_params=self.default_sampling_params,

177 )

178

179 + sampling_params: Union[SamplingParams, BeamSearchParams]

180 if request.use_beam_search:

181 sampling_params = request.to_beam_search_params(

182 max_tokens, self.default_sampling_params)

@@ -220,13 +212,25 @@ class OpenAIServingCompletion(OpenAIServing):

212 lora_request=lora_request,

213 )

214 else:

215 + engine_request, tokenization_kwargs = (

216 + await self._process_inputs(

217 + request_id_item,

218 + engine_prompt,

219 + sampling_params,

220 + lora_request=lora_request,

221 + trace_headers=trace_headers,

222 + priority=request.priority,

223 + ))

224 +

225 generator = self.engine_client.generate(

224 - engine_prompt,

226 + engine_request,

227 sampling_params,

228 request_id_item,

229 lora_request=lora_request,

230 trace_headers=trace_headers,

231 priority=request.priority,

232 + prompt_text=prompt_text,

233 + tokenization_kwargs=tokenization_kwargs,

234 )

235

236 generators.append(generator)

vllm/entrypoints/openai/serving_engine.py

+93 -10

@@ -7,7 +7,8 @@ import traceback

7 from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence

8 from concurrent.futures import ThreadPoolExecutor

9 from http import HTTPStatus

10 -from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union

10 +from typing import (Any, Callable, ClassVar, Generic, NamedTuple, Optional,

11 + TypeVar, Union)

13 import torch

14 from fastapi import Request

@@ -15,6 +16,11 @@ from pydantic import BaseModel, ConfigDict, Field

16 from starlette.datastructures import Headers

17 from typing_extensions import TypeIs

19 +from vllm.entrypoints.utils import _validate_truncation_size

20 +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs

21 +from vllm.v1.engine import EngineCoreRequest

22 +from vllm.v1.engine.processor import Processor

23 +

24 if sys.version_info >= (3, 12):

25 from typing import TypedDict

26 else:

@@ -134,6 +140,12 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:

140 and "prompt_embeds" in prompt)

141

142

143 +class PromptComponents(NamedTuple):

144 + text: Optional[str] = None

145 + token_ids: Optional[list[int]] = None

146 + embeds: Optional[torch.Tensor] = None

147 +

148 +

149 RequestT = TypeVar("RequestT", bound=AnyRequest)

150

151

@@ -239,6 +251,16 @@ class OpenAIServing:

251 AsyncMicrobatchTokenizer] = {}

252 self.log_error_stack = log_error_stack

253

254 + async def _get_processor(self) -> Processor:

255 + if not hasattr(self, "_processor"):

256 + vllm_config = await self.engine_client.get_vllm_config()

257 + if self.model_config.skip_tokenizer_init:

258 + tokenizer = None

259 + else:

260 + tokenizer = init_tokenizer_from_configs(self.model_config)

261 + self._processor = Processor(vllm_config, tokenizer)

262 + return self._processor

263 +

264 def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:

265 """

266 Get a Renderer instance with the provided tokenizer.

@@ -850,6 +872,36 @@ class OpenAIServing:

872

873 return conversation, [request_prompt], [engine_prompt]

874

875 + async def _process_inputs(

876 + self,

877 + request_id: str,

878 + engine_prompt: PromptType,

879 + sampling_params: SamplingParams,

880 + *,

881 + lora_request: Optional[LoRARequest],

882 + trace_headers: Optional[Mapping[str, str]],

883 + priority: int,

884 + ) -> tuple[EngineCoreRequest, dict[str, Any]]:

885 + """

886 + using the Processor to process inputs for AsyncLLM

887 + """

888 + tokenization_kwargs: dict[str, Any] = {}

889 + _validate_truncation_size(self.max_model_len,

890 + sampling_params.truncate_prompt_tokens,

891 + tokenization_kwargs)

892 +

893 + processor = await self._get_processor()

894 + engine_request = processor.process_inputs(

895 + request_id,

896 + engine_prompt,

897 + sampling_params,

898 + lora_request=lora_request,

899 + tokenization_kwargs=tokenization_kwargs,

900 + trace_headers=trace_headers,

901 + priority=priority,

902 + )

903 + return engine_request, tokenization_kwargs

904 +

905 async def _generate_with_builtin_tools(

906 self,

907 request_id: str,

@@ -861,6 +913,7 @@ class OpenAIServing:

913 priority: int = 0,

914 **kwargs,

915 ):

916 + prompt_text, _, _ = self._get_prompt_components(request_prompt)

917 orig_priority = priority

918 while True:

919 self._log_inputs(

@@ -869,14 +922,27 @@ class OpenAIServing:

922 params=sampling_params,

923 lora_request=lora_request,

924 )

872 - generator = self.engine_client.generate(

925 + trace_headers = kwargs.get("trace_headers")

926 + engine_request, tokenization_kwargs = (await self._process_inputs(

927 + request_id,

928 engine_prompt,

929 sampling_params,

930 + lora_request=lora_request,

931 + trace_headers=trace_headers,

932 + priority=priority,

933 + ))

934 +

935 + generator = self.engine_client.generate(

936 + engine_request,

937 + sampling_params,

938 request_id,

939 lora_request=lora_request,

940 priority=priority,

941 + prompt_text=prompt_text,

942 + tokenization_kwargs=tokenization_kwargs,

943 **kwargs,

944 )

945 +

946 async for res in generator:

947 context.append_output(res)

948 # NOTE(woosuk): The stop condition is handled by the engine.

@@ -905,6 +971,28 @@ class OpenAIServing:

971 # OPTIMIZATION

972 priority = orig_priority - 1

973

974 + def _get_prompt_components(

975 + self,

976 + inputs: Union[RequestPrompt, PromptType],

977 + ) -> PromptComponents:

978 + if isinstance(inputs, str):

979 + return PromptComponents(text=inputs)

980 + if isinstance(inputs, list):

981 + return PromptComponents(token_ids=inputs)

982 + if isinstance(inputs, dict):

983 + return PromptComponents(

984 + text=inputs.get("prompt"), # type: ignore[arg-type]

985 + token_ids=inputs.get(

986 + "prompt_token_ids"), # type: ignore[arg-type]

987 + embeds=inputs.get("prompt_embeds"),

988 + )

989 +

990 + return PromptComponents(

991 + text=getattr(inputs, "prompt", None),

992 + token_ids=getattr(inputs, "prompt_token_ids", None),

993 + embeds=getattr(inputs, "prompt_embeds", None),

994 + )

995 +

996 def _log_inputs(

997 self,

998 request_id: str,

@@ -915,14 +1003,9 @@ class OpenAIServing:

1003 ) -> None:

1004 if self.request_logger is None:

1005 return

918 - prompt, prompt_token_ids, prompt_embeds = None, None, None

919 - if isinstance(inputs, str):

920 - prompt = inputs

921 - elif isinstance(inputs, list):

922 - prompt_token_ids = inputs

923 - else:

924 - prompt = getattr(inputs, 'prompt', None)

925 - prompt_token_ids = getattr(inputs, 'prompt_token_ids', None)

1006 +

1007 + prompt, prompt_token_ids, prompt_embeds = (

1008 + self._get_prompt_components(inputs))

1009

1010 self.request_logger.log_inputs(

1011 request_id,

vllm/v1/engine/async_llm.py

+41 -30

@@ -261,7 +261,7 @@ class AsyncLLM(EngineClient):

261 async def add_request(

262 self,

263 request_id: str,

264 - prompt: PromptType,

264 + prompt: Union[EngineCoreRequest, PromptType],

265 params: Union[SamplingParams, PoolingParams],

266 arrival_time: Optional[float] = None,

267 lora_request: Optional[LoRARequest] = None,

@@ -269,6 +269,7 @@ class AsyncLLM(EngineClient):

269 trace_headers: Optional[Mapping[str, str]] = None,

270 priority: int = 0,

271 data_parallel_rank: Optional[int] = None,

272 + prompt_text: Optional[str] = None,

273 ) -> RequestOutputCollector:

274 """Add new request to the AsyncLLM."""

275

@@ -281,13 +282,20 @@ class AsyncLLM(EngineClient):

282 queue = RequestOutputCollector(output_kind=params.output_kind)

283

284 # Convert Input --> Request.

284 - request = self.processor.process_inputs(request_id, prompt, params,

285 - arrival_time, lora_request,

286 - tokenization_kwargs,

287 - trace_headers, priority,

288 - data_parallel_rank)

289 - prompt_text = prompt if isinstance(prompt,

290 - str) else prompt.get("prompt")

285 + if isinstance(prompt, EngineCoreRequest):

286 + request = prompt

287 + else:

288 + assert prompt_text is None

289 + logger.warning_once(

290 + "Processor has been moved under OpenAIServing and will "

291 + "be removed from AsyncLLM in v0.13.")

292 + request = self.processor.process_inputs(request_id, prompt, params,

293 + arrival_time, lora_request,

294 + tokenization_kwargs,

295 + trace_headers, priority,

296 + data_parallel_rank)

297 + prompt_text = (prompt if isinstance(prompt, str) else

298 + prompt.get("prompt"))

299

300 if is_pooling or params.n == 1:

301 await self._add_request(request, prompt_text, None, 0, queue)

@@ -332,10 +340,13 @@ class AsyncLLM(EngineClient):

340 # re-multiplexed in the API server anyhow.

341 async def generate(

342 self,

335 - prompt: PromptType,

343 + prompt: Union[EngineCoreRequest, PromptType],

344 sampling_params: SamplingParams,

345 request_id: str,

346 + *,

347 + prompt_text: Optional[str] = None,

348 lora_request: Optional[LoRARequest] = None,

349 + tokenization_kwargs: Optional[dict[str, Any]] = None,

350 trace_headers: Optional[Mapping[str, str]] = None,

351 priority: int = 0,

352 data_parallel_rank: Optional[int] = None,

@@ -368,25 +379,25 @@ class AsyncLLM(EngineClient):

379 # to handle startup failure gracefully in the OpenAI server.

380 self._run_output_handler()

381

371 - tokenization_kwargs: dict[str, Any] = {}

372 - truncate_prompt_tokens = sampling_params.truncate_prompt_tokens

373 -

374 - _validate_truncation_size(

375 - self.model_config.max_model_len,

376 - truncate_prompt_tokens,

377 - tokenization_kwargs,

378 - )

379 -

380 - q = await self.add_request(

381 - request_id,

382 - prompt,

383 - sampling_params,

384 - lora_request=lora_request,

385 - trace_headers=trace_headers,

386 - priority=priority,

387 - tokenization_kwargs=tokenization_kwargs,

388 - data_parallel_rank=data_parallel_rank,

389 - )

382 + if tokenization_kwargs is None:

383 + tokenization_kwargs = {}

384 + truncate_prompt_tokens = sampling_params.truncate_prompt_tokens

385 +

386 + _validate_truncation_size(

387 + self.model_config.max_model_len,

388 + truncate_prompt_tokens,

389 + tokenization_kwargs,

390 + )

391 +

392 + q = await self.add_request(request_id,

393 + prompt,

394 + sampling_params,

395 + lora_request=lora_request,

396 + tokenization_kwargs=tokenization_kwargs,

397 + trace_headers=trace_headers,

398 + priority=priority,

399 + data_parallel_rank=data_parallel_rank,

400 + prompt_text=prompt_text)

401

402 # The output_handler task pushes items into the queue.

403 # This task pulls from the queue and yields to caller.

@@ -535,7 +546,7 @@ class AsyncLLM(EngineClient):

546 self._run_output_handler()

547

548 if tokenization_kwargs is None:

538 - tokenization_kwargs = dict[str, Any]()

549 + tokenization_kwargs = {}

550 _validate_truncation_size(

551 self.model_config.max_model_len,

552 truncate_prompt_tokens,

@@ -547,9 +558,9 @@ class AsyncLLM(EngineClient):

558 prompt,

559 pooling_params,

560 lora_request=lora_request,

561 + tokenization_kwargs=tokenization_kwargs,

562 trace_headers=trace_headers,

563 priority=priority,

552 - tokenization_kwargs=tokenization_kwargs,

564 )

565

566 # The output_handler task pushes items into the queue.