@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a37d75bbec

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Front-end] microbatch tokenization (#19334)

Signed-off-by: zt2370 <ztang2370@gmail.com>

ztang2370 committed Jul 8, 2025 at 00:54 UTC a37d75bbec7ea1a1cadb738bf54b915875855428

3 files changed +288 -64

tests/entrypoints/openai/test_serving_chat.py

+23 -16

@@ -7,6 +7,8 @@ from dataclasses import dataclass, field

7 from typing import Any, Optional

8 from unittest.mock import MagicMock

10 +import pytest

11 +

12 from vllm.config import MultiModalConfig

13 from vllm.engine.multiprocessing.client import MQLLMEngineClient

14 from vllm.entrypoints.openai.protocol import ChatCompletionRequest

@@ -73,7 +75,8 @@ def test_async_serving_chat_init():

75 assert serving_completion.chat_template == CHAT_TEMPLATE

76 -def test_serving_chat_should_set_correct_max_tokens():

78 +@pytest.mark.asyncio

79 +async def test_serving_chat_should_set_correct_max_tokens():

80 mock_engine = MagicMock(spec=MQLLMEngineClient)

81 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)

82 mock_engine.errored = False

@@ -88,6 +91,7 @@ def test_serving_chat_should_set_correct_max_tokens():

91 chat_template=CHAT_TEMPLATE,

92 chat_template_content_format="auto",

93 request_logger=None)

94 +

95 req = ChatCompletionRequest(

96 model=MODEL_NAME,

97 messages=[{

@@ -98,13 +102,13 @@ def test_serving_chat_should_set_correct_max_tokens():

102 )

103

104 with suppress(Exception):

101 - asyncio.run(serving_chat.create_chat_completion(req))

105 + await serving_chat.create_chat_completion(req)

106

107 assert mock_engine.generate.call_args.args[1].max_tokens == 93

108

109 req.max_tokens = 10

110 with suppress(Exception):

107 - asyncio.run(serving_chat.create_chat_completion(req))

111 + await serving_chat.create_chat_completion(req)

112

113 assert mock_engine.generate.call_args.args[1].max_tokens == 10

114

@@ -143,7 +147,7 @@ def test_serving_chat_should_set_correct_max_tokens():

147 )

148

149 with suppress(Exception):

146 - asyncio.run(serving_chat.create_chat_completion(req))

150 + await serving_chat.create_chat_completion(req)

151

152 assert mock_engine.generate.call_args.args[1].max_tokens == 10

153

@@ -151,7 +155,7 @@ def test_serving_chat_should_set_correct_max_tokens():

155 req.max_tokens = 15

156

157 with suppress(Exception):

154 - asyncio.run(serving_chat.create_chat_completion(req))

158 + await serving_chat.create_chat_completion(req)

159

160 assert mock_engine.generate.call_args.args[1].max_tokens == 10

161

@@ -159,7 +163,7 @@ def test_serving_chat_should_set_correct_max_tokens():

163 req.max_tokens = 5

164

165 with suppress(Exception):

162 - asyncio.run(serving_chat.create_chat_completion(req))

166 + await serving_chat.create_chat_completion(req)

167

168 assert mock_engine.generate.call_args.args[1].max_tokens == 5

169

@@ -198,7 +202,7 @@ def test_serving_chat_should_set_correct_max_tokens():

202 )

203

204 with suppress(Exception):

201 - asyncio.run(serving_chat.create_chat_completion(req))

205 + await serving_chat.create_chat_completion(req)

206

207 assert mock_engine.generate.call_args.args[1].max_tokens == 93

208

@@ -206,7 +210,7 @@ def test_serving_chat_should_set_correct_max_tokens():

210 req.max_tokens = 100

211

212 with suppress(Exception):

209 - asyncio.run(serving_chat.create_chat_completion(req))

213 + await serving_chat.create_chat_completion(req)

214

215 assert mock_engine.generate.call_args.args[1].max_tokens == 93

216

@@ -214,12 +218,13 @@ def test_serving_chat_should_set_correct_max_tokens():

218 req.max_tokens = 5

219

220 with suppress(Exception):

217 - asyncio.run(serving_chat.create_chat_completion(req))

221 + await serving_chat.create_chat_completion(req)

222

223 assert mock_engine.generate.call_args.args[1].max_tokens == 5

224

225

222 -def test_serving_chat_could_load_correct_generation_config():

226 +@pytest.mark.asyncio

227 +async def test_serving_chat_could_load_correct_generation_config():

228

229 mock_model_config = MockModelConfig()

230 mock_model_config.diff_sampling_param = {

@@ -242,6 +247,7 @@ def test_serving_chat_could_load_correct_generation_config():

247 chat_template=CHAT_TEMPLATE,

248 chat_template_content_format="auto",

249 request_logger=None)

250 +

251 req = ChatCompletionRequest(

252 model=MODEL_NAME,

253 messages=[{

@@ -252,7 +258,7 @@ def test_serving_chat_could_load_correct_generation_config():

258 )

259

260 with suppress(Exception):

255 - asyncio.run(serving_chat.create_chat_completion(req))

261 + await serving_chat.create_chat_completion(req)

262

263 assert mock_engine.generate.call_args.args[1].temperature == 0.5

264 assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05

@@ -261,7 +267,7 @@ def test_serving_chat_could_load_correct_generation_config():

267 req.temperature = 0.1

268

269 with suppress(Exception):

264 - asyncio.run(serving_chat.create_chat_completion(req))

270 + await serving_chat.create_chat_completion(req)

271

272 assert mock_engine.generate.call_args.args[1].temperature == 0.1

273 assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05

@@ -270,13 +276,14 @@ def test_serving_chat_could_load_correct_generation_config():

276 req.temperature = 0.0

277

278 with suppress(Exception):

273 - asyncio.run(serving_chat.create_chat_completion(req))

279 + await serving_chat.create_chat_completion(req)

280

281 assert mock_engine.generate.call_args.args[1].temperature == 0.0

282 assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05

283

284

279 -def test_serving_chat_did_set_correct_cache_salt():

285 +@pytest.mark.asyncio

286 +async def test_serving_chat_did_set_correct_cache_salt():

287 mock_model_config = MockModelConfig()

288

289 mock_engine = MagicMock(spec=MQLLMEngineClient)

@@ -306,11 +313,11 @@ def test_serving_chat_did_set_correct_cache_salt():

313

314 # By default cache_salt in the engine prompt is not set

315 with suppress(Exception):

309 - asyncio.run(serving_chat.create_chat_completion(req))

316 + await serving_chat.create_chat_completion(req)

317 assert "cache_salt" not in mock_engine.generate.call_args.args[0]

318

319 # Test with certain cache_salt

320 req.cache_salt = "test_salt"

321 with suppress(Exception):

315 - asyncio.run(serving_chat.create_chat_completion(req))

322 + await serving_chat.create_chat_completion(req)

323 assert mock_engine.generate.call_args.args[0]["cache_salt"] == "test_salt"

vllm/entrypoints/openai/serving_engine.py

+73 -48

@@ -1,13 +1,13 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import asyncio

4 import base64

5 import io

6 import json

7 import sys

8 import time

8 -from collections.abc import (AsyncGenerator, Iterable, Iterator, Mapping,

9 - Sequence)

10 -from concurrent.futures.thread import ThreadPoolExecutor

9 +from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence

10 +from concurrent.futures import ThreadPoolExecutor

11 from http import HTTPStatus

12 from typing import (Annotated, Any, Callable, ClassVar, Generic, Optional,

13 TypeVar, Union, cast, overload)

@@ -79,8 +79,8 @@ from vllm.sequence import Logprob, PromptLogprobs

79 from vllm.tracing import (contains_trace_headers, extract_trace_headers,

80 log_tracing_disabled_warning)

81 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

82 -from vllm.utils import (is_list_of, make_async, merge_async_iterators,

83 - random_uuid)

82 +from vllm.utils import (AsyncMicrobatchTokenizer, is_list_of,

83 + merge_async_iterators, random_uuid)

85 logger = init_logger(__name__)

@@ -226,11 +226,19 @@ class OpenAIServing:

226

227 self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)

228

229 - self._tokenize_prompt_input_async = make_async(

230 - self._tokenize_prompt_input, executor=self._tokenizer_executor)

231 - self._tokenize_prompt_input_or_inputs_async = make_async(

232 - self._tokenize_prompt_input_or_inputs,

233 - executor=self._tokenizer_executor)

229 + self._async_tokenizer_pool: dict[AnyTokenizer,

230 + AsyncMicrobatchTokenizer] = {}

231 +

232 + def _get_async_tokenizer(self, tokenizer) -> AsyncMicrobatchTokenizer:

233 + """

234 + Return (and cache) an `AsyncMicrobatchTokenizer` bound to the

235 + given tokenizer.

236 + """

237 + async_tokenizer = self._async_tokenizer_pool.get(tokenizer)

238 + if async_tokenizer is None:

239 + async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)

240 + self._async_tokenizer_pool[tokenizer] = async_tokenizer

241 + return async_tokenizer

242

243 async def _preprocess(

244 self,

@@ -467,7 +475,7 @@ class OpenAIServing:

475 # if _check_model has been called earlier, this will be unreachable

476 raise ValueError(f"The model `{request.model}` does not exist.")

477

470 - def _normalize_prompt_text_to_input(

478 + async def _normalize_prompt_text_to_input(

479 self,

480 request: AnyRequest,

481 tokenizer: AnyTokenizer,

@@ -475,38 +483,44 @@ class OpenAIServing:

483 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]],

484 add_special_tokens: bool,

485 ) -> TextTokensPrompt:

486 + async_tokenizer = self._get_async_tokenizer(tokenizer)

487 +

488 if (self.model_config.encoder_config is not None

489 and self.model_config.encoder_config.get(

490 "do_lower_case", False)):

491 prompt = prompt.lower()

492

493 if truncate_prompt_tokens is None:

484 - encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)

494 + encoded = await async_tokenizer(

495 + prompt, add_special_tokens=add_special_tokens)

496 elif truncate_prompt_tokens < 0:

497 # Negative means we cap at the model's max length

487 - encoded = tokenizer(prompt,

488 - add_special_tokens=add_special_tokens,

489 - truncation=True,

490 - max_length=self.max_model_len)

498 + encoded = await async_tokenizer(

499 + prompt,

500 + add_special_tokens=add_special_tokens,

501 + truncation=True,

502 + max_length=self.max_model_len)

503 else:

492 - encoded = tokenizer(prompt,

493 - add_special_tokens=add_special_tokens,

494 - truncation=True,

495 - max_length=truncate_prompt_tokens)

504 + encoded = await async_tokenizer(

505 + prompt,

506 + add_special_tokens=add_special_tokens,

507 + truncation=True,

508 + max_length=truncate_prompt_tokens)

509

510 input_ids = encoded.input_ids

498 -

511 input_text = prompt

512

513 return self._validate_input(request, input_ids, input_text)

514

503 - def _normalize_prompt_tokens_to_input(

515 + async def _normalize_prompt_tokens_to_input(

516 self,

517 request: AnyRequest,

518 tokenizer: AnyTokenizer,

519 prompt_ids: list[int],

520 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],

521 ) -> TextTokensPrompt:

522 + async_tokenizer = self._get_async_tokenizer(tokenizer)

523 +

524 if truncate_prompt_tokens is None:

525 input_ids = prompt_ids

526 elif truncate_prompt_tokens < 0:

@@ -514,7 +528,7 @@ class OpenAIServing:

528 else:

529 input_ids = prompt_ids[-truncate_prompt_tokens:]

530

517 - input_text = tokenizer.decode(input_ids)

531 + input_text = await async_tokenizer.decode(input_ids)

532

533 return self._validate_input(request, input_ids, input_text)

534

@@ -578,7 +592,7 @@ class OpenAIServing:

592

593 return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)

594

581 - def _tokenize_prompt_input(

595 + async def _tokenize_prompt_input_async(

596 self,

597 request: AnyRequest,

598 tokenizer: AnyTokenizer,

@@ -591,23 +605,24 @@ class OpenAIServing:

605 [`_tokenize_prompt_input_or_inputs`][vllm.entrypoints.openai.serving_engine.OpenAIServing._tokenize_prompt_input_or_inputs]

606 that assumes single input.

607 """

594 - return next(

595 - self._tokenize_prompt_inputs(

608 + async for result in self._tokenize_prompt_inputs_async(

609 request,

610 tokenizer,

598 - [prompt_input],

611 + [prompt_input],

612 truncate_prompt_tokens=truncate_prompt_tokens,

613 add_special_tokens=add_special_tokens,

601 - ))

614 + ):

615 + return result

616 + raise ValueError("No results yielded from tokenization")

617

603 - def _tokenize_prompt_inputs(

618 + async def _tokenize_prompt_inputs_async(

619 self,

620 request: AnyRequest,

621 tokenizer: AnyTokenizer,

622 prompt_inputs: Iterable[Union[str, list[int]]],

623 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,

624 add_special_tokens: bool = True,

610 - ) -> Iterator[TextTokensPrompt]:

625 + ) -> AsyncGenerator[TextTokensPrompt, None]:

626 """

627 A simpler implementation of

628 [`_tokenize_prompt_input_or_inputs`][vllm.entrypoints.openai.serving_engine.OpenAIServing._tokenize_prompt_input_or_inputs]

@@ -615,7 +630,7 @@ class OpenAIServing:

630 """

631 for text in prompt_inputs:

632 if isinstance(text, str):

618 - yield self._normalize_prompt_text_to_input(

633 + yield await self._normalize_prompt_text_to_input(

634 request,

635 tokenizer,

636 prompt=text,

@@ -623,14 +638,14 @@ class OpenAIServing:

638 add_special_tokens=add_special_tokens,

639 )

640 else:

626 - yield self._normalize_prompt_tokens_to_input(

641 + yield await self._normalize_prompt_tokens_to_input(

642 request,

643 tokenizer,

644 prompt_ids=text,

645 truncate_prompt_tokens=truncate_prompt_tokens,

646 )

647

633 - def _tokenize_prompt_input_or_inputs(

648 + async def _tokenize_prompt_input_or_inputs_async(

649 self,

650 request: AnyRequest,

651 tokenizer: AnyTokenizer,

@@ -664,21 +679,31 @@ class OpenAIServing:

679 # VSCode Pyright extension should still work properly

680 # "is False" is required for Pyright to perform type narrowing

681 # See: https://github.com/microsoft/pyright/issues/7672

667 - inputs_text.extend([

668 - self._normalize_prompt_text_to_input(

669 - request,

670 - tokenizer,

671 - prompt=prompt_input["content"],

672 - truncate_prompt_tokens=truncate_prompt_tokens,

673 - add_special_tokens=add_special_tokens)

674 - if prompt_input["is_tokens"] is False else

675 - self._normalize_prompt_tokens_to_input(

676 - request,

677 - tokenizer,

678 - prompt_ids=prompt_input["content"],

679 - truncate_prompt_tokens=truncate_prompt_tokens)

680 - for prompt_input in parse_and_batch_prompt(input_or_inputs)

681 - ])

682 +

683 + # Parse and batch the input prompts

684 + batch_inputs = parse_and_batch_prompt(input_or_inputs)

685 +

686 + # Process each input in the batch concurrently

687 + tasks = []

688 + for prompt_input in batch_inputs:

689 + if prompt_input["is_tokens"] is False:

690 + task = self._normalize_prompt_text_to_input(

691 + request,

692 + tokenizer,

693 + prompt_input["content"],

694 + truncate_prompt_tokens=truncate_prompt_tokens,

695 + add_special_tokens=add_special_tokens)

696 + else:

697 + task = self._normalize_prompt_tokens_to_input(

698 + request,

699 + tokenizer,

700 + prompt_input["content"],

701 + truncate_prompt_tokens=truncate_prompt_tokens)

702 + tasks.append(task)

703 +

704 + # Wait for all tokenization tasks to complete

705 + results = await asyncio.gather(*tasks)

706 + inputs_text.extend(results)

707

708 return inputs_text, inputs_embeds

709

vllm/utils/__init__.py

+192

@@ -41,6 +41,7 @@ from collections import UserDict, defaultdict

41 from collections.abc import (AsyncGenerator, Awaitable, Collection, Generator,

42 Hashable, Iterable, Iterator, KeysView, Mapping,

43 Sequence)

44 +from concurrent.futures import ThreadPoolExecutor

45 from concurrent.futures.process import ProcessPoolExecutor

46 from dataclasses import dataclass, field

47 from functools import cache, lru_cache, partial, wraps

@@ -64,6 +65,7 @@ import zmq.asyncio

65 from packaging import version

66 from packaging.version import Version

67 from torch.library import Library

68 +from transformers.tokenization_utils_base import BatchEncoding

69 from typing_extensions import Never, ParamSpec, TypeIs, assert_never

71 import vllm.envs as envs

@@ -507,6 +509,196 @@ def random_uuid() -> str:

509 return str(uuid.uuid4().hex)

510

511

512 +class AsyncMicrobatchTokenizer:

513 + """Asynchronous tokenizer with micro-batching.

514 +

515 + Pulls pending encode/decode requests from a queue and batches them

516 + up to reduce overhead. A single-thread ThreadPoolExecutor is used

517 + so the event loop stays responsive.

518 + """

519 +

520 + def __init__(

521 + self,

522 + tokenizer,

523 + max_batch_size: int = 32,

524 + batch_wait_timeout_s: float = 0.002,

525 + ) -> None:

526 + self.tokenizer = tokenizer

527 + self.max_batch_size = max_batch_size

528 + self.batch_wait_timeout_s = batch_wait_timeout_s

529 +

530 + self._loop = asyncio.get_running_loop()

531 + self._queues: dict[tuple,

532 + asyncio.Queue[Union[tuple[str, dict,

533 + asyncio.Future],

534 + tuple[list[int],

535 + asyncio.Future]]]] = {}

536 + self._batcher_tasks: list[asyncio.Task] = []

537 +

538 + # Single-thread executor for blocking tokenizer calls.

539 + self._executor = ThreadPoolExecutor(max_workers=1)

540 +

541 + # === Public async API ===

542 + async def __call__(self, prompt, **kwargs):

543 + result_future: asyncio.Future = self._loop.create_future()

544 + key = self._queue_key("encode", kwargs)

545 + queue = self._get_queue(self._loop, key)

546 + await queue.put((prompt, kwargs, result_future))

547 + return await result_future

548 +

549 + async def decode(self, token_ids, **kwargs):

550 + result_future: asyncio.Future = self._loop.create_future()

551 + key = self._queue_key("decode", kwargs)

552 + queue = self._get_queue(self._loop, key)

553 + await queue.put((token_ids, result_future))

554 + return await result_future

555 +

556 + # === Internal helpers ===

557 + def _get_queue(

558 + self, loop: asyncio.AbstractEventLoop, key: tuple

559 + ) -> asyncio.Queue[Union[tuple[str, dict, asyncio.Future], tuple[

560 + list[int], asyncio.Future]]]:

561 + """Get the request queue for the given operation key, creating a new

562 + queue and batcher task if needed."""

563 + queue = self._queues.get(key)

564 + if queue is None:

565 + self._queues[key] = queue = asyncio.Queue()

566 + if key[0] == "encode":

567 + can_batch = key[1] != "other"

568 + coro = self._batch_encode_loop(queue, can_batch)

569 + else:

570 + assert key[0] == "decode", \

571 + f"Unknown operation type: {key[0]}."

572 + coro = self._batch_decode_loop(queue)

573 + self._batcher_tasks.append(loop.create_task(coro))

574 + return queue

575 +

576 + async def _batch_encode_loop(self, queue: asyncio.Queue, can_batch: bool):

577 + """Batch incoming encode requests for efficiency."""

578 + while True:

579 + prompt, kwargs, result_future = await queue.get()

580 + prompts = [prompt]

581 + kwargs_list = [kwargs]

582 + result_futures = [result_future]

583 + deadline = self._loop.time() + self.batch_wait_timeout_s

584 +

585 + while len(prompts) < self.max_batch_size:

586 + timeout = deadline - self._loop.time()

587 + if timeout <= 0:

588 + break

589 + try:

590 + prompt, kwargs, result_future = await asyncio.wait_for(

591 + queue.get(), timeout)

592 + prompts.append(prompt)

593 + result_futures.append(result_future)

594 + if not can_batch:

595 + kwargs_list.append(kwargs)

596 + except asyncio.TimeoutError:

597 + break

598 +

599 + try:

600 + # If every request uses identical kwargs we can run a single

601 + # batched tokenizer call for a big speed-up.

602 + if can_batch and len(prompts) > 1:

603 + encode_fn = partial(self.tokenizer, prompts, **kwargs)

604 + results = await self._loop.run_in_executor(

605 + self._executor, encode_fn)

606 +

607 + for i, fut in enumerate(result_futures):

608 + if not fut.done():

609 + data = {k: v[i] for k, v in results.items()}

610 + fut.set_result(BatchEncoding(data))

611 + else:

612 + encode_fn = lambda prompts=prompts, kwargs=kwargs_list: [

613 + self.tokenizer(p, **kw)

614 + for p, kw in zip(prompts, kwargs)

615 + ]

616 + results = await self._loop.run_in_executor(

617 + self._executor, encode_fn)

618 +

619 + for fut, res in zip(result_futures, results):

620 + if not fut.done():

621 + fut.set_result(res)

622 + except Exception as e:

623 + for fut in result_futures:

624 + if not fut.done():

625 + fut.set_exception(e)

626 +

627 + async def _batch_decode_loop(self, queue: asyncio.Queue):

628 + """Batch incoming decode requests for efficiency."""

629 + while True:

630 + token_ids, result_future = await queue.get()

631 + token_ids_list = [token_ids]

632 + result_futures = [result_future]

633 + deadline = self._loop.time() + self.batch_wait_timeout_s

634 +

635 + while len(token_ids_list) < self.max_batch_size:

636 + timeout = deadline - self._loop.time()

637 + if timeout <= 0:

638 + break

639 + try:

640 + token_ids, result_future = await asyncio.wait_for(

641 + queue.get(), timeout)

642 + token_ids_list.append(token_ids)

643 + result_futures.append(result_future)

644 + except asyncio.TimeoutError:

645 + break

646 +

647 + try:

648 + # Perform a single batched decode call for all requests

649 + results = await self._loop.run_in_executor(

650 + self._executor, self.tokenizer.batch_decode,

651 + token_ids_list)

652 + for fut, res in zip(result_futures, results):

653 + if not fut.done():

654 + fut.set_result(res)

655 + except Exception as e:

656 + for fut in result_futures:

657 + if not fut.done():

658 + fut.set_exception(e)

659 +

660 + def _queue_key(self, op: str, kwargs: dict) -> tuple:

661 + """

662 + Return a normalized key describing operation + kwargs.

663 +

664 + - `add_special_tokens`: {True/False}

665 + - `truncation`: {True/False}

666 + - If `truncation` is False (`max_length` is None),

667 + returns a key for a can_batch queue.

668 + - If `truncation` is True and `max_length` is None or equals

669 + `tokenizer.model_max_length`, returns a key for a can_batch queue.

670 + - Otherwise, returns a key for a cannot_batch queue.

671 +

672 + Examples:

673 + - Decode: ("decode",)

674 + - Encode typical:

675 + ("encode", add_special_tokens, bool_truncation, max_length_label)

676 + - Fallback: ("encode", "other")

677 + """

678 +

679 + if op == "decode":

680 + return ("decode", )

681 +

682 + add_special_tokens = kwargs.get("add_special_tokens", True)

683 + truncation = kwargs.get("truncation", False)

684 + max_length = kwargs.get("max_length")

685 +

686 + if not truncation:

687 + return ("encode", add_special_tokens, False, None)

688 +

689 + model_max = getattr(self.tokenizer, "model_max_length", None)

690 + if max_length is None or (model_max is not None

691 + and max_length == model_max):

692 + return ("encode", add_special_tokens, True, "model_max")

693 +

694 + return ("encode", "other")

695 +

696 + def __del__(self):

697 + for task in self._batcher_tasks:

698 + if not task.done():

699 + task.cancel()

700 +

701 +

702 def make_async(

703 func: Callable[P, T],

704 executor: Optional[concurrent.futures.Executor] = None