A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Deprecation] Remove `inputs` arg fallback in Engine classes (#18799)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jun 9, 2025 at 10:19 UTC
e31ae3de366dcf1898069dd57eebcb2983b5ab1d
4 files changed
+21
-268
vllm/engine/async_llm_engine.py
+9
-98
@@ -6,12 +6,10 @@ import copy
6
import time
7
import weakref
8
from functools import partial
9
-from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
10
- List, Mapping, Optional, Set, Tuple, Type, Union, overload)
9
+from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
10
+ Mapping, Optional, Set, Tuple, Type, Union)
11
from weakref import ReferenceType
12
13
-from typing_extensions import deprecated
14
-
13
import vllm.envs as envs
14
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
15
ParallelConfig, SchedulerConfig, VllmConfig)
@@ -36,7 +34,7 @@ from vllm.sampling_params import SamplingParams
34
from vllm.sequence import ExecuteModelRequest
35
from vllm.transformers_utils.tokenizer import AnyTokenizer
36
from vllm.usage.usage_lib import UsageContext
39
-from vllm.utils import Device, deprecate_kwargs, weak_bind
37
+from vllm.utils import Device, weak_bind
38
39
logger = init_logger(__name__)
40
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -429,24 +427,6 @@ class _AsyncLLMEngine(LLMEngine):
427
return await (
428
self.get_tokenizer_group().get_lora_tokenizer_async(lora_request))
429
432
- @overload
433
- @deprecated("'inputs' will be renamed to 'prompt")
434
- async def add_request_async(
435
- self,
436
- request_id: str,
437
- *,
438
- inputs: PromptType,
439
- params: Union[SamplingParams, PoolingParams],
440
- arrival_time: Optional[float] = None,
441
- lora_request: Optional[LoRARequest] = None,
442
- trace_headers: Optional[Mapping[str, str]] = None,
443
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
444
- priority: int = 0,
445
- data_parallel_rank: Optional[int] = None,
446
- ) -> None:
447
- ...
448
-
449
- @overload
430
async def add_request_async(
431
self,
432
request_id: str,
@@ -459,32 +439,10 @@ class _AsyncLLMEngine(LLMEngine):
439
priority: int = 0,
440
data_parallel_rank: Optional[int] = None,
441
) -> None:
462
- ...
463
-
464
- @deprecate_kwargs(
465
- "inputs",
466
- additional_message="Please use the 'prompt' parameter instead.",
467
- )
468
- async def add_request_async(
469
- self,
470
- request_id: str,
471
- prompt: Optional[PromptType] = None,
472
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
473
- arrival_time: Optional[float] = None,
474
- lora_request: Optional[LoRARequest] = None,
475
- trace_headers: Optional[Mapping[str, str]] = None,
476
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
477
- priority: int = 0,
478
- data_parallel_rank: Optional[int] = None,
479
- *,
480
- inputs: Optional[PromptType] = None, # DEPRECATED
481
- ) -> None:
482
- """Async version of
483
- [`add_request`][vllm.engine.llm_engine.LLMEngine.add_request]."""
484
- if inputs is not None:
485
- prompt = inputs
486
- assert prompt is not None and params is not None
487
-
442
+ """
443
+ Async version of
444
+ [`add_request`][vllm.engine.llm_engine.LLMEngine.add_request].
445
+ """
446
if lora_request is not None and not self.lora_config:
447
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
448
"not enabled!")
@@ -521,8 +479,7 @@ class _AsyncLLMEngine(LLMEngine):
479
params = await build_guided_decoding_logits_processor_async(
480
sampling_params=params,
481
tokenizer=await self.get_tokenizer_async(lora_request),
524
- default_guided_backend=self.decoding_config.
525
- guided_decoding_backend,
482
+ default_guided_backend=self.decoding_config.backend,
483
reasoning_backend=self.decoding_config.reasoning_backend,
484
model_config=self.model_config)
485
@@ -894,28 +851,7 @@ class AsyncLLMEngine(EngineClient):
851
raise
852
await asyncio.sleep(0)
853
897
- # This method does not need to be async, but kept that way
898
- # for backwards compatibility.
899
- @overload
900
- @deprecated("'inputs' will be renamed to 'prompt")
901
- def add_request(
902
- self,
903
- request_id: str,
904
- *,
905
- inputs: PromptType,
906
- params: Union[SamplingParams, PoolingParams],
907
- arrival_time: Optional[float] = None,
908
- lora_request: Optional[LoRARequest] = None,
909
- trace_headers: Optional[Mapping[str, str]] = None,
910
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
911
- priority: int = 0,
912
- data_parallel_rank: Optional[int] = None,
913
- ) -> Coroutine[None, None, AsyncGenerator[Union[
914
- RequestOutput, PoolingRequestOutput], None]]:
915
- ...
916
-
917
- @overload
918
- def add_request(
854
+ async def add_request(
855
self,
856
request_id: str,
857
prompt: PromptType,
@@ -926,32 +862,7 @@ class AsyncLLMEngine(EngineClient):
862
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
863
priority: int = 0,
864
data_parallel_rank: Optional[int] = None,
929
- ) -> Coroutine[None, None, AsyncGenerator[Union[
930
- RequestOutput, PoolingRequestOutput], None]]:
931
- ...
932
-
933
- @deprecate_kwargs(
934
- "inputs",
935
- additional_message="Please use the 'prompt' parameter instead.",
936
- )
937
- async def add_request(
938
- self,
939
- request_id: str,
940
- prompt: Optional[PromptType] = None,
941
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
942
- arrival_time: Optional[float] = None,
943
- lora_request: Optional[LoRARequest] = None,
944
- trace_headers: Optional[Mapping[str, str]] = None,
945
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
946
- priority: int = 0,
947
- data_parallel_rank: Optional[int] = None,
948
- *,
949
- inputs: Optional[PromptType] = None, # DEPRECATED
865
) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
951
- if inputs is not None:
952
- prompt = inputs
953
- assert prompt is not None and params is not None
954
-
866
if not self.is_running:
867
if self.start_engine_loop:
868
self.start_background_loop()
vllm/engine/llm_engine.py
+3
-45
@@ -11,10 +11,10 @@ from functools import partial
11
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
12
Iterable, List, Literal, Mapping, NamedTuple, Optional)
13
from typing import Sequence as GenericSequence
14
-from typing import Set, Type, Union, cast, overload
14
+from typing import Set, Type, Union, cast
15
16
import torch
17
-from typing_extensions import TypeVar, deprecated
17
+from typing_extensions import TypeVar
18
19
import vllm.envs as envs
20
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -58,8 +58,7 @@ from vllm.transformers_utils.tokenizer_group import (
58
TokenizerGroup, init_tokenizer_from_configs)
59
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
60
usage_message)
61
-from vllm.utils import (Counter, Device, deprecate_kwargs,
62
- resolve_obj_by_qualname, weak_bind)
61
+from vllm.utils import Counter, Device, resolve_obj_by_qualname, weak_bind
62
from vllm.version import __version__ as VLLM_VERSION
63
from vllm.worker.model_runner_base import InputProcessingError
64
@@ -629,7 +628,6 @@ class LLMEngine:
628
def stop_remote_worker_execution_loop(self) -> None:
629
self.model_executor.stop_remote_worker_execution_loop()
630
632
- @overload
631
def add_request(
632
self,
633
request_id: str,
@@ -641,42 +639,6 @@ class LLMEngine:
639
trace_headers: Optional[Mapping[str, str]] = None,
640
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
641
priority: int = 0,
644
- ) -> None:
645
- ...
646
-
647
- @overload
648
- @deprecated("'inputs' will be renamed to 'prompt")
649
- def add_request(
650
- self,
651
- request_id: str,
652
- *,
653
- inputs: PromptType,
654
- params: Union[SamplingParams, PoolingParams],
655
- arrival_time: Optional[float] = None,
656
- lora_request: Optional[LoRARequest] = None,
657
- trace_headers: Optional[Mapping[str, str]] = None,
658
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
659
- priority: int = 0,
660
- ) -> None:
661
- ...
662
-
663
- @deprecate_kwargs(
664
- "inputs",
665
- additional_message="Please use the 'prompt' parameter instead.",
666
- )
667
- def add_request(
668
- self,
669
- request_id: str,
670
- prompt: Optional[PromptType] = None,
671
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
672
- arrival_time: Optional[float] = None,
673
- lora_request: Optional[LoRARequest] = None,
674
- tokenization_kwargs: Optional[dict[str, Any]] = None,
675
- trace_headers: Optional[Mapping[str, str]] = None,
676
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
677
- priority: int = 0,
678
- *,
679
- inputs: Optional[PromptType] = None, # DEPRECATED
642
) -> None:
643
"""Add a request to the engine's request pool.
644
@@ -725,10 +687,6 @@ class LLMEngine:
687
>>> # continue the request processing
688
>>> ...
689
"""
728
- if inputs is not None:
729
- prompt = inputs
730
- assert prompt is not None and params is not None
731
-
690
if lora_request is not None and not self.lora_config:
691
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
692
"not enabled!")
vllm/engine/multiprocessing/__init__.py
+2
-43
@@ -4,9 +4,7 @@
4
import uuid
5
from dataclasses import dataclass, field
6
from enum import Enum
7
-from typing import List, Mapping, Optional, Union, overload
8
-
9
-from typing_extensions import deprecated
7
+from typing import List, Mapping, Optional, Union
8
9
from vllm import PoolingParams
10
from vllm.inputs import PromptType
@@ -14,7 +12,7 @@ from vllm.lora.request import LoRARequest
12
from vllm.outputs import RequestOutput
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sampling_params import SamplingParams
17
-from vllm.utils import Device, deprecate_kwargs
15
+from vllm.utils import Device
16
17
VLLM_RPC_SUCCESS_STR = "SUCCESS"
18
@@ -38,7 +36,6 @@ class RPCProcessRequest:
36
prompt_adapter_request: Optional[PromptAdapterRequest] = None
37
priority: int = 0
38
41
- @overload
39
def __init__(
40
self,
41
prompt: PromptType,
@@ -49,44 +46,6 @@ class RPCProcessRequest:
46
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
47
priority: int = 0,
48
) -> None:
52
- ...
53
-
54
- @overload
55
- @deprecated("'inputs' will be renamed to 'prompt")
56
- def __init__(
57
- self,
58
- *,
59
- inputs: PromptType,
60
- params: Union[SamplingParams, PoolingParams],
61
- request_id: str,
62
- lora_request: Optional[LoRARequest] = None,
63
- trace_headers: Optional[Mapping[str, str]] = None,
64
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
65
- priority: int = 0,
66
- ) -> None:
67
- ...
68
-
69
- @deprecate_kwargs(
70
- "inputs",
71
- additional_message="Please use the 'prompt' parameter instead.",
72
- )
73
- def __init__(
74
- self,
75
- prompt: Optional[PromptType] = None,
76
- params: Optional[Union[SamplingParams, PoolingParams]] = None,
77
- request_id: Optional[str] = None,
78
- lora_request: Optional[LoRARequest] = None,
79
- trace_headers: Optional[Mapping[str, str]] = None,
80
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
81
- priority: int = 0,
82
- *,
83
- inputs: Optional[PromptType] = None, # DEPRECATED
84
- ) -> None:
85
- if inputs is not None:
86
- prompt = inputs
87
- assert (prompt is not None and params is not None
88
- and request_id is not None)
89
-
49
super().__init__()
50
51
self.prompt = prompt
vllm/engine/multiprocessing/client.py
+7
-82
@@ -6,13 +6,12 @@ import copy
6
import pickle
7
from contextlib import contextmanager, suppress
8
from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
9
- Optional, Union, cast, overload)
9
+ Optional, Union, cast)
10
11
import cloudpickle
12
import psutil
13
import zmq
14
import zmq.asyncio
15
-from typing_extensions import deprecated
15
from zmq import Frame # type: ignore[attr-defined]
16
from zmq.asyncio import Socket
17
@@ -49,7 +48,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput
48
from vllm.prompt_adapter.request import PromptAdapterRequest
49
from vllm.sampling_params import SamplingParams
50
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
52
-from vllm.utils import Device, deprecate_kwargs
51
+from vllm.utils import Device
52
53
logger = init_logger(__name__)
54
@@ -442,7 +441,6 @@ class MQLLMEngineClient(EngineClient):
441
def dead_error(self) -> BaseException:
442
return ENGINE_DEAD_ERROR(self._errored_with)
443
445
- @overload
444
def generate(
445
self,
446
prompt: PromptType,
@@ -452,39 +450,6 @@ class MQLLMEngineClient(EngineClient):
450
trace_headers: Optional[Mapping[str, str]] = None,
451
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
452
priority: int = 0,
455
- ) -> AsyncGenerator[RequestOutput, None]:
456
- ...
457
-
458
- @overload
459
- @deprecated("'inputs' will be renamed to 'prompt")
460
- def generate(
461
- self,
462
- *,
463
- inputs: PromptType,
464
- sampling_params: SamplingParams,
465
- request_id: str,
466
- lora_request: Optional[LoRARequest] = None,
467
- trace_headers: Optional[Mapping[str, str]] = None,
468
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
469
- priority: int = 0,
470
- ) -> AsyncGenerator[RequestOutput, None]:
471
- ...
472
-
473
- @deprecate_kwargs(
474
- "inputs",
475
- additional_message="Please use the 'prompt' parameter instead.",
476
- )
477
- def generate(
478
- self,
479
- prompt: Optional[PromptType] = None,
480
- sampling_params: Optional[SamplingParams] = None,
481
- request_id: Optional[str] = None,
482
- lora_request: Optional[LoRARequest] = None,
483
- trace_headers: Optional[Mapping[str, str]] = None,
484
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
485
- priority: int = 0,
486
- *,
487
- inputs: Optional[PromptType] = None # DEPRECATED
453
) -> AsyncGenerator[RequestOutput, None]:
454
"""Generate outputs for a request.
455
@@ -506,16 +471,12 @@ class MQLLMEngineClient(EngineClient):
471
Any priority other than 0 will lead to an error if the
472
scheduling policy is not "priority".
473
"""
509
- if inputs is not None:
510
- prompt = inputs
511
- assert (prompt is not None and sampling_params is not None
512
- and request_id is not None)
513
-
514
- return self._process_request(prompt, sampling_params, request_id,
515
- lora_request, trace_headers,
516
- prompt_adapter_request, priority)
474
+ return cast(
475
+ AsyncGenerator[RequestOutput, None],
476
+ self._process_request(prompt, sampling_params, request_id,
477
+ lora_request, trace_headers,
478
+ prompt_adapter_request, priority))
479
518
- @overload
480
def encode(
481
self,
482
prompt: PromptType,
@@ -524,37 +485,6 @@ class MQLLMEngineClient(EngineClient):
485
lora_request: Optional[LoRARequest] = None,
486
trace_headers: Optional[Mapping[str, str]] = None,
487
priority: int = 0,
527
- ) -> AsyncGenerator[PoolingRequestOutput, None]:
528
- ...
529
-
530
- @overload
531
- @deprecated("'inputs' will be renamed to 'prompt")
532
- def encode(
533
- self,
534
- *,
535
- inputs: PromptType,
536
- pooling_params: PoolingParams,
537
- request_id: str,
538
- lora_request: Optional[LoRARequest] = None,
539
- trace_headers: Optional[Mapping[str, str]] = None,
540
- priority: int = 0,
541
- ) -> AsyncGenerator[PoolingRequestOutput, None]:
542
- ...
543
-
544
- @deprecate_kwargs(
545
- "inputs",
546
- additional_message="Please use the 'prompt' parameter instead.",
547
- )
548
- def encode(
549
- self,
550
- prompt: Optional[PromptType] = None,
551
- pooling_params: Optional[PoolingParams] = None,
552
- request_id: Optional[str] = None,
553
- lora_request: Optional[LoRARequest] = None,
554
- trace_headers: Optional[Mapping[str, str]] = None,
555
- priority: int = 0,
556
- *,
557
- inputs: Optional[PromptType] = None # DEPRECATED
488
) -> AsyncGenerator[PoolingRequestOutput, None]:
489
"""Generate outputs for a request from a pooling model.
490
@@ -575,11 +505,6 @@ class MQLLMEngineClient(EngineClient):
505
The output `PoolingRequestOutput` objects from the LLMEngine
506
for the request.
507
"""
578
- if inputs is not None:
579
- prompt = inputs
580
- assert (prompt is not None and pooling_params is not None
581
- and request_id is not None)
582
-
508
return cast(
509
AsyncGenerator[PoolingRequestOutput, None],
510
self._process_request(prompt,