A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[mypy] Enable following imports for entrypoints (#7248)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Fei <dfdfcai4@gmail.com>
Cyrus Leung committed
Aug 21, 2024 at 14:28 UTC
baaedfdb2d3f1d70b7dbcde08b083abfe6017a92
26 files changed
+480
-320
.github/workflows/mypy.yaml
-1
@@ -38,7 +38,6 @@ jobs:
38
mypy vllm/core --follow-imports skip
39
mypy vllm/distributed --follow-imports skip
40
mypy vllm/engine --follow-imports skip
41
- mypy vllm/entrypoints --follow-imports skip
41
mypy vllm/executor --follow-imports skip
42
mypy vllm/lora --follow-imports skip
43
mypy vllm/model_executor --follow-imports skip
docs/requirements-docs.txt
+1
-1
@@ -6,7 +6,7 @@ sphinx-argparse==0.4.0
6
msgspec
7
8
# packages to install to build the documentation
9
-pydantic
9
+pydantic >= 2.8
10
-f https://download.pytorch.org/whl/cpu
11
torch
12
py-cpuinfo
format.sh
-1
@@ -102,7 +102,6 @@ mypy vllm/attention --follow-imports skip
102
mypy vllm/core --follow-imports skip
103
mypy vllm/distributed --follow-imports skip
104
mypy vllm/engine --follow-imports skip
105
-mypy vllm/entrypoints --follow-imports skip
105
mypy vllm/executor --follow-imports skip
106
mypy vllm/lora --follow-imports skip
107
mypy vllm/model_executor --follow-imports skip
pyproject.toml
+1
@@ -56,6 +56,7 @@ files = [
56
"vllm/*.py",
57
"vllm/adapter_commons",
58
"vllm/assets",
59
+ "vllm/entrypoints",
60
"vllm/inputs",
61
"vllm/logging",
62
"vllm/multimodal",
requirements-common.txt
+1
-1
@@ -11,7 +11,7 @@ fastapi
11
aiohttp
12
openai >= 1.0 # Ensure modern openai package (ensure types module present)
13
uvicorn[standard]
14
-pydantic >= 2.0 # Required for OpenAI server.
14
+pydantic >= 2.8 # Required for OpenAI server.
15
pillow # Required for image processing
16
prometheus_client >= 0.18.0
17
prometheus-fastapi-instrumentator >= 7.0.0
tests/entrypoints/openai/test_chat.py
+83
-1
@@ -1,7 +1,7 @@
1
# imports for guided decoding tests
2
import json
3
import re
4
-from typing import List
4
+from typing import Dict, List, Optional
5
6
import jsonschema
7
import openai # use the official client for correctness check
@@ -174,6 +174,88 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
174
assert message.content is not None and len(message.content) >= 0
175
176
177
+@pytest.mark.asyncio
178
+@pytest.mark.parametrize(
179
+ "model_name, prompt_logprobs",
180
+ [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
181
+)
182
+async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
183
+ model_name: str,
184
+ prompt_logprobs: Optional[int]):
185
+ params: Dict = {
186
+ "messages": [{
187
+ "role": "system",
188
+ "content": "You are a helpful assistant."
189
+ }, {
190
+ "role": "user",
191
+ "content": "Who won the world series in 2020?"
192
+ }, {
193
+ "role":
194
+ "assistant",
195
+ "content":
196
+ "The Los Angeles Dodgers won the World Series in 2020."
197
+ }, {
198
+ "role": "user",
199
+ "content": "Where was it played?"
200
+ }],
201
+ "model":
202
+ model_name
203
+ }
204
+
205
+ if prompt_logprobs is not None:
206
+ params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
207
+
208
+ if prompt_logprobs is not None and prompt_logprobs < 0:
209
+ with pytest.raises(BadRequestError):
210
+ await client.chat.completions.create(**params)
211
+ else:
212
+ completion = await client.chat.completions.create(**params)
213
+ if prompt_logprobs is not None:
214
+ assert completion.prompt_logprobs is not None
215
+ assert len(completion.prompt_logprobs) > 0
216
+ else:
217
+ assert completion.prompt_logprobs is None
218
+
219
+
220
+@pytest.mark.asyncio
221
+@pytest.mark.parametrize(
222
+ "model_name",
223
+ [MODEL_NAME],
224
+)
225
+async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
226
+ model_name: str):
227
+ params: Dict = {
228
+ "messages": [{
229
+ "role": "system",
230
+ "content": "You are a helpful assistant."
231
+ }, {
232
+ "role": "user",
233
+ "content": "Who won the world series in 2020?"
234
+ }, {
235
+ "role":
236
+ "assistant",
237
+ "content":
238
+ "The Los Angeles Dodgers won the World Series in 2020."
239
+ }, {
240
+ "role": "user",
241
+ "content": "Where was it played?"
242
+ }],
243
+ "model":
244
+ model_name,
245
+ "extra_body": {
246
+ "prompt_logprobs": 1
247
+ }
248
+ }
249
+
250
+ completion_1 = await client.chat.completions.create(**params)
251
+
252
+ params["extra_body"] = {"prompt_logprobs": 2}
253
+ completion_2 = await client.chat.completions.create(**params)
254
+
255
+ assert len(completion_1.prompt_logprobs[3]) == 1
256
+ assert len(completion_2.prompt_logprobs[3]) == 2
257
+
258
+
259
@pytest.mark.asyncio
260
@pytest.mark.parametrize(
261
"model_name",
tests/entrypoints/openai/test_completion.py
+5
-96
@@ -3,7 +3,7 @@ import json
3
import re
4
import shutil
5
from tempfile import TemporaryDirectory
6
-from typing import Dict, List
6
+from typing import Dict, List, Optional
7
8
import jsonschema
9
import openai # use the official client for correctness check
@@ -268,92 +268,6 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
268
assert len(completion.choices[0].text) >= 0
269
270
271
-@pytest.mark.asyncio
272
-@pytest.mark.parametrize(
273
- "model_name, prompt_logprobs",
274
- [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
275
-)
276
-async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
277
- model_name: str, prompt_logprobs: int):
278
- params: Dict = {
279
- "messages": [{
280
- "role": "system",
281
- "content": "You are a helpful assistant."
282
- }, {
283
- "role": "user",
284
- "content": "Who won the world series in 2020?"
285
- }, {
286
- "role":
287
- "assistant",
288
- "content":
289
- "The Los Angeles Dodgers won the World Series in 2020."
290
- }, {
291
- "role": "user",
292
- "content": "Where was it played?"
293
- }],
294
- "model":
295
- model_name
296
- }
297
-
298
- if prompt_logprobs is not None:
299
- params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
300
-
301
- if prompt_logprobs and prompt_logprobs < 0:
302
- with pytest.raises(BadRequestError) as err_info:
303
- await client.chat.completions.create(**params)
304
- expected_err_string = (
305
- "Error code: 400 - {'object': 'error', 'message': "
306
- "'Prompt_logprobs set to invalid negative value: -1',"
307
- " 'type': 'BadRequestError', 'param': None, 'code': 400}")
308
- assert str(err_info.value) == expected_err_string
309
- else:
310
- completion = await client.chat.completions.create(**params)
311
- if prompt_logprobs and prompt_logprobs > 0:
312
- assert completion.prompt_logprobs is not None
313
- assert len(completion.prompt_logprobs) > 0
314
- else:
315
- assert completion.prompt_logprobs is None
316
-
317
-
318
-@pytest.mark.asyncio
319
-@pytest.mark.parametrize(
320
- "model_name",
321
- [MODEL_NAME],
322
-)
323
-async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
324
- model_name: str):
325
- params: Dict = {
326
- "messages": [{
327
- "role": "system",
328
- "content": "You are a helpful assistant."
329
- }, {
330
- "role": "user",
331
- "content": "Who won the world series in 2020?"
332
- }, {
333
- "role":
334
- "assistant",
335
- "content":
336
- "The Los Angeles Dodgers won the World Series in 2020."
337
- }, {
338
- "role": "user",
339
- "content": "Where was it played?"
340
- }],
341
- "model":
342
- model_name,
343
- "extra_body": {
344
- "prompt_logprobs": 1
345
- }
346
- }
347
-
348
- completion_1 = await client.chat.completions.create(**params)
349
-
350
- params["extra_body"] = {"prompt_logprobs": 2}
351
- completion_2 = await client.chat.completions.create(**params)
352
-
353
- assert len(completion_1.prompt_logprobs[3]) == 1
354
- assert len(completion_2.prompt_logprobs[3]) == 2
355
-
356
-
271
@pytest.mark.asyncio
272
@pytest.mark.parametrize("model_name, prompt_logprobs", [(MODEL_NAME, -1),
273
(MODEL_NAME, 0),
@@ -361,7 +275,7 @@ async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
275
(MODEL_NAME, None)])
276
async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
277
model_name: str,
364
- prompt_logprobs: int):
278
+ prompt_logprobs: Optional[int]):
279
params: Dict = {
280
"prompt": ["A robot may not injure another robot", "My name is"],
281
"model": model_name,
@@ -369,17 +283,12 @@ async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
283
if prompt_logprobs is not None:
284
params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
285
372
- if prompt_logprobs and prompt_logprobs < 0:
373
- with pytest.raises(BadRequestError) as err_info:
286
+ if prompt_logprobs is not None and prompt_logprobs < 0:
287
+ with pytest.raises(BadRequestError):
288
await client.completions.create(**params)
375
- expected_err_string = (
376
- "Error code: 400 - {'object': 'error', 'message': "
377
- "'Prompt_logprobs set to invalid negative value: -1',"
378
- " 'type': 'BadRequestError', 'param': None, 'code': 400}")
379
- assert str(err_info.value) == expected_err_string
289
else:
290
completion = await client.completions.create(**params)
382
- if prompt_logprobs and prompt_logprobs > 0:
291
+ if prompt_logprobs is not None:
292
assert completion.choices[0].prompt_logprobs is not None
293
assert len(completion.choices[0].prompt_logprobs) > 0
294
vllm/engine/async_llm_engine.py
+4
-4
@@ -6,7 +6,6 @@ from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6
Optional, Set, Tuple, Type, Union)
7
8
import torch
9
-from transformers import PreTrainedTokenizer
9
from typing_extensions import assert_never
10
11
import vllm.envs as envs
@@ -31,6 +30,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
30
from vllm.sampling_params import SamplingParams
31
from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
32
SequenceGroupMetadata)
33
+from vllm.transformers_utils.tokenizer import AnyTokenizer
34
from vllm.usage.usage_lib import UsageContext
35
from vllm.utils import print_warning_once
36
@@ -427,8 +427,8 @@ class _AsyncLLMEngine(LLMEngine):
427
lora_request: Optional[LoRARequest],
428
) -> List[int]:
429
"""Async version of :meth:`_tokenize_prompt`."""
430
- tokenizer = self.get_tokenizer_group("prompts must be None if "
431
- "skip_tokenizer_init is True")
430
+ tokenizer = self.get_tokenizer_group(
431
+ missing_msg="prompts must be None if skip_tokenizer_init is True")
432
433
return await tokenizer.encode_async(request_id=request_id,
434
prompt=prompt,
@@ -771,7 +771,7 @@ class AsyncLLMEngine:
771
async def get_tokenizer(
772
self,
773
lora_request: Optional[LoRARequest] = None,
774
- ) -> "PreTrainedTokenizer":
774
+ ) -> AnyTokenizer:
775
if self.engine_use_ray:
776
return await self.engine.get_tokenizer.remote( # type: ignore
777
lora_request)
vllm/engine/llm_engine.py
+21
-10
@@ -3,9 +3,9 @@ from contextlib import contextmanager
3
from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
4
Mapping, Optional)
5
from typing import Sequence as GenericSequence
6
-from typing import Set, Tuple, Type, TypeVar, Union
6
+from typing import Set, Tuple, Type, Union
7
8
-from typing_extensions import assert_never
8
+from typing_extensions import TypeVar, assert_never
9
10
import vllm.envs as envs
11
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -43,8 +43,9 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
43
init_tracer)
44
from vllm.transformers_utils.config import try_get_generation_config
45
from vllm.transformers_utils.detokenizer import Detokenizer
46
+from vllm.transformers_utils.tokenizer import AnyTokenizer
47
from vllm.transformers_utils.tokenizer_group import (
47
- AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48
+ BaseTokenizerGroup, init_tokenizer_from_configs)
49
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
50
usage_message)
51
from vllm.utils import Counter, Device
@@ -67,6 +68,7 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
68
return config.to_diff_dict()
69
70
71
+_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
72
_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
73
74
PromptComponents = Tuple[Optional[str], List[int],
@@ -493,12 +495,21 @@ class LLMEngine:
495
"skip_tokenizer_init is True")
496
497
def get_tokenizer_group(
496
- self,
497
- fail_msg: str = MISSING_TOKENIZER_GROUP_MSG) -> BaseTokenizerGroup:
498
- if self.tokenizer is None:
499
- raise ValueError(fail_msg)
498
+ self,
499
+ group_type: Type[_G] = BaseTokenizerGroup,
500
+ *,
501
+ missing_msg: str = MISSING_TOKENIZER_GROUP_MSG,
502
+ ) -> _G:
503
+ tokenizer_group = self.tokenizer
504
+
505
+ if tokenizer_group is None:
506
+ raise ValueError(missing_msg)
507
+ if not isinstance(tokenizer_group, group_type):
508
+ raise TypeError("Invalid type of tokenizer group. "
509
+ f"Expected type: {group_type}, but "
510
+ f"found type: {type(tokenizer_group)}")
511
501
- return self.tokenizer
512
+ return tokenizer_group
513
514
def get_tokenizer(
515
self,
@@ -693,8 +704,8 @@ class LLMEngine:
704
* prompt token ids
705
'''
706
696
- tokenizer = self.get_tokenizer_group("prompts must be None if "
697
- "skip_tokenizer_init is True")
707
+ tokenizer = self.get_tokenizer_group(
708
+ missing_msg="prompts must be None if skip_tokenizer_init is True")
709
710
return tokenizer.encode(request_id=request_id,
711
prompt=prompt,
vllm/engine/output_processor/interfaces.py
+2
-3
@@ -1,13 +1,12 @@
1
from abc import ABC, abstractmethod
2
from typing import Callable, List
3
4
-from transformers import PreTrainedTokenizer
5
-
4
from vllm.config import SchedulerConfig
5
from vllm.core.scheduler import Scheduler
6
from vllm.engine.output_processor.stop_checker import StopChecker
7
from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
8
from vllm.transformers_utils.detokenizer import Detokenizer
9
+from vllm.transformers_utils.tokenizer import AnyTokenizer
10
from vllm.utils import Counter
11
12
@@ -29,7 +28,7 @@ class SequenceGroupOutputProcessor(ABC):
28
detokenizer: Detokenizer,
29
scheduler: List[Scheduler],
30
seq_counter: Counter,
32
- get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
31
+ get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
32
stop_checker: "StopChecker",
33
):
34
"""Create an output processor.
vllm/engine/output_processor/multi_step.py
+2
-3
@@ -1,8 +1,6 @@
1
import functools
2
from typing import Callable, List
3
4
-from transformers import PreTrainedTokenizer
5
-
4
from vllm.core.scheduler import Scheduler
5
from vllm.engine.output_processor.interfaces import (
6
SequenceGroupOutputProcessor)
@@ -12,6 +10,7 @@ from vllm.sampling_params import SamplingParams
10
from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11
SequenceOutput, SequenceStatus)
12
from vllm.transformers_utils.detokenizer import Detokenizer
13
+from vllm.transformers_utils.tokenizer import AnyTokenizer
14
from vllm.utils import Counter
15
16
logger = init_logger(__name__)
@@ -36,7 +35,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
35
detokenizer: Detokenizer,
36
scheduler: List[Scheduler],
37
seq_counter: Counter,
39
- get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
38
+ get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
39
stop_checker: StopChecker,
40
):
41
self.detokenizer = detokenizer
vllm/engine/output_processor/stop_checker.py
+2
-4
@@ -1,10 +1,9 @@
1
from typing import Callable, Optional
2
3
-from transformers import PreTrainedTokenizer
4
-
3
from vllm.lora.request import LoRARequest
4
from vllm.sampling_params import SamplingParams
5
from vllm.sequence import Sequence, SequenceStatus
6
+from vllm.transformers_utils.tokenizer import AnyTokenizer
7
8
9
class StopChecker:
@@ -15,8 +14,7 @@ class StopChecker:
14
"""
15
16
def __init__(self, max_model_len: int,
18
- get_tokenizer_for_seq: Callable[[Sequence],
19
- PreTrainedTokenizer]):
17
+ get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer]):
18
# Do not use it directly, but use `self._get_max_model_len`.
19
self._max_model_len = max_model_len
20
self.get_tokenizer_for_seq = get_tokenizer_for_seq
vllm/engine/protocol.py
+11
-6
@@ -1,8 +1,6 @@
1
from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2
runtime_checkable)
3
4
-from transformers import PreTrainedTokenizer
5
-
4
from vllm.config import DecodingConfig, ModelConfig
5
from vllm.core.scheduler import SchedulerOutputs
6
from vllm.inputs.data import PromptInputs
@@ -12,6 +10,7 @@ from vllm.pooling_params import PoolingParams
10
from vllm.prompt_adapter.request import PromptAdapterRequest
11
from vllm.sampling_params import SamplingParams
12
from vllm.sequence import SamplerOutput
13
+from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
16
@runtime_checkable
@@ -40,6 +39,7 @@ class AsyncEngineClient(Protocol):
39
prompt_adapter_request: Optional[PromptAdapterRequest] = None
40
) -> AsyncGenerator[RequestOutput, None]:
41
"""Generates outputs for a request"""
42
+ ...
43
44
def encode(
45
self,
@@ -50,6 +50,7 @@ class AsyncEngineClient(Protocol):
50
trace_headers: Optional[Mapping[str, str]] = None,
51
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
52
"""Generate outputs for a request from an embedding model."""
53
+ ...
54
55
async def abort(self, request_id: str) -> None:
56
"""Abort a request.
@@ -60,25 +61,29 @@ class AsyncEngineClient(Protocol):
61
62
async def get_model_config(self) -> ModelConfig:
63
"""Get the model configuration of the vLLM engine."""
64
+ ...
65
66
async def get_decoding_config(self) -> DecodingConfig:
67
+ ...
68
"""Get the decoding configuration of the vLLM engine."""
69
70
async def get_tokenizer(
71
self,
72
lora_request: Optional[LoRARequest] = None,
70
- ) -> PreTrainedTokenizer:
71
- """Get the appropriate Tokenizer for the request"""
73
+ ) -> AnyTokenizer:
74
+ """Get the appropriate tokenizer for the request"""
75
+ ...
76
77
async def is_tracing_enabled(self) -> bool:
74
- pass
78
+ ...
79
80
async def do_log_stats(
81
self,
82
scheduler_outputs: Optional[SchedulerOutputs] = None,
83
model_output: Optional[List[SamplerOutput]] = None,
84
) -> None:
81
- pass
85
+ ...
86
87
async def check_health(self) -> None:
88
"""Raise if unhealthy"""
89
+ ...
vllm/entrypoints/api_server.py
+3
@@ -61,6 +61,7 @@ async def generate(request: Request) -> Response:
61
async def stream_results() -> AsyncGenerator[bytes, None]:
62
async for request_output in results_generator:
63
prompt = request_output.prompt
64
+ assert prompt is not None
65
text_outputs = [
66
prompt + output.text for output in request_output.outputs
67
]
@@ -80,6 +81,7 @@ async def generate(request: Request) -> Response:
81
82
assert final_output is not None
83
prompt = final_output.prompt
84
+ assert prompt is not None
85
text_outputs = [prompt + output.text for output in final_output.outputs]
86
ret = {"text": text_outputs}
87
return JSONResponse(ret)
@@ -115,6 +117,7 @@ async def run_server(args: Namespace,
117
logger.info("args: %s", args)
118
119
app = await init_app(args, llm_engine)
120
+ assert engine is not None
121
122
shutdown_task = await serve_http(
123
app,
vllm/entrypoints/chat_utils.py
+24
-18
@@ -3,7 +3,7 @@ from dataclasses import dataclass
3
from functools import lru_cache
4
from pathlib import Path
5
from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6
- Union, cast)
6
+ Union)
7
8
# yapf conflicts with isort for this block
9
# yapf: disable
@@ -15,9 +15,8 @@ from openai.types.chat import (
15
ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
16
# yapf: enable
17
# pydantic needs the TypedDict from typing_extensions
18
-from pydantic import ConfigDict
19
-from transformers import PreTrainedTokenizer
20
-from typing_extensions import Required, TypedDict
18
+from pydantic import ConfigDict, TypeAdapter
19
+from typing_extensions import Required, TypeAlias, TypedDict
20
21
from vllm.config import ModelConfig
22
from vllm.logger import init_logger
@@ -50,9 +49,9 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
49
"""The type of the content part."""
50
51
53
-ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
54
- ChatCompletionContentPartAudioParam,
55
- CustomChatCompletionContentPartParam]
52
+ChatCompletionContentPartParam: TypeAlias = Union[
53
+ OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
54
+ CustomChatCompletionContentPartParam, ]
55
56
57
class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -114,7 +113,7 @@ def load_chat_template(
113
114
115
@lru_cache(maxsize=None)
117
-def _mm_token_str(model_config: ModelConfig, tokenizer: PreTrainedTokenizer,
116
+def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117
modality: Literal["image", "audio"]) -> Optional[str]:
118
# TODO: Let user specify how to insert image tokens into prompt
119
# (similar to chat template)
@@ -151,11 +150,16 @@ def _get_full_multimodal_text_prompt(placeholder_token_str: str,
150
return f"{placeholder_token_str}\n{text_prompt}"
151
152
153
+_TextParser = TypeAdapter(ChatCompletionContentPartTextParam)
154
+_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam)
155
+_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
156
+
157
+
158
def _parse_chat_message_content_parts(
159
role: str,
160
parts: Iterable[ChatCompletionContentPartParam],
161
model_config: ModelConfig,
158
- tokenizer: PreTrainedTokenizer,
162
+ tokenizer: AnyTokenizer,
163
) -> ChatMessageParseResult:
164
texts: List[str] = []
165
mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -164,7 +168,7 @@ def _parse_chat_message_content_parts(
168
for part in parts:
169
part_type = part["type"]
170
if part_type == "text":
167
- text = cast(ChatCompletionContentPartTextParam, part)["text"]
171
+ text = _TextParser.validate_python(part)["text"]
172
texts.append(text)
173
elif part_type == "image_url":
174
modality = "image"
@@ -172,8 +176,7 @@ def _parse_chat_message_content_parts(
176
raise NotImplementedError(
177
"Multiple multimodal inputs is currently not supported.")
178
175
- image_url = cast(ChatCompletionContentPartImageParam,
176
- part)["image_url"]
179
+ image_url = _ImageParser.validate_python(part)["image_url"]
180
181
if image_url.get("detail", "auto") != "auto":
182
logger.warning(
@@ -188,8 +191,7 @@ def _parse_chat_message_content_parts(
191
raise NotImplementedError(
192
"Multiple multimodal inputs is currently not supported.")
193
191
- audio_url = cast(ChatCompletionContentPartAudioParam,
192
- part)["audio_url"]
194
+ audio_url = _AudioParser.validate_python(part)["audio_url"]
195
audio_future = async_get_and_parse_audio(audio_url["url"])
196
mm_futures.append(audio_future)
197
else:
@@ -219,7 +221,7 @@ def _parse_chat_message_content_parts(
221
def _parse_chat_message_content(
222
message: ChatCompletionMessageParam,
223
model_config: ModelConfig,
222
- tokenizer: PreTrainedTokenizer,
224
+ tokenizer: AnyTokenizer,
225
) -> ChatMessageParseResult:
226
role = message["role"]
227
content = message.get("content")
@@ -230,14 +232,18 @@ def _parse_chat_message_content(
232
messages = [ConversationMessage(role=role, content=content)]
233
return ChatMessageParseResult(messages=messages, mm_futures=[])
234
233
- return _parse_chat_message_content_parts(role, content, model_config,
234
- tokenizer)
235
+ return _parse_chat_message_content_parts(
236
+ role,
237
+ content, # type: ignore
238
+ model_config,
239
+ tokenizer,
240
+ )
241
242
243
def parse_chat_messages(
244
messages: List[ChatCompletionMessageParam],
245
model_config: ModelConfig,
240
- tokenizer: PreTrainedTokenizer,
246
+ tokenizer: AnyTokenizer,
247
) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
248
conversation: List[ConversationMessage] = []
249
mm_futures: List[Awaitable[MultiModalDataDict]] = []
vllm/entrypoints/llm.py
+15
-15
@@ -1,8 +1,7 @@
1
from contextlib import contextmanager
2
from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
3
4
-from tqdm.auto import tqdm
5
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4
+from tqdm import tqdm
5
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.engine.llm_engine import LLMEngine
@@ -20,7 +19,9 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
19
from vllm.pooling_params import PoolingParams
20
from vllm.prompt_adapter.request import PromptAdapterRequest
21
from vllm.sampling_params import SamplingParams
23
-from vllm.transformers_utils.tokenizer import get_cached_tokenizer
22
+from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23
+ get_cached_tokenizer)
24
+from vllm.transformers_utils.tokenizer_group import TokenizerGroup
25
from vllm.usage.usage_lib import UsageContext
26
from vllm.utils import Counter, deprecate_kwargs
27
@@ -122,7 +123,7 @@ class LLM:
123
tokenizer_revision: Optional[str] = None,
124
seed: int = 0,
125
gpu_memory_utilization: float = 0.9,
125
- swap_space: int = 4,
126
+ swap_space: float = 4,
127
cpu_offload_gb: float = 0,
128
enforce_eager: Optional[bool] = None,
129
max_context_len_to_capture: Optional[int] = None,
@@ -175,22 +176,19 @@ class LLM:
176
engine_args, usage_context=UsageContext.LLM_CLASS)
177
self.request_counter = Counter()
178
178
- def get_tokenizer(
179
- self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
180
- return self.llm_engine.tokenizer.tokenizer
179
+ def get_tokenizer(self) -> AnyTokenizer:
180
+ return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
181
+
182
+ def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
183
+ tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
184
182
- def set_tokenizer(
183
- self,
184
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
185
- ) -> None:
185
# While CachedTokenizer is dynamic, have no choice but
186
# compare class name. Misjudgment will arise from
187
# user-defined tokenizer started with 'Cached'
188
if tokenizer.__class__.__name__.startswith("Cached"):
190
- self.llm_engine.tokenizer.tokenizer = tokenizer
189
+ tokenizer_group.tokenizer = tokenizer
190
else:
192
- self.llm_engine.tokenizer.tokenizer = get_cached_tokenizer(
193
- tokenizer)
191
+ tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
192
193
@overload # LEGACY: single (prompt + optional token ids)
194
def generate(
@@ -578,6 +576,8 @@ class LLM:
576
577
inputs: List[PromptInputs] = []
578
for i in range(num_requests):
579
+ item: PromptInputs
580
+
581
if prompts is not None:
582
item = TextPrompt(prompt=prompts[i])
583
elif prompt_token_ids is not None:
@@ -635,7 +635,7 @@ class LLM:
635
self,
636
inputs: PromptInputs,
637
params: Union[SamplingParams, PoolingParams],
638
- lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
638
+ lora_request: Optional[LoRARequest] = None,
639
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
640
) -> None:
641
request_id = str(next(self.request_counter))
vllm/entrypoints/openai/api_server.py
+28
-20
@@ -15,6 +15,7 @@ from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
from fastapi.responses import JSONResponse, Response, StreamingResponse
17
from starlette.routing import Mount
18
+from typing_extensions import assert_never
19
20
import vllm.envs as envs
21
from vllm.config import ModelConfig
@@ -29,14 +30,16 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser
30
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31
ChatCompletionResponse,
32
CompletionRequest,
33
+ CompletionResponse,
34
DetokenizeRequest,
35
DetokenizeResponse,
34
- EmbeddingRequest, ErrorResponse,
36
+ EmbeddingRequest,
37
+ EmbeddingResponse, ErrorResponse,
38
TokenizeRequest,
39
TokenizeResponse)
40
+# yapf: enable
41
from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
42
from vllm.entrypoints.openai.rpc.server import run_rpc_server
39
-# yapf: enable
43
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
44
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
45
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
@@ -90,7 +93,8 @@ async def lifespan(app: FastAPI):
93
94
95
@asynccontextmanager
93
-async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
96
+async def build_async_engine_client(
97
+ args: Namespace) -> AsyncIterator[AsyncEngineClient]:
98
# Context manager to handle async_engine_client lifecycle
99
# Ensures everything is shutdown and cleaned up on error/exit
100
global engine_args
@@ -142,12 +146,15 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
146
logger.info("Started engine process with PID %d",
147
rpc_server_process.pid)
148
# Build RPCClient, which conforms to AsyncEngineClient Protocol.
145
- async_engine_client = AsyncEngineRPCClient(rpc_path)
149
+ # NOTE: Actually, this is not true yet. We still need to support
150
+ # embedding models via RPC (see TODO above)
151
+ rpc_client = AsyncEngineRPCClient(rpc_path)
152
+ async_engine_client = rpc_client # type: ignore
153
154
try:
155
while True:
156
try:
150
- await async_engine_client.setup()
157
+ await rpc_client.setup()
158
break
159
except TimeoutError as e:
160
if not rpc_server_process.is_alive():
@@ -161,7 +168,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
168
rpc_server_process.terminate()
169
170
# Close all open connections to the backend
164
- async_engine_client.close()
171
+ rpc_client.close()
172
173
# Wait for server process to join
174
rpc_server_process.join()
@@ -216,10 +223,11 @@ async def tokenize(request: TokenizeRequest):
223
if isinstance(generator, ErrorResponse):
224
return JSONResponse(content=generator.model_dump(),
225
status_code=generator.code)
219
- else:
220
- assert isinstance(generator, TokenizeResponse)
226
+ elif isinstance(generator, TokenizeResponse):
227
return JSONResponse(content=generator.model_dump())
228
229
+ assert_never(generator)
230
+
231
232
@router.post("/detokenize")
233
async def detokenize(request: DetokenizeRequest):
@@ -227,10 +235,11 @@ async def detokenize(request: DetokenizeRequest):
235
if isinstance(generator, ErrorResponse):
236
return JSONResponse(content=generator.model_dump(),
237
status_code=generator.code)
230
- else:
231
- assert isinstance(generator, DetokenizeResponse)
238
+ elif isinstance(generator, DetokenizeResponse):
239
return JSONResponse(content=generator.model_dump())
240
241
+ assert_never(generator)
242
+
243
244
@router.get("/v1/models")
245
async def show_available_models():
@@ -252,13 +261,11 @@ async def create_chat_completion(request: ChatCompletionRequest,
261
if isinstance(generator, ErrorResponse):
262
return JSONResponse(content=generator.model_dump(),
263
status_code=generator.code)
255
- if request.stream:
256
- return StreamingResponse(content=generator,
257
- media_type="text/event-stream")
258
- else:
259
- assert isinstance(generator, ChatCompletionResponse)
264
+ elif isinstance(generator, ChatCompletionResponse):
265
return JSONResponse(content=generator.model_dump())
266
267
+ return StreamingResponse(content=generator, media_type="text/event-stream")
268
+
269
270
@router.post("/v1/completions")
271
async def create_completion(request: CompletionRequest, raw_request: Request):
@@ -267,12 +274,11 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
274
if isinstance(generator, ErrorResponse):
275
return JSONResponse(content=generator.model_dump(),
276
status_code=generator.code)
270
- if request.stream:
271
- return StreamingResponse(content=generator,
272
- media_type="text/event-stream")
273
- else:
277
+ elif isinstance(generator, CompletionResponse):
278
return JSONResponse(content=generator.model_dump())
279
280
+ return StreamingResponse(content=generator, media_type="text/event-stream")
281
+
282
283
@router.post("/v1/embeddings")
284
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
@@ -281,9 +287,11 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
287
if isinstance(generator, ErrorResponse):
288
return JSONResponse(content=generator.model_dump(),
289
status_code=generator.code)
284
- else:
290
+ elif isinstance(generator, EmbeddingResponse):
291
return JSONResponse(content=generator.model_dump())
292
293
+ assert_never(generator)
294
+
295
296
def build_app(args: Namespace) -> FastAPI:
297
app = FastAPI(lifespan=lifespan)
vllm/entrypoints/openai/cli_args.py
+27
-4
@@ -7,6 +7,7 @@ purposes.
7
import argparse
8
import json
9
import ssl
10
+from typing import List, Optional, Sequence, Union
11
12
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
@@ -16,8 +17,19 @@ from vllm.utils import FlexibleArgumentParser
17
18
class LoRAParserAction(argparse.Action):
19
19
- def __call__(self, parser, namespace, values, option_string=None):
20
- lora_list = []
20
+ def __call__(
21
+ self,
22
+ parser: argparse.ArgumentParser,
23
+ namespace: argparse.Namespace,
24
+ values: Optional[Union[str, Sequence[str]]],
25
+ option_string: Optional[str] = None,
26
+ ):
27
+ if values is None:
28
+ values = []
29
+ if isinstance(values, str):
30
+ raise TypeError("Expected values to be a list")
31
+
32
+ lora_list: List[LoRAModulePath] = []
33
for item in values:
34
name, path = item.split('=')
35
lora_list.append(LoRAModulePath(name, path))
@@ -26,8 +38,19 @@ class LoRAParserAction(argparse.Action):
38
39
class PromptAdapterParserAction(argparse.Action):
40
29
- def __call__(self, parser, namespace, values, option_string=None):
30
- adapter_list = []
41
+ def __call__(
42
+ self,
43
+ parser: argparse.ArgumentParser,
44
+ namespace: argparse.Namespace,
45
+ values: Optional[Union[str, Sequence[str]]],
46
+ option_string: Optional[str] = None,
47
+ ):
48
+ if values is None:
49
+ values = []
50
+ if isinstance(values, str):
51
+ raise TypeError("Expected values to be a list")
52
+
53
+ adapter_list: List[PromptAdapterPath] = []
54
for item in values:
55
name, path = item.split('=')
56
adapter_list.append(PromptAdapterPath(name, path))
vllm/entrypoints/openai/logits_processors.py
+6
-5
@@ -2,9 +2,9 @@ from functools import lru_cache, partial
2
from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3
4
import torch
5
-from transformers import PreTrainedTokenizer
5
6
from vllm.sampling_params import LogitsProcessor
7
+from vllm.transformers_utils.tokenizer import AnyTokenizer
8
9
10
class AllowedTokenIdsLogitsProcessor:
@@ -51,10 +51,11 @@ def logit_bias_logits_processor(
51
52
53
def get_logits_processors(
54
- logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
55
- allowed_token_ids: Optional[List[int]],
56
- tokenizer: PreTrainedTokenizer) -> List[LogitsProcessor]:
57
- logits_processors = []
54
+ logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
55
+ allowed_token_ids: Optional[List[int]],
56
+ tokenizer: AnyTokenizer,
57
+) -> List[LogitsProcessor]:
58
+ logits_processors: List[LogitsProcessor] = []
59
if logit_bias:
60
try:
61
# Convert token_id to integer
vllm/entrypoints/openai/protocol.py
+58
-33
@@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optional, Union
6
7
import torch
8
from pydantic import BaseModel, ConfigDict, Field, model_validator
9
-from transformers import PreTrainedTokenizer
9
from typing_extensions import Annotated
10
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -14,11 +13,13 @@ from vllm.entrypoints.openai.logits_processors import get_logits_processors
13
from vllm.pooling_params import PoolingParams
14
from vllm.sampling_params import LogitsProcessor, SamplingParams
15
from vllm.sequence import Logprob
16
+from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.utils import random_uuid
18
19
# torch is mocked during docs generation,
20
# so we have to provide the values as literals
21
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
22
+_LONG_INFO: Union["torch.iinfo", Namespace]
23
24
try:
25
from sphinx.ext.autodoc.mock import _MockModule
@@ -235,13 +236,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
236
# doc: end-chat-completion-extra-params
237
238
def to_sampling_params(
238
- self, tokenizer: PreTrainedTokenizer,
239
+ self, tokenizer: AnyTokenizer,
240
guided_decode_logits_processor: Optional[LogitsProcessor],
241
default_max_tokens: int) -> SamplingParams:
242
max_tokens = self.max_tokens
243
if max_tokens is None:
244
max_tokens = default_max_tokens
245
246
+ prompt_logprobs = self.prompt_logprobs
247
+ if prompt_logprobs is None and self.echo:
248
+ prompt_logprobs = self.top_logprobs
249
+
250
# We now allow logprobs being true without top_logrobs.
251
logits_processors = get_logits_processors(
252
logit_bias=self.logit_bias,
@@ -251,7 +256,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
256
if guided_decode_logits_processor:
257
logits_processors.append(guided_decode_logits_processor)
258
254
- return SamplingParams(
259
+ return SamplingParams.from_optional(
260
n=self.n,
261
best_of=self.best_of,
262
presence_penalty=self.presence_penalty,
@@ -265,8 +270,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
270
stop=self.stop,
271
stop_token_ids=self.stop_token_ids,
272
logprobs=self.top_logprobs if self.logprobs else None,
268
- prompt_logprobs=self.prompt_logprobs if self.prompt_logprobs else
269
- (self.top_logprobs if self.echo else None),
273
+ prompt_logprobs=prompt_logprobs,
274
ignore_eos=self.ignore_eos,
275
max_tokens=max_tokens,
276
min_tokens=self.min_tokens,
@@ -280,14 +284,36 @@ class ChatCompletionRequest(OpenAIBaseModel):
284
truncate_prompt_tokens=self.truncate_prompt_tokens,
285
)
286
283
- @model_validator(mode='before')
287
+ @model_validator(mode="before")
288
@classmethod
285
- def validate_stream_options(cls, values):
286
- if (values.get('stream_options') is not None
287
- and not values.get('stream')):
289
+ def validate_stream_options(cls, data):
290
+ if data.get("stream_options") and not data.get("stream"):
291
raise ValueError(
289
- "stream_options can only be set if stream is true")
290
- return values
292
+ "Stream options can only be defined when `stream=True`.")
293
+
294
+ return data
295
+
296
+ @model_validator(mode="before")
297
+ @classmethod
298
+ def check_logprobs(cls, data):
299
+ if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
300
+ if data.get("stream") and prompt_logprobs > 0:
301
+ raise ValueError(
302
+ "`prompt_logprobs` are not available when `stream=True`.")
303
+
304
+ if prompt_logprobs < 0:
305
+ raise ValueError("`prompt_logprobs` must be a positive value.")
306
+
307
+ if (top_logprobs := data.get("top_logprobs")) is not None:
308
+ if top_logprobs < 0:
309
+ raise ValueError("`top_logprobs` must be a positive value.")
310
+
311
+ if not data.get("logprobs"):
312
+ raise ValueError(
313
+ "when using `top_logprobs`, `logprobs` must be set to true."
314
+ )
315
+
316
+ return data
317
318
@model_validator(mode="before")
319
@classmethod
@@ -320,19 +346,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
346
"When using `tool_choice`, `tools` must be set.")
347
return data
348
323
- @model_validator(mode="before")
324
- @classmethod
325
- def check_logprobs(cls, data):
326
- if "top_logprobs" in data and data["top_logprobs"] is not None:
327
- if "logprobs" not in data or data["logprobs"] is False:
328
- raise ValueError(
329
- "when using `top_logprobs`, `logprobs` must be set to true."
330
- )
331
- elif data["top_logprobs"] < 0:
332
- raise ValueError(
333
- "`top_logprobs` must be a value a positive value.")
334
- return data
335
-
349
350
class CompletionRequest(OpenAIBaseModel):
351
# Ordered by official OpenAI API documentation
@@ -422,13 +435,17 @@ class CompletionRequest(OpenAIBaseModel):
435
# doc: end-completion-extra-params
436
437
def to_sampling_params(
425
- self, tokenizer: PreTrainedTokenizer,
438
+ self, tokenizer: AnyTokenizer,
439
guided_decode_logits_processor: Optional[LogitsProcessor],
440
default_max_tokens: int) -> SamplingParams:
441
max_tokens = self.max_tokens
442
if max_tokens is None:
443
max_tokens = default_max_tokens
444
445
+ prompt_logprobs = self.prompt_logprobs
446
+ if prompt_logprobs is None and self.echo:
447
+ prompt_logprobs = self.logprobs
448
+
449
echo_without_generation = self.echo and self.max_tokens == 0
450
451
logits_processors = get_logits_processors(
@@ -439,7 +456,7 @@ class CompletionRequest(OpenAIBaseModel):
456
if guided_decode_logits_processor:
457
logits_processors.append(guided_decode_logits_processor)
458
442
- return SamplingParams(
459
+ return SamplingParams.from_optional(
460
n=self.n,
461
best_of=self.best_of,
462
presence_penalty=self.presence_penalty,
@@ -458,8 +475,7 @@ class CompletionRequest(OpenAIBaseModel):
475
min_tokens=self.min_tokens,
476
use_beam_search=self.use_beam_search,
477
early_stopping=self.early_stopping,
461
- prompt_logprobs=self.prompt_logprobs
462
- if self.prompt_logprobs else self.logprobs if self.echo else None,
478
+ prompt_logprobs=prompt_logprobs,
479
skip_special_tokens=self.skip_special_tokens,
480
spaces_between_special_tokens=self.spaces_between_special_tokens,
481
include_stop_str_in_output=self.include_stop_str_in_output,
@@ -485,9 +501,17 @@ class CompletionRequest(OpenAIBaseModel):
501
@model_validator(mode="before")
502
@classmethod
503
def check_logprobs(cls, data):
488
- if "logprobs" in data and data[
489
- "logprobs"] is not None and not data["logprobs"] >= 0:
490
- raise ValueError("if passed, `logprobs` must be a positive value.")
504
+ if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
505
+ if data.get("stream") and prompt_logprobs > 0:
506
+ raise ValueError(
507
+ "`prompt_logprobs` are not available when `stream=True`.")
508
+
509
+ if prompt_logprobs < 0:
510
+ raise ValueError("`prompt_logprobs` must be a positive value.")
511
+
512
+ if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
513
+ raise ValueError("`logprobs` must be a positive value.")
514
+
515
return data
516
517
@model_validator(mode="before")
@@ -495,7 +519,8 @@ class CompletionRequest(OpenAIBaseModel):
519
def validate_stream_options(cls, data):
520
if data.get("stream_options") and not data.get("stream"):
521
raise ValueError(
498
- "Stream options can only be defined when stream is true.")
522
+ "Stream options can only be defined when `stream=True`.")
523
+
524
return data
525
526
@@ -504,7 +529,7 @@ class EmbeddingRequest(OpenAIBaseModel):
529
# https://platform.openai.com/docs/api-reference/embeddings
530
model: str
531
input: Union[List[int], List[List[int]], str, List[str]]
507
- encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
532
+ encoding_format: Literal["float", "base64"] = "float"
533
dimensions: Optional[int] = None
534
user: Optional[str] = None
535
vllm/entrypoints/openai/rpc/server.py
+3
-3
@@ -23,8 +23,8 @@ class AsyncEngineRPCServer:
23
def __init__(self, async_engine_args: AsyncEngineArgs,
24
usage_context: UsageContext, rpc_path: str):
25
# Initialize engine first.
26
- self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
27
- usage_context)
26
+ self.engine = AsyncLLMEngine.from_engine_args(
27
+ async_engine_args, usage_context=usage_context)
28
29
# Initialize context.
30
self.context = zmq.asyncio.Context()
@@ -39,7 +39,7 @@ class AsyncEngineRPCServer:
39
self.context.destroy()
40
self.engine.shutdown_background_loop()
41
# Clear the engine reference so that it can be GC'ed.
42
- self.engine = None
42
+ del self.engine
43
44
async def get_model_config(self, identity):
45
"""Send the ModelConfig"""
vllm/entrypoints/openai/serving_chat.py
+33
-36
@@ -1,11 +1,10 @@
1
import asyncio
2
import time
3
-from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
3
+from typing import AsyncGenerator, AsyncIterator, Dict, Final, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Union
6
7
from fastapi import Request
8
-from transformers import PreTrainedTokenizer
8
9
from vllm.config import ModelConfig
10
from vllm.engine.protocol import AsyncEngineClient
@@ -24,13 +23,14 @@ from vllm.entrypoints.openai.protocol import (
23
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24
OpenAIServing,
25
PromptAdapterPath)
27
-from vllm.inputs import PromptInputs
26
+from vllm.inputs import TokensPrompt
27
from vllm.logger import init_logger
28
from vllm.multimodal import MultiModalDataDict
29
from vllm.outputs import RequestOutput
30
from vllm.sequence import Logprob
31
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
32
log_tracing_disabled_warning)
33
+from vllm.transformers_utils.tokenizer import AnyTokenizer
34
from vllm.utils import iterate_with_cancellation, random_uuid
35
36
logger = init_logger(__name__)
@@ -67,9 +67,9 @@ class OpenAIServingChat(OpenAIServing):
67
async def create_chat_completion(
68
self,
69
request: ChatCompletionRequest,
70
- raw_request: Optional[Request] = None
71
- ) -> Union[ErrorResponse, AsyncGenerator[str, None],
72
- ChatCompletionResponse]:
70
+ raw_request: Optional[Request] = None,
71
+ ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
72
+ ErrorResponse]:
73
"""Completion API similar to OpenAI's API.
74
75
See https://platform.openai.com/docs/api-reference/chat/create
@@ -83,16 +83,6 @@ class OpenAIServingChat(OpenAIServing):
83
if error_check_ret is not None:
84
return error_check_ret
85
86
- if request.prompt_logprobs is not None:
87
- if request.stream and request.prompt_logprobs > 0:
88
- return self.create_error_response(
89
- "Prompt_logprobs are not available when stream is enabled")
90
-
91
- if request.prompt_logprobs < 0:
92
- return self.create_error_response(
93
- f"Prompt_logprobs set to invalid "
94
- f"negative value: {request.prompt_logprobs}")
95
-
86
try:
87
(
88
lora_request,
@@ -160,9 +150,8 @@ class OpenAIServingChat(OpenAIServing):
150
lora_request=lora_request,
151
prompt_adapter_request=prompt_adapter_request)
152
163
- engine_inputs: PromptInputs = {
164
- "prompt_token_ids": prompt_inputs["prompt_token_ids"],
165
- }
153
+ engine_inputs = TokensPrompt(
154
+ prompt_token_ids=prompt_inputs["prompt_token_ids"])
155
if mm_data is not None:
156
engine_inputs["multi_modal_data"] = mm_data
157
@@ -214,11 +203,11 @@ class OpenAIServingChat(OpenAIServing):
203
result_generator: AsyncIterator[RequestOutput],
204
request_id: str,
205
conversation: List[ConversationMessage],
217
- tokenizer: PreTrainedTokenizer,
206
+ tokenizer: AnyTokenizer,
207
) -> AsyncGenerator[str, None]:
208
model_name = self.served_model_names[0]
209
created_time = int(time.time())
221
- chunk_object_type = "chat.completion.chunk"
210
+ chunk_object_type: Final = "chat.completion.chunk"
211
first_iteration = True
212
213
# Send response for each token for each request.n (index)
@@ -438,7 +427,7 @@ class OpenAIServingChat(OpenAIServing):
427
result_generator: AsyncIterator[RequestOutput],
428
request_id: str,
429
conversation: List[ConversationMessage],
441
- tokenizer: PreTrainedTokenizer,
430
+ tokenizer: AnyTokenizer,
431
) -> Union[ErrorResponse, ChatCompletionResponse]:
432
433
model_name = self.served_model_names[0]
@@ -523,7 +512,7 @@ class OpenAIServingChat(OpenAIServing):
512
513
def _get_top_logprobs(
514
self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
526
- tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
515
+ tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
516
return [
517
ChatCompletionLogProb(token=(token := self._get_decoded_token(
518
p[1],
@@ -541,12 +530,11 @@ class OpenAIServingChat(OpenAIServing):
530
self,
531
token_ids: GenericSequence[int],
532
top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
544
- tokenizer: PreTrainedTokenizer,
533
+ tokenizer: AnyTokenizer,
534
num_output_top_logprobs: Optional[int] = None,
535
) -> ChatCompletionLogProbs:
536
"""Create OpenAI-style logprobs."""
548
-
549
- logprobs_content = []
537
+ logprobs_content: List[ChatCompletionLogProbsContent] = []
538
539
for i, token_id in enumerate(token_ids):
540
step_top_logprobs = top_logprobs[i]
@@ -554,23 +542,32 @@ class OpenAIServingChat(OpenAIServing):
542
token = tokenizer.decode(token_id)
543
if self.return_tokens_as_token_ids:
544
token = f"token_id:{token_id}"
545
+
546
logprobs_content.append(
547
ChatCompletionLogProbsContent(
548
token=token,
560
- bytes=list(token.encode("utf-8", errors="replace"))))
549
+ bytes=list(token.encode("utf-8", errors="replace")),
550
+ ))
551
else:
552
+ step_token = step_top_logprobs[token_id]
553
+ step_decoded = step_token.decoded_token
554
+
555
logprobs_content.append(
556
ChatCompletionLogProbsContent(
557
token=self._get_decoded_token(
565
- step_top_logprobs[token_id], token_id, tokenizer,
566
- self.return_tokens_as_token_ids),
567
- logprob=max(step_top_logprobs[token_id].logprob,
568
- -9999.0),
569
- bytes=list(
570
- step_top_logprobs[token_id].decoded_token.encode(
571
- "utf-8", errors="replace")),
558
+ step_token,
559
+ token_id,
560
+ tokenizer,
561
+ self.return_tokens_as_token_ids,
562
+ ),
563
+ logprob=max(step_token.logprob, -9999.0),
564
+ bytes=None if step_decoded is None else list(
565
+ step_decoded.encode("utf-8", errors="replace")),
566
top_logprobs=self._get_top_logprobs(
573
- step_top_logprobs, num_output_top_logprobs,
574
- tokenizer)))
567
+ step_top_logprobs,
568
+ num_output_top_logprobs,
569
+ tokenizer,
570
+ ),
571
+ ))
572
573
return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+61
-36
@@ -3,10 +3,9 @@ import time
3
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
Optional)
5
from typing import Sequence as GenericSequence
6
-from typing import Tuple, cast
6
+from typing import Tuple, Union, cast
7
8
from fastapi import Request
9
-from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import AsyncEngineClient
@@ -19,7 +18,7 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
18
CompletionResponseChoice,
19
CompletionResponseStreamChoice,
20
CompletionStreamResponse,
22
- UsageInfo)
21
+ ErrorResponse, UsageInfo)
22
# yapf: enable
23
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24
OpenAIServing,
@@ -29,6 +28,7 @@ from vllm.outputs import RequestOutput
28
from vllm.sequence import Logprob
29
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
30
log_tracing_disabled_warning)
31
+from vllm.transformers_utils.tokenizer import AnyTokenizer
32
from vllm.utils import merge_async_iterators, random_uuid
33
34
logger = init_logger(__name__)
@@ -60,8 +60,11 @@ class OpenAIServingCompletion(OpenAIServing):
60
request_logger=request_logger,
61
return_tokens_as_token_ids=return_tokens_as_token_ids)
62
63
- async def create_completion(self, request: CompletionRequest,
64
- raw_request: Request):
63
+ async def create_completion(
64
+ self,
65
+ request: CompletionRequest,
66
+ raw_request: Request,
67
+ ) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]:
68
"""Completion API similar to OpenAI's API.
69
70
See https://platform.openai.com/docs/api-reference/completions/create
@@ -84,15 +87,6 @@ class OpenAIServingCompletion(OpenAIServing):
87
request_id = f"cmpl-{random_uuid()}"
88
created_time = int(time.time())
89
87
- if request.prompt_logprobs is not None:
88
- if request.stream and request.prompt_logprobs > 0:
89
- return self.create_error_response(
90
- "Prompt_logprobs are not available when stream is enabled")
91
- elif request.prompt_logprobs < 0:
92
- return self.create_error_response(
93
- f"Prompt_logprobs set to invalid negative "
94
- f"value: {request.prompt_logprobs}")
95
-
90
# Schedule the request and get the result generator.
91
generators: List[AsyncGenerator[RequestOutput, None]] = []
92
try:
@@ -153,9 +147,8 @@ class OpenAIServingCompletion(OpenAIServing):
147
# TODO: Use a vllm-specific Validation Error
148
return self.create_error_response(str(e))
149
156
- result_generator: AsyncIterator[Tuple[
157
- int, RequestOutput]] = merge_async_iterators(
158
- *generators, is_cancelled=raw_request.is_disconnected)
150
+ result_generator = merge_async_iterators(
151
+ *generators, is_cancelled=raw_request.is_disconnected)
152
153
# Similar to the OpenAI API, when n != best_of, we do not stream the
154
# results. In addition, we do not stream the results when use
@@ -227,7 +220,7 @@ class OpenAIServingCompletion(OpenAIServing):
220
created_time: int,
221
model_name: str,
222
num_prompts: int,
230
- tokenizer: PreTrainedTokenizer,
223
+ tokenizer: AnyTokenizer,
224
) -> AsyncGenerator[str, None]:
225
num_choices = 1 if request.n is None else request.n
226
previous_texts = [""] * num_choices * num_prompts
@@ -236,6 +229,13 @@ class OpenAIServingCompletion(OpenAIServing):
229
230
try:
231
async for prompt_idx, res in result_generator:
232
+ prompt_token_ids = res.prompt_token_ids
233
+ prompt_logprobs = res.prompt_logprobs
234
+ prompt_text = res.prompt
235
+
236
+ delta_token_ids: GenericSequence[int]
237
+ out_logprobs: Optional[GenericSequence[Optional[Dict[
238
+ int, Logprob]]]]
239
240
for output in res.outputs:
241
i = output.index + prompt_idx * num_choices
@@ -244,19 +244,25 @@ class OpenAIServingCompletion(OpenAIServing):
244
245
assert request.max_tokens is not None
246
if request.echo and request.max_tokens == 0:
247
+ assert prompt_text is not None
248
# only return the prompt
248
- delta_text = res.prompt
249
- delta_token_ids = res.prompt_token_ids
250
- out_logprobs = res.prompt_logprobs
249
+ delta_text = prompt_text
250
+ delta_token_ids = prompt_token_ids
251
+ out_logprobs = prompt_logprobs
252
has_echoed[i] = True
253
elif (request.echo and request.max_tokens > 0
254
and not has_echoed[i]):
255
+ assert prompt_text is not None
256
+ assert prompt_logprobs is not None
257
# echo the prompt and first token
255
- delta_text = res.prompt + output.text
256
- delta_token_ids = (res.prompt_token_ids +
257
- output.token_ids)
258
- out_logprobs = res.prompt_logprobs + (output.logprobs
259
- or [])
258
+ delta_text = prompt_text + output.text
259
+ delta_token_ids = [
260
+ *prompt_token_ids, *output.token_ids
261
+ ]
262
+ out_logprobs = [
263
+ *prompt_logprobs,
264
+ *(output.logprobs or []),
265
+ ]
266
has_echoed[i] = True
267
else:
268
# return just the delta
@@ -301,7 +307,7 @@ class OpenAIServingCompletion(OpenAIServing):
307
and request.stream_options.include_usage):
308
if (request.stream_options.continuous_usage_stats
309
or output.finish_reason is not None):
304
- prompt_tokens = len(res.prompt_token_ids)
310
+ prompt_tokens = len(prompt_token_ids)
311
completion_tokens = len(output.token_ids)
312
usage = UsageInfo(
313
prompt_tokens=prompt_tokens,
@@ -342,7 +348,7 @@ class OpenAIServingCompletion(OpenAIServing):
348
request_id: str,
349
created_time: int,
350
model_name: str,
345
- tokenizer: PreTrainedTokenizer,
351
+ tokenizer: AnyTokenizer,
352
) -> CompletionResponse:
353
choices: List[CompletionResponseChoice] = []
354
num_prompt_tokens = 0
@@ -353,16 +359,31 @@ class OpenAIServingCompletion(OpenAIServing):
359
prompt_logprobs = final_res.prompt_logprobs
360
prompt_text = final_res.prompt
361
362
+ token_ids: GenericSequence[int]
363
+ out_logprobs: Optional[GenericSequence[Optional[Dict[int,
364
+ Logprob]]]]
365
+
366
for output in final_res.outputs:
367
assert request.max_tokens is not None
368
if request.echo and request.max_tokens == 0:
369
+ assert prompt_text is not None
370
token_ids = prompt_token_ids
371
out_logprobs = prompt_logprobs
372
output_text = prompt_text
373
elif request.echo and request.max_tokens > 0:
363
- token_ids = prompt_token_ids + list(output.token_ids)
364
- out_logprobs = (prompt_logprobs + output.logprobs
365
- if request.logprobs is not None else None)
374
+ assert prompt_text is not None
375
+ token_ids = [*prompt_token_ids, *output.token_ids]
376
+
377
+ if request.logprobs is None:
378
+ out_logprobs = None
379
+ else:
380
+ assert prompt_logprobs is not None
381
+ assert output.logprobs is not None
382
+ out_logprobs = [
383
+ *prompt_logprobs,
384
+ *output.logprobs,
385
+ ]
386
+
387
output_text = prompt_text + output.text
388
else:
389
token_ids = output.token_ids
@@ -413,7 +434,7 @@ class OpenAIServingCompletion(OpenAIServing):
434
token_ids: GenericSequence[int],
435
top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
436
num_output_top_logprobs: int,
416
- tokenizer: PreTrainedTokenizer,
437
+ tokenizer: AnyTokenizer,
438
initial_text_offset: int = 0,
439
) -> CompletionLogProbs:
440
"""Create logprobs for OpenAI Completion API."""
@@ -430,17 +451,21 @@ class OpenAIServingCompletion(OpenAIServing):
451
token = tokenizer.decode(token_id)
452
if self.return_tokens_as_token_ids:
453
token = f"token_id:{token_id}"
454
+
455
out_tokens.append(token)
456
out_token_logprobs.append(None)
457
out_top_logprobs.append(None)
458
else:
459
+ step_token = step_top_logprobs[token_id]
460
+
461
token = self._get_decoded_token(
438
- step_top_logprobs[token_id],
462
+ step_token,
463
token_id,
464
tokenizer,
441
- return_as_token_id=self.return_tokens_as_token_ids)
442
- token_logprob = max(step_top_logprobs[token_id].logprob,
443
- -9999.0)
465
+ return_as_token_id=self.return_tokens_as_token_ids,
466
+ )
467
+ token_logprob = max(step_token.logprob, -9999.0)
468
+
469
out_tokens.append(token)
470
out_token_logprobs.append(token_logprob)
471
vllm/entrypoints/openai/serving_embedding.py
+26
-18
@@ -1,11 +1,11 @@
1
import asyncio
2
import base64
3
import time
4
-from typing import (AsyncGenerator, AsyncIterator, List, Optional, Tuple,
5
- Union, cast)
4
+from typing import AsyncGenerator, List, Literal, Optional, Union, cast
5
6
import numpy as np
7
from fastapi import Request
8
+from typing_extensions import assert_never
9
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import AsyncEngineClient
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
16
ErrorResponse, UsageInfo)
17
from vllm.entrypoints.openai.serving_engine import OpenAIServing
18
from vllm.logger import init_logger
19
-from vllm.outputs import EmbeddingRequestOutput
19
+from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
20
from vllm.utils import merge_async_iterators, random_uuid
21
22
logger = init_logger(__name__)
@@ -24,18 +24,28 @@ logger = init_logger(__name__)
24
TypeTokenIDs = List[int]
25
26
27
+def _get_embedding(
28
+ output: EmbeddingOutput,
29
+ encoding_format: Literal["float", "base64"],
30
+) -> Union[List[float], str]:
31
+ if encoding_format == "float":
32
+ return output.embedding
33
+ elif encoding_format == "base64":
34
+ embedding_bytes = np.array(output.embedding).tobytes()
35
+ return base64.b64encode(embedding_bytes).decode("utf-8")
36
+
37
+ assert_never(encoding_format)
38
+
39
+
40
def request_output_to_embedding_response(
41
final_res_batch: List[EmbeddingRequestOutput], request_id: str,
42
created_time: int, model_name: str,
30
- encoding_format: str) -> EmbeddingResponse:
43
+ encoding_format: Literal["float", "base64"]) -> EmbeddingResponse:
44
data: List[EmbeddingResponseData] = []
45
num_prompt_tokens = 0
46
for idx, final_res in enumerate(final_res_batch):
47
prompt_token_ids = final_res.prompt_token_ids
35
- embedding = final_res.outputs.embedding
36
- if encoding_format == "base64":
37
- embedding_bytes = np.array(embedding).tobytes()
38
- embedding = base64.b64encode(embedding_bytes).decode("utf-8")
48
+ embedding = _get_embedding(final_res.outputs, encoding_format)
49
embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
50
data.append(embedding_data)
51
@@ -76,8 +86,8 @@ class OpenAIServingEmbedding(OpenAIServing):
86
async def create_embedding(
87
self,
88
request: EmbeddingRequest,
79
- raw_request: Optional[Request] = None
80
- ) -> Union[ErrorResponse, EmbeddingResponse]:
89
+ raw_request: Optional[Request] = None,
90
+ ) -> Union[EmbeddingResponse, ErrorResponse]:
91
"""Completion API similar to OpenAI's API.
92
93
See https://platform.openai.com/docs/api-reference/embeddings/create
@@ -89,8 +99,7 @@ class OpenAIServingEmbedding(OpenAIServing):
99
if error_check_ret is not None:
100
return error_check_ret
101
92
- encoding_format = (request.encoding_format
93
- if request.encoding_format else "float")
102
+ encoding_format = request.encoding_format
103
if request.dimensions is not None:
104
return self.create_error_response(
105
"dimensions is currently not supported")
@@ -145,11 +154,10 @@ class OpenAIServingEmbedding(OpenAIServing):
154
# TODO: Use a vllm-specific Validation Error
155
return self.create_error_response(str(e))
156
148
- result_generator: AsyncIterator[Tuple[
149
- int, EmbeddingRequestOutput]] = merge_async_iterators(
150
- *generators,
151
- is_cancelled=raw_request.is_disconnected
152
- if raw_request else None)
157
+ result_generator = merge_async_iterators(
158
+ *generators,
159
+ is_cancelled=raw_request.is_disconnected if raw_request else None,
160
+ )
161
162
# Non-streaming response
163
final_res_batch: List[Optional[EmbeddingRequestOutput]]
@@ -175,7 +183,7 @@ class OpenAIServingEmbedding(OpenAIServing):
183
184
return response
185
178
- def _check_embedding_mode(self, embedding_mode: bool):
186
+ def _check_embedding_mode(self, embedding_mode: bool) -> bool:
187
if not embedding_mode:
188
logger.warning(
189
"embedding_mode is False. Embedding API will not work.")
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -31,7 +31,7 @@ from vllm.pooling_params import PoolingParams
31
from vllm.prompt_adapter.request import PromptAdapterRequest
32
from vllm.sampling_params import LogitsProcessor, SamplingParams
33
from vllm.sequence import Logprob
34
-from vllm.transformers_utils.tokenizer_group import AnyTokenizer
34
+from vllm.transformers_utils.tokenizer import AnyTokenizer
35
36
logger = init_logger(__name__)
37
vllm/sampling_params.py
+62
@@ -153,6 +153,68 @@ class SamplingParams(
153
output_text_buffer_length: int = 0
154
_all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
155
156
+ @staticmethod
157
+ def from_optional(
158
+ n: Optional[int] = 1,
159
+ best_of: Optional[int] = None,
160
+ presence_penalty: Optional[float] = 0.0,
161
+ frequency_penalty: Optional[float] = 0.0,
162
+ repetition_penalty: Optional[float] = 1.0,
163
+ temperature: Optional[float] = 1.0,
164
+ top_p: Optional[float] = 1.0,
165
+ top_k: int = -1,
166
+ min_p: float = 0.0,
167
+ seed: Optional[int] = None,
168
+ use_beam_search: bool = False,
169
+ length_penalty: float = 1.0,
170
+ early_stopping: Union[bool, str] = False,
171
+ stop: Optional[Union[str, List[str]]] = None,
172
+ stop_token_ids: Optional[List[int]] = None,
173
+ include_stop_str_in_output: bool = False,
174
+ ignore_eos: bool = False,
175
+ max_tokens: Optional[int] = 16,
176
+ min_tokens: int = 0,
177
+ logprobs: Optional[int] = None,
178
+ prompt_logprobs: Optional[int] = None,
179
+ detokenize: bool = True,
180
+ skip_special_tokens: bool = True,
181
+ spaces_between_special_tokens: bool = True,
182
+ logits_processors: Optional[List[LogitsProcessor]] = None,
183
+ truncate_prompt_tokens: Optional[Annotated[int,
184
+ msgspec.Meta(ge=1)]] = None,
185
+ ) -> "SamplingParams":
186
+ return SamplingParams(
187
+ n=1 if n is None else n,
188
+ best_of=best_of,
189
+ presence_penalty=0.0
190
+ if presence_penalty is None else presence_penalty,
191
+ frequency_penalty=0.0
192
+ if frequency_penalty is None else frequency_penalty,
193
+ repetition_penalty=1.0
194
+ if repetition_penalty is None else repetition_penalty,
195
+ temperature=1.0 if temperature is None else temperature,
196
+ top_p=1.0 if top_p is None else top_p,
197
+ top_k=top_k,
198
+ min_p=min_p,
199
+ seed=seed,
200
+ use_beam_search=use_beam_search,
201
+ length_penalty=length_penalty,
202
+ early_stopping=early_stopping,
203
+ stop=stop,
204
+ stop_token_ids=stop_token_ids,
205
+ include_stop_str_in_output=include_stop_str_in_output,
206
+ ignore_eos=ignore_eos,
207
+ max_tokens=max_tokens,
208
+ min_tokens=min_tokens,
209
+ logprobs=logprobs,
210
+ prompt_logprobs=prompt_logprobs,
211
+ detokenize=detokenize,
212
+ skip_special_tokens=skip_special_tokens,
213
+ spaces_between_special_tokens=spaces_between_special_tokens,
214
+ logits_processors=logits_processors,
215
+ truncate_prompt_tokens=truncate_prompt_tokens,
216
+ )
217
+
218
def __post_init__(self) -> None:
219
self.best_of = self.best_of or self.n
220
if 0 < self.temperature < _MAX_TEMP: