@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baaedfdb2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[mypy] Enable following imports for entrypoints (#7248)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Fei <dfdfcai4@gmail.com>

Cyrus Leung committed Aug 21, 2024 at 14:28 UTC baaedfdb2d3f1d70b7dbcde08b083abfe6017a92
26 files changed +480 -320
.github/workflows/mypy.yaml
-1
@@ -38,7 +38,6 @@ jobs:
38 mypy vllm/core --follow-imports skip
39 mypy vllm/distributed --follow-imports skip
40 mypy vllm/engine --follow-imports skip
41 - mypy vllm/entrypoints --follow-imports skip
41 mypy vllm/executor --follow-imports skip
42 mypy vllm/lora --follow-imports skip
43 mypy vllm/model_executor --follow-imports skip
docs/requirements-docs.txt
+1 -1
@@ -6,7 +6,7 @@ sphinx-argparse==0.4.0
6 msgspec
7
8 # packages to install to build the documentation
9 -pydantic
9 +pydantic >= 2.8
10 -f https://download.pytorch.org/whl/cpu
11 torch
12 py-cpuinfo
format.sh
-1
@@ -102,7 +102,6 @@ mypy vllm/attention --follow-imports skip
102 mypy vllm/core --follow-imports skip
103 mypy vllm/distributed --follow-imports skip
104 mypy vllm/engine --follow-imports skip
105 -mypy vllm/entrypoints --follow-imports skip
105 mypy vllm/executor --follow-imports skip
106 mypy vllm/lora --follow-imports skip
107 mypy vllm/model_executor --follow-imports skip
pyproject.toml
+1
@@ -56,6 +56,7 @@ files = [
56 "vllm/*.py",
57 "vllm/adapter_commons",
58 "vllm/assets",
59 + "vllm/entrypoints",
60 "vllm/inputs",
61 "vllm/logging",
62 "vllm/multimodal",
requirements-common.txt
+1 -1
@@ -11,7 +11,7 @@ fastapi
11 aiohttp
12 openai >= 1.0 # Ensure modern openai package (ensure types module present)
13 uvicorn[standard]
14 -pydantic >= 2.0 # Required for OpenAI server.
14 +pydantic >= 2.8 # Required for OpenAI server.
15 pillow # Required for image processing
16 prometheus_client >= 0.18.0
17 prometheus-fastapi-instrumentator >= 7.0.0
tests/entrypoints/openai/test_chat.py
+83 -1
@@ -1,7 +1,7 @@
1 # imports for guided decoding tests
2 import json
3 import re
4 -from typing import List
4 +from typing import Dict, List, Optional
5
6 import jsonschema
7 import openai # use the official client for correctness check
@@ -174,6 +174,88 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
174 assert message.content is not None and len(message.content) >= 0
175
176
177 +@pytest.mark.asyncio
178 +@pytest.mark.parametrize(
179 + "model_name, prompt_logprobs",
180 + [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
181 +)
182 +async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
183 + model_name: str,
184 + prompt_logprobs: Optional[int]):
185 + params: Dict = {
186 + "messages": [{
187 + "role": "system",
188 + "content": "You are a helpful assistant."
189 + }, {
190 + "role": "user",
191 + "content": "Who won the world series in 2020?"
192 + }, {
193 + "role":
194 + "assistant",
195 + "content":
196 + "The Los Angeles Dodgers won the World Series in 2020."
197 + }, {
198 + "role": "user",
199 + "content": "Where was it played?"
200 + }],
201 + "model":
202 + model_name
203 + }
204 +
205 + if prompt_logprobs is not None:
206 + params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
207 +
208 + if prompt_logprobs is not None and prompt_logprobs < 0:
209 + with pytest.raises(BadRequestError):
210 + await client.chat.completions.create(**params)
211 + else:
212 + completion = await client.chat.completions.create(**params)
213 + if prompt_logprobs is not None:
214 + assert completion.prompt_logprobs is not None
215 + assert len(completion.prompt_logprobs) > 0
216 + else:
217 + assert completion.prompt_logprobs is None
218 +
219 +
220 +@pytest.mark.asyncio
221 +@pytest.mark.parametrize(
222 + "model_name",
223 + [MODEL_NAME],
224 +)
225 +async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
226 + model_name: str):
227 + params: Dict = {
228 + "messages": [{
229 + "role": "system",
230 + "content": "You are a helpful assistant."
231 + }, {
232 + "role": "user",
233 + "content": "Who won the world series in 2020?"
234 + }, {
235 + "role":
236 + "assistant",
237 + "content":
238 + "The Los Angeles Dodgers won the World Series in 2020."
239 + }, {
240 + "role": "user",
241 + "content": "Where was it played?"
242 + }],
243 + "model":
244 + model_name,
245 + "extra_body": {
246 + "prompt_logprobs": 1
247 + }
248 + }
249 +
250 + completion_1 = await client.chat.completions.create(**params)
251 +
252 + params["extra_body"] = {"prompt_logprobs": 2}
253 + completion_2 = await client.chat.completions.create(**params)
254 +
255 + assert len(completion_1.prompt_logprobs[3]) == 1
256 + assert len(completion_2.prompt_logprobs[3]) == 2
257 +
258 +
259 @pytest.mark.asyncio
260 @pytest.mark.parametrize(
261 "model_name",
tests/entrypoints/openai/test_completion.py
+5 -96
@@ -3,7 +3,7 @@ import json
3 import re
4 import shutil
5 from tempfile import TemporaryDirectory
6 -from typing import Dict, List
6 +from typing import Dict, List, Optional
7
8 import jsonschema
9 import openai # use the official client for correctness check
@@ -268,92 +268,6 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
268 assert len(completion.choices[0].text) >= 0
269
270
271 -@pytest.mark.asyncio
272 -@pytest.mark.parametrize(
273 - "model_name, prompt_logprobs",
274 - [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
275 -)
276 -async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
277 - model_name: str, prompt_logprobs: int):
278 - params: Dict = {
279 - "messages": [{
280 - "role": "system",
281 - "content": "You are a helpful assistant."
282 - }, {
283 - "role": "user",
284 - "content": "Who won the world series in 2020?"
285 - }, {
286 - "role":
287 - "assistant",
288 - "content":
289 - "The Los Angeles Dodgers won the World Series in 2020."
290 - }, {
291 - "role": "user",
292 - "content": "Where was it played?"
293 - }],
294 - "model":
295 - model_name
296 - }
297 -
298 - if prompt_logprobs is not None:
299 - params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
300 -
301 - if prompt_logprobs and prompt_logprobs < 0:
302 - with pytest.raises(BadRequestError) as err_info:
303 - await client.chat.completions.create(**params)
304 - expected_err_string = (
305 - "Error code: 400 - {'object': 'error', 'message': "
306 - "'Prompt_logprobs set to invalid negative value: -1',"
307 - " 'type': 'BadRequestError', 'param': None, 'code': 400}")
308 - assert str(err_info.value) == expected_err_string
309 - else:
310 - completion = await client.chat.completions.create(**params)
311 - if prompt_logprobs and prompt_logprobs > 0:
312 - assert completion.prompt_logprobs is not None
313 - assert len(completion.prompt_logprobs) > 0
314 - else:
315 - assert completion.prompt_logprobs is None
316 -
317 -
318 -@pytest.mark.asyncio
319 -@pytest.mark.parametrize(
320 - "model_name",
321 - [MODEL_NAME],
322 -)
323 -async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
324 - model_name: str):
325 - params: Dict = {
326 - "messages": [{
327 - "role": "system",
328 - "content": "You are a helpful assistant."
329 - }, {
330 - "role": "user",
331 - "content": "Who won the world series in 2020?"
332 - }, {
333 - "role":
334 - "assistant",
335 - "content":
336 - "The Los Angeles Dodgers won the World Series in 2020."
337 - }, {
338 - "role": "user",
339 - "content": "Where was it played?"
340 - }],
341 - "model":
342 - model_name,
343 - "extra_body": {
344 - "prompt_logprobs": 1
345 - }
346 - }
347 -
348 - completion_1 = await client.chat.completions.create(**params)
349 -
350 - params["extra_body"] = {"prompt_logprobs": 2}
351 - completion_2 = await client.chat.completions.create(**params)
352 -
353 - assert len(completion_1.prompt_logprobs[3]) == 1
354 - assert len(completion_2.prompt_logprobs[3]) == 2
355 -
356 -
271 @pytest.mark.asyncio
272 @pytest.mark.parametrize("model_name, prompt_logprobs", [(MODEL_NAME, -1),
273 (MODEL_NAME, 0),
@@ -361,7 +275,7 @@ async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
275 (MODEL_NAME, None)])
276 async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
277 model_name: str,
364 - prompt_logprobs: int):
278 + prompt_logprobs: Optional[int]):
279 params: Dict = {
280 "prompt": ["A robot may not injure another robot", "My name is"],
281 "model": model_name,
@@ -369,17 +283,12 @@ async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
283 if prompt_logprobs is not None:
284 params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
285
372 - if prompt_logprobs and prompt_logprobs < 0:
373 - with pytest.raises(BadRequestError) as err_info:
286 + if prompt_logprobs is not None and prompt_logprobs < 0:
287 + with pytest.raises(BadRequestError):
288 await client.completions.create(**params)
375 - expected_err_string = (
376 - "Error code: 400 - {'object': 'error', 'message': "
377 - "'Prompt_logprobs set to invalid negative value: -1',"
378 - " 'type': 'BadRequestError', 'param': None, 'code': 400}")
379 - assert str(err_info.value) == expected_err_string
289 else:
290 completion = await client.completions.create(**params)
382 - if prompt_logprobs and prompt_logprobs > 0:
291 + if prompt_logprobs is not None:
292 assert completion.choices[0].prompt_logprobs is not None
293 assert len(completion.choices[0].prompt_logprobs) > 0
294
vllm/engine/async_llm_engine.py
+4 -4
@@ -6,7 +6,6 @@ from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6 Optional, Set, Tuple, Type, Union)
7
8 import torch
9 -from transformers import PreTrainedTokenizer
9 from typing_extensions import assert_never
10
11 import vllm.envs as envs
@@ -31,6 +30,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
30 from vllm.sampling_params import SamplingParams
31 from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
32 SequenceGroupMetadata)
33 +from vllm.transformers_utils.tokenizer import AnyTokenizer
34 from vllm.usage.usage_lib import UsageContext
35 from vllm.utils import print_warning_once
36
@@ -427,8 +427,8 @@ class _AsyncLLMEngine(LLMEngine):
427 lora_request: Optional[LoRARequest],
428 ) -> List[int]:
429 """Async version of :meth:`_tokenize_prompt`."""
430 - tokenizer = self.get_tokenizer_group("prompts must be None if "
431 - "skip_tokenizer_init is True")
430 + tokenizer = self.get_tokenizer_group(
431 + missing_msg="prompts must be None if skip_tokenizer_init is True")
432
433 return await tokenizer.encode_async(request_id=request_id,
434 prompt=prompt,
@@ -771,7 +771,7 @@ class AsyncLLMEngine:
771 async def get_tokenizer(
772 self,
773 lora_request: Optional[LoRARequest] = None,
774 - ) -> "PreTrainedTokenizer":
774 + ) -> AnyTokenizer:
775 if self.engine_use_ray:
776 return await self.engine.get_tokenizer.remote( # type: ignore
777 lora_request)
vllm/engine/llm_engine.py
+21 -10
@@ -3,9 +3,9 @@ from contextlib import contextmanager
3 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
4 Mapping, Optional)
5 from typing import Sequence as GenericSequence
6 -from typing import Set, Tuple, Type, TypeVar, Union
6 +from typing import Set, Tuple, Type, Union
7
8 -from typing_extensions import assert_never
8 +from typing_extensions import TypeVar, assert_never
9
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -43,8 +43,9 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
43 init_tracer)
44 from vllm.transformers_utils.config import try_get_generation_config
45 from vllm.transformers_utils.detokenizer import Detokenizer
46 +from vllm.transformers_utils.tokenizer import AnyTokenizer
47 from vllm.transformers_utils.tokenizer_group import (
47 - AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48 + BaseTokenizerGroup, init_tokenizer_from_configs)
49 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
50 usage_message)
51 from vllm.utils import Counter, Device
@@ -67,6 +68,7 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
68 return config.to_diff_dict()
69
70
71 +_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
72 _O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
73
74 PromptComponents = Tuple[Optional[str], List[int],
@@ -493,12 +495,21 @@ class LLMEngine:
495 "skip_tokenizer_init is True")
496
497 def get_tokenizer_group(
496 - self,
497 - fail_msg: str = MISSING_TOKENIZER_GROUP_MSG) -> BaseTokenizerGroup:
498 - if self.tokenizer is None:
499 - raise ValueError(fail_msg)
498 + self,
499 + group_type: Type[_G] = BaseTokenizerGroup,
500 + *,
501 + missing_msg: str = MISSING_TOKENIZER_GROUP_MSG,
502 + ) -> _G:
503 + tokenizer_group = self.tokenizer
504 +
505 + if tokenizer_group is None:
506 + raise ValueError(missing_msg)
507 + if not isinstance(tokenizer_group, group_type):
508 + raise TypeError("Invalid type of tokenizer group. "
509 + f"Expected type: {group_type}, but "
510 + f"found type: {type(tokenizer_group)}")
511
501 - return self.tokenizer
512 + return tokenizer_group
513
514 def get_tokenizer(
515 self,
@@ -693,8 +704,8 @@ class LLMEngine:
704 * prompt token ids
705 '''
706
696 - tokenizer = self.get_tokenizer_group("prompts must be None if "
697 - "skip_tokenizer_init is True")
707 + tokenizer = self.get_tokenizer_group(
708 + missing_msg="prompts must be None if skip_tokenizer_init is True")
709
710 return tokenizer.encode(request_id=request_id,
711 prompt=prompt,
vllm/engine/output_processor/interfaces.py
+2 -3
@@ -1,13 +1,12 @@
1 from abc import ABC, abstractmethod
2 from typing import Callable, List
3
4 -from transformers import PreTrainedTokenizer
5 -
4 from vllm.config import SchedulerConfig
5 from vllm.core.scheduler import Scheduler
6 from vllm.engine.output_processor.stop_checker import StopChecker
7 from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
8 from vllm.transformers_utils.detokenizer import Detokenizer
9 +from vllm.transformers_utils.tokenizer import AnyTokenizer
10 from vllm.utils import Counter
11
12
@@ -29,7 +28,7 @@ class SequenceGroupOutputProcessor(ABC):
28 detokenizer: Detokenizer,
29 scheduler: List[Scheduler],
30 seq_counter: Counter,
32 - get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
31 + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
32 stop_checker: "StopChecker",
33 ):
34 """Create an output processor.
vllm/engine/output_processor/multi_step.py
+2 -3
@@ -1,8 +1,6 @@
1 import functools
2 from typing import Callable, List
3
4 -from transformers import PreTrainedTokenizer
5 -
4 from vllm.core.scheduler import Scheduler
5 from vllm.engine.output_processor.interfaces import (
6 SequenceGroupOutputProcessor)
@@ -12,6 +10,7 @@ from vllm.sampling_params import SamplingParams
10 from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11 SequenceOutput, SequenceStatus)
12 from vllm.transformers_utils.detokenizer import Detokenizer
13 +from vllm.transformers_utils.tokenizer import AnyTokenizer
14 from vllm.utils import Counter
15
16 logger = init_logger(__name__)
@@ -36,7 +35,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
35 detokenizer: Detokenizer,
36 scheduler: List[Scheduler],
37 seq_counter: Counter,
39 - get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
38 + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
39 stop_checker: StopChecker,
40 ):
41 self.detokenizer = detokenizer
vllm/engine/output_processor/stop_checker.py
+2 -4
@@ -1,10 +1,9 @@
1 from typing import Callable, Optional
2
3 -from transformers import PreTrainedTokenizer
4 -
3 from vllm.lora.request import LoRARequest
4 from vllm.sampling_params import SamplingParams
5 from vllm.sequence import Sequence, SequenceStatus
6 +from vllm.transformers_utils.tokenizer import AnyTokenizer
7
8
9 class StopChecker:
@@ -15,8 +14,7 @@ class StopChecker:
14 """
15
16 def __init__(self, max_model_len: int,
18 - get_tokenizer_for_seq: Callable[[Sequence],
19 - PreTrainedTokenizer]):
17 + get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer]):
18 # Do not use it directly, but use `self._get_max_model_len`.
19 self._max_model_len = max_model_len
20 self.get_tokenizer_for_seq = get_tokenizer_for_seq
vllm/engine/protocol.py
+11 -6
@@ -1,8 +1,6 @@
1 from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2 runtime_checkable)
3
4 -from transformers import PreTrainedTokenizer
5 -
4 from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 from vllm.inputs.data import PromptInputs
@@ -12,6 +10,7 @@ from vllm.pooling_params import PoolingParams
10 from vllm.prompt_adapter.request import PromptAdapterRequest
11 from vllm.sampling_params import SamplingParams
12 from vllm.sequence import SamplerOutput
13 +from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
16 @runtime_checkable
@@ -40,6 +39,7 @@ class AsyncEngineClient(Protocol):
39 prompt_adapter_request: Optional[PromptAdapterRequest] = None
40 ) -> AsyncGenerator[RequestOutput, None]:
41 """Generates outputs for a request"""
42 + ...
43
44 def encode(
45 self,
@@ -50,6 +50,7 @@ class AsyncEngineClient(Protocol):
50 trace_headers: Optional[Mapping[str, str]] = None,
51 ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
52 """Generate outputs for a request from an embedding model."""
53 + ...
54
55 async def abort(self, request_id: str) -> None:
56 """Abort a request.
@@ -60,25 +61,29 @@ class AsyncEngineClient(Protocol):
61
62 async def get_model_config(self) -> ModelConfig:
63 """Get the model configuration of the vLLM engine."""
64 + ...
65
66 async def get_decoding_config(self) -> DecodingConfig:
67 + ...
68 """Get the decoding configuration of the vLLM engine."""
69
70 async def get_tokenizer(
71 self,
72 lora_request: Optional[LoRARequest] = None,
70 - ) -> PreTrainedTokenizer:
71 - """Get the appropriate Tokenizer for the request"""
73 + ) -> AnyTokenizer:
74 + """Get the appropriate tokenizer for the request"""
75 + ...
76
77 async def is_tracing_enabled(self) -> bool:
74 - pass
78 + ...
79
80 async def do_log_stats(
81 self,
82 scheduler_outputs: Optional[SchedulerOutputs] = None,
83 model_output: Optional[List[SamplerOutput]] = None,
84 ) -> None:
81 - pass
85 + ...
86
87 async def check_health(self) -> None:
88 """Raise if unhealthy"""
89 + ...
vllm/entrypoints/api_server.py
+3
@@ -61,6 +61,7 @@ async def generate(request: Request) -> Response:
61 async def stream_results() -> AsyncGenerator[bytes, None]:
62 async for request_output in results_generator:
63 prompt = request_output.prompt
64 + assert prompt is not None
65 text_outputs = [
66 prompt + output.text for output in request_output.outputs
67 ]
@@ -80,6 +81,7 @@ async def generate(request: Request) -> Response:
81
82 assert final_output is not None
83 prompt = final_output.prompt
84 + assert prompt is not None
85 text_outputs = [prompt + output.text for output in final_output.outputs]
86 ret = {"text": text_outputs}
87 return JSONResponse(ret)
@@ -115,6 +117,7 @@ async def run_server(args: Namespace,
117 logger.info("args: %s", args)
118
119 app = await init_app(args, llm_engine)
120 + assert engine is not None
121
122 shutdown_task = await serve_http(
123 app,
vllm/entrypoints/chat_utils.py
+24 -18
@@ -3,7 +3,7 @@ from dataclasses import dataclass
3 from functools import lru_cache
4 from pathlib import Path
5 from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6 - Union, cast)
6 + Union)
7
8 # yapf conflicts with isort for this block
9 # yapf: disable
@@ -15,9 +15,8 @@ from openai.types.chat import (
15 ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
16 # yapf: enable
17 # pydantic needs the TypedDict from typing_extensions
18 -from pydantic import ConfigDict
19 -from transformers import PreTrainedTokenizer
20 -from typing_extensions import Required, TypedDict
18 +from pydantic import ConfigDict, TypeAdapter
19 +from typing_extensions import Required, TypeAlias, TypedDict
20
21 from vllm.config import ModelConfig
22 from vllm.logger import init_logger
@@ -50,9 +49,9 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
49 """The type of the content part."""
50
51
53 -ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
54 - ChatCompletionContentPartAudioParam,
55 - CustomChatCompletionContentPartParam]
52 +ChatCompletionContentPartParam: TypeAlias = Union[
53 + OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
54 + CustomChatCompletionContentPartParam, ]
55
56
57 class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -114,7 +113,7 @@ def load_chat_template(
113
114
115 @lru_cache(maxsize=None)
117 -def _mm_token_str(model_config: ModelConfig, tokenizer: PreTrainedTokenizer,
116 +def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117 modality: Literal["image", "audio"]) -> Optional[str]:
118 # TODO: Let user specify how to insert image tokens into prompt
119 # (similar to chat template)
@@ -151,11 +150,16 @@ def _get_full_multimodal_text_prompt(placeholder_token_str: str,
150 return f"{placeholder_token_str}\n{text_prompt}"
151
152
153 +_TextParser = TypeAdapter(ChatCompletionContentPartTextParam)
154 +_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam)
155 +_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
156 +
157 +
158 def _parse_chat_message_content_parts(
159 role: str,
160 parts: Iterable[ChatCompletionContentPartParam],
161 model_config: ModelConfig,
158 - tokenizer: PreTrainedTokenizer,
162 + tokenizer: AnyTokenizer,
163 ) -> ChatMessageParseResult:
164 texts: List[str] = []
165 mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -164,7 +168,7 @@ def _parse_chat_message_content_parts(
168 for part in parts:
169 part_type = part["type"]
170 if part_type == "text":
167 - text = cast(ChatCompletionContentPartTextParam, part)["text"]
171 + text = _TextParser.validate_python(part)["text"]
172 texts.append(text)
173 elif part_type == "image_url":
174 modality = "image"
@@ -172,8 +176,7 @@ def _parse_chat_message_content_parts(
176 raise NotImplementedError(
177 "Multiple multimodal inputs is currently not supported.")
178
175 - image_url = cast(ChatCompletionContentPartImageParam,
176 - part)["image_url"]
179 + image_url = _ImageParser.validate_python(part)["image_url"]
180
181 if image_url.get("detail", "auto") != "auto":
182 logger.warning(
@@ -188,8 +191,7 @@ def _parse_chat_message_content_parts(
191 raise NotImplementedError(
192 "Multiple multimodal inputs is currently not supported.")
193
191 - audio_url = cast(ChatCompletionContentPartAudioParam,
192 - part)["audio_url"]
194 + audio_url = _AudioParser.validate_python(part)["audio_url"]
195 audio_future = async_get_and_parse_audio(audio_url["url"])
196 mm_futures.append(audio_future)
197 else:
@@ -219,7 +221,7 @@ def _parse_chat_message_content_parts(
221 def _parse_chat_message_content(
222 message: ChatCompletionMessageParam,
223 model_config: ModelConfig,
222 - tokenizer: PreTrainedTokenizer,
224 + tokenizer: AnyTokenizer,
225 ) -> ChatMessageParseResult:
226 role = message["role"]
227 content = message.get("content")
@@ -230,14 +232,18 @@ def _parse_chat_message_content(
232 messages = [ConversationMessage(role=role, content=content)]
233 return ChatMessageParseResult(messages=messages, mm_futures=[])
234
233 - return _parse_chat_message_content_parts(role, content, model_config,
234 - tokenizer)
235 + return _parse_chat_message_content_parts(
236 + role,
237 + content, # type: ignore
238 + model_config,
239 + tokenizer,
240 + )
241
242
243 def parse_chat_messages(
244 messages: List[ChatCompletionMessageParam],
245 model_config: ModelConfig,
240 - tokenizer: PreTrainedTokenizer,
246 + tokenizer: AnyTokenizer,
247 ) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
248 conversation: List[ConversationMessage] = []
249 mm_futures: List[Awaitable[MultiModalDataDict]] = []
vllm/entrypoints/llm.py
+15 -15
@@ -1,8 +1,7 @@
1 from contextlib import contextmanager
2 from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
3
4 -from tqdm.auto import tqdm
5 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4 +from tqdm import tqdm
5
6 from vllm.engine.arg_utils import EngineArgs
7 from vllm.engine.llm_engine import LLMEngine
@@ -20,7 +19,9 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
19 from vllm.pooling_params import PoolingParams
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sampling_params import SamplingParams
23 -from vllm.transformers_utils.tokenizer import get_cached_tokenizer
22 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23 + get_cached_tokenizer)
24 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
25 from vllm.usage.usage_lib import UsageContext
26 from vllm.utils import Counter, deprecate_kwargs
27
@@ -122,7 +123,7 @@ class LLM:
123 tokenizer_revision: Optional[str] = None,
124 seed: int = 0,
125 gpu_memory_utilization: float = 0.9,
125 - swap_space: int = 4,
126 + swap_space: float = 4,
127 cpu_offload_gb: float = 0,
128 enforce_eager: Optional[bool] = None,
129 max_context_len_to_capture: Optional[int] = None,
@@ -175,22 +176,19 @@ class LLM:
176 engine_args, usage_context=UsageContext.LLM_CLASS)
177 self.request_counter = Counter()
178
178 - def get_tokenizer(
179 - self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
180 - return self.llm_engine.tokenizer.tokenizer
179 + def get_tokenizer(self) -> AnyTokenizer:
180 + return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
181 +
182 + def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
183 + tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
184
182 - def set_tokenizer(
183 - self,
184 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
185 - ) -> None:
185 # While CachedTokenizer is dynamic, have no choice but
186 # compare class name. Misjudgment will arise from
187 # user-defined tokenizer started with 'Cached'
188 if tokenizer.__class__.__name__.startswith("Cached"):
190 - self.llm_engine.tokenizer.tokenizer = tokenizer
189 + tokenizer_group.tokenizer = tokenizer
190 else:
192 - self.llm_engine.tokenizer.tokenizer = get_cached_tokenizer(
193 - tokenizer)
191 + tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
192
193 @overload # LEGACY: single (prompt + optional token ids)
194 def generate(
@@ -578,6 +576,8 @@ class LLM:
576
577 inputs: List[PromptInputs] = []
578 for i in range(num_requests):
579 + item: PromptInputs
580 +
581 if prompts is not None:
582 item = TextPrompt(prompt=prompts[i])
583 elif prompt_token_ids is not None:
@@ -635,7 +635,7 @@ class LLM:
635 self,
636 inputs: PromptInputs,
637 params: Union[SamplingParams, PoolingParams],
638 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
638 + lora_request: Optional[LoRARequest] = None,
639 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
640 ) -> None:
641 request_id = str(next(self.request_counter))
vllm/entrypoints/openai/api_server.py
+28 -20
@@ -15,6 +15,7 @@ from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 from fastapi.responses import JSONResponse, Response, StreamingResponse
17 from starlette.routing import Mount
18 +from typing_extensions import assert_never
19
20 import vllm.envs as envs
21 from vllm.config import ModelConfig
@@ -29,14 +30,16 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser
30 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
31 ChatCompletionResponse,
32 CompletionRequest,
33 + CompletionResponse,
34 DetokenizeRequest,
35 DetokenizeResponse,
34 - EmbeddingRequest, ErrorResponse,
36 + EmbeddingRequest,
37 + EmbeddingResponse, ErrorResponse,
38 TokenizeRequest,
39 TokenizeResponse)
40 +# yapf: enable
41 from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
42 from vllm.entrypoints.openai.rpc.server import run_rpc_server
39 -# yapf: enable
43 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
44 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
45 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
@@ -90,7 +93,8 @@ async def lifespan(app: FastAPI):
93
94
95 @asynccontextmanager
93 -async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
96 +async def build_async_engine_client(
97 + args: Namespace) -> AsyncIterator[AsyncEngineClient]:
98 # Context manager to handle async_engine_client lifecycle
99 # Ensures everything is shutdown and cleaned up on error/exit
100 global engine_args
@@ -142,12 +146,15 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
146 logger.info("Started engine process with PID %d",
147 rpc_server_process.pid)
148 # Build RPCClient, which conforms to AsyncEngineClient Protocol.
145 - async_engine_client = AsyncEngineRPCClient(rpc_path)
149 + # NOTE: Actually, this is not true yet. We still need to support
150 + # embedding models via RPC (see TODO above)
151 + rpc_client = AsyncEngineRPCClient(rpc_path)
152 + async_engine_client = rpc_client # type: ignore
153
154 try:
155 while True:
156 try:
150 - await async_engine_client.setup()
157 + await rpc_client.setup()
158 break
159 except TimeoutError as e:
160 if not rpc_server_process.is_alive():
@@ -161,7 +168,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
168 rpc_server_process.terminate()
169
170 # Close all open connections to the backend
164 - async_engine_client.close()
171 + rpc_client.close()
172
173 # Wait for server process to join
174 rpc_server_process.join()
@@ -216,10 +223,11 @@ async def tokenize(request: TokenizeRequest):
223 if isinstance(generator, ErrorResponse):
224 return JSONResponse(content=generator.model_dump(),
225 status_code=generator.code)
219 - else:
220 - assert isinstance(generator, TokenizeResponse)
226 + elif isinstance(generator, TokenizeResponse):
227 return JSONResponse(content=generator.model_dump())
228
229 + assert_never(generator)
230 +
231
232 @router.post("/detokenize")
233 async def detokenize(request: DetokenizeRequest):
@@ -227,10 +235,11 @@ async def detokenize(request: DetokenizeRequest):
235 if isinstance(generator, ErrorResponse):
236 return JSONResponse(content=generator.model_dump(),
237 status_code=generator.code)
230 - else:
231 - assert isinstance(generator, DetokenizeResponse)
238 + elif isinstance(generator, DetokenizeResponse):
239 return JSONResponse(content=generator.model_dump())
240
241 + assert_never(generator)
242 +
243
244 @router.get("/v1/models")
245 async def show_available_models():
@@ -252,13 +261,11 @@ async def create_chat_completion(request: ChatCompletionRequest,
261 if isinstance(generator, ErrorResponse):
262 return JSONResponse(content=generator.model_dump(),
263 status_code=generator.code)
255 - if request.stream:
256 - return StreamingResponse(content=generator,
257 - media_type="text/event-stream")
258 - else:
259 - assert isinstance(generator, ChatCompletionResponse)
264 + elif isinstance(generator, ChatCompletionResponse):
265 return JSONResponse(content=generator.model_dump())
266
267 + return StreamingResponse(content=generator, media_type="text/event-stream")
268 +
269
270 @router.post("/v1/completions")
271 async def create_completion(request: CompletionRequest, raw_request: Request):
@@ -267,12 +274,11 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
274 if isinstance(generator, ErrorResponse):
275 return JSONResponse(content=generator.model_dump(),
276 status_code=generator.code)
270 - if request.stream:
271 - return StreamingResponse(content=generator,
272 - media_type="text/event-stream")
273 - else:
277 + elif isinstance(generator, CompletionResponse):
278 return JSONResponse(content=generator.model_dump())
279
280 + return StreamingResponse(content=generator, media_type="text/event-stream")
281 +
282
283 @router.post("/v1/embeddings")
284 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
@@ -281,9 +287,11 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
287 if isinstance(generator, ErrorResponse):
288 return JSONResponse(content=generator.model_dump(),
289 status_code=generator.code)
284 - else:
290 + elif isinstance(generator, EmbeddingResponse):
291 return JSONResponse(content=generator.model_dump())
292
293 + assert_never(generator)
294 +
295
296 def build_app(args: Namespace) -> FastAPI:
297 app = FastAPI(lifespan=lifespan)
vllm/entrypoints/openai/cli_args.py
+27 -4
@@ -7,6 +7,7 @@ purposes.
7 import argparse
8 import json
9 import ssl
10 +from typing import List, Optional, Sequence, Union
11
12 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
@@ -16,8 +17,19 @@ from vllm.utils import FlexibleArgumentParser
17
18 class LoRAParserAction(argparse.Action):
19
19 - def __call__(self, parser, namespace, values, option_string=None):
20 - lora_list = []
20 + def __call__(
21 + self,
22 + parser: argparse.ArgumentParser,
23 + namespace: argparse.Namespace,
24 + values: Optional[Union[str, Sequence[str]]],
25 + option_string: Optional[str] = None,
26 + ):
27 + if values is None:
28 + values = []
29 + if isinstance(values, str):
30 + raise TypeError("Expected values to be a list")
31 +
32 + lora_list: List[LoRAModulePath] = []
33 for item in values:
34 name, path = item.split('=')
35 lora_list.append(LoRAModulePath(name, path))
@@ -26,8 +38,19 @@ class LoRAParserAction(argparse.Action):
38
39 class PromptAdapterParserAction(argparse.Action):
40
29 - def __call__(self, parser, namespace, values, option_string=None):
30 - adapter_list = []
41 + def __call__(
42 + self,
43 + parser: argparse.ArgumentParser,
44 + namespace: argparse.Namespace,
45 + values: Optional[Union[str, Sequence[str]]],
46 + option_string: Optional[str] = None,
47 + ):
48 + if values is None:
49 + values = []
50 + if isinstance(values, str):
51 + raise TypeError("Expected values to be a list")
52 +
53 + adapter_list: List[PromptAdapterPath] = []
54 for item in values:
55 name, path = item.split('=')
56 adapter_list.append(PromptAdapterPath(name, path))
vllm/entrypoints/openai/logits_processors.py
+6 -5
@@ -2,9 +2,9 @@ from functools import lru_cache, partial
2 from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3
4 import torch
5 -from transformers import PreTrainedTokenizer
5
6 from vllm.sampling_params import LogitsProcessor
7 +from vllm.transformers_utils.tokenizer import AnyTokenizer
8
9
10 class AllowedTokenIdsLogitsProcessor:
@@ -51,10 +51,11 @@ def logit_bias_logits_processor(
51
52
53 def get_logits_processors(
54 - logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
55 - allowed_token_ids: Optional[List[int]],
56 - tokenizer: PreTrainedTokenizer) -> List[LogitsProcessor]:
57 - logits_processors = []
54 + logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
55 + allowed_token_ids: Optional[List[int]],
56 + tokenizer: AnyTokenizer,
57 +) -> List[LogitsProcessor]:
58 + logits_processors: List[LogitsProcessor] = []
59 if logit_bias:
60 try:
61 # Convert token_id to integer
vllm/entrypoints/openai/protocol.py
+58 -33
@@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optional, Union
6
7 import torch
8 from pydantic import BaseModel, ConfigDict, Field, model_validator
9 -from transformers import PreTrainedTokenizer
9 from typing_extensions import Annotated
10
11 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -14,11 +13,13 @@ from vllm.entrypoints.openai.logits_processors import get_logits_processors
13 from vllm.pooling_params import PoolingParams
14 from vllm.sampling_params import LogitsProcessor, SamplingParams
15 from vllm.sequence import Logprob
16 +from vllm.transformers_utils.tokenizer import AnyTokenizer
17 from vllm.utils import random_uuid
18
19 # torch is mocked during docs generation,
20 # so we have to provide the values as literals
21 _MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
22 +_LONG_INFO: Union["torch.iinfo", Namespace]
23
24 try:
25 from sphinx.ext.autodoc.mock import _MockModule
@@ -235,13 +236,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
236 # doc: end-chat-completion-extra-params
237
238 def to_sampling_params(
238 - self, tokenizer: PreTrainedTokenizer,
239 + self, tokenizer: AnyTokenizer,
240 guided_decode_logits_processor: Optional[LogitsProcessor],
241 default_max_tokens: int) -> SamplingParams:
242 max_tokens = self.max_tokens
243 if max_tokens is None:
244 max_tokens = default_max_tokens
245
246 + prompt_logprobs = self.prompt_logprobs
247 + if prompt_logprobs is None and self.echo:
248 + prompt_logprobs = self.top_logprobs
249 +
250 # We now allow logprobs being true without top_logrobs.
251 logits_processors = get_logits_processors(
252 logit_bias=self.logit_bias,
@@ -251,7 +256,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
256 if guided_decode_logits_processor:
257 logits_processors.append(guided_decode_logits_processor)
258
254 - return SamplingParams(
259 + return SamplingParams.from_optional(
260 n=self.n,
261 best_of=self.best_of,
262 presence_penalty=self.presence_penalty,
@@ -265,8 +270,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
270 stop=self.stop,
271 stop_token_ids=self.stop_token_ids,
272 logprobs=self.top_logprobs if self.logprobs else None,
268 - prompt_logprobs=self.prompt_logprobs if self.prompt_logprobs else
269 - (self.top_logprobs if self.echo else None),
273 + prompt_logprobs=prompt_logprobs,
274 ignore_eos=self.ignore_eos,
275 max_tokens=max_tokens,
276 min_tokens=self.min_tokens,
@@ -280,14 +284,36 @@ class ChatCompletionRequest(OpenAIBaseModel):
284 truncate_prompt_tokens=self.truncate_prompt_tokens,
285 )
286
283 - @model_validator(mode='before')
287 + @model_validator(mode="before")
288 @classmethod
285 - def validate_stream_options(cls, values):
286 - if (values.get('stream_options') is not None
287 - and not values.get('stream')):
289 + def validate_stream_options(cls, data):
290 + if data.get("stream_options") and not data.get("stream"):
291 raise ValueError(
289 - "stream_options can only be set if stream is true")
290 - return values
292 + "Stream options can only be defined when `stream=True`.")
293 +
294 + return data
295 +
296 + @model_validator(mode="before")
297 + @classmethod
298 + def check_logprobs(cls, data):
299 + if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
300 + if data.get("stream") and prompt_logprobs > 0:
301 + raise ValueError(
302 + "`prompt_logprobs` are not available when `stream=True`.")
303 +
304 + if prompt_logprobs < 0:
305 + raise ValueError("`prompt_logprobs` must be a positive value.")
306 +
307 + if (top_logprobs := data.get("top_logprobs")) is not None:
308 + if top_logprobs < 0:
309 + raise ValueError("`top_logprobs` must be a positive value.")
310 +
311 + if not data.get("logprobs"):
312 + raise ValueError(
313 + "when using `top_logprobs`, `logprobs` must be set to true."
314 + )
315 +
316 + return data
317
318 @model_validator(mode="before")
319 @classmethod
@@ -320,19 +346,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
346 "When using `tool_choice`, `tools` must be set.")
347 return data
348
323 - @model_validator(mode="before")
324 - @classmethod
325 - def check_logprobs(cls, data):
326 - if "top_logprobs" in data and data["top_logprobs"] is not None:
327 - if "logprobs" not in data or data["logprobs"] is False:
328 - raise ValueError(
329 - "when using `top_logprobs`, `logprobs` must be set to true."
330 - )
331 - elif data["top_logprobs"] < 0:
332 - raise ValueError(
333 - "`top_logprobs` must be a value a positive value.")
334 - return data
335 -
349
350 class CompletionRequest(OpenAIBaseModel):
351 # Ordered by official OpenAI API documentation
@@ -422,13 +435,17 @@ class CompletionRequest(OpenAIBaseModel):
435 # doc: end-completion-extra-params
436
437 def to_sampling_params(
425 - self, tokenizer: PreTrainedTokenizer,
438 + self, tokenizer: AnyTokenizer,
439 guided_decode_logits_processor: Optional[LogitsProcessor],
440 default_max_tokens: int) -> SamplingParams:
441 max_tokens = self.max_tokens
442 if max_tokens is None:
443 max_tokens = default_max_tokens
444
445 + prompt_logprobs = self.prompt_logprobs
446 + if prompt_logprobs is None and self.echo:
447 + prompt_logprobs = self.logprobs
448 +
449 echo_without_generation = self.echo and self.max_tokens == 0
450
451 logits_processors = get_logits_processors(
@@ -439,7 +456,7 @@ class CompletionRequest(OpenAIBaseModel):
456 if guided_decode_logits_processor:
457 logits_processors.append(guided_decode_logits_processor)
458
442 - return SamplingParams(
459 + return SamplingParams.from_optional(
460 n=self.n,
461 best_of=self.best_of,
462 presence_penalty=self.presence_penalty,
@@ -458,8 +475,7 @@ class CompletionRequest(OpenAIBaseModel):
475 min_tokens=self.min_tokens,
476 use_beam_search=self.use_beam_search,
477 early_stopping=self.early_stopping,
461 - prompt_logprobs=self.prompt_logprobs
462 - if self.prompt_logprobs else self.logprobs if self.echo else None,
478 + prompt_logprobs=prompt_logprobs,
479 skip_special_tokens=self.skip_special_tokens,
480 spaces_between_special_tokens=self.spaces_between_special_tokens,
481 include_stop_str_in_output=self.include_stop_str_in_output,
@@ -485,9 +501,17 @@ class CompletionRequest(OpenAIBaseModel):
501 @model_validator(mode="before")
502 @classmethod
503 def check_logprobs(cls, data):
488 - if "logprobs" in data and data[
489 - "logprobs"] is not None and not data["logprobs"] >= 0:
490 - raise ValueError("if passed, `logprobs` must be a positive value.")
504 + if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
505 + if data.get("stream") and prompt_logprobs > 0:
506 + raise ValueError(
507 + "`prompt_logprobs` are not available when `stream=True`.")
508 +
509 + if prompt_logprobs < 0:
510 + raise ValueError("`prompt_logprobs` must be a positive value.")
511 +
512 + if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
513 + raise ValueError("`logprobs` must be a positive value.")
514 +
515 return data
516
517 @model_validator(mode="before")
@@ -495,7 +519,8 @@ class CompletionRequest(OpenAIBaseModel):
519 def validate_stream_options(cls, data):
520 if data.get("stream_options") and not data.get("stream"):
521 raise ValueError(
498 - "Stream options can only be defined when stream is true.")
522 + "Stream options can only be defined when `stream=True`.")
523 +
524 return data
525
526
@@ -504,7 +529,7 @@ class EmbeddingRequest(OpenAIBaseModel):
529 # https://platform.openai.com/docs/api-reference/embeddings
530 model: str
531 input: Union[List[int], List[List[int]], str, List[str]]
507 - encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
532 + encoding_format: Literal["float", "base64"] = "float"
533 dimensions: Optional[int] = None
534 user: Optional[str] = None
535
vllm/entrypoints/openai/rpc/server.py
+3 -3
@@ -23,8 +23,8 @@ class AsyncEngineRPCServer:
23 def __init__(self, async_engine_args: AsyncEngineArgs,
24 usage_context: UsageContext, rpc_path: str):
25 # Initialize engine first.
26 - self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
27 - usage_context)
26 + self.engine = AsyncLLMEngine.from_engine_args(
27 + async_engine_args, usage_context=usage_context)
28
29 # Initialize context.
30 self.context = zmq.asyncio.Context()
@@ -39,7 +39,7 @@ class AsyncEngineRPCServer:
39 self.context.destroy()
40 self.engine.shutdown_background_loop()
41 # Clear the engine reference so that it can be GC'ed.
42 - self.engine = None
42 + del self.engine
43
44 async def get_model_config(self, identity):
45 """Send the ModelConfig"""
vllm/entrypoints/openai/serving_chat.py
+33 -36
@@ -1,11 +1,10 @@
1 import asyncio
2 import time
3 -from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
3 +from typing import AsyncGenerator, AsyncIterator, Dict, Final, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
7 from fastapi import Request
8 -from transformers import PreTrainedTokenizer
8
9 from vllm.config import ModelConfig
10 from vllm.engine.protocol import AsyncEngineClient
@@ -24,13 +23,14 @@ from vllm.entrypoints.openai.protocol import (
23 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24 OpenAIServing,
25 PromptAdapterPath)
27 -from vllm.inputs import PromptInputs
26 +from vllm.inputs import TokensPrompt
27 from vllm.logger import init_logger
28 from vllm.multimodal import MultiModalDataDict
29 from vllm.outputs import RequestOutput
30 from vllm.sequence import Logprob
31 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
32 log_tracing_disabled_warning)
33 +from vllm.transformers_utils.tokenizer import AnyTokenizer
34 from vllm.utils import iterate_with_cancellation, random_uuid
35
36 logger = init_logger(__name__)
@@ -67,9 +67,9 @@ class OpenAIServingChat(OpenAIServing):
67 async def create_chat_completion(
68 self,
69 request: ChatCompletionRequest,
70 - raw_request: Optional[Request] = None
71 - ) -> Union[ErrorResponse, AsyncGenerator[str, None],
72 - ChatCompletionResponse]:
70 + raw_request: Optional[Request] = None,
71 + ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
72 + ErrorResponse]:
73 """Completion API similar to OpenAI's API.
74
75 See https://platform.openai.com/docs/api-reference/chat/create
@@ -83,16 +83,6 @@ class OpenAIServingChat(OpenAIServing):
83 if error_check_ret is not None:
84 return error_check_ret
85
86 - if request.prompt_logprobs is not None:
87 - if request.stream and request.prompt_logprobs > 0:
88 - return self.create_error_response(
89 - "Prompt_logprobs are not available when stream is enabled")
90 -
91 - if request.prompt_logprobs < 0:
92 - return self.create_error_response(
93 - f"Prompt_logprobs set to invalid "
94 - f"negative value: {request.prompt_logprobs}")
95 -
86 try:
87 (
88 lora_request,
@@ -160,9 +150,8 @@ class OpenAIServingChat(OpenAIServing):
150 lora_request=lora_request,
151 prompt_adapter_request=prompt_adapter_request)
152
163 - engine_inputs: PromptInputs = {
164 - "prompt_token_ids": prompt_inputs["prompt_token_ids"],
165 - }
153 + engine_inputs = TokensPrompt(
154 + prompt_token_ids=prompt_inputs["prompt_token_ids"])
155 if mm_data is not None:
156 engine_inputs["multi_modal_data"] = mm_data
157
@@ -214,11 +203,11 @@ class OpenAIServingChat(OpenAIServing):
203 result_generator: AsyncIterator[RequestOutput],
204 request_id: str,
205 conversation: List[ConversationMessage],
217 - tokenizer: PreTrainedTokenizer,
206 + tokenizer: AnyTokenizer,
207 ) -> AsyncGenerator[str, None]:
208 model_name = self.served_model_names[0]
209 created_time = int(time.time())
221 - chunk_object_type = "chat.completion.chunk"
210 + chunk_object_type: Final = "chat.completion.chunk"
211 first_iteration = True
212
213 # Send response for each token for each request.n (index)
@@ -438,7 +427,7 @@ class OpenAIServingChat(OpenAIServing):
427 result_generator: AsyncIterator[RequestOutput],
428 request_id: str,
429 conversation: List[ConversationMessage],
441 - tokenizer: PreTrainedTokenizer,
430 + tokenizer: AnyTokenizer,
431 ) -> Union[ErrorResponse, ChatCompletionResponse]:
432
433 model_name = self.served_model_names[0]
@@ -523,7 +512,7 @@ class OpenAIServingChat(OpenAIServing):
512
513 def _get_top_logprobs(
514 self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
526 - tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
515 + tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
516 return [
517 ChatCompletionLogProb(token=(token := self._get_decoded_token(
518 p[1],
@@ -541,12 +530,11 @@ class OpenAIServingChat(OpenAIServing):
530 self,
531 token_ids: GenericSequence[int],
532 top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
544 - tokenizer: PreTrainedTokenizer,
533 + tokenizer: AnyTokenizer,
534 num_output_top_logprobs: Optional[int] = None,
535 ) -> ChatCompletionLogProbs:
536 """Create OpenAI-style logprobs."""
548 -
549 - logprobs_content = []
537 + logprobs_content: List[ChatCompletionLogProbsContent] = []
538
539 for i, token_id in enumerate(token_ids):
540 step_top_logprobs = top_logprobs[i]
@@ -554,23 +542,32 @@ class OpenAIServingChat(OpenAIServing):
542 token = tokenizer.decode(token_id)
543 if self.return_tokens_as_token_ids:
544 token = f"token_id:{token_id}"
545 +
546 logprobs_content.append(
547 ChatCompletionLogProbsContent(
548 token=token,
560 - bytes=list(token.encode("utf-8", errors="replace"))))
549 + bytes=list(token.encode("utf-8", errors="replace")),
550 + ))
551 else:
552 + step_token = step_top_logprobs[token_id]
553 + step_decoded = step_token.decoded_token
554 +
555 logprobs_content.append(
556 ChatCompletionLogProbsContent(
557 token=self._get_decoded_token(
565 - step_top_logprobs[token_id], token_id, tokenizer,
566 - self.return_tokens_as_token_ids),
567 - logprob=max(step_top_logprobs[token_id].logprob,
568 - -9999.0),
569 - bytes=list(
570 - step_top_logprobs[token_id].decoded_token.encode(
571 - "utf-8", errors="replace")),
558 + step_token,
559 + token_id,
560 + tokenizer,
561 + self.return_tokens_as_token_ids,
562 + ),
563 + logprob=max(step_token.logprob, -9999.0),
564 + bytes=None if step_decoded is None else list(
565 + step_decoded.encode("utf-8", errors="replace")),
566 top_logprobs=self._get_top_logprobs(
573 - step_top_logprobs, num_output_top_logprobs,
574 - tokenizer)))
567 + step_top_logprobs,
568 + num_output_top_logprobs,
569 + tokenizer,
570 + ),
571 + ))
572
573 return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+61 -36
@@ -3,10 +3,9 @@ import time
3 from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4 Optional)
5 from typing import Sequence as GenericSequence
6 -from typing import Tuple, cast
6 +from typing import Tuple, Union, cast
7
8 from fastapi import Request
9 -from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import AsyncEngineClient
@@ -19,7 +18,7 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
18 CompletionResponseChoice,
19 CompletionResponseStreamChoice,
20 CompletionStreamResponse,
22 - UsageInfo)
21 + ErrorResponse, UsageInfo)
22 # yapf: enable
23 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24 OpenAIServing,
@@ -29,6 +28,7 @@ from vllm.outputs import RequestOutput
28 from vllm.sequence import Logprob
29 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
30 log_tracing_disabled_warning)
31 +from vllm.transformers_utils.tokenizer import AnyTokenizer
32 from vllm.utils import merge_async_iterators, random_uuid
33
34 logger = init_logger(__name__)
@@ -60,8 +60,11 @@ class OpenAIServingCompletion(OpenAIServing):
60 request_logger=request_logger,
61 return_tokens_as_token_ids=return_tokens_as_token_ids)
62
63 - async def create_completion(self, request: CompletionRequest,
64 - raw_request: Request):
63 + async def create_completion(
64 + self,
65 + request: CompletionRequest,
66 + raw_request: Request,
67 + ) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]:
68 """Completion API similar to OpenAI's API.
69
70 See https://platform.openai.com/docs/api-reference/completions/create
@@ -84,15 +87,6 @@ class OpenAIServingCompletion(OpenAIServing):
87 request_id = f"cmpl-{random_uuid()}"
88 created_time = int(time.time())
89
87 - if request.prompt_logprobs is not None:
88 - if request.stream and request.prompt_logprobs > 0:
89 - return self.create_error_response(
90 - "Prompt_logprobs are not available when stream is enabled")
91 - elif request.prompt_logprobs < 0:
92 - return self.create_error_response(
93 - f"Prompt_logprobs set to invalid negative "
94 - f"value: {request.prompt_logprobs}")
95 -
90 # Schedule the request and get the result generator.
91 generators: List[AsyncGenerator[RequestOutput, None]] = []
92 try:
@@ -153,9 +147,8 @@ class OpenAIServingCompletion(OpenAIServing):
147 # TODO: Use a vllm-specific Validation Error
148 return self.create_error_response(str(e))
149
156 - result_generator: AsyncIterator[Tuple[
157 - int, RequestOutput]] = merge_async_iterators(
158 - *generators, is_cancelled=raw_request.is_disconnected)
150 + result_generator = merge_async_iterators(
151 + *generators, is_cancelled=raw_request.is_disconnected)
152
153 # Similar to the OpenAI API, when n != best_of, we do not stream the
154 # results. In addition, we do not stream the results when use
@@ -227,7 +220,7 @@ class OpenAIServingCompletion(OpenAIServing):
220 created_time: int,
221 model_name: str,
222 num_prompts: int,
230 - tokenizer: PreTrainedTokenizer,
223 + tokenizer: AnyTokenizer,
224 ) -> AsyncGenerator[str, None]:
225 num_choices = 1 if request.n is None else request.n
226 previous_texts = [""] * num_choices * num_prompts
@@ -236,6 +229,13 @@ class OpenAIServingCompletion(OpenAIServing):
229
230 try:
231 async for prompt_idx, res in result_generator:
232 + prompt_token_ids = res.prompt_token_ids
233 + prompt_logprobs = res.prompt_logprobs
234 + prompt_text = res.prompt
235 +
236 + delta_token_ids: GenericSequence[int]
237 + out_logprobs: Optional[GenericSequence[Optional[Dict[
238 + int, Logprob]]]]
239
240 for output in res.outputs:
241 i = output.index + prompt_idx * num_choices
@@ -244,19 +244,25 @@ class OpenAIServingCompletion(OpenAIServing):
244
245 assert request.max_tokens is not None
246 if request.echo and request.max_tokens == 0:
247 + assert prompt_text is not None
248 # only return the prompt
248 - delta_text = res.prompt
249 - delta_token_ids = res.prompt_token_ids
250 - out_logprobs = res.prompt_logprobs
249 + delta_text = prompt_text
250 + delta_token_ids = prompt_token_ids
251 + out_logprobs = prompt_logprobs
252 has_echoed[i] = True
253 elif (request.echo and request.max_tokens > 0
254 and not has_echoed[i]):
255 + assert prompt_text is not None
256 + assert prompt_logprobs is not None
257 # echo the prompt and first token
255 - delta_text = res.prompt + output.text
256 - delta_token_ids = (res.prompt_token_ids +
257 - output.token_ids)
258 - out_logprobs = res.prompt_logprobs + (output.logprobs
259 - or [])
258 + delta_text = prompt_text + output.text
259 + delta_token_ids = [
260 + *prompt_token_ids, *output.token_ids
261 + ]
262 + out_logprobs = [
263 + *prompt_logprobs,
264 + *(output.logprobs or []),
265 + ]
266 has_echoed[i] = True
267 else:
268 # return just the delta
@@ -301,7 +307,7 @@ class OpenAIServingCompletion(OpenAIServing):
307 and request.stream_options.include_usage):
308 if (request.stream_options.continuous_usage_stats
309 or output.finish_reason is not None):
304 - prompt_tokens = len(res.prompt_token_ids)
310 + prompt_tokens = len(prompt_token_ids)
311 completion_tokens = len(output.token_ids)
312 usage = UsageInfo(
313 prompt_tokens=prompt_tokens,
@@ -342,7 +348,7 @@ class OpenAIServingCompletion(OpenAIServing):
348 request_id: str,
349 created_time: int,
350 model_name: str,
345 - tokenizer: PreTrainedTokenizer,
351 + tokenizer: AnyTokenizer,
352 ) -> CompletionResponse:
353 choices: List[CompletionResponseChoice] = []
354 num_prompt_tokens = 0
@@ -353,16 +359,31 @@ class OpenAIServingCompletion(OpenAIServing):
359 prompt_logprobs = final_res.prompt_logprobs
360 prompt_text = final_res.prompt
361
362 + token_ids: GenericSequence[int]
363 + out_logprobs: Optional[GenericSequence[Optional[Dict[int,
364 + Logprob]]]]
365 +
366 for output in final_res.outputs:
367 assert request.max_tokens is not None
368 if request.echo and request.max_tokens == 0:
369 + assert prompt_text is not None
370 token_ids = prompt_token_ids
371 out_logprobs = prompt_logprobs
372 output_text = prompt_text
373 elif request.echo and request.max_tokens > 0:
363 - token_ids = prompt_token_ids + list(output.token_ids)
364 - out_logprobs = (prompt_logprobs + output.logprobs
365 - if request.logprobs is not None else None)
374 + assert prompt_text is not None
375 + token_ids = [*prompt_token_ids, *output.token_ids]
376 +
377 + if request.logprobs is None:
378 + out_logprobs = None
379 + else:
380 + assert prompt_logprobs is not None
381 + assert output.logprobs is not None
382 + out_logprobs = [
383 + *prompt_logprobs,
384 + *output.logprobs,
385 + ]
386 +
387 output_text = prompt_text + output.text
388 else:
389 token_ids = output.token_ids
@@ -413,7 +434,7 @@ class OpenAIServingCompletion(OpenAIServing):
434 token_ids: GenericSequence[int],
435 top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
436 num_output_top_logprobs: int,
416 - tokenizer: PreTrainedTokenizer,
437 + tokenizer: AnyTokenizer,
438 initial_text_offset: int = 0,
439 ) -> CompletionLogProbs:
440 """Create logprobs for OpenAI Completion API."""
@@ -430,17 +451,21 @@ class OpenAIServingCompletion(OpenAIServing):
451 token = tokenizer.decode(token_id)
452 if self.return_tokens_as_token_ids:
453 token = f"token_id:{token_id}"
454 +
455 out_tokens.append(token)
456 out_token_logprobs.append(None)
457 out_top_logprobs.append(None)
458 else:
459 + step_token = step_top_logprobs[token_id]
460 +
461 token = self._get_decoded_token(
438 - step_top_logprobs[token_id],
462 + step_token,
463 token_id,
464 tokenizer,
441 - return_as_token_id=self.return_tokens_as_token_ids)
442 - token_logprob = max(step_top_logprobs[token_id].logprob,
443 - -9999.0)
465 + return_as_token_id=self.return_tokens_as_token_ids,
466 + )
467 + token_logprob = max(step_token.logprob, -9999.0)
468 +
469 out_tokens.append(token)
470 out_token_logprobs.append(token_logprob)
471
vllm/entrypoints/openai/serving_embedding.py
+26 -18
@@ -1,11 +1,11 @@
1 import asyncio
2 import base64
3 import time
4 -from typing import (AsyncGenerator, AsyncIterator, List, Optional, Tuple,
5 - Union, cast)
4 +from typing import AsyncGenerator, List, Literal, Optional, Union, cast
5
6 import numpy as np
7 from fastapi import Request
8 +from typing_extensions import assert_never
9
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import AsyncEngineClient
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
16 ErrorResponse, UsageInfo)
17 from vllm.entrypoints.openai.serving_engine import OpenAIServing
18 from vllm.logger import init_logger
19 -from vllm.outputs import EmbeddingRequestOutput
19 +from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
20 from vllm.utils import merge_async_iterators, random_uuid
21
22 logger = init_logger(__name__)
@@ -24,18 +24,28 @@ logger = init_logger(__name__)
24 TypeTokenIDs = List[int]
25
26
27 +def _get_embedding(
28 + output: EmbeddingOutput,
29 + encoding_format: Literal["float", "base64"],
30 +) -> Union[List[float], str]:
31 + if encoding_format == "float":
32 + return output.embedding
33 + elif encoding_format == "base64":
34 + embedding_bytes = np.array(output.embedding).tobytes()
35 + return base64.b64encode(embedding_bytes).decode("utf-8")
36 +
37 + assert_never(encoding_format)
38 +
39 +
40 def request_output_to_embedding_response(
41 final_res_batch: List[EmbeddingRequestOutput], request_id: str,
42 created_time: int, model_name: str,
30 - encoding_format: str) -> EmbeddingResponse:
43 + encoding_format: Literal["float", "base64"]) -> EmbeddingResponse:
44 data: List[EmbeddingResponseData] = []
45 num_prompt_tokens = 0
46 for idx, final_res in enumerate(final_res_batch):
47 prompt_token_ids = final_res.prompt_token_ids
35 - embedding = final_res.outputs.embedding
36 - if encoding_format == "base64":
37 - embedding_bytes = np.array(embedding).tobytes()
38 - embedding = base64.b64encode(embedding_bytes).decode("utf-8")
48 + embedding = _get_embedding(final_res.outputs, encoding_format)
49 embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
50 data.append(embedding_data)
51
@@ -76,8 +86,8 @@ class OpenAIServingEmbedding(OpenAIServing):
86 async def create_embedding(
87 self,
88 request: EmbeddingRequest,
79 - raw_request: Optional[Request] = None
80 - ) -> Union[ErrorResponse, EmbeddingResponse]:
89 + raw_request: Optional[Request] = None,
90 + ) -> Union[EmbeddingResponse, ErrorResponse]:
91 """Completion API similar to OpenAI's API.
92
93 See https://platform.openai.com/docs/api-reference/embeddings/create
@@ -89,8 +99,7 @@ class OpenAIServingEmbedding(OpenAIServing):
99 if error_check_ret is not None:
100 return error_check_ret
101
92 - encoding_format = (request.encoding_format
93 - if request.encoding_format else "float")
102 + encoding_format = request.encoding_format
103 if request.dimensions is not None:
104 return self.create_error_response(
105 "dimensions is currently not supported")
@@ -145,11 +154,10 @@ class OpenAIServingEmbedding(OpenAIServing):
154 # TODO: Use a vllm-specific Validation Error
155 return self.create_error_response(str(e))
156
148 - result_generator: AsyncIterator[Tuple[
149 - int, EmbeddingRequestOutput]] = merge_async_iterators(
150 - *generators,
151 - is_cancelled=raw_request.is_disconnected
152 - if raw_request else None)
157 + result_generator = merge_async_iterators(
158 + *generators,
159 + is_cancelled=raw_request.is_disconnected if raw_request else None,
160 + )
161
162 # Non-streaming response
163 final_res_batch: List[Optional[EmbeddingRequestOutput]]
@@ -175,7 +183,7 @@ class OpenAIServingEmbedding(OpenAIServing):
183
184 return response
185
178 - def _check_embedding_mode(self, embedding_mode: bool):
186 + def _check_embedding_mode(self, embedding_mode: bool) -> bool:
187 if not embedding_mode:
188 logger.warning(
189 "embedding_mode is False. Embedding API will not work.")
vllm/entrypoints/openai/serving_engine.py
+1 -1
@@ -31,7 +31,7 @@ from vllm.pooling_params import PoolingParams
31 from vllm.prompt_adapter.request import PromptAdapterRequest
32 from vllm.sampling_params import LogitsProcessor, SamplingParams
33 from vllm.sequence import Logprob
34 -from vllm.transformers_utils.tokenizer_group import AnyTokenizer
34 +from vllm.transformers_utils.tokenizer import AnyTokenizer
35
36 logger = init_logger(__name__)
37
vllm/sampling_params.py
+62
@@ -153,6 +153,68 @@ class SamplingParams(
153 output_text_buffer_length: int = 0
154 _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
155
156 + @staticmethod
157 + def from_optional(
158 + n: Optional[int] = 1,
159 + best_of: Optional[int] = None,
160 + presence_penalty: Optional[float] = 0.0,
161 + frequency_penalty: Optional[float] = 0.0,
162 + repetition_penalty: Optional[float] = 1.0,
163 + temperature: Optional[float] = 1.0,
164 + top_p: Optional[float] = 1.0,
165 + top_k: int = -1,
166 + min_p: float = 0.0,
167 + seed: Optional[int] = None,
168 + use_beam_search: bool = False,
169 + length_penalty: float = 1.0,
170 + early_stopping: Union[bool, str] = False,
171 + stop: Optional[Union[str, List[str]]] = None,
172 + stop_token_ids: Optional[List[int]] = None,
173 + include_stop_str_in_output: bool = False,
174 + ignore_eos: bool = False,
175 + max_tokens: Optional[int] = 16,
176 + min_tokens: int = 0,
177 + logprobs: Optional[int] = None,
178 + prompt_logprobs: Optional[int] = None,
179 + detokenize: bool = True,
180 + skip_special_tokens: bool = True,
181 + spaces_between_special_tokens: bool = True,
182 + logits_processors: Optional[List[LogitsProcessor]] = None,
183 + truncate_prompt_tokens: Optional[Annotated[int,
184 + msgspec.Meta(ge=1)]] = None,
185 + ) -> "SamplingParams":
186 + return SamplingParams(
187 + n=1 if n is None else n,
188 + best_of=best_of,
189 + presence_penalty=0.0
190 + if presence_penalty is None else presence_penalty,
191 + frequency_penalty=0.0
192 + if frequency_penalty is None else frequency_penalty,
193 + repetition_penalty=1.0
194 + if repetition_penalty is None else repetition_penalty,
195 + temperature=1.0 if temperature is None else temperature,
196 + top_p=1.0 if top_p is None else top_p,
197 + top_k=top_k,
198 + min_p=min_p,
199 + seed=seed,
200 + use_beam_search=use_beam_search,
201 + length_penalty=length_penalty,
202 + early_stopping=early_stopping,
203 + stop=stop,
204 + stop_token_ids=stop_token_ids,
205 + include_stop_str_in_output=include_stop_str_in_output,
206 + ignore_eos=ignore_eos,
207 + max_tokens=max_tokens,
208 + min_tokens=min_tokens,
209 + logprobs=logprobs,
210 + prompt_logprobs=prompt_logprobs,
211 + detokenize=detokenize,
212 + skip_special_tokens=skip_special_tokens,
213 + spaces_between_special_tokens=spaces_between_special_tokens,
214 + logits_processors=logits_processors,
215 + truncate_prompt_tokens=truncate_prompt_tokens,
216 + )
217 +
218 def __post_init__(self) -> None:
219 self.best_of = self.best_of or self.n
220 if 0 < self.temperature < _MAX_TEMP: