A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
refactor complemention api for readability (#2499)
Simon Mo committed
Jan 18, 2024 at 16:45 UTC
dd7e8f5f643167e3f13045cf75cbead54cb2ccfe
6 files changed
+284
-253
tests/entrypoints/test_openai_server.py
+10
@@ -88,6 +88,16 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
88
assert completion.usage == openai.types.CompletionUsage(
89
completion_tokens=5, prompt_tokens=6, total_tokens=11)
90
91
+ # test using token IDs
92
+ completion = await client.completions.create(
93
+ model=MODEL_NAME,
94
+ prompt=[0, 0, 0, 0, 0],
95
+ max_tokens=5,
96
+ temperature=0.0,
97
+ )
98
+ assert completion.choices[0].text is not None and len(
99
+ completion.choices[0].text) >= 5
100
+
101
102
async def test_single_chat_session(server, client: openai.AsyncOpenAI):
103
messages = [{
vllm/entrypoints/openai/protocol.py
+45
@@ -6,6 +6,7 @@ from typing import Dict, List, Literal, Optional, Union
6
from pydantic import BaseModel, Field
7
8
from vllm.utils import random_uuid
9
+from vllm.sampling_params import SamplingParams
10
11
12
class ErrorResponse(BaseModel):
@@ -78,6 +79,26 @@ class ChatCompletionRequest(BaseModel):
79
repetition_penalty: Optional[float] = 1.0
80
min_p: Optional[float] = 0.0
81
82
+ def to_sampling_params(self) -> SamplingParams:
83
+ return SamplingParams(
84
+ n=self.n,
85
+ presence_penalty=self.presence_penalty,
86
+ frequency_penalty=self.frequency_penalty,
87
+ repetition_penalty=self.repetition_penalty,
88
+ temperature=self.temperature,
89
+ top_p=self.top_p,
90
+ min_p=self.min_p,
91
+ stop=self.stop,
92
+ stop_token_ids=self.stop_token_ids,
93
+ max_tokens=self.max_tokens,
94
+ best_of=self.best_of,
95
+ top_k=self.top_k,
96
+ ignore_eos=self.ignore_eos,
97
+ use_beam_search=self.use_beam_search,
98
+ skip_special_tokens=self.skip_special_tokens,
99
+ spaces_between_special_tokens=self.spaces_between_special_tokens,
100
+ )
101
+
102
103
class CompletionRequest(BaseModel):
104
model: str
@@ -107,6 +128,30 @@ class CompletionRequest(BaseModel):
128
repetition_penalty: Optional[float] = 1.0
129
min_p: Optional[float] = 0.0
130
131
+ def to_sampling_params(self):
132
+ echo_without_generation = self.echo and self.max_tokens == 0
133
+
134
+ return SamplingParams(
135
+ n=self.n,
136
+ best_of=self.best_of,
137
+ presence_penalty=self.presence_penalty,
138
+ frequency_penalty=self.frequency_penalty,
139
+ repetition_penalty=self.repetition_penalty,
140
+ temperature=self.temperature,
141
+ top_p=self.top_p,
142
+ top_k=self.top_k,
143
+ min_p=self.min_p,
144
+ stop=self.stop,
145
+ stop_token_ids=self.stop_token_ids,
146
+ ignore_eos=self.ignore_eos,
147
+ max_tokens=self.max_tokens if not echo_without_generation else 1,
148
+ logprobs=self.logprobs,
149
+ use_beam_search=self.use_beam_search,
150
+ prompt_logprobs=self.logprobs if self.echo else None,
151
+ skip_special_tokens=self.skip_special_tokens,
152
+ spaces_between_special_tokens=(self.spaces_between_special_tokens),
153
+ )
154
+
155
156
class LogProbs(BaseModel):
157
text_offset: List[int] = Field(default_factory=list)
vllm/entrypoints/openai/serving_chat.py
+3
-25
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.protocol import (
11
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12
UsageInfo)
13
from vllm.outputs import RequestOutput
14
-from vllm.sampling_params import SamplingParams
14
from vllm.entrypoints.openai.serving_engine import OpenAIServing
15
16
logger = init_logger(__name__)
@@ -60,32 +59,11 @@ class OpenAIServingChat(OpenAIServing):
59
f"Error in applying chat template from request: {str(e)}")
60
return self.create_error_response(str(e))
61
63
- token_ids, error_check_ret = await self._check_length(request,
64
- prompt=prompt)
65
- if error_check_ret is not None:
66
- return error_check_ret
67
-
62
request_id = f"cmpl-{random_uuid()}"
63
try:
70
- spaces_between_special_tokens = request.spaces_between_special_tokens
71
- sampling_params = SamplingParams(
72
- n=request.n,
73
- presence_penalty=request.presence_penalty,
74
- frequency_penalty=request.frequency_penalty,
75
- repetition_penalty=request.repetition_penalty,
76
- temperature=request.temperature,
77
- top_p=request.top_p,
78
- min_p=request.min_p,
79
- stop=request.stop,
80
- stop_token_ids=request.stop_token_ids,
81
- max_tokens=request.max_tokens,
82
- best_of=request.best_of,
83
- top_k=request.top_k,
84
- ignore_eos=request.ignore_eos,
85
- use_beam_search=request.use_beam_search,
86
- skip_special_tokens=request.skip_special_tokens,
87
- spaces_between_special_tokens=spaces_between_special_tokens,
88
- )
64
+ token_ids = self._validate_prompt_and_tokenize(request,
65
+ prompt=prompt)
66
+ sampling_params = request.to_sampling_params()
67
except ValueError as e:
68
return self.create_error_response(str(e))
69
vllm/entrypoints/openai/serving_completion.py
+210
-215
@@ -1,20 +1,194 @@
1
import time
2
from fastapi import Request
3
-from typing import AsyncGenerator, Optional
3
+from typing import AsyncGenerator, AsyncIterator
4
from vllm.logger import init_logger
5
from vllm.utils import random_uuid
6
from vllm.engine.async_llm_engine import AsyncLLMEngine
7
-from .protocol import (CompletionRequest, CompletionResponse,
8
- CompletionResponseChoice,
9
- CompletionResponseStreamChoice,
10
- CompletionStreamResponse, LogProbs, UsageInfo)
7
+from .protocol import (
8
+ CompletionRequest,
9
+ CompletionResponse,
10
+ CompletionResponseChoice,
11
+ CompletionResponseStreamChoice,
12
+ CompletionStreamResponse,
13
+ LogProbs,
14
+ UsageInfo,
15
+)
16
from vllm.outputs import RequestOutput
12
-from vllm.sampling_params import SamplingParams
17
from vllm.entrypoints.openai.serving_engine import OpenAIServing
18
19
logger = init_logger(__name__)
20
21
22
+async def completion_stream_generator(
23
+ request: CompletionRequest,
24
+ result_generator: AsyncIterator[RequestOutput],
25
+ echo_without_generation, create_logprobs_fn, request_id, created_time,
26
+ model_name) -> AsyncGenerator[str, None]:
27
+ previous_texts = [""] * request.n
28
+ previous_num_tokens = [0] * request.n
29
+ has_echoed = [False] * request.n
30
+
31
+ async for res in result_generator:
32
+ # TODO: handle client disconnect for streaming
33
+ for output in res.outputs:
34
+ i = output.index
35
+ delta_text = output.text[len(previous_texts[i]):]
36
+ token_ids = output.token_ids[previous_num_tokens[i]:]
37
+ if request.logprobs is not None:
38
+ top_logprobs = output.logprobs[previous_num_tokens[i]:]
39
+ else:
40
+ top_logprobs = None
41
+ offsets = len(previous_texts[i])
42
+ if request.echo and not has_echoed[i]:
43
+ if not echo_without_generation:
44
+ delta_text = res.prompt + delta_text
45
+ token_ids = res.prompt_token_ids + token_ids
46
+ if top_logprobs:
47
+ top_logprobs = res.prompt_logprobs + top_logprobs
48
+ else: # only just return the prompt
49
+ delta_text = res.prompt
50
+ token_ids = res.prompt_token_ids
51
+ if top_logprobs:
52
+ top_logprobs = res.prompt_logprobs
53
+ has_echoed[i] = True
54
+ if request.logprobs is not None:
55
+ logprobs = create_logprobs_fn(
56
+ token_ids=token_ids,
57
+ top_logprobs=top_logprobs,
58
+ num_output_top_logprobs=request.logprobs,
59
+ initial_text_offset=offsets,
60
+ )
61
+ else:
62
+ logprobs = None
63
+ previous_texts[i] = output.text
64
+ previous_num_tokens[i] = len(output.token_ids)
65
+ finish_reason = output.finish_reason
66
+ response_json = CompletionStreamResponse(
67
+ id=request_id,
68
+ created=created_time,
69
+ model=model_name,
70
+ choices=[
71
+ CompletionResponseStreamChoice(
72
+ index=i,
73
+ text=delta_text,
74
+ logprobs=logprobs,
75
+ finish_reason=finish_reason,
76
+ )
77
+ ]).json(exclude_unset=True, ensure_ascii=False)
78
+ yield f"data: {response_json}\n\n"
79
+
80
+ if output.finish_reason is not None:
81
+ logprobs = LogProbs() if request.logprobs is not None else None
82
+ prompt_tokens = len(res.prompt_token_ids)
83
+ completion_tokens = len(output.token_ids)
84
+ final_usage = UsageInfo(
85
+ prompt_tokens=prompt_tokens,
86
+ completion_tokens=completion_tokens,
87
+ total_tokens=prompt_tokens + completion_tokens,
88
+ )
89
+ response_json = CompletionStreamResponse(
90
+ id=request_id,
91
+ created=created_time,
92
+ model=model_name,
93
+ choices=[
94
+ CompletionResponseStreamChoice(
95
+ index=i,
96
+ text="",
97
+ logprobs=logprobs,
98
+ finish_reason=output.finish_reason,
99
+ )
100
+ ],
101
+ usage=final_usage,
102
+ ).json(exclude_unset=True, ensure_ascii=False)
103
+ yield f"data: {response_json}\n\n"
104
+
105
+ yield "data: [DONE]\n\n"
106
+
107
+
108
+def parse_prompt_format(prompt) -> tuple[bool, list]:
109
+ # get the prompt, openai supports the following
110
+ # "a string, array of strings, array of tokens, or array of token arrays."
111
+ prompt_is_tokens = False
112
+ prompts = [prompt] # case 1: a string
113
+ if isinstance(prompt, list):
114
+ if len(prompt) == 0:
115
+ raise ValueError("please provide at least one prompt")
116
+ elif isinstance(prompt[0], str):
117
+ prompt_is_tokens = False
118
+ prompts = prompt # case 2: array of strings
119
+ elif isinstance(prompt[0], int):
120
+ prompt_is_tokens = True
121
+ prompts = [prompt] # case 3: array of tokens
122
+ elif isinstance(prompt[0], list) and isinstance(prompt[0][0], int):
123
+ prompt_is_tokens = True
124
+ prompts = prompt # case 4: array of token arrays
125
+ else:
126
+ raise ValueError(
127
+ "prompt must be a string, array of strings, array of tokens, or array of token arrays"
128
+ )
129
+ return prompt_is_tokens, prompts
130
+
131
+
132
+def request_output_to_completion_response(final_res: RequestOutput, request,
133
+ echo_without_generation,
134
+ create_logprobs_fn, request_id,
135
+ created_time,
136
+ model_name) -> CompletionResponse:
137
+ assert final_res is not None
138
+ choices = []
139
+ prompt_token_ids = final_res.prompt_token_ids
140
+ prompt_logprobs = final_res.prompt_logprobs
141
+ prompt_text = final_res.prompt
142
+ for output in final_res.outputs:
143
+ if request.logprobs is not None:
144
+ if not echo_without_generation:
145
+ token_ids = output.token_ids
146
+ top_logprobs = output.logprobs
147
+ if request.echo:
148
+ token_ids = prompt_token_ids + token_ids
149
+ top_logprobs = prompt_logprobs + top_logprobs
150
+ else:
151
+ token_ids = prompt_token_ids
152
+ top_logprobs = prompt_logprobs
153
+ logprobs = create_logprobs_fn(
154
+ token_ids=token_ids,
155
+ top_logprobs=top_logprobs,
156
+ num_output_top_logprobs=request.logprobs,
157
+ )
158
+ else:
159
+ logprobs = None
160
+ if not echo_without_generation:
161
+ output_text = output.text
162
+ if request.echo:
163
+ output_text = prompt_text + output_text
164
+ else:
165
+ output_text = prompt_text
166
+ choice_data = CompletionResponseChoice(
167
+ index=output.index,
168
+ text=output_text,
169
+ logprobs=logprobs,
170
+ finish_reason=output.finish_reason,
171
+ )
172
+ choices.append(choice_data)
173
+
174
+ num_prompt_tokens = len(final_res.prompt_token_ids)
175
+ num_generated_tokens = sum(
176
+ len(output.token_ids) for output in final_res.outputs)
177
+ usage = UsageInfo(
178
+ prompt_tokens=num_prompt_tokens,
179
+ completion_tokens=num_generated_tokens,
180
+ total_tokens=num_prompt_tokens + num_generated_tokens,
181
+ )
182
+
183
+ return CompletionResponse(
184
+ id=request_id,
185
+ created=created_time,
186
+ model=model_name,
187
+ choices=choices,
188
+ usage=usage,
189
+ )
190
+
191
+
192
class OpenAIServingCompletion(OpenAIServing):
193
194
def __init__(self, engine: AsyncLLMEngine, served_model: str):
@@ -32,7 +206,6 @@ class OpenAIServingCompletion(OpenAIServing):
206
suffix)
207
- logit_bias (to be supported by vLLM engine)
208
"""
35
-
209
error_check_ret = await self._check_model(request)
210
if error_check_ret is not None:
211
return error_check_ret
@@ -40,83 +213,42 @@ class OpenAIServingCompletion(OpenAIServing):
213
# OpenAI API supports echoing the prompt when max_tokens is 0.
214
echo_without_generation = request.echo and request.max_tokens == 0
215
216
+ # Return error for unsupported features.
217
if request.suffix is not None:
44
- # The language models we currently support do not support suffix.
218
return self.create_error_response(
219
"suffix is not currently supported")
47
-
220
if request.logit_bias is not None and len(request.logit_bias) > 0:
49
- # TODO: support logit_bias in vLLM engine.
221
return self.create_error_response(
222
"logit_bias is not currently supported")
223
224
model_name = request.model
225
request_id = f"cmpl-{random_uuid()}"
226
+ created_time = int(time.monotonic())
227
56
- use_token_ids = False
57
- if isinstance(request.prompt, list):
58
- if len(request.prompt) == 0:
59
- return self.create_error_response(
60
- "please provide at least one prompt")
61
- first_element = request.prompt[0]
62
- if isinstance(first_element, int):
63
- use_token_ids = True
64
- prompt = request.prompt
65
- elif isinstance(first_element, (str, list)):
66
- # TODO: handles multiple prompt case in list[list[int]]
67
- if len(request.prompt) > 1:
68
- return self.create_error_response(
69
- "multiple prompts in a batch is not currently supported"
70
- )
71
- use_token_ids = not isinstance(first_element, str)
72
- prompt = request.prompt[0]
73
- else:
74
- prompt = request.prompt
228
+ # Schedule the request and get the result generator.
229
+ try:
230
+ sampling_params = request.to_sampling_params()
231
76
- if use_token_ids:
77
- _, error_check_ret = await self._check_length(request,
78
- prompt_ids=prompt)
79
- else:
80
- token_ids, error_check_ret = await self._check_length(
81
- request, prompt=prompt)
82
- if error_check_ret is not None:
83
- return error_check_ret
232
+ prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
233
85
- created_time = int(time.monotonic())
86
- try:
87
- spaces_between_special_tokens = request.spaces_between_special_tokens
88
- sampling_params = SamplingParams(
89
- n=request.n,
90
- best_of=request.best_of,
91
- presence_penalty=request.presence_penalty,
92
- frequency_penalty=request.frequency_penalty,
93
- repetition_penalty=request.repetition_penalty,
94
- temperature=request.temperature,
95
- top_p=request.top_p,
96
- top_k=request.top_k,
97
- min_p=request.min_p,
98
- stop=request.stop,
99
- stop_token_ids=request.stop_token_ids,
100
- ignore_eos=request.ignore_eos,
101
- max_tokens=request.max_tokens
102
- if not echo_without_generation else 1,
103
- logprobs=request.logprobs,
104
- use_beam_search=request.use_beam_search,
105
- prompt_logprobs=request.logprobs if request.echo else None,
106
- skip_special_tokens=request.skip_special_tokens,
107
- spaces_between_special_tokens=spaces_between_special_tokens,
108
- )
109
- except ValueError as e:
110
- return self.create_error_response(str(e))
234
+ if len(prompts) > 1:
235
+ raise ValueError(
236
+ "Batching in completion API is not supported.")
237
+ prompt = prompts[0]
238
+
239
+ if prompt_is_tokens:
240
+ input_ids = self._validate_prompt_and_tokenize(
241
+ request, prompt_ids=prompt)
242
+ else:
243
+ input_ids = self._validate_prompt_and_tokenize(request,
244
+ prompt=prompt)
245
112
- if use_token_ids:
246
result_generator = self.engine.generate(None,
247
sampling_params,
248
request_id,
116
- prompt_token_ids=prompt)
117
- else:
118
- result_generator = self.engine.generate(prompt, sampling_params,
119
- request_id, token_ids)
249
+ prompt_token_ids=input_ids)
250
+ except ValueError as e:
251
+ return self.create_error_response(str(e))
252
253
# Similar to the OpenAI API, when n != best_of, we do not stream the
254
# results. In addition, we do not stream the results when use beam search.
@@ -124,101 +256,13 @@ class OpenAIServingCompletion(OpenAIServing):
256
and (request.best_of is None or request.n == request.best_of)
257
and not request.use_beam_search)
258
127
- def create_stream_response_json(
128
- index: int,
129
- text: str,
130
- logprobs: Optional[LogProbs] = None,
131
- finish_reason: Optional[str] = None,
132
- usage: Optional[UsageInfo] = None,
133
- ) -> str:
134
- choice_data = CompletionResponseStreamChoice(
135
- index=index,
136
- text=text,
137
- logprobs=logprobs,
138
- finish_reason=finish_reason,
139
- )
140
- response = CompletionStreamResponse(
141
- id=request_id,
142
- created=created_time,
143
- model=model_name,
144
- choices=[choice_data],
145
- )
146
- if usage is not None:
147
- response.usage = usage
148
- response_json = response.json(exclude_unset=True,
149
- ensure_ascii=False)
150
-
151
- return response_json
152
-
153
- async def completion_stream_generator() -> AsyncGenerator[str, None]:
154
- previous_texts = [""] * request.n
155
- previous_num_tokens = [0] * request.n
156
- has_echoed = [False] * request.n
157
- async for res in result_generator:
158
- res: RequestOutput
159
- for output in res.outputs:
160
- i = output.index
161
- delta_text = output.text[len(previous_texts[i]):]
162
- token_ids = output.token_ids[previous_num_tokens[i]:]
163
- if request.logprobs is not None:
164
- top_logprobs = output.logprobs[previous_num_tokens[i]:]
165
- else:
166
- top_logprobs = None
167
- offsets = len(previous_texts[i])
168
- if request.echo and not has_echoed[i]:
169
- if not echo_without_generation:
170
- delta_text = res.prompt + delta_text
171
- token_ids = res.prompt_token_ids + token_ids
172
- if top_logprobs:
173
- top_logprobs = res.prompt_logprobs + top_logprobs
174
- else: # only just return the prompt
175
- delta_text = res.prompt
176
- token_ids = res.prompt_token_ids
177
- if top_logprobs:
178
- top_logprobs = res.prompt_logprobs
179
- has_echoed[i] = True
180
- if request.logprobs is not None:
181
- logprobs = self._create_logprobs(
182
- token_ids=token_ids,
183
- top_logprobs=top_logprobs,
184
- num_output_top_logprobs=request.logprobs,
185
- initial_text_offset=offsets,
186
- )
187
- else:
188
- logprobs = None
189
- previous_texts[i] = output.text
190
- previous_num_tokens[i] = len(output.token_ids)
191
- finish_reason = output.finish_reason
192
- response_json = create_stream_response_json(
193
- index=i,
194
- text=delta_text,
195
- logprobs=logprobs,
196
- finish_reason=finish_reason,
197
- )
198
- yield f"data: {response_json}\n\n"
199
- if output.finish_reason is not None:
200
- logprobs = (LogProbs()
201
- if request.logprobs is not None else None)
202
- prompt_tokens = len(res.prompt_token_ids)
203
- completion_tokens = len(output.token_ids)
204
- final_usage = UsageInfo(
205
- prompt_tokens=prompt_tokens,
206
- completion_tokens=completion_tokens,
207
- total_tokens=prompt_tokens + completion_tokens,
208
- )
209
- response_json = create_stream_response_json(
210
- index=i,
211
- text="",
212
- logprobs=logprobs,
213
- finish_reason=output.finish_reason,
214
- usage=final_usage,
215
- )
216
- yield f"data: {response_json}\n\n"
217
- yield "data: [DONE]\n\n"
218
-
259
# Streaming response
260
if stream:
221
- return completion_stream_generator()
261
+ return completion_stream_generator(request, result_generator,
262
+ echo_without_generation,
263
+ self._create_logprobs,
264
+ request_id, created_time,
265
+ model_name)
266
267
# Non-streaming response
268
final_res: RequestOutput = None
@@ -228,62 +272,13 @@ class OpenAIServingCompletion(OpenAIServing):
272
await self.engine.abort(request_id)
273
return self.create_error_response("Client disconnected")
274
final_res = res
231
- assert final_res is not None
232
- choices = []
233
- prompt_token_ids = final_res.prompt_token_ids
234
- prompt_logprobs = final_res.prompt_logprobs
235
- prompt_text = final_res.prompt
236
- for output in final_res.outputs:
237
- if request.logprobs is not None:
238
- if not echo_without_generation:
239
- token_ids = output.token_ids
240
- top_logprobs = output.logprobs
241
- if request.echo:
242
- token_ids = prompt_token_ids + token_ids
243
- top_logprobs = prompt_logprobs + top_logprobs
244
- else:
245
- token_ids = prompt_token_ids
246
- top_logprobs = prompt_logprobs
247
- logprobs = self._create_logprobs(
248
- token_ids=token_ids,
249
- top_logprobs=top_logprobs,
250
- num_output_top_logprobs=request.logprobs,
251
- )
252
- else:
253
- logprobs = None
254
- if not echo_without_generation:
255
- output_text = output.text
256
- if request.echo:
257
- output_text = prompt_text + output_text
258
- else:
259
- output_text = prompt_text
260
- choice_data = CompletionResponseChoice(
261
- index=output.index,
262
- text=output_text,
263
- logprobs=logprobs,
264
- finish_reason=output.finish_reason,
265
- )
266
- choices.append(choice_data)
267
-
268
- num_prompt_tokens = len(final_res.prompt_token_ids)
269
- num_generated_tokens = sum(
270
- len(output.token_ids) for output in final_res.outputs)
271
- usage = UsageInfo(
272
- prompt_tokens=num_prompt_tokens,
273
- completion_tokens=num_generated_tokens,
274
- total_tokens=num_prompt_tokens + num_generated_tokens,
275
- )
276
- response = CompletionResponse(
277
- id=request_id,
278
- created=created_time,
279
- model=model_name,
280
- choices=choices,
281
- usage=usage,
282
- )
275
+ response = request_output_to_completion_response(
276
+ final_res, request, echo_without_generation, self._create_logprobs,
277
+ request_id, created_time, model_name)
278
279
+ # When user requests streaming but we don't stream, we still need to
280
+ # return a streaming response with a single event.
281
if request.stream:
285
- # When user requests streaming but we don't stream, we still need to
286
- # return a streaming response with a single event.
282
response_json = response.json(ensure_ascii=False)
283
284
async def fake_stream_generator() -> AsyncGenerator[str, None]:
vllm/entrypoints/openai/serving_engine.py
+15
-12
@@ -1,6 +1,6 @@
1
import asyncio
2
from http import HTTPStatus
3
-from typing import Dict, List, Optional, Tuple, Union
3
+from typing import Dict, List, Optional, Union
4
from vllm.logger import init_logger
5
from vllm.transformers_utils.tokenizer import get_tokenizer
6
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -104,27 +104,30 @@ class OpenAIServing:
104
err_type="NotFoundError",
105
status_code=HTTPStatus.NOT_FOUND)
106
107
- async def _check_length(
108
- self,
109
- request: Union[ChatCompletionRequest, CompletionRequest],
110
- prompt: Optional[str] = None,
111
- prompt_ids: Optional[List[int]] = None
112
- ) -> Tuple[List[int], Optional[ErrorResponse]]:
113
- assert (not (prompt is None and prompt_ids is None)
114
- and not (prompt is not None and prompt_ids is not None)
115
- ), "Either prompt or prompt_ids should be provided."
107
+ def _validate_prompt_and_tokenize(
108
+ self,
109
+ request: Union[ChatCompletionRequest, CompletionRequest],
110
+ prompt: Optional[str] = None,
111
+ prompt_ids: Optional[List[int]] = None) -> List[int]:
112
+ if not (prompt or prompt_ids):
113
+ raise ValueError("Either prompt or prompt_ids should be provided.")
114
+ if (prompt and prompt_ids):
115
+ raise ValueError(
116
+ "Only one of prompt or prompt_ids should be provided.")
117
+
118
input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
119
prompt).input_ids
120
token_num = len(input_ids)
121
122
if request.max_tokens is None:
123
request.max_tokens = self.max_model_len - token_num
124
+
125
if token_num + request.max_tokens > self.max_model_len:
123
- return input_ids, self.create_error_response(
126
+ raise ValueError(
127
f"This model's maximum context length is {self.max_model_len} tokens. "
128
f"However, you requested {request.max_tokens + token_num} tokens "
129
f"({token_num} in the messages, "
130
f"{request.max_tokens} in the completion). "
131
f"Please reduce the length of the messages or completion.", )
132
else:
130
- return input_ids, None
133
+ return input_ids
vllm/model_executor/weight_utils.py
+1
-1
@@ -163,7 +163,7 @@ def prepare_hf_model_weights(
163
use_safetensors = True
164
break
165
166
- logger.info(f"Downloading model weights {allow_patterns}")
166
+ logger.info(f"Using model weights format {allow_patterns}")
167
# Use file lock to prevent multiple processes from
168
# downloading the same model weights at the same time.
169
with get_lock(model_name_or_path, cache_dir):