@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd7e8f5f64
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

refactor complemention api for readability (#2499)

Simon Mo committed Jan 18, 2024 at 16:45 UTC dd7e8f5f643167e3f13045cf75cbead54cb2ccfe
6 files changed +284 -253
tests/entrypoints/test_openai_server.py
+10
@@ -88,6 +88,16 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
88 assert completion.usage == openai.types.CompletionUsage(
89 completion_tokens=5, prompt_tokens=6, total_tokens=11)
90
91 + # test using token IDs
92 + completion = await client.completions.create(
93 + model=MODEL_NAME,
94 + prompt=[0, 0, 0, 0, 0],
95 + max_tokens=5,
96 + temperature=0.0,
97 + )
98 + assert completion.choices[0].text is not None and len(
99 + completion.choices[0].text) >= 5
100 +
101
102 async def test_single_chat_session(server, client: openai.AsyncOpenAI):
103 messages = [{
vllm/entrypoints/openai/protocol.py
+45
@@ -6,6 +6,7 @@ from typing import Dict, List, Literal, Optional, Union
6 from pydantic import BaseModel, Field
7
8 from vllm.utils import random_uuid
9 +from vllm.sampling_params import SamplingParams
10
11
12 class ErrorResponse(BaseModel):
@@ -78,6 +79,26 @@ class ChatCompletionRequest(BaseModel):
79 repetition_penalty: Optional[float] = 1.0
80 min_p: Optional[float] = 0.0
81
82 + def to_sampling_params(self) -> SamplingParams:
83 + return SamplingParams(
84 + n=self.n,
85 + presence_penalty=self.presence_penalty,
86 + frequency_penalty=self.frequency_penalty,
87 + repetition_penalty=self.repetition_penalty,
88 + temperature=self.temperature,
89 + top_p=self.top_p,
90 + min_p=self.min_p,
91 + stop=self.stop,
92 + stop_token_ids=self.stop_token_ids,
93 + max_tokens=self.max_tokens,
94 + best_of=self.best_of,
95 + top_k=self.top_k,
96 + ignore_eos=self.ignore_eos,
97 + use_beam_search=self.use_beam_search,
98 + skip_special_tokens=self.skip_special_tokens,
99 + spaces_between_special_tokens=self.spaces_between_special_tokens,
100 + )
101 +
102
103 class CompletionRequest(BaseModel):
104 model: str
@@ -107,6 +128,30 @@ class CompletionRequest(BaseModel):
128 repetition_penalty: Optional[float] = 1.0
129 min_p: Optional[float] = 0.0
130
131 + def to_sampling_params(self):
132 + echo_without_generation = self.echo and self.max_tokens == 0
133 +
134 + return SamplingParams(
135 + n=self.n,
136 + best_of=self.best_of,
137 + presence_penalty=self.presence_penalty,
138 + frequency_penalty=self.frequency_penalty,
139 + repetition_penalty=self.repetition_penalty,
140 + temperature=self.temperature,
141 + top_p=self.top_p,
142 + top_k=self.top_k,
143 + min_p=self.min_p,
144 + stop=self.stop,
145 + stop_token_ids=self.stop_token_ids,
146 + ignore_eos=self.ignore_eos,
147 + max_tokens=self.max_tokens if not echo_without_generation else 1,
148 + logprobs=self.logprobs,
149 + use_beam_search=self.use_beam_search,
150 + prompt_logprobs=self.logprobs if self.echo else None,
151 + skip_special_tokens=self.skip_special_tokens,
152 + spaces_between_special_tokens=(self.spaces_between_special_tokens),
153 + )
154 +
155
156 class LogProbs(BaseModel):
157 text_offset: List[int] = Field(default_factory=list)
vllm/entrypoints/openai/serving_chat.py
+3 -25
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.protocol import (
11 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12 UsageInfo)
13 from vllm.outputs import RequestOutput
14 -from vllm.sampling_params import SamplingParams
14 from vllm.entrypoints.openai.serving_engine import OpenAIServing
15
16 logger = init_logger(__name__)
@@ -60,32 +59,11 @@ class OpenAIServingChat(OpenAIServing):
59 f"Error in applying chat template from request: {str(e)}")
60 return self.create_error_response(str(e))
61
63 - token_ids, error_check_ret = await self._check_length(request,
64 - prompt=prompt)
65 - if error_check_ret is not None:
66 - return error_check_ret
67 -
62 request_id = f"cmpl-{random_uuid()}"
63 try:
70 - spaces_between_special_tokens = request.spaces_between_special_tokens
71 - sampling_params = SamplingParams(
72 - n=request.n,
73 - presence_penalty=request.presence_penalty,
74 - frequency_penalty=request.frequency_penalty,
75 - repetition_penalty=request.repetition_penalty,
76 - temperature=request.temperature,
77 - top_p=request.top_p,
78 - min_p=request.min_p,
79 - stop=request.stop,
80 - stop_token_ids=request.stop_token_ids,
81 - max_tokens=request.max_tokens,
82 - best_of=request.best_of,
83 - top_k=request.top_k,
84 - ignore_eos=request.ignore_eos,
85 - use_beam_search=request.use_beam_search,
86 - skip_special_tokens=request.skip_special_tokens,
87 - spaces_between_special_tokens=spaces_between_special_tokens,
88 - )
64 + token_ids = self._validate_prompt_and_tokenize(request,
65 + prompt=prompt)
66 + sampling_params = request.to_sampling_params()
67 except ValueError as e:
68 return self.create_error_response(str(e))
69
vllm/entrypoints/openai/serving_completion.py
+210 -215
@@ -1,20 +1,194 @@
1 import time
2 from fastapi import Request
3 -from typing import AsyncGenerator, Optional
3 +from typing import AsyncGenerator, AsyncIterator
4 from vllm.logger import init_logger
5 from vllm.utils import random_uuid
6 from vllm.engine.async_llm_engine import AsyncLLMEngine
7 -from .protocol import (CompletionRequest, CompletionResponse,
8 - CompletionResponseChoice,
9 - CompletionResponseStreamChoice,
10 - CompletionStreamResponse, LogProbs, UsageInfo)
7 +from .protocol import (
8 + CompletionRequest,
9 + CompletionResponse,
10 + CompletionResponseChoice,
11 + CompletionResponseStreamChoice,
12 + CompletionStreamResponse,
13 + LogProbs,
14 + UsageInfo,
15 +)
16 from vllm.outputs import RequestOutput
12 -from vllm.sampling_params import SamplingParams
17 from vllm.entrypoints.openai.serving_engine import OpenAIServing
18
19 logger = init_logger(__name__)
20
21
22 +async def completion_stream_generator(
23 + request: CompletionRequest,
24 + result_generator: AsyncIterator[RequestOutput],
25 + echo_without_generation, create_logprobs_fn, request_id, created_time,
26 + model_name) -> AsyncGenerator[str, None]:
27 + previous_texts = [""] * request.n
28 + previous_num_tokens = [0] * request.n
29 + has_echoed = [False] * request.n
30 +
31 + async for res in result_generator:
32 + # TODO: handle client disconnect for streaming
33 + for output in res.outputs:
34 + i = output.index
35 + delta_text = output.text[len(previous_texts[i]):]
36 + token_ids = output.token_ids[previous_num_tokens[i]:]
37 + if request.logprobs is not None:
38 + top_logprobs = output.logprobs[previous_num_tokens[i]:]
39 + else:
40 + top_logprobs = None
41 + offsets = len(previous_texts[i])
42 + if request.echo and not has_echoed[i]:
43 + if not echo_without_generation:
44 + delta_text = res.prompt + delta_text
45 + token_ids = res.prompt_token_ids + token_ids
46 + if top_logprobs:
47 + top_logprobs = res.prompt_logprobs + top_logprobs
48 + else: # only just return the prompt
49 + delta_text = res.prompt
50 + token_ids = res.prompt_token_ids
51 + if top_logprobs:
52 + top_logprobs = res.prompt_logprobs
53 + has_echoed[i] = True
54 + if request.logprobs is not None:
55 + logprobs = create_logprobs_fn(
56 + token_ids=token_ids,
57 + top_logprobs=top_logprobs,
58 + num_output_top_logprobs=request.logprobs,
59 + initial_text_offset=offsets,
60 + )
61 + else:
62 + logprobs = None
63 + previous_texts[i] = output.text
64 + previous_num_tokens[i] = len(output.token_ids)
65 + finish_reason = output.finish_reason
66 + response_json = CompletionStreamResponse(
67 + id=request_id,
68 + created=created_time,
69 + model=model_name,
70 + choices=[
71 + CompletionResponseStreamChoice(
72 + index=i,
73 + text=delta_text,
74 + logprobs=logprobs,
75 + finish_reason=finish_reason,
76 + )
77 + ]).json(exclude_unset=True, ensure_ascii=False)
78 + yield f"data: {response_json}\n\n"
79 +
80 + if output.finish_reason is not None:
81 + logprobs = LogProbs() if request.logprobs is not None else None
82 + prompt_tokens = len(res.prompt_token_ids)
83 + completion_tokens = len(output.token_ids)
84 + final_usage = UsageInfo(
85 + prompt_tokens=prompt_tokens,
86 + completion_tokens=completion_tokens,
87 + total_tokens=prompt_tokens + completion_tokens,
88 + )
89 + response_json = CompletionStreamResponse(
90 + id=request_id,
91 + created=created_time,
92 + model=model_name,
93 + choices=[
94 + CompletionResponseStreamChoice(
95 + index=i,
96 + text="",
97 + logprobs=logprobs,
98 + finish_reason=output.finish_reason,
99 + )
100 + ],
101 + usage=final_usage,
102 + ).json(exclude_unset=True, ensure_ascii=False)
103 + yield f"data: {response_json}\n\n"
104 +
105 + yield "data: [DONE]\n\n"
106 +
107 +
108 +def parse_prompt_format(prompt) -> tuple[bool, list]:
109 + # get the prompt, openai supports the following
110 + # "a string, array of strings, array of tokens, or array of token arrays."
111 + prompt_is_tokens = False
112 + prompts = [prompt] # case 1: a string
113 + if isinstance(prompt, list):
114 + if len(prompt) == 0:
115 + raise ValueError("please provide at least one prompt")
116 + elif isinstance(prompt[0], str):
117 + prompt_is_tokens = False
118 + prompts = prompt # case 2: array of strings
119 + elif isinstance(prompt[0], int):
120 + prompt_is_tokens = True
121 + prompts = [prompt] # case 3: array of tokens
122 + elif isinstance(prompt[0], list) and isinstance(prompt[0][0], int):
123 + prompt_is_tokens = True
124 + prompts = prompt # case 4: array of token arrays
125 + else:
126 + raise ValueError(
127 + "prompt must be a string, array of strings, array of tokens, or array of token arrays"
128 + )
129 + return prompt_is_tokens, prompts
130 +
131 +
132 +def request_output_to_completion_response(final_res: RequestOutput, request,
133 + echo_without_generation,
134 + create_logprobs_fn, request_id,
135 + created_time,
136 + model_name) -> CompletionResponse:
137 + assert final_res is not None
138 + choices = []
139 + prompt_token_ids = final_res.prompt_token_ids
140 + prompt_logprobs = final_res.prompt_logprobs
141 + prompt_text = final_res.prompt
142 + for output in final_res.outputs:
143 + if request.logprobs is not None:
144 + if not echo_without_generation:
145 + token_ids = output.token_ids
146 + top_logprobs = output.logprobs
147 + if request.echo:
148 + token_ids = prompt_token_ids + token_ids
149 + top_logprobs = prompt_logprobs + top_logprobs
150 + else:
151 + token_ids = prompt_token_ids
152 + top_logprobs = prompt_logprobs
153 + logprobs = create_logprobs_fn(
154 + token_ids=token_ids,
155 + top_logprobs=top_logprobs,
156 + num_output_top_logprobs=request.logprobs,
157 + )
158 + else:
159 + logprobs = None
160 + if not echo_without_generation:
161 + output_text = output.text
162 + if request.echo:
163 + output_text = prompt_text + output_text
164 + else:
165 + output_text = prompt_text
166 + choice_data = CompletionResponseChoice(
167 + index=output.index,
168 + text=output_text,
169 + logprobs=logprobs,
170 + finish_reason=output.finish_reason,
171 + )
172 + choices.append(choice_data)
173 +
174 + num_prompt_tokens = len(final_res.prompt_token_ids)
175 + num_generated_tokens = sum(
176 + len(output.token_ids) for output in final_res.outputs)
177 + usage = UsageInfo(
178 + prompt_tokens=num_prompt_tokens,
179 + completion_tokens=num_generated_tokens,
180 + total_tokens=num_prompt_tokens + num_generated_tokens,
181 + )
182 +
183 + return CompletionResponse(
184 + id=request_id,
185 + created=created_time,
186 + model=model_name,
187 + choices=choices,
188 + usage=usage,
189 + )
190 +
191 +
192 class OpenAIServingCompletion(OpenAIServing):
193
194 def __init__(self, engine: AsyncLLMEngine, served_model: str):
@@ -32,7 +206,6 @@ class OpenAIServingCompletion(OpenAIServing):
206 suffix)
207 - logit_bias (to be supported by vLLM engine)
208 """
35 -
209 error_check_ret = await self._check_model(request)
210 if error_check_ret is not None:
211 return error_check_ret
@@ -40,83 +213,42 @@ class OpenAIServingCompletion(OpenAIServing):
213 # OpenAI API supports echoing the prompt when max_tokens is 0.
214 echo_without_generation = request.echo and request.max_tokens == 0
215
216 + # Return error for unsupported features.
217 if request.suffix is not None:
44 - # The language models we currently support do not support suffix.
218 return self.create_error_response(
219 "suffix is not currently supported")
47 -
220 if request.logit_bias is not None and len(request.logit_bias) > 0:
49 - # TODO: support logit_bias in vLLM engine.
221 return self.create_error_response(
222 "logit_bias is not currently supported")
223
224 model_name = request.model
225 request_id = f"cmpl-{random_uuid()}"
226 + created_time = int(time.monotonic())
227
56 - use_token_ids = False
57 - if isinstance(request.prompt, list):
58 - if len(request.prompt) == 0:
59 - return self.create_error_response(
60 - "please provide at least one prompt")
61 - first_element = request.prompt[0]
62 - if isinstance(first_element, int):
63 - use_token_ids = True
64 - prompt = request.prompt
65 - elif isinstance(first_element, (str, list)):
66 - # TODO: handles multiple prompt case in list[list[int]]
67 - if len(request.prompt) > 1:
68 - return self.create_error_response(
69 - "multiple prompts in a batch is not currently supported"
70 - )
71 - use_token_ids = not isinstance(first_element, str)
72 - prompt = request.prompt[0]
73 - else:
74 - prompt = request.prompt
228 + # Schedule the request and get the result generator.
229 + try:
230 + sampling_params = request.to_sampling_params()
231
76 - if use_token_ids:
77 - _, error_check_ret = await self._check_length(request,
78 - prompt_ids=prompt)
79 - else:
80 - token_ids, error_check_ret = await self._check_length(
81 - request, prompt=prompt)
82 - if error_check_ret is not None:
83 - return error_check_ret
232 + prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
233
85 - created_time = int(time.monotonic())
86 - try:
87 - spaces_between_special_tokens = request.spaces_between_special_tokens
88 - sampling_params = SamplingParams(
89 - n=request.n,
90 - best_of=request.best_of,
91 - presence_penalty=request.presence_penalty,
92 - frequency_penalty=request.frequency_penalty,
93 - repetition_penalty=request.repetition_penalty,
94 - temperature=request.temperature,
95 - top_p=request.top_p,
96 - top_k=request.top_k,
97 - min_p=request.min_p,
98 - stop=request.stop,
99 - stop_token_ids=request.stop_token_ids,
100 - ignore_eos=request.ignore_eos,
101 - max_tokens=request.max_tokens
102 - if not echo_without_generation else 1,
103 - logprobs=request.logprobs,
104 - use_beam_search=request.use_beam_search,
105 - prompt_logprobs=request.logprobs if request.echo else None,
106 - skip_special_tokens=request.skip_special_tokens,
107 - spaces_between_special_tokens=spaces_between_special_tokens,
108 - )
109 - except ValueError as e:
110 - return self.create_error_response(str(e))
234 + if len(prompts) > 1:
235 + raise ValueError(
236 + "Batching in completion API is not supported.")
237 + prompt = prompts[0]
238 +
239 + if prompt_is_tokens:
240 + input_ids = self._validate_prompt_and_tokenize(
241 + request, prompt_ids=prompt)
242 + else:
243 + input_ids = self._validate_prompt_and_tokenize(request,
244 + prompt=prompt)
245
112 - if use_token_ids:
246 result_generator = self.engine.generate(None,
247 sampling_params,
248 request_id,
116 - prompt_token_ids=prompt)
117 - else:
118 - result_generator = self.engine.generate(prompt, sampling_params,
119 - request_id, token_ids)
249 + prompt_token_ids=input_ids)
250 + except ValueError as e:
251 + return self.create_error_response(str(e))
252
253 # Similar to the OpenAI API, when n != best_of, we do not stream the
254 # results. In addition, we do not stream the results when use beam search.
@@ -124,101 +256,13 @@ class OpenAIServingCompletion(OpenAIServing):
256 and (request.best_of is None or request.n == request.best_of)
257 and not request.use_beam_search)
258
127 - def create_stream_response_json(
128 - index: int,
129 - text: str,
130 - logprobs: Optional[LogProbs] = None,
131 - finish_reason: Optional[str] = None,
132 - usage: Optional[UsageInfo] = None,
133 - ) -> str:
134 - choice_data = CompletionResponseStreamChoice(
135 - index=index,
136 - text=text,
137 - logprobs=logprobs,
138 - finish_reason=finish_reason,
139 - )
140 - response = CompletionStreamResponse(
141 - id=request_id,
142 - created=created_time,
143 - model=model_name,
144 - choices=[choice_data],
145 - )
146 - if usage is not None:
147 - response.usage = usage
148 - response_json = response.json(exclude_unset=True,
149 - ensure_ascii=False)
150 -
151 - return response_json
152 -
153 - async def completion_stream_generator() -> AsyncGenerator[str, None]:
154 - previous_texts = [""] * request.n
155 - previous_num_tokens = [0] * request.n
156 - has_echoed = [False] * request.n
157 - async for res in result_generator:
158 - res: RequestOutput
159 - for output in res.outputs:
160 - i = output.index
161 - delta_text = output.text[len(previous_texts[i]):]
162 - token_ids = output.token_ids[previous_num_tokens[i]:]
163 - if request.logprobs is not None:
164 - top_logprobs = output.logprobs[previous_num_tokens[i]:]
165 - else:
166 - top_logprobs = None
167 - offsets = len(previous_texts[i])
168 - if request.echo and not has_echoed[i]:
169 - if not echo_without_generation:
170 - delta_text = res.prompt + delta_text
171 - token_ids = res.prompt_token_ids + token_ids
172 - if top_logprobs:
173 - top_logprobs = res.prompt_logprobs + top_logprobs
174 - else: # only just return the prompt
175 - delta_text = res.prompt
176 - token_ids = res.prompt_token_ids
177 - if top_logprobs:
178 - top_logprobs = res.prompt_logprobs
179 - has_echoed[i] = True
180 - if request.logprobs is not None:
181 - logprobs = self._create_logprobs(
182 - token_ids=token_ids,
183 - top_logprobs=top_logprobs,
184 - num_output_top_logprobs=request.logprobs,
185 - initial_text_offset=offsets,
186 - )
187 - else:
188 - logprobs = None
189 - previous_texts[i] = output.text
190 - previous_num_tokens[i] = len(output.token_ids)
191 - finish_reason = output.finish_reason
192 - response_json = create_stream_response_json(
193 - index=i,
194 - text=delta_text,
195 - logprobs=logprobs,
196 - finish_reason=finish_reason,
197 - )
198 - yield f"data: {response_json}\n\n"
199 - if output.finish_reason is not None:
200 - logprobs = (LogProbs()
201 - if request.logprobs is not None else None)
202 - prompt_tokens = len(res.prompt_token_ids)
203 - completion_tokens = len(output.token_ids)
204 - final_usage = UsageInfo(
205 - prompt_tokens=prompt_tokens,
206 - completion_tokens=completion_tokens,
207 - total_tokens=prompt_tokens + completion_tokens,
208 - )
209 - response_json = create_stream_response_json(
210 - index=i,
211 - text="",
212 - logprobs=logprobs,
213 - finish_reason=output.finish_reason,
214 - usage=final_usage,
215 - )
216 - yield f"data: {response_json}\n\n"
217 - yield "data: [DONE]\n\n"
218 -
259 # Streaming response
260 if stream:
221 - return completion_stream_generator()
261 + return completion_stream_generator(request, result_generator,
262 + echo_without_generation,
263 + self._create_logprobs,
264 + request_id, created_time,
265 + model_name)
266
267 # Non-streaming response
268 final_res: RequestOutput = None
@@ -228,62 +272,13 @@ class OpenAIServingCompletion(OpenAIServing):
272 await self.engine.abort(request_id)
273 return self.create_error_response("Client disconnected")
274 final_res = res
231 - assert final_res is not None
232 - choices = []
233 - prompt_token_ids = final_res.prompt_token_ids
234 - prompt_logprobs = final_res.prompt_logprobs
235 - prompt_text = final_res.prompt
236 - for output in final_res.outputs:
237 - if request.logprobs is not None:
238 - if not echo_without_generation:
239 - token_ids = output.token_ids
240 - top_logprobs = output.logprobs
241 - if request.echo:
242 - token_ids = prompt_token_ids + token_ids
243 - top_logprobs = prompt_logprobs + top_logprobs
244 - else:
245 - token_ids = prompt_token_ids
246 - top_logprobs = prompt_logprobs
247 - logprobs = self._create_logprobs(
248 - token_ids=token_ids,
249 - top_logprobs=top_logprobs,
250 - num_output_top_logprobs=request.logprobs,
251 - )
252 - else:
253 - logprobs = None
254 - if not echo_without_generation:
255 - output_text = output.text
256 - if request.echo:
257 - output_text = prompt_text + output_text
258 - else:
259 - output_text = prompt_text
260 - choice_data = CompletionResponseChoice(
261 - index=output.index,
262 - text=output_text,
263 - logprobs=logprobs,
264 - finish_reason=output.finish_reason,
265 - )
266 - choices.append(choice_data)
267 -
268 - num_prompt_tokens = len(final_res.prompt_token_ids)
269 - num_generated_tokens = sum(
270 - len(output.token_ids) for output in final_res.outputs)
271 - usage = UsageInfo(
272 - prompt_tokens=num_prompt_tokens,
273 - completion_tokens=num_generated_tokens,
274 - total_tokens=num_prompt_tokens + num_generated_tokens,
275 - )
276 - response = CompletionResponse(
277 - id=request_id,
278 - created=created_time,
279 - model=model_name,
280 - choices=choices,
281 - usage=usage,
282 - )
275 + response = request_output_to_completion_response(
276 + final_res, request, echo_without_generation, self._create_logprobs,
277 + request_id, created_time, model_name)
278
279 + # When user requests streaming but we don't stream, we still need to
280 + # return a streaming response with a single event.
281 if request.stream:
285 - # When user requests streaming but we don't stream, we still need to
286 - # return a streaming response with a single event.
282 response_json = response.json(ensure_ascii=False)
283
284 async def fake_stream_generator() -> AsyncGenerator[str, None]:
vllm/entrypoints/openai/serving_engine.py
+15 -12
@@ -1,6 +1,6 @@
1 import asyncio
2 from http import HTTPStatus
3 -from typing import Dict, List, Optional, Tuple, Union
3 +from typing import Dict, List, Optional, Union
4 from vllm.logger import init_logger
5 from vllm.transformers_utils.tokenizer import get_tokenizer
6 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -104,27 +104,30 @@ class OpenAIServing:
104 err_type="NotFoundError",
105 status_code=HTTPStatus.NOT_FOUND)
106
107 - async def _check_length(
108 - self,
109 - request: Union[ChatCompletionRequest, CompletionRequest],
110 - prompt: Optional[str] = None,
111 - prompt_ids: Optional[List[int]] = None
112 - ) -> Tuple[List[int], Optional[ErrorResponse]]:
113 - assert (not (prompt is None and prompt_ids is None)
114 - and not (prompt is not None and prompt_ids is not None)
115 - ), "Either prompt or prompt_ids should be provided."
107 + def _validate_prompt_and_tokenize(
108 + self,
109 + request: Union[ChatCompletionRequest, CompletionRequest],
110 + prompt: Optional[str] = None,
111 + prompt_ids: Optional[List[int]] = None) -> List[int]:
112 + if not (prompt or prompt_ids):
113 + raise ValueError("Either prompt or prompt_ids should be provided.")
114 + if (prompt and prompt_ids):
115 + raise ValueError(
116 + "Only one of prompt or prompt_ids should be provided.")
117 +
118 input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
119 prompt).input_ids
120 token_num = len(input_ids)
121
122 if request.max_tokens is None:
123 request.max_tokens = self.max_model_len - token_num
124 +
125 if token_num + request.max_tokens > self.max_model_len:
123 - return input_ids, self.create_error_response(
126 + raise ValueError(
127 f"This model's maximum context length is {self.max_model_len} tokens. "
128 f"However, you requested {request.max_tokens + token_num} tokens "
129 f"({token_num} in the messages, "
130 f"{request.max_tokens} in the completion). "
131 f"Please reduce the length of the messages or completion.", )
132 else:
130 - return input_ids, None
133 + return input_ids
vllm/model_executor/weight_utils.py
+1 -1
@@ -163,7 +163,7 @@ def prepare_hf_model_weights(
163 use_safetensors = True
164 break
165
166 - logger.info(f"Downloading model weights {allow_patterns}")
166 + logger.info(f"Using model weights format {allow_patterns}")
167 # Use file lock to prevent multiple processes from
168 # downloading the same model weights at the same time.
169 with get_lock(model_name_or_path, cache_dir):