A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Fix echo/logprob OpenAI completion bug (#3441)
Co-authored-by: Dylan Hawk <dylanwawk@gmail.com>
Dylan Hawk committed
Apr 11, 2024 at 15:15 UTC
95e7d4a97cd64f8c6dc226ec0bbceebef6458701
4 files changed
+73
-29
tests/entrypoints/test_openai_server.py
+31
@@ -742,5 +742,36 @@ number: "1" | "2"
742
assert content.strip() == ground_truth
743
744
745
+@pytest.mark.parametrize(
746
+ # first test base model, then test loras
747
+ "model_name",
748
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
749
+)
750
+async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
751
+ model_name: str):
752
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
753
+ # test using text and token IDs
754
+ for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
755
+ completion = await client.completions.create(model=model_name,
756
+ prompt=prompt,
757
+ max_tokens=5,
758
+ temperature=0.0,
759
+ echo=True,
760
+ logprobs=1)
761
+
762
+ prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
763
+ list) else prompt
764
+ assert (completion.choices[0].text is not None
765
+ and re.search(r"^" + prompt_text, completion.choices[0].text))
766
+ logprobs = completion.choices[0].logprobs
767
+ assert logprobs is not None
768
+ assert len(logprobs.text_offset) > 5
769
+ assert (len(logprobs.token_logprobs) > 5
770
+ and logprobs.token_logprobs[0] is None)
771
+ assert (len(logprobs.top_logprobs) > 5
772
+ and logprobs.top_logprobs[0] is None)
773
+ assert len(logprobs.tokens) > 5
774
+
775
+
776
if __name__ == "__main__":
777
pytest.main([__file__])
vllm/entrypoints/openai/serving_chat.py
+5
-4
@@ -63,8 +63,9 @@ class OpenAIServingChat(OpenAIServing):
63
64
request_id = f"cmpl-{random_uuid()}"
65
try:
66
- token_ids = self._validate_prompt_and_tokenize(request,
67
- prompt=prompt)
66
+ # Tokenize/detokenize depending on prompt format (string/token list)
67
+ prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
68
+ request, prompt=prompt)
69
sampling_params = request.to_sampling_params()
70
lora_request = self._maybe_get_lora(request)
71
guided_decode_logits_processor = (
@@ -78,8 +79,8 @@ class OpenAIServingChat(OpenAIServing):
79
except ValueError as e:
80
return self.create_error_response(str(e))
81
81
- result_generator = self.engine.generate(prompt, sampling_params,
82
- request_id, token_ids,
82
+ result_generator = self.engine.generate(prompt_text, sampling_params,
83
+ request_id, prompt_ids,
84
lora_request)
85
# Streaming response
86
if request.stream:
vllm/entrypoints/openai/serving_completion.py
+10
-5
@@ -136,23 +136,24 @@ class OpenAIServingCompletion(OpenAIServing):
136
137
for i, prompt in enumerate(prompts):
138
if prompt_is_tokens:
139
- input_ids = self._validate_prompt_and_tokenize(
139
+ prompt_formats = self._validate_prompt_and_tokenize(
140
request,
141
prompt_ids=prompt,
142
truncate_prompt_tokens=sampling_params.
143
truncate_prompt_tokens)
144
else:
145
- input_ids = self._validate_prompt_and_tokenize(
145
+ prompt_formats = self._validate_prompt_and_tokenize(
146
request,
147
prompt=prompt,
148
truncate_prompt_tokens=sampling_params.
149
truncate_prompt_tokens)
150
+ prompt_ids, prompt_text = prompt_formats
151
152
generators.append(
152
- self.engine.generate(prompt,
153
+ self.engine.generate(prompt_text,
154
sampling_params,
155
f"{request_id}-{i}",
155
- prompt_token_ids=input_ids,
156
+ prompt_token_ids=prompt_ids,
157
lora_request=lora_request))
158
except ValueError as e:
159
# TODO: Use a vllm-specific Validation Error
@@ -326,7 +327,8 @@ class OpenAIServingCompletion(OpenAIServing):
327
output_text = prompt_text
328
elif request.echo and request.max_tokens > 0:
329
token_ids = prompt_token_ids + output.token_ids
329
- top_logprobs = prompt_logprobs + output.logprobs
330
+ top_logprobs = (prompt_logprobs + output.logprobs
331
+ if request.logprobs else None)
332
output_text = prompt_text + output.text
333
else:
334
token_ids = output.token_ids
@@ -334,6 +336,9 @@ class OpenAIServingCompletion(OpenAIServing):
336
output_text = output.text
337
338
if request.logprobs is not None:
339
+ assert top_logprobs is not None, (
340
+ "top_logprobs must be provided when logprobs "
341
+ "is requested")
342
logprobs = self._create_logprobs(
343
token_ids=token_ids,
344
top_logprobs=top_logprobs,
vllm/entrypoints/openai/serving_engine.py
+27
-20
@@ -2,7 +2,7 @@ import asyncio
2
import json
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
-from typing import Dict, List, Optional, Union
5
+from typing import Dict, List, Optional, Tuple, Union
6
7
from pydantic import conint
8
@@ -99,27 +99,32 @@ class OpenAIServing:
99
last_token_len = 0
100
if num_output_top_logprobs:
101
logprobs.top_logprobs = []
102
+
103
for i, token_id in enumerate(token_ids):
104
step_top_logprobs = top_logprobs[i]
104
- if step_top_logprobs is not None:
105
- token_logprob = step_top_logprobs[token_id].logprob
105
+ if step_top_logprobs is None:
106
+ token = self.tokenizer.decode(token_id)
107
+ logprobs.tokens.append(token)
108
+ logprobs.token_logprobs.append(None)
109
+ logprobs.top_logprobs.append(None)
110
else:
107
- token_logprob = None
108
- token = step_top_logprobs[token_id].decoded_token
109
- logprobs.tokens.append(token)
110
- logprobs.token_logprobs.append(token_logprob)
111
+ token_logprob = step_top_logprobs[token_id].logprob
112
+ token = step_top_logprobs[token_id].decoded_token
113
+ logprobs.tokens.append(token)
114
+ logprobs.token_logprobs.append(token_logprob)
115
+
116
+ if num_output_top_logprobs:
117
+ logprobs.top_logprobs.append({
118
+ p.decoded_token: p.logprob
119
+ for i, p in step_top_logprobs.items()
120
+ } if step_top_logprobs else None)
121
+
122
if len(logprobs.text_offset) == 0:
123
logprobs.text_offset.append(initial_text_offset)
124
else:
125
logprobs.text_offset.append(logprobs.text_offset[-1] +
126
last_token_len)
127
last_token_len = len(token)
117
-
118
- if num_output_top_logprobs:
119
- logprobs.top_logprobs.append({
120
- p.decoded_token: p.logprob
121
- for i, p in step_top_logprobs.items()
122
- } if step_top_logprobs else None)
128
return logprobs
129
130
def create_error_response(
@@ -164,12 +169,12 @@ class OpenAIServing:
169
raise ValueError("The model `{request.model}` does not exist.")
170
171
def _validate_prompt_and_tokenize(
167
- self,
168
- request: Union[ChatCompletionRequest, CompletionRequest],
169
- prompt: Optional[str] = None,
170
- prompt_ids: Optional[List[int]] = None,
171
- truncate_prompt_tokens: Optional[conint(ge=1)] = None
172
- ) -> List[int]:
172
+ self,
173
+ request: Union[ChatCompletionRequest, CompletionRequest],
174
+ prompt: Optional[str] = None,
175
+ prompt_ids: Optional[List[int]] = None,
176
+ truncate_prompt_tokens: Optional[conint(ge=1)] = None
177
+ ) -> Tuple[List[int], str]:
178
if not (prompt or prompt_ids):
179
raise ValueError("Either prompt or prompt_ids should be provided.")
180
if (prompt and prompt_ids):
@@ -187,6 +192,8 @@ class OpenAIServing:
192
else:
193
input_ids = prompt_ids
194
195
+ input_text = prompt if prompt is not None else self.tokenizer.decode(
196
+ prompt_ids)
197
token_num = len(input_ids)
198
199
if request.max_tokens is None:
@@ -201,4 +208,4 @@ class OpenAIServing:
208
f"{request.max_tokens} in the completion). "
209
f"Please reduce the length of the messages or completion.", )
210
else:
204
- return input_ids
211
+ return input_ids, input_text