@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95e7d4a97c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Fix echo/logprob OpenAI completion bug (#3441)

Co-authored-by: Dylan Hawk <dylanwawk@gmail.com>

Dylan Hawk committed Apr 11, 2024 at 15:15 UTC 95e7d4a97cd64f8c6dc226ec0bbceebef6458701
4 files changed +73 -29
tests/entrypoints/test_openai_server.py
+31
@@ -742,5 +742,36 @@ number: "1" | "2"
742 assert content.strip() == ground_truth
743
744
745 +@pytest.mark.parametrize(
746 + # first test base model, then test loras
747 + "model_name",
748 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
749 +)
750 +async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
751 + model_name: str):
752 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
753 + # test using text and token IDs
754 + for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
755 + completion = await client.completions.create(model=model_name,
756 + prompt=prompt,
757 + max_tokens=5,
758 + temperature=0.0,
759 + echo=True,
760 + logprobs=1)
761 +
762 + prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
763 + list) else prompt
764 + assert (completion.choices[0].text is not None
765 + and re.search(r"^" + prompt_text, completion.choices[0].text))
766 + logprobs = completion.choices[0].logprobs
767 + assert logprobs is not None
768 + assert len(logprobs.text_offset) > 5
769 + assert (len(logprobs.token_logprobs) > 5
770 + and logprobs.token_logprobs[0] is None)
771 + assert (len(logprobs.top_logprobs) > 5
772 + and logprobs.top_logprobs[0] is None)
773 + assert len(logprobs.tokens) > 5
774 +
775 +
776 if __name__ == "__main__":
777 pytest.main([__file__])
vllm/entrypoints/openai/serving_chat.py
+5 -4
@@ -63,8 +63,9 @@ class OpenAIServingChat(OpenAIServing):
63
64 request_id = f"cmpl-{random_uuid()}"
65 try:
66 - token_ids = self._validate_prompt_and_tokenize(request,
67 - prompt=prompt)
66 + # Tokenize/detokenize depending on prompt format (string/token list)
67 + prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
68 + request, prompt=prompt)
69 sampling_params = request.to_sampling_params()
70 lora_request = self._maybe_get_lora(request)
71 guided_decode_logits_processor = (
@@ -78,8 +79,8 @@ class OpenAIServingChat(OpenAIServing):
79 except ValueError as e:
80 return self.create_error_response(str(e))
81
81 - result_generator = self.engine.generate(prompt, sampling_params,
82 - request_id, token_ids,
82 + result_generator = self.engine.generate(prompt_text, sampling_params,
83 + request_id, prompt_ids,
84 lora_request)
85 # Streaming response
86 if request.stream:
vllm/entrypoints/openai/serving_completion.py
+10 -5
@@ -136,23 +136,24 @@ class OpenAIServingCompletion(OpenAIServing):
136
137 for i, prompt in enumerate(prompts):
138 if prompt_is_tokens:
139 - input_ids = self._validate_prompt_and_tokenize(
139 + prompt_formats = self._validate_prompt_and_tokenize(
140 request,
141 prompt_ids=prompt,
142 truncate_prompt_tokens=sampling_params.
143 truncate_prompt_tokens)
144 else:
145 - input_ids = self._validate_prompt_and_tokenize(
145 + prompt_formats = self._validate_prompt_and_tokenize(
146 request,
147 prompt=prompt,
148 truncate_prompt_tokens=sampling_params.
149 truncate_prompt_tokens)
150 + prompt_ids, prompt_text = prompt_formats
151
152 generators.append(
152 - self.engine.generate(prompt,
153 + self.engine.generate(prompt_text,
154 sampling_params,
155 f"{request_id}-{i}",
155 - prompt_token_ids=input_ids,
156 + prompt_token_ids=prompt_ids,
157 lora_request=lora_request))
158 except ValueError as e:
159 # TODO: Use a vllm-specific Validation Error
@@ -326,7 +327,8 @@ class OpenAIServingCompletion(OpenAIServing):
327 output_text = prompt_text
328 elif request.echo and request.max_tokens > 0:
329 token_ids = prompt_token_ids + output.token_ids
329 - top_logprobs = prompt_logprobs + output.logprobs
330 + top_logprobs = (prompt_logprobs + output.logprobs
331 + if request.logprobs else None)
332 output_text = prompt_text + output.text
333 else:
334 token_ids = output.token_ids
@@ -334,6 +336,9 @@ class OpenAIServingCompletion(OpenAIServing):
336 output_text = output.text
337
338 if request.logprobs is not None:
339 + assert top_logprobs is not None, (
340 + "top_logprobs must be provided when logprobs "
341 + "is requested")
342 logprobs = self._create_logprobs(
343 token_ids=token_ids,
344 top_logprobs=top_logprobs,
vllm/entrypoints/openai/serving_engine.py
+27 -20
@@ -2,7 +2,7 @@ import asyncio
2 import json
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 -from typing import Dict, List, Optional, Union
5 +from typing import Dict, List, Optional, Tuple, Union
6
7 from pydantic import conint
8
@@ -99,27 +99,32 @@ class OpenAIServing:
99 last_token_len = 0
100 if num_output_top_logprobs:
101 logprobs.top_logprobs = []
102 +
103 for i, token_id in enumerate(token_ids):
104 step_top_logprobs = top_logprobs[i]
104 - if step_top_logprobs is not None:
105 - token_logprob = step_top_logprobs[token_id].logprob
105 + if step_top_logprobs is None:
106 + token = self.tokenizer.decode(token_id)
107 + logprobs.tokens.append(token)
108 + logprobs.token_logprobs.append(None)
109 + logprobs.top_logprobs.append(None)
110 else:
107 - token_logprob = None
108 - token = step_top_logprobs[token_id].decoded_token
109 - logprobs.tokens.append(token)
110 - logprobs.token_logprobs.append(token_logprob)
111 + token_logprob = step_top_logprobs[token_id].logprob
112 + token = step_top_logprobs[token_id].decoded_token
113 + logprobs.tokens.append(token)
114 + logprobs.token_logprobs.append(token_logprob)
115 +
116 + if num_output_top_logprobs:
117 + logprobs.top_logprobs.append({
118 + p.decoded_token: p.logprob
119 + for i, p in step_top_logprobs.items()
120 + } if step_top_logprobs else None)
121 +
122 if len(logprobs.text_offset) == 0:
123 logprobs.text_offset.append(initial_text_offset)
124 else:
125 logprobs.text_offset.append(logprobs.text_offset[-1] +
126 last_token_len)
127 last_token_len = len(token)
117 -
118 - if num_output_top_logprobs:
119 - logprobs.top_logprobs.append({
120 - p.decoded_token: p.logprob
121 - for i, p in step_top_logprobs.items()
122 - } if step_top_logprobs else None)
128 return logprobs
129
130 def create_error_response(
@@ -164,12 +169,12 @@ class OpenAIServing:
169 raise ValueError("The model `{request.model}` does not exist.")
170
171 def _validate_prompt_and_tokenize(
167 - self,
168 - request: Union[ChatCompletionRequest, CompletionRequest],
169 - prompt: Optional[str] = None,
170 - prompt_ids: Optional[List[int]] = None,
171 - truncate_prompt_tokens: Optional[conint(ge=1)] = None
172 - ) -> List[int]:
172 + self,
173 + request: Union[ChatCompletionRequest, CompletionRequest],
174 + prompt: Optional[str] = None,
175 + prompt_ids: Optional[List[int]] = None,
176 + truncate_prompt_tokens: Optional[conint(ge=1)] = None
177 + ) -> Tuple[List[int], str]:
178 if not (prompt or prompt_ids):
179 raise ValueError("Either prompt or prompt_ids should be provided.")
180 if (prompt and prompt_ids):
@@ -187,6 +192,8 @@ class OpenAIServing:
192 else:
193 input_ids = prompt_ids
194
195 + input_text = prompt if prompt is not None else self.tokenizer.decode(
196 + prompt_ids)
197 token_num = len(input_ids)
198
199 if request.max_tokens is None:
@@ -201,4 +208,4 @@ class OpenAIServing:
208 f"{request.max_tokens} in the completion). "
209 f"Please reduce the length of the messages or completion.", )
210 else:
204 - return input_ids
211 + return input_ids, input_text