A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support logit bias for OpenAI API (#3027)
Dylan Hawk committed
Feb 26, 2024 at 19:51 UTC
e0ade06d6305cf84b41c1962cdd9dfdbfee16ac9
4 files changed
+83
-12
tests/entrypoints/test_openai_server.py
+48
@@ -9,6 +9,8 @@ import ray # using Ray for overall ease of process management, parallel request
9
import openai # use the official client for correctness check
10
from huggingface_hub import snapshot_download # downloading lora to test lora requests
11
12
+from vllm.transformers_utils.tokenizer import get_tokenizer
13
+
14
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
15
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
16
LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
@@ -310,5 +312,51 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
312
assert texts[0] == texts[1]
313
314
315
+async def test_logits_bias(server, client: openai.AsyncOpenAI):
316
+ prompt = "Hello, my name is"
317
+ max_tokens = 5
318
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
319
+
320
+ # Test exclusive selection
321
+ token_id = 1000
322
+ completion = await client.completions.create(
323
+ model=MODEL_NAME,
324
+ prompt=prompt,
325
+ max_tokens=max_tokens,
326
+ temperature=0.0,
327
+ logit_bias={str(token_id): 100},
328
+ )
329
+ assert completion.choices[0].text is not None and len(
330
+ completion.choices[0].text) >= 5
331
+ response_tokens = tokenizer(completion.choices[0].text,
332
+ add_special_tokens=False)["input_ids"]
333
+ expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
334
+ add_special_tokens=False)["input_ids"]
335
+ assert all([
336
+ response == expected
337
+ for response, expected in zip(response_tokens, expected_tokens)
338
+ ])
339
+
340
+ # Test ban
341
+ completion = await client.completions.create(
342
+ model=MODEL_NAME,
343
+ prompt=prompt,
344
+ max_tokens=max_tokens,
345
+ temperature=0.0,
346
+ )
347
+ response_tokens = tokenizer(completion.choices[0].text,
348
+ add_special_tokens=False)["input_ids"]
349
+ first_response = completion.choices[0].text
350
+ completion = await client.completions.create(
351
+ model=MODEL_NAME,
352
+ prompt=prompt,
353
+ max_tokens=max_tokens,
354
+ temperature=0.0,
355
+ logit_bias={str(token): -100
356
+ for token in response_tokens},
357
+ )
358
+ assert first_response != completion.choices[0].text
359
+
360
+
361
if __name__ == "__main__":
362
pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+33
@@ -8,6 +8,8 @@ from pydantic import BaseModel, Field
8
from vllm.utils import random_uuid
9
from vllm.sampling_params import SamplingParams
10
11
+import torch
12
+
13
14
class ErrorResponse(BaseModel):
15
object: str = "error"
@@ -88,6 +90,21 @@ class ChatCompletionRequest(BaseModel):
90
def to_sampling_params(self) -> SamplingParams:
91
if self.logprobs and not self.top_logprobs:
92
raise ValueError("Top logprobs must be set when logprobs is.")
93
+
94
+ logits_processors = None
95
+ if self.logit_bias:
96
+
97
+ def logit_bias_logits_processor(
98
+ token_ids: List[int],
99
+ logits: torch.Tensor) -> torch.Tensor:
100
+ for token_id, bias in self.logit_bias.items():
101
+ # Clamp the bias between -100 and 100 per OpenAI API spec
102
+ bias = min(100, max(-100, bias))
103
+ logits[int(token_id)] += bias
104
+ return logits
105
+
106
+ logits_processors = [logit_bias_logits_processor]
107
+
108
return SamplingParams(
109
n=self.n,
110
presence_penalty=self.presence_penalty,
@@ -111,6 +128,7 @@ class ChatCompletionRequest(BaseModel):
128
spaces_between_special_tokens=self.spaces_between_special_tokens,
129
include_stop_str_in_output=self.include_stop_str_in_output,
130
length_penalty=self.length_penalty,
131
+ logits_processors=logits_processors,
132
)
133
134
@@ -149,6 +167,20 @@ class CompletionRequest(BaseModel):
167
def to_sampling_params(self):
168
echo_without_generation = self.echo and self.max_tokens == 0
169
170
+ logits_processors = None
171
+ if self.logit_bias:
172
+
173
+ def logit_bias_logits_processor(
174
+ token_ids: List[int],
175
+ logits: torch.Tensor) -> torch.Tensor:
176
+ for token_id, bias in self.logit_bias.items():
177
+ # Clamp the bias between -100 and 100 per OpenAI API spec
178
+ bias = min(100, max(-100, bias))
179
+ logits[int(token_id)] += bias
180
+ return logits
181
+
182
+ logits_processors = [logit_bias_logits_processor]
183
+
184
return SamplingParams(
185
n=self.n,
186
best_of=self.best_of,
@@ -172,6 +204,7 @@ class CompletionRequest(BaseModel):
204
spaces_between_special_tokens=(self.spaces_between_special_tokens),
205
include_stop_str_in_output=self.include_stop_str_in_output,
206
length_penalty=self.length_penalty,
207
+ logits_processors=logits_processors,
208
)
209
210
vllm/entrypoints/openai/serving_chat.py
+1
-7
@@ -39,19 +39,13 @@ class OpenAIServingChat(OpenAIServing):
39
See https://platform.openai.com/docs/api-reference/chat/create
40
for the API specification. This API mimics the OpenAI ChatCompletion API.
41
42
- NOTE: Currently we do not support the following features:
42
+ NOTE: Currently we do not support the following feature:
43
- function_call (Users should implement this by themselves)
44
- - logit_bias (to be supported by vLLM engine)
44
"""
45
error_check_ret = await self._check_model(request)
46
if error_check_ret is not None:
47
return error_check_ret
48
50
- if request.logit_bias is not None and len(request.logit_bias) > 0:
51
- # TODO: support logit_bias in vLLM engine.
52
- return self.create_error_response(
53
- "logit_bias is not currently supported")
54
-
49
try:
50
prompt = self.tokenizer.apply_chat_template(
51
conversation=request.messages,
vllm/entrypoints/openai/serving_completion.py
+1
-5
@@ -264,10 +264,9 @@ class OpenAIServingCompletion(OpenAIServing):
264
See https://platform.openai.com/docs/api-reference/completions/create
265
for the API specification. This API mimics the OpenAI Completion API.
266
267
- NOTE: Currently we do not support the following features:
267
+ NOTE: Currently we do not support the following feature:
268
- suffix (the language models we currently support do not support
269
suffix)
270
- - logit_bias (to be supported by vLLM engine)
270
"""
271
error_check_ret = await self._check_model(request)
272
if error_check_ret is not None:
@@ -277,9 +276,6 @@ class OpenAIServingCompletion(OpenAIServing):
276
if request.suffix is not None:
277
return self.create_error_response(
278
"suffix is not currently supported")
280
- if request.logit_bias is not None and len(request.logit_bias) > 0:
281
- return self.create_error_response(
282
- "logit_bias is not currently supported")
279
280
model_name = request.model
281
request_id = f"cmpl-{random_uuid()}"