A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] New `allowed_token_ids` decoding request parameter (#6753)
Nick Hill committed
Jul 29, 2024 at 16:37 UTC
9f69d8245a695918cb80c218a83c42e9fd409687
5 files changed
+114
-46
tests/entrypoints/openai/test_completion.py
+22
@@ -541,6 +541,28 @@ async def test_logits_bias(client: openai.AsyncOpenAI):
541
assert first_response != completion.choices[0].text
542
543
544
+@pytest.mark.asyncio
545
+async def test_allowed_token_ids(client: openai.AsyncOpenAI):
546
+ prompt = "Hello, my name is"
547
+ max_tokens = 1
548
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
549
+
550
+ # Test exclusive selection
551
+ allowed_ids = [21555, 21557, 21558]
552
+ completion = await client.completions.create(
553
+ model=MODEL_NAME,
554
+ prompt=prompt,
555
+ max_tokens=max_tokens,
556
+ temperature=0.0,
557
+ seed=42,
558
+ extra_body=dict(allowed_token_ids=allowed_ids),
559
+ logprobs=1,
560
+ )
561
+ response_tokens = completion.choices[0].logprobs.tokens
562
+ assert len(response_tokens) == 1
563
+ assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
564
+
565
+
566
@pytest.mark.asyncio
567
@pytest.mark.parametrize("guided_decoding_backend",
568
["outlines", "lm-format-enforcer"])
vllm/entrypoints/openai/logits_processors.py
new
+74
@@ -0,0 +1,74 @@
1
+from functools import lru_cache
2
+from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3
+
4
+import torch
5
+from transformers import PreTrainedTokenizer
6
+
7
+from vllm.sampling_params import LogitsProcessor
8
+
9
+
10
+class AllowedTokenIdsLogitsProcessor:
11
+ """Logits processor for constraining generated tokens to a
12
+ specific set of token ids."""
13
+
14
+ def __init__(self, allowed_ids: Iterable[int]):
15
+ self.allowed_ids: Optional[List[int]] = list(allowed_ids)
16
+ self.mask: Optional[torch.Tensor] = None
17
+
18
+ def __call__(self, token_ids: List[int],
19
+ logits: torch.Tensor) -> torch.Tensor:
20
+ if self.mask is None:
21
+ self.mask = torch.ones((logits.shape[-1], ),
22
+ dtype=torch.bool,
23
+ device=logits.device)
24
+ self.mask[self.allowed_ids] = False
25
+ self.allowed_ids = None
26
+ logits.masked_fill_(self.mask, float("-inf"))
27
+ return logits
28
+
29
+
30
+@lru_cache(maxsize=32)
31
+def _get_allowed_token_ids_logits_processor(
32
+ allowed_token_ids: FrozenSet[int],
33
+ vocab_size: int,
34
+) -> LogitsProcessor:
35
+ if not allowed_token_ids:
36
+ raise ValueError("Empty allowed_token_ids provided")
37
+ if not all(0 <= tid < vocab_size for tid in allowed_token_ids):
38
+ raise ValueError("allowed_token_ids contains "
39
+ "out-of-vocab token id")
40
+ return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41
+
42
+
43
+def get_logits_processors(
44
+ logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
45
+ allowed_token_ids: Optional[List[int]],
46
+ tokenizer: PreTrainedTokenizer) -> List[LogitsProcessor]:
47
+ logits_processors = []
48
+ if logit_bias:
49
+ try:
50
+ # Convert token_id to integer
51
+ # Clamp the bias between -100 and 100 per OpenAI API spec
52
+ clamped_logit_bias: Dict[int, float] = {
53
+ int(token_id): min(100.0, max(-100.0, bias))
54
+ for token_id, bias in logit_bias.items()
55
+ }
56
+ except ValueError as exc:
57
+ raise ValueError(
58
+ "Found token_id in logit_bias that is not "
59
+ "an integer or string representing an integer") from exc
60
+
61
+ def logit_bias_logits_processor(token_ids: List[int],
62
+ logits: torch.Tensor) -> torch.Tensor:
63
+ for token_id, bias in clamped_logit_bias.items():
64
+ logits[token_id] += bias
65
+ return logits
66
+
67
+ logits_processors.append(logit_bias_logits_processor)
68
+
69
+ if allowed_token_ids is not None:
70
+ logits_processors.append(
71
+ _get_allowed_token_ids_logits_processor(
72
+ frozenset(allowed_token_ids), tokenizer.vocab_size))
73
+
74
+ return logits_processors
vllm/entrypoints/openai/protocol.py
+16
-44
@@ -5,9 +5,11 @@ from typing import Any, Dict, List, Literal, Optional, Union
5
6
import torch
7
from pydantic import BaseModel, ConfigDict, Field, model_validator
8
+from transformers import PreTrainedTokenizer
9
from typing_extensions import Annotated
10
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
+from vllm.entrypoints.openai.logits_processors import get_logits_processors
13
from vllm.pooling_params import PoolingParams
14
from vllm.sampling_params import SamplingParams
15
from vllm.utils import random_uuid
@@ -213,30 +215,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
215
216
# doc: end-chat-completion-extra-params
217
216
- def to_sampling_params(self) -> SamplingParams:
218
+ def to_sampling_params(self,
219
+ tokenizer: PreTrainedTokenizer) -> SamplingParams:
220
# We now allow logprobs being true without top_logrobs.
221
219
- logits_processors = None
220
- if self.logit_bias:
221
- logit_bias: Dict[int, float] = {}
222
- try:
223
- for token_id, bias in self.logit_bias.items():
224
- # Convert token_id to integer before we add to LLMEngine
225
- # Clamp the bias between -100 and 100 per OpenAI API spec
226
- logit_bias[int(token_id)] = min(100, max(-100, bias))
227
- except ValueError as exc:
228
- raise ValueError(f"Found token_id `{token_id}` in logit_bias "
229
- f"but token_id must be an integer or string "
230
- f"representing an integer") from exc
231
-
232
- def logit_bias_logits_processor(
233
- token_ids: List[int],
234
- logits: torch.Tensor) -> torch.Tensor:
235
- for token_id, bias in logit_bias.items():
236
- logits[token_id] += bias
237
- return logits
238
-
239
- logits_processors = [logit_bias_logits_processor]
222
+ logits_processors = get_logits_processors(
223
+ logit_bias=self.logit_bias,
224
+ allowed_token_ids=None,
225
+ tokenizer=tokenizer,
226
+ )
227
228
return SamplingParams(
229
n=self.n,
@@ -358,6 +345,7 @@ class CompletionRequest(OpenAIBaseModel):
345
skip_special_tokens: bool = True
346
spaces_between_special_tokens: bool = True
347
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
348
+ allowed_token_ids: Optional[List[int]] = None
349
# doc: end-completion-sampling-params
350
351
# doc: begin-completion-extra-params
@@ -407,30 +395,14 @@ class CompletionRequest(OpenAIBaseModel):
395
396
# doc: end-completion-extra-params
397
410
- def to_sampling_params(self):
398
+ def to_sampling_params(self, tokenizer: PreTrainedTokenizer):
399
echo_without_generation = self.echo and self.max_tokens == 0
400
413
- logits_processors = None
414
- if self.logit_bias:
415
- logit_bias: Dict[int, float] = {}
416
- try:
417
- for token_id, bias in self.logit_bias.items():
418
- # Convert token_id to integer
419
- # Clamp the bias between -100 and 100 per OpenAI API spec
420
- logit_bias[int(token_id)] = min(100, max(-100, bias))
421
- except ValueError as exc:
422
- raise ValueError(f"Found token_id `{token_id}` in logit_bias "
423
- f"but token_id must be an integer or string "
424
- f"representing an integer") from exc
425
-
426
- def logit_bias_logits_processor(
427
- token_ids: List[int],
428
- logits: torch.Tensor) -> torch.Tensor:
429
- for token_id, bias in logit_bias.items():
430
- logits[token_id] += bias
431
- return logits
432
-
433
- logits_processors = [logit_bias_logits_processor]
401
+ logits_processors = get_logits_processors(
402
+ logit_bias=self.logit_bias,
403
+ allowed_token_ids=self.allowed_token_ids,
404
+ tokenizer=tokenizer,
405
+ )
406
407
return SamplingParams(
408
n=self.n,
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -134,7 +134,7 @@ class OpenAIServingChat(OpenAIServing):
134
135
request_id = f"chat-{random_uuid()}"
136
try:
137
- sampling_params = request.to_sampling_params()
137
+ sampling_params = request.to_sampling_params(tokenizer)
138
decoding_config = await self.engine.get_decoding_config()
139
guided_decoding_backend = request.guided_decoding_backend \
140
or decoding_config.guided_decoding_backend
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -95,7 +95,7 @@ class OpenAIServingCompletion(OpenAIServing):
95
96
tokenizer = await self.engine.get_tokenizer(lora_request)
97
98
- sampling_params = request.to_sampling_params()
98
+ sampling_params = request.to_sampling_params(tokenizer)
99
decoding_config = await self.engine.get_decoding_config()
100
guided_decoding_backend = request.guided_decoding_backend \
101
or decoding_config.guided_decoding_backend