@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f69d8245a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] New `allowed_token_ids` decoding request parameter (#6753)

Nick Hill committed Jul 29, 2024 at 16:37 UTC 9f69d8245a695918cb80c218a83c42e9fd409687
5 files changed +114 -46
tests/entrypoints/openai/test_completion.py
+22
@@ -541,6 +541,28 @@ async def test_logits_bias(client: openai.AsyncOpenAI):
541 assert first_response != completion.choices[0].text
542
543
544 +@pytest.mark.asyncio
545 +async def test_allowed_token_ids(client: openai.AsyncOpenAI):
546 + prompt = "Hello, my name is"
547 + max_tokens = 1
548 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
549 +
550 + # Test exclusive selection
551 + allowed_ids = [21555, 21557, 21558]
552 + completion = await client.completions.create(
553 + model=MODEL_NAME,
554 + prompt=prompt,
555 + max_tokens=max_tokens,
556 + temperature=0.0,
557 + seed=42,
558 + extra_body=dict(allowed_token_ids=allowed_ids),
559 + logprobs=1,
560 + )
561 + response_tokens = completion.choices[0].logprobs.tokens
562 + assert len(response_tokens) == 1
563 + assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
564 +
565 +
566 @pytest.mark.asyncio
567 @pytest.mark.parametrize("guided_decoding_backend",
568 ["outlines", "lm-format-enforcer"])
vllm/entrypoints/openai/logits_processors.py new
+74
@@ -0,0 +1,74 @@
1 +from functools import lru_cache
2 +from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3 +
4 +import torch
5 +from transformers import PreTrainedTokenizer
6 +
7 +from vllm.sampling_params import LogitsProcessor
8 +
9 +
10 +class AllowedTokenIdsLogitsProcessor:
11 + """Logits processor for constraining generated tokens to a
12 + specific set of token ids."""
13 +
14 + def __init__(self, allowed_ids: Iterable[int]):
15 + self.allowed_ids: Optional[List[int]] = list(allowed_ids)
16 + self.mask: Optional[torch.Tensor] = None
17 +
18 + def __call__(self, token_ids: List[int],
19 + logits: torch.Tensor) -> torch.Tensor:
20 + if self.mask is None:
21 + self.mask = torch.ones((logits.shape[-1], ),
22 + dtype=torch.bool,
23 + device=logits.device)
24 + self.mask[self.allowed_ids] = False
25 + self.allowed_ids = None
26 + logits.masked_fill_(self.mask, float("-inf"))
27 + return logits
28 +
29 +
30 +@lru_cache(maxsize=32)
31 +def _get_allowed_token_ids_logits_processor(
32 + allowed_token_ids: FrozenSet[int],
33 + vocab_size: int,
34 +) -> LogitsProcessor:
35 + if not allowed_token_ids:
36 + raise ValueError("Empty allowed_token_ids provided")
37 + if not all(0 <= tid < vocab_size for tid in allowed_token_ids):
38 + raise ValueError("allowed_token_ids contains "
39 + "out-of-vocab token id")
40 + return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41 +
42 +
43 +def get_logits_processors(
44 + logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
45 + allowed_token_ids: Optional[List[int]],
46 + tokenizer: PreTrainedTokenizer) -> List[LogitsProcessor]:
47 + logits_processors = []
48 + if logit_bias:
49 + try:
50 + # Convert token_id to integer
51 + # Clamp the bias between -100 and 100 per OpenAI API spec
52 + clamped_logit_bias: Dict[int, float] = {
53 + int(token_id): min(100.0, max(-100.0, bias))
54 + for token_id, bias in logit_bias.items()
55 + }
56 + except ValueError as exc:
57 + raise ValueError(
58 + "Found token_id in logit_bias that is not "
59 + "an integer or string representing an integer") from exc
60 +
61 + def logit_bias_logits_processor(token_ids: List[int],
62 + logits: torch.Tensor) -> torch.Tensor:
63 + for token_id, bias in clamped_logit_bias.items():
64 + logits[token_id] += bias
65 + return logits
66 +
67 + logits_processors.append(logit_bias_logits_processor)
68 +
69 + if allowed_token_ids is not None:
70 + logits_processors.append(
71 + _get_allowed_token_ids_logits_processor(
72 + frozenset(allowed_token_ids), tokenizer.vocab_size))
73 +
74 + return logits_processors
vllm/entrypoints/openai/protocol.py
+16 -44
@@ -5,9 +5,11 @@ from typing import Any, Dict, List, Literal, Optional, Union
5
6 import torch
7 from pydantic import BaseModel, ConfigDict, Field, model_validator
8 +from transformers import PreTrainedTokenizer
9 from typing_extensions import Annotated
10
11 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12 +from vllm.entrypoints.openai.logits_processors import get_logits_processors
13 from vllm.pooling_params import PoolingParams
14 from vllm.sampling_params import SamplingParams
15 from vllm.utils import random_uuid
@@ -213,30 +215,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
215
216 # doc: end-chat-completion-extra-params
217
216 - def to_sampling_params(self) -> SamplingParams:
218 + def to_sampling_params(self,
219 + tokenizer: PreTrainedTokenizer) -> SamplingParams:
220 # We now allow logprobs being true without top_logrobs.
221
219 - logits_processors = None
220 - if self.logit_bias:
221 - logit_bias: Dict[int, float] = {}
222 - try:
223 - for token_id, bias in self.logit_bias.items():
224 - # Convert token_id to integer before we add to LLMEngine
225 - # Clamp the bias between -100 and 100 per OpenAI API spec
226 - logit_bias[int(token_id)] = min(100, max(-100, bias))
227 - except ValueError as exc:
228 - raise ValueError(f"Found token_id `{token_id}` in logit_bias "
229 - f"but token_id must be an integer or string "
230 - f"representing an integer") from exc
231 -
232 - def logit_bias_logits_processor(
233 - token_ids: List[int],
234 - logits: torch.Tensor) -> torch.Tensor:
235 - for token_id, bias in logit_bias.items():
236 - logits[token_id] += bias
237 - return logits
238 -
239 - logits_processors = [logit_bias_logits_processor]
222 + logits_processors = get_logits_processors(
223 + logit_bias=self.logit_bias,
224 + allowed_token_ids=None,
225 + tokenizer=tokenizer,
226 + )
227
228 return SamplingParams(
229 n=self.n,
@@ -358,6 +345,7 @@ class CompletionRequest(OpenAIBaseModel):
345 skip_special_tokens: bool = True
346 spaces_between_special_tokens: bool = True
347 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
348 + allowed_token_ids: Optional[List[int]] = None
349 # doc: end-completion-sampling-params
350
351 # doc: begin-completion-extra-params
@@ -407,30 +395,14 @@ class CompletionRequest(OpenAIBaseModel):
395
396 # doc: end-completion-extra-params
397
410 - def to_sampling_params(self):
398 + def to_sampling_params(self, tokenizer: PreTrainedTokenizer):
399 echo_without_generation = self.echo and self.max_tokens == 0
400
413 - logits_processors = None
414 - if self.logit_bias:
415 - logit_bias: Dict[int, float] = {}
416 - try:
417 - for token_id, bias in self.logit_bias.items():
418 - # Convert token_id to integer
419 - # Clamp the bias between -100 and 100 per OpenAI API spec
420 - logit_bias[int(token_id)] = min(100, max(-100, bias))
421 - except ValueError as exc:
422 - raise ValueError(f"Found token_id `{token_id}` in logit_bias "
423 - f"but token_id must be an integer or string "
424 - f"representing an integer") from exc
425 -
426 - def logit_bias_logits_processor(
427 - token_ids: List[int],
428 - logits: torch.Tensor) -> torch.Tensor:
429 - for token_id, bias in logit_bias.items():
430 - logits[token_id] += bias
431 - return logits
432 -
433 - logits_processors = [logit_bias_logits_processor]
401 + logits_processors = get_logits_processors(
402 + logit_bias=self.logit_bias,
403 + allowed_token_ids=self.allowed_token_ids,
404 + tokenizer=tokenizer,
405 + )
406
407 return SamplingParams(
408 n=self.n,
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -134,7 +134,7 @@ class OpenAIServingChat(OpenAIServing):
134
135 request_id = f"chat-{random_uuid()}"
136 try:
137 - sampling_params = request.to_sampling_params()
137 + sampling_params = request.to_sampling_params(tokenizer)
138 decoding_config = await self.engine.get_decoding_config()
139 guided_decoding_backend = request.guided_decoding_backend \
140 or decoding_config.guided_decoding_backend
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -95,7 +95,7 @@ class OpenAIServingCompletion(OpenAIServing):
95
96 tokenizer = await self.engine.get_tokenizer(lora_request)
97
98 - sampling_params = request.to_sampling_params()
98 + sampling_params = request.to_sampling_params(tokenizer)
99 decoding_config = await self.engine.get_decoding_config()
100 guided_decoding_backend = request.guided_decoding_backend \
101 or decoding_config.guided_decoding_backend