A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix][Frontend] Guard against bad token ids (#9634)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Oct 29, 2024 at 16:13 UTC
67bdf8e523e4020a559b6d74981936c8156243f9
7 files changed
+89
-17
tests/entrypoints/llm/test_prompt_validation.py
+7
-1
@@ -4,6 +4,12 @@ from vllm import LLM
4
5
6
def test_empty_prompt():
7
- llm = LLM(model="gpt2")
7
+ llm = LLM(model="gpt2", enforce_eager=True)
8
with pytest.raises(ValueError, match='Prompt cannot be empty'):
9
llm.generate([""])
10
+
11
+
12
+def test_out_of_vocab_token():
13
+ llm = LLM(model="gpt2", enforce_eager=True)
14
+ with pytest.raises(ValueError, match='out of vocabulary'):
15
+ llm.generate({"prompt_token_ids": [999999]})
tests/entrypoints/openai/test_completion.py
+9
-9
@@ -157,15 +157,15 @@ async def test_added_lora_tokens(client: openai.AsyncOpenAI):
157
@pytest.mark.asyncio
158
async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
159
# test using token IDs
160
- completion = await client.completions.create(
161
- model=MODEL_NAME,
162
- prompt=[0, 0, 32000, 32001, 32002],
163
- echo=True,
164
- max_tokens=5,
165
- temperature=0.0,
166
- )
167
- # Added tokens should not appear in tokenized prompt
168
- assert "vllm" not in completion.choices[0].text
160
+ with pytest.raises(openai.BadRequestError, match="out of vocabulary"):
161
+ # Added tokens should be rejected by the base model
162
+ await client.completions.create(
163
+ model=MODEL_NAME,
164
+ prompt=[0, 0, 32000, 32001, 32002],
165
+ echo=True,
166
+ max_tokens=5,
167
+ temperature=0.0,
168
+ )
169
170
171
@pytest.mark.asyncio
tests/entrypoints/openai/test_prompt_validation.py
+15
@@ -20,3 +20,18 @@ async def test_empty_prompt():
20
prompt="",
21
max_tokens=5,
22
temperature=0.0)
23
+
24
+
25
+@pytest.mark.asyncio
26
+async def test_out_of_vocab_token_ids():
27
+ model_name = "gpt2"
28
+ server_args = ["--enforce-eager"]
29
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
30
+ client = remote_server.get_async_client()
31
+
32
+ with pytest.raises(openai.BadRequestError,
33
+ match=re.compile('.*out of vocabulary.*')):
34
+ await client.completions.create(model=model_name,
35
+ prompt=[999999],
36
+ max_tokens=5,
37
+ temperature=0.0)
vllm/engine/async_llm_engine.py
+12
-3
@@ -412,6 +412,12 @@ class _AsyncLLMEngine(LLMEngine):
412
"""Stop the remote worker execution loop."""
413
await self.model_executor.stop_remote_worker_execution_loop_async()
414
415
+ async def get_tokenizer_async(self,
416
+ lora_request: Optional[LoRARequest] = None
417
+ ) -> AnyTokenizer:
418
+ return await (
419
+ self.get_tokenizer_group().get_lora_tokenizer_async(lora_request))
420
+
421
@overload # DEPRECATED
422
async def add_request_async(
423
self,
@@ -472,6 +478,10 @@ class _AsyncLLMEngine(LLMEngine):
478
if arrival_time is None:
479
arrival_time = time.time()
480
481
+ if self.tokenizer is not None:
482
+ tokenizer = await self.get_tokenizer_async(lora_request)
483
+ self._validate_token_prompt(prompt, tokenizer=tokenizer)
484
+
485
preprocessed_inputs = await self.input_preprocessor.preprocess_async(
486
prompt,
487
request_id=request_id,
@@ -488,7 +498,7 @@ class _AsyncLLMEngine(LLMEngine):
498
# implementation in the LLMEngine
499
params = await build_guided_decoding_logits_processor_async(
500
sampling_params=params,
491
- tokenizer=self.get_tokenizer(lora_request),
501
+ tokenizer=await self.get_tokenizer_async(lora_request),
502
default_guided_backend=self.decoding_config.
503
guided_decoding_backend)
504
@@ -715,8 +725,7 @@ class AsyncLLMEngine(EngineClient):
725
self,
726
lora_request: Optional[LoRARequest] = None,
727
) -> AnyTokenizer:
718
- return await (self.engine.get_tokenizer_group().
719
- get_lora_tokenizer_async(lora_request))
728
+ return await self.engine.get_tokenizer_async(lora_request)
729
730
def start_background_loop(self) -> None:
731
"""Start the background loop."""
vllm/engine/llm_engine.py
+36
-4
@@ -10,7 +10,7 @@ from typing import Sequence as GenericSequence
10
from typing import Set, Type, Union, cast, overload
11
12
import torch
13
-from typing_extensions import TypeVar
13
+from typing_extensions import TypeIs, TypeVar
14
15
import vllm.envs as envs
16
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -32,7 +32,8 @@ from vllm.executor.executor_base import ExecutorBase
32
from vllm.executor.gpu_executor import GPUExecutor
33
from vllm.executor.ray_utils import initialize_ray_cluster
34
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
35
- EncoderDecoderInputs, InputRegistry, PromptType)
35
+ EncoderDecoderInputs, InputRegistry, PromptType,
36
+ TokensPrompt)
37
from vllm.inputs.preprocess import InputPreprocessor
38
from vllm.logger import init_logger
39
from vllm.logits_process import get_bad_words_logits_processors
@@ -667,7 +668,7 @@ class LLMEngine:
668
)
669
return None
670
670
- self._validate_model_inputs(processed_inputs)
671
+ self._validate_model_inputs(processed_inputs, lora_request)
672
# Create the sequences.
673
block_size = self.cache_config.block_size
674
seq_id = next(self.seq_counter)
@@ -829,6 +830,11 @@ class LLMEngine:
830
if arrival_time is None:
831
arrival_time = time.time()
832
833
+ if self.tokenizer is not None:
834
+ self._validate_token_prompt(
835
+ prompt,
836
+ tokenizer=self.get_tokenizer(lora_request=lora_request))
837
+
838
preprocessed_inputs = self.input_preprocessor.preprocess(
839
prompt,
840
request_id=request_id,
@@ -855,6 +861,31 @@ class LLMEngine:
861
priority=priority,
862
)
863
864
+ def _validate_token_prompt(self, prompt: PromptType,
865
+ tokenizer: AnyTokenizer):
866
+ # Guard against out-of-vocab tokens.
867
+ # For some tokenizers, tokenizer.decode will happily return empty text
868
+ # for token ids that are out of vocab, and we don't detect token ids
869
+ # that are greater than the max token id before running the model.
870
+ # However, these token ids will later crash a cuda kernel at runtime
871
+ # with an index out of bounds error. This will crash the entire engine.
872
+ # This needs to happen before multimodal input pre-processing, which
873
+ # may add dummy <image> tokens that aren't part of the tokenizer's
874
+ # vocabulary.
875
+ if self._is_token_prompt(prompt):
876
+ prompt_ids = prompt["prompt_token_ids"]
877
+ if len(prompt_ids) == 0:
878
+ # Empty prompt check is handled later
879
+ return
880
+ max_input_id = max(prompt_ids)
881
+ if max_input_id > tokenizer.max_token_id:
882
+ raise ValueError(
883
+ "Token id {} is out of vocabulary".format(max_input_id))
884
+
885
+ @staticmethod
886
+ def _is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
887
+ return isinstance(prompt, dict) and "prompt_token_ids" in prompt
888
+
889
def _create_sequence_group_with_sampling(
890
self,
891
request_id: str,
@@ -1942,7 +1973,8 @@ class LLMEngine:
1973
return self.input_preprocessor.is_encoder_decoder_model()
1974
1975
def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1945
- EncoderDecoderInputs]):
1976
+ EncoderDecoderInputs],
1977
+ lora_request: Optional[LoRARequest]):
1978
if self.model_config.is_multimodal_model:
1979
# For encoder-decoder multimodal models, the max_prompt_len
1980
# restricts the decoder prompt length
vllm/transformers_utils/tokenizer.py
+5
@@ -35,6 +35,7 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
35
tokenizer.all_special_tokens_extended)
36
tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
37
tokenizer_len = len(tokenizer)
38
+ max_token_id = max(tokenizer.get_vocab().values())
39
40
class CachedTokenizer(tokenizer.__class__): # type: ignore
41
@@ -50,6 +51,10 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
51
def all_special_tokens_extended(self):
52
return tokenizer_all_special_tokens_extended
53
54
+ @property
55
+ def max_token_id(self):
56
+ return max_token_id
57
+
58
def __len__(self):
59
return tokenizer_len
60
vllm/transformers_utils/tokenizers/mistral.py
+5
@@ -85,6 +85,7 @@ class MistralTokenizer:
85
raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
86
87
self.tokenizer = tokenizer_
88
+ self._max_token_id = max(self._vocab.values())
89
90
@classmethod
91
def from_pretrained(cls,
@@ -158,6 +159,10 @@ class MistralTokenizer:
159
def vocab_size(self) -> int:
160
return len(self._vocab)
161
162
+ @property
163
+ def max_token_id(self) -> int:
164
+ return self._max_token_id
165
+
166
def __len__(self) -> int:
167
return self.vocab_size
168