@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67bdf8e523
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][Frontend] Guard against bad token ids (#9634)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 29, 2024 at 16:13 UTC 67bdf8e523e4020a559b6d74981936c8156243f9
7 files changed +89 -17
tests/entrypoints/llm/test_prompt_validation.py
+7 -1
@@ -4,6 +4,12 @@ from vllm import LLM
4
5
6 def test_empty_prompt():
7 - llm = LLM(model="gpt2")
7 + llm = LLM(model="gpt2", enforce_eager=True)
8 with pytest.raises(ValueError, match='Prompt cannot be empty'):
9 llm.generate([""])
10 +
11 +
12 +def test_out_of_vocab_token():
13 + llm = LLM(model="gpt2", enforce_eager=True)
14 + with pytest.raises(ValueError, match='out of vocabulary'):
15 + llm.generate({"prompt_token_ids": [999999]})
tests/entrypoints/openai/test_completion.py
+9 -9
@@ -157,15 +157,15 @@ async def test_added_lora_tokens(client: openai.AsyncOpenAI):
157 @pytest.mark.asyncio
158 async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
159 # test using token IDs
160 - completion = await client.completions.create(
161 - model=MODEL_NAME,
162 - prompt=[0, 0, 32000, 32001, 32002],
163 - echo=True,
164 - max_tokens=5,
165 - temperature=0.0,
166 - )
167 - # Added tokens should not appear in tokenized prompt
168 - assert "vllm" not in completion.choices[0].text
160 + with pytest.raises(openai.BadRequestError, match="out of vocabulary"):
161 + # Added tokens should be rejected by the base model
162 + await client.completions.create(
163 + model=MODEL_NAME,
164 + prompt=[0, 0, 32000, 32001, 32002],
165 + echo=True,
166 + max_tokens=5,
167 + temperature=0.0,
168 + )
169
170
171 @pytest.mark.asyncio
tests/entrypoints/openai/test_prompt_validation.py
+15
@@ -20,3 +20,18 @@ async def test_empty_prompt():
20 prompt="",
21 max_tokens=5,
22 temperature=0.0)
23 +
24 +
25 +@pytest.mark.asyncio
26 +async def test_out_of_vocab_token_ids():
27 + model_name = "gpt2"
28 + server_args = ["--enforce-eager"]
29 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
30 + client = remote_server.get_async_client()
31 +
32 + with pytest.raises(openai.BadRequestError,
33 + match=re.compile('.*out of vocabulary.*')):
34 + await client.completions.create(model=model_name,
35 + prompt=[999999],
36 + max_tokens=5,
37 + temperature=0.0)
vllm/engine/async_llm_engine.py
+12 -3
@@ -412,6 +412,12 @@ class _AsyncLLMEngine(LLMEngine):
412 """Stop the remote worker execution loop."""
413 await self.model_executor.stop_remote_worker_execution_loop_async()
414
415 + async def get_tokenizer_async(self,
416 + lora_request: Optional[LoRARequest] = None
417 + ) -> AnyTokenizer:
418 + return await (
419 + self.get_tokenizer_group().get_lora_tokenizer_async(lora_request))
420 +
421 @overload # DEPRECATED
422 async def add_request_async(
423 self,
@@ -472,6 +478,10 @@ class _AsyncLLMEngine(LLMEngine):
478 if arrival_time is None:
479 arrival_time = time.time()
480
481 + if self.tokenizer is not None:
482 + tokenizer = await self.get_tokenizer_async(lora_request)
483 + self._validate_token_prompt(prompt, tokenizer=tokenizer)
484 +
485 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
486 prompt,
487 request_id=request_id,
@@ -488,7 +498,7 @@ class _AsyncLLMEngine(LLMEngine):
498 # implementation in the LLMEngine
499 params = await build_guided_decoding_logits_processor_async(
500 sampling_params=params,
491 - tokenizer=self.get_tokenizer(lora_request),
501 + tokenizer=await self.get_tokenizer_async(lora_request),
502 default_guided_backend=self.decoding_config.
503 guided_decoding_backend)
504
@@ -715,8 +725,7 @@ class AsyncLLMEngine(EngineClient):
725 self,
726 lora_request: Optional[LoRARequest] = None,
727 ) -> AnyTokenizer:
718 - return await (self.engine.get_tokenizer_group().
719 - get_lora_tokenizer_async(lora_request))
728 + return await self.engine.get_tokenizer_async(lora_request)
729
730 def start_background_loop(self) -> None:
731 """Start the background loop."""
vllm/engine/llm_engine.py
+36 -4
@@ -10,7 +10,7 @@ from typing import Sequence as GenericSequence
10 from typing import Set, Type, Union, cast, overload
11
12 import torch
13 -from typing_extensions import TypeVar
13 +from typing_extensions import TypeIs, TypeVar
14
15 import vllm.envs as envs
16 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -32,7 +32,8 @@ from vllm.executor.executor_base import ExecutorBase
32 from vllm.executor.gpu_executor import GPUExecutor
33 from vllm.executor.ray_utils import initialize_ray_cluster
34 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
35 - EncoderDecoderInputs, InputRegistry, PromptType)
35 + EncoderDecoderInputs, InputRegistry, PromptType,
36 + TokensPrompt)
37 from vllm.inputs.preprocess import InputPreprocessor
38 from vllm.logger import init_logger
39 from vllm.logits_process import get_bad_words_logits_processors
@@ -667,7 +668,7 @@ class LLMEngine:
668 )
669 return None
670
670 - self._validate_model_inputs(processed_inputs)
671 + self._validate_model_inputs(processed_inputs, lora_request)
672 # Create the sequences.
673 block_size = self.cache_config.block_size
674 seq_id = next(self.seq_counter)
@@ -829,6 +830,11 @@ class LLMEngine:
830 if arrival_time is None:
831 arrival_time = time.time()
832
833 + if self.tokenizer is not None:
834 + self._validate_token_prompt(
835 + prompt,
836 + tokenizer=self.get_tokenizer(lora_request=lora_request))
837 +
838 preprocessed_inputs = self.input_preprocessor.preprocess(
839 prompt,
840 request_id=request_id,
@@ -855,6 +861,31 @@ class LLMEngine:
861 priority=priority,
862 )
863
864 + def _validate_token_prompt(self, prompt: PromptType,
865 + tokenizer: AnyTokenizer):
866 + # Guard against out-of-vocab tokens.
867 + # For some tokenizers, tokenizer.decode will happily return empty text
868 + # for token ids that are out of vocab, and we don't detect token ids
869 + # that are greater than the max token id before running the model.
870 + # However, these token ids will later crash a cuda kernel at runtime
871 + # with an index out of bounds error. This will crash the entire engine.
872 + # This needs to happen before multimodal input pre-processing, which
873 + # may add dummy <image> tokens that aren't part of the tokenizer's
874 + # vocabulary.
875 + if self._is_token_prompt(prompt):
876 + prompt_ids = prompt["prompt_token_ids"]
877 + if len(prompt_ids) == 0:
878 + # Empty prompt check is handled later
879 + return
880 + max_input_id = max(prompt_ids)
881 + if max_input_id > tokenizer.max_token_id:
882 + raise ValueError(
883 + "Token id {} is out of vocabulary".format(max_input_id))
884 +
885 + @staticmethod
886 + def _is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
887 + return isinstance(prompt, dict) and "prompt_token_ids" in prompt
888 +
889 def _create_sequence_group_with_sampling(
890 self,
891 request_id: str,
@@ -1942,7 +1973,8 @@ class LLMEngine:
1973 return self.input_preprocessor.is_encoder_decoder_model()
1974
1975 def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1945 - EncoderDecoderInputs]):
1976 + EncoderDecoderInputs],
1977 + lora_request: Optional[LoRARequest]):
1978 if self.model_config.is_multimodal_model:
1979 # For encoder-decoder multimodal models, the max_prompt_len
1980 # restricts the decoder prompt length
vllm/transformers_utils/tokenizer.py
+5
@@ -35,6 +35,7 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
35 tokenizer.all_special_tokens_extended)
36 tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
37 tokenizer_len = len(tokenizer)
38 + max_token_id = max(tokenizer.get_vocab().values())
39
40 class CachedTokenizer(tokenizer.__class__): # type: ignore
41
@@ -50,6 +51,10 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
51 def all_special_tokens_extended(self):
52 return tokenizer_all_special_tokens_extended
53
54 + @property
55 + def max_token_id(self):
56 + return max_token_id
57 +
58 def __len__(self):
59 return tokenizer_len
60
vllm/transformers_utils/tokenizers/mistral.py
+5
@@ -85,6 +85,7 @@ class MistralTokenizer:
85 raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
86
87 self.tokenizer = tokenizer_
88 + self._max_token_id = max(self._vocab.values())
89
90 @classmethod
91 def from_pretrained(cls,
@@ -158,6 +159,10 @@ class MistralTokenizer:
159 def vocab_size(self) -> int:
160 return len(self._vocab)
161
162 + @property
163 + def max_token_id(self) -> int:
164 + return self._max_token_id
165 +
166 def __len__(self) -> int:
167 return self.vocab_size
168