A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix]: v1 engine - consider lora adapters in allowed_token_ids (#17855)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
May 11, 2025 at 03:53 UTC
8132365b746c974609b306c0af4291a6760bafbc
3 files changed
+154
-5
tests/lora/conftest.py
+14
-2
@@ -139,6 +139,12 @@ def dummy_model_gate_up() -> nn.Module:
139
return model
140
141
142
+@pytest.fixture(scope="session")
143
+def llama_2_7b_base_huggingface_id():
144
+ # used as a base model for testing with sql lora adapter
145
+ return "meta-llama/Llama-2-7b-hf"
146
+
147
+
148
@pytest.fixture(scope="session")
149
def sql_lora_huggingface_id():
150
# huggingface repo id is used to test lora runtime downloading.
@@ -198,6 +204,12 @@ def qwen2vl_lora_files():
204
return snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
205
206
207
+@pytest.fixture(scope="session")
208
+def qwen25vl_base_huggingface_id():
209
+ # used as a base model for testing with qwen25vl lora adapter
210
+ return "Qwen/Qwen2.5-VL-3B-Instruct"
211
+
212
+
213
@pytest.fixture(scope="session")
214
def qwen25vl_lora_files():
215
return snapshot_download(repo_id="jeeejeee/qwen25-vl-lora-pokemon")
@@ -261,8 +273,8 @@ def run_with_both_engines_lora(request, monkeypatch):
273
@pytest.fixture
274
def reset_default_device():
275
"""
264
- Some tests, such as `test_punica_ops.py`, explicitly set the
265
- default device, which can affect subsequent tests. Adding this fixture
276
+ Some tests, such as `test_punica_ops.py`, explicitly set the
277
+ default device, which can affect subsequent tests. Adding this fixture
278
helps avoid this problem.
279
"""
280
original_device = torch.get_default_device()
tests/lora/test_lora_allowed_token_ids.py
new
+134
@@ -0,0 +1,134 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import pytest
4
+
5
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
6
+ VllmConfig)
7
+from vllm.lora.request import LoRARequest
8
+from vllm.sampling_params import SamplingParams
9
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
10
+from vllm.v1.engine.processor import Processor
11
+
12
+
13
+def test_allowed_token_ids_with_lora_vocab(llama_2_7b_base_huggingface_id,
14
+ sql_lora_files):
15
+ """
16
+ Test that we properly resolve the range of allowed token ids for lora
17
+ adapters that define additional tokens.
18
+ """
19
+
20
+ # Setup a base model compatible with the sql_lora_files adapter and
21
+ # a known number of tokens in the base model.
22
+ model_config = ModelConfig(
23
+ model=llama_2_7b_base_huggingface_id,
24
+ tokenizer=llama_2_7b_base_huggingface_id,
25
+ tokenizer_mode="auto",
26
+ )
27
+
28
+ vllm_config = VllmConfig(
29
+ model_config=model_config,
30
+ cache_config=CacheConfig(),
31
+ device_config=DeviceConfig(),
32
+ lora_config=LoRAConfig(),
33
+ )
34
+
35
+ tokenizer = init_tokenizer_from_configs(
36
+ model_config=vllm_config.model_config,
37
+ scheduler_config=vllm_config.scheduler_config,
38
+ lora_config=vllm_config.lora_config)
39
+ processor = Processor(vllm_config, tokenizer)
40
+
41
+ lora_request = LoRARequest("1", 1, str(sql_lora_files))
42
+ request_id = "1"
43
+ prompt = "a prompt"
44
+
45
+ # tokens added in the lora adapter should not raise an error
46
+ lora_token_ids = [32000, 32001, 32002, 32003]
47
+ processor.process_inputs(
48
+ request_id,
49
+ prompt,
50
+ params=SamplingParams(allowed_token_ids=lora_token_ids),
51
+ lora_request=lora_request)
52
+
53
+ # tokens in the base model should not raise an error
54
+ base_token_ids = [1000, 1001, 1002, 1003]
55
+ processor.process_inputs(
56
+ request_id,
57
+ prompt,
58
+ params=SamplingParams(allowed_token_ids=base_token_ids),
59
+ lora_request=lora_request)
60
+
61
+ # tokens not in the lora adapter should raise an error
62
+ invalid_token_ids = [35000, 35001, 35002, 35003]
63
+ with pytest.raises(ValueError):
64
+ processor.process_inputs(
65
+ request_id,
66
+ prompt,
67
+ params=SamplingParams(allowed_token_ids=invalid_token_ids),
68
+ lora_request=lora_request)
69
+
70
+ # tokens in the lora adapter with no lora request should raise an error
71
+ with pytest.raises(ValueError):
72
+ processor.process_inputs(
73
+ request_id,
74
+ prompt,
75
+ params=SamplingParams(allowed_token_ids=lora_token_ids),
76
+ )
77
+
78
+
79
+def test_allowed_token_ids_with_lora_adapter_no_vocab(
80
+ qwen25vl_base_huggingface_id, qwen25vl_lora_files):
81
+ """
82
+ Test that we properly resolve the range of allowed token ids for lora
83
+ adapters that do not define additional tokens.
84
+ """
85
+
86
+ # Setup a base model compatible with the qwen25vl_lora_files adapter and
87
+ # a known number of tokens in the base model.
88
+ model_config = ModelConfig(
89
+ model=qwen25vl_base_huggingface_id,
90
+ tokenizer=qwen25vl_base_huggingface_id,
91
+ tokenizer_mode="auto",
92
+ )
93
+
94
+ vllm_config = VllmConfig(
95
+ model_config=model_config,
96
+ cache_config=CacheConfig(),
97
+ device_config=DeviceConfig(),
98
+ lora_config=LoRAConfig(),
99
+ )
100
+
101
+ tokenizer = init_tokenizer_from_configs(
102
+ model_config=vllm_config.model_config,
103
+ scheduler_config=vllm_config.scheduler_config,
104
+ lora_config=vllm_config.lora_config)
105
+ processor = Processor(vllm_config, tokenizer)
106
+
107
+ lora_request = LoRARequest("1", 1, str(qwen25vl_lora_files))
108
+ request_id = "1"
109
+ prompt = "a prompt"
110
+
111
+ # tokens in the base model should not raise an error
112
+ base_token_ids = [1000, 1001, 1002, 1003]
113
+ processor.process_inputs(
114
+ request_id,
115
+ prompt,
116
+ params=SamplingParams(allowed_token_ids=base_token_ids),
117
+ lora_request=lora_request)
118
+
119
+ # tokens in the base model with no lora request should not raise an error
120
+ base_token_ids = [1000, 1001, 1002, 1003]
121
+ processor.process_inputs(
122
+ request_id,
123
+ prompt,
124
+ params=SamplingParams(allowed_token_ids=base_token_ids),
125
+ )
126
+
127
+ # tokens not in the base model should raise an error
128
+ invalid_token_ids = [200000, 200001, 200002, 200003]
129
+ with pytest.raises(ValueError):
130
+ processor.process_inputs(
131
+ request_id,
132
+ prompt,
133
+ params=SamplingParams(allowed_token_ids=invalid_token_ids),
134
+ lora_request=lora_request)
vllm/v1/engine/processor.py
+6
-3
@@ -74,6 +74,7 @@ class Processor:
74
def _validate_sampling_params(
75
self,
76
params: SamplingParams,
77
+ lora_request: Optional[LoRARequest],
78
) -> None:
79
self._validate_structured_output(params)
80
self._validate_logit_bias(params)
@@ -82,7 +83,8 @@ class Processor:
83
return
84
if not params.allowed_token_ids:
85
raise ValueError("allowed_token_ids is not None and empty!")
85
- vocab_size = self.model_config.get_vocab_size()
86
+ tokenizer = self.tokenizer.get_lora_tokenizer(lora_request)
87
+ vocab_size = len(tokenizer)
88
if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):
89
raise ValueError(
90
"allowed_token_ids contains out-of-vocab token id!")
@@ -122,6 +124,7 @@ class Processor:
124
def _validate_params(
125
self,
126
params: Union[SamplingParams, PoolingParams],
127
+ lora_request: Optional[LoRARequest],
128
):
129
"""
130
Validate supported SamplingParam.
@@ -132,7 +135,7 @@ class Processor:
135
raise ValueError("V1 does not yet support Pooling models.")
136
137
self._validate_logprobs(params)
135
- self._validate_sampling_params(params)
138
+ self._validate_sampling_params(params, lora_request)
139
self._validate_supported_sampling_params(params)
140
141
def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
@@ -207,7 +210,7 @@ class Processor:
210
# TODO(woosuk): Support pooling models.
211
# TODO(woosuk): Support encoder-decoder models.
212
self._validate_lora(lora_request)
210
- self._validate_params(params)
213
+ self._validate_params(params, lora_request)
214
if priority != 0:
215
raise ValueError("V1 does not support priority yet.")
216
if trace_headers is not None: