A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Add user-configurable task for models that support both generation and embedding (#9424)
Cyrus Leung committed
Oct 19, 2024 at 02:31 UTC
051eaf6db3d8feeb0779a4e942aadc85eda2f8b2
33 files changed
+451
-201
docs/source/models/supported_models.rst
+8
@@ -294,6 +294,10 @@ Text Embedding
294
-
295
- ✅︎
296
297
+.. important::
298
+ Some model architectures support both generation and embedding tasks.
299
+ In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
300
+
301
Reward Modeling
302
---------------
303
@@ -482,6 +486,10 @@ Multimodal Embedding
486
- 🚧
487
- ✅︎
488
489
+.. important::
490
+ Some model architectures support both generation and embedding tasks.
491
+ In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
492
+
493
----
494
495
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
docs/source/models/vlm.rst
+2
-2
@@ -181,8 +181,8 @@ Below is an example on how to launch the same ``microsoft/Phi-3.5-vision-instruc
181
182
.. code-block:: bash
183
184
- vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
185
- --trust-remote-code --limit-mm-per-prompt image=2
184
+ vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
185
+ --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
186
187
.. important::
188
Since OpenAI Vision API is based on `Chat Completions <https://platform.openai.com/docs/api-reference/chat>`_ API,
examples/offline_inference_vision_language_embedding.py
+1
@@ -7,6 +7,7 @@ prompt = "<|image_1|> Represent the given image with the following question: Wha
7
# Create an LLM.
8
llm = LLM(
9
model="TIGER-Lab/VLM2Vec-Full",
10
+ task="embedding",
11
trust_remote_code=True,
12
max_model_len=4096,
13
max_num_seqs=2,
examples/openai_api_client_for_multimodal.py
+2
-2
@@ -7,8 +7,8 @@ Launch the vLLM server with the following command:
7
vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
8
9
(multi-image inference with Phi-3.5-vision-instruct)
10
-vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
11
- --trust-remote-code --limit-mm-per-prompt image=2
10
+vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
11
+ --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
12
13
(audio inference with Ultravox)
14
vllm serve fixie-ai/ultravox-v0_3 --max-model-len 4096
tests/conftest.py
+3
-1
@@ -25,7 +25,7 @@ from tests.models.utils import (TokensTextLogprobs,
25
from vllm import LLM, SamplingParams
26
from vllm.assets.image import ImageAsset
27
from vllm.assets.video import VideoAsset
28
-from vllm.config import TokenizerPoolConfig
28
+from vllm.config import TaskOption, TokenizerPoolConfig
29
from vllm.connections import global_http_connection
30
from vllm.distributed import (destroy_distributed_environment,
31
destroy_model_parallel,
@@ -619,6 +619,7 @@ class VllmRunner:
619
def __init__(
620
self,
621
model_name: str,
622
+ task: TaskOption = "auto",
623
tokenizer_name: Optional[str] = None,
624
# Use smaller max model length, otherwise bigger model cannot run due
625
# to kv cache size limit.
@@ -634,6 +635,7 @@ class VllmRunner:
635
) -> None:
636
self.model = LLM(
637
model=model_name,
638
+ task=task,
639
tokenizer=tokenizer_name,
640
trust_remote_code=True,
641
dtype=dtype,
tests/core/test_chunked_prefill_scheduler.py
+13
-2
@@ -33,7 +33,8 @@ def test_simple():
33
num_seq_group = 4
34
max_model_len = 16
35
max_num_batched_tokens = 64
36
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
36
+ scheduler_config = SchedulerConfig("generate",
37
+ max_num_batched_tokens,
38
num_seq_group,
39
max_model_len,
40
enable_chunked_prefill=True)
@@ -78,6 +79,7 @@ def test_chunk():
79
max_model_len = 80
80
max_num_batched_tokens = 64
81
scheduler_config = SchedulerConfig(
82
+ "generate",
83
max_num_batched_tokens,
84
max_seqs,
85
max_model_len,
@@ -126,6 +128,7 @@ def test_complex():
128
max_model_len = 80
129
max_num_batched_tokens = 64
130
scheduler_config = SchedulerConfig(
131
+ "generate",
132
max_num_batched_tokens,
133
max_seqs,
134
max_model_len,
@@ -196,6 +199,7 @@ def test_maximal_decoding():
199
max_model_len = 8
200
max_num_batched_tokens = 2
201
scheduler_config = SchedulerConfig(
202
+ "generate",
203
max_num_batched_tokens,
204
max_seqs,
205
max_model_len,
@@ -289,6 +293,7 @@ def test_prompt_limit():
293
max_model_len = 64
294
max_num_batched_tokens = 32
295
scheduler_config = SchedulerConfig(
296
+ "generate",
297
max_num_batched_tokens,
298
max_seqs,
299
max_model_len,
@@ -321,7 +326,8 @@ def test_prompt_limit_exceed():
326
max_seqs = 64
327
max_model_len = 32
328
max_num_batched_tokens = 64
324
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
329
+ scheduler_config = SchedulerConfig("generate",
330
+ max_num_batched_tokens,
331
max_seqs,
332
max_model_len,
333
enable_chunked_prefill=True)
@@ -348,6 +354,7 @@ def test_swap():
354
max_model_len = 200
355
max_num_batched_tokens = 30
356
scheduler_config = SchedulerConfig(
357
+ "generate",
358
max_num_batched_tokens,
359
max_seqs,
360
max_model_len,
@@ -404,6 +411,7 @@ def test_running_prefill_prioritized_over_swap():
411
max_model_len = 200
412
max_num_batched_tokens = 30
413
scheduler_config = SchedulerConfig(
414
+ "generate",
415
max_num_batched_tokens,
416
max_seqs,
417
max_model_len,
@@ -498,6 +506,7 @@ def test_chunked_prefill_preempt():
506
max_model_len = 200
507
max_num_batched_tokens = 30
508
scheduler_config = SchedulerConfig(
509
+ "generate",
510
max_num_batched_tokens,
511
max_seqs,
512
max_model_len,
@@ -563,6 +572,7 @@ def test_chunked_prefill_max_seqs():
572
max_model_len = 80
573
max_num_batched_tokens = 64
574
scheduler_config = SchedulerConfig(
575
+ "generate",
576
max_num_batched_tokens,
577
max_seqs,
578
max_model_len,
@@ -617,6 +627,7 @@ def test_perfix_caching():
627
max_model_len = 80
628
max_num_batched_tokens = 64
629
scheduler_config = SchedulerConfig(
630
+ "generate",
631
max_num_batched_tokens,
632
max_seqs,
633
max_model_len,
tests/core/test_scheduler.py
+32
-24
@@ -20,9 +20,10 @@ from .utils import (append_new_token, append_new_token_seq_group,
20
def test_scheduler_add_seq_group():
21
block_size = 4
22
scheduler_config = SchedulerConfig(
23
- 100,
24
- 64,
25
- 1,
23
+ "generate",
24
+ max_num_batched_tokens=100,
25
+ max_num_seqs=64,
26
+ max_model_len=1,
27
)
28
cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
29
cache_config.num_cpu_blocks = 4
@@ -42,9 +43,10 @@ def test_scheduler_add_seq_group():
43
def test_scheduler_abort_seq_group():
44
block_size = 4
45
scheduler_config = SchedulerConfig(
45
- 100,
46
- 64,
47
- 1,
46
+ "generate",
47
+ max_num_batched_tokens=100,
48
+ max_num_seqs=64,
49
+ max_model_len=1,
50
)
51
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
52
cache_config.num_cpu_blocks = 4
@@ -70,9 +72,10 @@ def test_scheduler_schedule_simple():
72
num_seq_group = 4
73
max_model_len = 16
74
scheduler_config = SchedulerConfig(
73
- 64,
74
- num_seq_group,
75
- max_model_len,
75
+ "generate",
76
+ max_num_batched_tokens=64,
77
+ max_num_seqs=num_seq_group,
78
+ max_model_len=max_model_len,
79
)
80
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
81
cache_config.num_cpu_blocks = 8
@@ -114,9 +117,10 @@ def test_scheduler_prefill_prioritized():
117
max_model_len = 30
118
max_batched_num_tokens = 30
119
scheduler_config = SchedulerConfig(
117
- max_batched_num_tokens,
118
- 2,
119
- max_model_len,
120
+ "generate",
121
+ max_num_batched_tokens=max_batched_num_tokens,
122
+ max_num_seqs=2,
123
+ max_model_len=max_model_len,
124
)
125
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
126
cache_config.num_cpu_blocks = 16
@@ -145,9 +149,10 @@ def test_scheduler_schedule_preempt_abort():
149
block_size = 4
150
max_model_len = 16
151
scheduler_config = SchedulerConfig(
148
- 64,
149
- 2,
150
- max_model_len,
152
+ "generate",
153
+ max_num_batched_tokens=64,
154
+ max_num_seqs=2,
155
+ max_model_len=max_model_len,
156
)
157
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
158
cache_config.num_cpu_blocks = 2
@@ -204,9 +209,10 @@ def test_scheduler_max_seqs():
209
max_seq_group = 2
210
max_model_len = 16
211
scheduler_config = SchedulerConfig(
207
- 64,
208
- max_seq_group,
209
- max_model_len,
212
+ "generate",
213
+ max_num_batched_tokens=64,
214
+ max_num_seqs=max_seq_group,
215
+ max_model_len=max_model_len,
216
)
217
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
218
cache_config.num_cpu_blocks = 8
@@ -248,9 +254,10 @@ def test_scheduler_max_seqs():
254
def test_scheduler_delay_factor():
255
block_size = 4
256
scheduler_config = SchedulerConfig(
251
- 100,
252
- 64,
253
- 16,
257
+ "generate",
258
+ max_num_batched_tokens=100,
259
+ max_num_seqs=64,
260
+ max_model_len=16,
261
delay_factor=0.5,
262
)
263
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
@@ -350,9 +357,10 @@ def initialize_scheduler(
357
):
358
block_size = block_size
359
scheduler_config = SchedulerConfig(
353
- max_token_budget,
354
- max_num_seqs,
355
- max_model_len,
360
+ "generate",
361
+ max_num_batched_tokens=max_token_budget,
362
+ max_num_seqs=max_num_seqs,
363
+ max_model_len=max_model_len,
364
)
365
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
366
cache_config.num_cpu_blocks = num_cpu_blocks
tests/core/test_scheduler_encoder_decoder.py
+6
-1
@@ -36,7 +36,12 @@ def test_scheduler_schedule_simple_encoder_decoder():
36
block_size = 4
37
num_seq_group = 4
38
max_model_len = 16
39
- scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
39
+ scheduler_config = SchedulerConfig(
40
+ task="generate",
41
+ max_num_batched_tokens=64,
42
+ max_num_seqs=num_seq_group,
43
+ max_model_len=max_model_len,
44
+ )
45
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
46
cache_config.num_cpu_blocks = 16 # enc and dec prompts per seq_group
47
cache_config.num_gpu_blocks = 16 # enc and dec prompts per seq_group
tests/distributed/test_pipeline_parallel.py
+19
-4
@@ -11,6 +11,7 @@ from typing import List, Literal, NamedTuple, Optional
11
12
import pytest
13
14
+from vllm.config import TaskOption
15
from vllm.logger import init_logger
16
17
from ..utils import compare_two_settings, fork_new_process_for_each_test
@@ -31,6 +32,7 @@ class ParallelSetup(NamedTuple):
32
class PPTestSettings:
33
parallel_setups: List[ParallelSetup]
34
distributed_backends: List[str]
35
+ task: TaskOption
36
trust_remote_code: bool
37
tokenizer_mode: Optional[str]
38
@@ -39,6 +41,7 @@ class PPTestSettings:
41
*,
42
tp_base: int = 1,
43
pp_base: int = 2,
44
+ task: TaskOption = "auto",
45
trust_remote_code: bool = False,
46
tokenizer_mode: Optional[str] = None,
47
):
@@ -66,6 +69,7 @@ class PPTestSettings:
69
chunked_prefill=False),
70
],
71
distributed_backends=["mp", "ray"],
72
+ task=task,
73
trust_remote_code=trust_remote_code,
74
tokenizer_mode=tokenizer_mode,
75
)
@@ -75,6 +79,7 @@ class PPTestSettings:
79
*,
80
tp_base: int = 1,
81
pp_base: int = 2,
82
+ task: TaskOption = "auto",
83
trust_remote_code: bool = False,
84
tokenizer_mode: Optional[str] = None,
85
):
@@ -86,6 +91,7 @@ class PPTestSettings:
91
chunked_prefill=False),
92
],
93
distributed_backends=["mp"],
94
+ task=task,
95
trust_remote_code=trust_remote_code,
96
tokenizer_mode=tokenizer_mode,
97
)
@@ -94,7 +100,7 @@ class PPTestSettings:
100
for parallel_setup in self.parallel_setups:
101
for distributed_backend in self.distributed_backends:
102
yield (model_name, parallel_setup, distributed_backend,
97
- self.trust_remote_code, self.tokenizer_mode)
103
+ self.task, self.trust_remote_code, self.tokenizer_mode)
104
105
106
# NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
@@ -213,6 +219,7 @@ def _compare_tp(
219
model_name: str,
220
parallel_setup: ParallelSetup,
221
distributed_backend: str,
222
+ task: TaskOption,
223
trust_remote_code: bool,
224
tokenizer_mode: Optional[str],
225
num_gpus_available: int,
@@ -240,6 +247,8 @@ def _compare_tp(
247
common_args.append("--enable-chunked-prefill")
248
if eager_mode:
249
common_args.append("--enforce-eager")
250
+ if task != "auto":
251
+ common_args.extend(["--task", task])
252
if trust_remote_code:
253
common_args.append("--trust-remote-code")
254
if tokenizer_mode:
@@ -297,7 +306,7 @@ def _compare_tp(
306
307
308
@pytest.mark.parametrize(
300
- ("model_name", "parallel_setup", "distributed_backend",
309
+ ("model_name", "parallel_setup", "distributed_backend", "task",
310
"trust_remote_code", "tokenizer_mode"),
311
[
312
params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
@@ -310,6 +319,7 @@ def test_tp_language_generation(
319
model_name: str,
320
parallel_setup: ParallelSetup,
321
distributed_backend: str,
322
+ task: TaskOption,
323
trust_remote_code: bool,
324
tokenizer_mode: Optional[str],
325
num_gpus_available,
@@ -317,6 +327,7 @@ def test_tp_language_generation(
327
_compare_tp(model_name,
328
parallel_setup,
329
distributed_backend,
330
+ task,
331
trust_remote_code,
332
tokenizer_mode,
333
num_gpus_available,
@@ -324,7 +335,7 @@ def test_tp_language_generation(
335
336
337
@pytest.mark.parametrize(
327
- ("model_name", "parallel_setup", "distributed_backend",
338
+ ("model_name", "parallel_setup", "distributed_backend", "task",
339
"trust_remote_code", "tokenizer_mode"),
340
[
341
params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
@@ -337,6 +348,7 @@ def test_tp_language_embedding(
348
model_name: str,
349
parallel_setup: ParallelSetup,
350
distributed_backend: str,
351
+ task: TaskOption,
352
trust_remote_code: bool,
353
tokenizer_mode: Optional[str],
354
num_gpus_available,
@@ -344,6 +356,7 @@ def test_tp_language_embedding(
356
_compare_tp(model_name,
357
parallel_setup,
358
distributed_backend,
359
+ task,
360
trust_remote_code,
361
tokenizer_mode,
362
num_gpus_available,
@@ -351,7 +364,7 @@ def test_tp_language_embedding(
364
365
366
@pytest.mark.parametrize(
354
- ("model_name", "parallel_setup", "distributed_backend",
367
+ ("model_name", "parallel_setup", "distributed_backend", "task",
368
"trust_remote_code", "tokenizer_mode"),
369
[
370
params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
@@ -364,6 +377,7 @@ def test_tp_multimodal_generation(
377
model_name: str,
378
parallel_setup: ParallelSetup,
379
distributed_backend: str,
380
+ task: TaskOption,
381
trust_remote_code: bool,
382
tokenizer_mode: Optional[str],
383
num_gpus_available,
@@ -371,6 +385,7 @@ def test_tp_multimodal_generation(
385
_compare_tp(model_name,
386
parallel_setup,
387
distributed_backend,
388
+ task,
389
trust_remote_code,
390
tokenizer_mode,
391
num_gpus_available,
tests/entrypoints/llm/test_chat.py
new
+92
@@ -0,0 +1,92 @@
1
+from typing import List
2
+
3
+import pytest
4
+
5
+from vllm import LLM
6
+
7
+from ..openai.test_vision import TEST_IMAGE_URLS
8
+
9
+
10
+def test_chat():
11
+ llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
12
+
13
+ prompt1 = "Explain the concept of entropy."
14
+ messages = [
15
+ {
16
+ "role": "system",
17
+ "content": "You are a helpful assistant"
18
+ },
19
+ {
20
+ "role": "user",
21
+ "content": prompt1
22
+ },
23
+ ]
24
+ outputs = llm.chat(messages)
25
+ assert len(outputs) == 1
26
+
27
+
28
+def test_multi_chat():
29
+ llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
30
+
31
+ prompt1 = "Explain the concept of entropy."
32
+ prompt2 = "Explain what among us is."
33
+
34
+ conversation1 = [
35
+ {
36
+ "role": "system",
37
+ "content": "You are a helpful assistant"
38
+ },
39
+ {
40
+ "role": "user",
41
+ "content": prompt1
42
+ },
43
+ ]
44
+
45
+ conversation2 = [
46
+ {
47
+ "role": "system",
48
+ "content": "You are a helpful assistant"
49
+ },
50
+ {
51
+ "role": "user",
52
+ "content": prompt2
53
+ },
54
+ ]
55
+
56
+ messages = [conversation1, conversation2]
57
+
58
+ outputs = llm.chat(messages)
59
+ assert len(outputs) == 2
60
+
61
+
62
+@pytest.mark.parametrize("image_urls",
63
+ [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
64
+def test_chat_multi_image(image_urls: List[str]):
65
+ llm = LLM(
66
+ model="microsoft/Phi-3.5-vision-instruct",
67
+ dtype="bfloat16",
68
+ max_model_len=4096,
69
+ max_num_seqs=5,
70
+ enforce_eager=True,
71
+ trust_remote_code=True,
72
+ limit_mm_per_prompt={"image": 2},
73
+ )
74
+
75
+ messages = [{
76
+ "role":
77
+ "user",
78
+ "content": [
79
+ *({
80
+ "type": "image_url",
81
+ "image_url": {
82
+ "url": image_url
83
+ }
84
+ } for image_url in image_urls),
85
+ {
86
+ "type": "text",
87
+ "text": "What's in this image?"
88
+ },
89
+ ],
90
+ }]
91
+ outputs = llm.chat(messages)
92
+ assert len(outputs) >= 0
tests/entrypoints/llm/test_generate.py
-88
@@ -6,7 +6,6 @@ import pytest
6
from vllm import LLM, RequestOutput, SamplingParams
7
8
from ...conftest import cleanup
9
-from ..openai.test_vision import TEST_IMAGE_URLS
9
10
MODEL_NAME = "facebook/opt-125m"
11
@@ -104,90 +103,3 @@ def test_multiple_sampling_params(llm: LLM):
103
# sampling_params is None, default params should be applied
104
outputs = llm.generate(PROMPTS, sampling_params=None)
105
assert len(PROMPTS) == len(outputs)
107
-
108
-
109
-def test_chat():
110
-
111
- llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
112
-
113
- prompt1 = "Explain the concept of entropy."
114
- messages = [
115
- {
116
- "role": "system",
117
- "content": "You are a helpful assistant"
118
- },
119
- {
120
- "role": "user",
121
- "content": prompt1
122
- },
123
- ]
124
- outputs = llm.chat(messages)
125
- assert len(outputs) == 1
126
-
127
-
128
-def test_multi_chat():
129
-
130
- llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
131
-
132
- prompt1 = "Explain the concept of entropy."
133
- prompt2 = "Explain what among us is."
134
-
135
- conversation1 = [
136
- {
137
- "role": "system",
138
- "content": "You are a helpful assistant"
139
- },
140
- {
141
- "role": "user",
142
- "content": prompt1
143
- },
144
- ]
145
-
146
- conversation2 = [
147
- {
148
- "role": "system",
149
- "content": "You are a helpful assistant"
150
- },
151
- {
152
- "role": "user",
153
- "content": prompt2
154
- },
155
- ]
156
-
157
- messages = [conversation1, conversation2]
158
-
159
- outputs = llm.chat(messages)
160
- assert len(outputs) == 2
161
-
162
-
163
-@pytest.mark.parametrize("image_urls",
164
- [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
165
-def test_chat_multi_image(image_urls: List[str]):
166
- llm = LLM(
167
- model="microsoft/Phi-3.5-vision-instruct",
168
- dtype="bfloat16",
169
- max_model_len=4096,
170
- max_num_seqs=5,
171
- enforce_eager=True,
172
- trust_remote_code=True,
173
- limit_mm_per_prompt={"image": 2},
174
- )
175
-
176
- messages = [{
177
- "role":
178
- "user",
179
- "content": [
180
- *({
181
- "type": "image_url",
182
- "image_url": {
183
- "url": image_url
184
- }
185
- } for image_url in image_urls),
186
- {
187
- "type": "text",
188
- "text": "What's in this image?"
189
- },
190
- ],
191
- }]
192
- outputs = llm.chat(messages)
193
- assert len(outputs) >= 0
tests/entrypoints/llm/test_init.py
new
+22
@@ -0,0 +1,22 @@
1
+import pytest
2
+
3
+from vllm import LLM
4
+
5
+from ...utils import error_on_warning
6
+
7
+MODEL_NAME = "facebook/opt-125m"
8
+
9
+
10
+def test_pos_args_deprecated():
11
+ with error_on_warning(DeprecationWarning):
12
+ LLM(model=MODEL_NAME, tokenizer=MODEL_NAME)
13
+
14
+ with error_on_warning(DeprecationWarning):
15
+ LLM(MODEL_NAME, tokenizer=MODEL_NAME)
16
+
17
+ with pytest.warns(DeprecationWarning, match="'tokenizer'"):
18
+ LLM(MODEL_NAME, MODEL_NAME)
19
+
20
+ with pytest.warns(DeprecationWarning,
21
+ match="'tokenizer', 'tokenizer_mode'"):
22
+ LLM(MODEL_NAME, MODEL_NAME, "auto")
tests/entrypoints/openai/test_serving_chat.py
+1
-1
@@ -22,12 +22,12 @@ class MockHFConfig:
22
23
@dataclass
24
class MockModelConfig:
25
+ task = "generate"
26
tokenizer = MODEL_NAME
27
trust_remote_code = False
28
tokenizer_mode = "auto"
29
max_model_len = 100
30
tokenizer_revision = None
30
- embedding_mode = False
31
multimodal_config = MultiModalConfig()
32
hf_config = MockHFConfig()
33
tests/entrypoints/openai/test_vision.py
+2
@@ -23,6 +23,8 @@ TEST_IMAGE_URLS = [
23
@pytest.fixture(scope="module")
24
def server():
25
args = [
26
+ "--task",
27
+ "generate",
28
"--dtype",
29
"bfloat16",
30
"--max-model-len",
tests/entrypoints/test_chat_utils.py
+2
-1
@@ -18,7 +18,8 @@ PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
18
@pytest.fixture(scope="module")
19
def phi3v_model_config():
20
return ModelConfig(PHI3V_MODEL_ID,
21
- PHI3V_MODEL_ID,
21
+ task="generate",
22
+ tokenizer=PHI3V_MODEL_ID,
23
tokenizer_mode="auto",
24
trust_remote_code=True,
25
dtype="bfloat16",
tests/lora/test_worker.py
+3
-2
@@ -15,7 +15,8 @@ def test_worker_apply_lora(sql_lora_files):
15
worker = Worker(
16
model_config=ModelConfig(
17
"meta-llama/Llama-2-7b-hf",
18
- "meta-llama/Llama-2-7b-hf",
18
+ task="auto",
19
+ tokenizer="meta-llama/Llama-2-7b-hf",
20
tokenizer_mode="auto",
21
trust_remote_code=False,
22
seed=0,
@@ -27,7 +28,7 @@ def test_worker_apply_lora(sql_lora_files):
28
load_format="dummy",
29
),
30
parallel_config=ParallelConfig(1, 1, False),
30
- scheduler_config=SchedulerConfig(32, 32, 32),
31
+ scheduler_config=SchedulerConfig("generate", 32, 32, 32),
32
device_config=DeviceConfig("cuda"),
33
cache_config=CacheConfig(block_size=16,
34
gpu_memory_utilization=1.,
tests/models/decoder_only/vision_language/test_phi3v.py
+1
@@ -89,6 +89,7 @@ def run_test(
89
90
# max_model_len should be greater than image_feature_size
91
with vllm_runner(model,
92
+ task="generate",
93
max_model_len=4096,
94
max_num_seqs=2,
95
dtype=dtype,
tests/models/embedding/vision_language/test_phi3v.py
+1
@@ -28,6 +28,7 @@ def test_models(
28
# if we run HF first, the cuda initialization will be done and it
29
# will hurt multiprocessing backend with fork method (the default method).
30
with vllm_runner(model,
31
+ task="embedding",
32
max_model_len=4096,
33
max_num_seqs=2,
34
dtype=dtype,
tests/models/utils.py
+4
-2
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
import torch
5
6
-from vllm.config import ModelConfig
6
+from vllm.config import ModelConfig, TaskOption
7
from vllm.inputs import InputContext
8
from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9
from vllm.utils import is_cpu
@@ -248,6 +248,7 @@ def check_logprobs_close(
248
249
250
def build_model_context(model_name: str,
251
+ task: TaskOption = "auto",
252
tokenizer_name: Optional[str] = None,
253
trust_remote_code: bool = False,
254
dtype: Optional[Union[str, torch.dtype]] = None,
@@ -273,7 +274,8 @@ def build_model_context(model_name: str,
274
275
model_config = ModelConfig(
276
model_name,
276
- tokenizer_name,
277
+ task=task,
278
+ tokenizer=tokenizer_name,
279
tokenizer_mode="auto",
280
trust_remote_code=trust_remote_code,
281
dtype=dtype,
tests/multimodal/test_mapper.py
+4
@@ -24,6 +24,7 @@ def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
24
25
model_config = ModelConfig(
26
model=MODEL_NAME,
27
+ task="auto",
28
tokenizer=MODEL_NAME,
29
tokenizer_mode="auto",
30
trust_remote_code=False,
@@ -67,6 +68,7 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
68
69
model_config = ModelConfig(
70
model=MODEL_NAME,
71
+ task="auto",
72
tokenizer=MODEL_NAME,
73
tokenizer_mode="auto",
74
trust_remote_code=False,
@@ -109,6 +111,7 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
111
112
model_config = ModelConfig(
113
model=MODEL_NAME,
114
+ task="auto",
115
tokenizer=MODEL_NAME,
116
tokenizer_mode="auto",
117
trust_remote_code=False,
@@ -139,6 +142,7 @@ def test_image_mapper_multi(image_assets, mm_registry, num_images):
142
143
model_config = ModelConfig(
144
model=MODEL_NAME,
145
+ task="auto",
146
tokenizer=MODEL_NAME,
147
tokenizer_mode="auto",
148
trust_remote_code=False,
tests/multimodal/test_processor_kwargs.py
+6
-1
@@ -221,6 +221,7 @@ def test_max_tokens_kwarg_overrides(num_crops):
221
expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
222
223
ctx = build_model_context(MULTIMODAL_MODEL_ID,
224
+ task="generate",
225
trust_remote_code=True,
226
mm_processor_kwargs=mm_processor_kwargs,
227
limit_mm_per_prompt={"image": 1})
@@ -256,6 +257,7 @@ def test_max_tokens_kwarg_overrides(num_crops):
257
def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
258
"""Ensure that max token calcs filters out invalid mm_processor_kwargs"""
259
ctx = build_model_context(MULTIMODAL_MODEL_ID,
260
+ task="generate",
261
trust_remote_code=True,
262
mm_processor_kwargs=mm_processor_kwargs,
263
limit_mm_per_prompt={"image": 1})
@@ -278,12 +280,13 @@ def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
280
281
### Test overrides for the mapper
282
@pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
281
-def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
283
+def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
284
"""Ensure that the mapper processor kwargs can fall back to HF models."""
285
# NOTE - we don't validate bad inputs for the default mapper, because it's
286
# through the automodel interface in transformers, so we can't easily
287
# inspect what kwargs are or are not allowed.
288
ctx = build_model_context(MULTIMODAL_MODEL_ID,
289
+ task="generate",
290
trust_remote_code=True,
291
mm_processor_kwargs={"num_crops": num_crops},
292
limit_mm_per_prompt={"image": 1})
@@ -311,6 +314,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
314
init_num_crops, inference_num_crops)
315
316
ctx = build_model_context(MULTIMODAL_MODEL_ID,
317
+ task="generate",
318
trust_remote_code=True,
319
mm_processor_kwargs=init_kwargs,
320
limit_mm_per_prompt={"image": 1})
@@ -348,6 +352,7 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
352
"""Ensure that custom mappers filters out invalid mm_processor_kwargs"""
353
# Should filter out the init time kwargs
354
ctx = build_model_context(MULTIMODAL_MODEL_ID,
355
+ task="generate",
356
trust_remote_code=True,
357
mm_processor_kwargs=mm_processor_kwargs,
358
limit_mm_per_prompt={"image": 1})
tests/quantization/test_configs.py
+2
-1
@@ -57,7 +57,8 @@ def test_auto_gptq(model_arg_exptype: Tuple[str, None, str]) -> None:
57
58
try:
59
model_config = ModelConfig(model_path,
60
- model_path,
60
+ task="auto",
61
+ tokenizer=model_path,
62
tokenizer_mode="auto",
63
trust_remote_code=False,
64
seed=0,
tests/test_config.py
+50
-7
@@ -2,6 +2,42 @@ import pytest
2
3
from vllm.config import ModelConfig
4
5
+
6
+@pytest.mark.parametrize(("model_id", "expected_task"), [
7
+ ("facebook/opt-125m", "generate"),
8
+ ("intfloat/e5-mistral-7b-instruct", "embedding"),
9
+])
10
+def test_auto_task(model_id, expected_task):
11
+ config = ModelConfig(
12
+ model_id,
13
+ task="auto",
14
+ tokenizer=model_id,
15
+ tokenizer_mode="auto",
16
+ trust_remote_code=False,
17
+ seed=0,
18
+ dtype="float16",
19
+ )
20
+
21
+ assert config.task == expected_task
22
+
23
+
24
+@pytest.mark.parametrize(("model_id", "bad_task"), [
25
+ ("facebook/opt-125m", "embedding"),
26
+ ("intfloat/e5-mistral-7b-instruct", "generate"),
27
+])
28
+def test_incorrect_task(model_id, bad_task):
29
+ with pytest.raises(ValueError, match=r"does not support the .* task"):
30
+ ModelConfig(
31
+ model_id,
32
+ task=bad_task,
33
+ tokenizer=model_id,
34
+ tokenizer_mode="auto",
35
+ trust_remote_code=False,
36
+ seed=0,
37
+ dtype="float16",
38
+ )
39
+
40
+
41
MODEL_IDS_EXPECTED = [
42
("Qwen/Qwen1.5-7B", 32768),
43
("mistralai/Mistral-7B-v0.1", 4096),
@@ -14,7 +50,8 @@ def test_disable_sliding_window(model_id_expected):
50
model_id, expected = model_id_expected
51
model_config = ModelConfig(
52
model_id,
17
- model_id,
53
+ task="auto",
54
+ tokenizer=model_id,
55
tokenizer_mode="auto",
56
trust_remote_code=False,
57
seed=0,
@@ -32,7 +69,8 @@ def test_get_sliding_window():
69
# when use_sliding_window is False.
70
qwen2_model_config = ModelConfig(
71
"Qwen/Qwen1.5-7B",
35
- "Qwen/Qwen1.5-7B",
72
+ task="auto",
73
+ tokenizer="Qwen/Qwen1.5-7B",
74
tokenizer_mode="auto",
75
trust_remote_code=False,
76
seed=0,
@@ -49,7 +87,8 @@ def test_get_sliding_window():
87
88
mistral_model_config = ModelConfig(
89
"mistralai/Mistral-7B-v0.1",
52
- "mistralai/Mistral-7B-v0.1",
90
+ task="auto",
91
+ tokenizer="mistralai/Mistral-7B-v0.1",
92
tokenizer_mode="auto",
93
trust_remote_code=False,
94
seed=0,
@@ -70,7 +109,8 @@ def test_rope_customization():
109
110
llama_model_config = ModelConfig(
111
"meta-llama/Meta-Llama-3-8B-Instruct",
73
- "meta-llama/Meta-Llama-3-8B-Instruct",
112
+ task="auto",
113
+ tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
114
tokenizer_mode="auto",
115
trust_remote_code=False,
116
dtype="float16",
@@ -82,7 +122,8 @@ def test_rope_customization():
122
123
llama_model_config = ModelConfig(
124
"meta-llama/Meta-Llama-3-8B-Instruct",
85
- "meta-llama/Meta-Llama-3-8B-Instruct",
125
+ task="auto",
126
+ tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
127
tokenizer_mode="auto",
128
trust_remote_code=False,
129
dtype="float16",
@@ -98,7 +139,8 @@ def test_rope_customization():
139
140
longchat_model_config = ModelConfig(
141
"lmsys/longchat-13b-16k",
101
- "lmsys/longchat-13b-16k",
142
+ task="auto",
143
+ tokenizer="lmsys/longchat-13b-16k",
144
tokenizer_mode="auto",
145
trust_remote_code=False,
146
dtype="float16",
@@ -112,7 +154,8 @@ def test_rope_customization():
154
155
longchat_model_config = ModelConfig(
156
"lmsys/longchat-13b-16k",
115
- "lmsys/longchat-13b-16k",
157
+ task="auto",
158
+ tokenizer="lmsys/longchat-13b-16k",
159
tokenizer_mode="auto",
160
trust_remote_code=False,
161
dtype="float16",
tests/test_utils.py
+6
-6
@@ -59,7 +59,7 @@ def test_deprecate_kwargs_always():
59
with pytest.warns(DeprecationWarning, match="'old_arg'"):
60
dummy(old_arg=1)
61
62
- with error_on_warning():
62
+ with error_on_warning(DeprecationWarning):
63
dummy(new_arg=1)
64
65
@@ -69,10 +69,10 @@ def test_deprecate_kwargs_never():
69
def dummy(*, old_arg: object = None, new_arg: object = None):
70
pass
71
72
- with error_on_warning():
72
+ with error_on_warning(DeprecationWarning):
73
dummy(old_arg=1)
74
75
- with error_on_warning():
75
+ with error_on_warning(DeprecationWarning):
76
dummy(new_arg=1)
77
78
@@ -86,15 +86,15 @@ def test_deprecate_kwargs_dynamic():
86
with pytest.warns(DeprecationWarning, match="'old_arg'"):
87
dummy(old_arg=1)
88
89
- with error_on_warning():
89
+ with error_on_warning(DeprecationWarning):
90
dummy(new_arg=1)
91
92
is_deprecated = False
93
94
- with error_on_warning():
94
+ with error_on_warning(DeprecationWarning):
95
dummy(old_arg=1)
96
97
- with error_on_warning():
97
+ with error_on_warning(DeprecationWarning):
98
dummy(new_arg=1)
99
100
tests/utils.py
+4
-4
@@ -8,7 +8,7 @@ import time
8
import warnings
9
from contextlib import contextmanager
10
from pathlib import Path
11
-from typing import Any, Callable, Dict, List, Literal, Optional, Union
11
+from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union
12
13
import openai
14
import pytest
@@ -454,13 +454,13 @@ def multi_process_parallel(
454
455
456
@contextmanager
457
-def error_on_warning():
457
+def error_on_warning(category: Type[Warning] = Warning):
458
"""
459
Within the scope of this context manager, tests will fail if any warning
460
- is emitted.
460
+ of the given category is emitted.
461
"""
462
with warnings.catch_warnings():
463
- warnings.simplefilter("error")
463
+ warnings.filterwarnings("error", category=category)
464
465
yield
466
vllm/config.py
+54
-23
@@ -1,8 +1,8 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping,
5
- Optional, Tuple, Type, Union)
4
+from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
5
+ Mapping, Optional, Set, Tuple, Type, Union)
6
7
import torch
8
from transformers import PretrainedConfig
@@ -33,6 +33,9 @@ logger = init_logger(__name__)
33
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
34
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
35
36
+Task = Literal["generate", "embedding"]
37
+TaskOption = Literal["auto", Task]
38
+
39
40
class ModelConfig:
41
"""Configuration for the model.
@@ -40,7 +43,11 @@ class ModelConfig:
43
Args:
44
model: Name or path of the huggingface model to use.
45
It is also used as the content for `model_name` tag in metrics
43
- output when `served_model_name` is not specified.
46
+ output when `served_model_name` is not specified.
47
+ task: The task to use the model for. Each vLLM instance only supports
48
+ one task, even if the same model can be used for multiple tasks.
49
+ When the model only supports one task, "auto" can be used to select
50
+ it; otherwise, you must specify explicitly which task to use.
51
tokenizer: Name or path of the huggingface tokenizer to use.
52
tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
53
available, "slow" will always use the slow tokenizer, and
@@ -108,6 +115,7 @@ class ModelConfig:
115
116
def __init__(self,
117
model: str,
118
+ task: TaskOption,
119
tokenizer: str,
120
tokenizer_mode: str,
121
trust_remote_code: bool,
@@ -207,7 +215,11 @@ class ModelConfig:
215
216
self.override_neuron_config = override_neuron_config if is_neuron(
217
) else None
210
- self._verify_embedding_mode()
218
+
219
+ supported_tasks, task = self._resolve_task(task, self.hf_config)
220
+ self.supported_tasks = supported_tasks
221
+ self.task: Final = task
222
+
223
self._verify_quantization()
224
self._verify_cuda_graph()
225
self._verify_bnb_config()
@@ -241,18 +253,41 @@ class ModelConfig:
253
"either 'auto', 'slow' or 'mistral'.")
254
self.tokenizer_mode = tokenizer_mode
255
244
- def _verify_embedding_mode(self) -> None:
245
- architectures = getattr(self.hf_config, "architectures", [])
256
+ def _resolve_task(
257
+ self,
258
+ task_option: TaskOption,
259
+ hf_config: PretrainedConfig,
260
+ ) -> Tuple[Set[Task], Task]:
261
+ architectures = getattr(hf_config, "architectures", [])
262
+
263
+ task_support: Dict[Task, bool] = {
264
+ # NOTE: Listed from highest to lowest priority,
265
+ # in case the model supports multiple of them
266
+ "generate": ModelRegistry.is_text_generation_model(architectures),
267
+ "embedding": ModelRegistry.is_embedding_model(architectures),
268
+ }
269
+ supported_tasks_lst: List[Task] = [
270
+ task for task, is_supported in task_support.items() if is_supported
271
+ ]
272
+ supported_tasks = set(supported_tasks_lst)
273
+
274
+ if task_option == "auto":
275
+ selected_task = next(iter(supported_tasks_lst))
276
247
- # TODO: Allow the same model architecture to be specified as either
248
- # generation or embedding model
249
- if "Phi3VForCausalLM" in architectures:
250
- # Match both remote and local names
251
- embedding_mode = "/VLM2Vec" in self.model
277
+ if len(supported_tasks) > 1:
278
+ logger.info(
279
+ "This model supports multiple tasks: %s. "
280
+ "Defaulting to '%s'.", supported_tasks, selected_task)
281
else:
253
- embedding_mode = ModelRegistry.is_embedding_model(architectures)
282
+ if task_option not in supported_tasks:
283
+ msg = (
284
+ f"This model does not support the '{task_option}' task. "
285
+ f"Supported tasks: {supported_tasks}")
286
+ raise ValueError(msg)
287
+
288
+ selected_task = task_option
289
255
- self.embedding_mode = embedding_mode
290
+ return supported_tasks, selected_task
291
292
def _parse_quant_hf_config(self):
293
quant_cfg = getattr(self.hf_config, "quantization_config", None)
@@ -401,7 +436,7 @@ class ModelConfig:
436
437
# Async postprocessor is not necessary with embedding mode
438
# since there is no token generation
404
- if self.embedding_mode:
439
+ if self.task == "embedding":
440
self.use_async_output_proc = False
441
442
# Reminder: Please update docs/source/serving/compatibility_matrix.rst
@@ -582,11 +617,6 @@ class ModelConfig:
617
(hasattr(self.hf_config, "text_config") and getattr(
618
self.hf_config.text_config, "is_encoder_decoder", False)))
619
585
- @property
586
- def is_embedding_model(self) -> bool:
587
- """Extract the embedding model flag."""
588
- return self.embedding_mode
589
-
620
@property
621
def is_multimodal_model(self) -> bool:
622
return self.multimodal_config is not None
@@ -943,6 +973,7 @@ class SchedulerConfig:
973
"""Scheduler configuration.
974
975
Args:
976
+ task: The task to use the model for.
977
max_num_batched_tokens: Maximum number of tokens to be processed in
978
a single iteration.
979
max_num_seqs: Maximum number of sequences to be processed in a single
@@ -957,7 +988,6 @@ class SchedulerConfig:
988
prompt latency) before scheduling next prompt.
989
enable_chunked_prefill: If True, prefill requests can be chunked based
990
on the remaining max_num_batched_tokens.
960
- embedding_mode: Whether the running model is for embedding.
991
preemption_mode: Whether to perform preemption by swapping or
992
recomputation. If not specified, we determine the mode as follows:
993
We use recomputation by default since it incurs lower overhead than
@@ -972,13 +1002,13 @@ class SchedulerConfig:
1002
"""
1003
1004
def __init__(self,
1005
+ task: Task,
1006
max_num_batched_tokens: Optional[int],
1007
max_num_seqs: int,
1008
max_model_len: int,
1009
num_lookahead_slots: int = 0,
1010
delay_factor: float = 0.0,
1011
enable_chunked_prefill: bool = False,
981
- embedding_mode: bool = False,
1012
is_multimodal_model: bool = False,
1013
preemption_mode: Optional[str] = None,
1014
num_scheduler_steps: int = 1,
@@ -1002,7 +1032,7 @@ class SchedulerConfig:
1032
# for higher throughput.
1033
max_num_batched_tokens = max(max_model_len, 2048)
1034
1005
- if embedding_mode:
1035
+ if task == "embedding":
1036
# For embedding, choose specific value for higher throughput
1037
max_num_batched_tokens = max(
1038
max_num_batched_tokens,
@@ -1022,12 +1052,12 @@ class SchedulerConfig:
1052
"Chunked prefill is enabled with max_num_batched_tokens=%d.",
1053
self.max_num_batched_tokens)
1054
1055
+ self.task: Final = task
1056
self.max_num_seqs = max_num_seqs
1057
self.max_model_len = max_model_len
1058
self.num_lookahead_slots = num_lookahead_slots
1059
self.delay_factor = delay_factor
1060
self.chunked_prefill_enabled = enable_chunked_prefill
1030
- self.embedding_mode = embedding_mode
1061
self.preemption_mode = preemption_mode
1062
self.num_scheduler_steps = num_scheduler_steps
1063
self.multi_step_stream_outputs = multi_step_stream_outputs
@@ -1239,6 +1269,7 @@ class SpeculativeConfig:
1269
ngram_prompt_lookup_min = 0
1270
draft_model_config = ModelConfig(
1271
model=speculative_model,
1272
+ task=target_model_config.task,
1273
tokenizer=target_model_config.tokenizer,
1274
tokenizer_mode=target_model_config.tokenizer_mode,
1275
trust_remote_code=target_model_config.trust_remote_code,
vllm/core/scheduler.py
+1
-1
@@ -313,7 +313,7 @@ class Scheduler:
313
self.lora_config = lora_config
314
315
version = "selfattn"
316
- if (self.scheduler_config.embedding_mode
316
+ if (self.scheduler_config.task == "embedding"
317
or self.cache_config.is_attention_free):
318
version = "placeholder"
319
vllm/engine/arg_utils.py
+14
-3
@@ -3,7 +3,7 @@ import dataclasses
3
import json
4
from dataclasses import dataclass
5
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6
- Tuple, Type, Union, cast)
6
+ Tuple, Type, Union, cast, get_args)
7
8
import torch
9
@@ -12,7 +12,7 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12
DeviceConfig, EngineConfig, LoadConfig, LoadFormat,
13
LoRAConfig, ModelConfig, ObservabilityConfig,
14
ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15
- SpeculativeConfig, TokenizerPoolConfig)
15
+ SpeculativeConfig, TaskOption, TokenizerPoolConfig)
16
from vllm.executor.executor_base import ExecutorBase
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -84,6 +84,7 @@ class EngineArgs:
84
model: str = 'facebook/opt-125m'
85
served_model_name: Optional[Union[str, List[str]]] = None
86
tokenizer: Optional[str] = None
87
+ task: TaskOption = "auto"
88
skip_tokenizer_init: bool = False
89
tokenizer_mode: str = 'auto'
90
trust_remote_code: bool = False
@@ -198,6 +199,15 @@ class EngineArgs:
199
type=str,
200
default=EngineArgs.model,
201
help='Name or path of the huggingface model to use.')
202
+ parser.add_argument(
203
+ '--task',
204
+ default=EngineArgs.task,
205
+ choices=get_args(TaskOption),
206
+ help='The task to use the model for. Each vLLM instance only '
207
+ 'supports one task, even if the same model can be used for '
208
+ 'multiple tasks. When the model only supports one task, "auto" '
209
+ 'can be used to select it; otherwise, you must specify explicitly '
210
+ 'which task to use.')
211
parser.add_argument(
212
'--tokenizer',
213
type=nullable_str,
@@ -838,6 +848,7 @@ class EngineArgs:
848
def create_model_config(self) -> ModelConfig:
849
return ModelConfig(
850
model=self.model,
851
+ task=self.task,
852
# We know this is not None because we set it in __post_init__
853
tokenizer=cast(str, self.tokenizer),
854
tokenizer_mode=self.tokenizer_mode,
@@ -1026,13 +1037,13 @@ class EngineArgs:
1037
" please file an issue with detailed information.")
1038
1039
scheduler_config = SchedulerConfig(
1040
+ task=model_config.task,
1041
max_num_batched_tokens=self.max_num_batched_tokens,
1042
max_num_seqs=self.max_num_seqs,
1043
max_model_len=model_config.max_model_len,
1044
num_lookahead_slots=num_lookahead_slots,
1045
delay_factor=self.scheduler_delay_factor,
1046
enable_chunked_prefill=self.enable_chunked_prefill,
1035
- embedding_mode=model_config.embedding_mode,
1047
is_multimodal_model=model_config.is_multimodal_model,
1048
preemption_mode=self.preemption_mode,
1049
num_scheduler_steps=self.num_scheduler_steps,
vllm/engine/llm_engine.py
+2
-5
@@ -344,7 +344,7 @@ class LLMEngine:
344
observability_config=self.observability_config,
345
)
346
347
- if not self.model_config.embedding_mode:
347
+ if self.model_config.task != "embedding":
348
self._initialize_kv_caches()
349
350
# If usage stat is enabled, collect relevant info.
@@ -1116,7 +1116,7 @@ class LLMEngine:
1116
seq_group.metrics.model_execute_time = (
1117
o.model_execute_time)
1118
1119
- if self.model_config.embedding_mode:
1119
+ if self.model_config.task == "embedding":
1120
self._process_sequence_group_outputs(seq_group, output)
1121
else:
1122
self.output_processor.process_prompt_logprob(seq_group, output)
@@ -1855,9 +1855,6 @@ class LLMEngine:
1855
def is_encoder_decoder_model(self):
1856
return self.input_preprocessor.is_encoder_decoder_model()
1857
1858
- def is_embedding_model(self):
1859
- return self.model_config.is_embedding_model
1860
-
1858
def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1859
EncoderDecoderInputs]):
1860
if self.model_config.is_multimodal_model:
vllm/entrypoints/llm.py
+44
-12
@@ -8,7 +8,7 @@ from tqdm import tqdm
8
9
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
10
BeamSearchSequence, get_beam_search_score)
11
-from vllm.engine.arg_utils import EngineArgs
11
+from vllm.engine.arg_utils import EngineArgs, TaskOption
12
from vllm.engine.llm_engine import LLMEngine
13
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
14
apply_hf_chat_template,
@@ -29,7 +29,7 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
29
get_cached_tokenizer)
30
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
31
from vllm.usage.usage_lib import UsageContext
32
-from vllm.utils import Counter, deprecate_kwargs, is_list_of
32
+from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
33
34
logger = init_logger(__name__)
35
@@ -108,6 +108,12 @@ class LLM:
108
DEPRECATE_LEGACY: ClassVar[bool] = False
109
"""A flag to toggle whether to deprecate the legacy generate/encode API."""
110
111
+ DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
112
+ """
113
+ A flag to toggle whether to deprecate positional arguments in
114
+ :meth:`LLM.__init__`.
115
+ """
116
+
117
@classmethod
118
@contextmanager
119
def deprecate_legacy_api(cls):
@@ -117,6 +123,13 @@ class LLM:
123
124
cls.DEPRECATE_LEGACY = False
125
126
+ @deprecate_args(
127
+ start_index=2, # Ignore self and model
128
+ is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
129
+ additional_message=(
130
+ "All positional arguments other than `model` will be "
131
+ "replaced with keyword arguments in an upcoming version."),
132
+ )
133
def __init__(
134
self,
135
model: str,
@@ -139,6 +152,8 @@ class LLM:
152
disable_custom_all_reduce: bool = False,
153
disable_async_output_proc: bool = False,
154
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
155
+ # After positional args are removed, move this right below `model`
156
+ task: TaskOption = "auto",
157
**kwargs,
158
) -> None:
159
'''
@@ -153,6 +168,7 @@ class LLM:
168
169
engine_args = EngineArgs(
170
model=model,
171
+ task=task,
172
tokenizer=tokenizer,
173
tokenizer_mode=tokenizer_mode,
174
skip_tokenizer_init=skip_tokenizer_init,
@@ -316,10 +332,21 @@ class LLM:
332
considered legacy and may be deprecated in the future. You should
333
instead pass them via the ``inputs`` parameter.
334
"""
319
- if self.llm_engine.model_config.embedding_mode:
320
- raise ValueError(
335
+ task = self.llm_engine.model_config.task
336
+ if task != "generate":
337
+ messages = [
338
"LLM.generate() is only supported for (conditional) generation "
322
- "models (XForCausalLM, XForConditionalGeneration).")
339
+ "models (XForCausalLM, XForConditionalGeneration).",
340
+ ]
341
+
342
+ supported_tasks = self.llm_engine.model_config.supported_tasks
343
+ if "generate" in supported_tasks:
344
+ messages.append(
345
+ "Your model supports the 'generate' task, but is "
346
+ f"currently initialized for the '{task}' task. Please "
347
+ "initialize the model using `--task generate`.")
348
+
349
+ raise ValueError(" ".join(messages))
350
351
if prompt_token_ids is not None:
352
parsed_prompts = self._convert_v1_inputs(
@@ -692,10 +719,18 @@ class LLM:
719
considered legacy and may be deprecated in the future. You should
720
instead pass them via the ``inputs`` parameter.
721
"""
695
- if not self.llm_engine.model_config.embedding_mode:
696
- raise ValueError(
697
- "LLM.encode() is only supported for embedding models (XModel)."
698
- )
722
+ task = self.llm_engine.model_config.task
723
+ if task != "embedding":
724
+ messages = ["LLM.encode() is only supported for embedding models."]
725
+
726
+ supported_tasks = self.llm_engine.model_config.supported_tasks
727
+ if "embedding" in supported_tasks:
728
+ messages.append(
729
+ "Your model supports the 'embedding' task, but is "
730
+ f"currently initialized for the '{task}' task. Please "
731
+ "initialize the model using `--task embedding`.")
732
+
733
+ raise ValueError(" ".join(messages))
734
735
if prompt_token_ids is not None:
736
parsed_prompts = self._convert_v1_inputs(
@@ -905,6 +940,3 @@ class LLM:
940
941
def _is_encoder_decoder_model(self):
942
return self.llm_engine.is_encoder_decoder_model()
908
-
909
- def _is_embedding_model(self):
910
- return self.llm_engine.is_embedding_model()
vllm/entrypoints/openai/serving_embedding.py
+2
-1
@@ -83,7 +83,8 @@ class OpenAIServingEmbedding(OpenAIServing):
83
lora_modules=None,
84
prompt_adapters=None,
85
request_logger=request_logger)
86
- self._enabled = self._check_embedding_mode(model_config.embedding_mode)
86
+ self._enabled = self._check_embedding_mode(
87
+ model_config.task == "embedding")
88
89
async def create_embedding(
90
self,
vllm/utils.py
+47
-3
@@ -1034,10 +1034,54 @@ def identity(value: T) -> T:
1034
F = TypeVar('F', bound=Callable[..., Any])
1035
1036
1037
+def deprecate_args(
1038
+ start_index: int,
1039
+ is_deprecated: Union[bool, Callable[[], bool]] = True,
1040
+ additional_message: Optional[str] = None,
1041
+) -> Callable[[F], F]:
1042
+
1043
+ if not callable(is_deprecated):
1044
+ is_deprecated = partial(identity, is_deprecated)
1045
+
1046
+ def wrapper(fn: F) -> F:
1047
+
1048
+ params = inspect.signature(fn).parameters
1049
+ pos_types = (
1050
+ inspect.Parameter.POSITIONAL_ONLY,
1051
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
1052
+ )
1053
+ pos_kws = [
1054
+ kw for kw, param in params.items() if param.kind in pos_types
1055
+ ]
1056
+
1057
+ @wraps(fn)
1058
+ def inner(*args, **kwargs):
1059
+ if is_deprecated():
1060
+ deprecated_args = pos_kws[start_index:len(args)]
1061
+ if deprecated_args:
1062
+ msg = (
1063
+ f"The positional arguments {deprecated_args} are "
1064
+ "deprecated and will be removed in a future update.")
1065
+ if additional_message is not None:
1066
+ msg += f" {additional_message}"
1067
+
1068
+ warnings.warn(
1069
+ DeprecationWarning(msg),
1070
+ stacklevel=3, # The inner function takes up one level
1071
+ )
1072
+
1073
+ return fn(*args, **kwargs)
1074
+
1075
+ return inner # type: ignore
1076
+
1077
+ return wrapper
1078
+
1079
+
1080
def deprecate_kwargs(
1038
- *kws: str,
1039
- is_deprecated: Union[bool, Callable[[], bool]] = True,
1040
- additional_message: Optional[str] = None) -> Callable[[F], F]:
1081
+ *kws: str,
1082
+ is_deprecated: Union[bool, Callable[[], bool]] = True,
1083
+ additional_message: Optional[str] = None,
1084
+) -> Callable[[F], F]:
1085
deprecated_kws = set(kws)
1086
1087
if not callable(is_deprecated):
vllm/worker/worker.py
+1
-4
@@ -92,7 +92,7 @@ class Worker(LocalOrDistributedWorkerBase):
92
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
93
if model_runner_cls is not None:
94
ModelRunnerClass = model_runner_cls
95
- elif self._is_embedding_model():
95
+ elif model_config.task == "embedding":
96
ModelRunnerClass = EmbeddingModelRunner
97
elif self._is_encoder_decoder_model():
98
ModelRunnerClass = EncoderDecoderModelRunner
@@ -147,9 +147,6 @@ class Worker(LocalOrDistributedWorkerBase):
147
def _is_encoder_decoder_model(self):
148
return self.model_config.is_encoder_decoder_model
149
150
- def _is_embedding_model(self):
151
- return self.model_config.is_embedding_model
152
-
150
def init_device(self) -> None:
151
if self.device_config.device.type == "cuda":
152
# torch.distributed.all_reduce does not free the input tensor until