@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 051eaf6db3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Add user-configurable task for models that support both generation and embedding (#9424)

Cyrus Leung committed Oct 19, 2024 at 02:31 UTC 051eaf6db3d8feeb0779a4e942aadc85eda2f8b2
33 files changed +451 -201
docs/source/models/supported_models.rst
+8
@@ -294,6 +294,10 @@ Text Embedding
294 -
295 - ✅︎
296
297 +.. important::
298 + Some model architectures support both generation and embedding tasks.
299 + In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
300 +
301 Reward Modeling
302 ---------------
303
@@ -482,6 +486,10 @@ Multimodal Embedding
486 - 🚧
487 - ✅︎
488
489 +.. important::
490 + Some model architectures support both generation and embedding tasks.
491 + In this case, you have to pass :code:`--task embedding` to run the model in embedding mode.
492 +
493 ----
494
495 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
docs/source/models/vlm.rst
+2 -2
@@ -181,8 +181,8 @@ Below is an example on how to launch the same ``microsoft/Phi-3.5-vision-instruc
181
182 .. code-block:: bash
183
184 - vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
185 - --trust-remote-code --limit-mm-per-prompt image=2
184 + vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
185 + --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
186
187 .. important::
188 Since OpenAI Vision API is based on `Chat Completions <https://platform.openai.com/docs/api-reference/chat>`_ API,
examples/offline_inference_vision_language_embedding.py
+1
@@ -7,6 +7,7 @@ prompt = "<|image_1|> Represent the given image with the following question: Wha
7 # Create an LLM.
8 llm = LLM(
9 model="TIGER-Lab/VLM2Vec-Full",
10 + task="embedding",
11 trust_remote_code=True,
12 max_model_len=4096,
13 max_num_seqs=2,
examples/openai_api_client_for_multimodal.py
+2 -2
@@ -7,8 +7,8 @@ Launch the vLLM server with the following command:
7 vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
8
9 (multi-image inference with Phi-3.5-vision-instruct)
10 -vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
11 - --trust-remote-code --limit-mm-per-prompt image=2
10 +vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
11 + --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
12
13 (audio inference with Ultravox)
14 vllm serve fixie-ai/ultravox-v0_3 --max-model-len 4096
tests/conftest.py
+3 -1
@@ -25,7 +25,7 @@ from tests.models.utils import (TokensTextLogprobs,
25 from vllm import LLM, SamplingParams
26 from vllm.assets.image import ImageAsset
27 from vllm.assets.video import VideoAsset
28 -from vllm.config import TokenizerPoolConfig
28 +from vllm.config import TaskOption, TokenizerPoolConfig
29 from vllm.connections import global_http_connection
30 from vllm.distributed import (destroy_distributed_environment,
31 destroy_model_parallel,
@@ -619,6 +619,7 @@ class VllmRunner:
619 def __init__(
620 self,
621 model_name: str,
622 + task: TaskOption = "auto",
623 tokenizer_name: Optional[str] = None,
624 # Use smaller max model length, otherwise bigger model cannot run due
625 # to kv cache size limit.
@@ -634,6 +635,7 @@ class VllmRunner:
635 ) -> None:
636 self.model = LLM(
637 model=model_name,
638 + task=task,
639 tokenizer=tokenizer_name,
640 trust_remote_code=True,
641 dtype=dtype,
tests/core/test_chunked_prefill_scheduler.py
+13 -2
@@ -33,7 +33,8 @@ def test_simple():
33 num_seq_group = 4
34 max_model_len = 16
35 max_num_batched_tokens = 64
36 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
36 + scheduler_config = SchedulerConfig("generate",
37 + max_num_batched_tokens,
38 num_seq_group,
39 max_model_len,
40 enable_chunked_prefill=True)
@@ -78,6 +79,7 @@ def test_chunk():
79 max_model_len = 80
80 max_num_batched_tokens = 64
81 scheduler_config = SchedulerConfig(
82 + "generate",
83 max_num_batched_tokens,
84 max_seqs,
85 max_model_len,
@@ -126,6 +128,7 @@ def test_complex():
128 max_model_len = 80
129 max_num_batched_tokens = 64
130 scheduler_config = SchedulerConfig(
131 + "generate",
132 max_num_batched_tokens,
133 max_seqs,
134 max_model_len,
@@ -196,6 +199,7 @@ def test_maximal_decoding():
199 max_model_len = 8
200 max_num_batched_tokens = 2
201 scheduler_config = SchedulerConfig(
202 + "generate",
203 max_num_batched_tokens,
204 max_seqs,
205 max_model_len,
@@ -289,6 +293,7 @@ def test_prompt_limit():
293 max_model_len = 64
294 max_num_batched_tokens = 32
295 scheduler_config = SchedulerConfig(
296 + "generate",
297 max_num_batched_tokens,
298 max_seqs,
299 max_model_len,
@@ -321,7 +326,8 @@ def test_prompt_limit_exceed():
326 max_seqs = 64
327 max_model_len = 32
328 max_num_batched_tokens = 64
324 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
329 + scheduler_config = SchedulerConfig("generate",
330 + max_num_batched_tokens,
331 max_seqs,
332 max_model_len,
333 enable_chunked_prefill=True)
@@ -348,6 +354,7 @@ def test_swap():
354 max_model_len = 200
355 max_num_batched_tokens = 30
356 scheduler_config = SchedulerConfig(
357 + "generate",
358 max_num_batched_tokens,
359 max_seqs,
360 max_model_len,
@@ -404,6 +411,7 @@ def test_running_prefill_prioritized_over_swap():
411 max_model_len = 200
412 max_num_batched_tokens = 30
413 scheduler_config = SchedulerConfig(
414 + "generate",
415 max_num_batched_tokens,
416 max_seqs,
417 max_model_len,
@@ -498,6 +506,7 @@ def test_chunked_prefill_preempt():
506 max_model_len = 200
507 max_num_batched_tokens = 30
508 scheduler_config = SchedulerConfig(
509 + "generate",
510 max_num_batched_tokens,
511 max_seqs,
512 max_model_len,
@@ -563,6 +572,7 @@ def test_chunked_prefill_max_seqs():
572 max_model_len = 80
573 max_num_batched_tokens = 64
574 scheduler_config = SchedulerConfig(
575 + "generate",
576 max_num_batched_tokens,
577 max_seqs,
578 max_model_len,
@@ -617,6 +627,7 @@ def test_perfix_caching():
627 max_model_len = 80
628 max_num_batched_tokens = 64
629 scheduler_config = SchedulerConfig(
630 + "generate",
631 max_num_batched_tokens,
632 max_seqs,
633 max_model_len,
tests/core/test_scheduler.py
+32 -24
@@ -20,9 +20,10 @@ from .utils import (append_new_token, append_new_token_seq_group,
20 def test_scheduler_add_seq_group():
21 block_size = 4
22 scheduler_config = SchedulerConfig(
23 - 100,
24 - 64,
25 - 1,
23 + "generate",
24 + max_num_batched_tokens=100,
25 + max_num_seqs=64,
26 + max_model_len=1,
27 )
28 cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
29 cache_config.num_cpu_blocks = 4
@@ -42,9 +43,10 @@ def test_scheduler_add_seq_group():
43 def test_scheduler_abort_seq_group():
44 block_size = 4
45 scheduler_config = SchedulerConfig(
45 - 100,
46 - 64,
47 - 1,
46 + "generate",
47 + max_num_batched_tokens=100,
48 + max_num_seqs=64,
49 + max_model_len=1,
50 )
51 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
52 cache_config.num_cpu_blocks = 4
@@ -70,9 +72,10 @@ def test_scheduler_schedule_simple():
72 num_seq_group = 4
73 max_model_len = 16
74 scheduler_config = SchedulerConfig(
73 - 64,
74 - num_seq_group,
75 - max_model_len,
75 + "generate",
76 + max_num_batched_tokens=64,
77 + max_num_seqs=num_seq_group,
78 + max_model_len=max_model_len,
79 )
80 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
81 cache_config.num_cpu_blocks = 8
@@ -114,9 +117,10 @@ def test_scheduler_prefill_prioritized():
117 max_model_len = 30
118 max_batched_num_tokens = 30
119 scheduler_config = SchedulerConfig(
117 - max_batched_num_tokens,
118 - 2,
119 - max_model_len,
120 + "generate",
121 + max_num_batched_tokens=max_batched_num_tokens,
122 + max_num_seqs=2,
123 + max_model_len=max_model_len,
124 )
125 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
126 cache_config.num_cpu_blocks = 16
@@ -145,9 +149,10 @@ def test_scheduler_schedule_preempt_abort():
149 block_size = 4
150 max_model_len = 16
151 scheduler_config = SchedulerConfig(
148 - 64,
149 - 2,
150 - max_model_len,
152 + "generate",
153 + max_num_batched_tokens=64,
154 + max_num_seqs=2,
155 + max_model_len=max_model_len,
156 )
157 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
158 cache_config.num_cpu_blocks = 2
@@ -204,9 +209,10 @@ def test_scheduler_max_seqs():
209 max_seq_group = 2
210 max_model_len = 16
211 scheduler_config = SchedulerConfig(
207 - 64,
208 - max_seq_group,
209 - max_model_len,
212 + "generate",
213 + max_num_batched_tokens=64,
214 + max_num_seqs=max_seq_group,
215 + max_model_len=max_model_len,
216 )
217 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
218 cache_config.num_cpu_blocks = 8
@@ -248,9 +254,10 @@ def test_scheduler_max_seqs():
254 def test_scheduler_delay_factor():
255 block_size = 4
256 scheduler_config = SchedulerConfig(
251 - 100,
252 - 64,
253 - 16,
257 + "generate",
258 + max_num_batched_tokens=100,
259 + max_num_seqs=64,
260 + max_model_len=16,
261 delay_factor=0.5,
262 )
263 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
@@ -350,9 +357,10 @@ def initialize_scheduler(
357 ):
358 block_size = block_size
359 scheduler_config = SchedulerConfig(
353 - max_token_budget,
354 - max_num_seqs,
355 - max_model_len,
360 + "generate",
361 + max_num_batched_tokens=max_token_budget,
362 + max_num_seqs=max_num_seqs,
363 + max_model_len=max_model_len,
364 )
365 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
366 cache_config.num_cpu_blocks = num_cpu_blocks
tests/core/test_scheduler_encoder_decoder.py
+6 -1
@@ -36,7 +36,12 @@ def test_scheduler_schedule_simple_encoder_decoder():
36 block_size = 4
37 num_seq_group = 4
38 max_model_len = 16
39 - scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
39 + scheduler_config = SchedulerConfig(
40 + task="generate",
41 + max_num_batched_tokens=64,
42 + max_num_seqs=num_seq_group,
43 + max_model_len=max_model_len,
44 + )
45 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
46 cache_config.num_cpu_blocks = 16 # enc and dec prompts per seq_group
47 cache_config.num_gpu_blocks = 16 # enc and dec prompts per seq_group
tests/distributed/test_pipeline_parallel.py
+19 -4
@@ -11,6 +11,7 @@ from typing import List, Literal, NamedTuple, Optional
11
12 import pytest
13
14 +from vllm.config import TaskOption
15 from vllm.logger import init_logger
16
17 from ..utils import compare_two_settings, fork_new_process_for_each_test
@@ -31,6 +32,7 @@ class ParallelSetup(NamedTuple):
32 class PPTestSettings:
33 parallel_setups: List[ParallelSetup]
34 distributed_backends: List[str]
35 + task: TaskOption
36 trust_remote_code: bool
37 tokenizer_mode: Optional[str]
38
@@ -39,6 +41,7 @@ class PPTestSettings:
41 *,
42 tp_base: int = 1,
43 pp_base: int = 2,
44 + task: TaskOption = "auto",
45 trust_remote_code: bool = False,
46 tokenizer_mode: Optional[str] = None,
47 ):
@@ -66,6 +69,7 @@ class PPTestSettings:
69 chunked_prefill=False),
70 ],
71 distributed_backends=["mp", "ray"],
72 + task=task,
73 trust_remote_code=trust_remote_code,
74 tokenizer_mode=tokenizer_mode,
75 )
@@ -75,6 +79,7 @@ class PPTestSettings:
79 *,
80 tp_base: int = 1,
81 pp_base: int = 2,
82 + task: TaskOption = "auto",
83 trust_remote_code: bool = False,
84 tokenizer_mode: Optional[str] = None,
85 ):
@@ -86,6 +91,7 @@ class PPTestSettings:
91 chunked_prefill=False),
92 ],
93 distributed_backends=["mp"],
94 + task=task,
95 trust_remote_code=trust_remote_code,
96 tokenizer_mode=tokenizer_mode,
97 )
@@ -94,7 +100,7 @@ class PPTestSettings:
100 for parallel_setup in self.parallel_setups:
101 for distributed_backend in self.distributed_backends:
102 yield (model_name, parallel_setup, distributed_backend,
97 - self.trust_remote_code, self.tokenizer_mode)
103 + self.task, self.trust_remote_code, self.tokenizer_mode)
104
105
106 # NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
@@ -213,6 +219,7 @@ def _compare_tp(
219 model_name: str,
220 parallel_setup: ParallelSetup,
221 distributed_backend: str,
222 + task: TaskOption,
223 trust_remote_code: bool,
224 tokenizer_mode: Optional[str],
225 num_gpus_available: int,
@@ -240,6 +247,8 @@ def _compare_tp(
247 common_args.append("--enable-chunked-prefill")
248 if eager_mode:
249 common_args.append("--enforce-eager")
250 + if task != "auto":
251 + common_args.extend(["--task", task])
252 if trust_remote_code:
253 common_args.append("--trust-remote-code")
254 if tokenizer_mode:
@@ -297,7 +306,7 @@ def _compare_tp(
306
307
308 @pytest.mark.parametrize(
300 - ("model_name", "parallel_setup", "distributed_backend",
309 + ("model_name", "parallel_setup", "distributed_backend", "task",
310 "trust_remote_code", "tokenizer_mode"),
311 [
312 params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
@@ -310,6 +319,7 @@ def test_tp_language_generation(
319 model_name: str,
320 parallel_setup: ParallelSetup,
321 distributed_backend: str,
322 + task: TaskOption,
323 trust_remote_code: bool,
324 tokenizer_mode: Optional[str],
325 num_gpus_available,
@@ -317,6 +327,7 @@ def test_tp_language_generation(
327 _compare_tp(model_name,
328 parallel_setup,
329 distributed_backend,
330 + task,
331 trust_remote_code,
332 tokenizer_mode,
333 num_gpus_available,
@@ -324,7 +335,7 @@ def test_tp_language_generation(
335
336
337 @pytest.mark.parametrize(
327 - ("model_name", "parallel_setup", "distributed_backend",
338 + ("model_name", "parallel_setup", "distributed_backend", "task",
339 "trust_remote_code", "tokenizer_mode"),
340 [
341 params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
@@ -337,6 +348,7 @@ def test_tp_language_embedding(
348 model_name: str,
349 parallel_setup: ParallelSetup,
350 distributed_backend: str,
351 + task: TaskOption,
352 trust_remote_code: bool,
353 tokenizer_mode: Optional[str],
354 num_gpus_available,
@@ -344,6 +356,7 @@ def test_tp_language_embedding(
356 _compare_tp(model_name,
357 parallel_setup,
358 distributed_backend,
359 + task,
360 trust_remote_code,
361 tokenizer_mode,
362 num_gpus_available,
@@ -351,7 +364,7 @@ def test_tp_language_embedding(
364
365
366 @pytest.mark.parametrize(
354 - ("model_name", "parallel_setup", "distributed_backend",
367 + ("model_name", "parallel_setup", "distributed_backend", "task",
368 "trust_remote_code", "tokenizer_mode"),
369 [
370 params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
@@ -364,6 +377,7 @@ def test_tp_multimodal_generation(
377 model_name: str,
378 parallel_setup: ParallelSetup,
379 distributed_backend: str,
380 + task: TaskOption,
381 trust_remote_code: bool,
382 tokenizer_mode: Optional[str],
383 num_gpus_available,
@@ -371,6 +385,7 @@ def test_tp_multimodal_generation(
385 _compare_tp(model_name,
386 parallel_setup,
387 distributed_backend,
388 + task,
389 trust_remote_code,
390 tokenizer_mode,
391 num_gpus_available,
tests/entrypoints/llm/test_chat.py new
+92
@@ -0,0 +1,92 @@
1 +from typing import List
2 +
3 +import pytest
4 +
5 +from vllm import LLM
6 +
7 +from ..openai.test_vision import TEST_IMAGE_URLS
8 +
9 +
10 +def test_chat():
11 + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
12 +
13 + prompt1 = "Explain the concept of entropy."
14 + messages = [
15 + {
16 + "role": "system",
17 + "content": "You are a helpful assistant"
18 + },
19 + {
20 + "role": "user",
21 + "content": prompt1
22 + },
23 + ]
24 + outputs = llm.chat(messages)
25 + assert len(outputs) == 1
26 +
27 +
28 +def test_multi_chat():
29 + llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
30 +
31 + prompt1 = "Explain the concept of entropy."
32 + prompt2 = "Explain what among us is."
33 +
34 + conversation1 = [
35 + {
36 + "role": "system",
37 + "content": "You are a helpful assistant"
38 + },
39 + {
40 + "role": "user",
41 + "content": prompt1
42 + },
43 + ]
44 +
45 + conversation2 = [
46 + {
47 + "role": "system",
48 + "content": "You are a helpful assistant"
49 + },
50 + {
51 + "role": "user",
52 + "content": prompt2
53 + },
54 + ]
55 +
56 + messages = [conversation1, conversation2]
57 +
58 + outputs = llm.chat(messages)
59 + assert len(outputs) == 2
60 +
61 +
62 +@pytest.mark.parametrize("image_urls",
63 + [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
64 +def test_chat_multi_image(image_urls: List[str]):
65 + llm = LLM(
66 + model="microsoft/Phi-3.5-vision-instruct",
67 + dtype="bfloat16",
68 + max_model_len=4096,
69 + max_num_seqs=5,
70 + enforce_eager=True,
71 + trust_remote_code=True,
72 + limit_mm_per_prompt={"image": 2},
73 + )
74 +
75 + messages = [{
76 + "role":
77 + "user",
78 + "content": [
79 + *({
80 + "type": "image_url",
81 + "image_url": {
82 + "url": image_url
83 + }
84 + } for image_url in image_urls),
85 + {
86 + "type": "text",
87 + "text": "What's in this image?"
88 + },
89 + ],
90 + }]
91 + outputs = llm.chat(messages)
92 + assert len(outputs) >= 0
tests/entrypoints/llm/test_generate.py
-88
@@ -6,7 +6,6 @@ import pytest
6 from vllm import LLM, RequestOutput, SamplingParams
7
8 from ...conftest import cleanup
9 -from ..openai.test_vision import TEST_IMAGE_URLS
9
10 MODEL_NAME = "facebook/opt-125m"
11
@@ -104,90 +103,3 @@ def test_multiple_sampling_params(llm: LLM):
103 # sampling_params is None, default params should be applied
104 outputs = llm.generate(PROMPTS, sampling_params=None)
105 assert len(PROMPTS) == len(outputs)
107 -
108 -
109 -def test_chat():
110 -
111 - llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
112 -
113 - prompt1 = "Explain the concept of entropy."
114 - messages = [
115 - {
116 - "role": "system",
117 - "content": "You are a helpful assistant"
118 - },
119 - {
120 - "role": "user",
121 - "content": prompt1
122 - },
123 - ]
124 - outputs = llm.chat(messages)
125 - assert len(outputs) == 1
126 -
127 -
128 -def test_multi_chat():
129 -
130 - llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
131 -
132 - prompt1 = "Explain the concept of entropy."
133 - prompt2 = "Explain what among us is."
134 -
135 - conversation1 = [
136 - {
137 - "role": "system",
138 - "content": "You are a helpful assistant"
139 - },
140 - {
141 - "role": "user",
142 - "content": prompt1
143 - },
144 - ]
145 -
146 - conversation2 = [
147 - {
148 - "role": "system",
149 - "content": "You are a helpful assistant"
150 - },
151 - {
152 - "role": "user",
153 - "content": prompt2
154 - },
155 - ]
156 -
157 - messages = [conversation1, conversation2]
158 -
159 - outputs = llm.chat(messages)
160 - assert len(outputs) == 2
161 -
162 -
163 -@pytest.mark.parametrize("image_urls",
164 - [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
165 -def test_chat_multi_image(image_urls: List[str]):
166 - llm = LLM(
167 - model="microsoft/Phi-3.5-vision-instruct",
168 - dtype="bfloat16",
169 - max_model_len=4096,
170 - max_num_seqs=5,
171 - enforce_eager=True,
172 - trust_remote_code=True,
173 - limit_mm_per_prompt={"image": 2},
174 - )
175 -
176 - messages = [{
177 - "role":
178 - "user",
179 - "content": [
180 - *({
181 - "type": "image_url",
182 - "image_url": {
183 - "url": image_url
184 - }
185 - } for image_url in image_urls),
186 - {
187 - "type": "text",
188 - "text": "What's in this image?"
189 - },
190 - ],
191 - }]
192 - outputs = llm.chat(messages)
193 - assert len(outputs) >= 0
tests/entrypoints/llm/test_init.py new
+22
@@ -0,0 +1,22 @@
1 +import pytest
2 +
3 +from vllm import LLM
4 +
5 +from ...utils import error_on_warning
6 +
7 +MODEL_NAME = "facebook/opt-125m"
8 +
9 +
10 +def test_pos_args_deprecated():
11 + with error_on_warning(DeprecationWarning):
12 + LLM(model=MODEL_NAME, tokenizer=MODEL_NAME)
13 +
14 + with error_on_warning(DeprecationWarning):
15 + LLM(MODEL_NAME, tokenizer=MODEL_NAME)
16 +
17 + with pytest.warns(DeprecationWarning, match="'tokenizer'"):
18 + LLM(MODEL_NAME, MODEL_NAME)
19 +
20 + with pytest.warns(DeprecationWarning,
21 + match="'tokenizer', 'tokenizer_mode'"):
22 + LLM(MODEL_NAME, MODEL_NAME, "auto")
tests/entrypoints/openai/test_serving_chat.py
+1 -1
@@ -22,12 +22,12 @@ class MockHFConfig:
22
23 @dataclass
24 class MockModelConfig:
25 + task = "generate"
26 tokenizer = MODEL_NAME
27 trust_remote_code = False
28 tokenizer_mode = "auto"
29 max_model_len = 100
30 tokenizer_revision = None
30 - embedding_mode = False
31 multimodal_config = MultiModalConfig()
32 hf_config = MockHFConfig()
33
tests/entrypoints/openai/test_vision.py
+2
@@ -23,6 +23,8 @@ TEST_IMAGE_URLS = [
23 @pytest.fixture(scope="module")
24 def server():
25 args = [
26 + "--task",
27 + "generate",
28 "--dtype",
29 "bfloat16",
30 "--max-model-len",
tests/entrypoints/test_chat_utils.py
+2 -1
@@ -18,7 +18,8 @@ PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
18 @pytest.fixture(scope="module")
19 def phi3v_model_config():
20 return ModelConfig(PHI3V_MODEL_ID,
21 - PHI3V_MODEL_ID,
21 + task="generate",
22 + tokenizer=PHI3V_MODEL_ID,
23 tokenizer_mode="auto",
24 trust_remote_code=True,
25 dtype="bfloat16",
tests/lora/test_worker.py
+3 -2
@@ -15,7 +15,8 @@ def test_worker_apply_lora(sql_lora_files):
15 worker = Worker(
16 model_config=ModelConfig(
17 "meta-llama/Llama-2-7b-hf",
18 - "meta-llama/Llama-2-7b-hf",
18 + task="auto",
19 + tokenizer="meta-llama/Llama-2-7b-hf",
20 tokenizer_mode="auto",
21 trust_remote_code=False,
22 seed=0,
@@ -27,7 +28,7 @@ def test_worker_apply_lora(sql_lora_files):
28 load_format="dummy",
29 ),
30 parallel_config=ParallelConfig(1, 1, False),
30 - scheduler_config=SchedulerConfig(32, 32, 32),
31 + scheduler_config=SchedulerConfig("generate", 32, 32, 32),
32 device_config=DeviceConfig("cuda"),
33 cache_config=CacheConfig(block_size=16,
34 gpu_memory_utilization=1.,
tests/models/decoder_only/vision_language/test_phi3v.py
+1
@@ -89,6 +89,7 @@ def run_test(
89
90 # max_model_len should be greater than image_feature_size
91 with vllm_runner(model,
92 + task="generate",
93 max_model_len=4096,
94 max_num_seqs=2,
95 dtype=dtype,
tests/models/embedding/vision_language/test_phi3v.py
+1
@@ -28,6 +28,7 @@ def test_models(
28 # if we run HF first, the cuda initialization will be done and it
29 # will hurt multiprocessing backend with fork method (the default method).
30 with vllm_runner(model,
31 + task="embedding",
32 max_model_len=4096,
33 max_num_seqs=2,
34 dtype=dtype,
tests/models/utils.py
+4 -2
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 import torch
5
6 -from vllm.config import ModelConfig
6 +from vllm.config import ModelConfig, TaskOption
7 from vllm.inputs import InputContext
8 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9 from vllm.utils import is_cpu
@@ -248,6 +248,7 @@ def check_logprobs_close(
248
249
250 def build_model_context(model_name: str,
251 + task: TaskOption = "auto",
252 tokenizer_name: Optional[str] = None,
253 trust_remote_code: bool = False,
254 dtype: Optional[Union[str, torch.dtype]] = None,
@@ -273,7 +274,8 @@ def build_model_context(model_name: str,
274
275 model_config = ModelConfig(
276 model_name,
276 - tokenizer_name,
277 + task=task,
278 + tokenizer=tokenizer_name,
279 tokenizer_mode="auto",
280 trust_remote_code=trust_remote_code,
281 dtype=dtype,
tests/multimodal/test_mapper.py
+4
@@ -24,6 +24,7 @@ def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
24
25 model_config = ModelConfig(
26 model=MODEL_NAME,
27 + task="auto",
28 tokenizer=MODEL_NAME,
29 tokenizer_mode="auto",
30 trust_remote_code=False,
@@ -67,6 +68,7 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
68
69 model_config = ModelConfig(
70 model=MODEL_NAME,
71 + task="auto",
72 tokenizer=MODEL_NAME,
73 tokenizer_mode="auto",
74 trust_remote_code=False,
@@ -109,6 +111,7 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
111
112 model_config = ModelConfig(
113 model=MODEL_NAME,
114 + task="auto",
115 tokenizer=MODEL_NAME,
116 tokenizer_mode="auto",
117 trust_remote_code=False,
@@ -139,6 +142,7 @@ def test_image_mapper_multi(image_assets, mm_registry, num_images):
142
143 model_config = ModelConfig(
144 model=MODEL_NAME,
145 + task="auto",
146 tokenizer=MODEL_NAME,
147 tokenizer_mode="auto",
148 trust_remote_code=False,
tests/multimodal/test_processor_kwargs.py
+6 -1
@@ -221,6 +221,7 @@ def test_max_tokens_kwarg_overrides(num_crops):
221 expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
222
223 ctx = build_model_context(MULTIMODAL_MODEL_ID,
224 + task="generate",
225 trust_remote_code=True,
226 mm_processor_kwargs=mm_processor_kwargs,
227 limit_mm_per_prompt={"image": 1})
@@ -256,6 +257,7 @@ def test_max_tokens_kwarg_overrides(num_crops):
257 def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
258 """Ensure that max token calcs filters out invalid mm_processor_kwargs"""
259 ctx = build_model_context(MULTIMODAL_MODEL_ID,
260 + task="generate",
261 trust_remote_code=True,
262 mm_processor_kwargs=mm_processor_kwargs,
263 limit_mm_per_prompt={"image": 1})
@@ -278,12 +280,13 @@ def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
280
281 ### Test overrides for the mapper
282 @pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
281 -def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
283 +def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
284 """Ensure that the mapper processor kwargs can fall back to HF models."""
285 # NOTE - we don't validate bad inputs for the default mapper, because it's
286 # through the automodel interface in transformers, so we can't easily
287 # inspect what kwargs are or are not allowed.
288 ctx = build_model_context(MULTIMODAL_MODEL_ID,
289 + task="generate",
290 trust_remote_code=True,
291 mm_processor_kwargs={"num_crops": num_crops},
292 limit_mm_per_prompt={"image": 1})
@@ -311,6 +314,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
314 init_num_crops, inference_num_crops)
315
316 ctx = build_model_context(MULTIMODAL_MODEL_ID,
317 + task="generate",
318 trust_remote_code=True,
319 mm_processor_kwargs=init_kwargs,
320 limit_mm_per_prompt={"image": 1})
@@ -348,6 +352,7 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
352 """Ensure that custom mappers filters out invalid mm_processor_kwargs"""
353 # Should filter out the init time kwargs
354 ctx = build_model_context(MULTIMODAL_MODEL_ID,
355 + task="generate",
356 trust_remote_code=True,
357 mm_processor_kwargs=mm_processor_kwargs,
358 limit_mm_per_prompt={"image": 1})
tests/quantization/test_configs.py
+2 -1
@@ -57,7 +57,8 @@ def test_auto_gptq(model_arg_exptype: Tuple[str, None, str]) -> None:
57
58 try:
59 model_config = ModelConfig(model_path,
60 - model_path,
60 + task="auto",
61 + tokenizer=model_path,
62 tokenizer_mode="auto",
63 trust_remote_code=False,
64 seed=0,
tests/test_config.py
+50 -7
@@ -2,6 +2,42 @@ import pytest
2
3 from vllm.config import ModelConfig
4
5 +
6 +@pytest.mark.parametrize(("model_id", "expected_task"), [
7 + ("facebook/opt-125m", "generate"),
8 + ("intfloat/e5-mistral-7b-instruct", "embedding"),
9 +])
10 +def test_auto_task(model_id, expected_task):
11 + config = ModelConfig(
12 + model_id,
13 + task="auto",
14 + tokenizer=model_id,
15 + tokenizer_mode="auto",
16 + trust_remote_code=False,
17 + seed=0,
18 + dtype="float16",
19 + )
20 +
21 + assert config.task == expected_task
22 +
23 +
24 +@pytest.mark.parametrize(("model_id", "bad_task"), [
25 + ("facebook/opt-125m", "embedding"),
26 + ("intfloat/e5-mistral-7b-instruct", "generate"),
27 +])
28 +def test_incorrect_task(model_id, bad_task):
29 + with pytest.raises(ValueError, match=r"does not support the .* task"):
30 + ModelConfig(
31 + model_id,
32 + task=bad_task,
33 + tokenizer=model_id,
34 + tokenizer_mode="auto",
35 + trust_remote_code=False,
36 + seed=0,
37 + dtype="float16",
38 + )
39 +
40 +
41 MODEL_IDS_EXPECTED = [
42 ("Qwen/Qwen1.5-7B", 32768),
43 ("mistralai/Mistral-7B-v0.1", 4096),
@@ -14,7 +50,8 @@ def test_disable_sliding_window(model_id_expected):
50 model_id, expected = model_id_expected
51 model_config = ModelConfig(
52 model_id,
17 - model_id,
53 + task="auto",
54 + tokenizer=model_id,
55 tokenizer_mode="auto",
56 trust_remote_code=False,
57 seed=0,
@@ -32,7 +69,8 @@ def test_get_sliding_window():
69 # when use_sliding_window is False.
70 qwen2_model_config = ModelConfig(
71 "Qwen/Qwen1.5-7B",
35 - "Qwen/Qwen1.5-7B",
72 + task="auto",
73 + tokenizer="Qwen/Qwen1.5-7B",
74 tokenizer_mode="auto",
75 trust_remote_code=False,
76 seed=0,
@@ -49,7 +87,8 @@ def test_get_sliding_window():
87
88 mistral_model_config = ModelConfig(
89 "mistralai/Mistral-7B-v0.1",
52 - "mistralai/Mistral-7B-v0.1",
90 + task="auto",
91 + tokenizer="mistralai/Mistral-7B-v0.1",
92 tokenizer_mode="auto",
93 trust_remote_code=False,
94 seed=0,
@@ -70,7 +109,8 @@ def test_rope_customization():
109
110 llama_model_config = ModelConfig(
111 "meta-llama/Meta-Llama-3-8B-Instruct",
73 - "meta-llama/Meta-Llama-3-8B-Instruct",
112 + task="auto",
113 + tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
114 tokenizer_mode="auto",
115 trust_remote_code=False,
116 dtype="float16",
@@ -82,7 +122,8 @@ def test_rope_customization():
122
123 llama_model_config = ModelConfig(
124 "meta-llama/Meta-Llama-3-8B-Instruct",
85 - "meta-llama/Meta-Llama-3-8B-Instruct",
125 + task="auto",
126 + tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
127 tokenizer_mode="auto",
128 trust_remote_code=False,
129 dtype="float16",
@@ -98,7 +139,8 @@ def test_rope_customization():
139
140 longchat_model_config = ModelConfig(
141 "lmsys/longchat-13b-16k",
101 - "lmsys/longchat-13b-16k",
142 + task="auto",
143 + tokenizer="lmsys/longchat-13b-16k",
144 tokenizer_mode="auto",
145 trust_remote_code=False,
146 dtype="float16",
@@ -112,7 +154,8 @@ def test_rope_customization():
154
155 longchat_model_config = ModelConfig(
156 "lmsys/longchat-13b-16k",
115 - "lmsys/longchat-13b-16k",
157 + task="auto",
158 + tokenizer="lmsys/longchat-13b-16k",
159 tokenizer_mode="auto",
160 trust_remote_code=False,
161 dtype="float16",
tests/test_utils.py
+6 -6
@@ -59,7 +59,7 @@ def test_deprecate_kwargs_always():
59 with pytest.warns(DeprecationWarning, match="'old_arg'"):
60 dummy(old_arg=1)
61
62 - with error_on_warning():
62 + with error_on_warning(DeprecationWarning):
63 dummy(new_arg=1)
64
65
@@ -69,10 +69,10 @@ def test_deprecate_kwargs_never():
69 def dummy(*, old_arg: object = None, new_arg: object = None):
70 pass
71
72 - with error_on_warning():
72 + with error_on_warning(DeprecationWarning):
73 dummy(old_arg=1)
74
75 - with error_on_warning():
75 + with error_on_warning(DeprecationWarning):
76 dummy(new_arg=1)
77
78
@@ -86,15 +86,15 @@ def test_deprecate_kwargs_dynamic():
86 with pytest.warns(DeprecationWarning, match="'old_arg'"):
87 dummy(old_arg=1)
88
89 - with error_on_warning():
89 + with error_on_warning(DeprecationWarning):
90 dummy(new_arg=1)
91
92 is_deprecated = False
93
94 - with error_on_warning():
94 + with error_on_warning(DeprecationWarning):
95 dummy(old_arg=1)
96
97 - with error_on_warning():
97 + with error_on_warning(DeprecationWarning):
98 dummy(new_arg=1)
99
100
tests/utils.py
+4 -4
@@ -8,7 +8,7 @@ import time
8 import warnings
9 from contextlib import contextmanager
10 from pathlib import Path
11 -from typing import Any, Callable, Dict, List, Literal, Optional, Union
11 +from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union
12
13 import openai
14 import pytest
@@ -454,13 +454,13 @@ def multi_process_parallel(
454
455
456 @contextmanager
457 -def error_on_warning():
457 +def error_on_warning(category: Type[Warning] = Warning):
458 """
459 Within the scope of this context manager, tests will fail if any warning
460 - is emitted.
460 + of the given category is emitted.
461 """
462 with warnings.catch_warnings():
463 - warnings.simplefilter("error")
463 + warnings.filterwarnings("error", category=category)
464
465 yield
466
vllm/config.py
+54 -23
@@ -1,8 +1,8 @@
1 import enum
2 import json
3 from dataclasses import dataclass, field, fields
4 -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping,
5 - Optional, Tuple, Type, Union)
4 +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
5 + Mapping, Optional, Set, Tuple, Type, Union)
6
7 import torch
8 from transformers import PretrainedConfig
@@ -33,6 +33,9 @@ logger = init_logger(__name__)
33 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
34 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
35
36 +Task = Literal["generate", "embedding"]
37 +TaskOption = Literal["auto", Task]
38 +
39
40 class ModelConfig:
41 """Configuration for the model.
@@ -40,7 +43,11 @@ class ModelConfig:
43 Args:
44 model: Name or path of the huggingface model to use.
45 It is also used as the content for `model_name` tag in metrics
43 - output when `served_model_name` is not specified.
46 + output when `served_model_name` is not specified.
47 + task: The task to use the model for. Each vLLM instance only supports
48 + one task, even if the same model can be used for multiple tasks.
49 + When the model only supports one task, "auto" can be used to select
50 + it; otherwise, you must specify explicitly which task to use.
51 tokenizer: Name or path of the huggingface tokenizer to use.
52 tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
53 available, "slow" will always use the slow tokenizer, and
@@ -108,6 +115,7 @@ class ModelConfig:
115
116 def __init__(self,
117 model: str,
118 + task: TaskOption,
119 tokenizer: str,
120 tokenizer_mode: str,
121 trust_remote_code: bool,
@@ -207,7 +215,11 @@ class ModelConfig:
215
216 self.override_neuron_config = override_neuron_config if is_neuron(
217 ) else None
210 - self._verify_embedding_mode()
218 +
219 + supported_tasks, task = self._resolve_task(task, self.hf_config)
220 + self.supported_tasks = supported_tasks
221 + self.task: Final = task
222 +
223 self._verify_quantization()
224 self._verify_cuda_graph()
225 self._verify_bnb_config()
@@ -241,18 +253,41 @@ class ModelConfig:
253 "either 'auto', 'slow' or 'mistral'.")
254 self.tokenizer_mode = tokenizer_mode
255
244 - def _verify_embedding_mode(self) -> None:
245 - architectures = getattr(self.hf_config, "architectures", [])
256 + def _resolve_task(
257 + self,
258 + task_option: TaskOption,
259 + hf_config: PretrainedConfig,
260 + ) -> Tuple[Set[Task], Task]:
261 + architectures = getattr(hf_config, "architectures", [])
262 +
263 + task_support: Dict[Task, bool] = {
264 + # NOTE: Listed from highest to lowest priority,
265 + # in case the model supports multiple of them
266 + "generate": ModelRegistry.is_text_generation_model(architectures),
267 + "embedding": ModelRegistry.is_embedding_model(architectures),
268 + }
269 + supported_tasks_lst: List[Task] = [
270 + task for task, is_supported in task_support.items() if is_supported
271 + ]
272 + supported_tasks = set(supported_tasks_lst)
273 +
274 + if task_option == "auto":
275 + selected_task = next(iter(supported_tasks_lst))
276
247 - # TODO: Allow the same model architecture to be specified as either
248 - # generation or embedding model
249 - if "Phi3VForCausalLM" in architectures:
250 - # Match both remote and local names
251 - embedding_mode = "/VLM2Vec" in self.model
277 + if len(supported_tasks) > 1:
278 + logger.info(
279 + "This model supports multiple tasks: %s. "
280 + "Defaulting to '%s'.", supported_tasks, selected_task)
281 else:
253 - embedding_mode = ModelRegistry.is_embedding_model(architectures)
282 + if task_option not in supported_tasks:
283 + msg = (
284 + f"This model does not support the '{task_option}' task. "
285 + f"Supported tasks: {supported_tasks}")
286 + raise ValueError(msg)
287 +
288 + selected_task = task_option
289
255 - self.embedding_mode = embedding_mode
290 + return supported_tasks, selected_task
291
292 def _parse_quant_hf_config(self):
293 quant_cfg = getattr(self.hf_config, "quantization_config", None)
@@ -401,7 +436,7 @@ class ModelConfig:
436
437 # Async postprocessor is not necessary with embedding mode
438 # since there is no token generation
404 - if self.embedding_mode:
439 + if self.task == "embedding":
440 self.use_async_output_proc = False
441
442 # Reminder: Please update docs/source/serving/compatibility_matrix.rst
@@ -582,11 +617,6 @@ class ModelConfig:
617 (hasattr(self.hf_config, "text_config") and getattr(
618 self.hf_config.text_config, "is_encoder_decoder", False)))
619
585 - @property
586 - def is_embedding_model(self) -> bool:
587 - """Extract the embedding model flag."""
588 - return self.embedding_mode
589 -
620 @property
621 def is_multimodal_model(self) -> bool:
622 return self.multimodal_config is not None
@@ -943,6 +973,7 @@ class SchedulerConfig:
973 """Scheduler configuration.
974
975 Args:
976 + task: The task to use the model for.
977 max_num_batched_tokens: Maximum number of tokens to be processed in
978 a single iteration.
979 max_num_seqs: Maximum number of sequences to be processed in a single
@@ -957,7 +988,6 @@ class SchedulerConfig:
988 prompt latency) before scheduling next prompt.
989 enable_chunked_prefill: If True, prefill requests can be chunked based
990 on the remaining max_num_batched_tokens.
960 - embedding_mode: Whether the running model is for embedding.
991 preemption_mode: Whether to perform preemption by swapping or
992 recomputation. If not specified, we determine the mode as follows:
993 We use recomputation by default since it incurs lower overhead than
@@ -972,13 +1002,13 @@ class SchedulerConfig:
1002 """
1003
1004 def __init__(self,
1005 + task: Task,
1006 max_num_batched_tokens: Optional[int],
1007 max_num_seqs: int,
1008 max_model_len: int,
1009 num_lookahead_slots: int = 0,
1010 delay_factor: float = 0.0,
1011 enable_chunked_prefill: bool = False,
981 - embedding_mode: bool = False,
1012 is_multimodal_model: bool = False,
1013 preemption_mode: Optional[str] = None,
1014 num_scheduler_steps: int = 1,
@@ -1002,7 +1032,7 @@ class SchedulerConfig:
1032 # for higher throughput.
1033 max_num_batched_tokens = max(max_model_len, 2048)
1034
1005 - if embedding_mode:
1035 + if task == "embedding":
1036 # For embedding, choose specific value for higher throughput
1037 max_num_batched_tokens = max(
1038 max_num_batched_tokens,
@@ -1022,12 +1052,12 @@ class SchedulerConfig:
1052 "Chunked prefill is enabled with max_num_batched_tokens=%d.",
1053 self.max_num_batched_tokens)
1054
1055 + self.task: Final = task
1056 self.max_num_seqs = max_num_seqs
1057 self.max_model_len = max_model_len
1058 self.num_lookahead_slots = num_lookahead_slots
1059 self.delay_factor = delay_factor
1060 self.chunked_prefill_enabled = enable_chunked_prefill
1030 - self.embedding_mode = embedding_mode
1061 self.preemption_mode = preemption_mode
1062 self.num_scheduler_steps = num_scheduler_steps
1063 self.multi_step_stream_outputs = multi_step_stream_outputs
@@ -1239,6 +1269,7 @@ class SpeculativeConfig:
1269 ngram_prompt_lookup_min = 0
1270 draft_model_config = ModelConfig(
1271 model=speculative_model,
1272 + task=target_model_config.task,
1273 tokenizer=target_model_config.tokenizer,
1274 tokenizer_mode=target_model_config.tokenizer_mode,
1275 trust_remote_code=target_model_config.trust_remote_code,
vllm/core/scheduler.py
+1 -1
@@ -313,7 +313,7 @@ class Scheduler:
313 self.lora_config = lora_config
314
315 version = "selfattn"
316 - if (self.scheduler_config.embedding_mode
316 + if (self.scheduler_config.task == "embedding"
317 or self.cache_config.is_attention_free):
318 version = "placeholder"
319
vllm/engine/arg_utils.py
+14 -3
@@ -3,7 +3,7 @@ import dataclasses
3 import json
4 from dataclasses import dataclass
5 from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6 - Tuple, Type, Union, cast)
6 + Tuple, Type, Union, cast, get_args)
7
8 import torch
9
@@ -12,7 +12,7 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12 DeviceConfig, EngineConfig, LoadConfig, LoadFormat,
13 LoRAConfig, ModelConfig, ObservabilityConfig,
14 ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15 - SpeculativeConfig, TokenizerPoolConfig)
15 + SpeculativeConfig, TaskOption, TokenizerPoolConfig)
16 from vllm.executor.executor_base import ExecutorBase
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -84,6 +84,7 @@ class EngineArgs:
84 model: str = 'facebook/opt-125m'
85 served_model_name: Optional[Union[str, List[str]]] = None
86 tokenizer: Optional[str] = None
87 + task: TaskOption = "auto"
88 skip_tokenizer_init: bool = False
89 tokenizer_mode: str = 'auto'
90 trust_remote_code: bool = False
@@ -198,6 +199,15 @@ class EngineArgs:
199 type=str,
200 default=EngineArgs.model,
201 help='Name or path of the huggingface model to use.')
202 + parser.add_argument(
203 + '--task',
204 + default=EngineArgs.task,
205 + choices=get_args(TaskOption),
206 + help='The task to use the model for. Each vLLM instance only '
207 + 'supports one task, even if the same model can be used for '
208 + 'multiple tasks. When the model only supports one task, "auto" '
209 + 'can be used to select it; otherwise, you must specify explicitly '
210 + 'which task to use.')
211 parser.add_argument(
212 '--tokenizer',
213 type=nullable_str,
@@ -838,6 +848,7 @@ class EngineArgs:
848 def create_model_config(self) -> ModelConfig:
849 return ModelConfig(
850 model=self.model,
851 + task=self.task,
852 # We know this is not None because we set it in __post_init__
853 tokenizer=cast(str, self.tokenizer),
854 tokenizer_mode=self.tokenizer_mode,
@@ -1026,13 +1037,13 @@ class EngineArgs:
1037 " please file an issue with detailed information.")
1038
1039 scheduler_config = SchedulerConfig(
1040 + task=model_config.task,
1041 max_num_batched_tokens=self.max_num_batched_tokens,
1042 max_num_seqs=self.max_num_seqs,
1043 max_model_len=model_config.max_model_len,
1044 num_lookahead_slots=num_lookahead_slots,
1045 delay_factor=self.scheduler_delay_factor,
1046 enable_chunked_prefill=self.enable_chunked_prefill,
1035 - embedding_mode=model_config.embedding_mode,
1047 is_multimodal_model=model_config.is_multimodal_model,
1048 preemption_mode=self.preemption_mode,
1049 num_scheduler_steps=self.num_scheduler_steps,
vllm/engine/llm_engine.py
+2 -5
@@ -344,7 +344,7 @@ class LLMEngine:
344 observability_config=self.observability_config,
345 )
346
347 - if not self.model_config.embedding_mode:
347 + if self.model_config.task != "embedding":
348 self._initialize_kv_caches()
349
350 # If usage stat is enabled, collect relevant info.
@@ -1116,7 +1116,7 @@ class LLMEngine:
1116 seq_group.metrics.model_execute_time = (
1117 o.model_execute_time)
1118
1119 - if self.model_config.embedding_mode:
1119 + if self.model_config.task == "embedding":
1120 self._process_sequence_group_outputs(seq_group, output)
1121 else:
1122 self.output_processor.process_prompt_logprob(seq_group, output)
@@ -1855,9 +1855,6 @@ class LLMEngine:
1855 def is_encoder_decoder_model(self):
1856 return self.input_preprocessor.is_encoder_decoder_model()
1857
1858 - def is_embedding_model(self):
1859 - return self.model_config.is_embedding_model
1860 -
1858 def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
1859 EncoderDecoderInputs]):
1860 if self.model_config.is_multimodal_model:
vllm/entrypoints/llm.py
+44 -12
@@ -8,7 +8,7 @@ from tqdm import tqdm
8
9 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
10 BeamSearchSequence, get_beam_search_score)
11 -from vllm.engine.arg_utils import EngineArgs
11 +from vllm.engine.arg_utils import EngineArgs, TaskOption
12 from vllm.engine.llm_engine import LLMEngine
13 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
14 apply_hf_chat_template,
@@ -29,7 +29,7 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
29 get_cached_tokenizer)
30 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
31 from vllm.usage.usage_lib import UsageContext
32 -from vllm.utils import Counter, deprecate_kwargs, is_list_of
32 +from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
33
34 logger = init_logger(__name__)
35
@@ -108,6 +108,12 @@ class LLM:
108 DEPRECATE_LEGACY: ClassVar[bool] = False
109 """A flag to toggle whether to deprecate the legacy generate/encode API."""
110
111 + DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
112 + """
113 + A flag to toggle whether to deprecate positional arguments in
114 + :meth:`LLM.__init__`.
115 + """
116 +
117 @classmethod
118 @contextmanager
119 def deprecate_legacy_api(cls):
@@ -117,6 +123,13 @@ class LLM:
123
124 cls.DEPRECATE_LEGACY = False
125
126 + @deprecate_args(
127 + start_index=2, # Ignore self and model
128 + is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
129 + additional_message=(
130 + "All positional arguments other than `model` will be "
131 + "replaced with keyword arguments in an upcoming version."),
132 + )
133 def __init__(
134 self,
135 model: str,
@@ -139,6 +152,8 @@ class LLM:
152 disable_custom_all_reduce: bool = False,
153 disable_async_output_proc: bool = False,
154 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
155 + # After positional args are removed, move this right below `model`
156 + task: TaskOption = "auto",
157 **kwargs,
158 ) -> None:
159 '''
@@ -153,6 +168,7 @@ class LLM:
168
169 engine_args = EngineArgs(
170 model=model,
171 + task=task,
172 tokenizer=tokenizer,
173 tokenizer_mode=tokenizer_mode,
174 skip_tokenizer_init=skip_tokenizer_init,
@@ -316,10 +332,21 @@ class LLM:
332 considered legacy and may be deprecated in the future. You should
333 instead pass them via the ``inputs`` parameter.
334 """
319 - if self.llm_engine.model_config.embedding_mode:
320 - raise ValueError(
335 + task = self.llm_engine.model_config.task
336 + if task != "generate":
337 + messages = [
338 "LLM.generate() is only supported for (conditional) generation "
322 - "models (XForCausalLM, XForConditionalGeneration).")
339 + "models (XForCausalLM, XForConditionalGeneration).",
340 + ]
341 +
342 + supported_tasks = self.llm_engine.model_config.supported_tasks
343 + if "generate" in supported_tasks:
344 + messages.append(
345 + "Your model supports the 'generate' task, but is "
346 + f"currently initialized for the '{task}' task. Please "
347 + "initialize the model using `--task generate`.")
348 +
349 + raise ValueError(" ".join(messages))
350
351 if prompt_token_ids is not None:
352 parsed_prompts = self._convert_v1_inputs(
@@ -692,10 +719,18 @@ class LLM:
719 considered legacy and may be deprecated in the future. You should
720 instead pass them via the ``inputs`` parameter.
721 """
695 - if not self.llm_engine.model_config.embedding_mode:
696 - raise ValueError(
697 - "LLM.encode() is only supported for embedding models (XModel)."
698 - )
722 + task = self.llm_engine.model_config.task
723 + if task != "embedding":
724 + messages = ["LLM.encode() is only supported for embedding models."]
725 +
726 + supported_tasks = self.llm_engine.model_config.supported_tasks
727 + if "embedding" in supported_tasks:
728 + messages.append(
729 + "Your model supports the 'embedding' task, but is "
730 + f"currently initialized for the '{task}' task. Please "
731 + "initialize the model using `--task embedding`.")
732 +
733 + raise ValueError(" ".join(messages))
734
735 if prompt_token_ids is not None:
736 parsed_prompts = self._convert_v1_inputs(
@@ -905,6 +940,3 @@ class LLM:
940
941 def _is_encoder_decoder_model(self):
942 return self.llm_engine.is_encoder_decoder_model()
908 -
909 - def _is_embedding_model(self):
910 - return self.llm_engine.is_embedding_model()
vllm/entrypoints/openai/serving_embedding.py
+2 -1
@@ -83,7 +83,8 @@ class OpenAIServingEmbedding(OpenAIServing):
83 lora_modules=None,
84 prompt_adapters=None,
85 request_logger=request_logger)
86 - self._enabled = self._check_embedding_mode(model_config.embedding_mode)
86 + self._enabled = self._check_embedding_mode(
87 + model_config.task == "embedding")
88
89 async def create_embedding(
90 self,
vllm/utils.py
+47 -3
@@ -1034,10 +1034,54 @@ def identity(value: T) -> T:
1034 F = TypeVar('F', bound=Callable[..., Any])
1035
1036
1037 +def deprecate_args(
1038 + start_index: int,
1039 + is_deprecated: Union[bool, Callable[[], bool]] = True,
1040 + additional_message: Optional[str] = None,
1041 +) -> Callable[[F], F]:
1042 +
1043 + if not callable(is_deprecated):
1044 + is_deprecated = partial(identity, is_deprecated)
1045 +
1046 + def wrapper(fn: F) -> F:
1047 +
1048 + params = inspect.signature(fn).parameters
1049 + pos_types = (
1050 + inspect.Parameter.POSITIONAL_ONLY,
1051 + inspect.Parameter.POSITIONAL_OR_KEYWORD,
1052 + )
1053 + pos_kws = [
1054 + kw for kw, param in params.items() if param.kind in pos_types
1055 + ]
1056 +
1057 + @wraps(fn)
1058 + def inner(*args, **kwargs):
1059 + if is_deprecated():
1060 + deprecated_args = pos_kws[start_index:len(args)]
1061 + if deprecated_args:
1062 + msg = (
1063 + f"The positional arguments {deprecated_args} are "
1064 + "deprecated and will be removed in a future update.")
1065 + if additional_message is not None:
1066 + msg += f" {additional_message}"
1067 +
1068 + warnings.warn(
1069 + DeprecationWarning(msg),
1070 + stacklevel=3, # The inner function takes up one level
1071 + )
1072 +
1073 + return fn(*args, **kwargs)
1074 +
1075 + return inner # type: ignore
1076 +
1077 + return wrapper
1078 +
1079 +
1080 def deprecate_kwargs(
1038 - *kws: str,
1039 - is_deprecated: Union[bool, Callable[[], bool]] = True,
1040 - additional_message: Optional[str] = None) -> Callable[[F], F]:
1081 + *kws: str,
1082 + is_deprecated: Union[bool, Callable[[], bool]] = True,
1083 + additional_message: Optional[str] = None,
1084 +) -> Callable[[F], F]:
1085 deprecated_kws = set(kws)
1086
1087 if not callable(is_deprecated):
vllm/worker/worker.py
+1 -4
@@ -92,7 +92,7 @@ class Worker(LocalOrDistributedWorkerBase):
92 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
93 if model_runner_cls is not None:
94 ModelRunnerClass = model_runner_cls
95 - elif self._is_embedding_model():
95 + elif model_config.task == "embedding":
96 ModelRunnerClass = EmbeddingModelRunner
97 elif self._is_encoder_decoder_model():
98 ModelRunnerClass = EncoderDecoderModelRunner
@@ -147,9 +147,6 @@ class Worker(LocalOrDistributedWorkerBase):
147 def _is_encoder_decoder_model(self):
148 return self.model_config.is_encoder_decoder_model
149
150 - def _is_embedding_model(self):
151 - return self.model_config.is_embedding_model
152 -
150 def init_device(self) -> None:
151 if self.device_config.device.type == "cuda":
152 # torch.distributed.all_reduce does not free the input tensor until