A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Feature] Add vision language model support. (#3042)
xwjiang2010 committed
Mar 25, 2024 at 14:16 UTC
64172a976c8d975b3aec946f1675716d2532d94f
28 files changed
+936
-94
.buildkite/download-images.sh
new
+18
@@ -0,0 +1,18 @@
1
+#!/bin/bash
2
+
3
+set -ex
4
+set -o pipefail
5
+
6
+(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
7
+
8
+# aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9
+mkdir -p images
10
+cd images
11
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_pixel_values.pt
12
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_image_features.pt
13
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_pixel_values.pt
14
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_image_features.pt
15
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
16
+wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
17
+
18
+cd -
.buildkite/test-pipeline.yaml
+7
-1
@@ -39,9 +39,15 @@ steps:
39
40
- label: Models Test
41
commands:
42
- - pytest -v -s models --forked
42
+ - bash ../.buildkite/download-images.sh
43
+ - pytest -v -s models --ignore=models/test_llava.py --forked
44
soft_fail: true
45
46
+- label: Llava Test
47
+ commands:
48
+ - bash ../.buildkite/download-images.sh
49
+ - pytest -v -s models/test_llava.py
50
+
51
- label: Prefix Caching Test
52
commands:
53
- pytest -v -s prefix_caching
examples/llava_example.py
new
+84
@@ -0,0 +1,84 @@
1
+import argparse
2
+import os
3
+import subprocess
4
+
5
+import torch
6
+
7
+from vllm import LLM
8
+from vllm.sequence import MultiModalData
9
+
10
+# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
11
+
12
+
13
+def run_llava_pixel_values():
14
+ llm = LLM(
15
+ model="llava-hf/llava-1.5-7b-hf",
16
+ image_input_type="pixel_values",
17
+ image_token_id=32000,
18
+ image_input_shape="1,3,336,336",
19
+ image_feature_size=576,
20
+ )
21
+
22
+ prompt = "<image>" * 576 + (
23
+ "\nUSER: What is the content of this image?\nASSISTANT:")
24
+
25
+ # This should be provided by another online or offline component.
26
+ images = torch.load("images/stop_sign_pixel_values.pt")
27
+
28
+ outputs = llm.generate(prompt,
29
+ multi_modal_data=MultiModalData(
30
+ type=MultiModalData.Type.IMAGE, data=images))
31
+ for o in outputs:
32
+ generated_text = o.outputs[0].text
33
+ print(generated_text)
34
+
35
+
36
+def run_llava_image_features():
37
+ llm = LLM(
38
+ model="llava-hf/llava-1.5-7b-hf",
39
+ image_input_type="image_features",
40
+ image_token_id=32000,
41
+ image_input_shape="1,576,1024",
42
+ image_feature_size=576,
43
+ )
44
+
45
+ prompt = "<image>" * 576 + (
46
+ "\nUSER: What is the content of this image?\nASSISTANT:")
47
+
48
+ # This should be provided by another online or offline component.
49
+ images = torch.load("images/stop_sign_image_features.pt")
50
+
51
+ outputs = llm.generate(prompt,
52
+ multi_modal_data=MultiModalData(
53
+ type=MultiModalData.Type.IMAGE, data=images))
54
+ for o in outputs:
55
+ generated_text = o.outputs[0].text
56
+ print(generated_text)
57
+
58
+
59
+def main(args):
60
+ if args.type == "pixel_values":
61
+ run_llava_pixel_values()
62
+ else:
63
+ run_llava_image_features()
64
+
65
+
66
+if __name__ == "__main__":
67
+ parser = argparse.ArgumentParser(description="Demo on Llava")
68
+ parser.add_argument("--type",
69
+ type=str,
70
+ choices=["pixel_values", "image_features"],
71
+ default="pixel_values",
72
+ help="image input type")
73
+ args = parser.parse_args()
74
+ # Download from s3
75
+ s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
76
+ local_directory = "images"
77
+
78
+ # Make sure the local directory exists or create it
79
+ os.makedirs(local_directory, exist_ok=True)
80
+
81
+ # Use AWS CLI to sync the directory
82
+ subprocess.check_call(
83
+ ["aws", "s3", "sync", s3_bucket_path, local_directory])
84
+ main(args)
requirements-dev.txt
+4
@@ -24,6 +24,10 @@ openai
24
requests
25
ray
26
peft
27
+awscli
28
29
# Benchmarking
30
aiohttp
31
+
32
+# Multimodal
33
+pillow
tests/conftest.py
+112
-14
@@ -3,16 +3,39 @@ from typing import List, Optional, Tuple
3
4
import pytest
5
import torch
6
-from transformers import AutoModelForCausalLM
6
+from PIL import Image
7
+from transformers import (AutoModelForCausalLM, AutoProcessor,
8
+ LlavaForConditionalGeneration)
9
10
from vllm import LLM, SamplingParams
9
-from vllm.config import TokenizerPoolConfig
11
+from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
12
+from vllm.sequence import MultiModalData
13
from vllm.transformers_utils.tokenizer import get_tokenizer
14
15
_TEST_DIR = os.path.dirname(__file__)
16
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
17
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
18
19
+# Multi modal related
20
+_PIXEL_VALUES_FILES = [
21
+ os.path.join(_TEST_DIR, "images", filename) for filename in
22
+ ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
23
+]
24
+_IMAGE_FEATURES_FILES = [
25
+ os.path.join(_TEST_DIR, "images", filename) for filename in
26
+ ["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
27
+]
28
+_IMAGE_FILES = [
29
+ os.path.join(_TEST_DIR, "images", filename)
30
+ for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
31
+]
32
+_IMAGE_PROMPTS = [
33
+ "<image>\nUSER: What's the content of the image?\nASSISTANT:",
34
+ "<image>\nUSER: What is the season?\nASSISTANT:"
35
+]
36
+assert len(_PIXEL_VALUES_FILES) == len(_IMAGE_FEATURES_FILES) == len(
37
+ _IMAGE_FILES) == len(_IMAGE_PROMPTS)
38
+
39
40
def _read_prompts(filename: str) -> List[str]:
41
with open(filename, "r") as f:
@@ -20,6 +43,39 @@ def _read_prompts(filename: str) -> List[str]:
43
return prompts
44
45
46
+@pytest.fixture(scope="session")
47
+def hf_image_prompts() -> List[str]:
48
+ return _IMAGE_PROMPTS
49
+
50
+
51
+@pytest.fixture(scope="session")
52
+def hf_images() -> List[Image.Image]:
53
+ return [Image.open(filename) for filename in _IMAGE_FILES]
54
+
55
+
56
+@pytest.fixture()
57
+def vllm_images(request) -> "torch.Tensor":
58
+ vision_language_config = request.getfixturevalue("model_and_config")[1]
59
+ all_images = []
60
+ if vision_language_config.image_input_type == (
61
+ VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
62
+ filenames = _IMAGE_FEATURES_FILES
63
+ else:
64
+ filenames = _PIXEL_VALUES_FILES
65
+ for filename in filenames:
66
+ all_images.append(torch.load(filename))
67
+ return torch.concat(all_images, dim=0)
68
+
69
+
70
+@pytest.fixture()
71
+def vllm_image_prompts(request) -> List[str]:
72
+ vision_language_config = request.getfixturevalue("model_and_config")[1]
73
+ return [
74
+ "<image>" * (vision_language_config.image_feature_size - 1) + p
75
+ for p in _IMAGE_PROMPTS
76
+ ]
77
+
78
+
79
@pytest.fixture
80
def example_prompts() -> List[str]:
81
prompts = []
@@ -42,6 +98,10 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
98
"float": torch.float,
99
}
100
101
+_VISION_LANGUAGE_MODELS = {
102
+ "llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
103
+}
104
+
105
106
class HfRunner:
107
@@ -53,11 +113,24 @@ class HfRunner:
113
) -> None:
114
assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
115
torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
56
- self.model = AutoModelForCausalLM.from_pretrained(
57
- model_name,
58
- torch_dtype=torch_dtype,
59
- trust_remote_code=True,
60
- ).cuda()
116
+ self.model_name = model_name
117
+ if model_name not in _VISION_LANGUAGE_MODELS:
118
+ self.model = AutoModelForCausalLM.from_pretrained(
119
+ model_name,
120
+ torch_dtype=torch_dtype,
121
+ trust_remote_code=True,
122
+ ).cuda()
123
+ self.processor = None
124
+ else:
125
+ self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
126
+ model_name,
127
+ torch_dtype=torch_dtype,
128
+ trust_remote_code=True,
129
+ ).cuda()
130
+ self.processor = AutoProcessor.from_pretrained(
131
+ model_name,
132
+ torch_dtype=torch_dtype,
133
+ )
134
if tokenizer_name is None:
135
tokenizer_name = model_name
136
self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
@@ -65,13 +138,28 @@ class HfRunner:
138
def generate(
139
self,
140
prompts: List[str],
141
+ images: Optional[List[Image.Image]] = None,
142
**kwargs,
143
) -> List[Tuple[List[int], str]]:
144
outputs: List[Tuple[List[int], str]] = []
71
- for prompt in prompts:
72
- input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
145
+ if images:
146
+ assert len(prompts) == len(images)
147
+ for i, prompt in enumerate(prompts):
148
+ if self.model_name not in _VISION_LANGUAGE_MODELS:
149
+ input_ids = self.tokenizer(prompt,
150
+ return_tensors="pt").input_ids
151
+ inputs = {"input_ids": input_ids.cuda()}
152
+ else:
153
+ image = images[i] if images else None
154
+ inputs = self.processor(text=prompt,
155
+ images=image,
156
+ return_tensors="pt")
157
+ inputs = {
158
+ key: value.cuda() if value is not None else None
159
+ for key, value in inputs.items()
160
+ }
161
output_ids = self.model.generate(
74
- input_ids.cuda(),
162
+ **inputs,
163
use_cache=True,
164
**kwargs,
165
)
@@ -88,10 +176,12 @@ class HfRunner:
176
self,
177
prompts: List[str],
178
max_tokens: int,
179
+ images: Optional["torch.Tensor"] = None,
180
) -> List[Tuple[List[int], str]]:
181
outputs = self.generate(prompts,
182
do_sample=False,
94
- max_new_tokens=max_tokens)
183
+ max_new_tokens=max_tokens,
184
+ images=images)
185
for i in range(len(outputs)):
186
output_ids, output_str = outputs[i]
187
outputs[i] = (output_ids[0], output_str[0])
@@ -183,9 +273,16 @@ class VllmRunner:
273
self,
274
prompts: List[str],
275
sampling_params: SamplingParams,
276
+ images: Optional["torch.Tensor"] = None,
277
) -> List[Tuple[List[int], str]]:
187
- req_outputs = self.model.generate(prompts,
188
- sampling_params=sampling_params)
278
+ if images is not None:
279
+ assert len(prompts) == images.shape[0]
280
+ req_outputs = self.model.generate(
281
+ prompts,
282
+ sampling_params=sampling_params,
283
+ multi_modal_data=MultiModalData(type=MultiModalData.Type.IMAGE,
284
+ data=images)
285
+ if images is not None else None)
286
outputs = []
287
for req_output in req_outputs:
288
prompt_str = req_output.prompt
@@ -222,9 +319,10 @@ class VllmRunner:
319
self,
320
prompts: List[str],
321
max_tokens: int,
322
+ images: Optional[torch.Tensor] = None,
323
) -> List[Tuple[List[int], str]]:
324
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
227
- outputs = self.generate(prompts, greedy_params)
325
+ outputs = self.generate(prompts, greedy_params, images=images)
326
return [(output_ids[0], output_str[0])
327
for output_ids, output_str in outputs]
328
tests/models/test_llava.py
new
+110
@@ -0,0 +1,110 @@
1
+import gc
2
+from dataclasses import fields
3
+from enum import Enum
4
+from typing import Dict, List, Tuple
5
+
6
+import pytest
7
+import torch
8
+from transformers import AutoTokenizer
9
+
10
+from vllm.config import VisionLanguageConfig
11
+
12
+model_and_vl_config = [
13
+ ("llava-hf/llava-1.5-7b-hf",
14
+ VisionLanguageConfig(
15
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
16
+ image_feature_size=576,
17
+ image_token_id=32000,
18
+ image_input_shape=(1, 3, 336, 336))),
19
+ ("llava-hf/llava-1.5-7b-hf",
20
+ VisionLanguageConfig(
21
+ image_input_type=VisionLanguageConfig.ImageInputType.IMAGE_FEATURES,
22
+ image_feature_size=576,
23
+ image_token_id=32000,
24
+ image_input_shape=(1, 576, 1024)))
25
+]
26
+
27
+
28
+def as_dict(vision_language_config: VisionLanguageConfig) -> Dict:
29
+ """Flatten vision language config to pure args.
30
+
31
+ Compatible with what llm entrypoint expects.
32
+ """
33
+ result = {}
34
+ for field in fields(vision_language_config):
35
+ value = getattr(vision_language_config, field.name)
36
+ if isinstance(value, Enum):
37
+ result[field.name] = value.name.lower()
38
+ elif isinstance(value, tuple):
39
+ result[field.name] = ",".join([str(item) for item in value])
40
+ else:
41
+ result[field.name] = value
42
+ return result
43
+
44
+
45
+def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
46
+ vision_language_config: VisionLanguageConfig,
47
+ model_id: str):
48
+ """Sanitize vllm output to be comparable with hf output.
49
+ The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
50
+ x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
51
+ It also reduces `output_str` from "<image><image>bla" to "bla".
52
+ """
53
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
54
+ image_token_str = tokenizer.decode(vision_language_config.image_token_id)
55
+ image_token_str_len = len(image_token_str)
56
+ input_ids, output_str = vllm_output
57
+ sanitized_input_ids = input_ids[0:2] + input_ids[2 + vision_language_config
58
+ .image_feature_size - 1:]
59
+ sanitzied_output_str = output_str[vision_language_config.
60
+ image_feature_size *
61
+ image_token_str_len:]
62
+ return sanitized_input_ids, sanitzied_output_str
63
+
64
+
65
+@pytest.mark.parametrize("worker_use_ray", [False])
66
+@pytest.mark.parametrize("model_and_config", model_and_vl_config)
67
+@pytest.mark.parametrize("dtype", ["half"])
68
+@pytest.mark.parametrize("max_tokens", [128])
69
+def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
70
+ vllm_image_prompts, vllm_images, model_and_config: tuple,
71
+ dtype: str, max_tokens: int, worker_use_ray: bool) -> None:
72
+ """Inference result should be the same between hf and vllm.
73
+
74
+ All the image fixtures for the test is under tests/images.
75
+ For huggingface runner, we provide the raw images as input.
76
+ For vllm runner, we provide image tensors and corresponding
77
+ vision language config as input.
78
+ Note, the text input is also adjusted to abide by vllm contract.
79
+ The text output is sanitized to be able to compare with hf.
80
+ """
81
+ model_id, vision_language_config = model_and_config
82
+ hf_model = hf_runner(model_id, dtype=dtype)
83
+ hf_outputs = hf_model.generate_greedy(hf_image_prompts,
84
+ max_tokens,
85
+ images=hf_images)
86
+ del hf_model
87
+
88
+ gc.collect()
89
+ torch.cuda.empty_cache()
90
+
91
+ vllm_model = vllm_runner(model_id,
92
+ dtype=dtype,
93
+ worker_use_ray=worker_use_ray,
94
+ **as_dict(vision_language_config))
95
+ vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
96
+ max_tokens,
97
+ images=vllm_images)
98
+ del vllm_model
99
+
100
+ gc.collect()
101
+ torch.cuda.empty_cache()
102
+
103
+ for i in range(len(hf_image_prompts)):
104
+ hf_output_ids, hf_output_str = hf_outputs[i]
105
+ vllm_output_ids, vllm_output_str = sanitize_vllm_output(
106
+ vllm_outputs[i], vision_language_config, model_id)
107
+ assert hf_output_str == vllm_output_str, (
108
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
109
+ assert hf_output_ids == vllm_output_ids, (
110
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/models/test_models.py
+1
-1
@@ -25,7 +25,7 @@ MODELS = [
25
26
27
@pytest.mark.parametrize("model", MODELS)
28
-@pytest.mark.parametrize("dtype", ["float"])
28
+@pytest.mark.parametrize("dtype", ["half"])
29
@pytest.mark.parametrize("max_tokens", [128])
30
def test_models(
31
hf_runner,
tests/spec_decode/utils.py
+1
-1
@@ -109,7 +109,7 @@ def create_worker(cls: type,
109
)
110
111
(model_config, cache_config, parallel_config, scheduler_config,
112
- device_config, _) = engine_args.create_engine_configs()
112
+ device_config, _, _) = engine_args.create_engine_configs()
113
114
distributed_init_method = get_distributed_init_method(
115
get_ip(), get_open_port())
tests/worker/test_model_runner.py
+1
-1
@@ -35,7 +35,7 @@ def test_prepare_prompt(batch_size):
35
prompt_len - 1)
36
selected_token_start_idx += prompt_len
37
(input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
38
- _) = (model_runner._prepare_prompt(seq_group_metadata_list))
38
+ _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
39
assert return_prompt_lens == prompt_lens
40
41
# Verify input metadata is correct for prompts.
tests/worker/test_swap.py
+1
-1
@@ -11,7 +11,7 @@ def test_swap() -> None:
11
dtype="half",
12
load_format="dummy")
13
(model_config, cache_config, parallel_config, scheduler_config,
14
- device_config, _) = engine_args.create_engine_configs()
14
+ device_config, _, _) = engine_args.create_engine_configs()
15
cache_config.num_gpu_blocks = 100
16
cache_config.num_cpu_blocks = 100
17
vllm/config.py
+62
-17
@@ -1,3 +1,4 @@
1
+import enum
2
import json
3
import os
4
from dataclasses import dataclass
@@ -8,7 +9,7 @@ from packaging.version import Version
9
from transformers import PretrainedConfig
10
11
from vllm.logger import init_logger
11
-from vllm.transformers_utils.config import get_config
12
+from vllm.transformers_utils.config import get_config, get_hf_text_config
13
from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
14
15
if TYPE_CHECKING:
@@ -118,8 +119,9 @@ class ModelConfig:
119
120
self.hf_config = get_config(self.model, trust_remote_code, revision,
121
code_revision)
121
- self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
122
- self.max_model_len = _get_and_verify_max_len(self.hf_config,
122
+ self.hf_text_config = get_hf_text_config(self.hf_config)
123
+ self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
124
+ self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
125
max_model_len)
126
self._verify_load_format()
127
self._verify_tokenizer_mode()
@@ -218,7 +220,7 @@ class ModelConfig:
220
self,
221
parallel_config: "ParallelConfig",
222
) -> None:
221
- total_num_attention_heads = self.hf_config.num_attention_heads
223
+ total_num_attention_heads = self.hf_text_config.num_attention_heads
224
tensor_parallel_size = parallel_config.tensor_parallel_size
225
if total_num_attention_heads % tensor_parallel_size != 0:
226
raise ValueError(
@@ -226,7 +228,7 @@ class ModelConfig:
228
" must be divisible by tensor parallel size "
229
f"({tensor_parallel_size}).")
230
229
- total_num_hidden_layers = self.hf_config.num_hidden_layers
231
+ total_num_hidden_layers = self.hf_text_config.num_hidden_layers
232
pipeline_parallel_size = parallel_config.pipeline_parallel_size
233
if total_num_hidden_layers % pipeline_parallel_size != 0:
234
raise ValueError(
@@ -241,22 +243,23 @@ class ModelConfig:
243
# Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
244
# addition to sliding window size. We check if that field is present
245
# and if it's False, return None.
244
- if (hasattr(self.hf_config, "use_sliding_window")
245
- and not self.hf_config.use_sliding_window):
246
+ if (hasattr(self.hf_text_config, "use_sliding_window")
247
+ and not self.hf_text_config.use_sliding_window):
248
return None
247
- return getattr(self.hf_config, "sliding_window", None)
249
+ return getattr(self.hf_text_config, "sliding_window", None)
250
251
def get_vocab_size(self) -> int:
250
- return self.hf_config.vocab_size
252
+ return self.hf_text_config.vocab_size
253
254
def get_hidden_size(self) -> int:
253
- return self.hf_config.hidden_size
255
+ return self.hf_text_config.hidden_size
256
257
def get_head_size(self) -> int:
256
- if hasattr(self.hf_config, "head_dim"):
257
- return self.hf_config.head_dim
258
+ if hasattr(self.hf_text_config, "head_dim"):
259
+ return self.hf_text_config.head_dim
260
# FIXME(woosuk): This may not be true for all models.
259
- return self.hf_config.hidden_size // self.hf_config.num_attention_heads
261
+ return (self.hf_text_config.hidden_size //
262
+ self.hf_text_config.num_attention_heads)
263
264
def get_total_num_kv_heads(self) -> int:
265
"""Returns the total number of KV heads."""
@@ -268,7 +271,7 @@ class ModelConfig:
271
new_decoder_arch_falcon = (
272
self.hf_config.model_type in falcon_model_types
273
and getattr(self.hf_config, "new_decoder_architecture", False))
271
- if not new_decoder_arch_falcon and getattr(self.hf_config,
274
+ if not new_decoder_arch_falcon and getattr(self.hf_text_config,
275
"multi_query", False):
276
# Multi-query attention, only one KV head.
277
# Currently, tensor parallelism is not supported in this case.
@@ -284,13 +287,13 @@ class ModelConfig:
287
"multi_query_group_num",
288
]
289
for attr in attributes:
287
- num_kv_heads = getattr(self.hf_config, attr, None)
290
+ num_kv_heads = getattr(self.hf_text_config, attr, None)
291
if num_kv_heads is not None:
292
return num_kv_heads
293
294
# For non-grouped-query attention models, the number of KV heads is
295
# equal to the number of attention heads.
293
- return self.hf_config.num_attention_heads
296
+ return self.hf_text_config.num_attention_heads
297
298
def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
299
"""Returns the number of KV heads per GPU."""
@@ -303,7 +306,7 @@ class ModelConfig:
306
total_num_kv_heads // parallel_config.tensor_parallel_size)
307
308
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
306
- total_num_hidden_layers = self.hf_config.num_hidden_layers
309
+ total_num_hidden_layers = self.hf_text_config.num_hidden_layers
310
return total_num_hidden_layers // parallel_config.pipeline_parallel_size
311
312
@@ -627,6 +630,48 @@ class LoRAConfig:
630
"LoRA is enabled.")
631
632
633
+@dataclass
634
+class VisionLanguageConfig:
635
+ """Configs the input data format and how models should run for
636
+ vision language models."""
637
+
638
+ class ImageInputType(enum.Enum):
639
+ """Image input type into the vision language model.
640
+
641
+ An image roughly goes through the following transformation:
642
+ Raw image --> pixel values --> image features --> image embeddings.
643
+
644
+ The difference between different image input types is where the
645
+ image encoder (pixel values --> image features) is run.
646
+ Different image input types also correspond to different tensor shapes.
647
+
648
+ For example, for Llava, PIXEL_VALUES: (1, 3, 336, 336).
649
+ IMAGE_FEATURES: (1, 576, 1024).
650
+ """
651
+ PIXEL_VALUES = enum.auto()
652
+ IMAGE_FEATURES = enum.auto()
653
+
654
+ image_input_type: ImageInputType
655
+ # The input id corresponding to image token.
656
+ image_token_id: int
657
+ # Used for running `run_prefill_max_token`.
658
+ # For models that support varying resolution, this corresponds to
659
+ # worst case scenario (biggest supported resolution).
660
+ image_input_shape: tuple
661
+ image_feature_size: int
662
+
663
+ @classmethod
664
+ def get_image_input_enum_type(
665
+ cls, value: str) -> "VisionLanguageConfig.ImageInputType":
666
+ """Get the image input type from a string."""
667
+ try:
668
+ return cls.ImageInputType[value.upper()]
669
+ except KeyError as e:
670
+ raise ValueError(f"{value} is not a valid choice. "
671
+ f"Expecting to choose from "
672
+ f"{[x.name for x in cls.ImageInputType]}.") from e
673
+
674
+
675
_STR_DTYPE_TO_TORCH_DTYPE = {
676
"half": torch.float16,
677
"float16": torch.float16,
vllm/core/scheduler.py
+6
@@ -388,6 +388,12 @@ class Scheduler:
388
computed_block_nums=self.block_manager.
389
get_common_computed_block_ids(seq_group),
390
state=seq_group.state,
391
+ # `multi_modal_data` will only be present for the 1st comm
392
+ # between engine and worker.
393
+ # the subsequent comms can still use delta, but
394
+ # `multi_modal_data` will be None.
395
+ multi_modal_data=seq_group.multi_modal_data
396
+ if scheduler_outputs.prompt_run else None,
397
)
398
seq_group_metadata_list.append(seq_group_metadata)
399
return seq_group_metadata_list, scheduler_outputs
vllm/engine/arg_utils.py
+53
-3
@@ -4,7 +4,9 @@ from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig, TokenizerPoolConfig)
7
+ ParallelConfig, SchedulerConfig, TokenizerPoolConfig,
8
+ VisionLanguageConfig)
9
+from vllm.utils import str_to_int_tuple
10
11
12
@dataclass
@@ -50,6 +52,11 @@ class EngineArgs:
52
max_cpu_loras: Optional[int] = None
53
device: str = 'auto'
54
ray_workers_use_nsight: bool = False
55
+ # Related to Vision-language models such as llava
56
+ image_input_type: Optional[str] = None
57
+ image_token_id: Optional[int] = None
58
+ image_input_shape: Optional[str] = None
59
+ image_feature_size: Optional[int] = None
60
scheduler_delay_factor: float = 0.0
61
62
def __post_init__(self):
@@ -305,6 +312,31 @@ class EngineArgs:
312
default=EngineArgs.device,
313
choices=["auto", "cuda", "neuron"],
314
help='Device type for vLLM execution.')
315
+ # Related to Vision-language models such as llava
316
+ parser.add_argument(
317
+ '--image-input-type',
318
+ type=str,
319
+ default=None,
320
+ choices=[
321
+ t.name.lower() for t in VisionLanguageConfig.ImageInputType
322
+ ],
323
+ help=('The image input type passed into vLLM. '
324
+ 'Should be one of "pixel_values" or "image_features".'))
325
+ parser.add_argument('--image-token-id',
326
+ type=int,
327
+ default=None,
328
+ help=('Input id for image token.'))
329
+ parser.add_argument(
330
+ '--image-input-shape',
331
+ type=str,
332
+ default=None,
333
+ help=('The biggest image input shape (worst for memory footprint) '
334
+ 'given an input type. Only used for vLLM\'s profile_run.'))
335
+ parser.add_argument(
336
+ '--image-feature-size',
337
+ type=int,
338
+ default=None,
339
+ help=('The image feature size along the context dimension.'))
340
parser.add_argument(
341
'--scheduler-delay-factor',
342
type=float,
@@ -324,7 +356,8 @@ class EngineArgs:
356
def create_engine_configs(
357
self,
358
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
327
- DeviceConfig, Optional[LoRAConfig]]:
359
+ DeviceConfig, Optional[LoRAConfig],
360
+ Optional[VisionLanguageConfig]]:
361
device_config = DeviceConfig(self.device)
362
model_config = ModelConfig(
363
self.model, self.tokenizer, self.tokenizer_mode,
@@ -358,8 +391,25 @@ class EngineArgs:
391
lora_dtype=self.lora_dtype,
392
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
393
and self.max_cpu_loras > 0 else None) if self.enable_lora else None
394
+
395
+ if self.image_input_type:
396
+ if (not self.image_token_id or not self.image_input_shape
397
+ or not self.image_feature_size):
398
+ raise ValueError(
399
+ 'Specify `image_token_id`, `image_input_shape` and '
400
+ '`image_feature_size` together with `image_input_type`.')
401
+ vision_language_config = VisionLanguageConfig(
402
+ image_input_type=VisionLanguageConfig.
403
+ get_image_input_enum_type(self.image_input_type),
404
+ image_token_id=self.image_token_id,
405
+ image_input_shape=str_to_int_tuple(self.image_input_shape),
406
+ image_feature_size=self.image_feature_size,
407
+ )
408
+ else:
409
+ vision_language_config = None
410
+
411
return (model_config, cache_config, parallel_config, scheduler_config,
362
- device_config, lora_config)
412
+ device_config, lora_config, vision_language_config)
413
414
415
@dataclass
vllm/engine/async_llm_engine.py
+16
-9
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
+from vllm.sequence import MultiModalData
19
20
logger = init_logger(__name__)
21
ENGINE_ITERATION_TIMEOUT_S = int(
@@ -240,6 +241,7 @@ class _AsyncLLMEngine(LLMEngine):
241
prompt_token_ids: Optional[List[int]] = None,
242
arrival_time: Optional[float] = None,
243
lora_request: Optional[LoRARequest] = None,
244
+ multi_modal_data: Optional[MultiModalData] = None,
245
) -> None:
246
if lora_request is not None and not self.lora_config:
247
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -252,14 +254,13 @@ class _AsyncLLMEngine(LLMEngine):
254
prompt_token_ids=prompt_token_ids,
255
lora_request=lora_request)
256
255
- return self.add_request(
256
- request_id,
257
- prompt=prompt,
258
- prompt_token_ids=prompt_token_ids,
259
- sampling_params=sampling_params,
260
- arrival_time=arrival_time,
261
- lora_request=lora_request,
262
- )
257
+ return self.add_request(request_id,
258
+ prompt=prompt,
259
+ prompt_token_ids=prompt_token_ids,
260
+ sampling_params=sampling_params,
261
+ arrival_time=arrival_time,
262
+ lora_request=lora_request,
263
+ multi_modal_data=multi_modal_data)
264
265
async def check_health_async(self) -> None:
266
self.model_executor.check_health()
@@ -486,6 +487,7 @@ class AsyncLLMEngine:
487
prompt_token_ids: Optional[List[int]] = None,
488
arrival_time: Optional[float] = None,
489
lora_request: Optional[LoRARequest] = None,
490
+ multi_modal_data: Optional[MultiModalData] = None,
491
) -> AsyncStream:
492
if self.log_requests:
493
shortened_prompt = prompt
@@ -534,7 +536,9 @@ class AsyncLLMEngine:
536
sampling_params=sampling_params,
537
prompt_token_ids=prompt_token_ids,
538
arrival_time=arrival_time,
537
- lora_request=lora_request)
539
+ lora_request=lora_request,
540
+ multi_modal_data=multi_modal_data,
541
+ )
542
543
return stream
544
@@ -545,6 +549,7 @@ class AsyncLLMEngine:
549
request_id: str,
550
prompt_token_ids: Optional[List[int]] = None,
551
lora_request: Optional[LoRARequest] = None,
552
+ multi_modal_data: Optional[MultiModalData] = None
553
) -> AsyncIterator[RequestOutput]:
554
"""Generate outputs for a request.
555
@@ -560,6 +565,7 @@ class AsyncLLMEngine:
565
prompt_token_ids: The token IDs of the prompt. If None, we
566
use the tokenizer to convert the prompts to token IDs.
567
lora_request: LoRA request to use for generation, if any.
568
+ multi_modal_data: Multi modal data per request.
569
570
Yields:
571
The output `RequestOutput` objects from the LLMEngine for the
@@ -619,6 +625,7 @@ class AsyncLLMEngine:
625
prompt_token_ids=prompt_token_ids,
626
arrival_time=arrival_time,
627
lora_request=lora_request,
628
+ multi_modal_data=multi_modal_data,
629
)
630
631
async for request_output in stream:
vllm/engine/llm_engine.py
+11
-6
@@ -5,7 +5,7 @@ from transformers import PreTrainedTokenizer
5
6
import vllm
7
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig)
8
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9
from vllm.core.scheduler import Scheduler, SchedulerOutputs
10
from vllm.engine.arg_utils import EngineArgs
11
from vllm.engine.metrics import StatLogger, Stats
@@ -15,8 +15,9 @@ from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
-from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19
- SequenceGroupOutput, SequenceOutput, SequenceStatus)
18
+from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
19
+ SequenceGroup, SequenceGroupOutput, SequenceOutput,
20
+ SequenceStatus)
21
from vllm.transformers_utils.detokenizer import Detokenizer
22
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
23
get_tokenizer_group)
@@ -62,6 +63,7 @@ class LLMEngine:
63
scheduler_config: SchedulerConfig,
64
device_config: DeviceConfig,
65
lora_config: Optional[LoRAConfig],
66
+ vision_language_config: Optional["VisionLanguageConfig"],
67
executor_class: Type[ExecutorBase],
68
log_stats: bool,
69
) -> None:
@@ -90,6 +92,7 @@ class LLMEngine:
92
self.model_config = model_config
93
self.cache_config = cache_config
94
self.lora_config = lora_config
95
+ self.vision_language_config = vision_language_config
96
self.parallel_config = parallel_config
97
self.scheduler_config = scheduler_config
98
self.device_config = device_config
@@ -102,7 +105,8 @@ class LLMEngine:
105
106
self.model_executor = executor_class(model_config, cache_config,
107
parallel_config, scheduler_config,
105
- device_config, lora_config)
108
+ device_config, lora_config,
109
+ vision_language_config)
110
111
# Ping the tokenizer to ensure liveness if it runs in a
112
# different process.
@@ -170,7 +174,6 @@ class LLMEngine:
174
trust_remote_code=self.model_config.trust_remote_code,
175
revision=self.model_config.tokenizer_revision)
176
init_kwargs.update(tokenizer_init_kwargs)
173
-
177
self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
178
self.parallel_config.tokenizer_pool_config, **init_kwargs)
179
@@ -212,6 +215,7 @@ class LLMEngine:
215
prompt_token_ids: Optional[List[int]] = None,
216
arrival_time: Optional[float] = None,
217
lora_request: Optional[LoRARequest] = None,
218
+ multi_modal_data: Optional[MultiModalData] = None,
219
) -> None:
220
"""Add a request to the engine's request pool.
221
@@ -228,6 +232,7 @@ class LLMEngine:
232
use the tokenizer to convert the prompts to token IDs.
233
arrival_time: The arrival time of the request. If None, we use
234
the current monotonic time.
235
+ multi_modal_data: Multi modal data per request.
236
237
Details:
238
- Set arrival_time to the current time if it is None.
@@ -288,7 +293,7 @@ class LLMEngine:
293
294
# Create the sequence group.
295
seq_group = SequenceGroup(request_id, [seq], sampling_params,
291
- arrival_time, lora_request)
296
+ arrival_time, lora_request, multi_modal_data)
297
298
# Add the sequence group to the scheduler.
299
self.scheduler.add_seq_group(seq_group)
vllm/entrypoints/llm.py
+21
-5
@@ -1,5 +1,6 @@
1
from typing import List, Optional, Union
2
3
+import torch
4
from tqdm import tqdm
5
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
@@ -8,6 +9,7 @@ from vllm.engine.llm_engine import LLMEngine
9
from vllm.lora.request import LoRARequest
10
from vllm.outputs import RequestOutput
11
from vllm.sampling_params import SamplingParams
12
+from vllm.sequence import MultiModalData
13
from vllm.utils import Counter
14
15
@@ -126,6 +128,7 @@ class LLM:
128
prompt_token_ids: Optional[List[List[int]]] = None,
129
use_tqdm: bool = True,
130
lora_request: Optional[LoRARequest] = None,
131
+ multi_modal_data: Optional[MultiModalData] = None,
132
) -> List[RequestOutput]:
133
"""Generates the completions for the input prompts.
134
@@ -141,6 +144,7 @@ class LLM:
144
use the tokenizer to convert the prompts to token IDs.
145
use_tqdm: Whether to use tqdm to display the progress bar.
146
lora_request: LoRA request to use for generation, if any.
147
+ multi_modal_data: Multi modal data.
148
149
Returns:
150
A list of `RequestOutput` objects containing the generated
@@ -160,6 +164,9 @@ class LLM:
164
# Use default sampling params.
165
sampling_params = SamplingParams()
166
167
+ if multi_modal_data:
168
+ multi_modal_data.data = multi_modal_data.data.to(torch.float16)
169
+
170
# Add requests to the engine.
171
num_requests = len(prompts) if prompts is not None else len(
172
prompt_token_ids)
@@ -167,10 +174,17 @@ class LLM:
174
prompt = prompts[i] if prompts is not None else None
175
token_ids = None if prompt_token_ids is None else prompt_token_ids[
176
i]
170
- self._add_request(prompt,
171
- sampling_params,
172
- token_ids,
173
- lora_request=lora_request)
177
+ self._add_request(
178
+ prompt,
179
+ sampling_params,
180
+ token_ids,
181
+ lora_request=lora_request,
182
+ # Get ith image while maintaining the batch dim.
183
+ multi_modal_data=MultiModalData(
184
+ type=multi_modal_data.type,
185
+ data=multi_modal_data.data[i].unsqueeze(0))
186
+ if multi_modal_data else None,
187
+ )
188
return self._run_engine(use_tqdm)
189
190
def _add_request(
@@ -179,13 +193,15 @@ class LLM:
193
sampling_params: SamplingParams,
194
prompt_token_ids: Optional[List[int]],
195
lora_request: Optional[LoRARequest] = None,
196
+ multi_modal_data: Optional[MultiModalData] = None,
197
) -> None:
198
request_id = str(next(self.request_counter))
199
self.llm_engine.add_request(request_id,
200
prompt,
201
sampling_params,
202
prompt_token_ids,
188
- lora_request=lora_request)
203
+ lora_request=lora_request,
204
+ multi_modal_data=multi_modal_data)
205
206
def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
207
# Initialize tqdm.
vllm/executor/executor_base.py
+2
-1
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
2
from typing import Dict, List, Optional
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig)
5
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
6
from vllm.lora.request import LoRARequest
7
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
@@ -24,6 +24,7 @@ class ExecutorBase(ABC):
24
scheduler_config: SchedulerConfig,
25
device_config: DeviceConfig,
26
lora_config: Optional[LoRAConfig],
27
+ vision_language_config: Optional[VisionLanguageConfig],
28
) -> None:
29
raise NotImplementedError
30
vllm/executor/gpu_executor.py
+4
-1
@@ -1,7 +1,7 @@
1
from typing import Dict, List, Optional
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
- ParallelConfig, SchedulerConfig)
4
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
from vllm.executor.utils import check_block_size_valid
7
from vllm.logger import init_logger
@@ -23,6 +23,7 @@ class GPUExecutor(ExecutorBase):
23
scheduler_config: SchedulerConfig,
24
device_config: DeviceConfig,
25
lora_config: Optional[LoRAConfig],
26
+ vision_language_config: Optional[VisionLanguageConfig],
27
) -> None:
28
self.model_config = model_config
29
self.cache_config = cache_config
@@ -30,6 +31,7 @@ class GPUExecutor(ExecutorBase):
31
self.parallel_config = parallel_config
32
self.scheduler_config = scheduler_config
33
self.device_config = device_config
34
+ self.vision_language_config = vision_language_config
35
36
# Instantiate the worker and load the model to GPU.
37
self._init_worker()
@@ -56,6 +58,7 @@ class GPUExecutor(ExecutorBase):
58
rank=0,
59
distributed_init_method=distributed_init_method,
60
lora_config=self.lora_config,
61
+ vision_language_config=self.vision_language_config,
62
kv_cache_dtype=self.cache_config.cache_dtype,
63
is_driver_worker=True,
64
)
vllm/executor/ray_gpu_executor.py
+4
-1
@@ -6,7 +6,7 @@ from collections import defaultdict
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig)
9
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
10
from vllm.engine.ray_utils import RayWorkerVllm, ray
11
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
12
from vllm.executor.utils import check_block_size_valid
@@ -40,6 +40,7 @@ class RayGPUExecutor(ExecutorBase):
40
scheduler_config: SchedulerConfig,
41
device_config: DeviceConfig,
42
lora_config: Optional[LoRAConfig],
43
+ vision_language_config: Optional[VisionLanguageConfig],
44
) -> None:
45
self.model_config = model_config
46
self.cache_config = cache_config
@@ -47,6 +48,7 @@ class RayGPUExecutor(ExecutorBase):
48
self.parallel_config = parallel_config
49
self.scheduler_config = scheduler_config
50
self.device_config = device_config
51
+ self.vision_language_config = vision_language_config
52
53
assert self.parallel_config.worker_use_ray
54
placement_group = self.parallel_config.placement_group
@@ -181,6 +183,7 @@ class RayGPUExecutor(ExecutorBase):
183
driver_rank,
184
distributed_init_method,
185
lora_config=self.lora_config,
186
+ vision_language_config=self.vision_language_config,
187
kv_cache_dtype=kv_cache_dtype,
188
is_driver_worker=True,
189
)
vllm/model_executor/model_loader.py
+11
-1
@@ -7,9 +7,14 @@ import torch.nn as nn
7
8
from vllm.config import DeviceConfig, ModelConfig
9
from vllm.model_executor.models import ModelRegistry
10
+from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11
from vllm.model_executor.weight_utils import (get_quant_config,
12
initialize_dummy_weights)
13
14
+_VISION_MODEL_CLASSES = [
15
+ LlavaForConditionalGeneration,
16
+]
17
+
18
19
@contextlib.contextmanager
20
def _set_default_torch_dtype(dtype: torch.dtype):
@@ -40,6 +45,7 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
45
def get_model(model_config: ModelConfig, device_config: DeviceConfig,
46
**kwargs) -> nn.Module:
47
lora_config = kwargs.get("lora_config", None)
48
+ vision_language_config = kwargs.get("vision_language_config", None)
49
model_class = _get_model_architecture(model_config)
50
51
# Get the (maybe quantized) linear method.
@@ -76,7 +82,11 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
82
"be added in the future. If this is important to you, "
83
"please open an issue on github.")
84
else:
79
- model = model_class(model_config.hf_config, linear_method)
85
+ if model_class not in _VISION_MODEL_CLASSES:
86
+ model = model_class(model_config.hf_config, linear_method)
87
+ else:
88
+ model = model_class(model_config.hf_config,
89
+ vision_language_config, linear_method)
90
if model_config.load_format == "dummy":
91
# NOTE(woosuk): For accurate performance evaluation, we assign
92
# random values to the weights.
vllm/model_executor/models/__init__.py
+2
@@ -29,6 +29,8 @@ _MODELS = {
29
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
30
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
31
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
32
+ "LlavaForConditionalGeneration":
33
+ ("llava", "LlavaForConditionalGeneration"),
34
# For decapoda-research/llama-*
35
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
36
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/llama.py
+9
-2
@@ -250,14 +250,21 @@ class LlamaModel(nn.Module):
250
])
251
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
252
253
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
254
+ return self.embed_tokens(input_ids)
255
+
256
def forward(
257
self,
255
- input_ids: torch.Tensor,
258
+ input_ids: Optional[torch.Tensor],
259
positions: torch.Tensor,
260
kv_caches: List[torch.Tensor],
261
attn_metadata: AttentionMetadata,
262
+ inputs_embeds: Optional[torch.Tensor] = None,
263
) -> torch.Tensor:
260
- hidden_states = self.embed_tokens(input_ids)
264
+ if inputs_embeds is not None:
265
+ hidden_states = inputs_embeds
266
+ else:
267
+ hidden_states = self.get_input_embeddings(input_ids)
268
residual = None
269
for i in range(len(self.layers)):
270
layer = self.layers[i]
vllm/model_executor/models/llava.py
new
+246
@@ -0,0 +1,246 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+from torch import nn
5
+# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
6
+# transformers' impl.
7
+from transformers import CLIPVisionModel, LlavaConfig
8
+
9
+from vllm.attention import AttentionMetadata
10
+from vllm.config import VisionLanguageConfig
11
+from vllm.model_executor.layers.activation import get_act_fn
12
+from vllm.model_executor.layers.linear import LinearMethodBase
13
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
+from vllm.model_executor.layers.sampler import Sampler
15
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16
+from vllm.model_executor.models.llama import LlamaModel
17
+from vllm.model_executor.sampling_metadata import SamplingMetadata
18
+from vllm.model_executor.weight_utils import (default_weight_loader,
19
+ hf_model_weights_iterator)
20
+from vllm.sequence import SamplerOutput
21
+
22
+KVCache = Tuple[torch.Tensor, torch.Tensor]
23
+
24
+_KEYS_TO_MODIFY_MAPPING = {
25
+ "language_model.lm_head": "lm_head",
26
+ "language_model.model": "language_model",
27
+}
28
+
29
+
30
+# TODO(xwjiang): Run benchmark and decide if TP.
31
+class LlavaMultiModalProjector(nn.Module):
32
+
33
+ def __init__(self, vision_hidden_size: int, text_hidden_size: int,
34
+ projector_hidden_act: str):
35
+ super().__init__()
36
+
37
+ self.linear_1 = nn.Linear(vision_hidden_size,
38
+ text_hidden_size,
39
+ bias=True)
40
+ self.act = get_act_fn(projector_hidden_act)
41
+ self.linear_2 = nn.Linear(text_hidden_size,
42
+ text_hidden_size,
43
+ bias=True)
44
+
45
+ def forward(self, image_features):
46
+ hidden_states = self.linear_1(image_features)
47
+ hidden_states = self.act(hidden_states)
48
+ hidden_states = self.linear_2(hidden_states)
49
+ return hidden_states
50
+
51
+
52
+def _merge_vision_embeddings(input_ids: torch.Tensor,
53
+ inputs_embeds: torch.Tensor,
54
+ vision_embeddings: torch.Tensor,
55
+ image_token_id: int):
56
+ """In place merges in vision_embeddings with inputs_embeds."""
57
+ mask = (input_ids == image_token_id)
58
+ inputs_embeds[mask] = vision_embeddings.view(-1,
59
+ vision_embeddings.shape[-1])
60
+
61
+
62
+class LlavaForConditionalGeneration(nn.Module):
63
+
64
+ def __init__(self,
65
+ config: "LlavaConfig",
66
+ vision_language_config: VisionLanguageConfig,
67
+ linear_method: Optional["LinearMethodBase"] = None) -> None:
68
+ super().__init__()
69
+ self.config = config
70
+
71
+ self.vision_language_config = vision_language_config
72
+
73
+ assert self.vision_language_config, (
74
+ "Provide `image_input_type` and other vision "
75
+ "related configurations through LLM entrypoint "
76
+ "or engine arguments.")
77
+
78
+ if self.vision_language_config.image_input_type == (
79
+ VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
80
+ self.vision_tower = CLIPVisionModel(config.vision_config)
81
+ else:
82
+ self.vision_tower = None
83
+
84
+ self.multi_modal_projector = LlavaMultiModalProjector(
85
+ vision_hidden_size=config.vision_config.hidden_size,
86
+ text_hidden_size=config.text_config.hidden_size,
87
+ projector_hidden_act=config.projector_hidden_act)
88
+
89
+ self.linear_method = linear_method
90
+ self.language_model = LlamaModel(config.text_config, linear_method)
91
+ self.unpadded_vocab_size = config.text_config.vocab_size
92
+ self.lm_head = ParallelLMHead(
93
+ self.unpadded_vocab_size,
94
+ config.text_config.hidden_size,
95
+ org_num_embeddings=self.language_model.org_vocab_size)
96
+ logit_scale = getattr(config, "logit_scale", 1.0)
97
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
98
+ config.vocab_size, logit_scale)
99
+ self.sampler = Sampler()
100
+
101
+ def forward(
102
+ self,
103
+ input_ids: torch.Tensor,
104
+ positions: torch.Tensor,
105
+ kv_caches: List[KVCache],
106
+ attn_metadata: AttentionMetadata,
107
+ image_input: Optional[torch.Tensor] = None
108
+ ) -> SamplerOutput: # noqa: E501
109
+ """Run forward pass for Llava 1.5.
110
+
111
+ One key thing to understand is the `input_ids` already accounts for the
112
+ positions of the to-be-inserted image embeddings.
113
+ Concretely, consider a text prompt:
114
+ "<image>\nUSER: What's the content of the image?\nASSISTANT:".
115
+ Tokenizer outputs:
116
+ [1, 32000, 29871, 13, 11889, 29901, 1724, 29915, 29879, 278,
117
+ 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901].
118
+ The to-be-inserted image has a size of 576 (24 * 24) along the context
119
+ length dimension.
120
+ `input_ids` is thus [1, 32000, ..., 32000, 29871, 13, 11889, 29901,
121
+ 1724, 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933,
122
+ 9047, 13566, 29901].
123
+ There will be 576 `32000` in the `input_ids`.
124
+ (32000 is the token id for `<image>`.)
125
+
126
+ This way, the `positions` and `attn_metadata` are consistent
127
+ with the `input_ids`.
128
+
129
+ The model takes two types of image inputs:
130
+ PIXEL_VALUES and IMAGE_FEATURES.
131
+ The following shows how each maps to huggingface implementation.
132
+ PIXEL_VALUES:
133
+ - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L353
134
+ IMAGE_FEATURES:
135
+ - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L430
136
+ before going through the multi modal projector.
137
+
138
+ Args:
139
+ input_ids: Flattened (concatenated) input_ids corresponding to a
140
+ batch.
141
+ image_input: A batch of image inputs.
142
+ For PIXEL_VALUES, expecting [1, 3, 336, 336].
143
+ For IMAGE_FEATURES, expecting [1, 576, 1024].
144
+ """
145
+ if image_input is not None:
146
+ if list(image_input.shape[1:]) != list(
147
+ self.vision_language_config.image_input_shape[1:]):
148
+ raise ValueError(
149
+ f"The expected image tensor shape is batch dimension "
150
+ f"plus "
151
+ f"{self.vision_language_config.image_input_shape[1:]}."
152
+ f" You supplied {image_input.shape}. "
153
+ f"If you are using vLLM's entrypoint, make sure your "
154
+ f"supplied image input is consistent with "
155
+ f"image_input_shape in engine args.")
156
+ if self.vision_tower is not None:
157
+ # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
158
+ image_outputs = self.vision_tower(image_input,
159
+ output_hidden_states=True)
160
+ image_features = image_outputs.hidden_states[
161
+ self.config.vision_feature_layer]
162
+ # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
163
+ if self.config.vision_feature_select_strategy == "default":
164
+ image_features = image_features[:, 1:]
165
+ elif self.config.vision_feature_select_strategy == "full":
166
+ image_features = image_features
167
+ else:
168
+ raise ValueError(
169
+ f"Unexpected select feature strategy: "
170
+ f"{self.config.vision_feature_select_strategy}")
171
+ else:
172
+ image_features = image_input
173
+ vision_embeddings = self.multi_modal_projector(image_features)
174
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
175
+ _merge_vision_embeddings(
176
+ input_ids, inputs_embeds, vision_embeddings,
177
+ self.vision_language_config.image_token_id)
178
+ input_ids = None
179
+ else:
180
+ inputs_embeds = None
181
+ hidden_states = self.language_model(input_ids,
182
+ positions,
183
+ kv_caches,
184
+ attn_metadata,
185
+ inputs_embeds=inputs_embeds)
186
+
187
+ return hidden_states
188
+
189
+ def compute_logits(self, hidden_states: torch.Tensor,
190
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
191
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
192
+ sampling_metadata)
193
+ return logits
194
+
195
+ def sample(
196
+ self,
197
+ logits: torch.Tensor,
198
+ sampling_metadata: SamplingMetadata,
199
+ ) -> Optional[SamplerOutput]:
200
+ next_tokens = self.sampler(logits, sampling_metadata)
201
+ return next_tokens
202
+
203
+ def load_weights(self,
204
+ model_name_or_path: str,
205
+ cache_dir: Optional[str] = None,
206
+ load_format: str = "auto",
207
+ revision: Optional[str] = None):
208
+ # only doing this for language model part for now.
209
+ stacked_params_mapping = [
210
+ # (param_name, shard_name, shard_id)
211
+ ("qkv_proj", "q_proj", "q"),
212
+ ("qkv_proj", "k_proj", "k"),
213
+ ("qkv_proj", "v_proj", "v"),
214
+ ("gate_up_proj", "gate_proj", 0),
215
+ ("gate_up_proj", "up_proj", 1),
216
+ ]
217
+ params_dict = dict(self.named_parameters())
218
+ for name, loaded_weight in hf_model_weights_iterator(
219
+ model_name_or_path, cache_dir, load_format, revision):
220
+ if "rotary_emb.inv_freq" in name:
221
+ continue
222
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
223
+ if key_to_modify in name:
224
+ name = name.replace(key_to_modify, new_key)
225
+ use_default_weight_loading = False
226
+ if "vision" in name:
227
+ if self.vision_tower is not None:
228
+ # We only do sharding for language model and
229
+ # not vision model for now.
230
+ use_default_weight_loading = True
231
+ else:
232
+ for (param_name, weight_name,
233
+ shard_id) in stacked_params_mapping:
234
+ if weight_name not in name:
235
+ continue
236
+ param = params_dict[name.replace(weight_name, param_name)]
237
+ weight_loader = param.weight_loader
238
+ weight_loader(param, loaded_weight, shard_id)
239
+ break
240
+ else:
241
+ use_default_weight_loading = True
242
+ if use_default_weight_loading:
243
+ param = params_dict[name]
244
+ weight_loader = getattr(param, "weight_loader",
245
+ default_weight_loader)
246
+ weight_loader(param, loaded_weight)
vllm/sequence.py
+25
@@ -303,6 +303,25 @@ class SequenceGroupState:
303
generator: Optional = None
304
305
306
+class MultiModalData:
307
+ """Multi modal request.
308
+
309
+ Args:
310
+ type: The data type.
311
+ data: The actual data.
312
+ The required shape and semantic meaning of it depends on the vision
313
+ language config of the hosted model.
314
+ See `VisionLanguageConfig` in `config.py`.
315
+ """
316
+
317
+ class Type(enum.Enum):
318
+ IMAGE = enum.auto()
319
+
320
+ def __init__(self, type: Type, data: "torch.Tensor"):
321
+ self.type = type
322
+ self.data = data
323
+
324
+
325
class SequenceGroup:
326
"""A group of sequences that are generated from the same prompt.
327
@@ -312,6 +331,7 @@ class SequenceGroup:
331
sampling_params: The sampling parameters used to generate the outputs.
332
arrival_time: The arrival time of the request.
333
lora_request: LoRA request.
334
+ multi_modal_data: Multi modal data associated with the request.
335
"""
336
337
def __init__(
@@ -321,6 +341,7 @@ class SequenceGroup:
341
sampling_params: SamplingParams,
342
arrival_time: float,
343
lora_request: Optional[LoRARequest] = None,
344
+ multi_modal_data: Optional[MultiModalData] = None,
345
) -> None:
346
self.request_id = request_id
347
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -333,6 +354,7 @@ class SequenceGroup:
354
self.lora_request = lora_request
355
self.prompt_logprobs: Optional[PromptLogprobs] = None
356
self.state = SequenceGroupState()
357
+ self.multi_modal_data = multi_modal_data
358
359
@property
360
def prompt(self) -> str:
@@ -450,6 +472,7 @@ class SequenceGroupMetadata:
472
numbers)
473
state: Internal state tied to this sequence group.
474
lora_request: LoRA request.
475
+ multi_modal_data: Multi modal data.
476
"""
477
478
def __init__(
@@ -462,6 +485,7 @@ class SequenceGroupMetadata:
485
lora_request: Optional[LoRARequest] = None,
486
computed_block_nums: Optional[List[int]] = None,
487
state: Optional[SequenceGroupState] = None,
488
+ multi_modal_data: Optional[MultiModalData] = None,
489
) -> None:
490
self.request_id = request_id
491
self.is_prompt = is_prompt
@@ -470,6 +494,7 @@ class SequenceGroupMetadata:
494
self.block_tables = block_tables
495
self.lora_request = lora_request
496
self.computed_block_nums = computed_block_nums
497
+ self.multi_modal_data = multi_modal_data
498
self.state = SequenceGroupState() if state is None else state
499
500
@property
vllm/transformers_utils/config.py
+14
@@ -40,3 +40,17 @@ def get_config(model: str,
40
revision=revision,
41
code_revision=code_revision)
42
return config
43
+
44
+
45
+def get_hf_text_config(config: PretrainedConfig):
46
+ """Get the "sub" config relevant to llm for multi modal models.
47
+ No op for pure text models.
48
+ """
49
+ if hasattr(config, "text_config"):
50
+ # The code operates under the assumption that text_config should have
51
+ # `num_attention_heads` (among others). Assert here to fail early
52
+ # if transformers config doesn't align with this assumption.
53
+ assert hasattr(config.text_config, "num_attention_heads")
54
+ return config.text_config
55
+ else:
56
+ return config
vllm/utils.py
+10
@@ -377,6 +377,16 @@ class CudaMemoryProfiler:
377
gc.collect()
378
379
380
+def str_to_int_tuple(s: str) -> Tuple[int]:
381
+ """Convert a string to a tuple of integers."""
382
+ try:
383
+ return tuple(map(int, s.split(",")))
384
+ except ValueError as e:
385
+ raise ValueError(
386
+ "String must be a series of integers separated by commas "
387
+ f"(e.g., 1, 2, 3). Given input: {s}") from e
388
+
389
+
390
def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
391
assert len(x) <= max_len
392
return x + [pad] * (max_len - len(x))
vllm/worker/model_runner.py
+85
-21
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9
from vllm.attention import AttentionMetadata, get_attn_backend
10
from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11
- SchedulerConfig)
11
+ SchedulerConfig, VisionLanguageConfig)
12
from vllm.logger import init_logger
13
from vllm.lora.layers import LoRAMapping
14
from vllm.lora.request import LoRARequest
@@ -21,7 +21,8 @@ from vllm.model_executor.parallel_utils.communication_op import (
21
from vllm.model_executor.parallel_utils.parallel_state import (
22
with_cupy_nccl_for_all_reduce)
23
from vllm.sampling_params import SamplingParams, SamplingType
24
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
24
+from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25
+ SequenceGroupMetadata)
26
from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d,
27
is_pin_memory_available, make_tensor_with_pad,
28
maybe_expand_dim)
@@ -49,6 +50,7 @@ class ModelRunner:
50
lora_config: Optional[LoRAConfig],
51
kv_cache_dtype: Optional[str] = "auto",
52
is_driver_worker: bool = False,
53
+ vision_language_config: Optional[VisionLanguageConfig] = None,
54
):
55
self.model_config = model_config
56
self.parallel_config = parallel_config
@@ -83,17 +85,20 @@ class ModelRunner:
85
self.graph_block_tables = None # Set after initial profiling.
86
self.pin_memory = is_pin_memory_available()
87
self.kv_cache_dtype = kv_cache_dtype
88
+ self.vision_language_config = vision_language_config
89
90
self.attn_backend = get_attn_backend(
91
self.model_config.dtype if model_config is not None else None)
92
93
def load_model(self) -> None:
94
with CudaMemoryProfiler() as m:
92
- self.model = get_model(self.model_config,
93
- self.device_config,
94
- lora_config=self.lora_config,
95
- parallel_config=self.parallel_config,
96
- scheduler_config=self.scheduler_config)
95
+ self.model = get_model(
96
+ self.model_config,
97
+ self.device_config,
98
+ lora_config=self.lora_config,
99
+ vision_language_config=self.vision_language_config,
100
+ parallel_config=self.parallel_config,
101
+ scheduler_config=self.scheduler_config)
102
103
self.model_memory_usage = m.consumed_memory
104
logger.info(f"Loading model weights took "
@@ -130,7 +135,8 @@ class ModelRunner:
135
self,
136
seq_group_metadata_list: List[SequenceGroupMetadata],
137
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
133
- List[int], List[int], List[int], Set[LoRARequest]]:
138
+ List[int], List[int], List[int], Set[LoRARequest],
139
+ torch.Tensor]:
140
assert len(seq_group_metadata_list) > 0
141
input_tokens: List[int] = []
142
input_positions: List[int] = []
@@ -143,6 +149,7 @@ class ModelRunner:
149
context_lens: List[int] = []
150
subquery_lens: List[int] = []
151
prefix_block_tables: List[List[int]] = []
152
+ multi_modal_input_list: List[torch.Tensor] = []
153
for seq_group_metadata in seq_group_metadata_list:
154
assert seq_group_metadata.is_prompt
155
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -188,6 +195,10 @@ class ModelRunner:
195
(prompt_len - computed_len
196
if seq_group_metadata.sampling_params.prompt_logprobs else 1))
197
198
+ if seq_group_metadata.multi_modal_data:
199
+ multi_modal_input_list.append(
200
+ seq_group_metadata.multi_modal_data.data)
201
+
202
if seq_group_metadata.block_tables is None:
203
# During memory profiling, the block tables are not initialized
204
# yet. In this case, we just use a dummy slot mapping.
@@ -236,6 +247,16 @@ class ModelRunner:
247
context_lens_tensor = torch.tensor(context_lens,
248
dtype=torch.int,
249
device=self.device)
250
+
251
+ if multi_modal_input_list:
252
+ assert self.vision_language_config, (
253
+ "Multi-modal inputs are only supported by "
254
+ "vision language models.")
255
+ multi_modal_input = torch.cat(multi_modal_input_list,
256
+ dim=0).to(self.device)
257
+ else:
258
+ multi_modal_input = None
259
+
260
# Prepare prefix block tables
261
max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
262
block_tables = make_tensor_with_pad(
@@ -291,7 +312,7 @@ class ModelRunner:
312
)
313
return (input_tokens, input_positions, attn_metadata, prompt_lens,
314
subquery_lens, lora_index_mapping, lora_prompt_mapping,
294
- lora_requests)
315
+ lora_requests, multi_modal_input)
316
317
def _prepare_decode(
318
self,
@@ -525,7 +546,7 @@ class ModelRunner:
546
self,
547
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
548
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
528
- Set[int], LoRAMapping]:
549
+ Set[int], LoRAMapping, torch.Tensor]:
550
if self.is_driver_worker:
551
# NOTE: We assume that all sequences in the group are all prompts or
552
# all decodes.
@@ -534,13 +555,15 @@ class ModelRunner:
555
if is_prompt:
556
(input_tokens, input_positions, attn_metadata, prompt_lens,
557
subquery_lens, lora_index_mapping, lora_prompt_mapping,
537
- lora_requests) = self._prepare_prompt(seq_group_metadata_list)
558
+ lora_requests, multi_modal_input
559
+ ) = self._prepare_prompt(seq_group_metadata_list)
560
else:
561
(input_tokens, input_positions, attn_metadata,
562
lora_index_mapping, lora_prompt_mapping,
563
lora_requests) = self._prepare_decode(seq_group_metadata_list)
564
prompt_lens = []
565
subquery_lens = None
566
+ multi_modal_input = None
567
sampling_metadata = self._prepare_sample(seq_group_metadata_list,
568
prompt_lens,
569
subquery_lens)
@@ -561,6 +584,7 @@ class ModelRunner:
584
sampling_metadata.selected_token_indices,
585
"lora_requests": lora_requests,
586
"lora_mapping": lora_mapping,
587
+ "multi_modal_input": multi_modal_input,
588
}
589
metadata_dict.update(attn_metadata.asdict_zerocopy())
590
broadcast_tensor_dict(metadata_dict, src=0)
@@ -572,6 +596,7 @@ class ModelRunner:
596
"selected_token_indices")
597
lora_mapping = metadata_dict.pop("lora_mapping")
598
lora_requests = metadata_dict.pop("lora_requests")
599
+ multi_modal_input = metadata_dict.pop("multi_modal_input")
600
attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
601
sampling_metadata = SamplingMetadata(
602
seq_groups=None,
@@ -584,7 +609,8 @@ class ModelRunner:
609
)
610
611
return (input_tokens, input_positions, attn_metadata,
587
- sampling_metadata, lora_requests, lora_mapping)
612
+ sampling_metadata, lora_requests, lora_mapping,
613
+ multi_modal_input)
614
615
@torch.inference_mode()
616
def execute_model(
@@ -593,8 +619,8 @@ class ModelRunner:
619
kv_caches: List[torch.Tensor],
620
) -> Optional[SamplerOutput]:
621
(input_tokens, input_positions, attn_metadata, sampling_metadata,
596
- lora_requests,
597
- lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
622
+ lora_requests, lora_mapping, multi_modal_input
623
+ ) = self.prepare_input_tensors(seq_group_metadata_list)
624
625
if self.lora_config:
626
self.set_active_loras(lora_requests, lora_mapping)
@@ -605,12 +631,15 @@ class ModelRunner:
631
model_executable = self.graph_runners[graph_batch_size]
632
else:
633
model_executable = self.model
608
- hidden_states = model_executable(
609
- input_ids=input_tokens,
610
- positions=input_positions,
611
- kv_caches=kv_caches,
612
- attn_metadata=attn_metadata,
613
- )
634
+ execute_model_kwargs = {
635
+ "input_ids": input_tokens,
636
+ "positions": input_positions,
637
+ "kv_caches": kv_caches,
638
+ "attn_metadata": attn_metadata,
639
+ }
640
+ if self.vision_language_config:
641
+ execute_model_kwargs.update({"image_input": multi_modal_input})
642
+ hidden_states = model_executable(**execute_model_kwargs)
643
644
# Compute the logits.
645
logits = self.model.compute_logits(hidden_states, sampling_metadata)
@@ -658,10 +687,22 @@ class ModelRunner:
687
# Profile memory usage with max_num_sequences sequences and the total
688
# number of tokens equal to max_num_batched_tokens.
689
seqs: List[SequenceGroupMetadata] = []
690
+ # Additional GPU memory may be needed for vision encoding, which needs
691
+ # to be accounted for when calculating the GPU blocks for
692
+ # vLLM blocker manager.
693
+ # To exercise the worst scenario for GPU memory consumption,
694
+ # the number of seqs (batch_size) is chosen to maximize the number
695
+ # of images processed.
696
+ if self.vision_language_config:
697
+ max_num_seqs = min(
698
+ max_num_seqs,
699
+ int(max_num_batched_tokens /
700
+ self.vision_language_config.image_feature_size))
701
for group_id in range(max_num_seqs):
702
seq_len = (max_num_batched_tokens // max_num_seqs +
703
(group_id < max_num_batched_tokens % max_num_seqs))
664
- seq_data = SequenceData([0] * seq_len)
704
+ seq_data, fake_multi_modal_input = _prepare_fake_inputs(
705
+ seq_len, self.vision_language_config)
706
seq = SequenceGroupMetadata(
707
request_id=str(group_id),
708
is_prompt=True,
@@ -670,6 +711,7 @@ class ModelRunner:
711
block_tables=None,
712
lora_request=dummy_lora_requests_per_seq[group_id]
713
if dummy_lora_requests_per_seq else None,
714
+ multi_modal_data=fake_multi_modal_input,
715
)
716
seqs.append(seq)
717
@@ -831,6 +873,7 @@ class CUDAGraphRunner:
873
kv_caches: List[torch.Tensor],
874
attn_metadata: AttentionMetadata,
875
memory_pool,
876
+ **kwargs,
877
) -> None:
878
assert self.graph is None
879
# Run the model once without capturing the graph.
@@ -842,6 +885,7 @@ class CUDAGraphRunner:
885
positions,
886
kv_caches,
887
attn_metadata,
888
+ **kwargs,
889
)
890
torch.cuda.synchronize()
891
@@ -856,6 +900,7 @@ class CUDAGraphRunner:
900
positions,
901
kv_caches,
902
attn_metadata,
903
+ **kwargs,
904
)
905
torch.cuda.synchronize()
906
@@ -877,6 +922,7 @@ class CUDAGraphRunner:
922
positions: torch.Tensor,
923
kv_caches: List[torch.Tensor],
924
attn_metadata: AttentionMetadata,
925
+ **kwargs,
926
) -> torch.Tensor:
927
# KV caches are fixed tensors, so we don't need to copy them.
928
del kv_caches
@@ -922,3 +968,21 @@ def _get_graph_batch_size(batch_size: int) -> int:
968
else:
969
return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
970
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
971
+
972
+
973
+def _prepare_fake_inputs(
974
+ seq_len: int, vision_language_config: Optional[VisionLanguageConfig]):
975
+ """Prepare fake inputs for profile run."""
976
+ if vision_language_config:
977
+ prompt_tokens = [
978
+ vision_language_config.image_token_id
979
+ ] * vision_language_config.image_feature_size + [0] * (
980
+ seq_len - vision_language_config.image_feature_size)
981
+ fake_image_input = MultiModalData(
982
+ type=MultiModalData.Type.IMAGE,
983
+ data=torch.zeros(vision_language_config.image_input_shape,
984
+ dtype=torch.float16))
985
+ else:
986
+ prompt_tokens = [0] * seq_len
987
+ fake_image_input = None
988
+ return SequenceData(prompt_tokens), fake_image_input
vllm/worker/worker.py
+16
-8
@@ -7,7 +7,7 @@ import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig)
10
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
from vllm.lora.request import LoRARequest
12
from vllm.model_executor import set_random_seed
13
from vllm.model_executor.parallel_utils import cupy_utils
@@ -39,6 +39,7 @@ class Worker:
39
rank: int,
40
distributed_init_method: str,
41
lora_config: Optional[LoRAConfig] = None,
42
+ vision_language_config: Optional[VisionLanguageConfig] = None,
43
kv_cache_dtype: Optional[str] = "auto",
44
is_driver_worker: bool = False,
45
) -> None:
@@ -54,13 +55,20 @@ class Worker:
55
if self.is_driver_worker:
56
assert self.rank == 0, "The driver worker must have rank 0."
57
57
- self.model_runner = ModelRunner(model_config,
58
- parallel_config,
59
- scheduler_config,
60
- device_config,
61
- lora_config=self.lora_config,
62
- kv_cache_dtype=kv_cache_dtype,
63
- is_driver_worker=is_driver_worker)
58
+ self.vision_language_config = vision_language_config
59
+ if self.vision_language_config:
60
+ assert not self.lora_config, (
61
+ "To be tested: vision language model with LoRA settings.")
62
+
63
+ self.model_runner = ModelRunner(
64
+ model_config,
65
+ parallel_config,
66
+ scheduler_config,
67
+ device_config,
68
+ lora_config=self.lora_config,
69
+ kv_cache_dtype=kv_cache_dtype,
70
+ is_driver_worker=is_driver_worker,
71
+ vision_language_config=vision_language_config)
72
# Uninitialized cache engine. Will be initialized by
73
# self.init_cache_engine().
74
self.cache_config = None