@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64172a976c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Feature] Add vision language model support. (#3042)

xwjiang2010 committed Mar 25, 2024 at 14:16 UTC 64172a976c8d975b3aec946f1675716d2532d94f
28 files changed +936 -94
.buildkite/download-images.sh new
+18
@@ -0,0 +1,18 @@
1 +#!/bin/bash
2 +
3 +set -ex
4 +set -o pipefail
5 +
6 +(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
7 +
8 +# aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9 +mkdir -p images
10 +cd images
11 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_pixel_values.pt
12 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_image_features.pt
13 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_pixel_values.pt
14 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_image_features.pt
15 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
16 +wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
17 +
18 +cd -
.buildkite/test-pipeline.yaml
+7 -1
@@ -39,9 +39,15 @@ steps:
39
40 - label: Models Test
41 commands:
42 - - pytest -v -s models --forked
42 + - bash ../.buildkite/download-images.sh
43 + - pytest -v -s models --ignore=models/test_llava.py --forked
44 soft_fail: true
45
46 +- label: Llava Test
47 + commands:
48 + - bash ../.buildkite/download-images.sh
49 + - pytest -v -s models/test_llava.py
50 +
51 - label: Prefix Caching Test
52 commands:
53 - pytest -v -s prefix_caching
examples/llava_example.py new
+84
@@ -0,0 +1,84 @@
1 +import argparse
2 +import os
3 +import subprocess
4 +
5 +import torch
6 +
7 +from vllm import LLM
8 +from vllm.sequence import MultiModalData
9 +
10 +# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
11 +
12 +
13 +def run_llava_pixel_values():
14 + llm = LLM(
15 + model="llava-hf/llava-1.5-7b-hf",
16 + image_input_type="pixel_values",
17 + image_token_id=32000,
18 + image_input_shape="1,3,336,336",
19 + image_feature_size=576,
20 + )
21 +
22 + prompt = "<image>" * 576 + (
23 + "\nUSER: What is the content of this image?\nASSISTANT:")
24 +
25 + # This should be provided by another online or offline component.
26 + images = torch.load("images/stop_sign_pixel_values.pt")
27 +
28 + outputs = llm.generate(prompt,
29 + multi_modal_data=MultiModalData(
30 + type=MultiModalData.Type.IMAGE, data=images))
31 + for o in outputs:
32 + generated_text = o.outputs[0].text
33 + print(generated_text)
34 +
35 +
36 +def run_llava_image_features():
37 + llm = LLM(
38 + model="llava-hf/llava-1.5-7b-hf",
39 + image_input_type="image_features",
40 + image_token_id=32000,
41 + image_input_shape="1,576,1024",
42 + image_feature_size=576,
43 + )
44 +
45 + prompt = "<image>" * 576 + (
46 + "\nUSER: What is the content of this image?\nASSISTANT:")
47 +
48 + # This should be provided by another online or offline component.
49 + images = torch.load("images/stop_sign_image_features.pt")
50 +
51 + outputs = llm.generate(prompt,
52 + multi_modal_data=MultiModalData(
53 + type=MultiModalData.Type.IMAGE, data=images))
54 + for o in outputs:
55 + generated_text = o.outputs[0].text
56 + print(generated_text)
57 +
58 +
59 +def main(args):
60 + if args.type == "pixel_values":
61 + run_llava_pixel_values()
62 + else:
63 + run_llava_image_features()
64 +
65 +
66 +if __name__ == "__main__":
67 + parser = argparse.ArgumentParser(description="Demo on Llava")
68 + parser.add_argument("--type",
69 + type=str,
70 + choices=["pixel_values", "image_features"],
71 + default="pixel_values",
72 + help="image input type")
73 + args = parser.parse_args()
74 + # Download from s3
75 + s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
76 + local_directory = "images"
77 +
78 + # Make sure the local directory exists or create it
79 + os.makedirs(local_directory, exist_ok=True)
80 +
81 + # Use AWS CLI to sync the directory
82 + subprocess.check_call(
83 + ["aws", "s3", "sync", s3_bucket_path, local_directory])
84 + main(args)
requirements-dev.txt
+4
@@ -24,6 +24,10 @@ openai
24 requests
25 ray
26 peft
27 +awscli
28
29 # Benchmarking
30 aiohttp
31 +
32 +# Multimodal
33 +pillow
tests/conftest.py
+112 -14
@@ -3,16 +3,39 @@ from typing import List, Optional, Tuple
3
4 import pytest
5 import torch
6 -from transformers import AutoModelForCausalLM
6 +from PIL import Image
7 +from transformers import (AutoModelForCausalLM, AutoProcessor,
8 + LlavaForConditionalGeneration)
9
10 from vllm import LLM, SamplingParams
9 -from vllm.config import TokenizerPoolConfig
11 +from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
12 +from vllm.sequence import MultiModalData
13 from vllm.transformers_utils.tokenizer import get_tokenizer
14
15 _TEST_DIR = os.path.dirname(__file__)
16 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
17 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
18
19 +# Multi modal related
20 +_PIXEL_VALUES_FILES = [
21 + os.path.join(_TEST_DIR, "images", filename) for filename in
22 + ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
23 +]
24 +_IMAGE_FEATURES_FILES = [
25 + os.path.join(_TEST_DIR, "images", filename) for filename in
26 + ["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
27 +]
28 +_IMAGE_FILES = [
29 + os.path.join(_TEST_DIR, "images", filename)
30 + for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
31 +]
32 +_IMAGE_PROMPTS = [
33 + "<image>\nUSER: What's the content of the image?\nASSISTANT:",
34 + "<image>\nUSER: What is the season?\nASSISTANT:"
35 +]
36 +assert len(_PIXEL_VALUES_FILES) == len(_IMAGE_FEATURES_FILES) == len(
37 + _IMAGE_FILES) == len(_IMAGE_PROMPTS)
38 +
39
40 def _read_prompts(filename: str) -> List[str]:
41 with open(filename, "r") as f:
@@ -20,6 +43,39 @@ def _read_prompts(filename: str) -> List[str]:
43 return prompts
44
45
46 +@pytest.fixture(scope="session")
47 +def hf_image_prompts() -> List[str]:
48 + return _IMAGE_PROMPTS
49 +
50 +
51 +@pytest.fixture(scope="session")
52 +def hf_images() -> List[Image.Image]:
53 + return [Image.open(filename) for filename in _IMAGE_FILES]
54 +
55 +
56 +@pytest.fixture()
57 +def vllm_images(request) -> "torch.Tensor":
58 + vision_language_config = request.getfixturevalue("model_and_config")[1]
59 + all_images = []
60 + if vision_language_config.image_input_type == (
61 + VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
62 + filenames = _IMAGE_FEATURES_FILES
63 + else:
64 + filenames = _PIXEL_VALUES_FILES
65 + for filename in filenames:
66 + all_images.append(torch.load(filename))
67 + return torch.concat(all_images, dim=0)
68 +
69 +
70 +@pytest.fixture()
71 +def vllm_image_prompts(request) -> List[str]:
72 + vision_language_config = request.getfixturevalue("model_and_config")[1]
73 + return [
74 + "<image>" * (vision_language_config.image_feature_size - 1) + p
75 + for p in _IMAGE_PROMPTS
76 + ]
77 +
78 +
79 @pytest.fixture
80 def example_prompts() -> List[str]:
81 prompts = []
@@ -42,6 +98,10 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
98 "float": torch.float,
99 }
100
101 +_VISION_LANGUAGE_MODELS = {
102 + "llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
103 +}
104 +
105
106 class HfRunner:
107
@@ -53,11 +113,24 @@ class HfRunner:
113 ) -> None:
114 assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
115 torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
56 - self.model = AutoModelForCausalLM.from_pretrained(
57 - model_name,
58 - torch_dtype=torch_dtype,
59 - trust_remote_code=True,
60 - ).cuda()
116 + self.model_name = model_name
117 + if model_name not in _VISION_LANGUAGE_MODELS:
118 + self.model = AutoModelForCausalLM.from_pretrained(
119 + model_name,
120 + torch_dtype=torch_dtype,
121 + trust_remote_code=True,
122 + ).cuda()
123 + self.processor = None
124 + else:
125 + self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
126 + model_name,
127 + torch_dtype=torch_dtype,
128 + trust_remote_code=True,
129 + ).cuda()
130 + self.processor = AutoProcessor.from_pretrained(
131 + model_name,
132 + torch_dtype=torch_dtype,
133 + )
134 if tokenizer_name is None:
135 tokenizer_name = model_name
136 self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
@@ -65,13 +138,28 @@ class HfRunner:
138 def generate(
139 self,
140 prompts: List[str],
141 + images: Optional[List[Image.Image]] = None,
142 **kwargs,
143 ) -> List[Tuple[List[int], str]]:
144 outputs: List[Tuple[List[int], str]] = []
71 - for prompt in prompts:
72 - input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
145 + if images:
146 + assert len(prompts) == len(images)
147 + for i, prompt in enumerate(prompts):
148 + if self.model_name not in _VISION_LANGUAGE_MODELS:
149 + input_ids = self.tokenizer(prompt,
150 + return_tensors="pt").input_ids
151 + inputs = {"input_ids": input_ids.cuda()}
152 + else:
153 + image = images[i] if images else None
154 + inputs = self.processor(text=prompt,
155 + images=image,
156 + return_tensors="pt")
157 + inputs = {
158 + key: value.cuda() if value is not None else None
159 + for key, value in inputs.items()
160 + }
161 output_ids = self.model.generate(
74 - input_ids.cuda(),
162 + **inputs,
163 use_cache=True,
164 **kwargs,
165 )
@@ -88,10 +176,12 @@ class HfRunner:
176 self,
177 prompts: List[str],
178 max_tokens: int,
179 + images: Optional["torch.Tensor"] = None,
180 ) -> List[Tuple[List[int], str]]:
181 outputs = self.generate(prompts,
182 do_sample=False,
94 - max_new_tokens=max_tokens)
183 + max_new_tokens=max_tokens,
184 + images=images)
185 for i in range(len(outputs)):
186 output_ids, output_str = outputs[i]
187 outputs[i] = (output_ids[0], output_str[0])
@@ -183,9 +273,16 @@ class VllmRunner:
273 self,
274 prompts: List[str],
275 sampling_params: SamplingParams,
276 + images: Optional["torch.Tensor"] = None,
277 ) -> List[Tuple[List[int], str]]:
187 - req_outputs = self.model.generate(prompts,
188 - sampling_params=sampling_params)
278 + if images is not None:
279 + assert len(prompts) == images.shape[0]
280 + req_outputs = self.model.generate(
281 + prompts,
282 + sampling_params=sampling_params,
283 + multi_modal_data=MultiModalData(type=MultiModalData.Type.IMAGE,
284 + data=images)
285 + if images is not None else None)
286 outputs = []
287 for req_output in req_outputs:
288 prompt_str = req_output.prompt
@@ -222,9 +319,10 @@ class VllmRunner:
319 self,
320 prompts: List[str],
321 max_tokens: int,
322 + images: Optional[torch.Tensor] = None,
323 ) -> List[Tuple[List[int], str]]:
324 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
227 - outputs = self.generate(prompts, greedy_params)
325 + outputs = self.generate(prompts, greedy_params, images=images)
326 return [(output_ids[0], output_str[0])
327 for output_ids, output_str in outputs]
328
tests/models/test_llava.py new
+110
@@ -0,0 +1,110 @@
1 +import gc
2 +from dataclasses import fields
3 +from enum import Enum
4 +from typing import Dict, List, Tuple
5 +
6 +import pytest
7 +import torch
8 +from transformers import AutoTokenizer
9 +
10 +from vllm.config import VisionLanguageConfig
11 +
12 +model_and_vl_config = [
13 + ("llava-hf/llava-1.5-7b-hf",
14 + VisionLanguageConfig(
15 + image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
16 + image_feature_size=576,
17 + image_token_id=32000,
18 + image_input_shape=(1, 3, 336, 336))),
19 + ("llava-hf/llava-1.5-7b-hf",
20 + VisionLanguageConfig(
21 + image_input_type=VisionLanguageConfig.ImageInputType.IMAGE_FEATURES,
22 + image_feature_size=576,
23 + image_token_id=32000,
24 + image_input_shape=(1, 576, 1024)))
25 +]
26 +
27 +
28 +def as_dict(vision_language_config: VisionLanguageConfig) -> Dict:
29 + """Flatten vision language config to pure args.
30 +
31 + Compatible with what llm entrypoint expects.
32 + """
33 + result = {}
34 + for field in fields(vision_language_config):
35 + value = getattr(vision_language_config, field.name)
36 + if isinstance(value, Enum):
37 + result[field.name] = value.name.lower()
38 + elif isinstance(value, tuple):
39 + result[field.name] = ",".join([str(item) for item in value])
40 + else:
41 + result[field.name] = value
42 + return result
43 +
44 +
45 +def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
46 + vision_language_config: VisionLanguageConfig,
47 + model_id: str):
48 + """Sanitize vllm output to be comparable with hf output.
49 + The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
50 + x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
51 + It also reduces `output_str` from "<image><image>bla" to "bla".
52 + """
53 + tokenizer = AutoTokenizer.from_pretrained(model_id)
54 + image_token_str = tokenizer.decode(vision_language_config.image_token_id)
55 + image_token_str_len = len(image_token_str)
56 + input_ids, output_str = vllm_output
57 + sanitized_input_ids = input_ids[0:2] + input_ids[2 + vision_language_config
58 + .image_feature_size - 1:]
59 + sanitzied_output_str = output_str[vision_language_config.
60 + image_feature_size *
61 + image_token_str_len:]
62 + return sanitized_input_ids, sanitzied_output_str
63 +
64 +
65 +@pytest.mark.parametrize("worker_use_ray", [False])
66 +@pytest.mark.parametrize("model_and_config", model_and_vl_config)
67 +@pytest.mark.parametrize("dtype", ["half"])
68 +@pytest.mark.parametrize("max_tokens", [128])
69 +def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
70 + vllm_image_prompts, vllm_images, model_and_config: tuple,
71 + dtype: str, max_tokens: int, worker_use_ray: bool) -> None:
72 + """Inference result should be the same between hf and vllm.
73 +
74 + All the image fixtures for the test is under tests/images.
75 + For huggingface runner, we provide the raw images as input.
76 + For vllm runner, we provide image tensors and corresponding
77 + vision language config as input.
78 + Note, the text input is also adjusted to abide by vllm contract.
79 + The text output is sanitized to be able to compare with hf.
80 + """
81 + model_id, vision_language_config = model_and_config
82 + hf_model = hf_runner(model_id, dtype=dtype)
83 + hf_outputs = hf_model.generate_greedy(hf_image_prompts,
84 + max_tokens,
85 + images=hf_images)
86 + del hf_model
87 +
88 + gc.collect()
89 + torch.cuda.empty_cache()
90 +
91 + vllm_model = vllm_runner(model_id,
92 + dtype=dtype,
93 + worker_use_ray=worker_use_ray,
94 + **as_dict(vision_language_config))
95 + vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
96 + max_tokens,
97 + images=vllm_images)
98 + del vllm_model
99 +
100 + gc.collect()
101 + torch.cuda.empty_cache()
102 +
103 + for i in range(len(hf_image_prompts)):
104 + hf_output_ids, hf_output_str = hf_outputs[i]
105 + vllm_output_ids, vllm_output_str = sanitize_vllm_output(
106 + vllm_outputs[i], vision_language_config, model_id)
107 + assert hf_output_str == vllm_output_str, (
108 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
109 + assert hf_output_ids == vllm_output_ids, (
110 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/models/test_models.py
+1 -1
@@ -25,7 +25,7 @@ MODELS = [
25
26
27 @pytest.mark.parametrize("model", MODELS)
28 -@pytest.mark.parametrize("dtype", ["float"])
28 +@pytest.mark.parametrize("dtype", ["half"])
29 @pytest.mark.parametrize("max_tokens", [128])
30 def test_models(
31 hf_runner,
tests/spec_decode/utils.py
+1 -1
@@ -109,7 +109,7 @@ def create_worker(cls: type,
109 )
110
111 (model_config, cache_config, parallel_config, scheduler_config,
112 - device_config, _) = engine_args.create_engine_configs()
112 + device_config, _, _) = engine_args.create_engine_configs()
113
114 distributed_init_method = get_distributed_init_method(
115 get_ip(), get_open_port())
tests/worker/test_model_runner.py
+1 -1
@@ -35,7 +35,7 @@ def test_prepare_prompt(batch_size):
35 prompt_len - 1)
36 selected_token_start_idx += prompt_len
37 (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
38 - _) = (model_runner._prepare_prompt(seq_group_metadata_list))
38 + _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
39 assert return_prompt_lens == prompt_lens
40
41 # Verify input metadata is correct for prompts.
tests/worker/test_swap.py
+1 -1
@@ -11,7 +11,7 @@ def test_swap() -> None:
11 dtype="half",
12 load_format="dummy")
13 (model_config, cache_config, parallel_config, scheduler_config,
14 - device_config, _) = engine_args.create_engine_configs()
14 + device_config, _, _) = engine_args.create_engine_configs()
15 cache_config.num_gpu_blocks = 100
16 cache_config.num_cpu_blocks = 100
17
vllm/config.py
+62 -17
@@ -1,3 +1,4 @@
1 +import enum
2 import json
3 import os
4 from dataclasses import dataclass
@@ -8,7 +9,7 @@ from packaging.version import Version
9 from transformers import PretrainedConfig
10
11 from vllm.logger import init_logger
11 -from vllm.transformers_utils.config import get_config
12 +from vllm.transformers_utils.config import get_config, get_hf_text_config
13 from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
14
15 if TYPE_CHECKING:
@@ -118,8 +119,9 @@ class ModelConfig:
119
120 self.hf_config = get_config(self.model, trust_remote_code, revision,
121 code_revision)
121 - self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
122 - self.max_model_len = _get_and_verify_max_len(self.hf_config,
122 + self.hf_text_config = get_hf_text_config(self.hf_config)
123 + self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
124 + self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
125 max_model_len)
126 self._verify_load_format()
127 self._verify_tokenizer_mode()
@@ -218,7 +220,7 @@ class ModelConfig:
220 self,
221 parallel_config: "ParallelConfig",
222 ) -> None:
221 - total_num_attention_heads = self.hf_config.num_attention_heads
223 + total_num_attention_heads = self.hf_text_config.num_attention_heads
224 tensor_parallel_size = parallel_config.tensor_parallel_size
225 if total_num_attention_heads % tensor_parallel_size != 0:
226 raise ValueError(
@@ -226,7 +228,7 @@ class ModelConfig:
228 " must be divisible by tensor parallel size "
229 f"({tensor_parallel_size}).")
230
229 - total_num_hidden_layers = self.hf_config.num_hidden_layers
231 + total_num_hidden_layers = self.hf_text_config.num_hidden_layers
232 pipeline_parallel_size = parallel_config.pipeline_parallel_size
233 if total_num_hidden_layers % pipeline_parallel_size != 0:
234 raise ValueError(
@@ -241,22 +243,23 @@ class ModelConfig:
243 # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
244 # addition to sliding window size. We check if that field is present
245 # and if it's False, return None.
244 - if (hasattr(self.hf_config, "use_sliding_window")
245 - and not self.hf_config.use_sliding_window):
246 + if (hasattr(self.hf_text_config, "use_sliding_window")
247 + and not self.hf_text_config.use_sliding_window):
248 return None
247 - return getattr(self.hf_config, "sliding_window", None)
249 + return getattr(self.hf_text_config, "sliding_window", None)
250
251 def get_vocab_size(self) -> int:
250 - return self.hf_config.vocab_size
252 + return self.hf_text_config.vocab_size
253
254 def get_hidden_size(self) -> int:
253 - return self.hf_config.hidden_size
255 + return self.hf_text_config.hidden_size
256
257 def get_head_size(self) -> int:
256 - if hasattr(self.hf_config, "head_dim"):
257 - return self.hf_config.head_dim
258 + if hasattr(self.hf_text_config, "head_dim"):
259 + return self.hf_text_config.head_dim
260 # FIXME(woosuk): This may not be true for all models.
259 - return self.hf_config.hidden_size // self.hf_config.num_attention_heads
261 + return (self.hf_text_config.hidden_size //
262 + self.hf_text_config.num_attention_heads)
263
264 def get_total_num_kv_heads(self) -> int:
265 """Returns the total number of KV heads."""
@@ -268,7 +271,7 @@ class ModelConfig:
271 new_decoder_arch_falcon = (
272 self.hf_config.model_type in falcon_model_types
273 and getattr(self.hf_config, "new_decoder_architecture", False))
271 - if not new_decoder_arch_falcon and getattr(self.hf_config,
274 + if not new_decoder_arch_falcon and getattr(self.hf_text_config,
275 "multi_query", False):
276 # Multi-query attention, only one KV head.
277 # Currently, tensor parallelism is not supported in this case.
@@ -284,13 +287,13 @@ class ModelConfig:
287 "multi_query_group_num",
288 ]
289 for attr in attributes:
287 - num_kv_heads = getattr(self.hf_config, attr, None)
290 + num_kv_heads = getattr(self.hf_text_config, attr, None)
291 if num_kv_heads is not None:
292 return num_kv_heads
293
294 # For non-grouped-query attention models, the number of KV heads is
295 # equal to the number of attention heads.
293 - return self.hf_config.num_attention_heads
296 + return self.hf_text_config.num_attention_heads
297
298 def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
299 """Returns the number of KV heads per GPU."""
@@ -303,7 +306,7 @@ class ModelConfig:
306 total_num_kv_heads // parallel_config.tensor_parallel_size)
307
308 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
306 - total_num_hidden_layers = self.hf_config.num_hidden_layers
309 + total_num_hidden_layers = self.hf_text_config.num_hidden_layers
310 return total_num_hidden_layers // parallel_config.pipeline_parallel_size
311
312
@@ -627,6 +630,48 @@ class LoRAConfig:
630 "LoRA is enabled.")
631
632
633 +@dataclass
634 +class VisionLanguageConfig:
635 + """Configs the input data format and how models should run for
636 + vision language models."""
637 +
638 + class ImageInputType(enum.Enum):
639 + """Image input type into the vision language model.
640 +
641 + An image roughly goes through the following transformation:
642 + Raw image --> pixel values --> image features --> image embeddings.
643 +
644 + The difference between different image input types is where the
645 + image encoder (pixel values --> image features) is run.
646 + Different image input types also correspond to different tensor shapes.
647 +
648 + For example, for Llava, PIXEL_VALUES: (1, 3, 336, 336).
649 + IMAGE_FEATURES: (1, 576, 1024).
650 + """
651 + PIXEL_VALUES = enum.auto()
652 + IMAGE_FEATURES = enum.auto()
653 +
654 + image_input_type: ImageInputType
655 + # The input id corresponding to image token.
656 + image_token_id: int
657 + # Used for running `run_prefill_max_token`.
658 + # For models that support varying resolution, this corresponds to
659 + # worst case scenario (biggest supported resolution).
660 + image_input_shape: tuple
661 + image_feature_size: int
662 +
663 + @classmethod
664 + def get_image_input_enum_type(
665 + cls, value: str) -> "VisionLanguageConfig.ImageInputType":
666 + """Get the image input type from a string."""
667 + try:
668 + return cls.ImageInputType[value.upper()]
669 + except KeyError as e:
670 + raise ValueError(f"{value} is not a valid choice. "
671 + f"Expecting to choose from "
672 + f"{[x.name for x in cls.ImageInputType]}.") from e
673 +
674 +
675 _STR_DTYPE_TO_TORCH_DTYPE = {
676 "half": torch.float16,
677 "float16": torch.float16,
vllm/core/scheduler.py
+6
@@ -388,6 +388,12 @@ class Scheduler:
388 computed_block_nums=self.block_manager.
389 get_common_computed_block_ids(seq_group),
390 state=seq_group.state,
391 + # `multi_modal_data` will only be present for the 1st comm
392 + # between engine and worker.
393 + # the subsequent comms can still use delta, but
394 + # `multi_modal_data` will be None.
395 + multi_modal_data=seq_group.multi_modal_data
396 + if scheduler_outputs.prompt_run else None,
397 )
398 seq_group_metadata_list.append(seq_group_metadata)
399 return seq_group_metadata_list, scheduler_outputs
vllm/engine/arg_utils.py
+53 -3
@@ -4,7 +4,9 @@ from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig, TokenizerPoolConfig)
7 + ParallelConfig, SchedulerConfig, TokenizerPoolConfig,
8 + VisionLanguageConfig)
9 +from vllm.utils import str_to_int_tuple
10
11
12 @dataclass
@@ -50,6 +52,11 @@ class EngineArgs:
52 max_cpu_loras: Optional[int] = None
53 device: str = 'auto'
54 ray_workers_use_nsight: bool = False
55 + # Related to Vision-language models such as llava
56 + image_input_type: Optional[str] = None
57 + image_token_id: Optional[int] = None
58 + image_input_shape: Optional[str] = None
59 + image_feature_size: Optional[int] = None
60 scheduler_delay_factor: float = 0.0
61
62 def __post_init__(self):
@@ -305,6 +312,31 @@ class EngineArgs:
312 default=EngineArgs.device,
313 choices=["auto", "cuda", "neuron"],
314 help='Device type for vLLM execution.')
315 + # Related to Vision-language models such as llava
316 + parser.add_argument(
317 + '--image-input-type',
318 + type=str,
319 + default=None,
320 + choices=[
321 + t.name.lower() for t in VisionLanguageConfig.ImageInputType
322 + ],
323 + help=('The image input type passed into vLLM. '
324 + 'Should be one of "pixel_values" or "image_features".'))
325 + parser.add_argument('--image-token-id',
326 + type=int,
327 + default=None,
328 + help=('Input id for image token.'))
329 + parser.add_argument(
330 + '--image-input-shape',
331 + type=str,
332 + default=None,
333 + help=('The biggest image input shape (worst for memory footprint) '
334 + 'given an input type. Only used for vLLM\'s profile_run.'))
335 + parser.add_argument(
336 + '--image-feature-size',
337 + type=int,
338 + default=None,
339 + help=('The image feature size along the context dimension.'))
340 parser.add_argument(
341 '--scheduler-delay-factor',
342 type=float,
@@ -324,7 +356,8 @@ class EngineArgs:
356 def create_engine_configs(
357 self,
358 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
327 - DeviceConfig, Optional[LoRAConfig]]:
359 + DeviceConfig, Optional[LoRAConfig],
360 + Optional[VisionLanguageConfig]]:
361 device_config = DeviceConfig(self.device)
362 model_config = ModelConfig(
363 self.model, self.tokenizer, self.tokenizer_mode,
@@ -358,8 +391,25 @@ class EngineArgs:
391 lora_dtype=self.lora_dtype,
392 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
393 and self.max_cpu_loras > 0 else None) if self.enable_lora else None
394 +
395 + if self.image_input_type:
396 + if (not self.image_token_id or not self.image_input_shape
397 + or not self.image_feature_size):
398 + raise ValueError(
399 + 'Specify `image_token_id`, `image_input_shape` and '
400 + '`image_feature_size` together with `image_input_type`.')
401 + vision_language_config = VisionLanguageConfig(
402 + image_input_type=VisionLanguageConfig.
403 + get_image_input_enum_type(self.image_input_type),
404 + image_token_id=self.image_token_id,
405 + image_input_shape=str_to_int_tuple(self.image_input_shape),
406 + image_feature_size=self.image_feature_size,
407 + )
408 + else:
409 + vision_language_config = None
410 +
411 return (model_config, cache_config, parallel_config, scheduler_config,
362 - device_config, lora_config)
412 + device_config, lora_config, vision_language_config)
413
414
415 @dataclass
vllm/engine/async_llm_engine.py
+16 -9
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 +from vllm.sequence import MultiModalData
19
20 logger = init_logger(__name__)
21 ENGINE_ITERATION_TIMEOUT_S = int(
@@ -240,6 +241,7 @@ class _AsyncLLMEngine(LLMEngine):
241 prompt_token_ids: Optional[List[int]] = None,
242 arrival_time: Optional[float] = None,
243 lora_request: Optional[LoRARequest] = None,
244 + multi_modal_data: Optional[MultiModalData] = None,
245 ) -> None:
246 if lora_request is not None and not self.lora_config:
247 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -252,14 +254,13 @@ class _AsyncLLMEngine(LLMEngine):
254 prompt_token_ids=prompt_token_ids,
255 lora_request=lora_request)
256
255 - return self.add_request(
256 - request_id,
257 - prompt=prompt,
258 - prompt_token_ids=prompt_token_ids,
259 - sampling_params=sampling_params,
260 - arrival_time=arrival_time,
261 - lora_request=lora_request,
262 - )
257 + return self.add_request(request_id,
258 + prompt=prompt,
259 + prompt_token_ids=prompt_token_ids,
260 + sampling_params=sampling_params,
261 + arrival_time=arrival_time,
262 + lora_request=lora_request,
263 + multi_modal_data=multi_modal_data)
264
265 async def check_health_async(self) -> None:
266 self.model_executor.check_health()
@@ -486,6 +487,7 @@ class AsyncLLMEngine:
487 prompt_token_ids: Optional[List[int]] = None,
488 arrival_time: Optional[float] = None,
489 lora_request: Optional[LoRARequest] = None,
490 + multi_modal_data: Optional[MultiModalData] = None,
491 ) -> AsyncStream:
492 if self.log_requests:
493 shortened_prompt = prompt
@@ -534,7 +536,9 @@ class AsyncLLMEngine:
536 sampling_params=sampling_params,
537 prompt_token_ids=prompt_token_ids,
538 arrival_time=arrival_time,
537 - lora_request=lora_request)
539 + lora_request=lora_request,
540 + multi_modal_data=multi_modal_data,
541 + )
542
543 return stream
544
@@ -545,6 +549,7 @@ class AsyncLLMEngine:
549 request_id: str,
550 prompt_token_ids: Optional[List[int]] = None,
551 lora_request: Optional[LoRARequest] = None,
552 + multi_modal_data: Optional[MultiModalData] = None
553 ) -> AsyncIterator[RequestOutput]:
554 """Generate outputs for a request.
555
@@ -560,6 +565,7 @@ class AsyncLLMEngine:
565 prompt_token_ids: The token IDs of the prompt. If None, we
566 use the tokenizer to convert the prompts to token IDs.
567 lora_request: LoRA request to use for generation, if any.
568 + multi_modal_data: Multi modal data per request.
569
570 Yields:
571 The output `RequestOutput` objects from the LLMEngine for the
@@ -619,6 +625,7 @@ class AsyncLLMEngine:
625 prompt_token_ids=prompt_token_ids,
626 arrival_time=arrival_time,
627 lora_request=lora_request,
628 + multi_modal_data=multi_modal_data,
629 )
630
631 async for request_output in stream:
vllm/engine/llm_engine.py
+11 -6
@@ -5,7 +5,7 @@ from transformers import PreTrainedTokenizer
5
6 import vllm
7 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig)
8 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9 from vllm.core.scheduler import Scheduler, SchedulerOutputs
10 from vllm.engine.arg_utils import EngineArgs
11 from vllm.engine.metrics import StatLogger, Stats
@@ -15,8 +15,9 @@ from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 -from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19 - SequenceGroupOutput, SequenceOutput, SequenceStatus)
18 +from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
19 + SequenceGroup, SequenceGroupOutput, SequenceOutput,
20 + SequenceStatus)
21 from vllm.transformers_utils.detokenizer import Detokenizer
22 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
23 get_tokenizer_group)
@@ -62,6 +63,7 @@ class LLMEngine:
63 scheduler_config: SchedulerConfig,
64 device_config: DeviceConfig,
65 lora_config: Optional[LoRAConfig],
66 + vision_language_config: Optional["VisionLanguageConfig"],
67 executor_class: Type[ExecutorBase],
68 log_stats: bool,
69 ) -> None:
@@ -90,6 +92,7 @@ class LLMEngine:
92 self.model_config = model_config
93 self.cache_config = cache_config
94 self.lora_config = lora_config
95 + self.vision_language_config = vision_language_config
96 self.parallel_config = parallel_config
97 self.scheduler_config = scheduler_config
98 self.device_config = device_config
@@ -102,7 +105,8 @@ class LLMEngine:
105
106 self.model_executor = executor_class(model_config, cache_config,
107 parallel_config, scheduler_config,
105 - device_config, lora_config)
108 + device_config, lora_config,
109 + vision_language_config)
110
111 # Ping the tokenizer to ensure liveness if it runs in a
112 # different process.
@@ -170,7 +174,6 @@ class LLMEngine:
174 trust_remote_code=self.model_config.trust_remote_code,
175 revision=self.model_config.tokenizer_revision)
176 init_kwargs.update(tokenizer_init_kwargs)
173 -
177 self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
178 self.parallel_config.tokenizer_pool_config, **init_kwargs)
179
@@ -212,6 +215,7 @@ class LLMEngine:
215 prompt_token_ids: Optional[List[int]] = None,
216 arrival_time: Optional[float] = None,
217 lora_request: Optional[LoRARequest] = None,
218 + multi_modal_data: Optional[MultiModalData] = None,
219 ) -> None:
220 """Add a request to the engine's request pool.
221
@@ -228,6 +232,7 @@ class LLMEngine:
232 use the tokenizer to convert the prompts to token IDs.
233 arrival_time: The arrival time of the request. If None, we use
234 the current monotonic time.
235 + multi_modal_data: Multi modal data per request.
236
237 Details:
238 - Set arrival_time to the current time if it is None.
@@ -288,7 +293,7 @@ class LLMEngine:
293
294 # Create the sequence group.
295 seq_group = SequenceGroup(request_id, [seq], sampling_params,
291 - arrival_time, lora_request)
296 + arrival_time, lora_request, multi_modal_data)
297
298 # Add the sequence group to the scheduler.
299 self.scheduler.add_seq_group(seq_group)
vllm/entrypoints/llm.py
+21 -5
@@ -1,5 +1,6 @@
1 from typing import List, Optional, Union
2
3 +import torch
4 from tqdm import tqdm
5 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
@@ -8,6 +9,7 @@ from vllm.engine.llm_engine import LLMEngine
9 from vllm.lora.request import LoRARequest
10 from vllm.outputs import RequestOutput
11 from vllm.sampling_params import SamplingParams
12 +from vllm.sequence import MultiModalData
13 from vllm.utils import Counter
14
15
@@ -126,6 +128,7 @@ class LLM:
128 prompt_token_ids: Optional[List[List[int]]] = None,
129 use_tqdm: bool = True,
130 lora_request: Optional[LoRARequest] = None,
131 + multi_modal_data: Optional[MultiModalData] = None,
132 ) -> List[RequestOutput]:
133 """Generates the completions for the input prompts.
134
@@ -141,6 +144,7 @@ class LLM:
144 use the tokenizer to convert the prompts to token IDs.
145 use_tqdm: Whether to use tqdm to display the progress bar.
146 lora_request: LoRA request to use for generation, if any.
147 + multi_modal_data: Multi modal data.
148
149 Returns:
150 A list of `RequestOutput` objects containing the generated
@@ -160,6 +164,9 @@ class LLM:
164 # Use default sampling params.
165 sampling_params = SamplingParams()
166
167 + if multi_modal_data:
168 + multi_modal_data.data = multi_modal_data.data.to(torch.float16)
169 +
170 # Add requests to the engine.
171 num_requests = len(prompts) if prompts is not None else len(
172 prompt_token_ids)
@@ -167,10 +174,17 @@ class LLM:
174 prompt = prompts[i] if prompts is not None else None
175 token_ids = None if prompt_token_ids is None else prompt_token_ids[
176 i]
170 - self._add_request(prompt,
171 - sampling_params,
172 - token_ids,
173 - lora_request=lora_request)
177 + self._add_request(
178 + prompt,
179 + sampling_params,
180 + token_ids,
181 + lora_request=lora_request,
182 + # Get ith image while maintaining the batch dim.
183 + multi_modal_data=MultiModalData(
184 + type=multi_modal_data.type,
185 + data=multi_modal_data.data[i].unsqueeze(0))
186 + if multi_modal_data else None,
187 + )
188 return self._run_engine(use_tqdm)
189
190 def _add_request(
@@ -179,13 +193,15 @@ class LLM:
193 sampling_params: SamplingParams,
194 prompt_token_ids: Optional[List[int]],
195 lora_request: Optional[LoRARequest] = None,
196 + multi_modal_data: Optional[MultiModalData] = None,
197 ) -> None:
198 request_id = str(next(self.request_counter))
199 self.llm_engine.add_request(request_id,
200 prompt,
201 sampling_params,
202 prompt_token_ids,
188 - lora_request=lora_request)
203 + lora_request=lora_request,
204 + multi_modal_data=multi_modal_data)
205
206 def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
207 # Initialize tqdm.
vllm/executor/executor_base.py
+2 -1
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
2 from typing import Dict, List, Optional
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig)
5 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
6 from vllm.lora.request import LoRARequest
7 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
@@ -24,6 +24,7 @@ class ExecutorBase(ABC):
24 scheduler_config: SchedulerConfig,
25 device_config: DeviceConfig,
26 lora_config: Optional[LoRAConfig],
27 + vision_language_config: Optional[VisionLanguageConfig],
28 ) -> None:
29 raise NotImplementedError
30
vllm/executor/gpu_executor.py
+4 -1
@@ -1,7 +1,7 @@
1 from typing import Dict, List, Optional
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 - ParallelConfig, SchedulerConfig)
4 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 from vllm.executor.utils import check_block_size_valid
7 from vllm.logger import init_logger
@@ -23,6 +23,7 @@ class GPUExecutor(ExecutorBase):
23 scheduler_config: SchedulerConfig,
24 device_config: DeviceConfig,
25 lora_config: Optional[LoRAConfig],
26 + vision_language_config: Optional[VisionLanguageConfig],
27 ) -> None:
28 self.model_config = model_config
29 self.cache_config = cache_config
@@ -30,6 +31,7 @@ class GPUExecutor(ExecutorBase):
31 self.parallel_config = parallel_config
32 self.scheduler_config = scheduler_config
33 self.device_config = device_config
34 + self.vision_language_config = vision_language_config
35
36 # Instantiate the worker and load the model to GPU.
37 self._init_worker()
@@ -56,6 +58,7 @@ class GPUExecutor(ExecutorBase):
58 rank=0,
59 distributed_init_method=distributed_init_method,
60 lora_config=self.lora_config,
61 + vision_language_config=self.vision_language_config,
62 kv_cache_dtype=self.cache_config.cache_dtype,
63 is_driver_worker=True,
64 )
vllm/executor/ray_gpu_executor.py
+4 -1
@@ -6,7 +6,7 @@ from collections import defaultdict
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig)
9 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
10 from vllm.engine.ray_utils import RayWorkerVllm, ray
11 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
12 from vllm.executor.utils import check_block_size_valid
@@ -40,6 +40,7 @@ class RayGPUExecutor(ExecutorBase):
40 scheduler_config: SchedulerConfig,
41 device_config: DeviceConfig,
42 lora_config: Optional[LoRAConfig],
43 + vision_language_config: Optional[VisionLanguageConfig],
44 ) -> None:
45 self.model_config = model_config
46 self.cache_config = cache_config
@@ -47,6 +48,7 @@ class RayGPUExecutor(ExecutorBase):
48 self.parallel_config = parallel_config
49 self.scheduler_config = scheduler_config
50 self.device_config = device_config
51 + self.vision_language_config = vision_language_config
52
53 assert self.parallel_config.worker_use_ray
54 placement_group = self.parallel_config.placement_group
@@ -181,6 +183,7 @@ class RayGPUExecutor(ExecutorBase):
183 driver_rank,
184 distributed_init_method,
185 lora_config=self.lora_config,
186 + vision_language_config=self.vision_language_config,
187 kv_cache_dtype=kv_cache_dtype,
188 is_driver_worker=True,
189 )
vllm/model_executor/model_loader.py
+11 -1
@@ -7,9 +7,14 @@ import torch.nn as nn
7
8 from vllm.config import DeviceConfig, ModelConfig
9 from vllm.model_executor.models import ModelRegistry
10 +from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11 from vllm.model_executor.weight_utils import (get_quant_config,
12 initialize_dummy_weights)
13
14 +_VISION_MODEL_CLASSES = [
15 + LlavaForConditionalGeneration,
16 +]
17 +
18
19 @contextlib.contextmanager
20 def _set_default_torch_dtype(dtype: torch.dtype):
@@ -40,6 +45,7 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
45 def get_model(model_config: ModelConfig, device_config: DeviceConfig,
46 **kwargs) -> nn.Module:
47 lora_config = kwargs.get("lora_config", None)
48 + vision_language_config = kwargs.get("vision_language_config", None)
49 model_class = _get_model_architecture(model_config)
50
51 # Get the (maybe quantized) linear method.
@@ -76,7 +82,11 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
82 "be added in the future. If this is important to you, "
83 "please open an issue on github.")
84 else:
79 - model = model_class(model_config.hf_config, linear_method)
85 + if model_class not in _VISION_MODEL_CLASSES:
86 + model = model_class(model_config.hf_config, linear_method)
87 + else:
88 + model = model_class(model_config.hf_config,
89 + vision_language_config, linear_method)
90 if model_config.load_format == "dummy":
91 # NOTE(woosuk): For accurate performance evaluation, we assign
92 # random values to the weights.
vllm/model_executor/models/__init__.py
+2
@@ -29,6 +29,8 @@ _MODELS = {
29 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
30 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
31 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
32 + "LlavaForConditionalGeneration":
33 + ("llava", "LlavaForConditionalGeneration"),
34 # For decapoda-research/llama-*
35 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
36 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/llama.py
+9 -2
@@ -250,14 +250,21 @@ class LlamaModel(nn.Module):
250 ])
251 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
252
253 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
254 + return self.embed_tokens(input_ids)
255 +
256 def forward(
257 self,
255 - input_ids: torch.Tensor,
258 + input_ids: Optional[torch.Tensor],
259 positions: torch.Tensor,
260 kv_caches: List[torch.Tensor],
261 attn_metadata: AttentionMetadata,
262 + inputs_embeds: Optional[torch.Tensor] = None,
263 ) -> torch.Tensor:
260 - hidden_states = self.embed_tokens(input_ids)
264 + if inputs_embeds is not None:
265 + hidden_states = inputs_embeds
266 + else:
267 + hidden_states = self.get_input_embeddings(input_ids)
268 residual = None
269 for i in range(len(self.layers)):
270 layer = self.layers[i]
vllm/model_executor/models/llava.py new
+246
@@ -0,0 +1,246 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import torch
4 +from torch import nn
5 +# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
6 +# transformers' impl.
7 +from transformers import CLIPVisionModel, LlavaConfig
8 +
9 +from vllm.attention import AttentionMetadata
10 +from vllm.config import VisionLanguageConfig
11 +from vllm.model_executor.layers.activation import get_act_fn
12 +from vllm.model_executor.layers.linear import LinearMethodBase
13 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 +from vllm.model_executor.layers.sampler import Sampler
15 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16 +from vllm.model_executor.models.llama import LlamaModel
17 +from vllm.model_executor.sampling_metadata import SamplingMetadata
18 +from vllm.model_executor.weight_utils import (default_weight_loader,
19 + hf_model_weights_iterator)
20 +from vllm.sequence import SamplerOutput
21 +
22 +KVCache = Tuple[torch.Tensor, torch.Tensor]
23 +
24 +_KEYS_TO_MODIFY_MAPPING = {
25 + "language_model.lm_head": "lm_head",
26 + "language_model.model": "language_model",
27 +}
28 +
29 +
30 +# TODO(xwjiang): Run benchmark and decide if TP.
31 +class LlavaMultiModalProjector(nn.Module):
32 +
33 + def __init__(self, vision_hidden_size: int, text_hidden_size: int,
34 + projector_hidden_act: str):
35 + super().__init__()
36 +
37 + self.linear_1 = nn.Linear(vision_hidden_size,
38 + text_hidden_size,
39 + bias=True)
40 + self.act = get_act_fn(projector_hidden_act)
41 + self.linear_2 = nn.Linear(text_hidden_size,
42 + text_hidden_size,
43 + bias=True)
44 +
45 + def forward(self, image_features):
46 + hidden_states = self.linear_1(image_features)
47 + hidden_states = self.act(hidden_states)
48 + hidden_states = self.linear_2(hidden_states)
49 + return hidden_states
50 +
51 +
52 +def _merge_vision_embeddings(input_ids: torch.Tensor,
53 + inputs_embeds: torch.Tensor,
54 + vision_embeddings: torch.Tensor,
55 + image_token_id: int):
56 + """In place merges in vision_embeddings with inputs_embeds."""
57 + mask = (input_ids == image_token_id)
58 + inputs_embeds[mask] = vision_embeddings.view(-1,
59 + vision_embeddings.shape[-1])
60 +
61 +
62 +class LlavaForConditionalGeneration(nn.Module):
63 +
64 + def __init__(self,
65 + config: "LlavaConfig",
66 + vision_language_config: VisionLanguageConfig,
67 + linear_method: Optional["LinearMethodBase"] = None) -> None:
68 + super().__init__()
69 + self.config = config
70 +
71 + self.vision_language_config = vision_language_config
72 +
73 + assert self.vision_language_config, (
74 + "Provide `image_input_type` and other vision "
75 + "related configurations through LLM entrypoint "
76 + "or engine arguments.")
77 +
78 + if self.vision_language_config.image_input_type == (
79 + VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
80 + self.vision_tower = CLIPVisionModel(config.vision_config)
81 + else:
82 + self.vision_tower = None
83 +
84 + self.multi_modal_projector = LlavaMultiModalProjector(
85 + vision_hidden_size=config.vision_config.hidden_size,
86 + text_hidden_size=config.text_config.hidden_size,
87 + projector_hidden_act=config.projector_hidden_act)
88 +
89 + self.linear_method = linear_method
90 + self.language_model = LlamaModel(config.text_config, linear_method)
91 + self.unpadded_vocab_size = config.text_config.vocab_size
92 + self.lm_head = ParallelLMHead(
93 + self.unpadded_vocab_size,
94 + config.text_config.hidden_size,
95 + org_num_embeddings=self.language_model.org_vocab_size)
96 + logit_scale = getattr(config, "logit_scale", 1.0)
97 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
98 + config.vocab_size, logit_scale)
99 + self.sampler = Sampler()
100 +
101 + def forward(
102 + self,
103 + input_ids: torch.Tensor,
104 + positions: torch.Tensor,
105 + kv_caches: List[KVCache],
106 + attn_metadata: AttentionMetadata,
107 + image_input: Optional[torch.Tensor] = None
108 + ) -> SamplerOutput: # noqa: E501
109 + """Run forward pass for Llava 1.5.
110 +
111 + One key thing to understand is the `input_ids` already accounts for the
112 + positions of the to-be-inserted image embeddings.
113 + Concretely, consider a text prompt:
114 + "<image>\nUSER: What's the content of the image?\nASSISTANT:".
115 + Tokenizer outputs:
116 + [1, 32000, 29871, 13, 11889, 29901, 1724, 29915, 29879, 278,
117 + 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901].
118 + The to-be-inserted image has a size of 576 (24 * 24) along the context
119 + length dimension.
120 + `input_ids` is thus [1, 32000, ..., 32000, 29871, 13, 11889, 29901,
121 + 1724, 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933,
122 + 9047, 13566, 29901].
123 + There will be 576 `32000` in the `input_ids`.
124 + (32000 is the token id for `<image>`.)
125 +
126 + This way, the `positions` and `attn_metadata` are consistent
127 + with the `input_ids`.
128 +
129 + The model takes two types of image inputs:
130 + PIXEL_VALUES and IMAGE_FEATURES.
131 + The following shows how each maps to huggingface implementation.
132 + PIXEL_VALUES:
133 + - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L353
134 + IMAGE_FEATURES:
135 + - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L430
136 + before going through the multi modal projector.
137 +
138 + Args:
139 + input_ids: Flattened (concatenated) input_ids corresponding to a
140 + batch.
141 + image_input: A batch of image inputs.
142 + For PIXEL_VALUES, expecting [1, 3, 336, 336].
143 + For IMAGE_FEATURES, expecting [1, 576, 1024].
144 + """
145 + if image_input is not None:
146 + if list(image_input.shape[1:]) != list(
147 + self.vision_language_config.image_input_shape[1:]):
148 + raise ValueError(
149 + f"The expected image tensor shape is batch dimension "
150 + f"plus "
151 + f"{self.vision_language_config.image_input_shape[1:]}."
152 + f" You supplied {image_input.shape}. "
153 + f"If you are using vLLM's entrypoint, make sure your "
154 + f"supplied image input is consistent with "
155 + f"image_input_shape in engine args.")
156 + if self.vision_tower is not None:
157 + # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
158 + image_outputs = self.vision_tower(image_input,
159 + output_hidden_states=True)
160 + image_features = image_outputs.hidden_states[
161 + self.config.vision_feature_layer]
162 + # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
163 + if self.config.vision_feature_select_strategy == "default":
164 + image_features = image_features[:, 1:]
165 + elif self.config.vision_feature_select_strategy == "full":
166 + image_features = image_features
167 + else:
168 + raise ValueError(
169 + f"Unexpected select feature strategy: "
170 + f"{self.config.vision_feature_select_strategy}")
171 + else:
172 + image_features = image_input
173 + vision_embeddings = self.multi_modal_projector(image_features)
174 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
175 + _merge_vision_embeddings(
176 + input_ids, inputs_embeds, vision_embeddings,
177 + self.vision_language_config.image_token_id)
178 + input_ids = None
179 + else:
180 + inputs_embeds = None
181 + hidden_states = self.language_model(input_ids,
182 + positions,
183 + kv_caches,
184 + attn_metadata,
185 + inputs_embeds=inputs_embeds)
186 +
187 + return hidden_states
188 +
189 + def compute_logits(self, hidden_states: torch.Tensor,
190 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
191 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
192 + sampling_metadata)
193 + return logits
194 +
195 + def sample(
196 + self,
197 + logits: torch.Tensor,
198 + sampling_metadata: SamplingMetadata,
199 + ) -> Optional[SamplerOutput]:
200 + next_tokens = self.sampler(logits, sampling_metadata)
201 + return next_tokens
202 +
203 + def load_weights(self,
204 + model_name_or_path: str,
205 + cache_dir: Optional[str] = None,
206 + load_format: str = "auto",
207 + revision: Optional[str] = None):
208 + # only doing this for language model part for now.
209 + stacked_params_mapping = [
210 + # (param_name, shard_name, shard_id)
211 + ("qkv_proj", "q_proj", "q"),
212 + ("qkv_proj", "k_proj", "k"),
213 + ("qkv_proj", "v_proj", "v"),
214 + ("gate_up_proj", "gate_proj", 0),
215 + ("gate_up_proj", "up_proj", 1),
216 + ]
217 + params_dict = dict(self.named_parameters())
218 + for name, loaded_weight in hf_model_weights_iterator(
219 + model_name_or_path, cache_dir, load_format, revision):
220 + if "rotary_emb.inv_freq" in name:
221 + continue
222 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
223 + if key_to_modify in name:
224 + name = name.replace(key_to_modify, new_key)
225 + use_default_weight_loading = False
226 + if "vision" in name:
227 + if self.vision_tower is not None:
228 + # We only do sharding for language model and
229 + # not vision model for now.
230 + use_default_weight_loading = True
231 + else:
232 + for (param_name, weight_name,
233 + shard_id) in stacked_params_mapping:
234 + if weight_name not in name:
235 + continue
236 + param = params_dict[name.replace(weight_name, param_name)]
237 + weight_loader = param.weight_loader
238 + weight_loader(param, loaded_weight, shard_id)
239 + break
240 + else:
241 + use_default_weight_loading = True
242 + if use_default_weight_loading:
243 + param = params_dict[name]
244 + weight_loader = getattr(param, "weight_loader",
245 + default_weight_loader)
246 + weight_loader(param, loaded_weight)
vllm/sequence.py
+25
@@ -303,6 +303,25 @@ class SequenceGroupState:
303 generator: Optional = None
304
305
306 +class MultiModalData:
307 + """Multi modal request.
308 +
309 + Args:
310 + type: The data type.
311 + data: The actual data.
312 + The required shape and semantic meaning of it depends on the vision
313 + language config of the hosted model.
314 + See `VisionLanguageConfig` in `config.py`.
315 + """
316 +
317 + class Type(enum.Enum):
318 + IMAGE = enum.auto()
319 +
320 + def __init__(self, type: Type, data: "torch.Tensor"):
321 + self.type = type
322 + self.data = data
323 +
324 +
325 class SequenceGroup:
326 """A group of sequences that are generated from the same prompt.
327
@@ -312,6 +331,7 @@ class SequenceGroup:
331 sampling_params: The sampling parameters used to generate the outputs.
332 arrival_time: The arrival time of the request.
333 lora_request: LoRA request.
334 + multi_modal_data: Multi modal data associated with the request.
335 """
336
337 def __init__(
@@ -321,6 +341,7 @@ class SequenceGroup:
341 sampling_params: SamplingParams,
342 arrival_time: float,
343 lora_request: Optional[LoRARequest] = None,
344 + multi_modal_data: Optional[MultiModalData] = None,
345 ) -> None:
346 self.request_id = request_id
347 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -333,6 +354,7 @@ class SequenceGroup:
354 self.lora_request = lora_request
355 self.prompt_logprobs: Optional[PromptLogprobs] = None
356 self.state = SequenceGroupState()
357 + self.multi_modal_data = multi_modal_data
358
359 @property
360 def prompt(self) -> str:
@@ -450,6 +472,7 @@ class SequenceGroupMetadata:
472 numbers)
473 state: Internal state tied to this sequence group.
474 lora_request: LoRA request.
475 + multi_modal_data: Multi modal data.
476 """
477
478 def __init__(
@@ -462,6 +485,7 @@ class SequenceGroupMetadata:
485 lora_request: Optional[LoRARequest] = None,
486 computed_block_nums: Optional[List[int]] = None,
487 state: Optional[SequenceGroupState] = None,
488 + multi_modal_data: Optional[MultiModalData] = None,
489 ) -> None:
490 self.request_id = request_id
491 self.is_prompt = is_prompt
@@ -470,6 +494,7 @@ class SequenceGroupMetadata:
494 self.block_tables = block_tables
495 self.lora_request = lora_request
496 self.computed_block_nums = computed_block_nums
497 + self.multi_modal_data = multi_modal_data
498 self.state = SequenceGroupState() if state is None else state
499
500 @property
vllm/transformers_utils/config.py
+14
@@ -40,3 +40,17 @@ def get_config(model: str,
40 revision=revision,
41 code_revision=code_revision)
42 return config
43 +
44 +
45 +def get_hf_text_config(config: PretrainedConfig):
46 + """Get the "sub" config relevant to llm for multi modal models.
47 + No op for pure text models.
48 + """
49 + if hasattr(config, "text_config"):
50 + # The code operates under the assumption that text_config should have
51 + # `num_attention_heads` (among others). Assert here to fail early
52 + # if transformers config doesn't align with this assumption.
53 + assert hasattr(config.text_config, "num_attention_heads")
54 + return config.text_config
55 + else:
56 + return config
vllm/utils.py
+10
@@ -377,6 +377,16 @@ class CudaMemoryProfiler:
377 gc.collect()
378
379
380 +def str_to_int_tuple(s: str) -> Tuple[int]:
381 + """Convert a string to a tuple of integers."""
382 + try:
383 + return tuple(map(int, s.split(",")))
384 + except ValueError as e:
385 + raise ValueError(
386 + "String must be a series of integers separated by commas "
387 + f"(e.g., 1, 2, 3). Given input: {s}") from e
388 +
389 +
390 def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
391 assert len(x) <= max_len
392 return x + [pad] * (max_len - len(x))
vllm/worker/model_runner.py
+85 -21
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9 from vllm.attention import AttentionMetadata, get_attn_backend
10 from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11 - SchedulerConfig)
11 + SchedulerConfig, VisionLanguageConfig)
12 from vllm.logger import init_logger
13 from vllm.lora.layers import LoRAMapping
14 from vllm.lora.request import LoRARequest
@@ -21,7 +21,8 @@ from vllm.model_executor.parallel_utils.communication_op import (
21 from vllm.model_executor.parallel_utils.parallel_state import (
22 with_cupy_nccl_for_all_reduce)
23 from vllm.sampling_params import SamplingParams, SamplingType
24 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
24 +from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25 + SequenceGroupMetadata)
26 from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d,
27 is_pin_memory_available, make_tensor_with_pad,
28 maybe_expand_dim)
@@ -49,6 +50,7 @@ class ModelRunner:
50 lora_config: Optional[LoRAConfig],
51 kv_cache_dtype: Optional[str] = "auto",
52 is_driver_worker: bool = False,
53 + vision_language_config: Optional[VisionLanguageConfig] = None,
54 ):
55 self.model_config = model_config
56 self.parallel_config = parallel_config
@@ -83,17 +85,20 @@ class ModelRunner:
85 self.graph_block_tables = None # Set after initial profiling.
86 self.pin_memory = is_pin_memory_available()
87 self.kv_cache_dtype = kv_cache_dtype
88 + self.vision_language_config = vision_language_config
89
90 self.attn_backend = get_attn_backend(
91 self.model_config.dtype if model_config is not None else None)
92
93 def load_model(self) -> None:
94 with CudaMemoryProfiler() as m:
92 - self.model = get_model(self.model_config,
93 - self.device_config,
94 - lora_config=self.lora_config,
95 - parallel_config=self.parallel_config,
96 - scheduler_config=self.scheduler_config)
95 + self.model = get_model(
96 + self.model_config,
97 + self.device_config,
98 + lora_config=self.lora_config,
99 + vision_language_config=self.vision_language_config,
100 + parallel_config=self.parallel_config,
101 + scheduler_config=self.scheduler_config)
102
103 self.model_memory_usage = m.consumed_memory
104 logger.info(f"Loading model weights took "
@@ -130,7 +135,8 @@ class ModelRunner:
135 self,
136 seq_group_metadata_list: List[SequenceGroupMetadata],
137 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
133 - List[int], List[int], List[int], Set[LoRARequest]]:
138 + List[int], List[int], List[int], Set[LoRARequest],
139 + torch.Tensor]:
140 assert len(seq_group_metadata_list) > 0
141 input_tokens: List[int] = []
142 input_positions: List[int] = []
@@ -143,6 +149,7 @@ class ModelRunner:
149 context_lens: List[int] = []
150 subquery_lens: List[int] = []
151 prefix_block_tables: List[List[int]] = []
152 + multi_modal_input_list: List[torch.Tensor] = []
153 for seq_group_metadata in seq_group_metadata_list:
154 assert seq_group_metadata.is_prompt
155 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -188,6 +195,10 @@ class ModelRunner:
195 (prompt_len - computed_len
196 if seq_group_metadata.sampling_params.prompt_logprobs else 1))
197
198 + if seq_group_metadata.multi_modal_data:
199 + multi_modal_input_list.append(
200 + seq_group_metadata.multi_modal_data.data)
201 +
202 if seq_group_metadata.block_tables is None:
203 # During memory profiling, the block tables are not initialized
204 # yet. In this case, we just use a dummy slot mapping.
@@ -236,6 +247,16 @@ class ModelRunner:
247 context_lens_tensor = torch.tensor(context_lens,
248 dtype=torch.int,
249 device=self.device)
250 +
251 + if multi_modal_input_list:
252 + assert self.vision_language_config, (
253 + "Multi-modal inputs are only supported by "
254 + "vision language models.")
255 + multi_modal_input = torch.cat(multi_modal_input_list,
256 + dim=0).to(self.device)
257 + else:
258 + multi_modal_input = None
259 +
260 # Prepare prefix block tables
261 max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
262 block_tables = make_tensor_with_pad(
@@ -291,7 +312,7 @@ class ModelRunner:
312 )
313 return (input_tokens, input_positions, attn_metadata, prompt_lens,
314 subquery_lens, lora_index_mapping, lora_prompt_mapping,
294 - lora_requests)
315 + lora_requests, multi_modal_input)
316
317 def _prepare_decode(
318 self,
@@ -525,7 +546,7 @@ class ModelRunner:
546 self,
547 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
548 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
528 - Set[int], LoRAMapping]:
549 + Set[int], LoRAMapping, torch.Tensor]:
550 if self.is_driver_worker:
551 # NOTE: We assume that all sequences in the group are all prompts or
552 # all decodes.
@@ -534,13 +555,15 @@ class ModelRunner:
555 if is_prompt:
556 (input_tokens, input_positions, attn_metadata, prompt_lens,
557 subquery_lens, lora_index_mapping, lora_prompt_mapping,
537 - lora_requests) = self._prepare_prompt(seq_group_metadata_list)
558 + lora_requests, multi_modal_input
559 + ) = self._prepare_prompt(seq_group_metadata_list)
560 else:
561 (input_tokens, input_positions, attn_metadata,
562 lora_index_mapping, lora_prompt_mapping,
563 lora_requests) = self._prepare_decode(seq_group_metadata_list)
564 prompt_lens = []
565 subquery_lens = None
566 + multi_modal_input = None
567 sampling_metadata = self._prepare_sample(seq_group_metadata_list,
568 prompt_lens,
569 subquery_lens)
@@ -561,6 +584,7 @@ class ModelRunner:
584 sampling_metadata.selected_token_indices,
585 "lora_requests": lora_requests,
586 "lora_mapping": lora_mapping,
587 + "multi_modal_input": multi_modal_input,
588 }
589 metadata_dict.update(attn_metadata.asdict_zerocopy())
590 broadcast_tensor_dict(metadata_dict, src=0)
@@ -572,6 +596,7 @@ class ModelRunner:
596 "selected_token_indices")
597 lora_mapping = metadata_dict.pop("lora_mapping")
598 lora_requests = metadata_dict.pop("lora_requests")
599 + multi_modal_input = metadata_dict.pop("multi_modal_input")
600 attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
601 sampling_metadata = SamplingMetadata(
602 seq_groups=None,
@@ -584,7 +609,8 @@ class ModelRunner:
609 )
610
611 return (input_tokens, input_positions, attn_metadata,
587 - sampling_metadata, lora_requests, lora_mapping)
612 + sampling_metadata, lora_requests, lora_mapping,
613 + multi_modal_input)
614
615 @torch.inference_mode()
616 def execute_model(
@@ -593,8 +619,8 @@ class ModelRunner:
619 kv_caches: List[torch.Tensor],
620 ) -> Optional[SamplerOutput]:
621 (input_tokens, input_positions, attn_metadata, sampling_metadata,
596 - lora_requests,
597 - lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
622 + lora_requests, lora_mapping, multi_modal_input
623 + ) = self.prepare_input_tensors(seq_group_metadata_list)
624
625 if self.lora_config:
626 self.set_active_loras(lora_requests, lora_mapping)
@@ -605,12 +631,15 @@ class ModelRunner:
631 model_executable = self.graph_runners[graph_batch_size]
632 else:
633 model_executable = self.model
608 - hidden_states = model_executable(
609 - input_ids=input_tokens,
610 - positions=input_positions,
611 - kv_caches=kv_caches,
612 - attn_metadata=attn_metadata,
613 - )
634 + execute_model_kwargs = {
635 + "input_ids": input_tokens,
636 + "positions": input_positions,
637 + "kv_caches": kv_caches,
638 + "attn_metadata": attn_metadata,
639 + }
640 + if self.vision_language_config:
641 + execute_model_kwargs.update({"image_input": multi_modal_input})
642 + hidden_states = model_executable(**execute_model_kwargs)
643
644 # Compute the logits.
645 logits = self.model.compute_logits(hidden_states, sampling_metadata)
@@ -658,10 +687,22 @@ class ModelRunner:
687 # Profile memory usage with max_num_sequences sequences and the total
688 # number of tokens equal to max_num_batched_tokens.
689 seqs: List[SequenceGroupMetadata] = []
690 + # Additional GPU memory may be needed for vision encoding, which needs
691 + # to be accounted for when calculating the GPU blocks for
692 + # vLLM blocker manager.
693 + # To exercise the worst scenario for GPU memory consumption,
694 + # the number of seqs (batch_size) is chosen to maximize the number
695 + # of images processed.
696 + if self.vision_language_config:
697 + max_num_seqs = min(
698 + max_num_seqs,
699 + int(max_num_batched_tokens /
700 + self.vision_language_config.image_feature_size))
701 for group_id in range(max_num_seqs):
702 seq_len = (max_num_batched_tokens // max_num_seqs +
703 (group_id < max_num_batched_tokens % max_num_seqs))
664 - seq_data = SequenceData([0] * seq_len)
704 + seq_data, fake_multi_modal_input = _prepare_fake_inputs(
705 + seq_len, self.vision_language_config)
706 seq = SequenceGroupMetadata(
707 request_id=str(group_id),
708 is_prompt=True,
@@ -670,6 +711,7 @@ class ModelRunner:
711 block_tables=None,
712 lora_request=dummy_lora_requests_per_seq[group_id]
713 if dummy_lora_requests_per_seq else None,
714 + multi_modal_data=fake_multi_modal_input,
715 )
716 seqs.append(seq)
717
@@ -831,6 +873,7 @@ class CUDAGraphRunner:
873 kv_caches: List[torch.Tensor],
874 attn_metadata: AttentionMetadata,
875 memory_pool,
876 + **kwargs,
877 ) -> None:
878 assert self.graph is None
879 # Run the model once without capturing the graph.
@@ -842,6 +885,7 @@ class CUDAGraphRunner:
885 positions,
886 kv_caches,
887 attn_metadata,
888 + **kwargs,
889 )
890 torch.cuda.synchronize()
891
@@ -856,6 +900,7 @@ class CUDAGraphRunner:
900 positions,
901 kv_caches,
902 attn_metadata,
903 + **kwargs,
904 )
905 torch.cuda.synchronize()
906
@@ -877,6 +922,7 @@ class CUDAGraphRunner:
922 positions: torch.Tensor,
923 kv_caches: List[torch.Tensor],
924 attn_metadata: AttentionMetadata,
925 + **kwargs,
926 ) -> torch.Tensor:
927 # KV caches are fixed tensors, so we don't need to copy them.
928 del kv_caches
@@ -922,3 +968,21 @@ def _get_graph_batch_size(batch_size: int) -> int:
968 else:
969 return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
970 _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
971 +
972 +
973 +def _prepare_fake_inputs(
974 + seq_len: int, vision_language_config: Optional[VisionLanguageConfig]):
975 + """Prepare fake inputs for profile run."""
976 + if vision_language_config:
977 + prompt_tokens = [
978 + vision_language_config.image_token_id
979 + ] * vision_language_config.image_feature_size + [0] * (
980 + seq_len - vision_language_config.image_feature_size)
981 + fake_image_input = MultiModalData(
982 + type=MultiModalData.Type.IMAGE,
983 + data=torch.zeros(vision_language_config.image_input_shape,
984 + dtype=torch.float16))
985 + else:
986 + prompt_tokens = [0] * seq_len
987 + fake_image_input = None
988 + return SequenceData(prompt_tokens), fake_image_input
vllm/worker/worker.py
+16 -8
@@ -7,7 +7,7 @@ import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig)
10 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 from vllm.lora.request import LoRARequest
12 from vllm.model_executor import set_random_seed
13 from vllm.model_executor.parallel_utils import cupy_utils
@@ -39,6 +39,7 @@ class Worker:
39 rank: int,
40 distributed_init_method: str,
41 lora_config: Optional[LoRAConfig] = None,
42 + vision_language_config: Optional[VisionLanguageConfig] = None,
43 kv_cache_dtype: Optional[str] = "auto",
44 is_driver_worker: bool = False,
45 ) -> None:
@@ -54,13 +55,20 @@ class Worker:
55 if self.is_driver_worker:
56 assert self.rank == 0, "The driver worker must have rank 0."
57
57 - self.model_runner = ModelRunner(model_config,
58 - parallel_config,
59 - scheduler_config,
60 - device_config,
61 - lora_config=self.lora_config,
62 - kv_cache_dtype=kv_cache_dtype,
63 - is_driver_worker=is_driver_worker)
58 + self.vision_language_config = vision_language_config
59 + if self.vision_language_config:
60 + assert not self.lora_config, (
61 + "To be tested: vision language model with LoRA settings.")
62 +
63 + self.model_runner = ModelRunner(
64 + model_config,
65 + parallel_config,
66 + scheduler_config,
67 + device_config,
68 + lora_config=self.lora_config,
69 + kv_cache_dtype=kv_cache_dtype,
70 + is_driver_worker=is_driver_worker,
71 + vision_language_config=vision_language_config)
72 # Uninitialized cache engine. Will be initialized by
73 # self.init_cache_engine().
74 self.cache_config = None