A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Add vision language model support to CPU backend (#3968)
Isotr0py committed
Apr 22, 2024 at 15:44 UTC
296cdf8ac7853b671d10f3df094288ea4f35bb38
3 files changed
+53
-32
vllm/executor/cpu_executor.py
+1
@@ -45,6 +45,7 @@ class CPUExecutor(ExecutorBase):
45
rank=0,
46
distributed_init_method=distributed_init_method,
47
lora_config=self.lora_config,
48
+ vision_language_config=self.vision_language_config,
49
kv_cache_dtype=self.cache_config.cache_dtype,
50
is_driver_worker=True,
51
)
vllm/worker/cpu_model_runner.py
+37
-23
@@ -5,7 +5,7 @@ from torch import nn
5
6
from vllm.attention import AttentionMetadata, get_attn_backend
7
from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig)
8
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9
from vllm.distributed import broadcast_tensor_dict
10
from vllm.logger import init_logger
11
from vllm.model_executor import SamplingMetadata
@@ -29,6 +29,7 @@ class CPUModelRunner:
29
device_config: DeviceConfig,
30
load_config: LoadConfig,
31
lora_config: Optional[LoRAConfig],
32
+ vision_language_config: Optional[VisionLanguageConfig],
33
kv_cache_dtype: Optional[str] = "auto",
34
is_driver_worker: bool = False,
35
*args,
@@ -38,6 +39,7 @@ class CPUModelRunner:
39
self.parallel_config = parallel_config
40
self.scheduler_config = scheduler_config
41
self.lora_config = lora_config
42
+ self.vision_language_config = vision_language_config
43
self.load_config = load_config
44
self.is_driver_worker = is_driver_worker
45
@@ -59,13 +61,14 @@ class CPUModelRunner:
61
self.block_size: int # Set after initial profiling.
62
63
def load_model(self) -> None:
62
- self.model = get_model(model_config=self.model_config,
63
- load_config=self.load_config,
64
- device_config=self.device_config,
65
- vision_language_config=None,
66
- lora_config=self.lora_config,
67
- parallel_config=self.parallel_config,
68
- scheduler_config=self.scheduler_config)
64
+ self.model = get_model(
65
+ model_config=self.model_config,
66
+ load_config=self.load_config,
67
+ device_config=self.device_config,
68
+ vision_language_config=self.vision_language_config,
69
+ lora_config=self.lora_config,
70
+ parallel_config=self.parallel_config,
71
+ scheduler_config=self.scheduler_config)
72
73
def _prepare_prompt(
74
self,
@@ -76,6 +79,7 @@ class CPUModelRunner:
79
input_positions: List[int] = []
80
slot_mapping: List[int] = []
81
prompt_lens: List[int] = []
82
+ multi_modal_input_list: List[torch.Tensor] = []
83
84
for seq_group_metadata in seq_group_metadata_list:
85
assert seq_group_metadata.is_prompt
@@ -96,6 +100,10 @@ class CPUModelRunner:
100
# is always the first token in the sequence.
101
input_positions.extend(list(range(computed_len, prompt_len)))
102
103
+ if seq_group_metadata.multi_modal_data:
104
+ multi_modal_input_list.append(
105
+ seq_group_metadata.multi_modal_data.data)
106
+
107
# Compute the slot mapping.
108
block_table = seq_group_metadata.block_tables[seq_id]
109
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
@@ -118,6 +126,15 @@ class CPUModelRunner:
126
slot = block_number * self.block_size + block_offset
127
slot_mapping.append(slot)
128
129
+ if multi_modal_input_list:
130
+ assert self.vision_language_config, (
131
+ "Multi-modal inputs are only supported by "
132
+ "vision language models.")
133
+ multi_modal_input = torch.cat(multi_modal_input_list,
134
+ dim=0).to(self.device)
135
+ else:
136
+ multi_modal_input = None
137
+
138
num_prompt_tokens = len(input_tokens)
139
140
input_tokens = torch.tensor(input_tokens,
@@ -144,12 +161,8 @@ class CPUModelRunner:
161
slot_mapping=slot_mapping,
162
kv_cache_dtype=self.kv_cache_dtype,
163
)
147
- return (
148
- input_tokens,
149
- input_positions,
150
- attn_metadata,
151
- prompt_lens,
152
- )
164
+ return (input_tokens, input_positions, attn_metadata, prompt_lens,
165
+ multi_modal_input)
166
167
def _prepare_decode(
168
self,
@@ -336,14 +349,16 @@ class CPUModelRunner:
349
seq_group_metadata_list: List[SequenceGroupMetadata],
350
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
351
SamplingMetadata]:
352
+ multi_modal_input = None
353
if self.is_driver_worker:
354
# NOTE: We assume that all sequences in the group are all prompts or
355
# all decodes.
356
is_prompt = seq_group_metadata_list[0].is_prompt
357
# Prepare input tensors.
358
if is_prompt:
345
- (input_tokens, input_positions, attn_metadata,
346
- prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
359
+ (input_tokens, input_positions, attn_metadata, prompt_lens,
360
+ multi_modal_input
361
+ ) = self._prepare_prompt(seq_group_metadata_list)
362
else:
363
(input_tokens, input_positions,
364
attn_metadata) = self._prepare_decode(seq_group_metadata_list)
@@ -376,12 +391,8 @@ class CPUModelRunner:
391
perform_sampling=False,
392
)
393
379
- return (
380
- input_tokens,
381
- input_positions,
382
- attn_metadata,
383
- sampling_metadata,
384
- )
394
+ return (input_tokens, input_positions, attn_metadata,
395
+ sampling_metadata, multi_modal_input)
396
397
@torch.inference_mode()
398
def execute_model(
@@ -389,7 +400,8 @@ class CPUModelRunner:
400
seq_group_metadata_list: List[SequenceGroupMetadata],
401
kv_caches: List[torch.Tensor],
402
) -> Optional[SamplerOutput]:
392
- (input_tokens, input_positions, attn_metadata, sampling_metadata
403
+ (input_tokens, input_positions, attn_metadata, sampling_metadata,
404
+ multi_modal_input
405
) = self.prepare_input_tensors(seq_group_metadata_list)
406
407
model_executable = self.model
@@ -399,6 +411,8 @@ class CPUModelRunner:
411
"kv_caches": kv_caches,
412
"attn_metadata": attn_metadata,
413
}
414
+ if self.vision_language_config:
415
+ execute_model_kwargs.update({"image_input": multi_modal_input})
416
417
hidden_states = model_executable(**execute_model_kwargs)
418
vllm/worker/cpu_worker.py
+15
-9
@@ -6,7 +6,8 @@ import torch.distributed
6
7
from vllm.attention import get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
- ModelConfig, ParallelConfig, SchedulerConfig)
9
+ ModelConfig, ParallelConfig, SchedulerConfig,
10
+ VisionLanguageConfig)
11
from vllm.distributed import (broadcast_tensor_dict,
12
ensure_model_parallel_initialized,
13
init_distributed_environment)
@@ -122,6 +123,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
123
rank: int,
124
distributed_init_method: str,
125
lora_config: Optional[LoRAConfig] = None,
126
+ vision_language_config: Optional[VisionLanguageConfig] = None,
127
kv_cache_dtype: Optional[str] = "auto",
128
is_driver_worker: bool = False,
129
) -> None:
@@ -135,21 +137,25 @@ class CPUWorker(LoraNotSupportedWorkerBase):
137
self.rank = rank
138
self.distributed_init_method = distributed_init_method
139
self.lora_config = lora_config
140
+ self.vision_language_config = vision_language_config
141
self.is_driver_worker = is_driver_worker
142
if self.is_driver_worker:
143
assert self.rank == 0, "The driver worker must have rank 0."
144
+
145
if self.model_config.trust_remote_code:
146
# note: lazy import to avoid importing torch before initializing
147
from vllm.utils import init_cached_hf_modules
148
init_cached_hf_modules()
145
- self.model_runner = CPUModelRunner(model_config,
146
- parallel_config,
147
- scheduler_config,
148
- device_config,
149
- load_config=self.load_config,
150
- lora_config=self.lora_config,
151
- kv_cache_dtype=kv_cache_dtype,
152
- is_driver_worker=is_driver_worker)
149
+ self.model_runner = CPUModelRunner(
150
+ model_config,
151
+ parallel_config,
152
+ scheduler_config,
153
+ device_config,
154
+ load_config=self.load_config,
155
+ lora_config=self.lora_config,
156
+ vision_language_config=self.vision_language_config,
157
+ kv_cache_dtype=kv_cache_dtype,
158
+ is_driver_worker=is_driver_worker)
159
# Uninitialized cache engine. Will be initialized by
160
# initialize_cache.
161
self.cache_engine: CPUCacheEngine