@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 296cdf8ac7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] Add vision language model support to CPU backend (#3968)

Isotr0py committed Apr 22, 2024 at 15:44 UTC 296cdf8ac7853b671d10f3df094288ea4f35bb38
3 files changed +53 -32
vllm/executor/cpu_executor.py
+1
@@ -45,6 +45,7 @@ class CPUExecutor(ExecutorBase):
45 rank=0,
46 distributed_init_method=distributed_init_method,
47 lora_config=self.lora_config,
48 + vision_language_config=self.vision_language_config,
49 kv_cache_dtype=self.cache_config.cache_dtype,
50 is_driver_worker=True,
51 )
vllm/worker/cpu_model_runner.py
+37 -23
@@ -5,7 +5,7 @@ from torch import nn
5
6 from vllm.attention import AttentionMetadata, get_attn_backend
7 from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig)
8 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9 from vllm.distributed import broadcast_tensor_dict
10 from vllm.logger import init_logger
11 from vllm.model_executor import SamplingMetadata
@@ -29,6 +29,7 @@ class CPUModelRunner:
29 device_config: DeviceConfig,
30 load_config: LoadConfig,
31 lora_config: Optional[LoRAConfig],
32 + vision_language_config: Optional[VisionLanguageConfig],
33 kv_cache_dtype: Optional[str] = "auto",
34 is_driver_worker: bool = False,
35 *args,
@@ -38,6 +39,7 @@ class CPUModelRunner:
39 self.parallel_config = parallel_config
40 self.scheduler_config = scheduler_config
41 self.lora_config = lora_config
42 + self.vision_language_config = vision_language_config
43 self.load_config = load_config
44 self.is_driver_worker = is_driver_worker
45
@@ -59,13 +61,14 @@ class CPUModelRunner:
61 self.block_size: int # Set after initial profiling.
62
63 def load_model(self) -> None:
62 - self.model = get_model(model_config=self.model_config,
63 - load_config=self.load_config,
64 - device_config=self.device_config,
65 - vision_language_config=None,
66 - lora_config=self.lora_config,
67 - parallel_config=self.parallel_config,
68 - scheduler_config=self.scheduler_config)
64 + self.model = get_model(
65 + model_config=self.model_config,
66 + load_config=self.load_config,
67 + device_config=self.device_config,
68 + vision_language_config=self.vision_language_config,
69 + lora_config=self.lora_config,
70 + parallel_config=self.parallel_config,
71 + scheduler_config=self.scheduler_config)
72
73 def _prepare_prompt(
74 self,
@@ -76,6 +79,7 @@ class CPUModelRunner:
79 input_positions: List[int] = []
80 slot_mapping: List[int] = []
81 prompt_lens: List[int] = []
82 + multi_modal_input_list: List[torch.Tensor] = []
83
84 for seq_group_metadata in seq_group_metadata_list:
85 assert seq_group_metadata.is_prompt
@@ -96,6 +100,10 @@ class CPUModelRunner:
100 # is always the first token in the sequence.
101 input_positions.extend(list(range(computed_len, prompt_len)))
102
103 + if seq_group_metadata.multi_modal_data:
104 + multi_modal_input_list.append(
105 + seq_group_metadata.multi_modal_data.data)
106 +
107 # Compute the slot mapping.
108 block_table = seq_group_metadata.block_tables[seq_id]
109 # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
@@ -118,6 +126,15 @@ class CPUModelRunner:
126 slot = block_number * self.block_size + block_offset
127 slot_mapping.append(slot)
128
129 + if multi_modal_input_list:
130 + assert self.vision_language_config, (
131 + "Multi-modal inputs are only supported by "
132 + "vision language models.")
133 + multi_modal_input = torch.cat(multi_modal_input_list,
134 + dim=0).to(self.device)
135 + else:
136 + multi_modal_input = None
137 +
138 num_prompt_tokens = len(input_tokens)
139
140 input_tokens = torch.tensor(input_tokens,
@@ -144,12 +161,8 @@ class CPUModelRunner:
161 slot_mapping=slot_mapping,
162 kv_cache_dtype=self.kv_cache_dtype,
163 )
147 - return (
148 - input_tokens,
149 - input_positions,
150 - attn_metadata,
151 - prompt_lens,
152 - )
164 + return (input_tokens, input_positions, attn_metadata, prompt_lens,
165 + multi_modal_input)
166
167 def _prepare_decode(
168 self,
@@ -336,14 +349,16 @@ class CPUModelRunner:
349 seq_group_metadata_list: List[SequenceGroupMetadata],
350 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
351 SamplingMetadata]:
352 + multi_modal_input = None
353 if self.is_driver_worker:
354 # NOTE: We assume that all sequences in the group are all prompts or
355 # all decodes.
356 is_prompt = seq_group_metadata_list[0].is_prompt
357 # Prepare input tensors.
358 if is_prompt:
345 - (input_tokens, input_positions, attn_metadata,
346 - prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
359 + (input_tokens, input_positions, attn_metadata, prompt_lens,
360 + multi_modal_input
361 + ) = self._prepare_prompt(seq_group_metadata_list)
362 else:
363 (input_tokens, input_positions,
364 attn_metadata) = self._prepare_decode(seq_group_metadata_list)
@@ -376,12 +391,8 @@ class CPUModelRunner:
391 perform_sampling=False,
392 )
393
379 - return (
380 - input_tokens,
381 - input_positions,
382 - attn_metadata,
383 - sampling_metadata,
384 - )
394 + return (input_tokens, input_positions, attn_metadata,
395 + sampling_metadata, multi_modal_input)
396
397 @torch.inference_mode()
398 def execute_model(
@@ -389,7 +400,8 @@ class CPUModelRunner:
400 seq_group_metadata_list: List[SequenceGroupMetadata],
401 kv_caches: List[torch.Tensor],
402 ) -> Optional[SamplerOutput]:
392 - (input_tokens, input_positions, attn_metadata, sampling_metadata
403 + (input_tokens, input_positions, attn_metadata, sampling_metadata,
404 + multi_modal_input
405 ) = self.prepare_input_tensors(seq_group_metadata_list)
406
407 model_executable = self.model
@@ -399,6 +411,8 @@ class CPUModelRunner:
411 "kv_caches": kv_caches,
412 "attn_metadata": attn_metadata,
413 }
414 + if self.vision_language_config:
415 + execute_model_kwargs.update({"image_input": multi_modal_input})
416
417 hidden_states = model_executable(**execute_model_kwargs)
418
vllm/worker/cpu_worker.py
+15 -9
@@ -6,7 +6,8 @@ import torch.distributed
6
7 from vllm.attention import get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 - ModelConfig, ParallelConfig, SchedulerConfig)
9 + ModelConfig, ParallelConfig, SchedulerConfig,
10 + VisionLanguageConfig)
11 from vllm.distributed import (broadcast_tensor_dict,
12 ensure_model_parallel_initialized,
13 init_distributed_environment)
@@ -122,6 +123,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
123 rank: int,
124 distributed_init_method: str,
125 lora_config: Optional[LoRAConfig] = None,
126 + vision_language_config: Optional[VisionLanguageConfig] = None,
127 kv_cache_dtype: Optional[str] = "auto",
128 is_driver_worker: bool = False,
129 ) -> None:
@@ -135,21 +137,25 @@ class CPUWorker(LoraNotSupportedWorkerBase):
137 self.rank = rank
138 self.distributed_init_method = distributed_init_method
139 self.lora_config = lora_config
140 + self.vision_language_config = vision_language_config
141 self.is_driver_worker = is_driver_worker
142 if self.is_driver_worker:
143 assert self.rank == 0, "The driver worker must have rank 0."
144 +
145 if self.model_config.trust_remote_code:
146 # note: lazy import to avoid importing torch before initializing
147 from vllm.utils import init_cached_hf_modules
148 init_cached_hf_modules()
145 - self.model_runner = CPUModelRunner(model_config,
146 - parallel_config,
147 - scheduler_config,
148 - device_config,
149 - load_config=self.load_config,
150 - lora_config=self.lora_config,
151 - kv_cache_dtype=kv_cache_dtype,
152 - is_driver_worker=is_driver_worker)
149 + self.model_runner = CPUModelRunner(
150 + model_config,
151 + parallel_config,
152 + scheduler_config,
153 + device_config,
154 + load_config=self.load_config,
155 + lora_config=self.lora_config,
156 + vision_language_config=self.vision_language_config,
157 + kv_cache_dtype=kv_cache_dtype,
158 + is_driver_worker=is_driver_worker)
159 # Uninitialized cache engine. Will be initialized by
160 # initialize_cache.
161 self.cache_engine: CPUCacheEngine