@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e09ce759aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[TPU] Remove multi-modal args in TPU backend (#6504)

Woosuk Kwon committed Jul 17, 2024 at 04:02 UTC e09ce759aa8d4fa41304df59b7e888fe12724d58
1 file changed +6 -40
vllm/worker/tpu_model_runner.py
+6 -40
@@ -1,5 +1,5 @@
1 import time
2 -from typing import List, Mapping, Optional, Tuple
2 +from typing import List, Optional, Tuple
3
4 import numpy as np
5 import torch
@@ -12,8 +12,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
12 from vllm.logger import init_logger
13 from vllm.model_executor.model_loader import get_model
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
16 - MultiModalInputs)
15 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
16 SamplerOutput, SequenceGroupMetadata,
17 SequenceOutput)
@@ -68,10 +66,6 @@ class TPUModelRunner:
66 False,
67 )
68
71 - # Multi-modal data support
72 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
73 - .create_input_mapper(self.model_config)
74 -
69 def load_model(self) -> None:
70 self.device = self.device_config.device
71
@@ -154,7 +148,7 @@ class TPUModelRunner:
148 # Dummy run.
149 num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
150 self.model(token_ids, position_ids, kv_caches, attn_metadata,
157 - input_lens, None, t, p, num_samples)
151 + input_lens, t, p, num_samples)
152
153 def warmup_model(
154 self,
@@ -199,14 +193,12 @@ class TPUModelRunner:
193 def _prepare_prompt(
194 self,
195 seq_group_metadata_list: List[SequenceGroupMetadata],
202 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor,
203 - Mapping[str, BatchedTensors]]:
196 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]:
197 assert len(seq_group_metadata_list) > 0
198 input_tokens: List[List[int]] = []
199 input_positions: List[List[int]] = []
200 prompt_lens: List[int] = []
201 slot_mapping: List[List[int]] = []
209 - multi_modal_inputs_list: List[MultiModalInputs] = []
202
203 for seq_group_metadata in seq_group_metadata_list:
204 assert seq_group_metadata.is_prompt
@@ -232,11 +224,6 @@ class TPUModelRunner:
224 slot = block_number * self.block_size + block_offset
225 slot_mapping[-1].append(slot)
226
235 - mm_data = seq_group_metadata.multi_modal_data
236 - if mm_data:
237 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
238 - multi_modal_inputs_list.append(mm_kwargs)
239 -
227 assert len(prompt_lens) > 0
228 num_prefills = len(prompt_lens)
229 num_prefill_tokens = sum(prompt_lens)
@@ -274,24 +261,17 @@ class TPUModelRunner:
261 block_tables=None,
262 context_lens=None,
263 )
277 -
278 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
279 - device=self.device)
280 -
281 - return (input_tokens, input_positions, attn_metadata, prompt_lens,
282 - multi_modal_kwargs)
264 + return input_tokens, input_positions, attn_metadata, prompt_lens
265
266 def _prepare_decode(
267 self,
268 seq_group_metadata_list: List[SequenceGroupMetadata],
287 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor,
288 - Mapping[str, BatchedTensors]]:
269 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]:
270 assert len(seq_group_metadata_list) > 0
271 input_tokens: List[List[int]] = []
272 input_positions: List[List[int]] = []
273 slot_mapping: List[List[int]] = []
274 context_lens: List[int] = []
294 - multi_modal_inputs_list: List[MultiModalInputs] = []
275
276 batch_idx = 0
277 for seq_group_metadata in seq_group_metadata_list:
@@ -317,11 +297,6 @@ class TPUModelRunner:
297 slot = block_number * self.block_size + block_offset
298 slot_mapping.append([slot])
299
320 - mm_data = seq_group_metadata.multi_modal_data
321 - if mm_data:
322 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
323 - multi_modal_inputs_list.append(mm_kwargs)
324 -
300 batch_size = _get_padded_batch_size(batch_idx)
301 num_paddings = batch_size - batch_idx
302 input_tokens = input_tokens + [[0]] * num_paddings
@@ -355,12 +330,7 @@ class TPUModelRunner:
330 block_tables=block_tables,
331 context_lens=context_lens,
332 )
358 -
359 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
360 - device=self.device)
361 -
362 - return (input_tokens, input_positions, attn_metadata, input_lens,
363 - multi_modal_kwargs)
333 + return input_tokens, input_positions, attn_metadata, input_lens
334
335 def _prepare_sample(
336 self,
@@ -513,7 +483,6 @@ class ModelWrapper(nn.Module):
483 kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
484 attn_metadata: AttentionMetadata,
485 input_lens: torch.Tensor,
516 - multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]],
486 t: torch.Tensor,
487 p: torch.Tensor,
488 num_samples: int,
@@ -527,8 +496,6 @@ class ModelWrapper(nn.Module):
496 memory profiling at initialization.
497 attn_metadata: The Pallas attention metadata.
498 input_lens: The actual input lengths of shape [batch_size].
530 - multi_modal_kwargs: Keyword arguments from multi-modal data to
531 - pass to the model.
499 t: The sampling temperature of shape [batch_size].
500 p: The top-p probability of shape [batch_size].
501 """
@@ -573,7 +540,6 @@ class ModelWrapper(nn.Module):
540 position_ids,
541 kv_caches,
542 attn_metadata,
576 - **(multi_modal_kwargs or {}),
543 )
544 hidden_states = hidden_states.flatten(0, 1)
545 logits = self.model.compute_logits(hidden_states, sampling_metadata)