A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Use input_ids as input for text-only models (#11032)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 11, 2024 at 10:49 UTC
d643c2aba1cd5421200f3a3bad1813dd067233b4
1 file changed
+47
-21
vllm/v1/worker/gpu_model_runner.py
+47
-21
@@ -61,6 +61,7 @@ class GPUModelRunner:
61
self.kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
62
cache_config.cache_dtype]
63
64
+ self.is_multimodal_model = model_config.is_multimodal_model
65
self.sliding_window = model_config.get_sliding_window()
66
self.block_size = cache_config.block_size
67
self.max_model_len = model_config.max_model_len
@@ -103,6 +104,11 @@ class GPUModelRunner:
104
# The batch sizes in the config are in descending order.
105
self.cudagraph_batch_sizes = list(
106
reversed(self.vllm_config.compilation_config.capture_sizes))
107
+
108
+ # Persistent buffers for CUDA graphs.
109
+ self.input_ids = torch.zeros(self.max_num_tokens,
110
+ dtype=torch.int32,
111
+ device=self.device)
112
self.positions = torch.zeros(self.max_num_tokens,
113
dtype=torch.int64,
114
device=self.device)
@@ -310,7 +316,8 @@ class GPUModelRunner:
316
seq_start_loc_np[0] = 0
317
np.cumsum(seq_lens, out=seq_start_loc_np[1:])
318
313
- input_ids = input_ids.to(self.device, non_blocking=True)
319
+ self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
320
+ non_blocking=True)
321
self.positions[:total_num_scheduled_tokens].copy_(positions,
322
non_blocking=True)
323
query_start_loc = query_start_loc.to(self.device, non_blocking=True)
@@ -331,7 +338,7 @@ class GPUModelRunner:
338
# token from the partial request.
339
# TODO: Support prompt logprobs.
340
logits_indices = query_start_loc[1:] - 1
334
- return input_ids, attn_metadata, logits_indices
341
+ return attn_metadata, logits_indices
342
343
def _prepare_sampling(
344
self,
@@ -427,13 +434,15 @@ class GPUModelRunner:
434
) -> ModelRunnerOutput:
435
self._update_states(scheduler_output)
436
430
- # Run the encoder.
431
- self._execute_encoder(scheduler_output)
432
- encoder_outputs = self._gather_encoder_outputs(scheduler_output)
437
+ if self.is_multimodal_model:
438
+ # Run the multimodal encoder if any.
439
+ self._execute_encoder(scheduler_output)
440
+ encoder_outputs = self._gather_encoder_outputs(scheduler_output)
441
+ else:
442
+ encoder_outputs = []
443
444
# Prepare the decoder inputs.
435
- input_ids, attn_metadata, logits_indices = self._prepare_inputs(
436
- scheduler_output)
445
+ attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
446
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
447
if (self.use_cuda_graph
448
and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
@@ -444,29 +453,39 @@ class GPUModelRunner:
453
else:
454
# Eager mode.
455
num_input_tokens = num_scheduled_tokens
447
-
456
attn_metadata.num_input_tokens = num_input_tokens
457
450
- # Get the inputs embeds.
451
- if encoder_outputs:
452
- inputs_embeds = self.model.get_input_embeddings(
453
- input_ids, encoder_outputs)
458
+ if self.is_multimodal_model:
459
+ # NOTE(woosuk): To unify token ids and soft tokens (vision
460
+ # embeddings), we always use embeddings (rather than token ids)
461
+ # as input to the multimodal model, even when the input is text.
462
+ input_ids = self.input_ids[:num_scheduled_tokens]
463
+ if encoder_outputs:
464
+ inputs_embeds = self.model.get_input_embeddings(
465
+ input_ids, encoder_outputs)
466
+ else:
467
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
468
+ # TODO(woosuk): Avoid the copy. Optimize.
469
+ self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
470
+ inputs_embeds = self.inputs_embeds[:num_input_tokens]
471
+ input_ids = None
472
else:
455
- inputs_embeds = self.model.get_input_embeddings(input_ids)
456
- # NOTE(woosuk): To unify token ids and soft tokens (vision embeddings),
457
- # always use embeddings (rather than token ids) as input to the model.
458
- # TODO(woosuk): Avoid the copy. Optimize.
459
- self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
473
+ # For text-only models, we use token ids as input.
474
+ # While it is possible to use embeddings as input just like the
475
+ # multimodal models, it is not desirable for performance since
476
+ # then the embedding layer is not included in the CUDA graph.
477
+ input_ids = self.input_ids[:num_input_tokens]
478
+ inputs_embeds = None
479
480
# Run the decoder.
481
# Use persistent buffers for CUDA graphs.
482
with set_forward_context(attn_metadata, self.vllm_config):
483
hidden_states = self.model(
465
- input_ids=None,
484
+ input_ids=input_ids,
485
positions=self.positions[:num_input_tokens],
486
kv_caches=self.kv_caches,
487
attn_metadata=None,
469
- inputs_embeds=self.inputs_embeds[:num_input_tokens],
488
+ inputs_embeds=inputs_embeds,
489
)
490
hidden_states = hidden_states[:num_scheduled_tokens]
491
hidden_states = hidden_states[logits_indices]
@@ -534,13 +553,20 @@ class GPUModelRunner:
553
num_tokens: int,
554
kv_caches: List[torch.Tensor],
555
) -> torch.Tensor:
556
+ if self.is_multimodal_model:
557
+ input_ids = None
558
+ inputs_embeds = self.inputs_embeds[:num_tokens]
559
+ else:
560
+ input_ids = self.input_ids[:num_tokens]
561
+ inputs_embeds = None
562
with set_forward_context(None, self.vllm_config):
563
hidden_states = model(
539
- input_ids=None,
564
+ input_ids=input_ids,
565
positions=self.positions[:num_tokens],
566
kv_caches=kv_caches,
567
attn_metadata=None,
543
- inputs_embeds=self.inputs_embeds[:num_tokens])
568
+ inputs_embeds=inputs_embeds,
569
+ )
570
return hidden_states
571
572
def profile_run(self) -> None: