@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d643c2aba1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Use input_ids as input for text-only models (#11032)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 11, 2024 at 10:49 UTC d643c2aba1cd5421200f3a3bad1813dd067233b4
1 file changed +47 -21
vllm/v1/worker/gpu_model_runner.py
+47 -21
@@ -61,6 +61,7 @@ class GPUModelRunner:
61 self.kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
62 cache_config.cache_dtype]
63
64 + self.is_multimodal_model = model_config.is_multimodal_model
65 self.sliding_window = model_config.get_sliding_window()
66 self.block_size = cache_config.block_size
67 self.max_model_len = model_config.max_model_len
@@ -103,6 +104,11 @@ class GPUModelRunner:
104 # The batch sizes in the config are in descending order.
105 self.cudagraph_batch_sizes = list(
106 reversed(self.vllm_config.compilation_config.capture_sizes))
107 +
108 + # Persistent buffers for CUDA graphs.
109 + self.input_ids = torch.zeros(self.max_num_tokens,
110 + dtype=torch.int32,
111 + device=self.device)
112 self.positions = torch.zeros(self.max_num_tokens,
113 dtype=torch.int64,
114 device=self.device)
@@ -310,7 +316,8 @@ class GPUModelRunner:
316 seq_start_loc_np[0] = 0
317 np.cumsum(seq_lens, out=seq_start_loc_np[1:])
318
313 - input_ids = input_ids.to(self.device, non_blocking=True)
319 + self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
320 + non_blocking=True)
321 self.positions[:total_num_scheduled_tokens].copy_(positions,
322 non_blocking=True)
323 query_start_loc = query_start_loc.to(self.device, non_blocking=True)
@@ -331,7 +338,7 @@ class GPUModelRunner:
338 # token from the partial request.
339 # TODO: Support prompt logprobs.
340 logits_indices = query_start_loc[1:] - 1
334 - return input_ids, attn_metadata, logits_indices
341 + return attn_metadata, logits_indices
342
343 def _prepare_sampling(
344 self,
@@ -427,13 +434,15 @@ class GPUModelRunner:
434 ) -> ModelRunnerOutput:
435 self._update_states(scheduler_output)
436
430 - # Run the encoder.
431 - self._execute_encoder(scheduler_output)
432 - encoder_outputs = self._gather_encoder_outputs(scheduler_output)
437 + if self.is_multimodal_model:
438 + # Run the multimodal encoder if any.
439 + self._execute_encoder(scheduler_output)
440 + encoder_outputs = self._gather_encoder_outputs(scheduler_output)
441 + else:
442 + encoder_outputs = []
443
444 # Prepare the decoder inputs.
435 - input_ids, attn_metadata, logits_indices = self._prepare_inputs(
436 - scheduler_output)
445 + attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
446 num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
447 if (self.use_cuda_graph
448 and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
@@ -444,29 +453,39 @@ class GPUModelRunner:
453 else:
454 # Eager mode.
455 num_input_tokens = num_scheduled_tokens
447 -
456 attn_metadata.num_input_tokens = num_input_tokens
457
450 - # Get the inputs embeds.
451 - if encoder_outputs:
452 - inputs_embeds = self.model.get_input_embeddings(
453 - input_ids, encoder_outputs)
458 + if self.is_multimodal_model:
459 + # NOTE(woosuk): To unify token ids and soft tokens (vision
460 + # embeddings), we always use embeddings (rather than token ids)
461 + # as input to the multimodal model, even when the input is text.
462 + input_ids = self.input_ids[:num_scheduled_tokens]
463 + if encoder_outputs:
464 + inputs_embeds = self.model.get_input_embeddings(
465 + input_ids, encoder_outputs)
466 + else:
467 + inputs_embeds = self.model.get_input_embeddings(input_ids)
468 + # TODO(woosuk): Avoid the copy. Optimize.
469 + self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
470 + inputs_embeds = self.inputs_embeds[:num_input_tokens]
471 + input_ids = None
472 else:
455 - inputs_embeds = self.model.get_input_embeddings(input_ids)
456 - # NOTE(woosuk): To unify token ids and soft tokens (vision embeddings),
457 - # always use embeddings (rather than token ids) as input to the model.
458 - # TODO(woosuk): Avoid the copy. Optimize.
459 - self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
473 + # For text-only models, we use token ids as input.
474 + # While it is possible to use embeddings as input just like the
475 + # multimodal models, it is not desirable for performance since
476 + # then the embedding layer is not included in the CUDA graph.
477 + input_ids = self.input_ids[:num_input_tokens]
478 + inputs_embeds = None
479
480 # Run the decoder.
481 # Use persistent buffers for CUDA graphs.
482 with set_forward_context(attn_metadata, self.vllm_config):
483 hidden_states = self.model(
465 - input_ids=None,
484 + input_ids=input_ids,
485 positions=self.positions[:num_input_tokens],
486 kv_caches=self.kv_caches,
487 attn_metadata=None,
469 - inputs_embeds=self.inputs_embeds[:num_input_tokens],
488 + inputs_embeds=inputs_embeds,
489 )
490 hidden_states = hidden_states[:num_scheduled_tokens]
491 hidden_states = hidden_states[logits_indices]
@@ -534,13 +553,20 @@ class GPUModelRunner:
553 num_tokens: int,
554 kv_caches: List[torch.Tensor],
555 ) -> torch.Tensor:
556 + if self.is_multimodal_model:
557 + input_ids = None
558 + inputs_embeds = self.inputs_embeds[:num_tokens]
559 + else:
560 + input_ids = self.input_ids[:num_tokens]
561 + inputs_embeds = None
562 with set_forward_context(None, self.vllm_config):
563 hidden_states = model(
539 - input_ids=None,
564 + input_ids=input_ids,
565 positions=self.positions[:num_tokens],
566 kv_caches=kv_caches,
567 attn_metadata=None,
543 - inputs_embeds=self.inputs_embeds[:num_tokens])
568 + inputs_embeds=inputs_embeds,
569 + )
570 return hidden_states
571
572 def profile_run(self) -> None: