@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c73026844
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][PP] Fix memory profiling in PP (#13315)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Feb 14, 2025 at 20:17 UTC 0c73026844e8a2d3ff017bf0e802b34bf8263aa0
1 file changed +6 -5
vllm/v1/worker/gpu_model_runner.py
+6 -5
@@ -1158,11 +1158,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1158 # Trigger compilation for general shape.
1159 hidden_states = self._dummy_run(self.max_num_tokens,
1160 dummy_kv_caches)
1161 - if not get_pp_group().is_last_rank:
1162 - return hidden_states
1163 - hidden_states = hidden_states[logit_indices]
1164 - logits = self.model.compute_logits(hidden_states, None)
1165 - # TODO(woosuk): Consider the memory usage of the sampler.
1161 + if get_pp_group().is_last_rank:
1162 + hidden_states = hidden_states[logit_indices]
1163 + logits = self.model.compute_logits(hidden_states, None)
1164 + # TODO(woosuk): Consider the memory usage of the sampler.
1165 + else:
1166 + logits = None
1167 torch.cuda.synchronize()
1168 del hidden_states, logits
1169 self.encoder_cache.clear()