A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][PP] Fix memory profiling in PP (#13315)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Feb 14, 2025 at 20:17 UTC
0c73026844e8a2d3ff017bf0e802b34bf8263aa0
1 file changed
+6
-5
vllm/v1/worker/gpu_model_runner.py
+6
-5
@@ -1158,11 +1158,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1158
# Trigger compilation for general shape.
1159
hidden_states = self._dummy_run(self.max_num_tokens,
1160
dummy_kv_caches)
1161
- if not get_pp_group().is_last_rank:
1162
- return hidden_states
1163
- hidden_states = hidden_states[logit_indices]
1164
- logits = self.model.compute_logits(hidden_states, None)
1165
- # TODO(woosuk): Consider the memory usage of the sampler.
1161
+ if get_pp_group().is_last_rank:
1162
+ hidden_states = hidden_states[logit_indices]
1163
+ logits = self.model.compute_logits(hidden_states, None)
1164
+ # TODO(woosuk): Consider the memory usage of the sampler.
1165
+ else:
1166
+ logits = None
1167
torch.cuda.synchronize()
1168
del hidden_states, logits
1169
self.encoder_cache.clear()