A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix] Fix sync_and_slice_intermediate_tensors (#21537)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Jul 25, 2025 at 17:07 UTC
cea96a015678c86789fa86a719ce7d6d176d78fd
1 file changed
+1
-1
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -1270,7 +1270,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1270
if sync_self:
1271
assert intermediate_tensors is not None
1272
for k, v in intermediate_tensors.items():
1273
- is_scattered = "residual" and is_residual_scattered
1273
+ is_scattered = k == "residual" and is_residual_scattered
1274
copy_len = num_tokens // tp if is_scattered else \
1275
num_tokens
1276
self.intermediate_tensors[k][:copy_len].copy_(