A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Remove num_tokens_across_dp (#14302)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Mar 5, 2025 at 18:55 UTC
a7ea35aa677272d36e01388f26801d673cbb680e
1 file changed
+1
-2
vllm/forward_context.py
+1
-2
@@ -27,7 +27,6 @@ batchsize_forward_time: defaultdict = defaultdict(list)
27
28
@dataclass
29
class DPMetadata:
30
- num_tokens_across_dp: list[int]
30
cu_tokens_across_dp_cpu: torch.Tensor
31
32
@@ -89,7 +88,7 @@ def set_forward_context(attn_metadata: Any,
88
from vllm.distributed.parallel_state import get_dp_group
89
dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group)
90
cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_tensor, dim=0)
92
- dp_metadata = DPMetadata(num_tokens_across_dp, cu_tokens_across_dp_cpu)
91
+ dp_metadata = DPMetadata(cu_tokens_across_dp_cpu)
92
93
global _forward_context
94
prev_context = _forward_context