@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 62467a834a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Avoid unnecessary multi-modal input data copy when len(batch) == 1 (#12722)

Signed-off-by: imkero <kerorek@outlook.com>

Kero Liang committed Feb 4, 2025 at 21:03 UTC 62467a834a94566e2d81a276817a20174b474151
1 file changed +16
vllm/multimodal/inputs.py
+16
@@ -212,6 +212,11 @@ class MultiModalBatchedField(BaseMultiModalField):
212
213 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
214 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
215 + if len(batch) == 1:
216 + # An optimization when `batch` contains only one tensor:
217 + # - produce exactly same result as `torch.stack(batch)`
218 + # - will achieve zero-copy if the tensor is contiguous
219 + return batch[0].unsqueeze(0).contiguous()
220 first_shape = batch[0].shape
221 if all(elem.shape == first_shape for elem in batch):
222 return torch.stack(batch)
@@ -235,6 +240,11 @@ class MultiModalFlatField(BaseMultiModalField):
240
241 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
242 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
243 + if len(batch) == 1:
244 + # An optimization when `batch` contains only one tensor:
245 + # - produce exactly same result as `torch.concat(batch)`
246 + # - will achieve zero-copy if the tensor is contiguous
247 + return batch[0].contiguous()
248 first_shape = batch[0].shape
249 if all(elem.shape[1:] == first_shape[1:] for elem in batch):
250 return torch.concat(batch)
@@ -407,6 +417,12 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
417 return stacked
418
419 tensors_ = cast(list[torch.Tensor], stacked)
420 + if len(tensors_) == 1:
421 + # An optimization when `tensors_` contains only one tensor:
422 + # - produce exactly same result as `torch.stack(tensors_)`
423 + # - will achieve zero-copy if the tensor is contiguous
424 + return tensors_[0].unsqueeze(0).contiguous()
425 +
426 if any(t.shape != tensors_[0].shape for t in tensors_):
427 # The tensors have incompatible shapes and can't be stacked.
428 return tensors_