A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Avoid unnecessary multi-modal input data copy when len(batch) == 1 (#12722)
Signed-off-by: imkero <kerorek@outlook.com>
Kero Liang committed
Feb 4, 2025 at 21:03 UTC
62467a834a94566e2d81a276817a20174b474151
1 file changed
+16
vllm/multimodal/inputs.py
+16
@@ -212,6 +212,11 @@ class MultiModalBatchedField(BaseMultiModalField):
212
213
def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
214
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
215
+ if len(batch) == 1:
216
+ # An optimization when `batch` contains only one tensor:
217
+ # - produce exactly same result as `torch.stack(batch)`
218
+ # - will achieve zero-copy if the tensor is contiguous
219
+ return batch[0].unsqueeze(0).contiguous()
220
first_shape = batch[0].shape
221
if all(elem.shape == first_shape for elem in batch):
222
return torch.stack(batch)
@@ -235,6 +240,11 @@ class MultiModalFlatField(BaseMultiModalField):
240
241
def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
242
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
243
+ if len(batch) == 1:
244
+ # An optimization when `batch` contains only one tensor:
245
+ # - produce exactly same result as `torch.concat(batch)`
246
+ # - will achieve zero-copy if the tensor is contiguous
247
+ return batch[0].contiguous()
248
first_shape = batch[0].shape
249
if all(elem.shape[1:] == first_shape[1:] for elem in batch):
250
return torch.concat(batch)
@@ -407,6 +417,12 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
417
return stacked
418
419
tensors_ = cast(list[torch.Tensor], stacked)
420
+ if len(tensors_) == 1:
421
+ # An optimization when `tensors_` contains only one tensor:
422
+ # - produce exactly same result as `torch.stack(tensors_)`
423
+ # - will achieve zero-copy if the tensor is contiguous
424
+ return tensors_[0].unsqueeze(0).contiguous()
425
+
426
if any(t.shape != tensors_[0].shape for t in tensors_):
427
# The tensors have incompatible shapes and can't be stacked.
428
return tensors_