@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4cfe5d2bca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] `multi_modal_kwargs` broadcast for CPU tensor parallel (#10541)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 23, 2024 at 13:25 UTC 4cfe5d2bcafe1f47d1df046e6788ebbe038eaf3f
2 files changed +2
vllm/worker/cpu_enc_dec_model_runner.py
+1
@@ -35,6 +35,7 @@ class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata):
35 "input_positions": self.input_positions,
36 "encoder_input_tokens": self.encoder_input_tokens,
37 "encoder_input_positions": self.encoder_input_positions,
38 + "multi_modal_kwargs": self.multi_modal_kwargs,
39 }
40 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
41 _add_sampling_metadata_broadcastable_dict(tensor_dict,
vllm/worker/cpu_model_runner.py
+1
@@ -83,6 +83,7 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
83 tensor_dict = {
84 "input_tokens": self.input_tokens,
85 "input_positions": self.input_positions,
86 + "multi_modal_kwargs": self.multi_modal_kwargs,
87 }
88 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
89 _add_sampling_metadata_broadcastable_dict(tensor_dict,