A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] `multi_modal_kwargs` broadcast for CPU tensor parallel (#10541)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 23, 2024 at 13:25 UTC
4cfe5d2bcafe1f47d1df046e6788ebbe038eaf3f
2 files changed
+2
vllm/worker/cpu_enc_dec_model_runner.py
+1
@@ -35,6 +35,7 @@ class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata):
35
"input_positions": self.input_positions,
36
"encoder_input_tokens": self.encoder_input_tokens,
37
"encoder_input_positions": self.encoder_input_positions,
38
+ "multi_modal_kwargs": self.multi_modal_kwargs,
39
}
40
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
41
_add_sampling_metadata_broadcastable_dict(tensor_dict,
vllm/worker/cpu_model_runner.py
+1
@@ -83,6 +83,7 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
83
tensor_dict = {
84
"input_tokens": self.input_tokens,
85
"input_positions": self.input_positions,
86
+ "multi_modal_kwargs": self.multi_modal_kwargs,
87
}
88
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
89
_add_sampling_metadata_broadcastable_dict(tensor_dict,