A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Fix VLM broadcast tests `tensor_parallel_size` passing (#10161)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 9, 2024 at 12:02 UTC
47672f38b58581cf2b7c33201e6ae01639c5ff51
2 files changed
+2
tests/models/decoder_only/vision_language/test_models.py
+1
@@ -34,6 +34,7 @@ COMMON_BROADCAST_SETTINGS = {
34
"dtype": "half",
35
"max_tokens": 5,
36
"tensor_parallel_size": 2,
37
+ "model_kwargs": {"device_map": "auto"},
38
"image_size_factors": [(.25, 0.5, 1.0)],
39
"distributed_executor_backend": (
40
"ray",
tests/models/decoder_only/vision_language/vlm_utils/types.py
+1
@@ -158,6 +158,7 @@ class VLMTestInfo(NamedTuple):
158
"max_model_len": self.max_model_len,
159
"max_num_seqs": self.max_num_seqs,
160
"task": self.task,
161
+ "tensor_parallel_size": self.tensor_parallel_size,
162
"hf_output_post_proc": self.hf_output_post_proc,
163
"vllm_output_post_proc": self.vllm_output_post_proc,
164
"auto_cls": self.auto_cls,