@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 47672f38b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI/Build] Fix VLM broadcast tests `tensor_parallel_size` passing (#10161)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 9, 2024 at 12:02 UTC 47672f38b58581cf2b7c33201e6ae01639c5ff51
2 files changed +2
tests/models/decoder_only/vision_language/test_models.py
+1
@@ -34,6 +34,7 @@ COMMON_BROADCAST_SETTINGS = {
34 "dtype": "half",
35 "max_tokens": 5,
36 "tensor_parallel_size": 2,
37 + "model_kwargs": {"device_map": "auto"},
38 "image_size_factors": [(.25, 0.5, 1.0)],
39 "distributed_executor_backend": (
40 "ray",
tests/models/decoder_only/vision_language/vlm_utils/types.py
+1
@@ -158,6 +158,7 @@ class VLMTestInfo(NamedTuple):
158 "max_model_len": self.max_model_len,
159 "max_num_seqs": self.max_num_seqs,
160 "task": self.task,
161 + "tensor_parallel_size": self.tensor_parallel_size,
162 "hf_output_post_proc": self.hf_output_post_proc,
163 "vllm_output_post_proc": self.vllm_output_post_proc,
164 "auto_cls": self.auto_cls,