@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bef180f009

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Enable the remaining multimodal tests in V1 (#25307)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Sep 21, 2025 at 01:50 UTC bef180f00978186fcc84f7ca6328a6ae6c39676d

8 files changed +195 -214

tests/conftest.py

+56 -20

@@ -19,6 +19,7 @@ import socket

19 import tempfile

20 import threading

21 from collections.abc import Generator

22 +from contextlib import nullcontext

23 from enum import Enum

24 from typing import Any, Callable, Optional, TypedDict, TypeVar, Union, cast

@@ -45,14 +46,14 @@ from vllm.connections import global_http_connection

46 from vllm.distributed import (cleanup_dist_env_and_memory,

47 init_distributed_environment,

48 initialize_model_parallel)

48 -from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,

49 - to_enc_dec_tuple_list, zip_enc_dec_prompts)

49 +from vllm.inputs import TextPrompt

50 from vllm.logger import init_logger

51 from vllm.multimodal.utils import fetch_image

52 from vllm.outputs import RequestOutput

53 from vllm.sampling_params import BeamSearchParams

54 from vllm.sequence import Logprob

55 from vllm.transformers_utils.utils import maybe_model_redirect

56 +from vllm.utils import set_default_torch_num_threads

58 logger = init_logger(__name__)

@@ -306,6 +307,35 @@ class HfRunner:

307 is_cross_encoder: bool = False,

308 skip_tokenizer_init: bool = False,

309 auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,

310 + # Set this to avoid hanging issue

311 + default_torch_num_threads: Optional[int] = None,

312 + ) -> None:

313 + init_ctx = (nullcontext() if default_torch_num_threads is None else

314 + set_default_torch_num_threads(default_torch_num_threads))

315 +

316 + with init_ctx:

317 + self._init(

318 + model_name=model_name,

319 + dtype=dtype,

320 + model_kwargs=model_kwargs,

321 + trust_remote_code=trust_remote_code,

322 + is_sentence_transformer=is_sentence_transformer,

323 + is_cross_encoder=is_cross_encoder,

324 + skip_tokenizer_init=skip_tokenizer_init,

325 + auto_cls=auto_cls,

326 + )

327 +

328 + def _init(

329 + self,

330 + model_name: str,

331 + dtype: str = "auto",

332 + *,

333 + model_kwargs: Optional[dict[str, Any]] = None,

334 + trust_remote_code: bool = True,

335 + is_sentence_transformer: bool = False,

336 + is_cross_encoder: bool = False,

337 + skip_tokenizer_init: bool = False,

338 + auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,

339 ) -> None:

340 model_name = maybe_model_redirect(model_name)

341 self.model_name = model_name

@@ -714,26 +744,32 @@ class VllmRunner:

744 enable_chunked_prefill: Optional[bool] = False,

745 swap_space: int = 4,

746 enforce_eager: Optional[bool] = False,

747 + # Set this to avoid hanging issue

748 + default_torch_num_threads: Optional[int] = None,

749 **kwargs,

750 ) -> None:

719 - self.llm = LLM(

720 - model=model_name,

721 - runner=runner,

722 - convert=convert,

723 - tokenizer=tokenizer_name,

724 - tokenizer_mode=tokenizer_mode,

725 - trust_remote_code=trust_remote_code,

726 - dtype=dtype,

727 - seed=seed,

728 - swap_space=swap_space,

729 - enforce_eager=enforce_eager,

730 - disable_log_stats=disable_log_stats,

731 - tensor_parallel_size=tensor_parallel_size,

732 - max_model_len=max_model_len,

733 - block_size=block_size,

734 - enable_chunked_prefill=enable_chunked_prefill,

735 - **kwargs,

736 - )

751 + init_ctx = (nullcontext() if default_torch_num_threads is None else

752 + set_default_torch_num_threads(default_torch_num_threads))

753 +

754 + with init_ctx:

755 + self.llm = LLM(

756 + model=model_name,

757 + runner=runner,

758 + convert=convert,

759 + tokenizer=tokenizer_name,

760 + tokenizer_mode=tokenizer_mode,

761 + trust_remote_code=trust_remote_code,

762 + dtype=dtype,

763 + seed=seed,

764 + swap_space=swap_space,

765 + enforce_eager=enforce_eager,

766 + disable_log_stats=disable_log_stats,

767 + tensor_parallel_size=tensor_parallel_size,

768 + max_model_len=max_model_len,

769 + block_size=block_size,

770 + enable_chunked_prefill=enable_chunked_prefill,

771 + **kwargs,

772 + )

773

774 def get_inputs(

775 self,

tests/models/multimodal/generation/test_common.py

+79 -79

@@ -32,13 +32,6 @@ from .vlm_utils.types import (CustomTestOptions, ExpandableVLMTestArgs,

32 if current_platform.is_rocm():

33 os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"

35 -REQUIRES_V0_MODELS = [

36 - # V1 Test: not enough KV cache space in C1.

37 - "fuyu",

38 - # V1 Test: Deadlock issue when processing mm_inputs

39 - "llava-onevision-transformers",

40 -]

41 -

35 # yapf: disable

36 COMMON_BROADCAST_SETTINGS = {

37 "test_type": VLMTestType.IMAGE,

@@ -186,8 +179,11 @@ VLM_TEST_SETTINGS = {

179 image_size_factors=[(0.25, 0.5, 1.0)],

180 vllm_runner_kwargs={

181 "model_impl": "transformers",

182 + "default_torch_num_threads": 1,

183 },

190 - marks=[pytest.mark.core_model],

184 + # FIXME: Investigate why the test hangs

185 + # when processing the 3rd prompt in vLLM

186 + marks=[pytest.mark.core_model, pytest.mark.skip(reason="Test hangs")],

187 ),

188 "idefics3-transformers": VLMTestInfo(

189 models=["HuggingFaceTB/SmolVLM-256M-Instruct"],

@@ -320,6 +316,7 @@ VLM_TEST_SETTINGS = {

316 vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,

317 num_logprobs=10,

318 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],

319 + marks=[large_gpu_mark(min_gb=32)],

320 ),

321 "gemma3": VLMTestInfo(

322 models=["google/gemma-3-4b-it"],

@@ -861,13 +858,14 @@ VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)

858 test_type=VLMTestType.IMAGE,

859 create_new_process_for_each_test=False,

860 ))

864 -def test_single_image_models(tmp_path: PosixPath, model_type: str,

865 - test_case: ExpandableVLMTestArgs,

866 - hf_runner: type[HfRunner],

867 - vllm_runner: type[VllmRunner],

868 - image_assets: ImageTestAssets, monkeypatch):

869 - if model_type in REQUIRES_V0_MODELS:

870 - monkeypatch.setenv("VLLM_USE_V1", "0")

861 +def test_single_image_models(

862 + tmp_path: PosixPath,

863 + model_type: str,

864 + test_case: ExpandableVLMTestArgs,

865 + hf_runner: type[HfRunner],

866 + vllm_runner: type[VllmRunner],

867 + image_assets: ImageTestAssets,

868 +):

869 model_test_info = VLM_TEST_SETTINGS[model_type]

870 runners.run_single_image_test(

871 tmp_path=tmp_path,

@@ -886,13 +884,14 @@ def test_single_image_models(tmp_path: PosixPath, model_type: str,

884 test_type=VLMTestType.MULTI_IMAGE,

885 create_new_process_for_each_test=False,

886 ))

889 -def test_multi_image_models(tmp_path: PosixPath, model_type: str,

890 - test_case: ExpandableVLMTestArgs,

891 - hf_runner: type[HfRunner],

892 - vllm_runner: type[VllmRunner],

893 - image_assets: ImageTestAssets, monkeypatch):

894 - if model_type in REQUIRES_V0_MODELS:

895 - monkeypatch.setenv("VLLM_USE_V1", "0")

887 +def test_multi_image_models(

888 + tmp_path: PosixPath,

889 + model_type: str,

890 + test_case: ExpandableVLMTestArgs,

891 + hf_runner: type[HfRunner],

892 + vllm_runner: type[VllmRunner],

893 + image_assets: ImageTestAssets,

894 +):

895 model_test_info = VLM_TEST_SETTINGS[model_type]

896 runners.run_multi_image_test(

897 tmp_path=tmp_path,

@@ -911,13 +910,13 @@ def test_multi_image_models(tmp_path: PosixPath, model_type: str,

910 test_type=VLMTestType.EMBEDDING,

911 create_new_process_for_each_test=False,

912 ))

914 -def test_image_embedding_models(model_type: str,

915 - test_case: ExpandableVLMTestArgs,

916 - hf_runner: type[HfRunner],

917 - vllm_runner: type[VllmRunner],

918 - image_assets: ImageTestAssets, monkeypatch):

919 - if model_type in REQUIRES_V0_MODELS:

920 - monkeypatch.setenv("VLLM_USE_V1", "0")

913 +def test_image_embedding_models(

914 + model_type: str,

915 + test_case: ExpandableVLMTestArgs,

916 + hf_runner: type[HfRunner],

917 + vllm_runner: type[VllmRunner],

918 + image_assets: ImageTestAssets,

919 +):

920 model_test_info = VLM_TEST_SETTINGS[model_type]

921 runners.run_embedding_test(

922 model_test_info=model_test_info,

@@ -935,11 +934,13 @@ def test_image_embedding_models(model_type: str,

934 test_type=VLMTestType.VIDEO,

935 create_new_process_for_each_test=False,

936 ))

938 -def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

939 - hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],

940 - video_assets: VideoTestAssets, monkeypatch):

941 - if model_type in REQUIRES_V0_MODELS:

942 - monkeypatch.setenv("VLLM_USE_V1", "0")

937 +def test_video_models(

938 + model_type: str,

939 + test_case: ExpandableVLMTestArgs,

940 + hf_runner: type[HfRunner],

941 + vllm_runner: type[VllmRunner],

942 + video_assets: VideoTestAssets,

943 +):

944 model_test_info = VLM_TEST_SETTINGS[model_type]

945 runners.run_video_test(

946 model_test_info=model_test_info,

@@ -957,11 +958,13 @@ def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

958 test_type=VLMTestType.AUDIO,

959 create_new_process_for_each_test=False,

960 ))

960 -def test_audio_models(model_type: str, test_case: ExpandableVLMTestArgs,

961 - hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],

962 - audio_assets: AudioTestAssets, monkeypatch):

963 - if model_type in REQUIRES_V0_MODELS:

964 - monkeypatch.setenv("VLLM_USE_V1", "0")

961 +def test_audio_models(

962 + model_type: str,

963 + test_case: ExpandableVLMTestArgs,

964 + hf_runner: type[HfRunner],

965 + vllm_runner: type[VllmRunner],

966 + audio_assets: AudioTestAssets,

967 +):

968 model_test_info = VLM_TEST_SETTINGS[model_type]

969 runners.run_audio_test(

970 model_test_info=model_test_info,

@@ -984,10 +987,7 @@ def test_custom_inputs_models(

987 test_case: ExpandableVLMTestArgs,

988 hf_runner: type[HfRunner],

989 vllm_runner: type[VllmRunner],

987 - monkeypatch,

990 ):

989 - if model_type in REQUIRES_V0_MODELS:

990 - monkeypatch.setenv("VLLM_USE_V1", "0")

991 model_test_info = VLM_TEST_SETTINGS[model_type]

992 runners.run_custom_inputs_test(

993 model_test_info=model_test_info,

@@ -1006,13 +1006,14 @@ def test_custom_inputs_models(

1006 create_new_process_for_each_test=True,

1007 ))

1008 @create_new_process_for_each_test()

1009 -def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

1010 - test_case: ExpandableVLMTestArgs,

1011 - hf_runner: type[HfRunner],

1012 - vllm_runner: type[VllmRunner],

1013 - image_assets: ImageTestAssets, monkeypatch):

1014 - if model_type in REQUIRES_V0_MODELS:

1015 - monkeypatch.setenv("VLLM_USE_V1", "0")

1009 +def test_single_image_models_heavy(

1010 + tmp_path: PosixPath,

1011 + model_type: str,

1012 + test_case: ExpandableVLMTestArgs,

1013 + hf_runner: type[HfRunner],

1014 + vllm_runner: type[VllmRunner],

1015 + image_assets: ImageTestAssets,

1016 +):

1017 model_test_info = VLM_TEST_SETTINGS[model_type]

1018 runners.run_single_image_test(

1019 tmp_path=tmp_path,

@@ -1032,13 +1033,14 @@ def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

1033 create_new_process_for_each_test=True,

1034 ))

1035 @create_new_process_for_each_test()

1035 -def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

1036 - test_case: ExpandableVLMTestArgs,

1037 - hf_runner: type[HfRunner],

1038 - vllm_runner: type[VllmRunner],

1039 - image_assets: ImageTestAssets, monkeypatch):

1040 - if model_type in REQUIRES_V0_MODELS:

1041 - monkeypatch.setenv("VLLM_USE_V1", "0")

1036 +def test_multi_image_models_heavy(

1037 + tmp_path: PosixPath,

1038 + model_type: str,

1039 + test_case: ExpandableVLMTestArgs,

1040 + hf_runner: type[HfRunner],

1041 + vllm_runner: type[VllmRunner],

1042 + image_assets: ImageTestAssets,

1043 +):

1044 model_test_info = VLM_TEST_SETTINGS[model_type]

1045 runners.run_multi_image_test(

1046 tmp_path=tmp_path,

@@ -1058,14 +1060,13 @@ def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

1060 create_new_process_for_each_test=True,

1061 ))

1062 @create_new_process_for_each_test()

1061 -def test_image_embedding_models_heavy(model_type: str,

1062 - test_case: ExpandableVLMTestArgs,

1063 - hf_runner: type[HfRunner],

1064 - vllm_runner: type[VllmRunner],

1065 - image_assets: ImageTestAssets,

1066 - monkeypatch):

1067 - if model_type in REQUIRES_V0_MODELS:

1068 - monkeypatch.setenv("VLLM_USE_V1", "0")

1063 +def test_image_embedding_models_heavy(

1064 + model_type: str,

1065 + test_case: ExpandableVLMTestArgs,

1066 + hf_runner: type[HfRunner],

1067 + vllm_runner: type[VllmRunner],

1068 + image_assets: ImageTestAssets,

1069 +):

1070 model_test_info = VLM_TEST_SETTINGS[model_type]

1071 runners.run_embedding_test(

1072 model_test_info=model_test_info,

@@ -1083,12 +1084,13 @@ def test_image_embedding_models_heavy(model_type: str,

1084 test_type=VLMTestType.VIDEO,

1085 create_new_process_for_each_test=True,

1086 ))

1086 -def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

1087 - hf_runner: type[HfRunner],

1088 - vllm_runner: type[VllmRunner],

1089 - video_assets: VideoTestAssets, monkeypatch):

1090 - if model_type in REQUIRES_V0_MODELS:

1091 - monkeypatch.setenv("VLLM_USE_V1", "0")

1087 +def test_video_models_heavy(

1088 + model_type: str,

1089 + test_case: ExpandableVLMTestArgs,

1090 + hf_runner: type[HfRunner],

1091 + vllm_runner: type[VllmRunner],

1092 + video_assets: VideoTestAssets,

1093 +):

1094 model_test_info = VLM_TEST_SETTINGS[model_type]

1095 runners.run_video_test(

1096 model_test_info=model_test_info,

@@ -1106,12 +1108,13 @@ def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

1108 test_type=VLMTestType.AUDIO,

1109 create_new_process_for_each_test=True,

1110 ))

1109 -def test_audio_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

1110 - hf_runner: type[HfRunner],

1111 - vllm_runner: type[VllmRunner],

1112 - audio_assets: AudioTestAssets, monkeypatch):

1113 - if model_type in REQUIRES_V0_MODELS:

1114 - monkeypatch.setenv("VLLM_USE_V1", "0")

1111 +def test_audio_models_heavy(

1112 + model_type: str,

1113 + test_case: ExpandableVLMTestArgs,

1114 + hf_runner: type[HfRunner],

1115 + vllm_runner: type[VllmRunner],

1116 + audio_assets: AudioTestAssets,

1117 +):

1118 model_test_info = VLM_TEST_SETTINGS[model_type]

1119 runners.run_audio_test(

1120 model_test_info=model_test_info,

@@ -1135,10 +1138,7 @@ def test_custom_inputs_models_heavy(

1138 test_case: ExpandableVLMTestArgs,

1139 hf_runner: type[HfRunner],

1140 vllm_runner: type[VllmRunner],

1138 - monkeypatch,

1141 ):

1140 - if model_type in REQUIRES_V0_MODELS:

1141 - monkeypatch.setenv("VLLM_USE_V1", "0")

1142 model_test_info = VLM_TEST_SETTINGS[model_type]

1143 runners.run_custom_inputs_test(

1144 model_test_info=model_test_info,

tests/models/multimodal/generation/test_pixtral.py

+2 -47

@@ -12,13 +12,12 @@ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer

12 from mistral_common.tokens.tokenizers.multimodal import image_from_chunk

13 from transformers import AutoProcessor

15 -from vllm import RequestOutput, SamplingParams, TextPrompt, TokensPrompt

15 +from vllm import SamplingParams, TextPrompt, TokensPrompt

16 from vllm.multimodal import MultiModalDataBuiltins

17 -from vllm.multimodal.inputs import PlaceholderRange

17 from vllm.sequence import Logprob, SampleLogprobs

19 from ....utils import VLLM_PATH, large_gpu_test

21 -from ...utils import check_logprobs_close, dummy_hf_overrides

20 +from ...utils import check_logprobs_close

22 if TYPE_CHECKING:

23 from _typeshed import StrPath

@@ -185,47 +184,3 @@ def test_chat(vllm_runner, max_model_len: int, model: str, dtype: str,

184 outputs_1_lst=logprobs,

185 name_0="h100_ref",

186 name_1="output")

188 -

189 -

190 -@pytest.mark.parametrize(

191 - "image_urls,expected_ranges",

192 - [(IMG_URLS[:1], [PlaceholderRange(offset=11, length=494)]),

193 - (IMG_URLS[1:4], [

194 - PlaceholderRange(offset=11, length=266),

195 - PlaceholderRange(offset=277, length=1056),

196 - PlaceholderRange(offset=1333, length=418)

197 - ])])

198 -def test_multi_modal_placeholders(vllm_runner, image_urls: list[str],

199 - expected_ranges: list[PlaceholderRange],

200 - local_asset_server, monkeypatch) -> None:

201 - local_image_urls = [local_asset_server.url_for(u) for u in image_urls]

202 - prompt = _create_engine_inputs_hf(local_image_urls)

203 -

204 - # This placeholder checking test only works with V0 engine

205 - # where `multi_modal_placeholders` is returned with `RequestOutput`

206 - monkeypatch.setenv("VLLM_USE_V1", "0")

207 - with vllm_runner(

208 - "mistral-community/pixtral-12b",

209 - max_model_len=8192,

210 - limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,

211 - load_format="dummy",

212 - hf_overrides=dummy_hf_overrides,

213 - ) as vllm_model:

214 - outputs = vllm_model.llm.generate(prompt)

215 -

216 - assert len(outputs) == 1, f"{len(outputs)=}"

217 - output: RequestOutput = outputs[0]

218 - assert hasattr(output,

219 - "multi_modal_placeholders"), f"{output.__dict__=}"

220 - assert "image" in output.multi_modal_placeholders, \

221 - f"{output.multi_modal_placeholders.keys()=}"

222 - image_placeholder_ranges: list[

223 - PlaceholderRange] = output.multi_modal_placeholders["image"]

224 - assert len(image_placeholder_ranges) == len(

225 - expected_ranges), f"{image_placeholder_ranges=}"

226 - for real_range, expected_range in zip(image_placeholder_ranges,

227 - expected_ranges):

228 - assert real_range.offset == expected_range.offset, \

229 - f"{real_range=} {expected_range=}"

230 - assert real_range.length == expected_range.length, \

231 - f"{real_range=} {expected_range=}"

tests/models/multimodal/generation/test_qwen2_vl.py

+3 -6

@@ -10,7 +10,6 @@ from PIL import Image

11 from vllm.multimodal.image import rescale_image_size

12 from vllm.multimodal.video import rescale_video_size, sample_frames_from_video

13 -from vllm.utils import set_default_torch_num_threads

14 from ....conftest import (IMAGE_ASSETS, VIDEO_ASSETS, PromptImageInput,

15 PromptVideoInput, VllmRunner)

@@ -264,8 +263,7 @@ def run_embedding_input_test(

263 processor = AutoProcessor.from_pretrained(model)

264

265 # max_model_len should be greater than image_feature_size

267 - with set_default_torch_num_threads(1):

268 - vllm_model = vllm_runner(

266 + with vllm_runner(

267 model,

268 runner="generate",

269 max_model_len=4000,

@@ -277,9 +275,8 @@ def run_embedding_input_test(

275 },

276 tensor_parallel_size=tensor_parallel_size,

277 distributed_executor_backend=distributed_executor_backend,

280 - )

281 -

282 - with vllm_model:

278 + default_torch_num_threads=1,

279 + ) as vllm_model:

280 outputs_per_case_for_original_input = [

281 vllm_model.generate_greedy_logprobs(prompts,

282 max_tokens,

tests/models/multimodal/pooling/test_prithvi_mae.py

+11 -15

@@ -4,8 +4,6 @@

4 import pytest

5 import torch

7 -from vllm.utils import set_default_torch_num_threads

8 -

7 from ....conftest import VllmRunner

@@ -30,19 +28,17 @@ def _run_test(

28 } for _ in range(10)

29 ]

33 - with (

34 - set_default_torch_num_threads(1),

35 - vllm_runner(

36 - model,

37 - runner="pooling",

38 - dtype=torch.float16,

39 - enforce_eager=True,

40 - skip_tokenizer_init=True,

41 - # Limit the maximum number of sequences to avoid the

42 - # test going OOM during the warmup run

43 - max_num_seqs=32,

44 - ) as vllm_model,

45 - ):

31 + with vllm_runner(

32 + model,

33 + runner="pooling",

34 + dtype="half",

35 + enforce_eager=True,

36 + skip_tokenizer_init=True,

37 + # Limit the maximum number of sequences to avoid the

38 + # test going OOM during the warmup run

39 + max_num_seqs=32,

40 + default_torch_num_threads=1,

41 + ) as vllm_model:

42 vllm_model.encode(prompt)

tests/models/quantization/test_awq.py

+20 -18

@@ -45,12 +45,15 @@ def run_awq_test(

45 # will hurt multiprocessing backend with fork method (the default method).

47 # max_model_len should be greater than image_feature_size

48 - with vllm_runner(source_model,

49 - max_model_len=4096,

50 - dtype=dtype,

51 - tensor_parallel_size=tensor_parallel_size,

52 - distributed_executor_backend=distributed_executor_backend,

53 - enforce_eager=True) as vllm_model:

48 + with vllm_runner(

49 + source_model,

50 + max_model_len=4096,

51 + dtype=dtype,

52 + tensor_parallel_size=tensor_parallel_size,

53 + distributed_executor_backend=distributed_executor_backend,

54 + enforce_eager=True,

55 + default_torch_num_threads=1,

56 + ) as vllm_model:

57 source_outputs_per_image = [

58 vllm_model.generate_greedy_logprobs(prompts,

59 max_tokens,

@@ -59,13 +62,16 @@ def run_awq_test(

62 for prompts, images in inputs_per_image

63 ]

62 - with vllm_runner(quant_model,

63 - quantization="awq",

64 - max_model_len=4096,

65 - dtype=dtype,

66 - tensor_parallel_size=tensor_parallel_size,

67 - distributed_executor_backend=distributed_executor_backend,

68 - enforce_eager=True) as vllm_model:

65 + with vllm_runner(

66 + quant_model,

67 + quantization="awq",

68 + max_model_len=4096,

69 + dtype=dtype,

70 + tensor_parallel_size=tensor_parallel_size,

71 + distributed_executor_backend=distributed_executor_backend,

72 + enforce_eager=True,

73 + default_torch_num_threads=1,

74 + ) as vllm_model:

75 quant_outputs_per_image = [

76 vllm_model.generate_greedy_logprobs(prompts,

77 max_tokens,

@@ -108,12 +114,8 @@ def run_awq_test(

114 @pytest.mark.parametrize("num_logprobs", [5])

115 @torch.inference_mode()

116 def test_awq_models(vllm_runner, image_assets, source_model, quant_model,

111 - size_factors, dtype, max_tokens, num_logprobs,

112 - monkeypatch) -> None:

117 + size_factors, dtype, max_tokens, num_logprobs) -> None:

118

114 - # Test V1: this test hangs during setup on single-scale input.

115 - # TODO: figure out why and re-enable this on V1.

116 - monkeypatch.setenv("VLLM_USE_V1", "0")

119 run_awq_test(

120 vllm_runner,

121 image_assets,

tests/models/quantization/test_bitsandbytes.py

+13 -15

@@ -5,10 +5,7 @@

5 Run `pytest tests/quantization/test_bitsandbytes.py`.

6 '''

8 -import gc

9 -

8 import pytest

11 -import torch

9 from transformers import BitsAndBytesConfig

11 from tests.quantization.utils import is_quant_method_supported

@@ -131,12 +128,15 @@ def test_4bit_bnb_moe_model(hf_runner, vllm_runner, example_prompts,

128 ))

129 with vllm_runner(model_name,

130 quantization='bitsandbytes',

134 - enforce_eager=False) as llm:

131 + enforce_eager=False,

132 + default_torch_num_threads=1) as llm:

133 vllm_outputs = llm.generate_greedy_logprobs(example_prompts,

134 max_tokens=32,

135 num_logprobs=5)

136

139 - with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:

137 + with hf_runner(model_name,

138 + model_kwargs=hf_model_kwargs,

139 + default_torch_num_threads=1) as llm:

140 transformers_outputs = llm.generate_greedy_logprobs_limit(

141 example_prompts, max_tokens=32, num_logprobs=5)

142 check_logprobs_close(

@@ -174,7 +174,8 @@ def test_4bit_bnb_embedding_model(

174 runner="pooling",

175 dtype=dtype,

176 gpu_memory_utilization=0.5,

177 - quantization="bitsandbytes") as vllm_model:

177 + quantization="bitsandbytes",

178 + default_torch_num_threads=1) as vllm_model:

179 vllm_outputs = vllm_model.embed(example_prompts)

180

181 hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(

@@ -184,6 +185,7 @@ def test_4bit_bnb_embedding_model(

185 dtype=dtype,

186 model_kwargs=hf_model_kwargs,

187 is_sentence_transformer=True,

188 + default_torch_num_threads=1,

189 ) as hf_model:

190 hf_outputs = hf_model.encode(example_prompts)

191

@@ -222,26 +224,22 @@ def validate_generated_texts(hf_runner,

224 with vllm_runner(model_name,

225 quantization=None if pre_quant else 'bitsandbytes',

226 tensor_parallel_size=vllm_tp_size,

225 - enforce_eager=False) as llm:

227 + enforce_eager=False,

228 + default_torch_num_threads=1) as llm:

229

230 vllm_outputs = llm.generate_greedy(prompts, max_tokens)

231 vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")

232

230 - # Clean up the GPU memory for the next test

231 - gc.collect()

232 - torch.cuda.empty_cache()

233 -

233 if hf_model_kwargs is None:

234 hf_model_kwargs = {}

235

236 # Run with HF runner

238 - with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:

237 + with hf_runner(model_name,

238 + model_kwargs=hf_model_kwargs,

239 + default_torch_num_threads=1) as llm:

240 hf_outputs = llm.generate_greedy(prompts, max_tokens)

241 hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")

242

242 - # Clean up the GPU memory for the next test

243 - gc.collect()

244 - torch.cuda.empty_cache()

243 # Compare the generated strings

244 for hf_log, vllm_log in zip(hf_logs, vllm_logs):

245 hf_str = hf_log["generated_text"]

tests/models/test_terratorch.py

+11 -14

@@ -5,7 +5,6 @@ import pytest

5 import torch

7 from tests.conftest import VllmRunner

8 -from vllm.utils import set_default_torch_num_threads

10 @pytest.mark.parametrize(

@@ -25,19 +24,17 @@ def test_inference(

24 prompt = dict(prompt_token_ids=[1],

25 multi_modal_data=dict(pixel_values=pixel_values,

26 location_coords=location_coords))

28 - with (

29 - set_default_torch_num_threads(1),

30 - vllm_runner(

31 - model,

32 - runner="pooling",

33 - dtype=torch.float16,

34 - enforce_eager=True,

35 - skip_tokenizer_init=True,

36 - # Limit the maximum number of sequences to avoid the

37 - # test going OOM during the warmup run

38 - max_num_seqs=32,

39 - ) as vllm_model,

40 - ):

27 + with vllm_runner(

28 + model,

29 + runner="pooling",

30 + dtype="half",

31 + enforce_eager=True,

32 + skip_tokenizer_init=True,

33 + # Limit the maximum number of sequences to avoid the

34 + # test going OOM during the warmup run

35 + max_num_seqs=32,

36 + default_torch_num_threads=1,

37 + ) as vllm_model:

39 vllm_output = vllm_model.llm.encode(prompt)

40 assert torch.equal(