@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51c2e1fcef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI/Build] Split up models tests (#10069)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 10, 2024 at 03:39 UTC 51c2e1fcef59ca42b378c433997c77affd114d30
21 files changed +115 -129
.buildkite/test-pipeline.yaml
+14 -10
@@ -305,7 +305,7 @@ steps:
305
306 ##### models test #####
307
308 -- label: Basic Models Test # 3min
308 +- label: Basic Models Test # 10min
309 source_file_dependencies:
310 - vllm/
311 - tests/models
@@ -314,23 +314,24 @@ steps:
314 - pytest -v -s models/test_oot_registration.py # it needs a clean process
315 - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
316
317 -- label: Decoder-only Language Models Test (Standard) # 35min
317 +- label: Decoder-only Language Models Test (Standard) # 18min
318 #mirror_hardwares: [amd]
319 source_file_dependencies:
320 - vllm/
321 - tests/models/decoder_only/language
322 commands:
323 - - pytest -v -s models/decoder_only/language/test_models.py
323 + - pytest -v -s models/decoder_only/language -m core_model
324 + - pytest -v -s models/decoder_only/language -m quant_model
325
325 -- label: Decoder-only Language Models Test (Extended) # 1h20min
326 +- label: Decoder-only Language Models Test (Extended) # 46min
327 nightly: true
328 source_file_dependencies:
329 - vllm/
330 - tests/models/decoder_only/language
331 commands:
331 - - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332 + - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
333
333 -- label: Decoder-only Multi-Modal Models Test (Standard) # 26min
334 +- label: Decoder-only Multi-Modal Models Test (Standard) # 22min
335 #mirror_hardwares: [amd]
336 source_file_dependencies:
337 - vllm/
@@ -339,21 +340,24 @@ steps:
340 commands:
341 - pytest -v -s models/decoder_only/audio_language -m core_model
342 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
343 + # No tests under this group for now
344 + # - pytest -v -s models/decoder_only/audio_language -m quant_model
345 + - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m quant_model
346
343 -- label: Decoder-only Multi-Modal Models Test (Extended)
347 +- label: Decoder-only Multi-Modal Models Test (Extended) # 1h10m
348 nightly: true
349 source_file_dependencies:
350 - vllm/
351 - tests/models/decoder_only/audio_language
352 - tests/models/decoder_only/vision_language
353 commands:
350 - - pytest -v -s models/decoder_only/audio_language -m 'not core_model'
354 + - pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
355 # HACK - run phi3v tests separately to sidestep this transformers bug
356 # https://github.com/huggingface/transformers/issues/34307
357 - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
354 - - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model'
358 + - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
359
356 -- label: Other Models Test # 6min
360 +- label: Other Models Test # 20min
361 #mirror_hardwares: [amd]
362 source_file_dependencies:
363 - vllm/
pyproject.toml
+1
@@ -95,6 +95,7 @@ markers = [
95 "skip_global_cleanup",
96 "core_model: enable this model test in each PR instead of only nightly",
97 "cpu_model: enable this model test in CPU tests",
98 + "quant_model: run this model test under Quantized category",
99 "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100 "skip_v1: do not run this test with v1",
101 ]
tests/models/decoder_only/language/test_aqlm.py
+1
@@ -38,6 +38,7 @@ ground_truth_generations = [
38 ]
39
40
41 +@pytest.mark.quant_model
42 @pytest.mark.skipif(not is_quant_method_supported("aqlm"),
43 reason="AQLM is not supported on this GPU type.")
44 @pytest.mark.parametrize("model", ["ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"])
tests/models/decoder_only/language/test_fp8.py
+1
@@ -15,6 +15,7 @@ from ...utils import check_logprobs_close
15 os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
17
18 +@pytest.mark.quant_model
19 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
20 reason="fp8 is not supported on this GPU type.")
21 @pytest.mark.parametrize(
tests/models/decoder_only/language/test_gguf.py
+16 -19
@@ -17,26 +17,21 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
17
18 MAX_MODEL_LEN = 1024
19
20 -# FIXME: Move this to confest
21 -MODELS = [
22 - ("meta-llama/Llama-3.2-1B-Instruct",
23 - hf_hub_download("bartowski/Llama-3.2-1B-Instruct-GGUF",
24 - filename="Llama-3.2-1B-Instruct-Q4_K_M.gguf")),
25 - ("meta-llama/Llama-3.2-1B-Instruct",
26 - hf_hub_download("bartowski/Llama-3.2-1B-Instruct-GGUF",
27 - filename="Llama-3.2-1B-Instruct-IQ4_XS.gguf")),
28 - ("Qwen/Qwen2-1.5B-Instruct",
29 - hf_hub_download("Qwen/Qwen2-1.5B-Instruct-GGUF",
30 - filename="qwen2-1_5b-instruct-q4_k_m.gguf")),
31 - ("Qwen/Qwen2-1.5B-Instruct",
32 - hf_hub_download("legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33 - filename="Qwen2-1.5B-Instruct.IQ4_XS.gguf")),
34 -]
35 -
20
21 @pytest.mark.skipif(not is_quant_method_supported("gguf"),
22 reason="gguf is not supported on this GPU type.")
39 -@pytest.mark.parametrize("model", MODELS)
23 +@pytest.mark.parametrize(("original_model", "gguf_id", "gguf_path"), [
24 + ("meta-llama/Llama-3.2-1B-Instruct",
25 + "bartowski/Llama-3.2-1B-Instruct-GGUF",
26 + "Llama-3.2-1B-Instruct-Q4_K_M.gguf"),
27 + ("meta-llama/Llama-3.2-1B-Instruct",
28 + "bartowski/Llama-3.2-1B-Instruct-GGUF",
29 + "Llama-3.2-1B-Instruct-IQ4_XS.gguf"),
30 + ("Qwen/Qwen2-1.5B-Instruct", "Qwen/Qwen2-1.5B-Instruct-GGUF",
31 + "qwen2-1_5b-instruct-q4_k_m.gguf"),
32 + ("Qwen/Qwen2-1.5B-Instruct", "legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33 + "Qwen2-1.5B-Instruct.IQ4_XS.gguf"),
34 +])
35 @pytest.mark.parametrize("dtype", ["half"])
36 @pytest.mark.parametrize("max_tokens", [32])
37 @pytest.mark.parametrize("num_logprobs", [5])
@@ -45,7 +40,9 @@ def test_models(
40 num_gpus_available,
41 vllm_runner,
42 example_prompts,
48 - model,
43 + original_model,
44 + gguf_id,
45 + gguf_path,
46 dtype: str,
47 max_tokens: int,
48 num_logprobs: int,
@@ -54,7 +51,7 @@ def test_models(
51 if num_gpus_available < tp_size:
52 pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
53
57 - original_model, gguf_model = model
54 + gguf_model = hf_hub_download(gguf_id, filename=gguf_path)
55
56 tokenizer = AutoTokenizer.from_pretrained(original_model)
57 messages = [[{
tests/models/decoder_only/language/test_gptq_marlin.py
+1
@@ -33,6 +33,7 @@ MODELS = [
33 ]
34
35
36 +@pytest.mark.quant_model
37 @pytest.mark.flaky(reruns=3)
38 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
39 reason="gptq_marlin is not supported on this GPU type.")
tests/models/decoder_only/language/test_gptq_marlin_24.py
+1
@@ -38,6 +38,7 @@ model_pairs = [
38 ]
39
40
41 +@pytest.mark.quant_model
42 @pytest.mark.flaky(reruns=2)
43 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin_24"),
44 reason="Marlin24 is not supported on this GPU type.")
tests/models/decoder_only/language/test_granite.py
+2 -1
@@ -7,7 +7,9 @@ import pytest
7 from ...utils import check_logprobs_close
8
9 MODELS = [
10 + # TODO(sang): Sliding window should be tested separately.
11 "ibm/PowerLM-3b",
12 + "ibm/PowerMoE-3b",
13 ]
14
15
@@ -24,7 +26,6 @@ def test_models(
26 max_tokens: int,
27 num_logprobs: int,
28 ) -> None:
27 - # TODO(sang): Sliding window should be tested separately.
29 with hf_runner(model, dtype=dtype) as hf_model:
30 hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 example_prompts, max_tokens, num_logprobs)
tests/models/decoder_only/language/test_granitemoe.py deleted
-39
@@ -1,39 +0,0 @@
1 -"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2 -
3 -Run `pytest tests/models/test_granite.py`.
4 -"""
5 -import pytest
6 -
7 -from ...utils import check_logprobs_close
8 -
9 -MODELS = [
10 - "ibm/PowerMoE-3b",
11 -]
12 -
13 -
14 -@pytest.mark.parametrize("model", MODELS)
15 -@pytest.mark.parametrize("dtype", ["bfloat16"])
16 -@pytest.mark.parametrize("max_tokens", [64])
17 -@pytest.mark.parametrize("num_logprobs", [5])
18 -def test_models(
19 - hf_runner,
20 - vllm_runner,
21 - example_prompts,
22 - model: str,
23 - dtype: str,
24 - max_tokens: int,
25 - num_logprobs: int,
26 -) -> None:
27 - with hf_runner(model, dtype=dtype) as hf_model:
28 - hf_outputs = hf_model.generate_greedy_logprobs_limit(
29 - example_prompts, max_tokens, num_logprobs)
30 -
31 - with vllm_runner(model, dtype=dtype) as vllm_model:
32 - vllm_outputs = vllm_model.generate_greedy_logprobs(
33 - example_prompts, max_tokens, num_logprobs)
34 - check_logprobs_close(
35 - outputs_0_lst=hf_outputs,
36 - outputs_1_lst=vllm_outputs,
37 - name_0="hf",
38 - name_1="vllm",
39 - )
tests/models/decoder_only/language/test_modelopt.py
+1
@@ -39,6 +39,7 @@ EXPECTED_STRS_MAP = {
39 @pytest.mark.skip(
40 reason=
41 "Prevent unstable test based on golden strings from breaking the build.")
42 +@pytest.mark.quant_model
43 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
44 reason="fp8 is not supported on this GPU type.")
45 @pytest.mark.parametrize("model_name", MODELS)
tests/models/decoder_only/language/test_models.py
+1 -3
@@ -1,8 +1,5 @@
1 """Compare the outputs of HF and vLLM when using greedy sampling.
2
3 -This test only tests small models. Big models such as 7B should be tested from
4 -test_big_models.py because it could use a larger instance to run tests.
5 -
3 Run `pytest tests/models/test_models.py`.
4 """
5 import pytest
@@ -35,6 +32,7 @@ if not current_platform.is_cpu():
32 target_dtype = "half"
33
34
35 +@pytest.mark.core_model
36 @pytest.mark.parametrize("model", MODELS)
37 @pytest.mark.parametrize("dtype", [target_dtype])
38 @pytest.mark.parametrize("max_tokens", [32])
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_llava_next.py
+3 -1
@@ -56,11 +56,13 @@ def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
56 ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
57 seq_len = 5000 # bigger than the max feature size for any image
58
59 - seq_data, mm_data = dummy_data_for_llava_next(
59 + dummy_data = dummy_data_for_llava_next(
60 ctx,
61 seq_len=seq_len,
62 mm_counts={"image": 1},
63 )
64 + seq_data = dummy_data.seq_data
65 + mm_data = dummy_data.multi_modal_data
66
67 # The dummy data dims should match the gridpoint with the biggest feat size
68 assert mm_data["image"].height == expected_size[0]
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2 -1
@@ -131,12 +131,13 @@ def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
131 mm_processor_kwargs=None,
132 )
133
134 - sequence_data, _, = dummy_data_for_phi3v(
134 + dummy_data = dummy_data_for_phi3v(
135 ctx=ctx,
136 seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137 mm_counts={"image": num_imgs},
138 num_crops=num_crops,
139 )
140 + sequence_data = dummy_data.seq_data
141 # Ensure we have the right number of placeholders per num_crops size
142 img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
143 assert img_tok_count == toks_per_img * num_imgs
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+11 -4
@@ -86,10 +86,17 @@ def test_qwen2_vl_dummy_data(dummy_data_for_qwen2_vl,
86
87 # NOTE: video value is required, but isn't actually used
88 # when making the dummy data except for error handling currently
89 - seq_data, mm_data = dummy_data_for_qwen2_vl(qwen2_vl_context, seq_len, {
90 - "image": 1,
91 - "video": 0
92 - }, **mm_processor_kwargs)
89 + dummy_data = dummy_data_for_qwen2_vl(
90 + ctx=qwen2_vl_context,
91 + seq_len=seq_len,
92 + mm_counts={
93 + "image": 1,
94 + "video": 0
95 + },
96 + **mm_processor_kwargs,
97 + )
98 + seq_data = dummy_data.seq_data
99 + mm_data = dummy_data.multi_modal_data
100
101 # Ensure we have the right number of placeholders for min/max pixel values
102 assert seq_data.get_token_ids().count(image_token_id) == token_count
tests/models/decoder_only/vision_language/test_awq.py renamed
+11 -8
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple, Type
1 +from typing import List, Optional, Type
2
3 import pytest
4 import torch
@@ -19,7 +19,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19 def run_awq_test(
20 vllm_runner: Type[VllmRunner],
21 image_assets: _ImageAssets,
22 - models: Tuple[str, str],
22 + source_model: str,
23 + quant_model: str,
24 *,
25 size_factors: List[float],
26 dtype: str,
@@ -28,8 +29,6 @@ def run_awq_test(
29 tensor_parallel_size: int,
30 distributed_executor_backend: Optional[str] = None,
31 ):
31 - source_model, quant_model = models
32 -
32 images = [asset.pil_image for asset in image_assets]
33
34 inputs_per_image = [(
@@ -84,8 +83,11 @@ def run_awq_test(
83 )
84
85
86 +@pytest.mark.quant_model
87 @pytest.mark.parametrize(
88 - "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
88 + ("source_model", "quant_model"),
89 + [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
90 +)
91 @pytest.mark.parametrize(
92 "size_factors",
93 [
@@ -103,12 +105,13 @@ def run_awq_test(
105 @pytest.mark.parametrize("max_tokens", [128])
106 @pytest.mark.parametrize("num_logprobs", [5])
107 @torch.inference_mode()
106 -def test_awq_models(vllm_runner, image_assets, models, size_factors,
107 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
108 +def test_awq_models(vllm_runner, image_assets, source_model, quant_model,
109 + size_factors, dtype, max_tokens, num_logprobs) -> None:
110 run_awq_test(
111 vllm_runner,
112 image_assets,
111 - models,
113 + source_model,
114 + quant_model,
115 size_factors=size_factors,
116 dtype=dtype,
117 max_tokens=max_tokens,
tests/models/decoder_only/vision_language/test_intern_vit.py
+9 -10
@@ -11,21 +11,17 @@ from ....conftest import _ImageAssets
11 # we use snapshot_download to prevent conflicts between
12 # dynamic_module and trust_remote_code for hf_runner
13 DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
14 -models = [
15 - snapshot_download("OpenGVLab/InternViT-300M-448px",
16 - allow_patterns=DOWNLOAD_PATTERN),
17 - snapshot_download("OpenGVLab/InternViT-6B-448px-V1-5",
18 - allow_patterns=DOWNLOAD_PATTERN),
19 -]
14
15
16 def run_intern_vit_test(
17 image_assets: _ImageAssets,
24 - model: str,
18 + model_id: str,
19 *,
20 dtype: str,
21 distributed_executor_backend: Optional[str] = None,
22 ):
23 + model = snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
24 +
25 img_processor = CLIPImageProcessor.from_pretrained(model)
26 images = [asset.pil_image for asset in image_assets]
27 pixel_values = [
@@ -67,12 +63,15 @@ def run_intern_vit_test(
63 assert cos_similar(vllm_output, hf_output).mean() > 0.99
64
65
70 -@pytest.mark.parametrize("model", models)
66 +@pytest.mark.parametrize("model_id", [
67 + "OpenGVLab/InternViT-300M-448px",
68 + "OpenGVLab/InternViT-6B-448px-V1-5",
69 +])
70 @pytest.mark.parametrize("dtype", [torch.half])
71 @torch.inference_mode()
73 -def test_models(dist_init, image_assets, model, dtype: str) -> None:
72 +def test_models(dist_init, image_assets, model_id, dtype: str) -> None:
73 run_intern_vit_test(
74 image_assets,
76 - model,
75 + model_id,
76 dtype=dtype,
77 )
tests/models/decoder_only/vision_language/test_models.py
+23 -23
@@ -130,8 +130,8 @@ VLM_TEST_SETTINGS = {
130 max_num_seqs=2,
131 auto_cls=AutoModelForVision2Seq,
132 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
133 - marks=[pytest.mark.core_model, pytest.mark.cpu_model],
133 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
134 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
135 ),
136 #### Extended model tests
137 "blip2": VLMTestInfo(
@@ -159,9 +159,9 @@ VLM_TEST_SETTINGS = {
159 dtype="bfloat16",
160 marks=[
161 pytest.mark.skipif(
162 - transformers.__version__.startswith("4.46"),
162 + transformers.__version__ < "4.46.2",
163 reason="Model broken in HF, see huggingface/transformers#34379"
164 - )
164 + ),
165 ]
166 ),
167 "fuyu": VLMTestInfo(
@@ -185,8 +185,8 @@ VLM_TEST_SETTINGS = {
185 max_num_seqs=2,
186 dtype="bfloat16",
187 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
188 - marks=[large_gpu_mark(min_gb=48)],
188 patch_hf_runner=model_utils.glm_patch_hf_runner,
189 + marks=[large_gpu_mark(min_gb=48)],
190 ),
191 "h2ovl": VLMTestInfo(
192 models = [
@@ -205,6 +205,22 @@ VLM_TEST_SETTINGS = {
205 use_tokenizer_eos=True,
206 patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
207 ),
208 + "idefics3": VLMTestInfo(
209 + models=["HuggingFaceM4/Idefics3-8B-Llama3"],
210 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
211 + prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
212 + img_idx_to_prompt=lambda idx: "<image>",
213 + max_model_len=8192,
214 + max_num_seqs=2,
215 + auto_cls=AutoModelForVision2Seq,
216 + marks=[
217 + pytest.mark.skipif(
218 + transformers.__version__ < "4.46.0",
219 + reason="Model introduced in HF >= 4.46.0"
220 + ),
221 + large_gpu_mark(min_gb=48),
222 + ],
223 + ),
224 "intern_vl": VLMTestInfo(
225 models=[
226 "OpenGVLab/InternVL2-1B",
@@ -263,7 +279,6 @@ VLM_TEST_SETTINGS = {
279 runner_mm_key="videos",
280 )],
281 ),
266 - # FIXME
282 "llava_next_video": VLMTestInfo(
283 models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
284 test_type=VLMTestType.VIDEO,
@@ -275,7 +290,7 @@ VLM_TEST_SETTINGS = {
290 image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
291 marks=[
292 pytest.mark.skipif(
278 - transformers.__version__.startswith("4.46"),
293 + transformers.__version__ < "4.46.2",
294 reason="Model broken with changes in transformers 4.46"
295 )
296 ],
@@ -316,6 +331,7 @@ VLM_TEST_SETTINGS = {
331 max_model_len=8192,
332 max_num_seqs=2,
333 auto_cls=AutoModelForVision2Seq,
334 + marks=[large_gpu_mark(min_gb=48)],
335 ),
336 "qwen": VLMTestInfo(
337 models=["Qwen/Qwen-VL"],
@@ -327,22 +343,6 @@ VLM_TEST_SETTINGS = {
343 vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
344 prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
345 ),
330 - "idefics3": VLMTestInfo(
331 - models=["HuggingFaceM4/Idefics3-8B-Llama3"],
332 - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
333 - prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
334 - img_idx_to_prompt=lambda idx: "<image>",
335 - max_model_len=8192,
336 - max_num_seqs=2,
337 - auto_cls=AutoModelForVision2Seq,
338 - marks=[
339 - pytest.mark.skipif(
340 - transformers.__version__ < "4.46.0",
341 - reason="Model introduced in HF >= 4.46.0"
342 - ),
343 - large_gpu_mark(min_gb=48),
344 - ],
345 - ),
346 ### Tensor parallel / multi-gpu broadcast tests
347 "broadcast-chameleon": VLMTestInfo(
348 models=["facebook/chameleon-7b"],
@@ -362,7 +362,7 @@ VLM_TEST_SETTINGS = {
362 reason="Need at least 2 GPUs to run the test.",
363 ),
364 pytest.mark.skipif(
365 - transformers.__version__.startswith("4.46"),
365 + transformers.__version__ < "4.46.2",
366 reason="Model broken in HF, see huggingface/transformers#34379"
367 )
368 ],
vllm/config.py
+8 -1
@@ -1,7 +1,8 @@
1 +import copy
2 import enum
3 import json
4 import warnings
4 -from dataclasses import dataclass, field
5 +from dataclasses import dataclass, field, replace
6 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
7 Mapping, Optional, Set, Tuple, Type, Union)
8
@@ -2078,6 +2079,12 @@ class VllmConfig:
2079 return quant_config
2080 return None
2081
2082 + def with_hf_config(self, hf_config: PretrainedConfig) -> "VllmConfig":
2083 + model_config = copy.deepcopy(self.model_config)
2084 + model_config.hf_config = hf_config
2085 +
2086 + return replace(self, model_config=model_config)
2087 +
2088 def __post_init__(self):
2089 """Verify configs are valid & consistent with each other.
2090 """
vllm/model_executor/models/fuyu.py
+2 -4
@@ -229,7 +229,6 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
229 def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
230 super().__init__()
231 config = vllm_config.model_config.hf_config
232 - cache_config = vllm_config.cache_config
232 quant_config = vllm_config.quant_config
233 multimodal_config = vllm_config.model_config.multimodal_config
234 self.config = config
@@ -246,9 +245,8 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
245 quant_config=quant_config,
246 gather_output=True,
247 )
249 - self.language_model = PersimmonForCausalLM(config.text_config,
250 - cache_config=cache_config,
251 - quant_config=quant_config)
248 + self.language_model = PersimmonForCausalLM(
249 + vllm_config.with_hf_config(config.text_config))
250 self.make_empty_intermediate_tensors = (
251 self.language_model.make_empty_intermediate_tensors)
252
vllm/model_executor/models/internlm2_ve.py
+3 -1
@@ -164,10 +164,12 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
164 vllm_config: VllmConfig,
165 prefix: str = "",
166 ) -> None:
167 + super().__init__(vllm_config, prefix=prefix)
168 +
169 config = vllm_config.model_config.hf_config
170 cache_config = vllm_config.cache_config
171 quant_config = vllm_config.quant_config
170 - super().__init__(config, cache_config, quant_config)
172 +
173 self.model = InternLM2VEModel(config,
174 cache_config,
175 quant_config,
vllm/model_executor/models/utils.py
+4 -4
@@ -241,11 +241,11 @@ def init_vllm_registered_model(
241 based on the arguments passed to the outer vLLM model.
242 """
243 model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
244 - import copy
245 - copied_config = copy.deepcopy(vllm_config)
246 - copied_config.model_config.hf_config = hf_config
244
248 - return model_class(vllm_config=copied_config, prefix=prefix)
245 + return model_class(
246 + vllm_config=vllm_config.with_hf_config(hf_config),
247 + prefix=prefix,
248 + )
249
250
251 @overload