A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Split up models tests (#10069)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 10, 2024 at 03:39 UTC
51c2e1fcef59ca42b378c433997c77affd114d30
21 files changed
+115
-129
.buildkite/test-pipeline.yaml
+14
-10
@@ -305,7 +305,7 @@ steps:
305
306
##### models test #####
307
308
-- label: Basic Models Test # 3min
308
+- label: Basic Models Test # 10min
309
source_file_dependencies:
310
- vllm/
311
- tests/models
@@ -314,23 +314,24 @@ steps:
314
- pytest -v -s models/test_oot_registration.py # it needs a clean process
315
- pytest -v -s models/*.py --ignore=models/test_oot_registration.py
316
317
-- label: Decoder-only Language Models Test (Standard) # 35min
317
+- label: Decoder-only Language Models Test (Standard) # 18min
318
#mirror_hardwares: [amd]
319
source_file_dependencies:
320
- vllm/
321
- tests/models/decoder_only/language
322
commands:
323
- - pytest -v -s models/decoder_only/language/test_models.py
323
+ - pytest -v -s models/decoder_only/language -m core_model
324
+ - pytest -v -s models/decoder_only/language -m quant_model
325
325
-- label: Decoder-only Language Models Test (Extended) # 1h20min
326
+- label: Decoder-only Language Models Test (Extended) # 46min
327
nightly: true
328
source_file_dependencies:
329
- vllm/
330
- tests/models/decoder_only/language
331
commands:
331
- - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332
+ - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
333
333
-- label: Decoder-only Multi-Modal Models Test (Standard) # 26min
334
+- label: Decoder-only Multi-Modal Models Test (Standard) # 22min
335
#mirror_hardwares: [amd]
336
source_file_dependencies:
337
- vllm/
@@ -339,21 +340,24 @@ steps:
340
commands:
341
- pytest -v -s models/decoder_only/audio_language -m core_model
342
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
343
+ # No tests under this group for now
344
+ # - pytest -v -s models/decoder_only/audio_language -m quant_model
345
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m quant_model
346
343
-- label: Decoder-only Multi-Modal Models Test (Extended)
347
+- label: Decoder-only Multi-Modal Models Test (Extended) # 1h10m
348
nightly: true
349
source_file_dependencies:
350
- vllm/
351
- tests/models/decoder_only/audio_language
352
- tests/models/decoder_only/vision_language
353
commands:
350
- - pytest -v -s models/decoder_only/audio_language -m 'not core_model'
354
+ - pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
355
# HACK - run phi3v tests separately to sidestep this transformers bug
356
# https://github.com/huggingface/transformers/issues/34307
357
- pytest -v -s models/decoder_only/vision_language/test_phi3v.py
354
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model'
358
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
359
356
-- label: Other Models Test # 6min
360
+- label: Other Models Test # 20min
361
#mirror_hardwares: [amd]
362
source_file_dependencies:
363
- vllm/
pyproject.toml
+1
@@ -95,6 +95,7 @@ markers = [
95
"skip_global_cleanup",
96
"core_model: enable this model test in each PR instead of only nightly",
97
"cpu_model: enable this model test in CPU tests",
98
+ "quant_model: run this model test under Quantized category",
99
"distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100
"skip_v1: do not run this test with v1",
101
]
tests/models/decoder_only/language/test_aqlm.py
+1
@@ -38,6 +38,7 @@ ground_truth_generations = [
38
]
39
40
41
+@pytest.mark.quant_model
42
@pytest.mark.skipif(not is_quant_method_supported("aqlm"),
43
reason="AQLM is not supported on this GPU type.")
44
@pytest.mark.parametrize("model", ["ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"])
tests/models/decoder_only/language/test_fp8.py
+1
@@ -15,6 +15,7 @@ from ...utils import check_logprobs_close
15
os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
17
18
+@pytest.mark.quant_model
19
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
20
reason="fp8 is not supported on this GPU type.")
21
@pytest.mark.parametrize(
tests/models/decoder_only/language/test_gguf.py
+16
-19
@@ -17,26 +17,21 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
17
18
MAX_MODEL_LEN = 1024
19
20
-# FIXME: Move this to confest
21
-MODELS = [
22
- ("meta-llama/Llama-3.2-1B-Instruct",
23
- hf_hub_download("bartowski/Llama-3.2-1B-Instruct-GGUF",
24
- filename="Llama-3.2-1B-Instruct-Q4_K_M.gguf")),
25
- ("meta-llama/Llama-3.2-1B-Instruct",
26
- hf_hub_download("bartowski/Llama-3.2-1B-Instruct-GGUF",
27
- filename="Llama-3.2-1B-Instruct-IQ4_XS.gguf")),
28
- ("Qwen/Qwen2-1.5B-Instruct",
29
- hf_hub_download("Qwen/Qwen2-1.5B-Instruct-GGUF",
30
- filename="qwen2-1_5b-instruct-q4_k_m.gguf")),
31
- ("Qwen/Qwen2-1.5B-Instruct",
32
- hf_hub_download("legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33
- filename="Qwen2-1.5B-Instruct.IQ4_XS.gguf")),
34
-]
35
-
20
21
@pytest.mark.skipif(not is_quant_method_supported("gguf"),
22
reason="gguf is not supported on this GPU type.")
39
-@pytest.mark.parametrize("model", MODELS)
23
+@pytest.mark.parametrize(("original_model", "gguf_id", "gguf_path"), [
24
+ ("meta-llama/Llama-3.2-1B-Instruct",
25
+ "bartowski/Llama-3.2-1B-Instruct-GGUF",
26
+ "Llama-3.2-1B-Instruct-Q4_K_M.gguf"),
27
+ ("meta-llama/Llama-3.2-1B-Instruct",
28
+ "bartowski/Llama-3.2-1B-Instruct-GGUF",
29
+ "Llama-3.2-1B-Instruct-IQ4_XS.gguf"),
30
+ ("Qwen/Qwen2-1.5B-Instruct", "Qwen/Qwen2-1.5B-Instruct-GGUF",
31
+ "qwen2-1_5b-instruct-q4_k_m.gguf"),
32
+ ("Qwen/Qwen2-1.5B-Instruct", "legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33
+ "Qwen2-1.5B-Instruct.IQ4_XS.gguf"),
34
+])
35
@pytest.mark.parametrize("dtype", ["half"])
36
@pytest.mark.parametrize("max_tokens", [32])
37
@pytest.mark.parametrize("num_logprobs", [5])
@@ -45,7 +40,9 @@ def test_models(
40
num_gpus_available,
41
vllm_runner,
42
example_prompts,
48
- model,
43
+ original_model,
44
+ gguf_id,
45
+ gguf_path,
46
dtype: str,
47
max_tokens: int,
48
num_logprobs: int,
@@ -54,7 +51,7 @@ def test_models(
51
if num_gpus_available < tp_size:
52
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
53
57
- original_model, gguf_model = model
54
+ gguf_model = hf_hub_download(gguf_id, filename=gguf_path)
55
56
tokenizer = AutoTokenizer.from_pretrained(original_model)
57
messages = [[{
tests/models/decoder_only/language/test_gptq_marlin.py
+1
@@ -33,6 +33,7 @@ MODELS = [
33
]
34
35
36
+@pytest.mark.quant_model
37
@pytest.mark.flaky(reruns=3)
38
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
39
reason="gptq_marlin is not supported on this GPU type.")
tests/models/decoder_only/language/test_gptq_marlin_24.py
+1
@@ -38,6 +38,7 @@ model_pairs = [
38
]
39
40
41
+@pytest.mark.quant_model
42
@pytest.mark.flaky(reruns=2)
43
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin_24"),
44
reason="Marlin24 is not supported on this GPU type.")
tests/models/decoder_only/language/test_granite.py
+2
-1
@@ -7,7 +7,9 @@ import pytest
7
from ...utils import check_logprobs_close
8
9
MODELS = [
10
+ # TODO(sang): Sliding window should be tested separately.
11
"ibm/PowerLM-3b",
12
+ "ibm/PowerMoE-3b",
13
]
14
15
@@ -24,7 +26,6 @@ def test_models(
26
max_tokens: int,
27
num_logprobs: int,
28
) -> None:
27
- # TODO(sang): Sliding window should be tested separately.
29
with hf_runner(model, dtype=dtype) as hf_model:
30
hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
example_prompts, max_tokens, num_logprobs)
tests/models/decoder_only/language/test_granitemoe.py
deleted
-39
@@ -1,39 +0,0 @@
1
-"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2
-
3
-Run `pytest tests/models/test_granite.py`.
4
-"""
5
-import pytest
6
-
7
-from ...utils import check_logprobs_close
8
-
9
-MODELS = [
10
- "ibm/PowerMoE-3b",
11
-]
12
-
13
-
14
-@pytest.mark.parametrize("model", MODELS)
15
-@pytest.mark.parametrize("dtype", ["bfloat16"])
16
-@pytest.mark.parametrize("max_tokens", [64])
17
-@pytest.mark.parametrize("num_logprobs", [5])
18
-def test_models(
19
- hf_runner,
20
- vllm_runner,
21
- example_prompts,
22
- model: str,
23
- dtype: str,
24
- max_tokens: int,
25
- num_logprobs: int,
26
-) -> None:
27
- with hf_runner(model, dtype=dtype) as hf_model:
28
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
29
- example_prompts, max_tokens, num_logprobs)
30
-
31
- with vllm_runner(model, dtype=dtype) as vllm_model:
32
- vllm_outputs = vllm_model.generate_greedy_logprobs(
33
- example_prompts, max_tokens, num_logprobs)
34
- check_logprobs_close(
35
- outputs_0_lst=hf_outputs,
36
- outputs_1_lst=vllm_outputs,
37
- name_0="hf",
38
- name_1="vllm",
39
- )
tests/models/decoder_only/language/test_modelopt.py
+1
@@ -39,6 +39,7 @@ EXPECTED_STRS_MAP = {
39
@pytest.mark.skip(
40
reason=
41
"Prevent unstable test based on golden strings from breaking the build.")
42
+@pytest.mark.quant_model
43
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
44
reason="fp8 is not supported on this GPU type.")
45
@pytest.mark.parametrize("model_name", MODELS)
tests/models/decoder_only/language/test_models.py
+1
-3
@@ -1,8 +1,5 @@
1
"""Compare the outputs of HF and vLLM when using greedy sampling.
2
3
-This test only tests small models. Big models such as 7B should be tested from
4
-test_big_models.py because it could use a larger instance to run tests.
5
-
3
Run `pytest tests/models/test_models.py`.
4
"""
5
import pytest
@@ -35,6 +32,7 @@ if not current_platform.is_cpu():
32
target_dtype = "half"
33
34
35
+@pytest.mark.core_model
36
@pytest.mark.parametrize("model", MODELS)
37
@pytest.mark.parametrize("dtype", [target_dtype])
38
@pytest.mark.parametrize("max_tokens", [32])
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_llava_next.py
+3
-1
@@ -56,11 +56,13 @@ def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
56
ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
57
seq_len = 5000 # bigger than the max feature size for any image
58
59
- seq_data, mm_data = dummy_data_for_llava_next(
59
+ dummy_data = dummy_data_for_llava_next(
60
ctx,
61
seq_len=seq_len,
62
mm_counts={"image": 1},
63
)
64
+ seq_data = dummy_data.seq_data
65
+ mm_data = dummy_data.multi_modal_data
66
67
# The dummy data dims should match the gridpoint with the biggest feat size
68
assert mm_data["image"].height == expected_size[0]
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2
-1
@@ -131,12 +131,13 @@ def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
131
mm_processor_kwargs=None,
132
)
133
134
- sequence_data, _, = dummy_data_for_phi3v(
134
+ dummy_data = dummy_data_for_phi3v(
135
ctx=ctx,
136
seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137
mm_counts={"image": num_imgs},
138
num_crops=num_crops,
139
)
140
+ sequence_data = dummy_data.seq_data
141
# Ensure we have the right number of placeholders per num_crops size
142
img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
143
assert img_tok_count == toks_per_img * num_imgs
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+11
-4
@@ -86,10 +86,17 @@ def test_qwen2_vl_dummy_data(dummy_data_for_qwen2_vl,
86
87
# NOTE: video value is required, but isn't actually used
88
# when making the dummy data except for error handling currently
89
- seq_data, mm_data = dummy_data_for_qwen2_vl(qwen2_vl_context, seq_len, {
90
- "image": 1,
91
- "video": 0
92
- }, **mm_processor_kwargs)
89
+ dummy_data = dummy_data_for_qwen2_vl(
90
+ ctx=qwen2_vl_context,
91
+ seq_len=seq_len,
92
+ mm_counts={
93
+ "image": 1,
94
+ "video": 0
95
+ },
96
+ **mm_processor_kwargs,
97
+ )
98
+ seq_data = dummy_data.seq_data
99
+ mm_data = dummy_data.multi_modal_data
100
101
# Ensure we have the right number of placeholders for min/max pixel values
102
assert seq_data.get_token_ids().count(image_token_id) == token_count
tests/models/decoder_only/vision_language/test_awq.py
renamed
+11
-8
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple, Type
1
+from typing import List, Optional, Type
2
3
import pytest
4
import torch
@@ -19,7 +19,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19
def run_awq_test(
20
vllm_runner: Type[VllmRunner],
21
image_assets: _ImageAssets,
22
- models: Tuple[str, str],
22
+ source_model: str,
23
+ quant_model: str,
24
*,
25
size_factors: List[float],
26
dtype: str,
@@ -28,8 +29,6 @@ def run_awq_test(
29
tensor_parallel_size: int,
30
distributed_executor_backend: Optional[str] = None,
31
):
31
- source_model, quant_model = models
32
-
32
images = [asset.pil_image for asset in image_assets]
33
34
inputs_per_image = [(
@@ -84,8 +83,11 @@ def run_awq_test(
83
)
84
85
86
+@pytest.mark.quant_model
87
@pytest.mark.parametrize(
88
- "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
88
+ ("source_model", "quant_model"),
89
+ [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
90
+)
91
@pytest.mark.parametrize(
92
"size_factors",
93
[
@@ -103,12 +105,13 @@ def run_awq_test(
105
@pytest.mark.parametrize("max_tokens", [128])
106
@pytest.mark.parametrize("num_logprobs", [5])
107
@torch.inference_mode()
106
-def test_awq_models(vllm_runner, image_assets, models, size_factors,
107
- dtype: str, max_tokens: int, num_logprobs: int) -> None:
108
+def test_awq_models(vllm_runner, image_assets, source_model, quant_model,
109
+ size_factors, dtype, max_tokens, num_logprobs) -> None:
110
run_awq_test(
111
vllm_runner,
112
image_assets,
111
- models,
113
+ source_model,
114
+ quant_model,
115
size_factors=size_factors,
116
dtype=dtype,
117
max_tokens=max_tokens,
tests/models/decoder_only/vision_language/test_intern_vit.py
+9
-10
@@ -11,21 +11,17 @@ from ....conftest import _ImageAssets
11
# we use snapshot_download to prevent conflicts between
12
# dynamic_module and trust_remote_code for hf_runner
13
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
14
-models = [
15
- snapshot_download("OpenGVLab/InternViT-300M-448px",
16
- allow_patterns=DOWNLOAD_PATTERN),
17
- snapshot_download("OpenGVLab/InternViT-6B-448px-V1-5",
18
- allow_patterns=DOWNLOAD_PATTERN),
19
-]
14
15
16
def run_intern_vit_test(
17
image_assets: _ImageAssets,
24
- model: str,
18
+ model_id: str,
19
*,
20
dtype: str,
21
distributed_executor_backend: Optional[str] = None,
22
):
23
+ model = snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
24
+
25
img_processor = CLIPImageProcessor.from_pretrained(model)
26
images = [asset.pil_image for asset in image_assets]
27
pixel_values = [
@@ -67,12 +63,15 @@ def run_intern_vit_test(
63
assert cos_similar(vllm_output, hf_output).mean() > 0.99
64
65
70
-@pytest.mark.parametrize("model", models)
66
+@pytest.mark.parametrize("model_id", [
67
+ "OpenGVLab/InternViT-300M-448px",
68
+ "OpenGVLab/InternViT-6B-448px-V1-5",
69
+])
70
@pytest.mark.parametrize("dtype", [torch.half])
71
@torch.inference_mode()
73
-def test_models(dist_init, image_assets, model, dtype: str) -> None:
72
+def test_models(dist_init, image_assets, model_id, dtype: str) -> None:
73
run_intern_vit_test(
74
image_assets,
76
- model,
75
+ model_id,
76
dtype=dtype,
77
)
tests/models/decoder_only/vision_language/test_models.py
+23
-23
@@ -130,8 +130,8 @@ VLM_TEST_SETTINGS = {
130
max_num_seqs=2,
131
auto_cls=AutoModelForVision2Seq,
132
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
133
- marks=[pytest.mark.core_model, pytest.mark.cpu_model],
133
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
134
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
135
),
136
#### Extended model tests
137
"blip2": VLMTestInfo(
@@ -159,9 +159,9 @@ VLM_TEST_SETTINGS = {
159
dtype="bfloat16",
160
marks=[
161
pytest.mark.skipif(
162
- transformers.__version__.startswith("4.46"),
162
+ transformers.__version__ < "4.46.2",
163
reason="Model broken in HF, see huggingface/transformers#34379"
164
- )
164
+ ),
165
]
166
),
167
"fuyu": VLMTestInfo(
@@ -185,8 +185,8 @@ VLM_TEST_SETTINGS = {
185
max_num_seqs=2,
186
dtype="bfloat16",
187
get_stop_token_ids=lambda tok: [151329, 151336, 151338],
188
- marks=[large_gpu_mark(min_gb=48)],
188
patch_hf_runner=model_utils.glm_patch_hf_runner,
189
+ marks=[large_gpu_mark(min_gb=48)],
190
),
191
"h2ovl": VLMTestInfo(
192
models = [
@@ -205,6 +205,22 @@ VLM_TEST_SETTINGS = {
205
use_tokenizer_eos=True,
206
patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
207
),
208
+ "idefics3": VLMTestInfo(
209
+ models=["HuggingFaceM4/Idefics3-8B-Llama3"],
210
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
211
+ prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
212
+ img_idx_to_prompt=lambda idx: "<image>",
213
+ max_model_len=8192,
214
+ max_num_seqs=2,
215
+ auto_cls=AutoModelForVision2Seq,
216
+ marks=[
217
+ pytest.mark.skipif(
218
+ transformers.__version__ < "4.46.0",
219
+ reason="Model introduced in HF >= 4.46.0"
220
+ ),
221
+ large_gpu_mark(min_gb=48),
222
+ ],
223
+ ),
224
"intern_vl": VLMTestInfo(
225
models=[
226
"OpenGVLab/InternVL2-1B",
@@ -263,7 +279,6 @@ VLM_TEST_SETTINGS = {
279
runner_mm_key="videos",
280
)],
281
),
266
- # FIXME
282
"llava_next_video": VLMTestInfo(
283
models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
284
test_type=VLMTestType.VIDEO,
@@ -275,7 +290,7 @@ VLM_TEST_SETTINGS = {
290
image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
291
marks=[
292
pytest.mark.skipif(
278
- transformers.__version__.startswith("4.46"),
293
+ transformers.__version__ < "4.46.2",
294
reason="Model broken with changes in transformers 4.46"
295
)
296
],
@@ -316,6 +331,7 @@ VLM_TEST_SETTINGS = {
331
max_model_len=8192,
332
max_num_seqs=2,
333
auto_cls=AutoModelForVision2Seq,
334
+ marks=[large_gpu_mark(min_gb=48)],
335
),
336
"qwen": VLMTestInfo(
337
models=["Qwen/Qwen-VL"],
@@ -327,22 +343,6 @@ VLM_TEST_SETTINGS = {
343
vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
344
prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
345
),
330
- "idefics3": VLMTestInfo(
331
- models=["HuggingFaceM4/Idefics3-8B-Llama3"],
332
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
333
- prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
334
- img_idx_to_prompt=lambda idx: "<image>",
335
- max_model_len=8192,
336
- max_num_seqs=2,
337
- auto_cls=AutoModelForVision2Seq,
338
- marks=[
339
- pytest.mark.skipif(
340
- transformers.__version__ < "4.46.0",
341
- reason="Model introduced in HF >= 4.46.0"
342
- ),
343
- large_gpu_mark(min_gb=48),
344
- ],
345
- ),
346
### Tensor parallel / multi-gpu broadcast tests
347
"broadcast-chameleon": VLMTestInfo(
348
models=["facebook/chameleon-7b"],
@@ -362,7 +362,7 @@ VLM_TEST_SETTINGS = {
362
reason="Need at least 2 GPUs to run the test.",
363
),
364
pytest.mark.skipif(
365
- transformers.__version__.startswith("4.46"),
365
+ transformers.__version__ < "4.46.2",
366
reason="Model broken in HF, see huggingface/transformers#34379"
367
)
368
],
vllm/config.py
+8
-1
@@ -1,7 +1,8 @@
1
+import copy
2
import enum
3
import json
4
import warnings
4
-from dataclasses import dataclass, field
5
+from dataclasses import dataclass, field, replace
6
from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
7
Mapping, Optional, Set, Tuple, Type, Union)
8
@@ -2078,6 +2079,12 @@ class VllmConfig:
2079
return quant_config
2080
return None
2081
2082
+ def with_hf_config(self, hf_config: PretrainedConfig) -> "VllmConfig":
2083
+ model_config = copy.deepcopy(self.model_config)
2084
+ model_config.hf_config = hf_config
2085
+
2086
+ return replace(self, model_config=model_config)
2087
+
2088
def __post_init__(self):
2089
"""Verify configs are valid & consistent with each other.
2090
"""
vllm/model_executor/models/fuyu.py
+2
-4
@@ -229,7 +229,6 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
229
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
230
super().__init__()
231
config = vllm_config.model_config.hf_config
232
- cache_config = vllm_config.cache_config
232
quant_config = vllm_config.quant_config
233
multimodal_config = vllm_config.model_config.multimodal_config
234
self.config = config
@@ -246,9 +245,8 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
245
quant_config=quant_config,
246
gather_output=True,
247
)
249
- self.language_model = PersimmonForCausalLM(config.text_config,
250
- cache_config=cache_config,
251
- quant_config=quant_config)
248
+ self.language_model = PersimmonForCausalLM(
249
+ vllm_config.with_hf_config(config.text_config))
250
self.make_empty_intermediate_tensors = (
251
self.language_model.make_empty_intermediate_tensors)
252
vllm/model_executor/models/internlm2_ve.py
+3
-1
@@ -164,10 +164,12 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
164
vllm_config: VllmConfig,
165
prefix: str = "",
166
) -> None:
167
+ super().__init__(vllm_config, prefix=prefix)
168
+
169
config = vllm_config.model_config.hf_config
170
cache_config = vllm_config.cache_config
171
quant_config = vllm_config.quant_config
170
- super().__init__(config, cache_config, quant_config)
172
+
173
self.model = InternLM2VEModel(config,
174
cache_config,
175
quant_config,
vllm/model_executor/models/utils.py
+4
-4
@@ -241,11 +241,11 @@ def init_vllm_registered_model(
241
based on the arguments passed to the outer vLLM model.
242
"""
243
model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
244
- import copy
245
- copied_config = copy.deepcopy(vllm_config)
246
- copied_config.model_config.hf_config = hf_config
244
248
- return model_class(vllm_config=copied_config, prefix=prefix)
245
+ return model_class(
246
+ vllm_config=vllm_config.with_hf_config(hf_config),
247
+ prefix=prefix,
248
+ )
249
250
251
@overload