@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b377d6feb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[BugFix] Fix test breakages from transformers 4.45 upgrade (#8829)

Nick Hill committed Sep 27, 2024 at 00:46 UTC 4b377d6febed7ddd964f1b96079d7e78c231325e
13 files changed +62 -49
.buildkite/test-pipeline.yaml
+3 -6
@@ -83,7 +83,6 @@ steps:
83
84 - label: Entrypoints Test # 20min
85 working_dir: "/vllm-workspace/tests"
86 - soft_fail: true
86 fast_check: true
87 mirror_hardwares: [amd]
88 source_file_dependencies:
@@ -96,7 +95,8 @@ steps:
95 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
96 - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
97 - pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
99 - - pytest -v -s entrypoints/openai
98 + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py
99 + - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
100 - pytest -v -s entrypoints/test_chat_utils.py
101 - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
102
@@ -178,7 +178,6 @@ steps:
178 - pytest -v -s prefix_caching
179
180 - label: Samplers Test # 18min
181 - soft_fail: true
181 source_file_dependencies:
182 - vllm/model_executor/layers
183 - vllm/sampling_metadata.py
@@ -206,7 +205,6 @@ steps:
205
206 - label: LoRA Test %N # 30min each
207 mirror_hardwares: [amd]
209 - soft_fail: true
208 source_file_dependencies:
209 - vllm/lora
210 - tests/lora
@@ -311,7 +309,6 @@ steps:
309 - pytest -v -s models/decoder_only/language
310
311 - label: Decoder-only Multi-Modal Models Test # 56min
314 - soft_fail: true
312 #mirror_hardwares: [amd]
313 source_file_dependencies:
314 - vllm/
@@ -463,7 +460,7 @@ steps:
460 # NOTE: don't test llama model here, it seems hf implementation is buggy
461 # see https://github.com/vllm-project/vllm/pull/5689 for details
462 - pytest -v -s distributed/test_custom_all_reduce.py
466 - - TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
463 + - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
464 - pytest -v -s -x lora/test_mixtral.py
465
466 - label: LM Eval Large Models # optional
tests/conftest.py
-1
@@ -699,7 +699,6 @@ class VllmRunner:
699 if videos is not None:
700 for i, video in enumerate(videos):
701 inputs[i]["multi_modal_data"] = {"video": video}
702 - print(f"[INPUTS!!!!]: {inputs}, {sampling_params}")
702
703 req_outputs = self.model.generate(inputs,
704 sampling_params=sampling_params)
tests/distributed/test_pipeline_parallel.py
-7
@@ -8,8 +8,6 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
8 import os
9
10 import pytest
11 -from packaging import version
12 -from transformers import __version__ as transformers_version
11
12 from vllm.logger import init_logger
13
@@ -49,11 +47,6 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
47 pytest.skip("Skipping multi-node pipeline parallel test for "
48 "multiprocessing distributed backend")
49
52 - # Skip tests that require transformers>=4.45.0
53 - if "Qwen2-VL" in MODEL_NAME and version.parse(
54 - transformers_version) < version.parse("4.45.0.dev0"):
55 - pytest.skip("This test requires transformers>=4.45.0")
56 -
50 pp_args = [
51 # use half precision for speed and memory savings in CI environment
52 "--dtype",
tests/engine/test_custom_executor.py
+4 -4
@@ -48,9 +48,9 @@ def test_custom_executor_type_checking(model):
48
49
50 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
51 -def test_custom_executor(model, tmpdir):
51 +def test_custom_executor(model, tmp_path):
52 cwd = os.path.abspath(".")
53 - os.chdir(tmpdir)
53 + os.chdir(tmp_path)
54 try:
55 assert not os.path.exists(".marker")
56
@@ -68,9 +68,9 @@ def test_custom_executor(model, tmpdir):
68
69
70 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
71 -def test_custom_executor_async(model, tmpdir):
71 +def test_custom_executor_async(model, tmp_path):
72 cwd = os.path.abspath(".")
73 - os.chdir(tmpdir)
73 + os.chdir(tmp_path)
74 try:
75 assert not os.path.exists(".marker")
76
tests/entrypoints/openai/test_serving_chat.py
+6
@@ -15,6 +15,11 @@ CHAT_TEMPLATE = "Dummy chat template for testing {}"
15 BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
16
17
18 +@dataclass
19 +class MockHFConfig:
20 + model_type: str = "any"
21 +
22 +
23 @dataclass
24 class MockModelConfig:
25 tokenizer = MODEL_NAME
@@ -24,6 +29,7 @@ class MockModelConfig:
29 tokenizer_revision = None
30 embedding_mode = False
31 multimodal_config = MultiModalConfig()
32 + hf_config = MockHFConfig()
33
34
35 @dataclass
tests/lora/test_tokenizer_group.py
+2 -2
@@ -41,7 +41,7 @@ async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
41 lora_request)
42
43
44 -def test_get_lora_tokenizer(sql_lora_files, tmpdir):
44 +def test_get_lora_tokenizer(sql_lora_files, tmp_path):
45 lora_request = None
46 tokenizer = get_lora_tokenizer(lora_request)
47 assert not tokenizer
@@ -50,6 +50,6 @@ def test_get_lora_tokenizer(sql_lora_files, tmpdir):
50 tokenizer = get_lora_tokenizer(lora_request)
51 assert tokenizer.get_added_vocab()
52
53 - lora_request = LoRARequest("1", 1, str(tmpdir))
53 + lora_request = LoRARequest("1", 1, str(tmp_path))
54 tokenizer = get_lora_tokenizer(lora_request)
55 assert not tokenizer
tests/models/decoder_only/language/test_granite.py
-4
@@ -3,7 +3,6 @@
3 Run `pytest tests/models/test_granite.py`.
4 """
5 import pytest
6 -import transformers
6
7 from ...utils import check_logprobs_close
8
@@ -12,9 +11,6 @@ MODELS = [
11 ]
12
13
15 -# GraniteForCausalLM will be in transformers >= 4.45
16 -@pytest.mark.skipif(transformers.__version__ < "4.45",
17 - reason="granite model test requires transformers >= 4.45")
14 @pytest.mark.parametrize("model", MODELS)
15 @pytest.mark.parametrize("dtype", ["bfloat16"])
16 @pytest.mark.parametrize("max_tokens", [64])
tests/models/decoder_only/vision_language/test_llava_next_video.py
-5
@@ -1,7 +1,6 @@
1 from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 -import transformers
4 from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6 from vllm.multimodal.utils import (rescale_video_size, resize_video,
@@ -158,8 +157,6 @@ def run_test(
157 )
158
159
161 -@pytest.mark.skipif(transformers.__version__ < "4.45",
162 - reason="Waiting for next transformers release")
160 @pytest.mark.parametrize("model", models)
161 @pytest.mark.parametrize(
162 "size_factors",
@@ -203,8 +200,6 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
200 )
201
202
206 -@pytest.mark.skipif(transformers.__version__ < "4.45",
207 - reason="Waiting for next transformers release")
203 @pytest.mark.parametrize("model", models)
204 @pytest.mark.parametrize(
205 "sizes",
tests/models/decoder_only/vision_language/test_llava_onevision.py
+5 -8
@@ -1,7 +1,6 @@
1 from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 -import transformers
4 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 BatchEncoding)
6
@@ -166,8 +165,6 @@ def run_video_test(
165 )
166
167
169 -@pytest.mark.skipif(transformers.__version__ < "4.45",
170 - reason="Waiting for next transformers release")
168 @pytest.mark.parametrize("model", models)
169 @pytest.mark.parametrize(
170 "size_factors",
@@ -211,8 +208,6 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
208 )
209
210
214 -@pytest.mark.skipif(transformers.__version__ < "4.45",
215 - reason="Waiting for next transformers release")
211 @pytest.mark.parametrize("model", models)
212 @pytest.mark.parametrize(
213 "sizes",
@@ -259,7 +254,9 @@ def run_image_test(
254 # max_model_len should be greater than image_feature_size
255 with vllm_runner(model,
256 dtype=dtype,
262 - max_model_len=32768,
257 + max_num_seqs=1,
258 + max_model_len=16384,
259 + gpu_memory_utilization=0.98,
260 tensor_parallel_size=tensor_parallel_size,
261 distributed_executor_backend=distributed_executor_backend,
262 enforce_eager=True,
@@ -305,8 +302,8 @@ def run_image_test(
302 )
303
304
308 -@pytest.mark.skipif(transformers.__version__ < "4.45",
309 - reason="Waiting for next transformers release")
305 +# FIXME: Swap to a smaller model for this architecture
306 +@pytest.mark.skip(reason="Model OOMing on CI")
307 @pytest.mark.parametrize("model", models)
308 @pytest.mark.parametrize("dtype", ["half"])
309 @pytest.mark.parametrize("max_tokens", [128])
tests/models/test_registry.py
-6
@@ -1,15 +1,9 @@
1 import pytest
2 -import transformers
2
3 from vllm.model_executor.models import _MODELS, ModelRegistry
4
5
6 @pytest.mark.parametrize("model_cls", _MODELS)
7 def test_registry_imports(model_cls):
9 - if (model_cls in ("LlavaOnevisionForConditionalGeneration",
10 - "Qwen2VLForConditionalGeneration")
11 - and transformers.__version__ < "4.45"):
12 - pytest.skip("Waiting for next transformers release")
13 -
8 # Ensure all model classes can be imported successfully
9 ModelRegistry.resolve_model_cls([model_cls])
tests/samplers/test_sampler.py
+15 -3
@@ -1,5 +1,6 @@
1 import itertools
2 import random
3 +from dataclasses import dataclass
4 from typing import Dict, List, Optional, Tuple
5 from unittest.mock import Mock, patch
6
@@ -596,8 +597,19 @@ def test_sampler_top_k_top_p(seed: int, device: str):
597 generation_config = GenerationConfig(top_k=top_k,
598 top_p=top_p,
599 do_sample=True)
599 - warpers = generation_model._get_logits_warper(generation_config, device)
600 - assert len(warpers) == 2 # top_p and top_k
600 +
601 + @dataclass
602 + class MockConfig:
603 + is_encoder_decoder: bool = False
604 +
605 + generation_model.config = MockConfig() # needed by the following method
606 + generation_model._prepare_special_tokens(generation_config, device=device)
607 + processors = generation_model._get_logits_processor(generation_config,
608 + None,
609 + None,
610 + None, [],
611 + device=device)
612 + assert len(processors) == 2 # top_p and top_k
613
614 seq_group_metadata_list: List[SequenceGroupMetadata] = []
615 seq_lens: List[int] = []
@@ -639,7 +651,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
651
652 assert sample_probs is not None
653
642 - hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
654 + hf_probs = processors(torch.zeros_like(fake_logits), fake_logits.clone())
655 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
656 torch.testing.assert_close(hf_probs, sample_probs, rtol=0.0, atol=1e-5)
657 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -152,13 +152,13 @@ class OpenAIServingChat(OpenAIServing):
152 **(request.chat_template_kwargs or {}),
153 )
154 except Exception as e:
155 - logger.error("Error in applying chat template from request: %s", e)
155 + logger.exception("Error in applying chat template from request")
156 return self.create_error_response(str(e))
157
158 try:
159 mm_data = await mm_data_future
160 except Exception as e:
161 - logger.error("Error in loading multi-modal data: %s", e)
161 + logger.exception("Error in loading multi-modal data")
162 return self.create_error_response(str(e))
163
164 # validation for OpenAI tools
vllm/transformers_utils/tokenizer.py
+25 -1
@@ -1,6 +1,7 @@
1 import os
2 import warnings
3 from pathlib import Path
4 +from types import MethodType
5 from typing import Optional, Union
6
7 import huggingface_hub
@@ -152,6 +153,29 @@ def get_tokenizer(
153 else:
154 raise e
155
156 + # NOTE: We can remove this after https://github.com/THUDM/ChatGLM3/issues/1324
157 + if type(tokenizer).__name__ in ("ChatGLMTokenizer",
158 + "ChatGLM4Tokenizer"):
159 + assert isinstance(tokenizer, PreTrainedTokenizer)
160 + orig_pad = tokenizer._pad
161 +
162 + # Patch _pad method to accept `padding_side`
163 + def _pad(
164 + self: PreTrainedTokenizer,
165 + *args,
166 + padding_side: Optional[str] = None,
167 + **kwargs,
168 + ):
169 + if (padding_side is not None
170 + and padding_side != self.padding_side):
171 + msg = ("`padding_side` argument is not supported by "
172 + "ChatGLMTokenizer and will be ignored.")
173 + warnings.warn(msg, stacklevel=2)
174 +
175 + return orig_pad(*args, **kwargs)
176 +
177 + tokenizer._pad = MethodType(_pad, tokenizer)
178 +
179 if not isinstance(tokenizer, PreTrainedTokenizerFast):
180 logger.warning(
181 "Using a slow tokenizer. This might cause a significant "
@@ -167,7 +191,7 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
191 return None
192 try:
193 tokenizer = get_tokenizer(lora_request.lora_path, *args, **kwargs)
170 - except OSError as e:
194 + except Exception as e:
195 # No tokenizer was found in the LoRA folder,
196 # use base model tokenizer
197 logger.warning(