A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Test] improve robustness of test (hf_runner) (#5347)
[CI/Test] improve robustness of test by replacing del with context manager (hf_runner) (#5347)
youkaichao committed
Jun 7, 2024 at 22:31 UTC
9fb900f90cbb5614c3e7d67446325ad8b7ac04b2
14 files changed
+48
-61
tests/basic_correctness/test_basic_correctness.py
+2
-3
@@ -43,9 +43,8 @@ def test_models(
43
if backend_by_env_var == "FLASHINFER" and enforce_eager is False:
44
pytest.skip("Skipping non-eager test for FlashInferBackend.")
45
46
- hf_model = hf_runner(model, dtype=dtype)
47
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
- del hf_model
46
+ with hf_runner(model, dtype=dtype) as hf_model:
47
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49
vllm_model = vllm_runner(model,
50
dtype=dtype,
tests/basic_correctness/test_chunked_prefill.py
+2
-3
@@ -40,9 +40,8 @@ def test_models(
40
enable_chunked_prefill = True
41
max_num_batched_tokens = chunked_prefill_token_size
42
43
- hf_model = hf_runner(model, dtype=dtype)
44
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
- del hf_model
43
+ with hf_runner(model, dtype=dtype) as hf_model:
44
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
46
vllm_model = vllm_runner(
47
model,
tests/basic_correctness/test_preemption.py
+7
-10
@@ -43,9 +43,8 @@ def test_chunked_prefill_recompute(
43
enable_chunked_prefill = True
44
max_num_batched_tokens = chunked_prefill_token_size
45
46
- hf_model = hf_runner(model, dtype=dtype)
47
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
- del hf_model
46
+ with hf_runner(model, dtype=dtype) as hf_model:
47
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49
vllm_model = vllm_runner(
50
model,
@@ -82,9 +81,8 @@ def test_preemption(
81
) -> None:
82
"""By default, recompute preemption is enabled"""
83
85
- hf_model = hf_runner(model, dtype=dtype)
86
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
87
- del hf_model
84
+ with hf_runner(model, dtype=dtype) as hf_model:
85
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
86
87
vllm_model = vllm_runner(
88
model,
@@ -137,10 +135,9 @@ def test_swap(
135
) -> None:
136
"""Use beam search enables swapping."""
137
example_prompts = example_prompts[:1]
140
- hf_model = hf_runner(model, dtype=dtype)
141
- hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
142
- max_tokens)
143
- del hf_model
138
+ with hf_runner(model, dtype=dtype) as hf_model:
139
+ hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
140
+ max_tokens)
141
142
vllm_model = vllm_runner(
143
model,
tests/conftest.py
+4
-1
@@ -354,7 +354,10 @@ class HfRunner:
354
def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:
355
return self.model.encode(prompts)
356
357
- def __del__(self):
357
+ def __enter__(self):
358
+ return self
359
+
360
+ def __exit__(self, exc_type, exc_value, traceback):
361
del self.model
362
cleanup()
363
tests/distributed/test_basic_distributed_correctness.py
+2
-3
@@ -42,9 +42,8 @@ def test_models(
42
backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43
enforce_eager = backend_by_env_var == "FLASHINFER"
44
45
- hf_model = hf_runner(model, dtype=dtype)
46
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
- del hf_model
45
+ with hf_runner(model, dtype=dtype) as hf_model:
46
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
48
vllm_model = vllm_runner(
49
model,
tests/distributed/test_chunked_prefill_distributed.py
+2
-3
@@ -45,9 +45,8 @@ def test_models(
45
enable_chunked_prefill = True
46
max_num_batched_tokens = chunked_prefill_token_size
47
48
- hf_model = hf_runner(model, dtype=dtype)
49
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50
- del hf_model
48
+ with hf_runner(model, dtype=dtype) as hf_model:
49
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50
51
vllm_model = vllm_runner(
52
model,
tests/models/test_big_models.py
+2
-3
@@ -34,9 +34,8 @@ def test_models(
34
dtype: str,
35
max_tokens: int,
36
) -> None:
37
- hf_model = hf_runner(model, dtype=dtype)
38
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
- del hf_model
37
+ with hf_runner(model, dtype=dtype) as hf_model:
38
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40
vllm_model = vllm_runner(model, dtype=dtype)
41
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
tests/models/test_embedding.py
+2
-3
@@ -28,9 +28,8 @@ def test_models(
28
model: str,
29
dtype: str,
30
) -> None:
31
- hf_model = hf_runner(model, dtype=dtype, is_embedding_model=True)
32
- hf_outputs = hf_model.encode(example_prompts)
33
- del hf_model
31
+ with hf_runner(model, dtype=dtype, is_embedding_model=True) as hf_model:
32
+ hf_outputs = hf_model.encode(example_prompts)
33
34
vllm_model = vllm_runner(model, dtype=dtype)
35
vllm_outputs = vllm_model.encode(example_prompts)
tests/models/test_llava.py
+4
-5
@@ -84,11 +84,10 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
84
"""
85
model_id, vlm_config = model_and_config
86
87
- hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
88
- hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89
- max_tokens,
90
- images=hf_images)
91
- del hf_model
87
+ with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
88
+ hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89
+ max_tokens,
90
+ images=hf_images)
91
92
vllm_image_prompts = [
93
p.replace("<image>", "<image>" * vlm_config.image_feature_size)
tests/models/test_mistral.py
+3
-4
@@ -26,10 +26,9 @@ def test_models(
26
num_logprobs: int,
27
) -> None:
28
# TODO(sang): Sliding window should be tested separately.
29
- hf_model = hf_runner(model, dtype=dtype)
30
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
- example_prompts, max_tokens, num_logprobs)
32
- del hf_model
29
+ with hf_runner(model, dtype=dtype) as hf_model:
30
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
+ example_prompts, max_tokens, num_logprobs)
32
33
vllm_model = vllm_runner(model, dtype=dtype)
34
vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
tests/models/test_models.py
+2
-3
@@ -34,9 +34,8 @@ def test_models(
34
# To pass the small model tests, we need full precision.
35
assert dtype == "float"
36
37
- hf_model = hf_runner(model, dtype=dtype)
38
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
- del hf_model
37
+ with hf_runner(model, dtype=dtype) as hf_model:
38
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40
vllm_model = vllm_runner(model, dtype=dtype)
41
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
tests/samplers/test_beam_search.py
+3
-4
@@ -30,10 +30,9 @@ def test_beam_search_single_input(
30
beam_width: int,
31
) -> None:
32
example_prompts = example_prompts[:1]
33
- hf_model = hf_runner(model, dtype=dtype)
34
- hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
35
- max_tokens)
36
- del hf_model
33
+ with hf_runner(model, dtype=dtype) as hf_model:
34
+ hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
35
+ max_tokens)
36
37
vllm_model = vllm_runner(model, dtype=dtype)
38
vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
tests/samplers/test_logprobs.py
+5
-6
@@ -32,12 +32,11 @@ def test_get_prompt_logprobs(
32
max_num_batched_tokens = chunked_prefill_token_size
33
34
max_tokens = 5
35
- hf_model = hf_runner(model, dtype=dtype)
36
- hf_logprobs = hf_model.generate_greedy_logprobs(
37
- example_prompts,
38
- max_tokens=max_tokens,
39
- )
40
- del hf_model
35
+ with hf_runner(model, dtype=dtype) as hf_model:
36
+ hf_logprobs = hf_model.generate_greedy_logprobs(
37
+ example_prompts,
38
+ max_tokens=max_tokens,
39
+ )
40
41
vllm_model = vllm_runner(
42
model,
tests/tensorizer_loader/test_tensorizer.py
+8
-10
@@ -116,16 +116,14 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
116
117
def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
118
tmp_path):
119
- hf_model = hf_runner(model_ref)
120
- model_path = tmp_path / (model_ref + ".tensors")
121
- max_tokens = 50
122
- outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
123
- with open_stream(model_path, "wb+") as stream:
124
- serializer = TensorSerializer(stream)
125
- serializer.write_module(hf_model.model)
126
- del hf_model
127
- gc.collect()
128
- torch.cuda.empty_cache()
119
+ with hf_runner(model_ref) as hf_model:
120
+ model_path = tmp_path / (model_ref + ".tensors")
121
+ max_tokens = 50
122
+ outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
123
+ with open_stream(model_path, "wb+") as stream:
124
+ serializer = TensorSerializer(stream)
125
+ serializer.write_module(hf_model.model)
126
+
127
loaded_hf_model = vllm_runner(model_ref,
128
load_format="tensorizer",
129
model_loader_extra_config=TensorizerConfig(