@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9fb900f90c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Test] improve robustness of test (hf_runner) (#5347)

[CI/Test] improve robustness of test by replacing del with context manager (hf_runner) (#5347)

youkaichao committed Jun 7, 2024 at 22:31 UTC 9fb900f90cbb5614c3e7d67446325ad8b7ac04b2
14 files changed +48 -61
tests/basic_correctness/test_basic_correctness.py
+2 -3
@@ -43,9 +43,8 @@ def test_models(
43 if backend_by_env_var == "FLASHINFER" and enforce_eager is False:
44 pytest.skip("Skipping non-eager test for FlashInferBackend.")
45
46 - hf_model = hf_runner(model, dtype=dtype)
47 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48 - del hf_model
46 + with hf_runner(model, dtype=dtype) as hf_model:
47 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49 vllm_model = vllm_runner(model,
50 dtype=dtype,
tests/basic_correctness/test_chunked_prefill.py
+2 -3
@@ -40,9 +40,8 @@ def test_models(
40 enable_chunked_prefill = True
41 max_num_batched_tokens = chunked_prefill_token_size
42
43 - hf_model = hf_runner(model, dtype=dtype)
44 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45 - del hf_model
43 + with hf_runner(model, dtype=dtype) as hf_model:
44 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
46 vllm_model = vllm_runner(
47 model,
tests/basic_correctness/test_preemption.py
+7 -10
@@ -43,9 +43,8 @@ def test_chunked_prefill_recompute(
43 enable_chunked_prefill = True
44 max_num_batched_tokens = chunked_prefill_token_size
45
46 - hf_model = hf_runner(model, dtype=dtype)
47 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48 - del hf_model
46 + with hf_runner(model, dtype=dtype) as hf_model:
47 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49 vllm_model = vllm_runner(
50 model,
@@ -82,9 +81,8 @@ def test_preemption(
81 ) -> None:
82 """By default, recompute preemption is enabled"""
83
85 - hf_model = hf_runner(model, dtype=dtype)
86 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
87 - del hf_model
84 + with hf_runner(model, dtype=dtype) as hf_model:
85 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
86
87 vllm_model = vllm_runner(
88 model,
@@ -137,10 +135,9 @@ def test_swap(
135 ) -> None:
136 """Use beam search enables swapping."""
137 example_prompts = example_prompts[:1]
140 - hf_model = hf_runner(model, dtype=dtype)
141 - hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
142 - max_tokens)
143 - del hf_model
138 + with hf_runner(model, dtype=dtype) as hf_model:
139 + hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
140 + max_tokens)
141
142 vllm_model = vllm_runner(
143 model,
tests/conftest.py
+4 -1
@@ -354,7 +354,10 @@ class HfRunner:
354 def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:
355 return self.model.encode(prompts)
356
357 - def __del__(self):
357 + def __enter__(self):
358 + return self
359 +
360 + def __exit__(self, exc_type, exc_value, traceback):
361 del self.model
362 cleanup()
363
tests/distributed/test_basic_distributed_correctness.py
+2 -3
@@ -42,9 +42,8 @@ def test_models(
42 backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43 enforce_eager = backend_by_env_var == "FLASHINFER"
44
45 - hf_model = hf_runner(model, dtype=dtype)
46 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47 - del hf_model
45 + with hf_runner(model, dtype=dtype) as hf_model:
46 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
48 vllm_model = vllm_runner(
49 model,
tests/distributed/test_chunked_prefill_distributed.py
+2 -3
@@ -45,9 +45,8 @@ def test_models(
45 enable_chunked_prefill = True
46 max_num_batched_tokens = chunked_prefill_token_size
47
48 - hf_model = hf_runner(model, dtype=dtype)
49 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50 - del hf_model
48 + with hf_runner(model, dtype=dtype) as hf_model:
49 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50
51 vllm_model = vllm_runner(
52 model,
tests/models/test_big_models.py
+2 -3
@@ -34,9 +34,8 @@ def test_models(
34 dtype: str,
35 max_tokens: int,
36 ) -> None:
37 - hf_model = hf_runner(model, dtype=dtype)
38 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39 - del hf_model
37 + with hf_runner(model, dtype=dtype) as hf_model:
38 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40 vllm_model = vllm_runner(model, dtype=dtype)
41 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
tests/models/test_embedding.py
+2 -3
@@ -28,9 +28,8 @@ def test_models(
28 model: str,
29 dtype: str,
30 ) -> None:
31 - hf_model = hf_runner(model, dtype=dtype, is_embedding_model=True)
32 - hf_outputs = hf_model.encode(example_prompts)
33 - del hf_model
31 + with hf_runner(model, dtype=dtype, is_embedding_model=True) as hf_model:
32 + hf_outputs = hf_model.encode(example_prompts)
33
34 vllm_model = vllm_runner(model, dtype=dtype)
35 vllm_outputs = vllm_model.encode(example_prompts)
tests/models/test_llava.py
+4 -5
@@ -84,11 +84,10 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
84 """
85 model_id, vlm_config = model_and_config
86
87 - hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
88 - hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89 - max_tokens,
90 - images=hf_images)
91 - del hf_model
87 + with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
88 + hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89 + max_tokens,
90 + images=hf_images)
91
92 vllm_image_prompts = [
93 p.replace("<image>", "<image>" * vlm_config.image_feature_size)
tests/models/test_mistral.py
+3 -4
@@ -26,10 +26,9 @@ def test_models(
26 num_logprobs: int,
27 ) -> None:
28 # TODO(sang): Sliding window should be tested separately.
29 - hf_model = hf_runner(model, dtype=dtype)
30 - hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 - example_prompts, max_tokens, num_logprobs)
32 - del hf_model
29 + with hf_runner(model, dtype=dtype) as hf_model:
30 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 + example_prompts, max_tokens, num_logprobs)
32
33 vllm_model = vllm_runner(model, dtype=dtype)
34 vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
tests/models/test_models.py
+2 -3
@@ -34,9 +34,8 @@ def test_models(
34 # To pass the small model tests, we need full precision.
35 assert dtype == "float"
36
37 - hf_model = hf_runner(model, dtype=dtype)
38 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39 - del hf_model
37 + with hf_runner(model, dtype=dtype) as hf_model:
38 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40 vllm_model = vllm_runner(model, dtype=dtype)
41 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
tests/samplers/test_beam_search.py
+3 -4
@@ -30,10 +30,9 @@ def test_beam_search_single_input(
30 beam_width: int,
31 ) -> None:
32 example_prompts = example_prompts[:1]
33 - hf_model = hf_runner(model, dtype=dtype)
34 - hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
35 - max_tokens)
36 - del hf_model
33 + with hf_runner(model, dtype=dtype) as hf_model:
34 + hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
35 + max_tokens)
36
37 vllm_model = vllm_runner(model, dtype=dtype)
38 vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
tests/samplers/test_logprobs.py
+5 -6
@@ -32,12 +32,11 @@ def test_get_prompt_logprobs(
32 max_num_batched_tokens = chunked_prefill_token_size
33
34 max_tokens = 5
35 - hf_model = hf_runner(model, dtype=dtype)
36 - hf_logprobs = hf_model.generate_greedy_logprobs(
37 - example_prompts,
38 - max_tokens=max_tokens,
39 - )
40 - del hf_model
35 + with hf_runner(model, dtype=dtype) as hf_model:
36 + hf_logprobs = hf_model.generate_greedy_logprobs(
37 + example_prompts,
38 + max_tokens=max_tokens,
39 + )
40
41 vllm_model = vllm_runner(
42 model,
tests/tensorizer_loader/test_tensorizer.py
+8 -10
@@ -116,16 +116,14 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
116
117 def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
118 tmp_path):
119 - hf_model = hf_runner(model_ref)
120 - model_path = tmp_path / (model_ref + ".tensors")
121 - max_tokens = 50
122 - outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
123 - with open_stream(model_path, "wb+") as stream:
124 - serializer = TensorSerializer(stream)
125 - serializer.write_module(hf_model.model)
126 - del hf_model
127 - gc.collect()
128 - torch.cuda.empty_cache()
119 + with hf_runner(model_ref) as hf_model:
120 + model_path = tmp_path / (model_ref + ".tensors")
121 + max_tokens = 50
122 + outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
123 + with open_stream(model_path, "wb+") as stream:
124 + serializer = TensorSerializer(stream)
125 + serializer.write_module(hf_model.model)
126 +
127 loaded_hf_model = vllm_runner(model_ref,
128 load_format="tensorizer",
129 model_loader_extra_config=TensorizerConfig(