@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8ea5e44a43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Test] improve robustness of test (vllm_runner) (#5357)

[CI/Test] improve robustness of test by replacing del with context manager (vllm_runner) (#5357)

youkaichao committed Jun 8, 2024 at 01:59 UTC 8ea5e44a435e8731fd6f5ba4c329dd112752532a
28 files changed +455 -494
tests/basic_correctness/test_basic_correctness.py
+5 -6
@@ -46,12 +46,11 @@ def test_models(
46 with hf_runner(model, dtype=dtype) as hf_model:
47 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49 - vllm_model = vllm_runner(model,
50 - dtype=dtype,
51 - enforce_eager=enforce_eager,
52 - gpu_memory_utilization=0.7)
53 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54 - del vllm_model
49 + with vllm_runner(model,
50 + dtype=dtype,
51 + enforce_eager=enforce_eager,
52 + gpu_memory_utilization=0.7) as vllm_model:
53 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
55 for i in range(len(example_prompts)):
56 hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_chunked_prefill.py
+10 -11
@@ -43,17 +43,16 @@ def test_models(
43 with hf_runner(model, dtype=dtype) as hf_model:
44 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
46 - vllm_model = vllm_runner(
47 - model,
48 - dtype=dtype,
49 - max_num_batched_tokens=max_num_batched_tokens,
50 - enable_chunked_prefill=enable_chunked_prefill,
51 - tensor_parallel_size=tensor_parallel_size,
52 - enforce_eager=enforce_eager,
53 - max_num_seqs=max_num_seqs,
54 - )
55 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56 - del vllm_model
46 + with vllm_runner(
47 + model,
48 + dtype=dtype,
49 + max_num_batched_tokens=max_num_batched_tokens,
50 + enable_chunked_prefill=enable_chunked_prefill,
51 + tensor_parallel_size=tensor_parallel_size,
52 + enforce_eager=enforce_eager,
53 + max_num_seqs=max_num_seqs,
54 + ) as vllm_model:
55 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56
57 for i in range(len(example_prompts)):
58 hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_preemption.py
+73 -75
@@ -46,17 +46,16 @@ def test_chunked_prefill_recompute(
46 with hf_runner(model, dtype=dtype) as hf_model:
47 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49 - vllm_model = vllm_runner(
50 - model,
51 - dtype=dtype,
52 - max_num_batched_tokens=max_num_batched_tokens,
53 - enable_chunked_prefill=enable_chunked_prefill,
54 - max_num_seqs=max_num_seqs,
55 - )
56 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
58 - ARTIFICIAL_PREEMPTION_MAX_CNT)
59 - del vllm_model
49 + with vllm_runner(
50 + model,
51 + dtype=dtype,
52 + max_num_batched_tokens=max_num_batched_tokens,
53 + enable_chunked_prefill=enable_chunked_prefill,
54 + max_num_seqs=max_num_seqs,
55 + ) as vllm_model:
56 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
58 + ARTIFICIAL_PREEMPTION_MAX_CNT)
59
60 for i in range(len(example_prompts)):
61 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -84,17 +83,16 @@ def test_preemption(
83 with hf_runner(model, dtype=dtype) as hf_model:
84 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
85
87 - vllm_model = vllm_runner(
88 - model,
89 - dtype=dtype,
90 - disable_log_stats=False,
91 - )
92 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
93 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
94 - ARTIFICIAL_PREEMPTION_MAX_CNT)
95 - total_preemption = (
96 - vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
97 - del vllm_model
86 + with vllm_runner(
87 + model,
88 + dtype=dtype,
89 + disable_log_stats=False,
90 + ) as vllm_model:
91 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
92 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
93 + ARTIFICIAL_PREEMPTION_MAX_CNT)
94 + total_preemption = (
95 + vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
96
97 for i in range(len(example_prompts)):
98 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -139,19 +137,18 @@ def test_swap(
137 hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
138 max_tokens)
139
142 - vllm_model = vllm_runner(
143 - model,
144 - dtype=dtype,
145 - swap_space=10,
146 - disable_log_stats=False,
147 - )
148 - vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
149 - max_tokens)
150 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
151 - ARTIFICIAL_PREEMPTION_MAX_CNT)
152 - total_preemption = (
153 - vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
154 - del vllm_model
140 + with vllm_runner(
141 + model,
142 + dtype=dtype,
143 + swap_space=10,
144 + disable_log_stats=False,
145 + ) as vllm_model:
146 + vllm_outputs = vllm_model.generate_beam_search(example_prompts,
147 + beam_width, max_tokens)
148 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
149 + ARTIFICIAL_PREEMPTION_MAX_CNT)
150 + total_preemption = (
151 + vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
152
153 for i in range(len(example_prompts)):
154 hf_output_ids, _ = hf_outputs[i]
@@ -196,28 +193,28 @@ def test_swap_infeasible(
193 decode_blocks = max_tokens // BLOCK_SIZE
194 example_prompts = example_prompts[:1]
195
199 - vllm_model = vllm_runner(
200 - model,
201 - dtype=dtype,
202 - swap_space=10,
203 - block_size=BLOCK_SIZE,
204 - # Since beam search have more than 1 sequence, prefill + decode blocks
205 - # are not enough to finish.
206 - num_gpu_blocks_override=prefill_blocks + decode_blocks,
207 - max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
208 - )
209 - sampling_params = SamplingParams(n=beam_width,
210 - use_beam_search=True,
211 - temperature=0.0,
212 - max_tokens=max_tokens,
213 - ignore_eos=True)
214 - req_outputs = vllm_model.model.generate(
215 - example_prompts,
216 - sampling_params=sampling_params,
217 - )
218 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
219 - ARTIFICIAL_PREEMPTION_MAX_CNT)
220 - del vllm_model
196 + with vllm_runner(
197 + model,
198 + dtype=dtype,
199 + swap_space=10,
200 + block_size=BLOCK_SIZE,
201 + # Since beam search have more than 1 sequence, prefill +
202 + # decode blocks are not enough to finish.
203 + num_gpu_blocks_override=prefill_blocks + decode_blocks,
204 + max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
205 + ) as vllm_model:
206 + sampling_params = SamplingParams(n=beam_width,
207 + use_beam_search=True,
208 + temperature=0.0,
209 + max_tokens=max_tokens,
210 + ignore_eos=True)
211 + req_outputs = vllm_model.model.generate(
212 + example_prompts,
213 + sampling_params=sampling_params,
214 + )
215 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
216 + ARTIFICIAL_PREEMPTION_MAX_CNT)
217 +
218 # Verify the request is ignored and not hang.
219 assert req_outputs[0].outputs[0].finish_reason == "length"
220
@@ -236,25 +233,26 @@ def test_preemption_infeasible(
233 BLOCK_SIZE = 16
234 prefill_blocks = 2
235 decode_blocks = max_tokens // BLOCK_SIZE
239 - vllm_model = vllm_runner(
240 - model,
241 - dtype=dtype,
242 - block_size=BLOCK_SIZE,
243 - # Not enough gpu blocks to complete a single sequence.
244 - # preemption should happen, and the sequence should be
245 - # ignored instead of hanging forever.
246 - num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
247 - max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
248 - )
249 - sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
250 - req_outputs = vllm_model.model.generate(
251 - example_prompts,
252 - sampling_params=sampling_params,
253 - )
236 + with vllm_runner(
237 + model,
238 + dtype=dtype,
239 + block_size=BLOCK_SIZE,
240 + # Not enough gpu blocks to complete a single sequence.
241 + # preemption should happen, and the sequence should be
242 + # ignored instead of hanging forever.
243 + num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
244 + max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
245 + ) as vllm_model:
246 + sampling_params = SamplingParams(max_tokens=max_tokens,
247 + ignore_eos=True)
248 + req_outputs = vllm_model.model.generate(
249 + example_prompts,
250 + sampling_params=sampling_params,
251 + )
252 +
253 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
254 + ARTIFICIAL_PREEMPTION_MAX_CNT)
255
255 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
256 - ARTIFICIAL_PREEMPTION_MAX_CNT)
257 - del vllm_model
256 # Verify the request is ignored and not hang.
257 for req_output in req_outputs:
258 outputs = req_output.outputs
tests/conftest.py
+4 -1
@@ -493,7 +493,10 @@ class VllmRunner:
493 outputs.append(embedding)
494 return outputs
495
496 - def __del__(self):
496 + def __enter__(self):
497 + return self
498 +
499 + def __exit__(self, exc_type, exc_value, traceback):
500 del self.model
501 cleanup()
502
tests/distributed/test_basic_distributed_correctness.py
+7 -8
@@ -45,14 +45,13 @@ def test_models(
45 with hf_runner(model, dtype=dtype) as hf_model:
46 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
48 - vllm_model = vllm_runner(
49 - model,
50 - dtype=dtype,
51 - tensor_parallel_size=2,
52 - enforce_eager=enforce_eager,
53 - distributed_executor_backend=distributed_executor_backend)
54 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
55 - del vllm_model
48 + with vllm_runner(model,
49 + dtype=dtype,
50 + tensor_parallel_size=2,
51 + enforce_eager=enforce_eager,
52 + distributed_executor_backend=distributed_executor_backend
53 + ) as vllm_model:
54 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
55
56 for i in range(len(example_prompts)):
57 hf_output_ids, hf_output_str = hf_outputs[i]
tests/distributed/test_chunked_prefill_distributed.py
+10 -11
@@ -48,17 +48,16 @@ def test_models(
48 with hf_runner(model, dtype=dtype) as hf_model:
49 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50
51 - vllm_model = vllm_runner(
52 - model,
53 - dtype=dtype,
54 - tensor_parallel_size=2,
55 - max_num_seqs=max_num_seqs,
56 - enable_chunked_prefill=enable_chunked_prefill,
57 - max_num_batched_tokens=max_num_batched_tokens,
58 - distributed_executor_backend=distributed_executor_backend,
59 - )
60 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61 - del vllm_model
51 + with vllm_runner(
52 + model,
53 + dtype=dtype,
54 + tensor_parallel_size=2,
55 + max_num_seqs=max_num_seqs,
56 + enable_chunked_prefill=enable_chunked_prefill,
57 + max_num_batched_tokens=max_num_batched_tokens,
58 + distributed_executor_backend=distributed_executor_backend,
59 + ) as vllm_model:
60 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61
62 for i in range(len(example_prompts)):
63 hf_output_ids, hf_output_str = hf_outputs[i]
tests/engine/test_stop_reason.py
+2 -3
@@ -19,9 +19,8 @@ MAX_TOKENS = 1024
19
20 @pytest.fixture
21 def vllm_model(vllm_runner):
22 - vllm_model = vllm_runner(MODEL)
23 - yield vllm_model
24 - del vllm_model
22 + with vllm_runner(MODEL) as vllm_model:
23 + yield vllm_model
24
25
26 def test_stop_reason(vllm_model, example_prompts):
tests/engine/test_stop_strings.py
+2 -1
@@ -10,7 +10,8 @@ MAX_TOKENS = 200
10
11 @pytest.fixture(scope="session")
12 def vllm_model(vllm_runner):
13 - return vllm_runner(MODEL)
13 + with vllm_runner(MODEL) as vllm_model:
14 + yield vllm_model
15
16
17 @pytest.mark.skip_global_cleanup
tests/metrics/test_metrics.py
+42 -42
@@ -23,23 +23,25 @@ def test_metric_counter_prompt_tokens(
23 dtype: str,
24 max_tokens: int,
25 ) -> None:
26 - vllm_model = vllm_runner(model,
27 - dtype=dtype,
28 - disable_log_stats=False,
29 - gpu_memory_utilization=0.4)
30 - tokenizer = vllm_model.model.get_tokenizer()
31 - prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
32 - # This test needs at least 2 prompts in a batch of different lengths to
33 - # verify their token count is correct despite padding.
34 - assert len(example_prompts) > 1, "at least 2 prompts are required"
35 - assert prompt_token_counts[0] != prompt_token_counts[1], (
36 - "prompts of different lengths are required")
37 - vllm_prompt_token_count = sum(prompt_token_counts)
38 -
39 - _ = vllm_model.generate_greedy(example_prompts, max_tokens)
40 - stat_logger = vllm_model.model.llm_engine.stat_logger
41 - metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
42 - **stat_logger.labels)._value.get()
26 + with vllm_runner(model,
27 + dtype=dtype,
28 + disable_log_stats=False,
29 + gpu_memory_utilization=0.4) as vllm_model:
30 + tokenizer = vllm_model.model.get_tokenizer()
31 + prompt_token_counts = [
32 + len(tokenizer.encode(p)) for p in example_prompts
33 + ]
34 + # This test needs at least 2 prompts in a batch of different lengths to
35 + # verify their token count is correct despite padding.
36 + assert len(example_prompts) > 1, "at least 2 prompts are required"
37 + assert prompt_token_counts[0] != prompt_token_counts[1], (
38 + "prompts of different lengths are required")
39 + vllm_prompt_token_count = sum(prompt_token_counts)
40 +
41 + _ = vllm_model.generate_greedy(example_prompts, max_tokens)
42 + stat_logger = vllm_model.model.llm_engine.stat_logger
43 + metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
44 + **stat_logger.labels)._value.get()
45
46 assert vllm_prompt_token_count == metric_count, (
47 f"prompt token count: {vllm_prompt_token_count!r}\n"
@@ -56,22 +58,22 @@ def test_metric_counter_generation_tokens(
58 dtype: str,
59 max_tokens: int,
60 ) -> None:
59 - vllm_model = vllm_runner(model,
60 - dtype=dtype,
61 - disable_log_stats=False,
62 - gpu_memory_utilization=0.4)
63 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
64 - tokenizer = vllm_model.model.get_tokenizer()
65 - stat_logger = vllm_model.model.llm_engine.stat_logger
66 - metric_count = stat_logger.metrics.counter_generation_tokens.labels(
67 - **stat_logger.labels)._value.get()
68 - vllm_generation_count = 0
69 - for i in range(len(example_prompts)):
70 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
71 - prompt_ids = tokenizer.encode(example_prompts[i])
72 - # vllm_output_ids contains both prompt tokens and generation tokens.
73 - # We're interested only in the count of the generation tokens.
74 - vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
61 + with vllm_runner(model,
62 + dtype=dtype,
63 + disable_log_stats=False,
64 + gpu_memory_utilization=0.4) as vllm_model:
65 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66 + tokenizer = vllm_model.model.get_tokenizer()
67 + stat_logger = vllm_model.model.llm_engine.stat_logger
68 + metric_count = stat_logger.metrics.counter_generation_tokens.labels(
69 + **stat_logger.labels)._value.get()
70 + vllm_generation_count = 0
71 + for i in range(len(example_prompts)):
72 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
73 + prompt_ids = tokenizer.encode(example_prompts[i])
74 + # vllm_output_ids contains both prompt tokens and generation tokens.
75 + # We're interested only in the count of the generation tokens.
76 + vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
77
78 assert vllm_generation_count == metric_count, (
79 f"generation token count: {vllm_generation_count!r}\n"
@@ -85,15 +87,13 @@ def test_metric_counter_generation_tokens(
87 [None, [], ["ModelName0"], ["ModelName0", "ModelName1", "ModelName2"]])
88 def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
89 served_model_name: List[str]) -> None:
88 - vllm_model = vllm_runner(model,
89 - dtype=dtype,
90 - disable_log_stats=False,
91 - gpu_memory_utilization=0.3,
92 - served_model_name=served_model_name)
93 - stat_logger = vllm_model.model.llm_engine.stat_logger
94 - metrics_tag_content = stat_logger.labels["model_name"]
95 -
96 - del vllm_model
90 + with vllm_runner(model,
91 + dtype=dtype,
92 + disable_log_stats=False,
93 + gpu_memory_utilization=0.3,
94 + served_model_name=served_model_name) as vllm_model:
95 + stat_logger = vllm_model.model.llm_engine.stat_logger
96 + metrics_tag_content = stat_logger.labels["model_name"]
97
98 if served_model_name is None or served_model_name == []:
99 assert metrics_tag_content == model, (
tests/models/test_aqlm.py
+3 -4
@@ -82,10 +82,9 @@ def test_models(
82 num_logprobs: int,
83 ) -> None:
84
85 - vllm_model = vllm_runner(model, dtype=dtype)
86 - vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
87 - max_tokens,
88 - num_logprobs)
85 + with vllm_runner(model, dtype=dtype) as vllm_model:
86 + vllm_outputs = vllm_model.generate_greedy_logprobs(
87 + example_prompts, max_tokens, num_logprobs)
88
89 # loop through the prompts to compare against the ground truth generations
90 for prompt_idx in range(len(example_prompts)):
tests/models/test_big_models.py
+7 -9
@@ -37,9 +37,8 @@ def test_models(
37 with hf_runner(model, dtype=dtype) as hf_model:
38 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40 - vllm_model = vllm_runner(model, dtype=dtype)
41 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42 - del vllm_model
40 + with vllm_runner(model, dtype=dtype) as vllm_model:
41 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
43 for i in range(len(example_prompts)):
44 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -57,9 +56,8 @@ def test_model_print(
56 model: str,
57 dtype: str,
58 ) -> None:
60 - vllm_model = vllm_runner(model, dtype=dtype)
61 - # This test is for verifying whether the model's extra_repr
62 - # can be printed correctly.
63 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
64 - model_runner.model)
65 - del vllm_model
59 + with vllm_runner(model, dtype=dtype) as vllm_model:
60 + # This test is for verifying whether the model's extra_repr
61 + # can be printed correctly.
62 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
63 + model_runner.model)
tests/models/test_embedding.py
+2 -3
@@ -31,9 +31,8 @@ def test_models(
31 with hf_runner(model, dtype=dtype, is_embedding_model=True) as hf_model:
32 hf_outputs = hf_model.encode(example_prompts)
33
34 - vllm_model = vllm_runner(model, dtype=dtype)
35 - vllm_outputs = vllm_model.encode(example_prompts)
36 - del vllm_model
34 + with vllm_runner(model, dtype=dtype) as vllm_model:
35 + vllm_outputs = vllm_model.encode(example_prompts)
36
37 similarities = compare_embeddings(hf_outputs, vllm_outputs)
38 all_similarities = torch.stack(similarities)
tests/models/test_gptq_marlin.py
+17 -20
@@ -70,32 +70,29 @@ def test_models(
70 model_name, revision = model
71
72 # Run marlin.
73 - gptq_marlin_model = vllm_runner(model_name=model_name,
74 - revision=revision,
75 - dtype=dtype,
76 - quantization="marlin",
77 - max_model_len=MAX_MODEL_LEN,
78 - tensor_parallel_size=1)
79 -
80 - gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
81 - example_prompts[:-1], max_tokens, num_logprobs)
82 - del gptq_marlin_model
73 + with vllm_runner(model_name=model_name,
74 + revision=revision,
75 + dtype=dtype,
76 + quantization="marlin",
77 + max_model_len=MAX_MODEL_LEN,
78 + tensor_parallel_size=1) as gptq_marlin_model:
79 +
80 + gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
81 + example_prompts[:-1], max_tokens, num_logprobs)
82 _ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
83
84 # Run gptq.
85 # The naive gptq kernel doesn't support bf16 yet.
86 # Here we always compare fp16/bf16 gpt marlin kernel
87 # to fp16 gptq kernel.
89 - gptq_model = vllm_runner(model_name=model_name,
90 - revision=revision,
91 - dtype="half",
92 - quantization="gptq",
93 - max_model_len=MAX_MODEL_LEN,
94 - tensor_parallel_size=1)
95 - gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts[:-1],
96 - max_tokens,
97 - num_logprobs)
98 - del gptq_model
88 + with vllm_runner(model_name=model_name,
89 + revision=revision,
90 + dtype="half",
91 + quantization="gptq",
92 + max_model_len=MAX_MODEL_LEN,
93 + tensor_parallel_size=1) as gptq_model:
94 + gptq_outputs = gptq_model.generate_greedy_logprobs(
95 + example_prompts[:-1], max_tokens, num_logprobs)
96
97 check_logprobs_close(
98 outputs_0_lst=gptq_outputs,
tests/models/test_gptq_marlin_24.py
+9 -13
@@ -61,20 +61,16 @@ def test_models(
61 max_tokens: int,
62 num_logprobs: int,
63 ) -> None:
64 - marlin_24_model = vllm_runner(model_pair.model_marlin,
65 - dtype=dtype,
66 - quantization="gptq_marlin_24")
67 - marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
68 - example_prompts, max_tokens, num_logprobs)
69 - del marlin_24_model
64 + with vllm_runner(model_pair.model_marlin,
65 + dtype=dtype,
66 + quantization="gptq_marlin_24") as marlin_24_model:
67 + marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
68 + example_prompts, max_tokens, num_logprobs)
69
71 - gptq_model = vllm_runner(model_pair.model_gptq,
72 - dtype=dtype,
73 - quantization="gptq")
74 - gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
75 - max_tokens,
76 - num_logprobs)
77 - del gptq_model
70 + with vllm_runner(model_pair.model_gptq, dtype=dtype,
71 + quantization="gptq") as gptq_model:
72 + gptq_outputs = gptq_model.generate_greedy_logprobs(
73 + example_prompts, max_tokens, num_logprobs)
74
75 check_logprobs_close(
76 outputs_0_lst=gptq_outputs,
tests/models/test_llava.py
+7 -8
@@ -94,14 +94,13 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
94 for p in HF_IMAGE_PROMPTS
95 ]
96
97 - vllm_model = vllm_runner(model_id,
98 - dtype=dtype,
99 - enforce_eager=True,
100 - **vlm_config.as_cli_args_dict())
101 - vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
102 - max_tokens,
103 - images=vllm_images)
104 - del vllm_model
97 + with vllm_runner(model_id,
98 + dtype=dtype,
99 + enforce_eager=True,
100 + **vlm_config.as_cli_args_dict()) as vllm_model:
101 + vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
102 + max_tokens,
103 + images=vllm_images)
104
105 for i in range(len(HF_IMAGE_PROMPTS)):
106 hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/test_marlin.py
+10 -14
@@ -59,20 +59,16 @@ def test_models(
59 max_tokens: int,
60 num_logprobs: int,
61 ) -> None:
62 - marlin_model = vllm_runner(model_pair.model_marlin,
63 - dtype=dtype,
64 - quantization="marlin")
65 - marlin_outputs = marlin_model.generate_greedy_logprobs(
66 - example_prompts, max_tokens, num_logprobs)
67 - del marlin_model
68 -
69 - gptq_model = vllm_runner(model_pair.model_gptq,
70 - dtype=dtype,
71 - quantization="gptq")
72 - gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
73 - max_tokens,
74 - num_logprobs)
75 - del gptq_model
62 + with vllm_runner(model_pair.model_marlin,
63 + dtype=dtype,
64 + quantization="marlin") as marlin_model:
65 + marlin_outputs = marlin_model.generate_greedy_logprobs(
66 + example_prompts, max_tokens, num_logprobs)
67 +
68 + with vllm_runner(model_pair.model_gptq, dtype=dtype,
69 + quantization="gptq") as gptq_model:
70 + gptq_outputs = gptq_model.generate_greedy_logprobs(
71 + example_prompts, max_tokens, num_logprobs)
72
73 check_logprobs_close(
74 outputs_0_lst=gptq_outputs,
tests/models/test_mistral.py
+3 -5
@@ -30,11 +30,9 @@ def test_models(
30 hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 example_prompts, max_tokens, num_logprobs)
32
33 - vllm_model = vllm_runner(model, dtype=dtype)
34 - vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
35 - max_tokens,
36 - num_logprobs)
37 - del vllm_model
33 + with vllm_runner(model, dtype=dtype) as vllm_model:
34 + vllm_outputs = vllm_model.generate_greedy_logprobs(
35 + example_prompts, max_tokens, num_logprobs)
36 check_logprobs_close(
37 outputs_0_lst=hf_outputs,
38 outputs_1_lst=vllm_outputs,
tests/models/test_models.py
+7 -9
@@ -37,9 +37,8 @@ def test_models(
37 with hf_runner(model, dtype=dtype) as hf_model:
38 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40 - vllm_model = vllm_runner(model, dtype=dtype)
41 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42 - del vllm_model
40 + with vllm_runner(model, dtype=dtype) as vllm_model:
41 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
43 for i in range(len(example_prompts)):
44 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -57,9 +56,8 @@ def test_model_print(
56 model: str,
57 dtype: str,
58 ) -> None:
60 - vllm_model = vllm_runner(model, dtype=dtype)
61 - # This test is for verifying whether the model's extra_repr
62 - # can be printed correctly.
63 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
64 - model_runner.model)
65 - del vllm_model
59 + with vllm_runner(model, dtype=dtype) as vllm_model:
60 + # This test is for verifying whether the model's extra_repr
61 + # can be printed correctly.
62 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
63 + model_runner.model)
tests/quantization/test_bitsandbytes.py
+62 -62
@@ -16,65 +16,65 @@ capability = capability[0] * 10 + capability[1]
16 capability < QUANTIZATION_METHODS['bitsandbytes'].get_min_capability(),
17 reason='bitsandbytes is not supported on this GPU type.')
18 def test_load_bnb_model(vllm_runner) -> None:
19 - llm = vllm_runner('huggyllama/llama-7b',
20 - quantization='bitsandbytes',
21 - load_format='bitsandbytes',
22 - enforce_eager=True)
23 -
24 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
25 -
26 - # check the weights in MLP & SelfAttention are quantized to torch.uint8
27 - qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28 - assert qweight.dtype == torch.uint8, (
29 - f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30 -
31 - qweight = model.model.layers[0].mlp.down_proj.qweight
32 - assert qweight.dtype == torch.uint8, (
33 - f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34 -
35 - qweight = model.model.layers[0].self_attn.o_proj.qweight
36 - assert qweight.dtype == torch.uint8, (
37 - f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38 -
39 - qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40 - assert qweight.dtype == torch.uint8, (
41 - f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42 -
43 - # some weights should not be quantized
44 - weight = model.lm_head.weight
45 - assert weight.dtype != torch.uint8, (
46 - 'lm_head weight dtype should not be torch.uint8')
47 -
48 - weight = model.model.embed_tokens.weight
49 - assert weight.dtype != torch.uint8, (
50 - 'embed_tokens weight dtype should not be torch.uint8')
51 -
52 - weight = model.model.layers[0].input_layernorm.weight
53 - assert weight.dtype != torch.uint8, (
54 - 'input_layernorm weight dtype should not be torch.uint8')
55 -
56 - weight = model.model.layers[0].post_attention_layernorm.weight
57 - assert weight.dtype != torch.uint8, (
58 - 'input_layernorm weight dtype should not be torch.uint8')
59 -
60 - # check the output of the model is expected
61 - sampling_params = SamplingParams(temperature=0.0,
62 - logprobs=1,
63 - prompt_logprobs=1,
64 - max_tokens=8)
65 -
66 - prompts = ['That which does not kill us', 'To be or not to be,']
67 - expected_outputs = [
68 - 'That which does not kill us makes us stronger.',
69 - 'To be or not to be, that is the question.'
70 - ]
71 - outputs = llm.generate(prompts, sampling_params=sampling_params)
72 -
73 - assert len(outputs) == len(prompts)
74 -
75 - for index in range(len(outputs)):
76 - # compare the first line of the output
77 - actual_output = outputs[index][1][0].split('\n', 1)[0]
78 - expected_output = expected_outputs[index].split('\n', 1)[0]
79 - assert actual_output == expected_output, (
80 - f'Expected: {expected_output}, but got: {actual_output}')
19 + with vllm_runner('huggyllama/llama-7b',
20 + quantization='bitsandbytes',
21 + load_format='bitsandbytes',
22 + enforce_eager=True) as llm:
23 +
24 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
25 +
26 + # check the weights in MLP & SelfAttention are quantized to torch.uint8
27 + qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28 + assert qweight.dtype == torch.uint8, (
29 + f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30 +
31 + qweight = model.model.layers[0].mlp.down_proj.qweight
32 + assert qweight.dtype == torch.uint8, (
33 + f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34 +
35 + qweight = model.model.layers[0].self_attn.o_proj.qweight
36 + assert qweight.dtype == torch.uint8, (
37 + f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38 +
39 + qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40 + assert qweight.dtype == torch.uint8, (
41 + f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42 +
43 + # some weights should not be quantized
44 + weight = model.lm_head.weight
45 + assert weight.dtype != torch.uint8, (
46 + 'lm_head weight dtype should not be torch.uint8')
47 +
48 + weight = model.model.embed_tokens.weight
49 + assert weight.dtype != torch.uint8, (
50 + 'embed_tokens weight dtype should not be torch.uint8')
51 +
52 + weight = model.model.layers[0].input_layernorm.weight
53 + assert weight.dtype != torch.uint8, (
54 + 'input_layernorm weight dtype should not be torch.uint8')
55 +
56 + weight = model.model.layers[0].post_attention_layernorm.weight
57 + assert weight.dtype != torch.uint8, (
58 + 'input_layernorm weight dtype should not be torch.uint8')
59 +
60 + # check the output of the model is expected
61 + sampling_params = SamplingParams(temperature=0.0,
62 + logprobs=1,
63 + prompt_logprobs=1,
64 + max_tokens=8)
65 +
66 + prompts = ['That which does not kill us', 'To be or not to be,']
67 + expected_outputs = [
68 + 'That which does not kill us makes us stronger.',
69 + 'To be or not to be, that is the question.'
70 + ]
71 + outputs = llm.generate(prompts, sampling_params=sampling_params)
72 +
73 + assert len(outputs) == len(prompts)
74 +
75 + for index in range(len(outputs)):
76 + # compare the first line of the output
77 + actual_output = outputs[index][1][0].split('\n', 1)[0]
78 + expected_output = expected_outputs[index].split('\n', 1)[0]
79 + assert actual_output == expected_output, (
80 + f'Expected: {expected_output}, but got: {actual_output}')
tests/quantization/test_compressed_tensors.py
+33 -30
@@ -12,42 +12,45 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
12
13 def test_compressed_tensors_w8a8_static_setup(vllm_runner):
14 model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
15 - llm = vllm_runner(model_path, quantization="sparseml", enforce_eager=True)
16 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
17 - layer = model.model.layers[0]
15 + with vllm_runner(model_path, quantization="sparseml",
16 + enforce_eager=True) as llm:
17 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
18 + layer = model.model.layers[0]
19
19 - qkv_proj = layer.self_attn.qkv_proj
20 - o_proj = layer.self_attn.o_proj
21 - gate_up_proj = layer.mlp.gate_up_proj
22 - down_proj = layer.mlp.down_proj
20 + qkv_proj = layer.self_attn.qkv_proj
21 + o_proj = layer.self_attn.o_proj
22 + gate_up_proj = layer.mlp.gate_up_proj
23 + down_proj = layer.mlp.down_proj
24
24 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
25 - assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
26 - assert isinstance(gate_up_proj.quant_method, CompressedTensorsLinearMethod)
27 - assert isinstance(down_proj.quant_method, CompressedTensorsLinearMethod)
25 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
26 + assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
27 + assert isinstance(gate_up_proj.quant_method,
28 + CompressedTensorsLinearMethod)
29 + assert isinstance(down_proj.quant_method,
30 + CompressedTensorsLinearMethod)
31
29 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
32 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
33
31 - assert qkv_proj.weight.dtype is torch.int8
32 - assert o_proj.weight.dtype is torch.int8
33 - assert gate_up_proj.weight.dtype is torch.int8
34 + assert qkv_proj.weight.dtype is torch.int8
35 + assert o_proj.weight.dtype is torch.int8
36 + assert gate_up_proj.weight.dtype is torch.int8
37
35 - assert qkv_proj.weight_scale.shard_splitter is not None
36 - assert qkv_proj.weight_scale.logical_widths is not None
37 - assert qkv_proj.input_scale.dtype is torch.float32
38 + assert qkv_proj.weight_scale.shard_splitter is not None
39 + assert qkv_proj.weight_scale.logical_widths is not None
40 + assert qkv_proj.input_scale.dtype is torch.float32
41
42
43 def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
44 model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
42 - llm = vllm_runner(model_path,
43 - quantization="sparseml",
44 - enforce_eager=True,
45 - dtype=torch.float16)
46 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
47 - layer = model.model.layers[0]
48 -
49 - qkv_proj = layer.self_attn.qkv_proj
50 -
51 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
52 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
53 - assert qkv_proj.weight.dtype is torch.int8
45 + with vllm_runner(model_path,
46 + quantization="sparseml",
47 + enforce_eager=True,
48 + dtype=torch.float16) as llm:
49 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
50 + layer = model.model.layers[0]
51 +
52 + qkv_proj = layer.self_attn.qkv_proj
53 +
54 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
55 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
56 + assert qkv_proj.weight.dtype is torch.int8
tests/quantization/test_fp8.py
+5 -5
@@ -16,9 +16,9 @@ capability = capability[0] * 10 + capability[1]
16 capability < QUANTIZATION_METHODS["fp8"].get_min_capability(),
17 reason="FP8 is not supported on this GPU type.")
18 def test_load_fp16_model(vllm_runner) -> None:
19 - llm = vllm_runner("facebook/opt-125m", quantization="fp8")
19 + with vllm_runner("facebook/opt-125m", quantization="fp8") as llm:
20
21 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
22 - fc1 = model.model.decoder.layers[0].fc1
23 - assert isinstance(fc1.quant_method, Fp8LinearMethod)
24 - assert fc1.weight.dtype == torch.float8_e4m3fn
21 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
22 + fc1 = model.model.decoder.layers[0].fc1
23 + assert isinstance(fc1.quant_method, Fp8LinearMethod)
24 + assert fc1.weight.dtype == torch.float8_e4m3fn
tests/samplers/test_beam_search.py
+3 -10
@@ -2,10 +2,8 @@
2
3 Run `pytest tests/samplers/test_beam_search.py`.
4 """
5 -import gc
5
6 import pytest
8 -import torch
7
8 # FIXME(zhuohan): The test can not pass if we:
9 # 1. Increase max_tokens to 256.
@@ -34,14 +32,9 @@ def test_beam_search_single_input(
32 hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
33 max_tokens)
34
37 - vllm_model = vllm_runner(model, dtype=dtype)
38 - vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
39 - max_tokens)
40 - del vllm_model
41 - # NOTE(woosuk): For some reason, the following GC is required to avoid
42 - # GPU OOM errors in the following tests using `vllm_runner`.
43 - gc.collect()
44 - torch.cuda.empty_cache()
35 + with vllm_runner(model, dtype=dtype) as vllm_model:
36 + vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37 + beam_width, max_tokens)
38
39 for i in range(len(example_prompts)):
40 hf_output_ids, _ = hf_outputs[i]
tests/samplers/test_ignore_eos.py
+8 -7
@@ -22,11 +22,12 @@ def test_ignore_eos(
22 dtype: str,
23 max_tokens: int,
24 ) -> None:
25 - vllm_model = vllm_runner(model, dtype=dtype)
26 - sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
25 + with vllm_runner(model, dtype=dtype) as vllm_model:
26 + sampling_params = SamplingParams(max_tokens=max_tokens,
27 + ignore_eos=True)
28
28 - for prompt in example_prompts:
29 - ignore_eos_output = vllm_model.model.generate(
30 - prompt, sampling_params=sampling_params)
31 - output_length = len(ignore_eos_output[0].outputs[0].token_ids)
32 - assert output_length == max_tokens
29 + for prompt in example_prompts:
30 + ignore_eos_output = vllm_model.model.generate(
31 + prompt, sampling_params=sampling_params)
32 + output_length = len(ignore_eos_output[0].outputs[0].token_ids)
33 + assert output_length == max_tokens
tests/samplers/test_logits_processor.py
+41 -41
@@ -14,46 +14,46 @@ def test_logits_processor_force_generate(
14 model: str,
15 dtype: str,
16 ) -> None:
17 - vllm_model = vllm_runner(model, dtype=dtype)
18 - tokenizer = vllm_model.model.get_tokenizer()
19 - repeat_times = 2
20 - enforced_answers = " vLLM"
21 - vllm_token_ids = tokenizer.encode(enforced_answers,
22 - add_special_tokens=False)
23 - max_tokens = len(vllm_token_ids) * repeat_times
24 -
25 - def pick_vllm(token_ids, logits):
26 - token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27 - logits[token_id] = torch.finfo(logits.dtype).max
28 - return logits
29 -
30 - params_with_logprobs = SamplingParams(
31 - logits_processors=[pick_vllm],
32 - prompt_logprobs=3,
33 - max_tokens=max_tokens,
34 - )
35 -
36 - # test logits_processors when prompt_logprobs is not None
37 - vllm_model.model._add_request(
38 - example_prompts[0],
39 - params=params_with_logprobs,
40 - )
41 -
42 - # test prompt_logprobs is not None
43 - vllm_model.model._add_request(
44 - example_prompts[1],
45 - params=SamplingParams(
17 + with vllm_runner(model, dtype=dtype) as vllm_model:
18 + tokenizer = vllm_model.model.get_tokenizer()
19 + repeat_times = 2
20 + enforced_answers = " vLLM"
21 + vllm_token_ids = tokenizer.encode(enforced_answers,
22 + add_special_tokens=False)
23 + max_tokens = len(vllm_token_ids) * repeat_times
24 +
25 + def pick_vllm(token_ids, logits):
26 + token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27 + logits[token_id] = torch.finfo(logits.dtype).max
28 + return logits
29 +
30 + params_with_logprobs = SamplingParams(
31 + logits_processors=[pick_vllm],
32 prompt_logprobs=3,
33 max_tokens=max_tokens,
48 - ),
49 - )
50 -
51 - # test grouped requests
52 - vllm_model.model._add_request(
53 - example_prompts[2],
54 - params=SamplingParams(max_tokens=max_tokens),
55 - )
56 -
57 - outputs = vllm_model.model._run_engine(use_tqdm=False)
58 -
59 - assert outputs[0].outputs[0].text == enforced_answers * repeat_times
34 + )
35 +
36 + # test logits_processors when prompt_logprobs is not None
37 + vllm_model.model._add_request(
38 + example_prompts[0],
39 + params=params_with_logprobs,
40 + )
41 +
42 + # test prompt_logprobs is not None
43 + vllm_model.model._add_request(
44 + example_prompts[1],
45 + params=SamplingParams(
46 + prompt_logprobs=3,
47 + max_tokens=max_tokens,
48 + ),
49 + )
50 +
51 + # test grouped requests
52 + vllm_model.model._add_request(
53 + example_prompts[2],
54 + params=SamplingParams(max_tokens=max_tokens),
55 + )
56 +
57 + outputs = vllm_model.model._run_engine(use_tqdm=False)
58 +
59 + assert outputs[0].outputs[0].text == enforced_answers * repeat_times
tests/samplers/test_logprobs.py
+15 -15
@@ -38,21 +38,21 @@ def test_get_prompt_logprobs(
38 max_tokens=max_tokens,
39 )
40
41 - vllm_model = vllm_runner(
42 - model,
43 - dtype=dtype,
44 - max_logprobs=num_top_logprobs,
45 - enable_chunked_prefill=enable_chunked_prefill,
46 - max_num_batched_tokens=max_num_batched_tokens,
47 - max_num_seqs=max_num_seqs,
48 - )
49 - vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
50 - logprobs=num_top_logprobs,
51 - prompt_logprobs=num_top_logprobs,
52 - temperature=0.0,
53 - detokenize=detokenize)
54 - vllm_results = vllm_model.model.generate(
55 - example_prompts, sampling_params=vllm_sampling_params)
41 + with vllm_runner(
42 + model,
43 + dtype=dtype,
44 + max_logprobs=num_top_logprobs,
45 + enable_chunked_prefill=enable_chunked_prefill,
46 + max_num_batched_tokens=max_num_batched_tokens,
47 + max_num_seqs=max_num_seqs,
48 + ) as vllm_model:
49 + vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
50 + logprobs=num_top_logprobs,
51 + prompt_logprobs=num_top_logprobs,
52 + temperature=0.0,
53 + detokenize=detokenize)
54 + vllm_results = vllm_model.model.generate(
55 + example_prompts, sampling_params=vllm_sampling_params)
56
57 # Test whether logprobs are included in the results.
58 for result in vllm_results:
tests/samplers/test_ranks.py
+21 -17
@@ -17,16 +17,27 @@ def test_ranks(
17 num_top_logprobs = 5
18 num_prompt_logprobs = 5
19
20 - vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
21 -
22 - ## Test greedy logprobs ranks
23 - vllm_sampling_params = SamplingParams(temperature=0.0,
24 - top_p=1.0,
25 - max_tokens=max_tokens,
26 - logprobs=num_top_logprobs,
27 - prompt_logprobs=num_prompt_logprobs)
28 - vllm_results = vllm_model.generate_w_logprobs(example_prompts,
29 - vllm_sampling_params)
20 + with vllm_runner(model, dtype=dtype,
21 + max_logprobs=num_top_logprobs) as vllm_model:
22 +
23 + ## Test greedy logprobs ranks
24 + vllm_sampling_params = SamplingParams(
25 + temperature=0.0,
26 + top_p=1.0,
27 + max_tokens=max_tokens,
28 + logprobs=num_top_logprobs,
29 + prompt_logprobs=num_prompt_logprobs)
30 + vllm_results = vllm_model.generate_w_logprobs(example_prompts,
31 + vllm_sampling_params)
32 +
33 + ## Test non-greedy logprobs ranks
34 + sampling_params = SamplingParams(temperature=1.0,
35 + top_p=1.0,
36 + max_tokens=max_tokens,
37 + logprobs=num_top_logprobs,
38 + prompt_logprobs=num_prompt_logprobs)
39 + res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
40 +
41 for result in vllm_results:
42 assert result[2] is not None
43 assert len(result[2]) == len(result[0])
@@ -35,13 +46,6 @@ def test_ranks(
46 assert token in logprobs
47 assert logprobs[token].rank == 1
48
38 - ## Test non-greedy logprobs ranks
39 - sampling_params = SamplingParams(temperature=1.0,
40 - top_p=1.0,
41 - max_tokens=max_tokens,
42 - logprobs=num_top_logprobs,
43 - prompt_logprobs=num_prompt_logprobs)
44 - res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
49 for result in res:
50 assert result[2] is not None
51 assert len(result[2]) == len(result[0])
tests/samplers/test_seeded_generate.py
+2 -3
@@ -17,9 +17,8 @@ RANDOM_SEEDS = list(range(5))
17
18 @pytest.fixture
19 def vllm_model(vllm_runner):
20 - vllm_model = vllm_runner(MODEL, dtype="half")
21 - yield vllm_model
22 - del vllm_model
20 + with vllm_runner(MODEL, dtype="half") as vllm_model:
21 + yield vllm_model
22
23
24 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
tests/tensorizer_loader/test_tensorizer.py
+45 -61
@@ -1,4 +1,3 @@
1 -import gc
1 import json
2 import os
3 import subprocess
@@ -7,7 +6,6 @@ from unittest.mock import MagicMock, patch
6 import openai
7 import pytest
8 import ray
10 -import torch
9
10 from vllm import SamplingParams
11 # yapf: disable
@@ -71,47 +69,43 @@ def test_can_deserialize_s3(vllm_runner):
69 model_ref = "EleutherAI/pythia-1.4b"
70 tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
71
74 - loaded_hf_model = vllm_runner(model_ref,
72 + with vllm_runner(model_ref,
73 load_format="tensorizer",
74 model_loader_extra_config=TensorizerConfig(
75 tensorizer_uri=tensorized_path,
76 num_readers=1,
77 s3_endpoint="object.ord1.coreweave.com",
80 - ))
78 + )) as loaded_hf_model:
79
82 - deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
80 + deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params) # noqa: E501
81
84 - assert deserialized_outputs
82 + assert deserialized_outputs
83
84
85 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
86 def test_deserialized_encrypted_vllm_model_has_same_outputs(
87 vllm_runner, tmp_path):
90 - vllm_model = vllm_runner(model_ref)
91 - model_path = tmp_path / (model_ref + ".tensors")
92 - key_path = tmp_path / (model_ref + ".key")
93 - outputs = vllm_model.generate(prompts, sampling_params)
94 -
95 - config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
96 - serialize_vllm_model(vllm_model.model.llm_engine,
97 - config_for_serializing,
98 - encryption_key_path=key_path)
88 + with vllm_runner(model_ref) as vllm_model:
89 + model_path = tmp_path / (model_ref + ".tensors")
90 + key_path = tmp_path / (model_ref + ".key")
91 + outputs = vllm_model.generate(prompts, sampling_params)
92
100 - del vllm_model
101 - gc.collect()
102 - torch.cuda.empty_cache()
93 + config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
94 + serialize_vllm_model(vllm_model.model.llm_engine,
95 + config_for_serializing,
96 + encryption_key_path=key_path)
97
98 config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
99 encryption_keyfile=key_path)
100
107 - loaded_vllm_model = vllm_runner(
101 + with vllm_runner(
102 model_ref,
103 load_format="tensorizer",
110 - model_loader_extra_config=config_for_deserializing)
104 + model_loader_extra_config=config_for_deserializing) as loaded_vllm_model: # noqa: E501
105
112 - deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
106 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params) # noqa: E501
107
114 - assert outputs == deserialized_outputs
108 + assert outputs == deserialized_outputs
109
110
111 def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
@@ -124,17 +118,17 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
118 serializer = TensorSerializer(stream)
119 serializer.write_module(hf_model.model)
120
127 - loaded_hf_model = vllm_runner(model_ref,
121 + with vllm_runner(model_ref,
122 load_format="tensorizer",
123 model_loader_extra_config=TensorizerConfig(
124 tensorizer_uri=model_path,
125 num_readers=1,
132 - ))
126 + )) as loaded_hf_model:
127
134 - deserialized_outputs = loaded_hf_model.generate_greedy(
135 - prompts, max_tokens=max_tokens)
128 + deserialized_outputs = loaded_hf_model.generate_greedy(
129 + prompts, max_tokens=max_tokens)
130
137 - assert outputs == deserialized_outputs
131 + assert outputs == deserialized_outputs
132
133
134 def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
@@ -148,16 +142,13 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
142 test_prompts = create_test_prompts(lora_path)
143
144 # Serialize model before deserializing and binding LoRA adapters
151 - vllm_model = vllm_runner(model_ref, )
152 - model_path = tmp_path / (model_ref + ".tensors")
145 + with vllm_runner(model_ref, ) as vllm_model:
146 + model_path = tmp_path / (model_ref + ".tensors")
147
154 - serialize_vllm_model(vllm_model.model.llm_engine,
155 - TensorizerConfig(tensorizer_uri=model_path))
148 + serialize_vllm_model(vllm_model.model.llm_engine,
149 + TensorizerConfig(tensorizer_uri=model_path))
150
157 - del vllm_model
158 - gc.collect()
159 - torch.cuda.empty_cache()
160 - loaded_vllm_model = vllm_runner(
151 + with vllm_runner(
152 model_ref,
153 load_format="tensorizer",
154 model_loader_extra_config=TensorizerConfig(
@@ -170,10 +161,10 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
161 max_cpu_loras=2,
162 max_num_seqs=50,
163 max_model_len=1000,
173 - )
174 - process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
164 + ) as loaded_vllm_model:
165 + process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
166
176 - assert loaded_vllm_model
167 + assert loaded_vllm_model
168
169
170 def test_load_without_tensorizer_load_format(vllm_runner):
@@ -186,19 +177,15 @@ def test_load_without_tensorizer_load_format(vllm_runner):
177 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
178 def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
179 ## Serialize model
189 - vllm_model = vllm_runner(model_ref, )
190 - model_path = tmp_path / (model_ref + ".tensors")
191 -
192 - serialize_vllm_model(vllm_model.model.llm_engine,
193 - TensorizerConfig(tensorizer_uri=model_path))
180 + with vllm_runner(model_ref, ) as vllm_model:
181 + model_path = tmp_path / (model_ref + ".tensors")
182
195 - model_loader_extra_config = {
196 - "tensorizer_uri": str(model_path),
197 - }
183 + serialize_vllm_model(vllm_model.model.llm_engine,
184 + TensorizerConfig(tensorizer_uri=model_path))
185
199 - del vllm_model
200 - gc.collect()
201 - torch.cuda.empty_cache()
186 + model_loader_extra_config = {
187 + "tensorizer_uri": str(model_path),
188 + }
189
190 ## Start OpenAI API server
191 openai_args = [
@@ -260,18 +247,15 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
247 model_path = tmp_path / (model_ref + ".tensors")
248 config = TensorizerConfig(tensorizer_uri=str(model_path))
249
263 - vllm_model = vllm_runner(model_ref)
264 - outputs = vllm_model.generate(prompts, sampling_params)
265 - serialize_vllm_model(vllm_model.model.llm_engine, config)
250 + with vllm_runner(model_ref) as vllm_model:
251 + outputs = vllm_model.generate(prompts, sampling_params)
252 + serialize_vllm_model(vllm_model.model.llm_engine, config)
253
267 - assert is_vllm_tensorized(config)
268 - del vllm_model
269 - gc.collect()
270 - torch.cuda.empty_cache()
254 + assert is_vllm_tensorized(config)
255
272 - loaded_vllm_model = vllm_runner(model_ref,
273 - load_format="tensorizer",
274 - model_loader_extra_config=config)
275 - deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
256 + with vllm_runner(model_ref,
257 + load_format="tensorizer",
258 + model_loader_extra_config=config) as loaded_vllm_model:
259 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params) # noqa: E501
260
277 - assert outputs == deserialized_outputs
261 + assert outputs == deserialized_outputs