A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Test] improve robustness of test (vllm_runner) (#5357)
[CI/Test] improve robustness of test by replacing del with context manager (vllm_runner) (#5357)
youkaichao committed
Jun 8, 2024 at 01:59 UTC
8ea5e44a435e8731fd6f5ba4c329dd112752532a
28 files changed
+455
-494
tests/basic_correctness/test_basic_correctness.py
+5
-6
@@ -46,12 +46,11 @@ def test_models(
46
with hf_runner(model, dtype=dtype) as hf_model:
47
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49
- vllm_model = vllm_runner(model,
50
- dtype=dtype,
51
- enforce_eager=enforce_eager,
52
- gpu_memory_utilization=0.7)
53
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
- del vllm_model
49
+ with vllm_runner(model,
50
+ dtype=dtype,
51
+ enforce_eager=enforce_eager,
52
+ gpu_memory_utilization=0.7) as vllm_model:
53
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
55
for i in range(len(example_prompts)):
56
hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_chunked_prefill.py
+10
-11
@@ -43,17 +43,16 @@ def test_models(
43
with hf_runner(model, dtype=dtype) as hf_model:
44
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
46
- vllm_model = vllm_runner(
47
- model,
48
- dtype=dtype,
49
- max_num_batched_tokens=max_num_batched_tokens,
50
- enable_chunked_prefill=enable_chunked_prefill,
51
- tensor_parallel_size=tensor_parallel_size,
52
- enforce_eager=enforce_eager,
53
- max_num_seqs=max_num_seqs,
54
- )
55
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56
- del vllm_model
46
+ with vllm_runner(
47
+ model,
48
+ dtype=dtype,
49
+ max_num_batched_tokens=max_num_batched_tokens,
50
+ enable_chunked_prefill=enable_chunked_prefill,
51
+ tensor_parallel_size=tensor_parallel_size,
52
+ enforce_eager=enforce_eager,
53
+ max_num_seqs=max_num_seqs,
54
+ ) as vllm_model:
55
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56
57
for i in range(len(example_prompts)):
58
hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_preemption.py
+73
-75
@@ -46,17 +46,16 @@ def test_chunked_prefill_recompute(
46
with hf_runner(model, dtype=dtype) as hf_model:
47
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
48
49
- vllm_model = vllm_runner(
50
- model,
51
- dtype=dtype,
52
- max_num_batched_tokens=max_num_batched_tokens,
53
- enable_chunked_prefill=enable_chunked_prefill,
54
- max_num_seqs=max_num_seqs,
55
- )
56
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
58
- ARTIFICIAL_PREEMPTION_MAX_CNT)
59
- del vllm_model
49
+ with vllm_runner(
50
+ model,
51
+ dtype=dtype,
52
+ max_num_batched_tokens=max_num_batched_tokens,
53
+ enable_chunked_prefill=enable_chunked_prefill,
54
+ max_num_seqs=max_num_seqs,
55
+ ) as vllm_model:
56
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
58
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
59
60
for i in range(len(example_prompts)):
61
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -84,17 +83,16 @@ def test_preemption(
83
with hf_runner(model, dtype=dtype) as hf_model:
84
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
85
87
- vllm_model = vllm_runner(
88
- model,
89
- dtype=dtype,
90
- disable_log_stats=False,
91
- )
92
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
93
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
94
- ARTIFICIAL_PREEMPTION_MAX_CNT)
95
- total_preemption = (
96
- vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
97
- del vllm_model
86
+ with vllm_runner(
87
+ model,
88
+ dtype=dtype,
89
+ disable_log_stats=False,
90
+ ) as vllm_model:
91
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
92
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
93
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
94
+ total_preemption = (
95
+ vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
96
97
for i in range(len(example_prompts)):
98
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -139,19 +137,18 @@ def test_swap(
137
hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
138
max_tokens)
139
142
- vllm_model = vllm_runner(
143
- model,
144
- dtype=dtype,
145
- swap_space=10,
146
- disable_log_stats=False,
147
- )
148
- vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
149
- max_tokens)
150
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
151
- ARTIFICIAL_PREEMPTION_MAX_CNT)
152
- total_preemption = (
153
- vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
154
- del vllm_model
140
+ with vllm_runner(
141
+ model,
142
+ dtype=dtype,
143
+ swap_space=10,
144
+ disable_log_stats=False,
145
+ ) as vllm_model:
146
+ vllm_outputs = vllm_model.generate_beam_search(example_prompts,
147
+ beam_width, max_tokens)
148
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
149
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
150
+ total_preemption = (
151
+ vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
152
153
for i in range(len(example_prompts)):
154
hf_output_ids, _ = hf_outputs[i]
@@ -196,28 +193,28 @@ def test_swap_infeasible(
193
decode_blocks = max_tokens // BLOCK_SIZE
194
example_prompts = example_prompts[:1]
195
199
- vllm_model = vllm_runner(
200
- model,
201
- dtype=dtype,
202
- swap_space=10,
203
- block_size=BLOCK_SIZE,
204
- # Since beam search have more than 1 sequence, prefill + decode blocks
205
- # are not enough to finish.
206
- num_gpu_blocks_override=prefill_blocks + decode_blocks,
207
- max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
208
- )
209
- sampling_params = SamplingParams(n=beam_width,
210
- use_beam_search=True,
211
- temperature=0.0,
212
- max_tokens=max_tokens,
213
- ignore_eos=True)
214
- req_outputs = vllm_model.model.generate(
215
- example_prompts,
216
- sampling_params=sampling_params,
217
- )
218
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
219
- ARTIFICIAL_PREEMPTION_MAX_CNT)
220
- del vllm_model
196
+ with vllm_runner(
197
+ model,
198
+ dtype=dtype,
199
+ swap_space=10,
200
+ block_size=BLOCK_SIZE,
201
+ # Since beam search have more than 1 sequence, prefill +
202
+ # decode blocks are not enough to finish.
203
+ num_gpu_blocks_override=prefill_blocks + decode_blocks,
204
+ max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
205
+ ) as vllm_model:
206
+ sampling_params = SamplingParams(n=beam_width,
207
+ use_beam_search=True,
208
+ temperature=0.0,
209
+ max_tokens=max_tokens,
210
+ ignore_eos=True)
211
+ req_outputs = vllm_model.model.generate(
212
+ example_prompts,
213
+ sampling_params=sampling_params,
214
+ )
215
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
216
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
217
+
218
# Verify the request is ignored and not hang.
219
assert req_outputs[0].outputs[0].finish_reason == "length"
220
@@ -236,25 +233,26 @@ def test_preemption_infeasible(
233
BLOCK_SIZE = 16
234
prefill_blocks = 2
235
decode_blocks = max_tokens // BLOCK_SIZE
239
- vllm_model = vllm_runner(
240
- model,
241
- dtype=dtype,
242
- block_size=BLOCK_SIZE,
243
- # Not enough gpu blocks to complete a single sequence.
244
- # preemption should happen, and the sequence should be
245
- # ignored instead of hanging forever.
246
- num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
247
- max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
248
- )
249
- sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
250
- req_outputs = vllm_model.model.generate(
251
- example_prompts,
252
- sampling_params=sampling_params,
253
- )
236
+ with vllm_runner(
237
+ model,
238
+ dtype=dtype,
239
+ block_size=BLOCK_SIZE,
240
+ # Not enough gpu blocks to complete a single sequence.
241
+ # preemption should happen, and the sequence should be
242
+ # ignored instead of hanging forever.
243
+ num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
244
+ max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
245
+ ) as vllm_model:
246
+ sampling_params = SamplingParams(max_tokens=max_tokens,
247
+ ignore_eos=True)
248
+ req_outputs = vllm_model.model.generate(
249
+ example_prompts,
250
+ sampling_params=sampling_params,
251
+ )
252
+
253
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
254
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
255
255
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
256
- ARTIFICIAL_PREEMPTION_MAX_CNT)
257
- del vllm_model
256
# Verify the request is ignored and not hang.
257
for req_output in req_outputs:
258
outputs = req_output.outputs
tests/conftest.py
+4
-1
@@ -493,7 +493,10 @@ class VllmRunner:
493
outputs.append(embedding)
494
return outputs
495
496
- def __del__(self):
496
+ def __enter__(self):
497
+ return self
498
+
499
+ def __exit__(self, exc_type, exc_value, traceback):
500
del self.model
501
cleanup()
502
tests/distributed/test_basic_distributed_correctness.py
+7
-8
@@ -45,14 +45,13 @@ def test_models(
45
with hf_runner(model, dtype=dtype) as hf_model:
46
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
48
- vllm_model = vllm_runner(
49
- model,
50
- dtype=dtype,
51
- tensor_parallel_size=2,
52
- enforce_eager=enforce_eager,
53
- distributed_executor_backend=distributed_executor_backend)
54
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
55
- del vllm_model
48
+ with vllm_runner(model,
49
+ dtype=dtype,
50
+ tensor_parallel_size=2,
51
+ enforce_eager=enforce_eager,
52
+ distributed_executor_backend=distributed_executor_backend
53
+ ) as vllm_model:
54
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
55
56
for i in range(len(example_prompts)):
57
hf_output_ids, hf_output_str = hf_outputs[i]
tests/distributed/test_chunked_prefill_distributed.py
+10
-11
@@ -48,17 +48,16 @@ def test_models(
48
with hf_runner(model, dtype=dtype) as hf_model:
49
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
50
51
- vllm_model = vllm_runner(
52
- model,
53
- dtype=dtype,
54
- tensor_parallel_size=2,
55
- max_num_seqs=max_num_seqs,
56
- enable_chunked_prefill=enable_chunked_prefill,
57
- max_num_batched_tokens=max_num_batched_tokens,
58
- distributed_executor_backend=distributed_executor_backend,
59
- )
60
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61
- del vllm_model
51
+ with vllm_runner(
52
+ model,
53
+ dtype=dtype,
54
+ tensor_parallel_size=2,
55
+ max_num_seqs=max_num_seqs,
56
+ enable_chunked_prefill=enable_chunked_prefill,
57
+ max_num_batched_tokens=max_num_batched_tokens,
58
+ distributed_executor_backend=distributed_executor_backend,
59
+ ) as vllm_model:
60
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61
62
for i in range(len(example_prompts)):
63
hf_output_ids, hf_output_str = hf_outputs[i]
tests/engine/test_stop_reason.py
+2
-3
@@ -19,9 +19,8 @@ MAX_TOKENS = 1024
19
20
@pytest.fixture
21
def vllm_model(vllm_runner):
22
- vllm_model = vllm_runner(MODEL)
23
- yield vllm_model
24
- del vllm_model
22
+ with vllm_runner(MODEL) as vllm_model:
23
+ yield vllm_model
24
25
26
def test_stop_reason(vllm_model, example_prompts):
tests/engine/test_stop_strings.py
+2
-1
@@ -10,7 +10,8 @@ MAX_TOKENS = 200
10
11
@pytest.fixture(scope="session")
12
def vllm_model(vllm_runner):
13
- return vllm_runner(MODEL)
13
+ with vllm_runner(MODEL) as vllm_model:
14
+ yield vllm_model
15
16
17
@pytest.mark.skip_global_cleanup
tests/metrics/test_metrics.py
+42
-42
@@ -23,23 +23,25 @@ def test_metric_counter_prompt_tokens(
23
dtype: str,
24
max_tokens: int,
25
) -> None:
26
- vllm_model = vllm_runner(model,
27
- dtype=dtype,
28
- disable_log_stats=False,
29
- gpu_memory_utilization=0.4)
30
- tokenizer = vllm_model.model.get_tokenizer()
31
- prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
32
- # This test needs at least 2 prompts in a batch of different lengths to
33
- # verify their token count is correct despite padding.
34
- assert len(example_prompts) > 1, "at least 2 prompts are required"
35
- assert prompt_token_counts[0] != prompt_token_counts[1], (
36
- "prompts of different lengths are required")
37
- vllm_prompt_token_count = sum(prompt_token_counts)
38
-
39
- _ = vllm_model.generate_greedy(example_prompts, max_tokens)
40
- stat_logger = vllm_model.model.llm_engine.stat_logger
41
- metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
42
- **stat_logger.labels)._value.get()
26
+ with vllm_runner(model,
27
+ dtype=dtype,
28
+ disable_log_stats=False,
29
+ gpu_memory_utilization=0.4) as vllm_model:
30
+ tokenizer = vllm_model.model.get_tokenizer()
31
+ prompt_token_counts = [
32
+ len(tokenizer.encode(p)) for p in example_prompts
33
+ ]
34
+ # This test needs at least 2 prompts in a batch of different lengths to
35
+ # verify their token count is correct despite padding.
36
+ assert len(example_prompts) > 1, "at least 2 prompts are required"
37
+ assert prompt_token_counts[0] != prompt_token_counts[1], (
38
+ "prompts of different lengths are required")
39
+ vllm_prompt_token_count = sum(prompt_token_counts)
40
+
41
+ _ = vllm_model.generate_greedy(example_prompts, max_tokens)
42
+ stat_logger = vllm_model.model.llm_engine.stat_logger
43
+ metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
44
+ **stat_logger.labels)._value.get()
45
46
assert vllm_prompt_token_count == metric_count, (
47
f"prompt token count: {vllm_prompt_token_count!r}\n"
@@ -56,22 +58,22 @@ def test_metric_counter_generation_tokens(
58
dtype: str,
59
max_tokens: int,
60
) -> None:
59
- vllm_model = vllm_runner(model,
60
- dtype=dtype,
61
- disable_log_stats=False,
62
- gpu_memory_utilization=0.4)
63
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
64
- tokenizer = vllm_model.model.get_tokenizer()
65
- stat_logger = vllm_model.model.llm_engine.stat_logger
66
- metric_count = stat_logger.metrics.counter_generation_tokens.labels(
67
- **stat_logger.labels)._value.get()
68
- vllm_generation_count = 0
69
- for i in range(len(example_prompts)):
70
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
71
- prompt_ids = tokenizer.encode(example_prompts[i])
72
- # vllm_output_ids contains both prompt tokens and generation tokens.
73
- # We're interested only in the count of the generation tokens.
74
- vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
61
+ with vllm_runner(model,
62
+ dtype=dtype,
63
+ disable_log_stats=False,
64
+ gpu_memory_utilization=0.4) as vllm_model:
65
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66
+ tokenizer = vllm_model.model.get_tokenizer()
67
+ stat_logger = vllm_model.model.llm_engine.stat_logger
68
+ metric_count = stat_logger.metrics.counter_generation_tokens.labels(
69
+ **stat_logger.labels)._value.get()
70
+ vllm_generation_count = 0
71
+ for i in range(len(example_prompts)):
72
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
73
+ prompt_ids = tokenizer.encode(example_prompts[i])
74
+ # vllm_output_ids contains both prompt tokens and generation tokens.
75
+ # We're interested only in the count of the generation tokens.
76
+ vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
77
78
assert vllm_generation_count == metric_count, (
79
f"generation token count: {vllm_generation_count!r}\n"
@@ -85,15 +87,13 @@ def test_metric_counter_generation_tokens(
87
[None, [], ["ModelName0"], ["ModelName0", "ModelName1", "ModelName2"]])
88
def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
89
served_model_name: List[str]) -> None:
88
- vllm_model = vllm_runner(model,
89
- dtype=dtype,
90
- disable_log_stats=False,
91
- gpu_memory_utilization=0.3,
92
- served_model_name=served_model_name)
93
- stat_logger = vllm_model.model.llm_engine.stat_logger
94
- metrics_tag_content = stat_logger.labels["model_name"]
95
-
96
- del vllm_model
90
+ with vllm_runner(model,
91
+ dtype=dtype,
92
+ disable_log_stats=False,
93
+ gpu_memory_utilization=0.3,
94
+ served_model_name=served_model_name) as vllm_model:
95
+ stat_logger = vllm_model.model.llm_engine.stat_logger
96
+ metrics_tag_content = stat_logger.labels["model_name"]
97
98
if served_model_name is None or served_model_name == []:
99
assert metrics_tag_content == model, (
tests/models/test_aqlm.py
+3
-4
@@ -82,10 +82,9 @@ def test_models(
82
num_logprobs: int,
83
) -> None:
84
85
- vllm_model = vllm_runner(model, dtype=dtype)
86
- vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
87
- max_tokens,
88
- num_logprobs)
85
+ with vllm_runner(model, dtype=dtype) as vllm_model:
86
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
87
+ example_prompts, max_tokens, num_logprobs)
88
89
# loop through the prompts to compare against the ground truth generations
90
for prompt_idx in range(len(example_prompts)):
tests/models/test_big_models.py
+7
-9
@@ -37,9 +37,8 @@ def test_models(
37
with hf_runner(model, dtype=dtype) as hf_model:
38
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40
- vllm_model = vllm_runner(model, dtype=dtype)
41
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
- del vllm_model
40
+ with vllm_runner(model, dtype=dtype) as vllm_model:
41
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
43
for i in range(len(example_prompts)):
44
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -57,9 +56,8 @@ def test_model_print(
56
model: str,
57
dtype: str,
58
) -> None:
60
- vllm_model = vllm_runner(model, dtype=dtype)
61
- # This test is for verifying whether the model's extra_repr
62
- # can be printed correctly.
63
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
64
- model_runner.model)
65
- del vllm_model
59
+ with vllm_runner(model, dtype=dtype) as vllm_model:
60
+ # This test is for verifying whether the model's extra_repr
61
+ # can be printed correctly.
62
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
63
+ model_runner.model)
tests/models/test_embedding.py
+2
-3
@@ -31,9 +31,8 @@ def test_models(
31
with hf_runner(model, dtype=dtype, is_embedding_model=True) as hf_model:
32
hf_outputs = hf_model.encode(example_prompts)
33
34
- vllm_model = vllm_runner(model, dtype=dtype)
35
- vllm_outputs = vllm_model.encode(example_prompts)
36
- del vllm_model
34
+ with vllm_runner(model, dtype=dtype) as vllm_model:
35
+ vllm_outputs = vllm_model.encode(example_prompts)
36
37
similarities = compare_embeddings(hf_outputs, vllm_outputs)
38
all_similarities = torch.stack(similarities)
tests/models/test_gptq_marlin.py
+17
-20
@@ -70,32 +70,29 @@ def test_models(
70
model_name, revision = model
71
72
# Run marlin.
73
- gptq_marlin_model = vllm_runner(model_name=model_name,
74
- revision=revision,
75
- dtype=dtype,
76
- quantization="marlin",
77
- max_model_len=MAX_MODEL_LEN,
78
- tensor_parallel_size=1)
79
-
80
- gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
81
- example_prompts[:-1], max_tokens, num_logprobs)
82
- del gptq_marlin_model
73
+ with vllm_runner(model_name=model_name,
74
+ revision=revision,
75
+ dtype=dtype,
76
+ quantization="marlin",
77
+ max_model_len=MAX_MODEL_LEN,
78
+ tensor_parallel_size=1) as gptq_marlin_model:
79
+
80
+ gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
81
+ example_prompts[:-1], max_tokens, num_logprobs)
82
_ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
83
84
# Run gptq.
85
# The naive gptq kernel doesn't support bf16 yet.
86
# Here we always compare fp16/bf16 gpt marlin kernel
87
# to fp16 gptq kernel.
89
- gptq_model = vllm_runner(model_name=model_name,
90
- revision=revision,
91
- dtype="half",
92
- quantization="gptq",
93
- max_model_len=MAX_MODEL_LEN,
94
- tensor_parallel_size=1)
95
- gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts[:-1],
96
- max_tokens,
97
- num_logprobs)
98
- del gptq_model
88
+ with vllm_runner(model_name=model_name,
89
+ revision=revision,
90
+ dtype="half",
91
+ quantization="gptq",
92
+ max_model_len=MAX_MODEL_LEN,
93
+ tensor_parallel_size=1) as gptq_model:
94
+ gptq_outputs = gptq_model.generate_greedy_logprobs(
95
+ example_prompts[:-1], max_tokens, num_logprobs)
96
97
check_logprobs_close(
98
outputs_0_lst=gptq_outputs,
tests/models/test_gptq_marlin_24.py
+9
-13
@@ -61,20 +61,16 @@ def test_models(
61
max_tokens: int,
62
num_logprobs: int,
63
) -> None:
64
- marlin_24_model = vllm_runner(model_pair.model_marlin,
65
- dtype=dtype,
66
- quantization="gptq_marlin_24")
67
- marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
68
- example_prompts, max_tokens, num_logprobs)
69
- del marlin_24_model
64
+ with vllm_runner(model_pair.model_marlin,
65
+ dtype=dtype,
66
+ quantization="gptq_marlin_24") as marlin_24_model:
67
+ marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
68
+ example_prompts, max_tokens, num_logprobs)
69
71
- gptq_model = vllm_runner(model_pair.model_gptq,
72
- dtype=dtype,
73
- quantization="gptq")
74
- gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
75
- max_tokens,
76
- num_logprobs)
77
- del gptq_model
70
+ with vllm_runner(model_pair.model_gptq, dtype=dtype,
71
+ quantization="gptq") as gptq_model:
72
+ gptq_outputs = gptq_model.generate_greedy_logprobs(
73
+ example_prompts, max_tokens, num_logprobs)
74
75
check_logprobs_close(
76
outputs_0_lst=gptq_outputs,
tests/models/test_llava.py
+7
-8
@@ -94,14 +94,13 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
94
for p in HF_IMAGE_PROMPTS
95
]
96
97
- vllm_model = vllm_runner(model_id,
98
- dtype=dtype,
99
- enforce_eager=True,
100
- **vlm_config.as_cli_args_dict())
101
- vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
102
- max_tokens,
103
- images=vllm_images)
104
- del vllm_model
97
+ with vllm_runner(model_id,
98
+ dtype=dtype,
99
+ enforce_eager=True,
100
+ **vlm_config.as_cli_args_dict()) as vllm_model:
101
+ vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
102
+ max_tokens,
103
+ images=vllm_images)
104
105
for i in range(len(HF_IMAGE_PROMPTS)):
106
hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/test_marlin.py
+10
-14
@@ -59,20 +59,16 @@ def test_models(
59
max_tokens: int,
60
num_logprobs: int,
61
) -> None:
62
- marlin_model = vllm_runner(model_pair.model_marlin,
63
- dtype=dtype,
64
- quantization="marlin")
65
- marlin_outputs = marlin_model.generate_greedy_logprobs(
66
- example_prompts, max_tokens, num_logprobs)
67
- del marlin_model
68
-
69
- gptq_model = vllm_runner(model_pair.model_gptq,
70
- dtype=dtype,
71
- quantization="gptq")
72
- gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
73
- max_tokens,
74
- num_logprobs)
75
- del gptq_model
62
+ with vllm_runner(model_pair.model_marlin,
63
+ dtype=dtype,
64
+ quantization="marlin") as marlin_model:
65
+ marlin_outputs = marlin_model.generate_greedy_logprobs(
66
+ example_prompts, max_tokens, num_logprobs)
67
+
68
+ with vllm_runner(model_pair.model_gptq, dtype=dtype,
69
+ quantization="gptq") as gptq_model:
70
+ gptq_outputs = gptq_model.generate_greedy_logprobs(
71
+ example_prompts, max_tokens, num_logprobs)
72
73
check_logprobs_close(
74
outputs_0_lst=gptq_outputs,
tests/models/test_mistral.py
+3
-5
@@ -30,11 +30,9 @@ def test_models(
30
hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
example_prompts, max_tokens, num_logprobs)
32
33
- vllm_model = vllm_runner(model, dtype=dtype)
34
- vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
35
- max_tokens,
36
- num_logprobs)
37
- del vllm_model
33
+ with vllm_runner(model, dtype=dtype) as vllm_model:
34
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
35
+ example_prompts, max_tokens, num_logprobs)
36
check_logprobs_close(
37
outputs_0_lst=hf_outputs,
38
outputs_1_lst=vllm_outputs,
tests/models/test_models.py
+7
-9
@@ -37,9 +37,8 @@ def test_models(
37
with hf_runner(model, dtype=dtype) as hf_model:
38
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
40
- vllm_model = vllm_runner(model, dtype=dtype)
41
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
- del vllm_model
40
+ with vllm_runner(model, dtype=dtype) as vllm_model:
41
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
43
for i in range(len(example_prompts)):
44
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -57,9 +56,8 @@ def test_model_print(
56
model: str,
57
dtype: str,
58
) -> None:
60
- vllm_model = vllm_runner(model, dtype=dtype)
61
- # This test is for verifying whether the model's extra_repr
62
- # can be printed correctly.
63
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
64
- model_runner.model)
65
- del vllm_model
59
+ with vllm_runner(model, dtype=dtype) as vllm_model:
60
+ # This test is for verifying whether the model's extra_repr
61
+ # can be printed correctly.
62
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
63
+ model_runner.model)
tests/quantization/test_bitsandbytes.py
+62
-62
@@ -16,65 +16,65 @@ capability = capability[0] * 10 + capability[1]
16
capability < QUANTIZATION_METHODS['bitsandbytes'].get_min_capability(),
17
reason='bitsandbytes is not supported on this GPU type.')
18
def test_load_bnb_model(vllm_runner) -> None:
19
- llm = vllm_runner('huggyllama/llama-7b',
20
- quantization='bitsandbytes',
21
- load_format='bitsandbytes',
22
- enforce_eager=True)
23
-
24
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
25
-
26
- # check the weights in MLP & SelfAttention are quantized to torch.uint8
27
- qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28
- assert qweight.dtype == torch.uint8, (
29
- f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30
-
31
- qweight = model.model.layers[0].mlp.down_proj.qweight
32
- assert qweight.dtype == torch.uint8, (
33
- f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34
-
35
- qweight = model.model.layers[0].self_attn.o_proj.qweight
36
- assert qweight.dtype == torch.uint8, (
37
- f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38
-
39
- qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40
- assert qweight.dtype == torch.uint8, (
41
- f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42
-
43
- # some weights should not be quantized
44
- weight = model.lm_head.weight
45
- assert weight.dtype != torch.uint8, (
46
- 'lm_head weight dtype should not be torch.uint8')
47
-
48
- weight = model.model.embed_tokens.weight
49
- assert weight.dtype != torch.uint8, (
50
- 'embed_tokens weight dtype should not be torch.uint8')
51
-
52
- weight = model.model.layers[0].input_layernorm.weight
53
- assert weight.dtype != torch.uint8, (
54
- 'input_layernorm weight dtype should not be torch.uint8')
55
-
56
- weight = model.model.layers[0].post_attention_layernorm.weight
57
- assert weight.dtype != torch.uint8, (
58
- 'input_layernorm weight dtype should not be torch.uint8')
59
-
60
- # check the output of the model is expected
61
- sampling_params = SamplingParams(temperature=0.0,
62
- logprobs=1,
63
- prompt_logprobs=1,
64
- max_tokens=8)
65
-
66
- prompts = ['That which does not kill us', 'To be or not to be,']
67
- expected_outputs = [
68
- 'That which does not kill us makes us stronger.',
69
- 'To be or not to be, that is the question.'
70
- ]
71
- outputs = llm.generate(prompts, sampling_params=sampling_params)
72
-
73
- assert len(outputs) == len(prompts)
74
-
75
- for index in range(len(outputs)):
76
- # compare the first line of the output
77
- actual_output = outputs[index][1][0].split('\n', 1)[0]
78
- expected_output = expected_outputs[index].split('\n', 1)[0]
79
- assert actual_output == expected_output, (
80
- f'Expected: {expected_output}, but got: {actual_output}')
19
+ with vllm_runner('huggyllama/llama-7b',
20
+ quantization='bitsandbytes',
21
+ load_format='bitsandbytes',
22
+ enforce_eager=True) as llm:
23
+
24
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
25
+
26
+ # check the weights in MLP & SelfAttention are quantized to torch.uint8
27
+ qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28
+ assert qweight.dtype == torch.uint8, (
29
+ f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30
+
31
+ qweight = model.model.layers[0].mlp.down_proj.qweight
32
+ assert qweight.dtype == torch.uint8, (
33
+ f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34
+
35
+ qweight = model.model.layers[0].self_attn.o_proj.qweight
36
+ assert qweight.dtype == torch.uint8, (
37
+ f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38
+
39
+ qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40
+ assert qweight.dtype == torch.uint8, (
41
+ f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42
+
43
+ # some weights should not be quantized
44
+ weight = model.lm_head.weight
45
+ assert weight.dtype != torch.uint8, (
46
+ 'lm_head weight dtype should not be torch.uint8')
47
+
48
+ weight = model.model.embed_tokens.weight
49
+ assert weight.dtype != torch.uint8, (
50
+ 'embed_tokens weight dtype should not be torch.uint8')
51
+
52
+ weight = model.model.layers[0].input_layernorm.weight
53
+ assert weight.dtype != torch.uint8, (
54
+ 'input_layernorm weight dtype should not be torch.uint8')
55
+
56
+ weight = model.model.layers[0].post_attention_layernorm.weight
57
+ assert weight.dtype != torch.uint8, (
58
+ 'input_layernorm weight dtype should not be torch.uint8')
59
+
60
+ # check the output of the model is expected
61
+ sampling_params = SamplingParams(temperature=0.0,
62
+ logprobs=1,
63
+ prompt_logprobs=1,
64
+ max_tokens=8)
65
+
66
+ prompts = ['That which does not kill us', 'To be or not to be,']
67
+ expected_outputs = [
68
+ 'That which does not kill us makes us stronger.',
69
+ 'To be or not to be, that is the question.'
70
+ ]
71
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
72
+
73
+ assert len(outputs) == len(prompts)
74
+
75
+ for index in range(len(outputs)):
76
+ # compare the first line of the output
77
+ actual_output = outputs[index][1][0].split('\n', 1)[0]
78
+ expected_output = expected_outputs[index].split('\n', 1)[0]
79
+ assert actual_output == expected_output, (
80
+ f'Expected: {expected_output}, but got: {actual_output}')
tests/quantization/test_compressed_tensors.py
+33
-30
@@ -12,42 +12,45 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
12
13
def test_compressed_tensors_w8a8_static_setup(vllm_runner):
14
model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
15
- llm = vllm_runner(model_path, quantization="sparseml", enforce_eager=True)
16
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
17
- layer = model.model.layers[0]
15
+ with vllm_runner(model_path, quantization="sparseml",
16
+ enforce_eager=True) as llm:
17
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
18
+ layer = model.model.layers[0]
19
19
- qkv_proj = layer.self_attn.qkv_proj
20
- o_proj = layer.self_attn.o_proj
21
- gate_up_proj = layer.mlp.gate_up_proj
22
- down_proj = layer.mlp.down_proj
20
+ qkv_proj = layer.self_attn.qkv_proj
21
+ o_proj = layer.self_attn.o_proj
22
+ gate_up_proj = layer.mlp.gate_up_proj
23
+ down_proj = layer.mlp.down_proj
24
24
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
25
- assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
26
- assert isinstance(gate_up_proj.quant_method, CompressedTensorsLinearMethod)
27
- assert isinstance(down_proj.quant_method, CompressedTensorsLinearMethod)
25
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
26
+ assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
27
+ assert isinstance(gate_up_proj.quant_method,
28
+ CompressedTensorsLinearMethod)
29
+ assert isinstance(down_proj.quant_method,
30
+ CompressedTensorsLinearMethod)
31
29
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
32
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
33
31
- assert qkv_proj.weight.dtype is torch.int8
32
- assert o_proj.weight.dtype is torch.int8
33
- assert gate_up_proj.weight.dtype is torch.int8
34
+ assert qkv_proj.weight.dtype is torch.int8
35
+ assert o_proj.weight.dtype is torch.int8
36
+ assert gate_up_proj.weight.dtype is torch.int8
37
35
- assert qkv_proj.weight_scale.shard_splitter is not None
36
- assert qkv_proj.weight_scale.logical_widths is not None
37
- assert qkv_proj.input_scale.dtype is torch.float32
38
+ assert qkv_proj.weight_scale.shard_splitter is not None
39
+ assert qkv_proj.weight_scale.logical_widths is not None
40
+ assert qkv_proj.input_scale.dtype is torch.float32
41
42
43
def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
44
model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
42
- llm = vllm_runner(model_path,
43
- quantization="sparseml",
44
- enforce_eager=True,
45
- dtype=torch.float16)
46
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
47
- layer = model.model.layers[0]
48
-
49
- qkv_proj = layer.self_attn.qkv_proj
50
-
51
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
52
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
53
- assert qkv_proj.weight.dtype is torch.int8
45
+ with vllm_runner(model_path,
46
+ quantization="sparseml",
47
+ enforce_eager=True,
48
+ dtype=torch.float16) as llm:
49
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
50
+ layer = model.model.layers[0]
51
+
52
+ qkv_proj = layer.self_attn.qkv_proj
53
+
54
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
55
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
56
+ assert qkv_proj.weight.dtype is torch.int8
tests/quantization/test_fp8.py
+5
-5
@@ -16,9 +16,9 @@ capability = capability[0] * 10 + capability[1]
16
capability < QUANTIZATION_METHODS["fp8"].get_min_capability(),
17
reason="FP8 is not supported on this GPU type.")
18
def test_load_fp16_model(vllm_runner) -> None:
19
- llm = vllm_runner("facebook/opt-125m", quantization="fp8")
19
+ with vllm_runner("facebook/opt-125m", quantization="fp8") as llm:
20
21
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
22
- fc1 = model.model.decoder.layers[0].fc1
23
- assert isinstance(fc1.quant_method, Fp8LinearMethod)
24
- assert fc1.weight.dtype == torch.float8_e4m3fn
21
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
22
+ fc1 = model.model.decoder.layers[0].fc1
23
+ assert isinstance(fc1.quant_method, Fp8LinearMethod)
24
+ assert fc1.weight.dtype == torch.float8_e4m3fn
tests/samplers/test_beam_search.py
+3
-10
@@ -2,10 +2,8 @@
2
3
Run `pytest tests/samplers/test_beam_search.py`.
4
"""
5
-import gc
5
6
import pytest
8
-import torch
7
8
# FIXME(zhuohan): The test can not pass if we:
9
# 1. Increase max_tokens to 256.
@@ -34,14 +32,9 @@ def test_beam_search_single_input(
32
hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
33
max_tokens)
34
37
- vllm_model = vllm_runner(model, dtype=dtype)
38
- vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
39
- max_tokens)
40
- del vllm_model
41
- # NOTE(woosuk): For some reason, the following GC is required to avoid
42
- # GPU OOM errors in the following tests using `vllm_runner`.
43
- gc.collect()
44
- torch.cuda.empty_cache()
35
+ with vllm_runner(model, dtype=dtype) as vllm_model:
36
+ vllm_outputs = vllm_model.generate_beam_search(example_prompts,
37
+ beam_width, max_tokens)
38
39
for i in range(len(example_prompts)):
40
hf_output_ids, _ = hf_outputs[i]
tests/samplers/test_ignore_eos.py
+8
-7
@@ -22,11 +22,12 @@ def test_ignore_eos(
22
dtype: str,
23
max_tokens: int,
24
) -> None:
25
- vllm_model = vllm_runner(model, dtype=dtype)
26
- sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
25
+ with vllm_runner(model, dtype=dtype) as vllm_model:
26
+ sampling_params = SamplingParams(max_tokens=max_tokens,
27
+ ignore_eos=True)
28
28
- for prompt in example_prompts:
29
- ignore_eos_output = vllm_model.model.generate(
30
- prompt, sampling_params=sampling_params)
31
- output_length = len(ignore_eos_output[0].outputs[0].token_ids)
32
- assert output_length == max_tokens
29
+ for prompt in example_prompts:
30
+ ignore_eos_output = vllm_model.model.generate(
31
+ prompt, sampling_params=sampling_params)
32
+ output_length = len(ignore_eos_output[0].outputs[0].token_ids)
33
+ assert output_length == max_tokens
tests/samplers/test_logits_processor.py
+41
-41
@@ -14,46 +14,46 @@ def test_logits_processor_force_generate(
14
model: str,
15
dtype: str,
16
) -> None:
17
- vllm_model = vllm_runner(model, dtype=dtype)
18
- tokenizer = vllm_model.model.get_tokenizer()
19
- repeat_times = 2
20
- enforced_answers = " vLLM"
21
- vllm_token_ids = tokenizer.encode(enforced_answers,
22
- add_special_tokens=False)
23
- max_tokens = len(vllm_token_ids) * repeat_times
24
-
25
- def pick_vllm(token_ids, logits):
26
- token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27
- logits[token_id] = torch.finfo(logits.dtype).max
28
- return logits
29
-
30
- params_with_logprobs = SamplingParams(
31
- logits_processors=[pick_vllm],
32
- prompt_logprobs=3,
33
- max_tokens=max_tokens,
34
- )
35
-
36
- # test logits_processors when prompt_logprobs is not None
37
- vllm_model.model._add_request(
38
- example_prompts[0],
39
- params=params_with_logprobs,
40
- )
41
-
42
- # test prompt_logprobs is not None
43
- vllm_model.model._add_request(
44
- example_prompts[1],
45
- params=SamplingParams(
17
+ with vllm_runner(model, dtype=dtype) as vllm_model:
18
+ tokenizer = vllm_model.model.get_tokenizer()
19
+ repeat_times = 2
20
+ enforced_answers = " vLLM"
21
+ vllm_token_ids = tokenizer.encode(enforced_answers,
22
+ add_special_tokens=False)
23
+ max_tokens = len(vllm_token_ids) * repeat_times
24
+
25
+ def pick_vllm(token_ids, logits):
26
+ token_id = vllm_token_ids[len(token_ids) % len(vllm_token_ids)]
27
+ logits[token_id] = torch.finfo(logits.dtype).max
28
+ return logits
29
+
30
+ params_with_logprobs = SamplingParams(
31
+ logits_processors=[pick_vllm],
32
prompt_logprobs=3,
33
max_tokens=max_tokens,
48
- ),
49
- )
50
-
51
- # test grouped requests
52
- vllm_model.model._add_request(
53
- example_prompts[2],
54
- params=SamplingParams(max_tokens=max_tokens),
55
- )
56
-
57
- outputs = vllm_model.model._run_engine(use_tqdm=False)
58
-
59
- assert outputs[0].outputs[0].text == enforced_answers * repeat_times
34
+ )
35
+
36
+ # test logits_processors when prompt_logprobs is not None
37
+ vllm_model.model._add_request(
38
+ example_prompts[0],
39
+ params=params_with_logprobs,
40
+ )
41
+
42
+ # test prompt_logprobs is not None
43
+ vllm_model.model._add_request(
44
+ example_prompts[1],
45
+ params=SamplingParams(
46
+ prompt_logprobs=3,
47
+ max_tokens=max_tokens,
48
+ ),
49
+ )
50
+
51
+ # test grouped requests
52
+ vllm_model.model._add_request(
53
+ example_prompts[2],
54
+ params=SamplingParams(max_tokens=max_tokens),
55
+ )
56
+
57
+ outputs = vllm_model.model._run_engine(use_tqdm=False)
58
+
59
+ assert outputs[0].outputs[0].text == enforced_answers * repeat_times
tests/samplers/test_logprobs.py
+15
-15
@@ -38,21 +38,21 @@ def test_get_prompt_logprobs(
38
max_tokens=max_tokens,
39
)
40
41
- vllm_model = vllm_runner(
42
- model,
43
- dtype=dtype,
44
- max_logprobs=num_top_logprobs,
45
- enable_chunked_prefill=enable_chunked_prefill,
46
- max_num_batched_tokens=max_num_batched_tokens,
47
- max_num_seqs=max_num_seqs,
48
- )
49
- vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
50
- logprobs=num_top_logprobs,
51
- prompt_logprobs=num_top_logprobs,
52
- temperature=0.0,
53
- detokenize=detokenize)
54
- vllm_results = vllm_model.model.generate(
55
- example_prompts, sampling_params=vllm_sampling_params)
41
+ with vllm_runner(
42
+ model,
43
+ dtype=dtype,
44
+ max_logprobs=num_top_logprobs,
45
+ enable_chunked_prefill=enable_chunked_prefill,
46
+ max_num_batched_tokens=max_num_batched_tokens,
47
+ max_num_seqs=max_num_seqs,
48
+ ) as vllm_model:
49
+ vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
50
+ logprobs=num_top_logprobs,
51
+ prompt_logprobs=num_top_logprobs,
52
+ temperature=0.0,
53
+ detokenize=detokenize)
54
+ vllm_results = vllm_model.model.generate(
55
+ example_prompts, sampling_params=vllm_sampling_params)
56
57
# Test whether logprobs are included in the results.
58
for result in vllm_results:
tests/samplers/test_ranks.py
+21
-17
@@ -17,16 +17,27 @@ def test_ranks(
17
num_top_logprobs = 5
18
num_prompt_logprobs = 5
19
20
- vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
21
-
22
- ## Test greedy logprobs ranks
23
- vllm_sampling_params = SamplingParams(temperature=0.0,
24
- top_p=1.0,
25
- max_tokens=max_tokens,
26
- logprobs=num_top_logprobs,
27
- prompt_logprobs=num_prompt_logprobs)
28
- vllm_results = vllm_model.generate_w_logprobs(example_prompts,
29
- vllm_sampling_params)
20
+ with vllm_runner(model, dtype=dtype,
21
+ max_logprobs=num_top_logprobs) as vllm_model:
22
+
23
+ ## Test greedy logprobs ranks
24
+ vllm_sampling_params = SamplingParams(
25
+ temperature=0.0,
26
+ top_p=1.0,
27
+ max_tokens=max_tokens,
28
+ logprobs=num_top_logprobs,
29
+ prompt_logprobs=num_prompt_logprobs)
30
+ vllm_results = vllm_model.generate_w_logprobs(example_prompts,
31
+ vllm_sampling_params)
32
+
33
+ ## Test non-greedy logprobs ranks
34
+ sampling_params = SamplingParams(temperature=1.0,
35
+ top_p=1.0,
36
+ max_tokens=max_tokens,
37
+ logprobs=num_top_logprobs,
38
+ prompt_logprobs=num_prompt_logprobs)
39
+ res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
40
+
41
for result in vllm_results:
42
assert result[2] is not None
43
assert len(result[2]) == len(result[0])
@@ -35,13 +46,6 @@ def test_ranks(
46
assert token in logprobs
47
assert logprobs[token].rank == 1
48
38
- ## Test non-greedy logprobs ranks
39
- sampling_params = SamplingParams(temperature=1.0,
40
- top_p=1.0,
41
- max_tokens=max_tokens,
42
- logprobs=num_top_logprobs,
43
- prompt_logprobs=num_prompt_logprobs)
44
- res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
49
for result in res:
50
assert result[2] is not None
51
assert len(result[2]) == len(result[0])
tests/samplers/test_seeded_generate.py
+2
-3
@@ -17,9 +17,8 @@ RANDOM_SEEDS = list(range(5))
17
18
@pytest.fixture
19
def vllm_model(vllm_runner):
20
- vllm_model = vllm_runner(MODEL, dtype="half")
21
- yield vllm_model
22
- del vllm_model
20
+ with vllm_runner(MODEL, dtype="half") as vllm_model:
21
+ yield vllm_model
22
23
24
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
tests/tensorizer_loader/test_tensorizer.py
+45
-61
@@ -1,4 +1,3 @@
1
-import gc
1
import json
2
import os
3
import subprocess
@@ -7,7 +6,6 @@ from unittest.mock import MagicMock, patch
6
import openai
7
import pytest
8
import ray
10
-import torch
9
10
from vllm import SamplingParams
11
# yapf: disable
@@ -71,47 +69,43 @@ def test_can_deserialize_s3(vllm_runner):
69
model_ref = "EleutherAI/pythia-1.4b"
70
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
71
74
- loaded_hf_model = vllm_runner(model_ref,
72
+ with vllm_runner(model_ref,
73
load_format="tensorizer",
74
model_loader_extra_config=TensorizerConfig(
75
tensorizer_uri=tensorized_path,
76
num_readers=1,
77
s3_endpoint="object.ord1.coreweave.com",
80
- ))
78
+ )) as loaded_hf_model:
79
82
- deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
80
+ deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params) # noqa: E501
81
84
- assert deserialized_outputs
82
+ assert deserialized_outputs
83
84
85
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
86
def test_deserialized_encrypted_vllm_model_has_same_outputs(
87
vllm_runner, tmp_path):
90
- vllm_model = vllm_runner(model_ref)
91
- model_path = tmp_path / (model_ref + ".tensors")
92
- key_path = tmp_path / (model_ref + ".key")
93
- outputs = vllm_model.generate(prompts, sampling_params)
94
-
95
- config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
96
- serialize_vllm_model(vllm_model.model.llm_engine,
97
- config_for_serializing,
98
- encryption_key_path=key_path)
88
+ with vllm_runner(model_ref) as vllm_model:
89
+ model_path = tmp_path / (model_ref + ".tensors")
90
+ key_path = tmp_path / (model_ref + ".key")
91
+ outputs = vllm_model.generate(prompts, sampling_params)
92
100
- del vllm_model
101
- gc.collect()
102
- torch.cuda.empty_cache()
93
+ config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
94
+ serialize_vllm_model(vllm_model.model.llm_engine,
95
+ config_for_serializing,
96
+ encryption_key_path=key_path)
97
98
config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
99
encryption_keyfile=key_path)
100
107
- loaded_vllm_model = vllm_runner(
101
+ with vllm_runner(
102
model_ref,
103
load_format="tensorizer",
110
- model_loader_extra_config=config_for_deserializing)
104
+ model_loader_extra_config=config_for_deserializing) as loaded_vllm_model: # noqa: E501
105
112
- deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
106
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params) # noqa: E501
107
114
- assert outputs == deserialized_outputs
108
+ assert outputs == deserialized_outputs
109
110
111
def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
@@ -124,17 +118,17 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
118
serializer = TensorSerializer(stream)
119
serializer.write_module(hf_model.model)
120
127
- loaded_hf_model = vllm_runner(model_ref,
121
+ with vllm_runner(model_ref,
122
load_format="tensorizer",
123
model_loader_extra_config=TensorizerConfig(
124
tensorizer_uri=model_path,
125
num_readers=1,
132
- ))
126
+ )) as loaded_hf_model:
127
134
- deserialized_outputs = loaded_hf_model.generate_greedy(
135
- prompts, max_tokens=max_tokens)
128
+ deserialized_outputs = loaded_hf_model.generate_greedy(
129
+ prompts, max_tokens=max_tokens)
130
137
- assert outputs == deserialized_outputs
131
+ assert outputs == deserialized_outputs
132
133
134
def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
@@ -148,16 +142,13 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
142
test_prompts = create_test_prompts(lora_path)
143
144
# Serialize model before deserializing and binding LoRA adapters
151
- vllm_model = vllm_runner(model_ref, )
152
- model_path = tmp_path / (model_ref + ".tensors")
145
+ with vllm_runner(model_ref, ) as vllm_model:
146
+ model_path = tmp_path / (model_ref + ".tensors")
147
154
- serialize_vllm_model(vllm_model.model.llm_engine,
155
- TensorizerConfig(tensorizer_uri=model_path))
148
+ serialize_vllm_model(vllm_model.model.llm_engine,
149
+ TensorizerConfig(tensorizer_uri=model_path))
150
157
- del vllm_model
158
- gc.collect()
159
- torch.cuda.empty_cache()
160
- loaded_vllm_model = vllm_runner(
151
+ with vllm_runner(
152
model_ref,
153
load_format="tensorizer",
154
model_loader_extra_config=TensorizerConfig(
@@ -170,10 +161,10 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
161
max_cpu_loras=2,
162
max_num_seqs=50,
163
max_model_len=1000,
173
- )
174
- process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
164
+ ) as loaded_vllm_model:
165
+ process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
166
176
- assert loaded_vllm_model
167
+ assert loaded_vllm_model
168
169
170
def test_load_without_tensorizer_load_format(vllm_runner):
@@ -186,19 +177,15 @@ def test_load_without_tensorizer_load_format(vllm_runner):
177
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
178
def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
179
## Serialize model
189
- vllm_model = vllm_runner(model_ref, )
190
- model_path = tmp_path / (model_ref + ".tensors")
191
-
192
- serialize_vllm_model(vllm_model.model.llm_engine,
193
- TensorizerConfig(tensorizer_uri=model_path))
180
+ with vllm_runner(model_ref, ) as vllm_model:
181
+ model_path = tmp_path / (model_ref + ".tensors")
182
195
- model_loader_extra_config = {
196
- "tensorizer_uri": str(model_path),
197
- }
183
+ serialize_vllm_model(vllm_model.model.llm_engine,
184
+ TensorizerConfig(tensorizer_uri=model_path))
185
199
- del vllm_model
200
- gc.collect()
201
- torch.cuda.empty_cache()
186
+ model_loader_extra_config = {
187
+ "tensorizer_uri": str(model_path),
188
+ }
189
190
## Start OpenAI API server
191
openai_args = [
@@ -260,18 +247,15 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
247
model_path = tmp_path / (model_ref + ".tensors")
248
config = TensorizerConfig(tensorizer_uri=str(model_path))
249
263
- vllm_model = vllm_runner(model_ref)
264
- outputs = vllm_model.generate(prompts, sampling_params)
265
- serialize_vllm_model(vllm_model.model.llm_engine, config)
250
+ with vllm_runner(model_ref) as vllm_model:
251
+ outputs = vllm_model.generate(prompts, sampling_params)
252
+ serialize_vllm_model(vllm_model.model.llm_engine, config)
253
267
- assert is_vllm_tensorized(config)
268
- del vllm_model
269
- gc.collect()
270
- torch.cuda.empty_cache()
254
+ assert is_vllm_tensorized(config)
255
272
- loaded_vllm_model = vllm_runner(model_ref,
273
- load_format="tensorizer",
274
- model_loader_extra_config=config)
275
- deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
256
+ with vllm_runner(model_ref,
257
+ load_format="tensorizer",
258
+ model_loader_extra_config=config) as loaded_vllm_model:
259
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params) # noqa: E501
260
277
- assert outputs == deserialized_outputs
261
+ assert outputs == deserialized_outputs