A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bug fix][Core] assert num_new_tokens == 1 fails when SamplingParams.n is not 1 and max_tokens is large & Add tests for preemption (#4451)
SangBin Cho committed
May 2, 2024 at 11:24 UTC
0d62fe58dbb58cfe4132005ce7ff37319d66981d
6 files changed
+172
-13
.buildkite/test-pipeline.yaml
+1
@@ -17,6 +17,7 @@ steps:
17
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
19
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
20
+ - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
21
22
- label: Core Test
23
command: pytest -v -s core
tests/basic_correctness/test_chunked_prefill.py
-1
@@ -55,7 +55,6 @@ def test_models(
55
)
56
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57
del vllm_model
58
- print(vllm_outputs[0])
58
59
for i in range(len(example_prompts)):
60
hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_preemption.py
new
+138
@@ -0,0 +1,138 @@
1
+"""Compare the short outputs of HF and vLLM when using greedy sampling.
2
+
3
+VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 has to be set before running this test.
4
+
5
+Run `VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1
6
+pytest tests/basic_correctness/test_preemption.py`.
7
+"""
8
+import pytest
9
+
10
+from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
11
+ ENABLE_ARTIFICIAL_PREEMPT)
12
+
13
+MODELS = [
14
+ "facebook/opt-125m",
15
+]
16
+
17
+assert ENABLE_ARTIFICIAL_PREEMPT is True, (
18
+ "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1. "
19
+ "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest "
20
+ "tests/basic_correctness/test_preemption.py`")
21
+
22
+
23
+@pytest.mark.parametrize("model", MODELS)
24
+@pytest.mark.parametrize("dtype", ["half"])
25
+@pytest.mark.parametrize("max_tokens", [96])
26
+@pytest.mark.parametrize("chunked_prefill_token_size", [16])
27
+def test_chunked_prefill_recompute(
28
+ hf_runner,
29
+ vllm_runner,
30
+ example_prompts,
31
+ model: str,
32
+ dtype: str,
33
+ max_tokens: int,
34
+ chunked_prefill_token_size: int,
35
+) -> None:
36
+ """Ensure that chunked prefill works with preemption."""
37
+ max_num_seqs = min(chunked_prefill_token_size, 256)
38
+ enable_chunked_prefill = False
39
+ max_num_batched_tokens = None
40
+ if chunked_prefill_token_size != -1:
41
+ enable_chunked_prefill = True
42
+ max_num_batched_tokens = chunked_prefill_token_size
43
+
44
+ hf_model = hf_runner(model, dtype=dtype)
45
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
46
+ del hf_model
47
+
48
+ vllm_model = vllm_runner(
49
+ model,
50
+ dtype=dtype,
51
+ max_num_batched_tokens=max_num_batched_tokens,
52
+ enable_chunked_prefill=enable_chunked_prefill,
53
+ max_num_seqs=max_num_seqs,
54
+ )
55
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
57
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
58
+ del vllm_model
59
+
60
+ for i in range(len(example_prompts)):
61
+ hf_output_ids, hf_output_str = hf_outputs[i]
62
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
63
+ assert hf_output_str == vllm_output_str, (
64
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
65
+ assert hf_output_ids == vllm_output_ids, (
66
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
67
+
68
+
69
+@pytest.mark.parametrize("model", MODELS)
70
+@pytest.mark.parametrize("dtype", ["float"])
71
+@pytest.mark.parametrize("max_tokens", [96])
72
+def test_preemption(
73
+ hf_runner,
74
+ vllm_runner,
75
+ example_prompts,
76
+ model: str,
77
+ dtype: str,
78
+ max_tokens: int,
79
+) -> None:
80
+ """By default, recompute preemption is enabled"""
81
+
82
+ hf_model = hf_runner(model, dtype=dtype)
83
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
84
+ del hf_model
85
+
86
+ vllm_model = vllm_runner(
87
+ model,
88
+ dtype=dtype,
89
+ )
90
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
91
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
92
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
93
+ del vllm_model
94
+
95
+ for i in range(len(example_prompts)):
96
+ hf_output_ids, hf_output_str = hf_outputs[i]
97
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
98
+ assert hf_output_str == vllm_output_str, (
99
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
100
+ assert hf_output_ids == vllm_output_ids, (
101
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
102
+
103
+
104
+@pytest.mark.parametrize("model", MODELS)
105
+@pytest.mark.parametrize("dtype", ["float"])
106
+@pytest.mark.parametrize("max_tokens", [96])
107
+@pytest.mark.parametrize("beam_width", [4])
108
+def test_swap(
109
+ hf_runner,
110
+ vllm_runner,
111
+ example_prompts,
112
+ model: str,
113
+ dtype: str,
114
+ max_tokens: int,
115
+ beam_width: int,
116
+) -> None:
117
+ """Use beam search enables swapping."""
118
+ example_prompts = example_prompts[:1]
119
+ hf_model = hf_runner(model, dtype=dtype)
120
+ hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
121
+ max_tokens)
122
+ del hf_model
123
+
124
+ vllm_model = vllm_runner(model, dtype=dtype, swap_space=10)
125
+ vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
126
+ max_tokens)
127
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
128
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
129
+ del vllm_model
130
+
131
+ for i in range(len(example_prompts)):
132
+ hf_output_ids, _ = hf_outputs[i]
133
+ vllm_output_ids, _ = vllm_outputs[i]
134
+ assert len(hf_output_ids) == len(vllm_output_ids)
135
+ for j in range(len(hf_output_ids)):
136
+ assert hf_output_ids[j] == vllm_output_ids[j], (
137
+ f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
138
+ f"vLLM: {vllm_output_ids}")
tests/conftest.py
+2
-1
@@ -296,6 +296,7 @@ class VllmRunner:
296
tensor_parallel_size: int = 1,
297
block_size: int = 16,
298
enable_chunked_prefill: bool = False,
299
+ swap_space=4,
300
**kwargs,
301
) -> None:
302
self.model = LLM(
@@ -303,7 +304,7 @@ class VllmRunner:
304
tokenizer=tokenizer_name,
305
trust_remote_code=True,
306
dtype=dtype,
306
- swap_space=0,
307
+ swap_space=swap_space,
308
disable_log_stats=disable_log_stats,
309
tensor_parallel_size=tensor_parallel_size,
310
max_model_len=max_model_len,
tests/spec_decode/test_spec_decode_worker.py
+3
-3
@@ -33,7 +33,7 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
33
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
34
metrics_collector)
35
36
- exception_secret = 'artifical stop'
36
+ exception_secret = 'artificial stop'
37
draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
38
39
execute_model_data, _, _ = create_batch(batch_size, k)
@@ -101,7 +101,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
101
proposal_probs=proposal_probs,
102
proposal_lens=proposal_lens)
103
104
- exception_secret = 'artifical stop'
104
+ exception_secret = 'artificial stop'
105
target_worker.execute_model.side_effect = ValueError(exception_secret)
106
107
with pytest.raises(ValueError, match=exception_secret):
@@ -197,7 +197,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
197
198
target_worker.execute_model.return_value = [target_output[0]]
199
200
- exception_secret = 'artifical stop'
200
+ exception_secret = 'artificial stop'
201
rejection_sampler.side_effect = ValueError(exception_secret)
202
203
with pytest.raises(ValueError, match=exception_secret):
vllm/core/scheduler.py
+28
-8
@@ -1,4 +1,6 @@
1
import enum
2
+import os
3
+import random
4
import time
5
from collections import deque
6
from dataclasses import dataclass, field
@@ -15,6 +17,13 @@ from vllm.utils import merge_dicts
17
18
logger = init_logger(__name__)
19
20
+# Test-only. If configured, decode is preempted with
21
+# ARTIFICIAL_PREEMPTION_PROB% probability.
22
+ENABLE_ARTIFICIAL_PREEMPT = bool(
23
+ os.getenv("VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT", False)) # noqa
24
+ARTIFICIAL_PREEMPTION_PROB = 0.5
25
+ARTIFICIAL_PREEMPTION_MAX_CNT = 500
26
+
27
28
class PreemptionMode(enum.Enum):
29
"""Preemption modes.
@@ -286,6 +295,13 @@ class Scheduler:
295
# Latency of the last prompt step
296
self.last_prompt_latency = 0.0
297
298
+ # The following field is test-only. It is used to inject artificial
299
+ # preemption.
300
+ self.enable_artificial_preemption = ENABLE_ARTIFICIAL_PREEMPT
301
+ self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT
302
+ if self.enable_artificial_preemption
303
+ else 0)
304
+
305
@property
306
def lora_enabled(self) -> bool:
307
return bool(self.lora_config)
@@ -386,15 +402,13 @@ class Scheduler:
402
# groups to preempt.
403
now = time.time()
404
running_queue = policy.sort_by_priority(now, running_queue)
389
-
405
while running_queue:
406
seq_group = running_queue[0]
407
num_running_tokens = self._get_num_new_tokens(
408
seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
409
395
- # We can have up to 1 running prefill at any given time in running
396
- # queue, which means we can guarantee chunk size is at least 1.
397
- assert num_running_tokens != 0
410
+ if num_running_tokens == 0:
411
+ break
412
413
running_queue.popleft()
414
while not self._can_append_slots(seq_group):
@@ -449,9 +463,6 @@ class Scheduler:
463
if curr_loras is not None and seq_group.lora_int_id > 0:
464
curr_loras.add(seq_group.lora_int_id)
465
452
- # Make sure all queues are updated.
453
- assert len(running_queue) == 0
454
-
466
return running_queue, SchedulerRunningOutputs(
467
decode_seq_groups=decode_seq_groups,
468
prefill_seq_groups=prefill_seq_groups,
@@ -545,7 +556,6 @@ class Scheduler:
556
ScheduledSequenceGroup(seq_group,
557
token_chunk_size=num_new_tokens))
558
else:
548
- assert num_new_tokens == 1
559
decode_seq_groups.append(
560
ScheduledSequenceGroup(seq_group, token_chunk_size=1))
561
budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
@@ -868,6 +878,13 @@ class Scheduler:
878
"""Determine whether or not we have enough space in the KV cache to
879
continue generation of the sequence group.
880
"""
881
+ # It is True only for testing case to trigger artificial preemption.
882
+ if (self.enable_artificial_preemption
883
+ and random.uniform(0, 1) < ARTIFICIAL_PREEMPTION_PROB
884
+ and self.artificial_preempt_cnt > 0):
885
+ self.artificial_preempt_cnt -= 1
886
+ return False
887
+
888
# Appending slots only occurs in decoding.
889
is_prefill = False
890
@@ -1116,11 +1133,14 @@ class Scheduler:
1133
if `enable_chunking` is True. If a sequence group has multiple
1134
sequences (e.g., running beam search), it means it is in decoding
1135
phase, so chunking doesn't happen.
1136
+
1137
+ Returns 0 if the new token cannot be computed due to token budget.
1138
"""
1139
num_new_tokens = 0
1140
seqs = seq_group.get_seqs(status=status)
1141
for seq in seqs:
1142
num_new_tokens += seq.get_num_new_tokens()
1143
+ assert num_new_tokens > 0
1144
# Chunk if a running request cannot fit in.
1145
# If number of seq > 1, it means it is doing beam search in a
1146
# decode phase. Do not chunk in that case.