A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix][V1] GPUModelRunner._update_states should return True when there is a finished request in batch (#13126)
Kero Liang committed
Feb 14, 2025 at 14:39 UTC
b0ccfc565ac263b4097175b3ba33ad1d5dc5df8a
2 files changed
+238
-1
tests/v1/worker/test_gpu_model_runner.py
new
+236
@@ -0,0 +1,236 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import pytest
3
+
4
+from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
5
+from vllm.sampling_params import SamplingParams
6
+from vllm.v1.core.scheduler_output import (CachedRequestData, NewRequestData,
7
+ SchedulerOutput)
8
+from vllm.v1.worker.gpu_model_runner import GPUModelRunner
9
+
10
+
11
+@pytest.fixture
12
+def model_runner():
13
+ scheduler_config = SchedulerConfig(
14
+ max_num_seqs=10,
15
+ max_num_batched_tokens=512,
16
+ max_model_len=512,
17
+ )
18
+ model_config = ModelConfig(
19
+ model="facebook/opt-125m",
20
+ task="generate",
21
+ tokenizer="facebook/opt-125m",
22
+ tokenizer_mode="auto",
23
+ trust_remote_code=True,
24
+ dtype="float16",
25
+ seed=42,
26
+ )
27
+ cache_config = CacheConfig(
28
+ block_size=16,
29
+ gpu_memory_utilization=0.9,
30
+ swap_space=0,
31
+ cache_dtype="auto",
32
+ )
33
+ vllm_config = VllmConfig(
34
+ model_config=model_config,
35
+ cache_config=cache_config,
36
+ scheduler_config=scheduler_config,
37
+ )
38
+
39
+ device = "cuda"
40
+ return GPUModelRunner(vllm_config, device)
41
+
42
+
43
+def _schedule_new_request(*req_ids: str) -> SchedulerOutput:
44
+ new_reqs = []
45
+ num_scheduled_tokens = {}
46
+ total_num_scheduled_tokens = 0
47
+ for req_id in req_ids:
48
+ new_reqs.append(
49
+ NewRequestData(
50
+ req_id=req_id,
51
+ prompt_token_ids=[1, 2, 3],
52
+ prompt="test",
53
+ mm_inputs=[],
54
+ mm_hashes=[],
55
+ mm_positions=[],
56
+ sampling_params=SamplingParams(),
57
+ block_ids=[0],
58
+ num_computed_tokens=0,
59
+ lora_request=None,
60
+ ))
61
+ num_scheduled_tokens[req_id] = 3
62
+ total_num_scheduled_tokens += num_scheduled_tokens[req_id]
63
+
64
+ return SchedulerOutput(
65
+ scheduled_new_reqs=new_reqs,
66
+ scheduled_cached_reqs=[],
67
+ num_scheduled_tokens=num_scheduled_tokens,
68
+ total_num_scheduled_tokens=total_num_scheduled_tokens,
69
+ scheduled_encoder_inputs={},
70
+ num_common_prefix_blocks=0,
71
+ finished_req_ids=set(),
72
+ free_encoder_input_ids=[],
73
+ )
74
+
75
+
76
+def _is_req_scheduled(model_runner, req_id: str) -> bool:
77
+ return req_id in model_runner.input_batch.req_id_to_index
78
+
79
+
80
+def _is_req_added(model_runner, req_id: str) -> bool:
81
+ return req_id in model_runner.requests
82
+
83
+
84
+def test_update_states_new_request(model_runner):
85
+ req_id = "req_0"
86
+
87
+ # new req
88
+ scheduler_output = _schedule_new_request(req_id)
89
+
90
+ batch_changed = model_runner._update_states(scheduler_output)
91
+ assert batch_changed is True
92
+ assert _is_req_added(model_runner, req_id)
93
+ assert _is_req_scheduled(model_runner, req_id)
94
+
95
+
96
+def test_update_states_request_finished(model_runner):
97
+ req_id = "req_0"
98
+
99
+ # new req
100
+ scheduler_output = _schedule_new_request(req_id)
101
+
102
+ model_runner._update_states(scheduler_output)
103
+ assert _is_req_added(model_runner, req_id)
104
+ assert _is_req_scheduled(model_runner, req_id)
105
+
106
+ # finish req
107
+ scheduler_output = SchedulerOutput(
108
+ scheduled_new_reqs=[],
109
+ scheduled_cached_reqs=[],
110
+ num_scheduled_tokens={},
111
+ total_num_scheduled_tokens=0,
112
+ scheduled_encoder_inputs={},
113
+ num_common_prefix_blocks=0,
114
+ finished_req_ids={req_id},
115
+ free_encoder_input_ids=[],
116
+ )
117
+
118
+ batch_changed = model_runner._update_states(scheduler_output)
119
+ assert batch_changed is True
120
+ assert not _is_req_added(model_runner, req_id)
121
+ assert not _is_req_scheduled(model_runner, req_id)
122
+
123
+
124
+def test_update_states_request_resumed(model_runner):
125
+ req_id = "req_0"
126
+
127
+ # new req
128
+ scheduler_output = _schedule_new_request(req_id)
129
+
130
+ model_runner._update_states(scheduler_output)
131
+ assert _is_req_added(model_runner, req_id)
132
+ assert _is_req_scheduled(model_runner, req_id)
133
+
134
+ # unschedule req
135
+ scheduler_output = SchedulerOutput(
136
+ scheduled_new_reqs=[],
137
+ scheduled_cached_reqs=[],
138
+ num_scheduled_tokens={},
139
+ total_num_scheduled_tokens=0,
140
+ scheduled_encoder_inputs={},
141
+ num_common_prefix_blocks=0,
142
+ finished_req_ids={},
143
+ free_encoder_input_ids=[],
144
+ )
145
+
146
+ model_runner._update_states(scheduler_output)
147
+ assert _is_req_added(model_runner, req_id)
148
+ assert not _is_req_scheduled(model_runner, req_id)
149
+
150
+ # resume req
151
+ cached_req_data = CachedRequestData(
152
+ req_id=req_id,
153
+ resumed_from_preemption=False,
154
+ new_block_ids=[],
155
+ num_computed_tokens=0,
156
+ )
157
+
158
+ scheduler_output = SchedulerOutput(
159
+ scheduled_new_reqs=[],
160
+ scheduled_cached_reqs=[cached_req_data],
161
+ num_scheduled_tokens={req_id: 1},
162
+ total_num_scheduled_tokens=1,
163
+ scheduled_encoder_inputs={},
164
+ num_common_prefix_blocks=0,
165
+ finished_req_ids=set(),
166
+ free_encoder_input_ids=[],
167
+ )
168
+
169
+ batch_changed = model_runner._update_states(scheduler_output)
170
+ assert batch_changed is True
171
+ assert _is_req_added(model_runner, req_id)
172
+ assert _is_req_scheduled(model_runner, req_id)
173
+
174
+
175
+def test_update_states_no_changes(model_runner):
176
+ req_id = "req_0"
177
+
178
+ # new req
179
+ scheduler_output = _schedule_new_request(req_id)
180
+
181
+ model_runner._update_states(scheduler_output)
182
+ assert _is_req_added(model_runner, req_id)
183
+ assert _is_req_scheduled(model_runner, req_id)
184
+
185
+ # schedule req
186
+ scheduler_output = SchedulerOutput(
187
+ scheduled_new_reqs=[],
188
+ scheduled_cached_reqs=[],
189
+ num_scheduled_tokens={req_id: 1},
190
+ total_num_scheduled_tokens=1,
191
+ scheduled_encoder_inputs={},
192
+ num_common_prefix_blocks=0,
193
+ finished_req_ids=set(),
194
+ free_encoder_input_ids=[],
195
+ )
196
+
197
+ batch_changed = model_runner._update_states(scheduler_output)
198
+ assert batch_changed is False
199
+ assert _is_req_added(model_runner, req_id)
200
+ assert _is_req_scheduled(model_runner, req_id)
201
+
202
+
203
+def test_update_states_request_unscheduled(model_runner):
204
+ req_ids = ("req_0", "req_1")
205
+
206
+ # new reqs
207
+ scheduler_output = _schedule_new_request(*req_ids)
208
+
209
+ model_runner._update_states(scheduler_output)
210
+
211
+ assert _is_req_added(model_runner, req_ids[0])
212
+ assert _is_req_scheduled(model_runner, req_ids[0])
213
+
214
+ assert _is_req_added(model_runner, req_ids[1])
215
+ assert _is_req_scheduled(model_runner, req_ids[1])
216
+
217
+ # unschedule req_1
218
+ scheduler_output = SchedulerOutput(
219
+ scheduled_new_reqs=[],
220
+ scheduled_cached_reqs=[],
221
+ num_scheduled_tokens={req_ids[0]: 1},
222
+ total_num_scheduled_tokens=1,
223
+ scheduled_encoder_inputs={},
224
+ num_common_prefix_blocks=0,
225
+ finished_req_ids=set(),
226
+ free_encoder_input_ids=[],
227
+ )
228
+
229
+ batch_changed = model_runner._update_states(scheduler_output)
230
+ assert batch_changed is True
231
+
232
+ assert _is_req_added(model_runner, req_ids[0])
233
+ assert _is_req_scheduled(model_runner, req_ids[0])
234
+
235
+ assert _is_req_added(model_runner, req_ids[1])
236
+ assert not _is_req_scheduled(model_runner, req_ids[1])
vllm/v1/worker/gpu_model_runner.py
+2
-1
@@ -363,7 +363,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
363
# Condense the batched states if there are empty indices.
364
if removed_req_indices:
365
self.input_batch.condense(removed_req_indices)
366
- return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
366
+ return (len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
367
+ or len(scheduler_output.finished_req_ids) > 0)
368
369
def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
370
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens