A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix for Spec model TP + Chunked Prefill (#10232)
Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com> Signed-off-by: Sourashis Roy <sroy@roblox.com> Co-authored-by: Sourashis Roy <sroy@roblox.com>
Murali Andoorveedu committed
Nov 26, 2024 at 09:11 UTC
db66e018eaabcc5e5855e994b49931dbb4800ce1
8 files changed
+145
-73
docs/source/serving/compatibility_matrix.rst
+1
-1
@@ -118,7 +118,7 @@ Feature x Feature
118
-
119
-
120
* - :ref:`SD <spec_decode>`
121
- - ✗
121
+ - ✅
122
- ✅
123
- ✗
124
- ✅
tests/core/test_chunked_prefill_scheduler.py
+39
@@ -413,6 +413,45 @@ def test_chunked_prefill_preempt():
413
assert out.num_batched_tokens == max_num_batched_tokens
414
415
416
+@pytest.mark.parametrize("num_scheduler_steps", [1, 5])
417
+def test_chunked_prefill_spec_prefill(num_scheduler_steps):
418
+ """Verify that the num_lookahead_slots is set appropriately for an all"""
419
+ """prefill batch depending on whether multi-step scheduling is enabled"""
420
+ """or not"""
421
+ block_size = 4
422
+ max_seqs = 30
423
+ max_model_len = 200
424
+ max_num_batched_tokens = 30
425
+ num_lookahead_slots = 4
426
+ scheduler_config = SchedulerConfig(
427
+ "generate",
428
+ max_num_batched_tokens,
429
+ max_seqs,
430
+ max_model_len,
431
+ enable_chunked_prefill=True,
432
+ num_lookahead_slots=num_lookahead_slots,
433
+ num_scheduler_steps=num_scheduler_steps,
434
+ )
435
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
436
+ cache_config.num_cpu_blocks = 16
437
+ cache_config.num_gpu_blocks = 16
438
+ scheduler = Scheduler(scheduler_config, cache_config, None)
439
+
440
+ _, seq_group = create_dummy_prompt("1",
441
+ prompt_length=30,
442
+ block_size=block_size)
443
+ scheduler.add_seq_group(seq_group)
444
+ _, out = schedule_and_update_computed_tokens(scheduler)
445
+ # The request is chunked.
446
+ # prefill scheduled now.
447
+ assert len(out.scheduled_seq_groups) == 1
448
+ assert out.num_prefill_groups == 1
449
+ assert out.num_batched_tokens == max_num_batched_tokens
450
+ print(out.num_lookahead_slots)
451
+ assert out.num_lookahead_slots == (0 if (num_scheduler_steps == 1) else
452
+ num_lookahead_slots)
453
+
454
+
455
def test_chunked_prefill_max_seqs():
456
block_size = 4
457
max_seqs = 2
tests/spec_decode/e2e/test_compatibility.py
-46
@@ -50,49 +50,3 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
50
with pytest.raises(ValueError, match="cannot be larger than"):
51
get_output_from_llm_generator(test_llm_generator, prompts,
52
sampling_params)
53
-
54
-
55
-@pytest.mark.parametrize("common_llm_kwargs",
56
- [{
57
- "model": "meta-llama/Llama-2-7b-chat-hf",
58
- "speculative_model": "JackFram/llama-68m",
59
- "num_speculative_tokens": 5,
60
- "enable_chunked_prefill": "True",
61
- }])
62
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [
63
- {
64
- "tensor_parallel_size": 2,
65
- "speculative_draft_tensor_parallel_size": 2,
66
- },
67
- {
68
- "tensor_parallel_size": 4,
69
- "speculative_draft_tensor_parallel_size": 4,
70
- },
71
- {
72
- "tensor_parallel_size": 8,
73
- "speculative_draft_tensor_parallel_size": 8,
74
- },
75
-])
76
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
77
-@pytest.mark.parametrize("seed", [1])
78
-def test_spec_decode_xfail_chunked_prefill_draft_model_tp_not_one(
79
- test_llm_generator):
80
- """Verify that speculative decoding fails if chunked prefill is enabled for
81
- draft model with tensor parallelism of more than 1.
82
- """
83
- output_len = 128
84
- temperature = 0.0
85
-
86
- prompts = [
87
- "Hello, my name is",
88
- ]
89
-
90
- sampling_params = SamplingParams(
91
- max_tokens=output_len,
92
- ignore_eos=True,
93
- temperature=temperature,
94
- )
95
-
96
- with pytest.raises(ValueError, match="with tensor parallel size 1"):
97
- get_output_from_llm_generator(test_llm_generator, prompts,
98
- sampling_params)
tests/spec_decode/e2e/test_integration_dist_tp2.py
+57
@@ -115,3 +115,60 @@ def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
115
max_output_len=32,
116
seed=seed,
117
temperature=0.0)
118
+
119
+
120
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
121
+ reason="Need at least 2 GPUs to run the test.")
122
+@pytest.mark.parametrize(
123
+ "common_llm_kwargs",
124
+ [[
125
+ # Skip cuda graph recording for fast test.
126
+ "--enforce-eager",
127
+ "--tensor_parallel_size",
128
+ "2",
129
+
130
+ # precision
131
+ "--dtype",
132
+ "bfloat16",
133
+ ]])
134
+@pytest.mark.parametrize(
135
+ "per_test_common_llm_kwargs",
136
+ [["--enable-chunked-prefill", "False"],
137
+ [
138
+ "--enable-chunked-prefill", "True", "--max-num-batched-tokens", "4",
139
+ "--max-num-seqs", "4"
140
+ ]])
141
+@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
142
+@pytest.mark.parametrize("model, test_llm_kwargs",
143
+ [("JackFram/llama-68m", [
144
+ "--speculative-model",
145
+ "JackFram/llama-68m",
146
+ "--num_speculative-tokens",
147
+ "3",
148
+ ]),
149
+ ("JackFram/llama-68m", [
150
+ "--speculative-model",
151
+ "JackFram/llama-68m",
152
+ "--num_speculative-tokens",
153
+ "3",
154
+ "--speculative-draft-tensor-parallel-size",
155
+ "1",
156
+ ])])
157
+@pytest.mark.parametrize("batch_size", [2])
158
+@pytest.mark.parametrize("seed", [1])
159
+def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
160
+ per_test_common_llm_kwargs,
161
+ baseline_llm_kwargs, test_llm_kwargs,
162
+ batch_size: int, seed: int):
163
+ """Verify spec decode works well with same and different TP size for
164
+ the draft model with chunked prefill.
165
+ """
166
+ run_equality_correctness_test_tp(model,
167
+ common_llm_kwargs,
168
+ per_test_common_llm_kwargs,
169
+ baseline_llm_kwargs,
170
+ test_llm_kwargs,
171
+ batch_size,
172
+ max_output_len=32,
173
+ seed=seed,
174
+ temperature=0.0)
tests/spec_decode/test_spec_decode_worker.py
+2
-1
@@ -867,7 +867,8 @@ def test_chunked_prefill_flow(k: int, batch_size: int, batch_composition: str):
867
target_group_metadata_list = prefill + decodes
868
execute_model_req = ExecuteModelRequest(
869
seq_group_metadata_list=target_group_metadata_list,
870
- num_lookahead_slots=k)
870
+ # For prefill only batches we expect num_lookahead_slots = 0.
871
+ num_lookahead_slots=k if n_decodes > 0 else 0)
872
873
target_token_ids = torch.randint(low=0,
874
high=vocab_size,
vllm/config.py
-10
@@ -1409,16 +1409,6 @@ class SpeculativeConfig:
1409
draft_hf_config
1410
)
1411
1412
- if (enable_chunked_prefill and \
1413
- speculative_draft_tensor_parallel_size != 1):
1414
- # TODO - Investigate why the error reported in
1415
- # https://github.com/vllm-project/vllm/pull/9291#issuecomment-2463266258
1416
- # is happening and re-enable it.
1417
- raise ValueError(
1418
- "Chunked prefill and speculative decoding can be enabled "
1419
- "simultaneously only for draft models with tensor "
1420
- "parallel size 1.")
1421
-
1412
draft_model_config.max_model_len = (
1413
SpeculativeConfig._maybe_override_draft_max_model_len(
1414
speculative_max_model_len,
vllm/core/scheduler.py
+19
-9
@@ -1201,15 +1201,25 @@ class Scheduler:
1201
# Update swapped requests.
1202
self.swapped.extend(running_scheduled.swapped_out)
1203
# Put prefills first due to Attention backend ordering assumption.
1204
+ scheduled_seq_groups = (prefills.seq_groups +
1205
+ running_scheduled.prefill_seq_groups +
1206
+ swapped_in.prefill_seq_groups +
1207
+ running_scheduled.decode_seq_groups +
1208
+ swapped_in.decode_seq_groups)
1209
+ num_prefill_groups = (len(prefills.seq_groups) +
1210
+ len(swapped_in.prefill_seq_groups) +
1211
+ len(running_scheduled.prefill_seq_groups))
1212
+ # If all prompts, then we set num_lookahead_slots to 0
1213
+ # this allows us to go through the `no_spec` path in
1214
+ # `spec_decode_worker.py`
1215
+ all_prefills = (len(scheduled_seq_groups) == num_prefill_groups)
1216
+ num_lookahead_slots = (0 if
1217
+ (all_prefills
1218
+ and not self.scheduler_config.is_multi_step)
1219
+ else running_scheduled.num_lookahead_slots)
1220
return SchedulerOutputs(
1205
- scheduled_seq_groups=(prefills.seq_groups +
1206
- running_scheduled.prefill_seq_groups +
1207
- swapped_in.prefill_seq_groups +
1208
- running_scheduled.decode_seq_groups +
1209
- swapped_in.decode_seq_groups),
1210
- num_prefill_groups=(len(prefills.seq_groups) +
1211
- len(swapped_in.prefill_seq_groups) +
1212
- len(running_scheduled.prefill_seq_groups)),
1221
+ scheduled_seq_groups=scheduled_seq_groups,
1222
+ num_prefill_groups=num_prefill_groups,
1223
num_batched_tokens=budget.num_batched_tokens +
1224
budget.num_cached_tokens,
1225
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
@@ -1218,7 +1228,7 @@ class Scheduler:
1228
swapped_in.blocks_to_copy,
1229
ignored_seq_groups=prefills.ignored_seq_groups +
1230
swapped_in.infeasible_seq_groups,
1221
- num_lookahead_slots=running_scheduled.num_lookahead_slots,
1231
+ num_lookahead_slots=num_lookahead_slots,
1232
running_queue_size=len(self.running),
1233
preempted=(len(running_scheduled.preempted) +
1234
len(running_scheduled.swapped_out)),
vllm/spec_decode/spec_decode_worker.py
+27
-6
@@ -408,7 +408,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
408
disable_all_speculation = self._should_disable_all_speculation(
409
execute_model_req)
410
num_lookahead_slots = execute_model_req.num_lookahead_slots
411
-
411
+ all_prompt = True
412
+ atleast_one_prompt = False
413
+ all_zero_spec_tokens = True
414
+ for sgm in execute_model_req.seq_group_metadata_list:
415
+ all_prompt = all_prompt and sgm.is_prompt
416
+ atleast_one_prompt = atleast_one_prompt or sgm.is_prompt
417
+ all_zero_spec_tokens = all_zero_spec_tokens and (
418
+ sgm.num_speculative_tokens == 0)
419
+
420
+ if all_prompt and execute_model_req.seq_group_metadata_list:
421
+ assert num_lookahead_slots == 0, (
422
+ "Prompt only runs should have num_lookahead_slots equal to 0. "
423
+ "This should never happen, please file a bug at "
424
+ "https://github.com/vllm-project/vllm/issues")
425
# Speculative decoding is disabled in the following cases:
426
# 1. Prefill phase: Speculative decoding is not
427
# used during the prefill phase.
@@ -419,11 +432,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
432
# In any of these cases, the proposer and scorer workers
433
# are called normally.
434
# We expect `num_speculative_tokens` to be None for prefills.
422
- no_spec = all(
423
- sgm.is_prompt for sgm in execute_model_req.seq_group_metadata_list
424
- ) or num_lookahead_slots == 0 or disable_all_speculation or all(
425
- sgm.num_speculative_tokens == 0
426
- for sgm in execute_model_req.seq_group_metadata_list)
435
+ no_spec = (num_lookahead_slots == 0 or disable_all_speculation
436
+ or all_zero_spec_tokens)
437
438
# Broadcast how many lookahead slots are scheduled for this step, and
439
# whether all speculation is disabled, to all non-driver workers.
@@ -442,6 +452,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
452
num_lookahead_slots=num_lookahead_slots,
453
no_spec=no_spec,
454
disable_all_speculation=disable_all_speculation,
455
+ # When both chunked prefill and speculative decoding are enabled
456
+ # it is possible that the same batch contains both prefill
457
+ # and decodes. If that happens in the scorer we run the batch
458
+ # as one single forward pass. However, in the proposer we
459
+ # run them as 2 different batches - one for prefill and
460
+ # the other for decodes. The variable indicates to the non-driver
461
+ # worker that there are prefills as part of the speculative batch
462
+ # and hence it needs to run an extra prefill forward pass.
463
+ run_spec_proposer_for_prefill=atleast_one_prompt,
464
)
465
broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
466
@@ -653,6 +672,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
672
673
if not data["no_spec"]:
674
self.scorer_worker.execute_model()
675
+ if data["run_spec_proposer_for_prefill"]:
676
+ self.proposer_worker.execute_model()
677
678
return True
679