@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db66e018ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix for Spec model TP + Chunked Prefill (#10232)

Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com> Signed-off-by: Sourashis Roy <sroy@roblox.com> Co-authored-by: Sourashis Roy <sroy@roblox.com>

Murali Andoorveedu committed Nov 26, 2024 at 09:11 UTC db66e018eaabcc5e5855e994b49931dbb4800ce1
8 files changed +145 -73
docs/source/serving/compatibility_matrix.rst
+1 -1
@@ -118,7 +118,7 @@ Feature x Feature
118 -
119 -
120 * - :ref:`SD <spec_decode>`
121 - - ✗
121 + - ✅
122 - ✅
123 - ✗
124 - ✅
tests/core/test_chunked_prefill_scheduler.py
+39
@@ -413,6 +413,45 @@ def test_chunked_prefill_preempt():
413 assert out.num_batched_tokens == max_num_batched_tokens
414
415
416 +@pytest.mark.parametrize("num_scheduler_steps", [1, 5])
417 +def test_chunked_prefill_spec_prefill(num_scheduler_steps):
418 + """Verify that the num_lookahead_slots is set appropriately for an all"""
419 + """prefill batch depending on whether multi-step scheduling is enabled"""
420 + """or not"""
421 + block_size = 4
422 + max_seqs = 30
423 + max_model_len = 200
424 + max_num_batched_tokens = 30
425 + num_lookahead_slots = 4
426 + scheduler_config = SchedulerConfig(
427 + "generate",
428 + max_num_batched_tokens,
429 + max_seqs,
430 + max_model_len,
431 + enable_chunked_prefill=True,
432 + num_lookahead_slots=num_lookahead_slots,
433 + num_scheduler_steps=num_scheduler_steps,
434 + )
435 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
436 + cache_config.num_cpu_blocks = 16
437 + cache_config.num_gpu_blocks = 16
438 + scheduler = Scheduler(scheduler_config, cache_config, None)
439 +
440 + _, seq_group = create_dummy_prompt("1",
441 + prompt_length=30,
442 + block_size=block_size)
443 + scheduler.add_seq_group(seq_group)
444 + _, out = schedule_and_update_computed_tokens(scheduler)
445 + # The request is chunked.
446 + # prefill scheduled now.
447 + assert len(out.scheduled_seq_groups) == 1
448 + assert out.num_prefill_groups == 1
449 + assert out.num_batched_tokens == max_num_batched_tokens
450 + print(out.num_lookahead_slots)
451 + assert out.num_lookahead_slots == (0 if (num_scheduler_steps == 1) else
452 + num_lookahead_slots)
453 +
454 +
455 def test_chunked_prefill_max_seqs():
456 block_size = 4
457 max_seqs = 2
tests/spec_decode/e2e/test_compatibility.py
-46
@@ -50,49 +50,3 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
50 with pytest.raises(ValueError, match="cannot be larger than"):
51 get_output_from_llm_generator(test_llm_generator, prompts,
52 sampling_params)
53 -
54 -
55 -@pytest.mark.parametrize("common_llm_kwargs",
56 - [{
57 - "model": "meta-llama/Llama-2-7b-chat-hf",
58 - "speculative_model": "JackFram/llama-68m",
59 - "num_speculative_tokens": 5,
60 - "enable_chunked_prefill": "True",
61 - }])
62 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [
63 - {
64 - "tensor_parallel_size": 2,
65 - "speculative_draft_tensor_parallel_size": 2,
66 - },
67 - {
68 - "tensor_parallel_size": 4,
69 - "speculative_draft_tensor_parallel_size": 4,
70 - },
71 - {
72 - "tensor_parallel_size": 8,
73 - "speculative_draft_tensor_parallel_size": 8,
74 - },
75 -])
76 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
77 -@pytest.mark.parametrize("seed", [1])
78 -def test_spec_decode_xfail_chunked_prefill_draft_model_tp_not_one(
79 - test_llm_generator):
80 - """Verify that speculative decoding fails if chunked prefill is enabled for
81 - draft model with tensor parallelism of more than 1.
82 - """
83 - output_len = 128
84 - temperature = 0.0
85 -
86 - prompts = [
87 - "Hello, my name is",
88 - ]
89 -
90 - sampling_params = SamplingParams(
91 - max_tokens=output_len,
92 - ignore_eos=True,
93 - temperature=temperature,
94 - )
95 -
96 - with pytest.raises(ValueError, match="with tensor parallel size 1"):
97 - get_output_from_llm_generator(test_llm_generator, prompts,
98 - sampling_params)
tests/spec_decode/e2e/test_integration_dist_tp2.py
+57
@@ -115,3 +115,60 @@ def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
115 max_output_len=32,
116 seed=seed,
117 temperature=0.0)
118 +
119 +
120 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
121 + reason="Need at least 2 GPUs to run the test.")
122 +@pytest.mark.parametrize(
123 + "common_llm_kwargs",
124 + [[
125 + # Skip cuda graph recording for fast test.
126 + "--enforce-eager",
127 + "--tensor_parallel_size",
128 + "2",
129 +
130 + # precision
131 + "--dtype",
132 + "bfloat16",
133 + ]])
134 +@pytest.mark.parametrize(
135 + "per_test_common_llm_kwargs",
136 + [["--enable-chunked-prefill", "False"],
137 + [
138 + "--enable-chunked-prefill", "True", "--max-num-batched-tokens", "4",
139 + "--max-num-seqs", "4"
140 + ]])
141 +@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
142 +@pytest.mark.parametrize("model, test_llm_kwargs",
143 + [("JackFram/llama-68m", [
144 + "--speculative-model",
145 + "JackFram/llama-68m",
146 + "--num_speculative-tokens",
147 + "3",
148 + ]),
149 + ("JackFram/llama-68m", [
150 + "--speculative-model",
151 + "JackFram/llama-68m",
152 + "--num_speculative-tokens",
153 + "3",
154 + "--speculative-draft-tensor-parallel-size",
155 + "1",
156 + ])])
157 +@pytest.mark.parametrize("batch_size", [2])
158 +@pytest.mark.parametrize("seed", [1])
159 +def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
160 + per_test_common_llm_kwargs,
161 + baseline_llm_kwargs, test_llm_kwargs,
162 + batch_size: int, seed: int):
163 + """Verify spec decode works well with same and different TP size for
164 + the draft model with chunked prefill.
165 + """
166 + run_equality_correctness_test_tp(model,
167 + common_llm_kwargs,
168 + per_test_common_llm_kwargs,
169 + baseline_llm_kwargs,
170 + test_llm_kwargs,
171 + batch_size,
172 + max_output_len=32,
173 + seed=seed,
174 + temperature=0.0)
tests/spec_decode/test_spec_decode_worker.py
+2 -1
@@ -867,7 +867,8 @@ def test_chunked_prefill_flow(k: int, batch_size: int, batch_composition: str):
867 target_group_metadata_list = prefill + decodes
868 execute_model_req = ExecuteModelRequest(
869 seq_group_metadata_list=target_group_metadata_list,
870 - num_lookahead_slots=k)
870 + # For prefill only batches we expect num_lookahead_slots = 0.
871 + num_lookahead_slots=k if n_decodes > 0 else 0)
872
873 target_token_ids = torch.randint(low=0,
874 high=vocab_size,
vllm/config.py
-10
@@ -1409,16 +1409,6 @@ class SpeculativeConfig:
1409 draft_hf_config
1410 )
1411
1412 - if (enable_chunked_prefill and \
1413 - speculative_draft_tensor_parallel_size != 1):
1414 - # TODO - Investigate why the error reported in
1415 - # https://github.com/vllm-project/vllm/pull/9291#issuecomment-2463266258
1416 - # is happening and re-enable it.
1417 - raise ValueError(
1418 - "Chunked prefill and speculative decoding can be enabled "
1419 - "simultaneously only for draft models with tensor "
1420 - "parallel size 1.")
1421 -
1412 draft_model_config.max_model_len = (
1413 SpeculativeConfig._maybe_override_draft_max_model_len(
1414 speculative_max_model_len,
vllm/core/scheduler.py
+19 -9
@@ -1201,15 +1201,25 @@ class Scheduler:
1201 # Update swapped requests.
1202 self.swapped.extend(running_scheduled.swapped_out)
1203 # Put prefills first due to Attention backend ordering assumption.
1204 + scheduled_seq_groups = (prefills.seq_groups +
1205 + running_scheduled.prefill_seq_groups +
1206 + swapped_in.prefill_seq_groups +
1207 + running_scheduled.decode_seq_groups +
1208 + swapped_in.decode_seq_groups)
1209 + num_prefill_groups = (len(prefills.seq_groups) +
1210 + len(swapped_in.prefill_seq_groups) +
1211 + len(running_scheduled.prefill_seq_groups))
1212 + # If all prompts, then we set num_lookahead_slots to 0
1213 + # this allows us to go through the `no_spec` path in
1214 + # `spec_decode_worker.py`
1215 + all_prefills = (len(scheduled_seq_groups) == num_prefill_groups)
1216 + num_lookahead_slots = (0 if
1217 + (all_prefills
1218 + and not self.scheduler_config.is_multi_step)
1219 + else running_scheduled.num_lookahead_slots)
1220 return SchedulerOutputs(
1205 - scheduled_seq_groups=(prefills.seq_groups +
1206 - running_scheduled.prefill_seq_groups +
1207 - swapped_in.prefill_seq_groups +
1208 - running_scheduled.decode_seq_groups +
1209 - swapped_in.decode_seq_groups),
1210 - num_prefill_groups=(len(prefills.seq_groups) +
1211 - len(swapped_in.prefill_seq_groups) +
1212 - len(running_scheduled.prefill_seq_groups)),
1221 + scheduled_seq_groups=scheduled_seq_groups,
1222 + num_prefill_groups=num_prefill_groups,
1223 num_batched_tokens=budget.num_batched_tokens +
1224 budget.num_cached_tokens,
1225 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
@@ -1218,7 +1228,7 @@ class Scheduler:
1228 swapped_in.blocks_to_copy,
1229 ignored_seq_groups=prefills.ignored_seq_groups +
1230 swapped_in.infeasible_seq_groups,
1221 - num_lookahead_slots=running_scheduled.num_lookahead_slots,
1231 + num_lookahead_slots=num_lookahead_slots,
1232 running_queue_size=len(self.running),
1233 preempted=(len(running_scheduled.preempted) +
1234 len(running_scheduled.swapped_out)),
vllm/spec_decode/spec_decode_worker.py
+27 -6
@@ -408,7 +408,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
408 disable_all_speculation = self._should_disable_all_speculation(
409 execute_model_req)
410 num_lookahead_slots = execute_model_req.num_lookahead_slots
411 -
411 + all_prompt = True
412 + atleast_one_prompt = False
413 + all_zero_spec_tokens = True
414 + for sgm in execute_model_req.seq_group_metadata_list:
415 + all_prompt = all_prompt and sgm.is_prompt
416 + atleast_one_prompt = atleast_one_prompt or sgm.is_prompt
417 + all_zero_spec_tokens = all_zero_spec_tokens and (
418 + sgm.num_speculative_tokens == 0)
419 +
420 + if all_prompt and execute_model_req.seq_group_metadata_list:
421 + assert num_lookahead_slots == 0, (
422 + "Prompt only runs should have num_lookahead_slots equal to 0. "
423 + "This should never happen, please file a bug at "
424 + "https://github.com/vllm-project/vllm/issues")
425 # Speculative decoding is disabled in the following cases:
426 # 1. Prefill phase: Speculative decoding is not
427 # used during the prefill phase.
@@ -419,11 +432,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
432 # In any of these cases, the proposer and scorer workers
433 # are called normally.
434 # We expect `num_speculative_tokens` to be None for prefills.
422 - no_spec = all(
423 - sgm.is_prompt for sgm in execute_model_req.seq_group_metadata_list
424 - ) or num_lookahead_slots == 0 or disable_all_speculation or all(
425 - sgm.num_speculative_tokens == 0
426 - for sgm in execute_model_req.seq_group_metadata_list)
435 + no_spec = (num_lookahead_slots == 0 or disable_all_speculation
436 + or all_zero_spec_tokens)
437
438 # Broadcast how many lookahead slots are scheduled for this step, and
439 # whether all speculation is disabled, to all non-driver workers.
@@ -442,6 +452,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
452 num_lookahead_slots=num_lookahead_slots,
453 no_spec=no_spec,
454 disable_all_speculation=disable_all_speculation,
455 + # When both chunked prefill and speculative decoding are enabled
456 + # it is possible that the same batch contains both prefill
457 + # and decodes. If that happens in the scorer we run the batch
458 + # as one single forward pass. However, in the proposer we
459 + # run them as 2 different batches - one for prefill and
460 + # the other for decodes. The variable indicates to the non-driver
461 + # worker that there are prefills as part of the speculative batch
462 + # and hence it needs to run an extra prefill forward pass.
463 + run_spec_proposer_for_prefill=atleast_one_prompt,
464 )
465 broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
466
@@ -653,6 +672,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
672
673 if not data["no_spec"]:
674 self.scorer_worker.execute_model()
675 + if data["run_spec_proposer_for_prefill"]:
676 + self.proposer_worker.execute_model()
677
678 return True
679