@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c63c34f72
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix][SpecDecode] kv corruption with bonus tokens in spec decode (#9730)

Co-authored-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>

Sungjae Lee committed Nov 6, 2024 at 10:45 UTC 0c63c34f725f0b519fa094fbeca6e3cf12c911c1
4 files changed +159 -10
tests/spec_decode/test_multi_step_worker.py
+107
@@ -5,6 +5,8 @@ from unittest.mock import MagicMock
5 import pytest
6 import torch
7
8 +from vllm.attention.selector import (_Backend,
9 + global_force_attn_backend_context_manager)
10 from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.model_executor.utils import set_random_seed
12 from vllm.sequence import (ExecuteModelRequest, HiddenStates, Logprob,
@@ -303,6 +305,7 @@ def test_multi_step_with_batch_expansion_correct_output():
305 seed,
306 model_runner_cls=TP1DraftModelRunner,
307 )
308 + multi_step_worker.set_include_gpu_probs_tensor()
309 worker = create_worker(
310 Worker,
311 model_name,
@@ -397,6 +400,7 @@ def test_multi_step_with_batch_expansion_incorrect_output():
400 seed,
401 model_runner_cls=TP1DraftModelRunner,
402 )
403 + multi_step_worker.set_include_gpu_probs_tensor()
404 worker = create_worker(
405 Worker,
406 model_name,
@@ -477,6 +481,109 @@ def test_multi_step_with_batch_expansion_incorrect_output():
481 assert (num_mismatch > 0)
482
483
484 +@torch.inference_mode()
485 +@pytest.mark.parametrize('num_steps', [1, 2, 3, 4])
486 +# The choice of backends forces the multi_step_worker to choose between
487 +# the vanilla model_runner and TP1DraftModelRunner and that we can test
488 +# both code paths.
489 +@pytest.mark.parametrize('attn_backend',
490 + [_Backend.XFORMERS, _Backend.FLASH_ATTN])
491 +def test_multi_step_correct_kvcache(num_steps, attn_backend):
492 + """Verify that the KV cache of the draft model
493 + is correctly updated for sequences with bonus token.
494 + """
495 + seed = 100
496 + model_name = "JackFram/llama-68m"
497 +
498 + block_size = 16
499 + num_gpu_blocks = 2048 // block_size
500 + batch_size = 1
501 +
502 + with global_force_attn_backend_context_manager(attn_backend):
503 + dtype = 'float16' if attn_backend == _Backend.FLASH_ATTN else 'float32'
504 + multi_step_worker = create_worker(MultiStepWorker,
505 + model_name,
506 + block_size,
507 + num_gpu_blocks,
508 + seed,
509 + model_runner_cls=TP1DraftModelRunner,
510 + dtype=dtype)
511 + multi_step_worker.set_include_gpu_probs_tensor()
512 + worker = create_worker(Worker,
513 + model_name,
514 + block_size,
515 + num_gpu_blocks,
516 + seed,
517 + dtype=dtype)
518 +
519 + prompts = [[0] for _ in range(batch_size)]
520 + # Already generate two tokens for the sequence
521 + # so that we can simulate the bonus token case
522 + multi_step_continuations = [[
523 + random.randint(0, 1000),
524 + random.randint(0, 1000)
525 + ] for _ in prompts]
526 + final_prompt_lens = [len(prompt) + 2 + num_steps for prompt in prompts]
527 +
528 + seq_ids_with_bonus_token_in_last_step = set(range(batch_size))
529 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
530 + prompts,
531 + num_gpu_blocks,
532 + block_size,
533 + continuations=multi_step_continuations,
534 + final_prompt_lens=final_prompt_lens)
535 +
536 + # Run multi-step.
537 + zero_kv_cache(multi_step_worker.cache_engine)
538 + multi_step_worker.sampler_output(execute_model_req=ExecuteModelRequest(
539 + seq_group_metadata_list=seq_group_metadata_list),
540 + sample_len=num_steps,
541 + seq_ids_with_bonus_token_in_last_step=
542 + seq_ids_with_bonus_token_in_last_step)
543 +
544 + # Run single-step repeatedly.
545 + zero_kv_cache(worker.cache_engine)
546 + # Generate the kv cache for the bonus token first
547 + single_step_continuations = [c[:1] for c in multi_step_continuations]
548 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
549 + prompts,
550 + num_gpu_blocks,
551 + block_size,
552 + continuations=single_step_continuations,
553 + final_prompt_lens=final_prompt_lens)
554 + single_step_output = worker.execute_model(
555 + execute_model_req=ExecuteModelRequest(
556 + seq_group_metadata_list=seq_group_metadata_list))
557 + for _ in range(num_steps):
558 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
559 + prompts,
560 + num_gpu_blocks,
561 + block_size,
562 + continuations=multi_step_continuations,
563 + final_prompt_lens=final_prompt_lens)
564 +
565 + single_step_output = worker.execute_model(
566 + execute_model_req=ExecuteModelRequest(
567 + seq_group_metadata_list=seq_group_metadata_list))
568 +
569 + for i, seq_group_output in enumerate(single_step_output[-1]):
570 + multi_step_continuations[i].append(
571 + seq_group_output.samples[0].output_token)
572 +
573 + # Verify that the KV cache of the single-step and
574 + # multi-step workers are the same.
575 + single_step_gpu_cache = worker.cache_engine[0].gpu_cache
576 + multi_step_gpu_cache = multi_step_worker.cache_engine[0].gpu_cache
577 + num_layers = len(single_step_gpu_cache)
578 + allclose = lambda a, b: torch.allclose(
579 + a.cuda(), b.cuda(), rtol=1e-2, atol=1e-2)
580 + for i in range(num_layers):
581 + assert allclose(single_step_gpu_cache[i][0],
582 + multi_step_gpu_cache[i][0])
583 + assert allclose(single_step_gpu_cache[i][1],
584 + multi_step_gpu_cache[i][1])
585 +
586 +
587 @torch.inference_mode()
588 def test_draft_proposals_full_speculation_len():
589 """Verify Top1Proposer correctly handles case where all sequences
tests/spec_decode/utils.py
+3 -1
@@ -68,12 +68,14 @@ def create_worker(cls: Callable[..., T],
68 seed: int,
69 is_driver_worker: bool = True,
70 enforce_eager: bool = True,
71 - model_runner_cls: Optional[ModelRunner] = None) -> T:
71 + model_runner_cls: Optional[ModelRunner] = None,
72 + dtype: Optional[str] = "auto") -> T:
73 engine_args = EngineArgs(
74 model=model_name,
75 seed=seed,
76 block_size=block_size,
77 enforce_eager=enforce_eager,
78 + dtype=dtype,
79 )
80 engine_config = engine_args.create_engine_config()
81
vllm/spec_decode/draft_model_runner.py
+30 -5
@@ -54,6 +54,8 @@ class TP1DraftModelRunner(ModelRunner):
54
55 super().__init__(*args, **kwargs)
56
57 + self.indices_of_seq_with_bonus_tokens = None
58 +
59 def _update_sampling_metadata(self, sampling_metadata, num_seqs,
60 num_queries):
61
@@ -159,6 +161,10 @@ class TP1DraftModelRunner(ModelRunner):
161 # TODO: Add soft-tuning prompt adapter support
162 return not self.prompt_adapter_config
163
164 + def set_indices_of_seq_with_bonus_tokens(self,
165 + indices_of_seq_with_bonus_tokens):
166 + self.indices_of_seq_with_bonus_tokens = indices_of_seq_with_bonus_tokens
167 +
168 @torch.inference_mode()
169 def execute_model(
170 self,
@@ -284,11 +290,30 @@ class TP1DraftModelRunner(ModelRunner):
290 model_input.sampling_metadata)
291
292 # Sample the next token.
287 - outputs.append(
288 - self.model.sample(
289 - logits=logits,
290 - sampling_metadata=model_input.sampling_metadata,
291 - ))
293 + output = self.model.sample(
294 + logits=logits,
295 + sampling_metadata=model_input.sampling_metadata,
296 + )
297 + outputs.append(output)
298 +
299 + if model_input.attn_metadata.num_prefills == 0 \
300 + and self.indices_of_seq_with_bonus_tokens is not None:
301 + assert output.sampled_token_ids is not None
302 + # output.sampled_token_ids should be of shape (num_seqs, 1)
303 + nums_seqs, num_tokens_per_seq = output.sampled_token_ids.shape
304 + assert num_tokens_per_seq == 1
305 + count = 0
306 + for i in range(nums_seqs):
307 + bonus_seq_idx = self.indices_of_seq_with_bonus_tokens[
308 + count]
309 + if i != bonus_seq_idx:
310 + # The following might cause a cpu->gpu sync
311 + # However, the performance impact is negligible as we
312 + # benchmarked on H100.
313 + output.sampled_token_ids[
314 + i, :] = model_input.input_tokens[bonus_seq_idx]
315 + else:
316 + count += 1
317
318 # Prepare inputs for the next step
319 if step != num_steps - 1:
vllm/spec_decode/multi_step_worker.py
+19 -4
@@ -81,6 +81,8 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
81 # Here we run the draft_model_runner with multi-step prepare
82 # on the GPU directly
83 expanded_request.num_steps = sample_len
84 + self.model_runner.set_indices_of_seq_with_bonus_tokens(
85 + indices_of_seq_with_bonus_tokens)
86 model_outputs = self.execute_model(
87 execute_model_req=expanded_request)
88 else:
@@ -97,7 +99,8 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
99 model_output = model_output[0]
100
101 self._append_new_tokens(
100 - model_output, expanded_request.seq_group_metadata_list)
102 + model_output, expanded_request.seq_group_metadata_list,
103 + indices_of_seq_with_bonus_tokens)
104 model_outputs.append(model_output)
105
106 filtered_model_outputs = self._filter_model_output(
@@ -221,13 +224,15 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
224 @staticmethod
225 def _append_new_tokens(
226 model_output: List[SamplerOutput],
224 - seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
227 + seq_group_metadata_list: List[SequenceGroupMetadata],
228 + indices_of_seq_with_bonus_tokens: List[int]) -> None:
229 """Given model output from a single run, append the tokens to the
230 sequences. This is normally done outside of the worker, but it is
231 required if the worker is to perform multiple forward passes.
232 """
229 - for seq_group_metadata, sequence_group_outputs in zip(
230 - seq_group_metadata_list, model_output):
233 + count = 0
234 + for index, (seq_group_metadata, sequence_group_outputs) in enumerate(
235 + zip(seq_group_metadata_list, model_output)):
236 seq_group_metadata.is_prompt = False
237
238 for seq_output in sequence_group_outputs.samples:
@@ -237,6 +242,16 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
242
243 token_id = seq_output.output_token
244 token_logprob = seq_output.logprobs[token_id]
245 + # Determine the actual token ID to be generated,
246 + # considering bonus tokens
247 + if index != indices_of_seq_with_bonus_tokens[count]:
248 + bonus_seq_metadata = seq_group_metadata_list[
249 + indices_of_seq_with_bonus_tokens[count]]
250 + _, bonus_token_seq_data = next(
251 + iter(bonus_seq_metadata.seq_data.items()))
252 + token_id = bonus_token_seq_data.output_token_ids[-1]
253 + else:
254 + count += 1
255
256 seq.append_token_id(token_id, token_logprob.logprob)
257 seq.update_num_computed_tokens(1)