A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU] Implement multi-step scheduling (#8489)
Woosuk Kwon committed
Sep 14, 2024 at 16:58 UTC
50e9ec41fc2dbd1b80e7ec488650c327bdf81798
5 files changed
+279
-76
vllm/config.py
+1
-1
@@ -379,7 +379,7 @@ class ModelConfig:
379
self.use_async_output_proc = False
380
return
381
382
- if self.enforce_eager:
382
+ if device_config.device_type == "cuda" and self.enforce_eager:
383
logger.warning(
384
"To see benefits of async output processing, enable CUDA "
385
"graph. Since, enforce-eager is enabled, async output "
vllm/executor/ray_tpu_executor.py
+6
-2
@@ -68,8 +68,12 @@ class RayTPUExecutor(TPUExecutor):
68
)
69
70
assert self.speculative_config is None
71
- worker_module_name = "vllm.worker.tpu_worker"
72
- worker_class_name = "TPUWorker"
71
+ if self.scheduler_config.is_multi_step:
72
+ worker_module_name = "vllm.worker.multi_step_tpu_worker"
73
+ worker_class_name = "MultiStepTPUWorker"
74
+ else:
75
+ worker_module_name = "vllm.worker.tpu_worker"
76
+ worker_class_name = "TPUWorker"
77
78
# GKE does not fetch environment information from metadata server
79
# and instead sets these from within the Ray process. Therefore we
vllm/executor/tpu_executor.py
+11
-5
@@ -62,11 +62,17 @@ class TPUExecutor(ExecutorBase):
62
rank: int = 0,
63
distributed_init_method: Optional[str] = None,
64
):
65
- from vllm.worker.tpu_worker import TPUWorker
66
-
67
- worker = TPUWorker(**self._get_worker_kwargs(local_rank, rank,
68
- distributed_init_method))
69
- return worker
65
+ if self.scheduler_config.is_multi_step:
66
+ from vllm.worker.multi_step_tpu_worker import MultiStepTPUWorker
67
+ worker = MultiStepTPUWorker(**self._get_worker_kwargs(
68
+ local_rank, rank, distributed_init_method))
69
+ return worker
70
+ else:
71
+ from vllm.worker.tpu_worker import TPUWorker
72
+
73
+ worker = TPUWorker(**self._get_worker_kwargs(
74
+ local_rank, rank, distributed_init_method))
75
+ return worker
76
77
def initialize_cache(
78
self,
vllm/worker/multi_step_tpu_worker.py
new
+105
@@ -0,0 +1,105 @@
1
+import dataclasses
2
+from typing import Dict, Optional, Tuple
3
+
4
+import torch
5
+
6
+from vllm.distributed import broadcast_tensor_dict
7
+from vllm.sequence import ExecuteModelRequest
8
+from vllm.worker.tpu_model_runner import ModelInputForTPU
9
+from vllm.worker.tpu_worker import TPUWorker
10
+from vllm.worker.worker_base import WorkerInput
11
+
12
+
13
+class MultiStepTPUWorker(TPUWorker):
14
+
15
+ def __init__(self, *args, **kwargs):
16
+ super().__init__(*args, **kwargs)
17
+ self.cached_model_input: Optional[ModelInputForTPU] = None
18
+
19
+ def _get_driver_input_and_broadcast(
20
+ self, execute_model_req: ExecuteModelRequest
21
+ ) -> Tuple[ModelInputForTPU, WorkerInput, Dict[str, torch.Tensor]]:
22
+ assert self.is_driver_worker
23
+ assert execute_model_req.virtual_engine == 0
24
+
25
+ is_first_multi_step = execute_model_req.is_first_multi_step
26
+ is_last_step = execute_model_req.is_last_step
27
+ if is_first_multi_step:
28
+ worker_input: WorkerInput = self.prepare_worker_input(
29
+ execute_model_req=execute_model_req)
30
+ worker_input = dataclasses.replace(
31
+ worker_input,
32
+ num_steps=execute_model_req.num_lookahead_slots + 1)
33
+ model_input: ModelInputForTPU = (
34
+ self.model_runner.prepare_model_input(
35
+ execute_model_req.seq_group_metadata_list,
36
+ execute_model_req.virtual_engine,
37
+ execute_model_req.finished_requests_ids))
38
+
39
+ if execute_model_req.async_callback:
40
+ model_input = dataclasses.replace(
41
+ model_input,
42
+ async_callback=execute_model_req.async_callback)
43
+ else:
44
+ assert self.cached_model_input is not None
45
+ model_input = self.cached_model_input
46
+ worker_input = WorkerInput()
47
+ model_input = dataclasses.replace(
48
+ model_input,
49
+ is_first_multi_step=is_first_multi_step,
50
+ is_last_step=is_last_step)
51
+
52
+ if self.do_metadata_broadcast:
53
+ if is_first_multi_step:
54
+ broadcast_data = worker_input.as_broadcastable_tensor_dict()
55
+ broadcast_data.update(
56
+ model_input.as_broadcastable_tensor_dict())
57
+ broadcast_tensor_dict(broadcast_data, src=0)
58
+ else:
59
+ broadcast_data = {
60
+ "is_first_multi_step": is_first_multi_step,
61
+ "is_last_step": is_last_step,
62
+ }
63
+ broadcast_tensor_dict(broadcast_data, src=0)
64
+
65
+ # Retuning empty dict here to keep this compatible with
66
+ # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast`
67
+ return model_input, worker_input, {}
68
+
69
+ def prepare_input(
70
+ self,
71
+ execute_model_req: Optional[ExecuteModelRequest] = None,
72
+ ) -> Optional[Tuple[ModelInputForTPU, WorkerInput, Dict[str,
73
+ torch.Tensor]]]:
74
+ if self.is_driver_worker:
75
+ if execute_model_req is None:
76
+ if self.do_metadata_broadcast:
77
+ broadcast_tensor_dict({}, src=0)
78
+ return None
79
+
80
+ model_input, worker_input, _ = self._get_driver_input_and_broadcast(
81
+ execute_model_req)
82
+ if model_input.is_first_multi_step:
83
+ self.cached_model_input = model_input
84
+ return model_input, worker_input, {}
85
+ else:
86
+ broadcast_data = broadcast_tensor_dict(src=0)
87
+ if not broadcast_data:
88
+ return None
89
+
90
+ if len(broadcast_data) == 2:
91
+ assert self.cached_model_input is not None
92
+ self.cached_model_input = dataclasses.replace(
93
+ self.cached_model_input,
94
+ is_first_multi_step=broadcast_data["is_first_multi_step"],
95
+ is_last_step=broadcast_data["is_last_step"])
96
+ empty_worker_input = WorkerInput()
97
+ return self.cached_model_input, empty_worker_input, {}
98
+
99
+ worker_input = WorkerInput.from_broadcasted_tensor_dict(
100
+ broadcast_data)
101
+ model_input = (
102
+ self.model_runner.
103
+ make_model_input_from_broadcasted_tensor_dict(broadcast_data))
104
+ self.cached_model_input = model_input
105
+ return model_input, worker_input, {}
vllm/worker/tpu_model_runner.py
+156
-68
@@ -51,6 +51,8 @@ class ModelInputForTPU(ModelRunnerInputBase):
51
num_samples: int
52
best_of: List[int]
53
seq_groups: List[List[int]]
54
+ is_first_multi_step: bool = True
55
+ is_last_step: bool = True
56
virtual_engine: int = 0
57
async_callback: Optional[Callable] = None
58
@@ -65,6 +67,8 @@ class ModelInputForTPU(ModelRunnerInputBase):
67
"num_samples": self.num_samples,
68
"best_of": self.best_of,
69
"seq_groups": self.seq_groups,
70
+ "is_first_multi_step": self.is_first_multi_step,
71
+ "is_last_step": self.is_last_step,
72
"virtual_engine": self.virtual_engine,
73
}
74
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
@@ -118,6 +122,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
122
self.block_size,
123
False,
124
)
125
+ self.cached_step_outputs: List[torch.Tensor] = []
126
127
def load_model(self) -> None:
128
self.device = self.device_config.device
@@ -518,97 +523,159 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
523
num_steps: int = 1,
524
) -> List[SamplerOutput]:
525
assert intermediate_tensors is None
521
- if num_steps > 1:
522
- raise ValueError(
523
- "TPUModelRunner does not support multi-step execution.")
524
-
525
- def _execute_model(*args):
526
- """Move input args from CPU to device and execute the model."""
527
-
528
- new_args = []
529
- for arg in args:
530
- if isinstance(arg, torch.Tensor):
531
- arg = arg.to(self.device)
532
- elif isinstance(arg, AttentionMetadata):
533
- arg.slot_mapping = arg.slot_mapping.to(self.device)
534
- if getattr(arg, "block_tables", None) is not None:
535
- arg.block_tables = arg.block_tables.to(self.device)
536
- if getattr(arg, "context_lens", None) is not None:
537
- arg.context_lens = arg.context_lens.to(self.device)
538
- new_args.append(arg)
539
- return self.model(*new_args, is_prompt=is_prompt)
540
-
541
- num_prefills = model_input.attn_metadata.num_prefills
542
- is_prompt = num_prefills > 0
526
+ if not model_input.is_first_multi_step:
527
+ if not model_input.is_last_step:
528
+ return []
529
+
530
+ use_async_out_proc = model_input.async_callback is not None
531
+ sampler_outputs = []
532
+ num_outputs = len(self.cached_step_outputs)
533
+ for i in range(num_outputs):
534
+ next_token_ids = self.cached_step_outputs.pop(0)
535
+ next_token_ids = next_token_ids.cpu().tolist()
536
+ sampler_output = _make_decode_output(next_token_ids,
537
+ model_input.seq_groups)
538
+ sampler_outputs.append(sampler_output)
539
+
540
+ if i < num_outputs - 1 and use_async_out_proc:
541
+ assert model_input.async_callback is not None
542
+ ctx = model_input.async_callback.keywords[ # type: ignore
543
+ "ctx"]
544
+ ctx.append_output(
545
+ outputs=[sampler_output],
546
+ seq_group_metadata_list=ctx.seq_group_metadata_list,
547
+ scheduler_outputs=ctx.scheduler_outputs,
548
+ is_async=False,
549
+ is_last_step=False)
550
+ model_input.async_callback()
551
+ if use_async_out_proc:
552
+ return [sampler_outputs[-1]]
553
+ else:
554
+ return sampler_outputs
555
+
556
+ is_prompt = model_input.attn_metadata.num_prefills > 0
557
if is_prompt:
558
+ assert num_steps == 1
559
# NOTE(woosuk): Since the FlashAttention kernel does not support
560
# ragged inputs, we split the prompts into different batches and
561
# process them separately. This is a temporary hack that should be
562
# optimized by using SplashAttention.
548
- next_token_ids = []
563
orig_slot_mapping = model_input.attn_metadata.slot_mapping
564
batch_size = model_input.input_lens.shape[0]
565
start_idx = 0
566
+ next_token_ids = []
567
for i in range(batch_size):
568
# Get the actual prefill_len.
569
prefill_len = model_input.input_lens[i:i + 1].item()
570
prefill_len = _get_padded_prefill_len(prefill_len)
571
end_idx = start_idx + prefill_len
572
558
- model_input.attn_metadata.slot_mapping = orig_slot_mapping[
559
- None, start_idx:end_idx]
560
- model_input.attn_metadata.num_prefills = 1
561
- output_token_ids = _execute_model(
562
- model_input.token_ids[None, start_idx:end_idx],
563
- model_input.position_ids[None, start_idx:end_idx],
564
- model_input.attn_metadata, model_input.input_lens[i:i + 1],
565
- model_input.t[i:i + 1], model_input.p[i:i + 1],
566
- model_input.num_samples, kv_caches)
567
- if i == 0 and model_input.async_callback is not None:
568
- model_input.async_callback()
569
- # Retrieve the outputs to CPU.
570
- next_token_ids += output_token_ids.cpu().tolist()
573
+ token_ids = model_input.token_ids[None, start_idx:end_idx].to(
574
+ self.device)
575
+ position_ids = model_input.position_ids[None,
576
+ start_idx:end_idx].to(
577
+ self.device)
578
+ attn_metadata = model_input.attn_metadata
579
+ attn_metadata.num_prefills = 1
580
+ attn_metadata.slot_mapping = orig_slot_mapping[
581
+ None, start_idx:end_idx].to(self.device)
582
+ input_lens = model_input.input_lens[i:i + 1].to(self.device)
583
+ t = model_input.t[i:i + 1].to(self.device)
584
+ p = model_input.p[i:i + 1].to(self.device)
585
+ output_token_ids = self.model(token_ids,
586
+ position_ids,
587
+ attn_metadata,
588
+ input_lens,
589
+ t,
590
+ p,
591
+ model_input.num_samples,
592
+ kv_caches,
593
+ is_prompt=True)
594
+ next_token_ids.append(output_token_ids[0])
595
start_idx = end_idx
572
- else:
573
- # Execute the model.
574
- output_token_ids = _execute_model(
575
- model_input.token_ids, model_input.position_ids,
576
- model_input.attn_metadata, model_input.input_lens,
577
- model_input.t, model_input.p, model_input.num_samples,
578
- kv_caches)
596
+
597
if model_input.async_callback is not None:
598
model_input.async_callback()
599
# Retrieve the outputs to CPU.
582
- next_token_ids = output_token_ids.cpu().tolist()
583
-
584
- # NOTE(woosuk): Minimal code to construct the sampler outputs.
585
- # The TPU backend does not reuse the sampler, since the TPU backend
586
- # does not support the advanced sampling parameters such as logprobs.
587
- zero_logprob = Logprob(0.0)
588
- batch_idx = 0
589
- sampler_outputs = []
590
- for seq_group in model_input.seq_groups:
591
- seq_ids = seq_group
592
- seq_outputs = []
593
- if is_prompt:
600
+ next_token_ids = [
601
+ output_token_ids.cpu().tolist()
602
+ for output_token_ids in next_token_ids
603
+ ]
604
+
605
+ # NOTE(woosuk): Minimal code to construct the sampler outputs.
606
+ # The TPU backend does not reuse the sampler, since the TPU backend
607
+ # does not support advanced sampling parameters such as logprobs.
608
+ zero_logprob = Logprob(0.0)
609
+ sampler_outputs = []
610
+ for i, seq_group in enumerate(model_input.seq_groups):
611
+ seq_ids = seq_group
612
assert len(seq_ids) == 1
613
seq_id = seq_ids[0]
596
- for i in range(model_input.best_of[batch_idx]):
597
- next_token_id = next_token_ids[batch_idx][i]
614
+ seq_outputs = []
615
+ for j in range(model_input.best_of[i]):
616
+ next_token_id = next_token_ids[i][j]
617
seq_outputs.append(
618
SequenceOutput(seq_id, next_token_id,
619
{next_token_id: zero_logprob}))
601
- batch_idx += 1
602
- else:
603
- for seq_id in seq_ids:
604
- next_token_id = next_token_ids[batch_idx]
605
- seq_outputs.append(
606
- SequenceOutput(seq_id, next_token_id,
607
- {next_token_id: zero_logprob}))
608
- batch_idx += 1
609
- sampler_outputs.append(
610
- CompletionSequenceGroupOutput(seq_outputs, None))
611
- return [SamplerOutput(sampler_outputs)]
620
+ sampler_outputs.append(
621
+ CompletionSequenceGroupOutput(seq_outputs, None))
622
+ return [SamplerOutput(sampler_outputs)]
623
+ else:
624
+ token_ids = model_input.token_ids.to(self.device)
625
+ position_ids = model_input.position_ids.to(self.device)
626
+ attn_metadata = model_input.attn_metadata
627
+ attn_metadata.slot_mapping = attn_metadata.slot_mapping.to(
628
+ self.device)
629
+ attn_metadata.block_tables = attn_metadata.block_tables.to(
630
+ self.device)
631
+ attn_metadata.context_lens = attn_metadata.context_lens.to(
632
+ self.device)
633
+ t = model_input.t.to(self.device)
634
+ p = model_input.p.to(self.device)
635
+ input_lens = model_input.input_lens.to(self.device)
636
+ for i in range(num_steps):
637
+ slot_mapping = attn_metadata.slot_mapping
638
+ output_token_ids = self.model(token_ids,
639
+ position_ids,
640
+ attn_metadata,
641
+ input_lens,
642
+ t,
643
+ p,
644
+ model_input.num_samples,
645
+ kv_caches,
646
+ is_prompt=False)
647
+ self.cached_step_outputs.append(output_token_ids)
648
+
649
+ if i < num_steps - 1:
650
+ # Prepare the inputs for the next step.
651
+ token_ids = output_token_ids.unsqueeze(dim=1).int()
652
+ position_ids = position_ids + 1
653
+ attn_metadata.context_lens = attn_metadata.context_lens + 1
654
+
655
+ block_tables = attn_metadata.block_tables
656
+ block_number = block_tables.gather(
657
+ 1,
658
+ position_ids.long() // self.block_size)
659
+ block_offset = position_ids % self.block_size
660
+
661
+ is_padding = slot_mapping == _PAD_SLOT_ID
662
+ slot_mapping = block_number * self.block_size + block_offset
663
+ slot_mapping = slot_mapping.long()
664
+ slot_mapping = torch.where(is_padding, _PAD_SLOT_ID,
665
+ slot_mapping)
666
+ attn_metadata.slot_mapping = slot_mapping
667
+
668
+ if model_input.async_callback is not None:
669
+ model_input.async_callback()
670
+
671
+ if num_steps > 1:
672
+ return []
673
+ # Retrieve the outputs to CPU.
674
+ next_token_ids = self.cached_step_outputs.pop(0)
675
+ next_token_ids = next_token_ids.cpu().tolist()
676
+ sampler_output = _make_decode_output(next_token_ids,
677
+ model_input.seq_groups)
678
+ return [sampler_output]
679
680
681
class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
@@ -756,3 +823,24 @@ def _apply_top_p(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor:
823
cutoff_logit = torch.gather(logits_sorted, -1, cutoff_index)
824
logits = logits.masked_fill_(logits < cutoff_logit, -float("inf"))
825
return logits
826
+
827
+
828
+def _make_decode_output(
829
+ next_token_ids: List[int],
830
+ seq_groups: List[List[int]],
831
+) -> SamplerOutput:
832
+ zero_logprob = Logprob(0.0)
833
+ sampler_outputs = []
834
+ batch_idx = 0
835
+ for seq_group in seq_groups:
836
+ seq_ids = seq_group
837
+ seq_outputs = []
838
+ for seq_id in seq_ids:
839
+ next_token_id = next_token_ids[batch_idx]
840
+ seq_outputs.append(
841
+ SequenceOutput(seq_id, next_token_id,
842
+ {next_token_id: zero_logprob}))
843
+ batch_idx += 1
844
+ sampler_outputs.append(CompletionSequenceGroupOutput(
845
+ seq_outputs, None))
846
+ return SamplerOutput(sampler_outputs)