@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 50e9ec41fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[TPU] Implement multi-step scheduling (#8489)

Woosuk Kwon committed Sep 14, 2024 at 16:58 UTC 50e9ec41fc2dbd1b80e7ec488650c327bdf81798
5 files changed +279 -76
vllm/config.py
+1 -1
@@ -379,7 +379,7 @@ class ModelConfig:
379 self.use_async_output_proc = False
380 return
381
382 - if self.enforce_eager:
382 + if device_config.device_type == "cuda" and self.enforce_eager:
383 logger.warning(
384 "To see benefits of async output processing, enable CUDA "
385 "graph. Since, enforce-eager is enabled, async output "
vllm/executor/ray_tpu_executor.py
+6 -2
@@ -68,8 +68,12 @@ class RayTPUExecutor(TPUExecutor):
68 )
69
70 assert self.speculative_config is None
71 - worker_module_name = "vllm.worker.tpu_worker"
72 - worker_class_name = "TPUWorker"
71 + if self.scheduler_config.is_multi_step:
72 + worker_module_name = "vllm.worker.multi_step_tpu_worker"
73 + worker_class_name = "MultiStepTPUWorker"
74 + else:
75 + worker_module_name = "vllm.worker.tpu_worker"
76 + worker_class_name = "TPUWorker"
77
78 # GKE does not fetch environment information from metadata server
79 # and instead sets these from within the Ray process. Therefore we
vllm/executor/tpu_executor.py
+11 -5
@@ -62,11 +62,17 @@ class TPUExecutor(ExecutorBase):
62 rank: int = 0,
63 distributed_init_method: Optional[str] = None,
64 ):
65 - from vllm.worker.tpu_worker import TPUWorker
66 -
67 - worker = TPUWorker(**self._get_worker_kwargs(local_rank, rank,
68 - distributed_init_method))
69 - return worker
65 + if self.scheduler_config.is_multi_step:
66 + from vllm.worker.multi_step_tpu_worker import MultiStepTPUWorker
67 + worker = MultiStepTPUWorker(**self._get_worker_kwargs(
68 + local_rank, rank, distributed_init_method))
69 + return worker
70 + else:
71 + from vllm.worker.tpu_worker import TPUWorker
72 +
73 + worker = TPUWorker(**self._get_worker_kwargs(
74 + local_rank, rank, distributed_init_method))
75 + return worker
76
77 def initialize_cache(
78 self,
vllm/worker/multi_step_tpu_worker.py new
+105
@@ -0,0 +1,105 @@
1 +import dataclasses
2 +from typing import Dict, Optional, Tuple
3 +
4 +import torch
5 +
6 +from vllm.distributed import broadcast_tensor_dict
7 +from vllm.sequence import ExecuteModelRequest
8 +from vllm.worker.tpu_model_runner import ModelInputForTPU
9 +from vllm.worker.tpu_worker import TPUWorker
10 +from vllm.worker.worker_base import WorkerInput
11 +
12 +
13 +class MultiStepTPUWorker(TPUWorker):
14 +
15 + def __init__(self, *args, **kwargs):
16 + super().__init__(*args, **kwargs)
17 + self.cached_model_input: Optional[ModelInputForTPU] = None
18 +
19 + def _get_driver_input_and_broadcast(
20 + self, execute_model_req: ExecuteModelRequest
21 + ) -> Tuple[ModelInputForTPU, WorkerInput, Dict[str, torch.Tensor]]:
22 + assert self.is_driver_worker
23 + assert execute_model_req.virtual_engine == 0
24 +
25 + is_first_multi_step = execute_model_req.is_first_multi_step
26 + is_last_step = execute_model_req.is_last_step
27 + if is_first_multi_step:
28 + worker_input: WorkerInput = self.prepare_worker_input(
29 + execute_model_req=execute_model_req)
30 + worker_input = dataclasses.replace(
31 + worker_input,
32 + num_steps=execute_model_req.num_lookahead_slots + 1)
33 + model_input: ModelInputForTPU = (
34 + self.model_runner.prepare_model_input(
35 + execute_model_req.seq_group_metadata_list,
36 + execute_model_req.virtual_engine,
37 + execute_model_req.finished_requests_ids))
38 +
39 + if execute_model_req.async_callback:
40 + model_input = dataclasses.replace(
41 + model_input,
42 + async_callback=execute_model_req.async_callback)
43 + else:
44 + assert self.cached_model_input is not None
45 + model_input = self.cached_model_input
46 + worker_input = WorkerInput()
47 + model_input = dataclasses.replace(
48 + model_input,
49 + is_first_multi_step=is_first_multi_step,
50 + is_last_step=is_last_step)
51 +
52 + if self.do_metadata_broadcast:
53 + if is_first_multi_step:
54 + broadcast_data = worker_input.as_broadcastable_tensor_dict()
55 + broadcast_data.update(
56 + model_input.as_broadcastable_tensor_dict())
57 + broadcast_tensor_dict(broadcast_data, src=0)
58 + else:
59 + broadcast_data = {
60 + "is_first_multi_step": is_first_multi_step,
61 + "is_last_step": is_last_step,
62 + }
63 + broadcast_tensor_dict(broadcast_data, src=0)
64 +
65 + # Retuning empty dict here to keep this compatible with
66 + # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast`
67 + return model_input, worker_input, {}
68 +
69 + def prepare_input(
70 + self,
71 + execute_model_req: Optional[ExecuteModelRequest] = None,
72 + ) -> Optional[Tuple[ModelInputForTPU, WorkerInput, Dict[str,
73 + torch.Tensor]]]:
74 + if self.is_driver_worker:
75 + if execute_model_req is None:
76 + if self.do_metadata_broadcast:
77 + broadcast_tensor_dict({}, src=0)
78 + return None
79 +
80 + model_input, worker_input, _ = self._get_driver_input_and_broadcast(
81 + execute_model_req)
82 + if model_input.is_first_multi_step:
83 + self.cached_model_input = model_input
84 + return model_input, worker_input, {}
85 + else:
86 + broadcast_data = broadcast_tensor_dict(src=0)
87 + if not broadcast_data:
88 + return None
89 +
90 + if len(broadcast_data) == 2:
91 + assert self.cached_model_input is not None
92 + self.cached_model_input = dataclasses.replace(
93 + self.cached_model_input,
94 + is_first_multi_step=broadcast_data["is_first_multi_step"],
95 + is_last_step=broadcast_data["is_last_step"])
96 + empty_worker_input = WorkerInput()
97 + return self.cached_model_input, empty_worker_input, {}
98 +
99 + worker_input = WorkerInput.from_broadcasted_tensor_dict(
100 + broadcast_data)
101 + model_input = (
102 + self.model_runner.
103 + make_model_input_from_broadcasted_tensor_dict(broadcast_data))
104 + self.cached_model_input = model_input
105 + return model_input, worker_input, {}
vllm/worker/tpu_model_runner.py
+156 -68
@@ -51,6 +51,8 @@ class ModelInputForTPU(ModelRunnerInputBase):
51 num_samples: int
52 best_of: List[int]
53 seq_groups: List[List[int]]
54 + is_first_multi_step: bool = True
55 + is_last_step: bool = True
56 virtual_engine: int = 0
57 async_callback: Optional[Callable] = None
58
@@ -65,6 +67,8 @@ class ModelInputForTPU(ModelRunnerInputBase):
67 "num_samples": self.num_samples,
68 "best_of": self.best_of,
69 "seq_groups": self.seq_groups,
70 + "is_first_multi_step": self.is_first_multi_step,
71 + "is_last_step": self.is_last_step,
72 "virtual_engine": self.virtual_engine,
73 }
74 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
@@ -118,6 +122,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
122 self.block_size,
123 False,
124 )
125 + self.cached_step_outputs: List[torch.Tensor] = []
126
127 def load_model(self) -> None:
128 self.device = self.device_config.device
@@ -518,97 +523,159 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
523 num_steps: int = 1,
524 ) -> List[SamplerOutput]:
525 assert intermediate_tensors is None
521 - if num_steps > 1:
522 - raise ValueError(
523 - "TPUModelRunner does not support multi-step execution.")
524 -
525 - def _execute_model(*args):
526 - """Move input args from CPU to device and execute the model."""
527 -
528 - new_args = []
529 - for arg in args:
530 - if isinstance(arg, torch.Tensor):
531 - arg = arg.to(self.device)
532 - elif isinstance(arg, AttentionMetadata):
533 - arg.slot_mapping = arg.slot_mapping.to(self.device)
534 - if getattr(arg, "block_tables", None) is not None:
535 - arg.block_tables = arg.block_tables.to(self.device)
536 - if getattr(arg, "context_lens", None) is not None:
537 - arg.context_lens = arg.context_lens.to(self.device)
538 - new_args.append(arg)
539 - return self.model(*new_args, is_prompt=is_prompt)
540 -
541 - num_prefills = model_input.attn_metadata.num_prefills
542 - is_prompt = num_prefills > 0
526 + if not model_input.is_first_multi_step:
527 + if not model_input.is_last_step:
528 + return []
529 +
530 + use_async_out_proc = model_input.async_callback is not None
531 + sampler_outputs = []
532 + num_outputs = len(self.cached_step_outputs)
533 + for i in range(num_outputs):
534 + next_token_ids = self.cached_step_outputs.pop(0)
535 + next_token_ids = next_token_ids.cpu().tolist()
536 + sampler_output = _make_decode_output(next_token_ids,
537 + model_input.seq_groups)
538 + sampler_outputs.append(sampler_output)
539 +
540 + if i < num_outputs - 1 and use_async_out_proc:
541 + assert model_input.async_callback is not None
542 + ctx = model_input.async_callback.keywords[ # type: ignore
543 + "ctx"]
544 + ctx.append_output(
545 + outputs=[sampler_output],
546 + seq_group_metadata_list=ctx.seq_group_metadata_list,
547 + scheduler_outputs=ctx.scheduler_outputs,
548 + is_async=False,
549 + is_last_step=False)
550 + model_input.async_callback()
551 + if use_async_out_proc:
552 + return [sampler_outputs[-1]]
553 + else:
554 + return sampler_outputs
555 +
556 + is_prompt = model_input.attn_metadata.num_prefills > 0
557 if is_prompt:
558 + assert num_steps == 1
559 # NOTE(woosuk): Since the FlashAttention kernel does not support
560 # ragged inputs, we split the prompts into different batches and
561 # process them separately. This is a temporary hack that should be
562 # optimized by using SplashAttention.
548 - next_token_ids = []
563 orig_slot_mapping = model_input.attn_metadata.slot_mapping
564 batch_size = model_input.input_lens.shape[0]
565 start_idx = 0
566 + next_token_ids = []
567 for i in range(batch_size):
568 # Get the actual prefill_len.
569 prefill_len = model_input.input_lens[i:i + 1].item()
570 prefill_len = _get_padded_prefill_len(prefill_len)
571 end_idx = start_idx + prefill_len
572
558 - model_input.attn_metadata.slot_mapping = orig_slot_mapping[
559 - None, start_idx:end_idx]
560 - model_input.attn_metadata.num_prefills = 1
561 - output_token_ids = _execute_model(
562 - model_input.token_ids[None, start_idx:end_idx],
563 - model_input.position_ids[None, start_idx:end_idx],
564 - model_input.attn_metadata, model_input.input_lens[i:i + 1],
565 - model_input.t[i:i + 1], model_input.p[i:i + 1],
566 - model_input.num_samples, kv_caches)
567 - if i == 0 and model_input.async_callback is not None:
568 - model_input.async_callback()
569 - # Retrieve the outputs to CPU.
570 - next_token_ids += output_token_ids.cpu().tolist()
573 + token_ids = model_input.token_ids[None, start_idx:end_idx].to(
574 + self.device)
575 + position_ids = model_input.position_ids[None,
576 + start_idx:end_idx].to(
577 + self.device)
578 + attn_metadata = model_input.attn_metadata
579 + attn_metadata.num_prefills = 1
580 + attn_metadata.slot_mapping = orig_slot_mapping[
581 + None, start_idx:end_idx].to(self.device)
582 + input_lens = model_input.input_lens[i:i + 1].to(self.device)
583 + t = model_input.t[i:i + 1].to(self.device)
584 + p = model_input.p[i:i + 1].to(self.device)
585 + output_token_ids = self.model(token_ids,
586 + position_ids,
587 + attn_metadata,
588 + input_lens,
589 + t,
590 + p,
591 + model_input.num_samples,
592 + kv_caches,
593 + is_prompt=True)
594 + next_token_ids.append(output_token_ids[0])
595 start_idx = end_idx
572 - else:
573 - # Execute the model.
574 - output_token_ids = _execute_model(
575 - model_input.token_ids, model_input.position_ids,
576 - model_input.attn_metadata, model_input.input_lens,
577 - model_input.t, model_input.p, model_input.num_samples,
578 - kv_caches)
596 +
597 if model_input.async_callback is not None:
598 model_input.async_callback()
599 # Retrieve the outputs to CPU.
582 - next_token_ids = output_token_ids.cpu().tolist()
583 -
584 - # NOTE(woosuk): Minimal code to construct the sampler outputs.
585 - # The TPU backend does not reuse the sampler, since the TPU backend
586 - # does not support the advanced sampling parameters such as logprobs.
587 - zero_logprob = Logprob(0.0)
588 - batch_idx = 0
589 - sampler_outputs = []
590 - for seq_group in model_input.seq_groups:
591 - seq_ids = seq_group
592 - seq_outputs = []
593 - if is_prompt:
600 + next_token_ids = [
601 + output_token_ids.cpu().tolist()
602 + for output_token_ids in next_token_ids
603 + ]
604 +
605 + # NOTE(woosuk): Minimal code to construct the sampler outputs.
606 + # The TPU backend does not reuse the sampler, since the TPU backend
607 + # does not support advanced sampling parameters such as logprobs.
608 + zero_logprob = Logprob(0.0)
609 + sampler_outputs = []
610 + for i, seq_group in enumerate(model_input.seq_groups):
611 + seq_ids = seq_group
612 assert len(seq_ids) == 1
613 seq_id = seq_ids[0]
596 - for i in range(model_input.best_of[batch_idx]):
597 - next_token_id = next_token_ids[batch_idx][i]
614 + seq_outputs = []
615 + for j in range(model_input.best_of[i]):
616 + next_token_id = next_token_ids[i][j]
617 seq_outputs.append(
618 SequenceOutput(seq_id, next_token_id,
619 {next_token_id: zero_logprob}))
601 - batch_idx += 1
602 - else:
603 - for seq_id in seq_ids:
604 - next_token_id = next_token_ids[batch_idx]
605 - seq_outputs.append(
606 - SequenceOutput(seq_id, next_token_id,
607 - {next_token_id: zero_logprob}))
608 - batch_idx += 1
609 - sampler_outputs.append(
610 - CompletionSequenceGroupOutput(seq_outputs, None))
611 - return [SamplerOutput(sampler_outputs)]
620 + sampler_outputs.append(
621 + CompletionSequenceGroupOutput(seq_outputs, None))
622 + return [SamplerOutput(sampler_outputs)]
623 + else:
624 + token_ids = model_input.token_ids.to(self.device)
625 + position_ids = model_input.position_ids.to(self.device)
626 + attn_metadata = model_input.attn_metadata
627 + attn_metadata.slot_mapping = attn_metadata.slot_mapping.to(
628 + self.device)
629 + attn_metadata.block_tables = attn_metadata.block_tables.to(
630 + self.device)
631 + attn_metadata.context_lens = attn_metadata.context_lens.to(
632 + self.device)
633 + t = model_input.t.to(self.device)
634 + p = model_input.p.to(self.device)
635 + input_lens = model_input.input_lens.to(self.device)
636 + for i in range(num_steps):
637 + slot_mapping = attn_metadata.slot_mapping
638 + output_token_ids = self.model(token_ids,
639 + position_ids,
640 + attn_metadata,
641 + input_lens,
642 + t,
643 + p,
644 + model_input.num_samples,
645 + kv_caches,
646 + is_prompt=False)
647 + self.cached_step_outputs.append(output_token_ids)
648 +
649 + if i < num_steps - 1:
650 + # Prepare the inputs for the next step.
651 + token_ids = output_token_ids.unsqueeze(dim=1).int()
652 + position_ids = position_ids + 1
653 + attn_metadata.context_lens = attn_metadata.context_lens + 1
654 +
655 + block_tables = attn_metadata.block_tables
656 + block_number = block_tables.gather(
657 + 1,
658 + position_ids.long() // self.block_size)
659 + block_offset = position_ids % self.block_size
660 +
661 + is_padding = slot_mapping == _PAD_SLOT_ID
662 + slot_mapping = block_number * self.block_size + block_offset
663 + slot_mapping = slot_mapping.long()
664 + slot_mapping = torch.where(is_padding, _PAD_SLOT_ID,
665 + slot_mapping)
666 + attn_metadata.slot_mapping = slot_mapping
667 +
668 + if model_input.async_callback is not None:
669 + model_input.async_callback()
670 +
671 + if num_steps > 1:
672 + return []
673 + # Retrieve the outputs to CPU.
674 + next_token_ids = self.cached_step_outputs.pop(0)
675 + next_token_ids = next_token_ids.cpu().tolist()
676 + sampler_output = _make_decode_output(next_token_ids,
677 + model_input.seq_groups)
678 + return [sampler_output]
679
680
681 class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
@@ -756,3 +823,24 @@ def _apply_top_p(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor:
823 cutoff_logit = torch.gather(logits_sorted, -1, cutoff_index)
824 logits = logits.masked_fill_(logits < cutoff_logit, -float("inf"))
825 return logits
826 +
827 +
828 +def _make_decode_output(
829 + next_token_ids: List[int],
830 + seq_groups: List[List[int]],
831 +) -> SamplerOutput:
832 + zero_logprob = Logprob(0.0)
833 + sampler_outputs = []
834 + batch_idx = 0
835 + for seq_group in seq_groups:
836 + seq_ids = seq_group
837 + seq_outputs = []
838 + for seq_id in seq_ids:
839 + next_token_id = next_token_ids[batch_idx]
840 + seq_outputs.append(
841 + SequenceOutput(seq_id, next_token_id,
842 + {next_token_id: zero_logprob}))
843 + batch_idx += 1
844 + sampler_outputs.append(CompletionSequenceGroupOutput(
845 + seq_outputs, None))
846 + return SamplerOutput(sampler_outputs)