@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa63e710c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Perf] Reduce scheduling overhead in model runner after cuda sync (#12094)

Signed-off-by: Keyun Tong <tongkeyun@gmail.com>

Keyun Tong committed Jan 26, 2025 at 00:42 UTC fa63e710c7fbaae3a445f669d3b5ba6b9a4ef412
3 files changed +21 -13
vllm/v1/outputs.py
+1 -1
@@ -8,7 +8,7 @@ import torch
8 class SamplerOutput:
9
10 # [num_reqs]
11 - sampled_token_ids: List[int]
11 + sampled_token_ids: torch.Tensor
12
13 # [num_reqs, max_num_logprobs + 1]
14 logprob_token_ids: Optional[torch.Tensor]
vllm/v1/sample/sampler.py
+1 -2
@@ -50,9 +50,8 @@ class Sampler(nn.Module):
50 # Use int32 to reduce the tensor size.
51 sampled = sampled.to(torch.int32)
52
53 - # NOTE: CPU-GPU synchronization happens here.
53 sampler_output = SamplerOutput(
55 - sampled_token_ids=sampled.tolist(),
54 + sampled_token_ids=sampled,
55 logprob_token_ids=topk_indices,
56 logprobs=topk_logprobs,
57 prompt_logprob_token_ids=None,
vllm/v1/worker/gpu_model_runner.py
+19 -10
@@ -775,10 +775,10 @@ class GPUModelRunner:
775 sampling_metadata=sampling_metadata,
776 )
777
778 - sampled_token_ids = sampler_output.sampled_token_ids
778 # TODO(woosuk): The following loop can be slow since it iterates over
779 # the requests one by one. Optimize.
780 num_reqs = self.input_batch.num_reqs
781 + request_seq_lens: List[Tuple[int, CachedRequestState, int]] = []
782 for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
783 assert req_id is not None
784 req_state = self.requests[req_id]
@@ -787,10 +787,10 @@ class GPUModelRunner:
787 assert seq_len <= req_state.num_tokens
788 if seq_len == req_state.num_tokens:
789 # Append the sampled token to the output token ids.
790 - token_id = sampled_token_ids[i]
791 - self.input_batch.token_ids_cpu[i, seq_len] = token_id
790 self.input_batch.num_tokens[i] += 1
793 - req_state.output_token_ids.append(token_id)
791 + # OPTIMIZATION: Priming the state updates for later updates.
792 + req_state.output_token_ids.append(0)
793 + request_seq_lens.append((i, req_state, seq_len))
794 else:
795 # Ignore the sampled token from the partial request.
796 # Rewind the generator state as if the token was not sampled.
@@ -799,6 +799,21 @@ class GPUModelRunner:
799 # This relies on cuda-specific torch-internal impl details
800 generator.set_offset(generator.get_offset() - 4)
801
802 + # num_reqs entries should be non-None
803 + assert all(
804 + req_id is not None for req_id in
805 + self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
806 + req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
807 +
808 + # NOTE: GPU -> CPU Sync happens here.
809 + # Move as many CPU operations as possible before this sync point.
810 + sampled_token_ids = sampler_output.sampled_token_ids.tolist()
811 + # Update with the actual token ids
812 + for i, req_state, seq_len in request_seq_lens:
813 + token_id = sampled_token_ids[i]
814 + self.input_batch.token_ids_cpu[i, seq_len] = token_id
815 + req_state.output_token_ids[-1] = token_id
816 +
817 if sampler_output.logprob_token_ids is None:
818 logprob_token_ids = None
819 else:
@@ -808,12 +823,6 @@ class GPUModelRunner:
823 else:
824 logprobs = sampler_output.logprobs.cpu()
825
811 - # num_reqs entries should be non-None
812 - assert all(
813 - req_id is not None for req_id in
814 - self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
815 - req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
816 -
826 model_runner_output = ModelRunnerOutput(
827 req_ids=req_ids,
828 req_id_to_index=self.input_batch.req_id_to_index,