A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Perf] Reduce scheduling overhead in model runner after cuda sync (#12094)
Signed-off-by: Keyun Tong <tongkeyun@gmail.com>
Keyun Tong committed
Jan 26, 2025 at 00:42 UTC
fa63e710c7fbaae3a445f669d3b5ba6b9a4ef412
3 files changed
+21
-13
vllm/v1/outputs.py
+1
-1
@@ -8,7 +8,7 @@ import torch
8
class SamplerOutput:
9
10
# [num_reqs]
11
- sampled_token_ids: List[int]
11
+ sampled_token_ids: torch.Tensor
12
13
# [num_reqs, max_num_logprobs + 1]
14
logprob_token_ids: Optional[torch.Tensor]
vllm/v1/sample/sampler.py
+1
-2
@@ -50,9 +50,8 @@ class Sampler(nn.Module):
50
# Use int32 to reduce the tensor size.
51
sampled = sampled.to(torch.int32)
52
53
- # NOTE: CPU-GPU synchronization happens here.
53
sampler_output = SamplerOutput(
55
- sampled_token_ids=sampled.tolist(),
54
+ sampled_token_ids=sampled,
55
logprob_token_ids=topk_indices,
56
logprobs=topk_logprobs,
57
prompt_logprob_token_ids=None,
vllm/v1/worker/gpu_model_runner.py
+19
-10
@@ -775,10 +775,10 @@ class GPUModelRunner:
775
sampling_metadata=sampling_metadata,
776
)
777
778
- sampled_token_ids = sampler_output.sampled_token_ids
778
# TODO(woosuk): The following loop can be slow since it iterates over
779
# the requests one by one. Optimize.
780
num_reqs = self.input_batch.num_reqs
781
+ request_seq_lens: List[Tuple[int, CachedRequestState, int]] = []
782
for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
783
assert req_id is not None
784
req_state = self.requests[req_id]
@@ -787,10 +787,10 @@ class GPUModelRunner:
787
assert seq_len <= req_state.num_tokens
788
if seq_len == req_state.num_tokens:
789
# Append the sampled token to the output token ids.
790
- token_id = sampled_token_ids[i]
791
- self.input_batch.token_ids_cpu[i, seq_len] = token_id
790
self.input_batch.num_tokens[i] += 1
793
- req_state.output_token_ids.append(token_id)
791
+ # OPTIMIZATION: Priming the state updates for later updates.
792
+ req_state.output_token_ids.append(0)
793
+ request_seq_lens.append((i, req_state, seq_len))
794
else:
795
# Ignore the sampled token from the partial request.
796
# Rewind the generator state as if the token was not sampled.
@@ -799,6 +799,21 @@ class GPUModelRunner:
799
# This relies on cuda-specific torch-internal impl details
800
generator.set_offset(generator.get_offset() - 4)
801
802
+ # num_reqs entries should be non-None
803
+ assert all(
804
+ req_id is not None for req_id in
805
+ self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
806
+ req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
807
+
808
+ # NOTE: GPU -> CPU Sync happens here.
809
+ # Move as many CPU operations as possible before this sync point.
810
+ sampled_token_ids = sampler_output.sampled_token_ids.tolist()
811
+ # Update with the actual token ids
812
+ for i, req_state, seq_len in request_seq_lens:
813
+ token_id = sampled_token_ids[i]
814
+ self.input_batch.token_ids_cpu[i, seq_len] = token_id
815
+ req_state.output_token_ids[-1] = token_id
816
+
817
if sampler_output.logprob_token_ids is None:
818
logprob_token_ids = None
819
else:
@@ -808,12 +823,6 @@ class GPUModelRunner:
823
else:
824
logprobs = sampler_output.logprobs.cpu()
825
811
- # num_reqs entries should be non-None
812
- assert all(
813
- req_id is not None for req_id in
814
- self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
815
- req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
816
-
826
model_runner_output = ModelRunnerOutput(
827
req_ids=req_ids,
828
req_id_to_index=self.input_batch.req_id_to_index,