A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Support per-request seed (#9945)
Signed-off-by: Nick Hill <nickhill@us.ibm.com>
Nick Hill committed
Nov 3, 2024 at 17:14 UTC
1f1b6d6eda3ea5fbdf4566632ac8a9fa61b31593
3 files changed
+41
-48
vllm/v1/sample/metadata.py
+2
-3
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import List, Optional
2
+from typing import Dict
3
4
import torch
5
@@ -16,7 +16,6 @@ class SamplingMetadata:
16
no_top_p: bool
17
no_top_k: bool
18
19
- generators: List[Optional[torch.Generator]]
20
- no_generator: bool
19
+ generators: Dict[int, torch.Generator]
20
21
max_num_logprobs: int
vllm/v1/sample/sampler.py
+10
-13
@@ -1,5 +1,5 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
-from typing import List, Optional
2
+from typing import Dict
3
4
import torch
5
import torch.nn as nn
@@ -84,22 +84,21 @@ class Sampler(nn.Module):
84
def random_sample(
85
self,
86
probs: torch.Tensor,
87
- generators: List[Optional[torch.Generator]],
88
- no_generator: bool,
87
+ generators: Dict[int, torch.Generator],
88
) -> torch.Tensor:
89
q = torch.empty_like(probs)
90
# NOTE(woosuk): To batch-process the requests without their own seeds,
91
# which is the common case, we first assume that every request does
92
# not have its own seed. Then, we overwrite the values for the requests
93
# that have their own seeds.
95
- q.exponential_()
96
- if not no_generator:
97
- assert len(generators) == probs.shape[0]
94
+ if len(generators) != probs.shape[0]:
95
+ # This might still be done here unnecessarily if there are greedies
96
+ q.exponential_()
97
+ if generators:
98
# TODO(woosuk): This can be slow because we handle each request
99
# one by one. Optimize this.
100
- for i, generator in enumerate(generators):
101
- if generator is not None:
102
- q[i].exponential_(generator=generator)
100
+ for i, generator in generators.items():
101
+ q[i].exponential_(generator=generator)
102
return probs.div_(q).argmax(dim=-1).view(-1)
103
104
def sample(
@@ -112,13 +111,11 @@ class Sampler(nn.Module):
111
if sampling_metadata.all_greedy:
112
return self.greedy_sample(probs)
113
if sampling_metadata.all_random:
115
- return self.random_sample(probs, sampling_metadata.generators,
116
- sampling_metadata.no_generator)
114
+ return self.random_sample(probs, sampling_metadata.generators)
115
116
greedy_sampled = self.greedy_sample(probs)
117
random_sampled = self.random_sample(probs,
120
- sampling_metadata.generators,
121
- sampling_metadata.no_generator)
118
+ sampling_metadata.generators)
119
sampled = torch.where(
120
sampling_metadata.temperature < _SAMPLING_EPS,
121
greedy_sampled,
vllm/v1/worker/gpu_model_runner.py
+29
-32
@@ -128,13 +128,20 @@ class GPUModelRunner:
128
# Add new requests to the cached states.
129
for req_data in scheduler_output.scheduled_new_reqs:
130
req_id = req_data.req_id
131
+ sampling_params = req_data.sampling_params
132
+ if sampling_params.seed is not None:
133
+ generator = torch.Generator(device=self.device)
134
+ generator.manual_seed(sampling_params.seed)
135
+ else:
136
+ generator = None
137
+
138
self.requests[req_id] = CachedRequestState(
139
req_id=req_id,
140
prompt_token_ids=req_data.prompt_token_ids,
141
prompt=req_data.prompt,
142
multi_modal_data=req_data.multi_modal_data,
136
- sampling_params=req_data.sampling_params,
137
- generator=None, # TODO
143
+ sampling_params=sampling_params,
144
+ generator=generator,
145
block_ids=req_data.block_ids,
146
num_computed_tokens=req_data.num_computed_tokens,
147
output_token_ids=[],
@@ -342,11 +349,9 @@ class GPUModelRunner:
349
else:
350
# Ignore the sampled token from the partial request.
351
# Rewind the generator state as if the token was not sampled.
345
- generator = self.input_batch.generators[i]
352
+ generator = self.input_batch.generators.get(i)
353
if generator is not None:
347
- offset = generator.get_offset()
348
- generator = generator.set_offset(offset - 1)
349
- self.input_batch.generators[i] = generator
354
+ generator.set_offset(generator.get_offset() - 1)
355
356
if sampler_output.logprob_token_ids is None:
357
logprob_token_ids = None
@@ -494,8 +499,8 @@ class InputBatch:
499
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
500
self.top_k_reqs: Set[str] = set()
501
497
- self.generators: List[Optional[torch.Generator]] = [None
498
- ] * max_num_reqs
502
+ # req_index -> generator
503
+ self.generators: Dict[int, torch.Generator] = {}
504
505
self.num_logprobs: Dict[str, int] = {}
506
self.prompt_logprob_reqs: Set[str] = set()
@@ -509,8 +514,9 @@ class InputBatch:
514
req_index = self.num_reqs
515
assert req_index < self.max_num_reqs
516
512
- self.req_ids[req_index] = request.req_id
513
- self.req_id_to_index[request.req_id] = req_index
517
+ req_id = request.req_id
518
+ self.req_ids[req_index] = req_id
519
+ self.req_id_to_index[req_id] = req_index
520
521
# Copy the prompt token ids and output token ids.
522
num_prompt_tokens = len(request.prompt_token_ids)
@@ -528,27 +534,24 @@ class InputBatch:
534
sampling_params = request.sampling_params
535
self.temperature_cpu[req_index] = sampling_params.temperature
536
if sampling_params.sampling_type == SamplingType.GREEDY:
531
- self.greedy_reqs.add(req_index)
532
- elif sampling_params.sampling_type == SamplingType.RANDOM:
533
- self.random_reqs.add(req_index)
534
- elif sampling_params.sampling_type == SamplingType.RANDOM_SEED:
535
- # TODO(woosuk): Support per-request random seed.
536
- raise NotImplementedError("Per-request seed is not supported yet.")
537
+ self.greedy_reqs.add(req_id)
538
+ else:
539
+ self.random_reqs.add(req_id)
540
541
self.top_p_cpu[req_index] = sampling_params.top_p
542
if sampling_params.top_p < 1:
540
- self.top_p_reqs.add(req_index)
543
+ self.top_p_reqs.add(req_id)
544
self.top_k_cpu[req_index] = sampling_params.top_k
545
if sampling_params.top_k > 0:
543
- self.top_k_reqs.add(req_index)
546
+ self.top_k_reqs.add(req_id)
547
548
self.generators[req_index] = request.generator
549
550
num_logprobs = sampling_params.logprobs
551
if num_logprobs is not None and num_logprobs > 0:
549
- self.num_logprobs[request.req_id] = num_logprobs
552
+ self.num_logprobs[req_id] = num_logprobs
553
if sampling_params.prompt_logprobs:
551
- self.prompt_logprob_reqs.add(req_index)
554
+ self.prompt_logprob_reqs.add(req_id)
555
556
def remove_request(self, req_id: str) -> Optional[int]:
557
req_index = self.req_id_to_index.pop(req_id, None)
@@ -560,7 +563,7 @@ class InputBatch:
563
self.random_reqs.discard(req_id)
564
self.top_p_reqs.discard(req_id)
565
self.top_k_reqs.discard(req_id)
563
- self.generators[req_index] = None
566
+ self.generators.pop(req_index, None)
567
self.num_logprobs.pop(req_id, None)
568
self.prompt_logprob_reqs.discard(req_id)
569
return req_index
@@ -612,7 +615,9 @@ class InputBatch:
615
last_req_index]
616
self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
617
self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
615
- self.generators[empty_index] = self.generators[last_req_index]
618
+ generator = self.generators.pop(last_req_index, None)
619
+ if generator is not None:
620
+ self.generators[empty_index] = generator
621
622
# Decrement last_req_index since it is now empty.
623
last_req_index -= 1
@@ -636,8 +641,7 @@ class InputBatch:
641
top_k=self.top_k[:self.num_reqs],
642
no_top_p=self.no_top_p,
643
no_top_k=self.no_top_k,
639
- generators=self.generators[:self.num_reqs],
640
- no_generator=self.no_generator,
644
+ generators=self.generators,
645
max_num_logprobs=self.max_num_logprobs,
646
)
647
@@ -661,16 +665,9 @@ class InputBatch:
665
def no_top_k(self) -> bool:
666
return len(self.top_k_reqs) == 0
667
664
- @property
665
- def no_generator(self) -> bool:
666
- return len(self.generators) == 0
667
-
668
@property
669
def max_num_logprobs(self) -> int:
670
- if self.num_logprobs:
671
- return max(self.num_logprobs.values())
672
- else:
673
- return 0
670
+ return max(self.num_logprobs.values()) if self.num_logprobs else 0
671
672
@property
673
def no_logprob(self) -> bool: