@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f1b6d6eda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Support per-request seed (#9945)

Signed-off-by: Nick Hill <nickhill@us.ibm.com>

Nick Hill committed Nov 3, 2024 at 17:14 UTC 1f1b6d6eda3ea5fbdf4566632ac8a9fa61b31593
3 files changed +41 -48
vllm/v1/sample/metadata.py
+2 -3
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import List, Optional
2 +from typing import Dict
3
4 import torch
5
@@ -16,7 +16,6 @@ class SamplingMetadata:
16 no_top_p: bool
17 no_top_k: bool
18
19 - generators: List[Optional[torch.Generator]]
20 - no_generator: bool
19 + generators: Dict[int, torch.Generator]
20
21 max_num_logprobs: int
vllm/v1/sample/sampler.py
+10 -13
@@ -1,5 +1,5 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 -from typing import List, Optional
2 +from typing import Dict
3
4 import torch
5 import torch.nn as nn
@@ -84,22 +84,21 @@ class Sampler(nn.Module):
84 def random_sample(
85 self,
86 probs: torch.Tensor,
87 - generators: List[Optional[torch.Generator]],
88 - no_generator: bool,
87 + generators: Dict[int, torch.Generator],
88 ) -> torch.Tensor:
89 q = torch.empty_like(probs)
90 # NOTE(woosuk): To batch-process the requests without their own seeds,
91 # which is the common case, we first assume that every request does
92 # not have its own seed. Then, we overwrite the values for the requests
93 # that have their own seeds.
95 - q.exponential_()
96 - if not no_generator:
97 - assert len(generators) == probs.shape[0]
94 + if len(generators) != probs.shape[0]:
95 + # This might still be done here unnecessarily if there are greedies
96 + q.exponential_()
97 + if generators:
98 # TODO(woosuk): This can be slow because we handle each request
99 # one by one. Optimize this.
100 - for i, generator in enumerate(generators):
101 - if generator is not None:
102 - q[i].exponential_(generator=generator)
100 + for i, generator in generators.items():
101 + q[i].exponential_(generator=generator)
102 return probs.div_(q).argmax(dim=-1).view(-1)
103
104 def sample(
@@ -112,13 +111,11 @@ class Sampler(nn.Module):
111 if sampling_metadata.all_greedy:
112 return self.greedy_sample(probs)
113 if sampling_metadata.all_random:
115 - return self.random_sample(probs, sampling_metadata.generators,
116 - sampling_metadata.no_generator)
114 + return self.random_sample(probs, sampling_metadata.generators)
115
116 greedy_sampled = self.greedy_sample(probs)
117 random_sampled = self.random_sample(probs,
120 - sampling_metadata.generators,
121 - sampling_metadata.no_generator)
118 + sampling_metadata.generators)
119 sampled = torch.where(
120 sampling_metadata.temperature < _SAMPLING_EPS,
121 greedy_sampled,
vllm/v1/worker/gpu_model_runner.py
+29 -32
@@ -128,13 +128,20 @@ class GPUModelRunner:
128 # Add new requests to the cached states.
129 for req_data in scheduler_output.scheduled_new_reqs:
130 req_id = req_data.req_id
131 + sampling_params = req_data.sampling_params
132 + if sampling_params.seed is not None:
133 + generator = torch.Generator(device=self.device)
134 + generator.manual_seed(sampling_params.seed)
135 + else:
136 + generator = None
137 +
138 self.requests[req_id] = CachedRequestState(
139 req_id=req_id,
140 prompt_token_ids=req_data.prompt_token_ids,
141 prompt=req_data.prompt,
142 multi_modal_data=req_data.multi_modal_data,
136 - sampling_params=req_data.sampling_params,
137 - generator=None, # TODO
143 + sampling_params=sampling_params,
144 + generator=generator,
145 block_ids=req_data.block_ids,
146 num_computed_tokens=req_data.num_computed_tokens,
147 output_token_ids=[],
@@ -342,11 +349,9 @@ class GPUModelRunner:
349 else:
350 # Ignore the sampled token from the partial request.
351 # Rewind the generator state as if the token was not sampled.
345 - generator = self.input_batch.generators[i]
352 + generator = self.input_batch.generators.get(i)
353 if generator is not None:
347 - offset = generator.get_offset()
348 - generator = generator.set_offset(offset - 1)
349 - self.input_batch.generators[i] = generator
354 + generator.set_offset(generator.get_offset() - 1)
355
356 if sampler_output.logprob_token_ids is None:
357 logprob_token_ids = None
@@ -494,8 +499,8 @@ class InputBatch:
499 self.top_k_cpu = self.top_k_cpu_tensor.numpy()
500 self.top_k_reqs: Set[str] = set()
501
497 - self.generators: List[Optional[torch.Generator]] = [None
498 - ] * max_num_reqs
502 + # req_index -> generator
503 + self.generators: Dict[int, torch.Generator] = {}
504
505 self.num_logprobs: Dict[str, int] = {}
506 self.prompt_logprob_reqs: Set[str] = set()
@@ -509,8 +514,9 @@ class InputBatch:
514 req_index = self.num_reqs
515 assert req_index < self.max_num_reqs
516
512 - self.req_ids[req_index] = request.req_id
513 - self.req_id_to_index[request.req_id] = req_index
517 + req_id = request.req_id
518 + self.req_ids[req_index] = req_id
519 + self.req_id_to_index[req_id] = req_index
520
521 # Copy the prompt token ids and output token ids.
522 num_prompt_tokens = len(request.prompt_token_ids)
@@ -528,27 +534,24 @@ class InputBatch:
534 sampling_params = request.sampling_params
535 self.temperature_cpu[req_index] = sampling_params.temperature
536 if sampling_params.sampling_type == SamplingType.GREEDY:
531 - self.greedy_reqs.add(req_index)
532 - elif sampling_params.sampling_type == SamplingType.RANDOM:
533 - self.random_reqs.add(req_index)
534 - elif sampling_params.sampling_type == SamplingType.RANDOM_SEED:
535 - # TODO(woosuk): Support per-request random seed.
536 - raise NotImplementedError("Per-request seed is not supported yet.")
537 + self.greedy_reqs.add(req_id)
538 + else:
539 + self.random_reqs.add(req_id)
540
541 self.top_p_cpu[req_index] = sampling_params.top_p
542 if sampling_params.top_p < 1:
540 - self.top_p_reqs.add(req_index)
543 + self.top_p_reqs.add(req_id)
544 self.top_k_cpu[req_index] = sampling_params.top_k
545 if sampling_params.top_k > 0:
543 - self.top_k_reqs.add(req_index)
546 + self.top_k_reqs.add(req_id)
547
548 self.generators[req_index] = request.generator
549
550 num_logprobs = sampling_params.logprobs
551 if num_logprobs is not None and num_logprobs > 0:
549 - self.num_logprobs[request.req_id] = num_logprobs
552 + self.num_logprobs[req_id] = num_logprobs
553 if sampling_params.prompt_logprobs:
551 - self.prompt_logprob_reqs.add(req_index)
554 + self.prompt_logprob_reqs.add(req_id)
555
556 def remove_request(self, req_id: str) -> Optional[int]:
557 req_index = self.req_id_to_index.pop(req_id, None)
@@ -560,7 +563,7 @@ class InputBatch:
563 self.random_reqs.discard(req_id)
564 self.top_p_reqs.discard(req_id)
565 self.top_k_reqs.discard(req_id)
563 - self.generators[req_index] = None
566 + self.generators.pop(req_index, None)
567 self.num_logprobs.pop(req_id, None)
568 self.prompt_logprob_reqs.discard(req_id)
569 return req_index
@@ -612,7 +615,9 @@ class InputBatch:
615 last_req_index]
616 self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
617 self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
615 - self.generators[empty_index] = self.generators[last_req_index]
618 + generator = self.generators.pop(last_req_index, None)
619 + if generator is not None:
620 + self.generators[empty_index] = generator
621
622 # Decrement last_req_index since it is now empty.
623 last_req_index -= 1
@@ -636,8 +641,7 @@ class InputBatch:
641 top_k=self.top_k[:self.num_reqs],
642 no_top_p=self.no_top_p,
643 no_top_k=self.no_top_k,
639 - generators=self.generators[:self.num_reqs],
640 - no_generator=self.no_generator,
644 + generators=self.generators,
645 max_num_logprobs=self.max_num_logprobs,
646 )
647
@@ -661,16 +665,9 @@ class InputBatch:
665 def no_top_k(self) -> bool:
666 return len(self.top_k_reqs) == 0
667
664 - @property
665 - def no_generator(self) -> bool:
666 - return len(self.generators) == 0
667 -
668 @property
669 def max_num_logprobs(self) -> int:
670 - if self.num_logprobs:
671 - return max(self.num_logprobs.values())
672 - else:
673 - return 0
670 + return max(self.num_logprobs.values()) if self.num_logprobs else 0
671
672 @property
673 def no_logprob(self) -> bool: