A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Do not store `None` in self.generators (#11038)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 9, 2024 at 15:08 UTC
6faec545057e6152e92e8ab619fc018e20864943
1 file changed
+6
-1
vllm/v1/worker/gpu_input_batch.py
+6
-1
@@ -102,6 +102,8 @@ class InputBatch:
102
self.top_k_reqs: Set[str] = set()
103
104
# req_index -> generator
105
+ # NOTE(woosuk): The indices of the requests that do not have their own
106
+ # generator should not be included in the dictionary.
107
self.generators: Dict[int, torch.Generator] = {}
108
109
self.num_logprobs: Dict[str, int] = {}
@@ -147,7 +149,10 @@ class InputBatch:
149
if sampling_params.top_k > 0:
150
self.top_k_reqs.add(req_id)
151
150
- self.generators[req_index] = request.generator
152
+ # NOTE(woosuk): self.generators should not include the requests that
153
+ # do not have their own generator.
154
+ if request.generator is not None:
155
+ self.generators[req_index] = request.generator
156
157
num_logprobs = sampling_params.logprobs
158
if num_logprobs is not None and num_logprobs > 0: