@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6faec54505
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Do not store `None` in self.generators (#11038)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 9, 2024 at 15:08 UTC 6faec545057e6152e92e8ab619fc018e20864943
1 file changed +6 -1
vllm/v1/worker/gpu_input_batch.py
+6 -1
@@ -102,6 +102,8 @@ class InputBatch:
102 self.top_k_reqs: Set[str] = set()
103
104 # req_index -> generator
105 + # NOTE(woosuk): The indices of the requests that do not have their own
106 + # generator should not be included in the dictionary.
107 self.generators: Dict[int, torch.Generator] = {}
108
109 self.num_logprobs: Dict[str, int] = {}
@@ -147,7 +149,10 @@ class InputBatch:
149 if sampling_params.top_k > 0:
150 self.top_k_reqs.add(req_id)
151
150 - self.generators[req_index] = request.generator
152 + # NOTE(woosuk): self.generators should not include the requests that
153 + # do not have their own generator.
154 + if request.generator is not None:
155 + self.generators[req_index] = request.generator
156
157 num_logprobs = sampling_params.logprobs
158 if num_logprobs is not None and num_logprobs > 0: