A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[BugFix] Fix seeded random sampling with encoder-decoder models (#8870)
Co-authored-by: Roger Wang <ywang@roblox.com>
Nick Hill committed
Sep 29, 2024 at 10:43 UTC
31f46a0d35da80118bac5f80c533019cd50ddd9a
1 file changed
+3
-1
vllm/worker/enc_dec_model_runner.py
+3
-1
@@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
268
encoder_input_positions=encoder_input_positions_tensor,
269
)
270
271
+ generators = self.get_generators(finished_requests_ids)
272
sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
273
model_input.seq_lens,
274
model_input.query_lens,
275
self.device,
275
- self.pin_memory)
276
+ self.pin_memory,
277
+ generators=generators)
278
is_prompt = (seq_group_metadata_list[0].is_prompt
279
if seq_group_metadata_list else None)
280
return dataclasses.replace(model_input,