@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 31f46a0d35
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] Fix seeded random sampling with encoder-decoder models (#8870)

Co-authored-by: Roger Wang <ywang@roblox.com>

Nick Hill committed Sep 29, 2024 at 10:43 UTC 31f46a0d35da80118bac5f80c533019cd50ddd9a
1 file changed +3 -1
vllm/worker/enc_dec_model_runner.py
+3 -1
@@ -268,11 +268,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
268 encoder_input_positions=encoder_input_positions_tensor,
269 )
270
271 + generators = self.get_generators(finished_requests_ids)
272 sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
273 model_input.seq_lens,
274 model_input.query_lens,
275 self.device,
275 - self.pin_memory)
276 + self.pin_memory,
277 + generators=generators)
278 is_prompt = (seq_group_metadata_list[0].is_prompt
279 if seq_group_metadata_list else None)
280 return dataclasses.replace(model_input,