@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b9241be3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding] [Bugfix] Fix overallocation in ngram + spec logprobs (#4672)

Cade Daniel committed May 8, 2024 at 16:24 UTC 8b9241be3a0020724e145bf600d9710b3d59b167
1 file changed +1 -1
vllm/spec_decode/ngram_worker.py
+1 -1
@@ -138,7 +138,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
138 SamplerOutput(
139 outputs=None,
140 sampled_token_probs=token_probs[i],
141 - logprobs=token_logprobs,
141 + logprobs=token_logprobs[i],
142 sampled_token_ids=token_ids[i],
143 ))
144 return outputs, False