A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding] [Bugfix] Fix overallocation in ngram + spec logprobs (#4672)
Cade Daniel committed
May 8, 2024 at 16:24 UTC
8b9241be3a0020724e145bf600d9710b3d59b167
1 file changed
+1
-1
vllm/spec_decode/ngram_worker.py
+1
-1
@@ -138,7 +138,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
138
SamplerOutput(
139
outputs=None,
140
sampled_token_probs=token_probs[i],
141
- logprobs=token_logprobs,
141
+ logprobs=token_logprobs[i],
142
sampled_token_ids=token_ids[i],
143
))
144
return outputs, False