A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix crash with llama 3.2 vision models and guided decoding (#9631)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: pavlo-ruban <pavlo.ruban@servicenow.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Travis Johnson committed
Oct 25, 2024 at 16:42 UTC
6567e13724110fac2042d06a9e4c01fd822e8909
1 file changed
+11
-3
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+11
-3
@@ -15,11 +15,11 @@
15
# limitations under the License.
16
import copy
17
import json
18
-import math
18
from collections import defaultdict
19
from functools import lru_cache
20
from typing import Callable, DefaultDict, Dict, List, Union
21
22
+import numpy as np
23
import torch
24
from lark import Lark
25
from outlines import grammars
@@ -77,9 +77,17 @@ class BaseLogitsProcessor:
77
f"Unsupported instruction type {type(instruction)}")
78
79
mask = torch.full((scores.shape[-1], ),
80
- -math.inf,
80
+ -torch.inf,
81
device=scores.device)
82
- mask[allowed_tokens] = 0
82
+ # The tokenizer may support more token ids than the model can generate,
83
+ # eg. Llama 3.2 Vision models have an `<|image|>` token with id 128256
84
+ # but scores.shape == torch.Size([128256])
85
+ # Using NumPy is faster for filtering token ids
86
+ allowed_tokens = np.array(allowed_tokens, dtype=np.int64)
87
+ allowed_tokens = torch.tensor(allowed_tokens, device=scores.device)
88
+ allowed_tokens = allowed_tokens.masked_select(
89
+ allowed_tokens < scores.shape[-1])
90
+ mask.index_fill_(0, allowed_tokens, 0)
91
scores.add_(mask)
92
return scores
93