@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6567e13724
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix crash with llama 3.2 vision models and guided decoding (#9631)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: pavlo-ruban <pavlo.ruban@servicenow.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Travis Johnson committed Oct 25, 2024 at 16:42 UTC 6567e13724110fac2042d06a9e4c01fd822e8909
1 file changed +11 -3
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+11 -3
@@ -15,11 +15,11 @@
15 # limitations under the License.
16 import copy
17 import json
18 -import math
18 from collections import defaultdict
19 from functools import lru_cache
20 from typing import Callable, DefaultDict, Dict, List, Union
21
22 +import numpy as np
23 import torch
24 from lark import Lark
25 from outlines import grammars
@@ -77,9 +77,17 @@ class BaseLogitsProcessor:
77 f"Unsupported instruction type {type(instruction)}")
78
79 mask = torch.full((scores.shape[-1], ),
80 - -math.inf,
80 + -torch.inf,
81 device=scores.device)
82 - mask[allowed_tokens] = 0
82 + # The tokenizer may support more token ids than the model can generate,
83 + # eg. Llama 3.2 Vision models have an `<|image|>` token with id 128256
84 + # but scores.shape == torch.Size([128256])
85 + # Using NumPy is faster for filtering token ids
86 + allowed_tokens = np.array(allowed_tokens, dtype=np.int64)
87 + allowed_tokens = torch.tensor(allowed_tokens, device=scores.device)
88 + allowed_tokens = allowed_tokens.masked_select(
89 + allowed_tokens < scores.shape[-1])
90 + mask.index_fill_(0, allowed_tokens, 0)
91 scores.add_(mask)
92 return scores
93