@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a740a11ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix token padding for chameleon (#6724)

Roger Wang committed Jul 24, 2024 at 01:05 UTC 0a740a11ba055d6a2ae4786db6510684bfb7a887
1 file changed +2 -1
vllm/model_executor/models/chameleon.py
+2 -1
@@ -125,7 +125,8 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
125
126 # Appending sep token for chat mode to follow default processor
127 # behavior
128 - new_prompt += tokenizer.sep_token
128 + if new_prompt is not None:
129 + new_prompt += tokenizer.sep_token
130 new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
131
132 # NOTE: Create a defensive copy of the original inputs