A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix token padding for chameleon (#6724)
Roger Wang committed
Jul 24, 2024 at 01:05 UTC
0a740a11ba055d6a2ae4786db6510684bfb7a887
1 file changed
+2
-1
vllm/model_executor/models/chameleon.py
+2
-1
@@ -125,7 +125,8 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
125
126
# Appending sep token for chat mode to follow default processor
127
# behavior
128
- new_prompt += tokenizer.sep_token
128
+ if new_prompt is not None:
129
+ new_prompt += tokenizer.sep_token
130
new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
131
132
# NOTE: Create a defensive copy of the original inputs