A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[BugFix] Pass tokenizer_config to local_tokenizer_group (#3754)
Signed-off-by: Tao He <sighingnow@gmail.com>
Tao He committed
Apr 4, 2024 at 11:31 UTC
294f8f6665b1731d1a8786e518d9862f7a7043df
1 file changed
+1
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+1
@@ -51,6 +51,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
51
enable_lora=enable_lora,
52
max_num_seqs=max_num_seqs,
53
max_input_length=max_input_length,
54
+ **tokenizer_config,
55
)
56
57
ray_tokenizer_group_cls = ray.remote(