@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 17d8fc1806
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[bugfix] Fix example/tensorize_vllm_model tests (#10595)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 24, 2024 at 09:22 UTC 17d8fc1806c61e3f859a45b69be9f8dccf9a5fcc
1 file changed +3 -1
vllm/model_executor/model_loader/tensorizer.py
+3 -1
@@ -19,6 +19,7 @@ from vllm.engine.llm_engine import LLMEngine
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 +from vllm.plugins import set_current_vllm_config
23 from vllm.utils import FlexibleArgumentParser
24
25 tensorizer_error_msg = None
@@ -284,7 +285,8 @@ class TensorizerAgent:
285 model_args = self.tensorizer_config.hf_config
286 model_args.torch_dtype = self.tensorizer_config.dtype
287 assert self.tensorizer_config.model_class is not None
287 - with no_init_or_tensor():
288 + # TODO: Do we need to consider old-style model class?
289 + with no_init_or_tensor(), set_current_vllm_config(self.vllm_config):
290 return self.tensorizer_config.model_class(
291 vllm_config=self.vllm_config, )
292