A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[bugfix] Fix example/tensorize_vllm_model tests (#10595)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 24, 2024 at 09:22 UTC
17d8fc1806c61e3f859a45b69be9f8dccf9a5fcc
1 file changed
+3
-1
vllm/model_executor/model_loader/tensorizer.py
+3
-1
@@ -19,6 +19,7 @@ from vllm.engine.llm_engine import LLMEngine
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
22
+from vllm.plugins import set_current_vllm_config
23
from vllm.utils import FlexibleArgumentParser
24
25
tensorizer_error_msg = None
@@ -284,7 +285,8 @@ class TensorizerAgent:
285
model_args = self.tensorizer_config.hf_config
286
model_args.torch_dtype = self.tensorizer_config.dtype
287
assert self.tensorizer_config.model_class is not None
287
- with no_init_or_tensor():
288
+ # TODO: Do we need to consider old-style model class?
289
+ with no_init_or_tensor(), set_current_vllm_config(self.vllm_config):
290
return self.tensorizer_config.model_class(
291
vllm_config=self.vllm_config, )
292