A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Fix early CUDA init via get_architecture_class_name import (#3770)
Signed-off-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Lei Wen <wenlei03@qiyi.com>
leiwen83 committed
Apr 3, 2024 at 02:56 UTC
ad6eca408bc515391992db0f8becaa78e23d1150
1 file changed
+2
-1
vllm/engine/llm_engine.py
+2
-1
@@ -13,7 +13,6 @@ from vllm.engine.ray_utils import initialize_ray_cluster
13
from vllm.executor.executor_base import ExecutorBase
14
from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
-from vllm.model_executor.model_loader import get_architecture_class_name
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
@@ -115,6 +114,8 @@ class LLMEngine:
114
115
# If usage stat is enabled, collect relevant info.
116
if is_usage_stats_enabled():
117
+ from vllm.model_executor.model_loader import (
118
+ get_architecture_class_name)
119
usage_message.report_usage(
120
get_architecture_class_name(model_config),
121
usage_context,