@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad6eca408b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Fix early CUDA init via get_architecture_class_name import (#3770)

Signed-off-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Lei Wen <wenlei03@qiyi.com>

leiwen83 committed Apr 3, 2024 at 02:56 UTC ad6eca408bc515391992db0f8becaa78e23d1150
1 file changed +2 -1
vllm/engine/llm_engine.py
+2 -1
@@ -13,7 +13,6 @@ from vllm.engine.ray_utils import initialize_ray_cluster
13 from vllm.executor.executor_base import ExecutorBase
14 from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 -from vllm.model_executor.model_loader import get_architecture_class_name
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
@@ -115,6 +114,8 @@ class LLMEngine:
114
115 # If usage stat is enabled, collect relevant info.
116 if is_usage_stats_enabled():
117 + from vllm.model_executor.model_loader import (
118 + get_architecture_class_name)
119 usage_message.report_usage(
120 get_architecture_class_name(model_config),
121 usage_context,