A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Docs] Lazy import gguf (#20785)
Signed-off-by: simon-mo <simon.mo@hey.com>
Simon Mo committed
Jul 10, 2025 at 16:06 UTC
b854321ffe50fd04c6b1ac58eecdab4caf5b4295
2 files changed
+10
-2
vllm/entrypoints/score_utils.py
+5
-1
@@ -11,7 +11,6 @@ from vllm.entrypoints.chat_utils import (
11
ChatCompletionContentPartImageParam, ChatCompletionContentPartTextParam,
12
MultiModalItemTracker, _ContentPart, _parse_chat_message_content_part)
13
from vllm.inputs import TokensPrompt
14
-from vllm.model_executor.model_loader import get_model_cls
14
from vllm.model_executor.models.interfaces import supports_score_template
15
from vllm.multimodal.inputs import MultiModalDataDict
16
from vllm.outputs import PoolingRequestOutput
@@ -140,6 +139,8 @@ def apply_score_template(
139
prompt_1: str,
140
prompt_2: str,
141
) -> str:
142
+ # NOTE(Simon): lazy import to avoid bring in all dependencies (e.g. gguf)
143
+ from vllm.model_executor.model_loader import get_model_cls
144
145
model = get_model_cls(model_config)
146
if supports_score_template(model):
@@ -162,6 +163,9 @@ def post_process_tokens(
163
Note:
164
This is an in-place operation.
165
"""
166
+ # NOTE(Simon): lazy import to avoid bring in all dependencies (e.g. gguf)
167
+ from vllm.model_executor.model_loader import get_model_cls
168
+
169
model = get_model_cls(model_config)
170
if supports_score_template(model):
171
model.post_process_tokens(prompt)
vllm/model_executor/model_loader/weight_utils.py
+5
-1
@@ -14,7 +14,6 @@ from pathlib import Path
14
from typing import Any, Callable, Optional, Union
15
16
import filelock
17
-import gguf
17
import huggingface_hub.constants
18
import numpy as np
19
import torch
@@ -40,6 +39,11 @@ except (ImportError, OSError):
39
SafetensorsStreamer = runai_model_streamer.placeholder_attr(
40
"SafetensorsStreamer")
41
42
+try:
43
+ import gguf
44
+except ImportError:
45
+ gguf = PlaceholderModule("gguf")
46
+
47
try:
48
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
49
except ImportError: