A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[CI/Build][CPU] Fix CPU CI by lazy importing triton FP8 kernels (#11618)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Dec 30, 2024 at 18:17 UTC
5dbf854553cb6ac97f0c633ed36ba64e0fc9bb29
1 file changed
+3
-2
vllm/model_executor/layers/quantization/fp8.py
+3
-2
@@ -15,8 +15,6 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig, QuantizeMethodBase)
17
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
18
-from vllm.model_executor.layers.quantization.utils.fp8_utils import (
19
- apply_w8a8_block_fp8_linear)
18
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
19
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
20
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -337,6 +335,9 @@ class Fp8LinearMethod(LinearMethodBase):
335
size_k=layer.input_size_per_partition,
336
bias=bias)
337
338
+ # Note: lazy import to avoid triton import error.
339
+ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
340
+ apply_w8a8_block_fp8_linear)
341
if self.block_quant:
342
assert self.quant_config.weight_block_size is not None
343
return apply_w8a8_block_fp8_linear(