@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5dbf854553
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[CI/Build][CPU] Fix CPU CI by lazy importing triton FP8 kernels (#11618)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Dec 30, 2024 at 18:17 UTC 5dbf854553cb6ac97f0c633ed36ba64e0fc9bb29
1 file changed +3 -2
vllm/model_executor/layers/quantization/fp8.py
+3 -2
@@ -15,8 +15,6 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
18 -from vllm.model_executor.layers.quantization.utils.fp8_utils import (
19 - apply_w8a8_block_fp8_linear)
18 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
19 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
20 from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -337,6 +335,9 @@ class Fp8LinearMethod(LinearMethodBase):
335 size_k=layer.input_size_per_partition,
336 bias=bias)
337
338 + # Note: lazy import to avoid triton import error.
339 + from vllm.model_executor.layers.quantization.utils.fp8_utils import (
340 + apply_w8a8_block_fp8_linear)
341 if self.block_quant:
342 assert self.quant_config.weight_block_size is not None
343 return apply_w8a8_block_fp8_linear(