A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[CI] Add smoke test for non-uniform AutoFP8 quantization (#6702)
Michael Goin committed
Jul 23, 2024 at 18:45 UTC
01c16ede6b4fc2e07b6eb5a4f60a64a1f365e460
1 file changed
+1
tests/quantization/test_fp8.py
+1
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.fp8 import (Fp8KVCacheMethod,
13
MODELS = [
14
"neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
15
"nm-testing/Phi-3-mini-128k-instruct-FP8",
16
+ "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
17
]
18
19