@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01c16ede6b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[CI] Add smoke test for non-uniform AutoFP8 quantization (#6702)

Michael Goin committed Jul 23, 2024 at 18:45 UTC 01c16ede6b4fc2e07b6eb5a4f60a64a1f365e460
1 file changed +1
tests/quantization/test_fp8.py
+1
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.fp8 import (Fp8KVCacheMethod,
13 MODELS = [
14 "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
15 "nm-testing/Phi-3-mini-128k-instruct-FP8",
16 + "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
17 ]
18
19