A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
add gptq_marlin test for bug report https://github.com/vllm-project/vllm/issues/5088 (#5145)
Alexander Matveev committed
Jun 15, 2024 at 13:38 UTC
d919ecc771ece6995a949c3d4284c534a2bd0890
1 file changed
+3
tests/models/test_gptq_marlin.py
+3
@@ -40,6 +40,9 @@ MODELS = [
40
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
41
# 8-bit, act_order==True, group_size=32
42
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
43
+
44
+ # 4-bit, act_order==True, group_size=128
45
+ ("TechxGenus/gemma-1.1-2b-it-GPTQ", "main")
46
]
47
48