@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d919ecc771
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

add gptq_marlin test for bug report https://github.com/vllm-project/vllm/issues/5088 (#5145)

Alexander Matveev committed Jun 15, 2024 at 13:38 UTC d919ecc771ece6995a949c3d4284c534a2bd0890
1 file changed +3
tests/models/test_gptq_marlin.py
+3
@@ -40,6 +40,9 @@ MODELS = [
40 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
41 # 8-bit, act_order==True, group_size=32
42 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
43 +
44 + # 4-bit, act_order==True, group_size=128
45 + ("TechxGenus/gemma-1.1-2b-it-GPTQ", "main")
46 ]
47
48