A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] add w8a8 asym models (#11075)
Dipika Sikka committed
Dec 23, 2024 at 13:33 UTC
8cef6e02dcba8a1fa680cd130222bd3d47d54796
1 file changed
+10
-6
tests/quantization/test_compressed_tensors.py
+10
-6
@@ -79,12 +79,12 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
79
assert output
80
81
82
-@pytest.mark.parametrize(
83
- "model_path",
84
- [
85
- "neuralmagic/Llama-3.2-1B-quantized.w8a8"
86
- # TODO static & asymmetric
87
- ])
82
+@pytest.mark.parametrize("model_path", [
83
+ "neuralmagic/Llama-3.2-1B-quantized.w8a8",
84
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
85
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
86
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
87
+])
88
@pytest.mark.parametrize("max_tokens", [32])
89
@pytest.mark.parametrize("num_logprobs", [10])
90
def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
@@ -92,6 +92,10 @@ def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
92
max_tokens, num_logprobs):
93
dtype = "bfloat16"
94
95
+ # skip language translation prompt for the static per tensor asym model
96
+ if model_path == "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym": # noqa: E501
97
+ example_prompts = example_prompts[0:-1]
98
+
99
with hf_runner(model_path, dtype=dtype) as hf_model:
100
hf_outputs = hf_model.generate_greedy_logprobs_limit(
101
example_prompts, max_tokens, num_logprobs)