@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8cef6e02dc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] add w8a8 asym models (#11075)

Dipika Sikka committed Dec 23, 2024 at 13:33 UTC 8cef6e02dcba8a1fa680cd130222bd3d47d54796
1 file changed +10 -6
tests/quantization/test_compressed_tensors.py
+10 -6
@@ -79,12 +79,12 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
79 assert output
80
81
82 -@pytest.mark.parametrize(
83 - "model_path",
84 - [
85 - "neuralmagic/Llama-3.2-1B-quantized.w8a8"
86 - # TODO static & asymmetric
87 - ])
82 +@pytest.mark.parametrize("model_path", [
83 + "neuralmagic/Llama-3.2-1B-quantized.w8a8",
84 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
85 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
86 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
87 +])
88 @pytest.mark.parametrize("max_tokens", [32])
89 @pytest.mark.parametrize("num_logprobs", [10])
90 def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
@@ -92,6 +92,10 @@ def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
92 max_tokens, num_logprobs):
93 dtype = "bfloat16"
94
95 + # skip language translation prompt for the static per tensor asym model
96 + if model_path == "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym": # noqa: E501
97 + example_prompts = example_prompts[0:-1]
98 +
99 with hf_runner(model_path, dtype=dtype) as hf_model:
100 hf_outputs = hf_model.generate_greedy_logprobs_limit(
101 example_prompts, max_tokens, num_logprobs)