@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f192aeba74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Enable some fp8 and quantized fullgraph tests (#10171)

Signed-off-by: Bill Nell <bill@neuralmagic.com>

bnellnm committed Nov 9, 2024 at 03:01 UTC f192aeba74ebf5a6d1a0fccc9a84e8fe99f8c619
1 file changed +13 -16
tests/compile/utils.py
+13 -16
@@ -9,29 +9,26 @@ from vllm.platforms import current_platform
9
10 TEST_MODELS = [
11 ("facebook/opt-125m", {}),
12 - # TODO: add fake implementation for compressed-tensors
13 - # ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
14 - # "dtype": torch.float16,
15 - # "quantization": "compressed-tensors"
16 - # }),
12 + ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
13 + "dtype": torch.float16,
14 + "quantization": "compressed-tensors"
15 + }),
16 ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
17 "dtype": torch.float16,
18 "quantization": "fp8"
19 }),
21 - # TODO: add fake implementation for compressed-tensors
22 - # ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
23 - # "quantization": "compressed-tensors"
24 - # }),
20 + ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21 + "quantization": "compressed-tensors"
22 + }),
23 ("meta-llama/Meta-Llama-3-8B", {}),
24 ]
25
28 -# TODO: enable in pytorch 2.5
29 -if False and is_quant_method_supported("aqlm"): # noqa: SIM223
26 +if is_quant_method_supported("aqlm"):
27 TEST_MODELS.append(("ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf", {
28 "quantization": "aqlm"
29 }))
30
34 -# TODO: enable in pytorch 2.5
31 +# TODO: figure out why this fails.
32 if False and is_quant_method_supported("gguf"): # noqa: SIM223
33 TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {
34 "quantization": "gguf"
@@ -71,13 +68,13 @@ def check_full_graph_support(model,
68 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
69 os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
70
74 - # Inductor doesn't support fp8 and the base meta llama uses too
75 - # much memory.
76 - quantization = model_kwargs.get("quantization")
77 - if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
71 + # The base meta llama uses too much memory.
72 + if (model == "meta-llama/Meta-Llama-3-8B"
73 and optimization_level >= CompilationLevel.PIECEWISE):
74 return
75
76 + print(f"MODEL={model}")
77 +
78 prompts = [
79 "Hello, my name is",
80 "The president of the United States is",