A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Enable some fp8 and quantized fullgraph tests (#10171)
Signed-off-by: Bill Nell <bill@neuralmagic.com>
bnellnm committed
Nov 9, 2024 at 03:01 UTC
f192aeba74ebf5a6d1a0fccc9a84e8fe99f8c619
1 file changed
+13
-16
tests/compile/utils.py
+13
-16
@@ -9,29 +9,26 @@ from vllm.platforms import current_platform
9
10
TEST_MODELS = [
11
("facebook/opt-125m", {}),
12
- # TODO: add fake implementation for compressed-tensors
13
- # ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
14
- # "dtype": torch.float16,
15
- # "quantization": "compressed-tensors"
16
- # }),
12
+ ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
13
+ "dtype": torch.float16,
14
+ "quantization": "compressed-tensors"
15
+ }),
16
("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
17
"dtype": torch.float16,
18
"quantization": "fp8"
19
}),
21
- # TODO: add fake implementation for compressed-tensors
22
- # ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
23
- # "quantization": "compressed-tensors"
24
- # }),
20
+ ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21
+ "quantization": "compressed-tensors"
22
+ }),
23
("meta-llama/Meta-Llama-3-8B", {}),
24
]
25
28
-# TODO: enable in pytorch 2.5
29
-if False and is_quant_method_supported("aqlm"): # noqa: SIM223
26
+if is_quant_method_supported("aqlm"):
27
TEST_MODELS.append(("ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf", {
28
"quantization": "aqlm"
29
}))
30
34
-# TODO: enable in pytorch 2.5
31
+# TODO: figure out why this fails.
32
if False and is_quant_method_supported("gguf"): # noqa: SIM223
33
TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {
34
"quantization": "gguf"
@@ -71,13 +68,13 @@ def check_full_graph_support(model,
68
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
69
os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
70
74
- # Inductor doesn't support fp8 and the base meta llama uses too
75
- # much memory.
76
- quantization = model_kwargs.get("quantization")
77
- if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
71
+ # The base meta llama uses too much memory.
72
+ if (model == "meta-llama/Meta-Llama-3-8B"
73
and optimization_level >= CompilationLevel.PIECEWISE):
74
return
75
76
+ print(f"MODEL={model}")
77
+
78
prompts = [
79
"Hello, my name is",
80
"The president of the United States is",