@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8dbfcd35bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ CI/Build ] Added E2E Test For Compressed Tensors (#5839)

Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 29, 2024 at 09:12 UTC 8dbfcd35bf2313dedc9e947a991b6e0044248589
4 files changed +57 -1
requirements-test.txt
+2
@@ -14,6 +14,8 @@ peft
14 requests
15 ray
16 sentence-transformers # required for embedding
17 +sparseml==1.8.0 # required for compressed-tensors
18 +compressed-tensors==0.4.0 # required for compressed-tensors
19
20 # Benchmarking
21 aiohttp
tests/conftest.py
+4
@@ -176,6 +176,7 @@ class HfRunner:
176 model_kwargs: Optional[Dict[str, Any]] = None,
177 is_embedding_model: bool = False,
178 is_vision_model: bool = False,
179 + is_sparseml_model: bool = False,
180 ) -> None:
181 assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
182 torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
@@ -193,6 +194,9 @@ class HfRunner:
194 else:
195 if is_vision_model:
196 auto_cls = AutoModelForVision2Seq
197 + elif is_sparseml_model:
198 + from sparseml.transformers import SparseAutoModelForCausalLM
199 + auto_cls = SparseAutoModelForCausalLM
200 else:
201 auto_cls = AutoModelForCausalLM
202
tests/models/test_compressed_tensors.py new
+49
@@ -0,0 +1,49 @@
1 +"""Compares vllm vs sparseml for compressed-tensors
2 +
3 +Note: vllm and sparseml do not have bitwise correctness,
4 +so in this test, we just confirm that the top selected
5 +tokens of the are in the top 5 selections of each other.
6 +"""
7 +
8 +import pytest
9 +
10 +from tests.quantization.utils import is_quant_method_supported
11 +
12 +from .utils import check_logprobs_close
13 +
14 +MODELS = [
15 + "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test",
16 +]
17 +
18 +MAX_TOKENS = 32
19 +NUM_LOGPROBS = 5
20 +
21 +
22 +@pytest.mark.skipif(
23 + not is_quant_method_supported("compressed-tensors"),
24 + reason="compressed-tensors is not supported on this machine type.")
25 +@pytest.mark.parametrize("model_name", MODELS)
26 +def test_models(
27 + vllm_runner,
28 + hf_runner,
29 + example_prompts,
30 + model_name,
31 +) -> None:
32 + # Run sparseml.
33 + with hf_runner(model_name=model_name,
34 + is_sparseml_model=True) as sparseml_model:
35 +
36 + sparseml_outputs = sparseml_model.generate_greedy_logprobs_limit(
37 + example_prompts, MAX_TOKENS, NUM_LOGPROBS)
38 +
39 + # Run vllm.
40 + with vllm_runner(model_name=model_name) as vllm_model:
41 + vllm_outputs = vllm_model.generate_greedy_logprobs(
42 + example_prompts, MAX_TOKENS, NUM_LOGPROBS)
43 +
44 + check_logprobs_close(
45 + outputs_0_lst=sparseml_outputs,
46 + outputs_1_lst=vllm_outputs,
47 + name_0="sparseml",
48 + name_1="vllm",
49 + )
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -1
@@ -34,7 +34,8 @@ class CompressedTensorsConfig(QuantizationConfig):
34 return [torch.float16, torch.bfloat16]
35
36 # Need to figure it out
37 - def get_min_capability(self) -> int:
37 + @classmethod
38 + def get_min_capability(cls) -> int:
39 return 60
40
41 def get_name(self) -> str: