A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[CI/Build] Remove sparseml requirement from testing (#7037)
Michael Goin committed
Aug 1, 2024 at 15:00 UTC
fb3db616881d7225c4bbe64bb709ea6bcd6157f7
4 files changed
+1
-58
requirements-test.txt
-1
@@ -14,7 +14,6 @@ peft
14
requests
15
ray
16
sentence-transformers # required for embedding
17
-sparseml==1.8.0 # required for compressed-tensors
17
compressed-tensors==0.4.0 # required for compressed-tensors
18
timm # required for internvl test
19
tests/conftest.py
-4
@@ -152,7 +152,6 @@ class HfRunner:
152
model_kwargs: Optional[Dict[str, Any]] = None,
153
is_embedding_model: bool = False,
154
is_vision_model: bool = False,
155
- is_sparseml_model: bool = False,
155
) -> None:
156
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
157
@@ -169,9 +168,6 @@ class HfRunner:
168
else:
169
if is_vision_model:
170
auto_cls = AutoModelForVision2Seq
172
- elif is_sparseml_model:
173
- from sparseml.transformers import SparseAutoModelForCausalLM
174
- auto_cls = SparseAutoModelForCausalLM
171
else:
172
auto_cls = AutoModelForCausalLM
173
tests/models/test_compressed_tensors.py
deleted
-52
@@ -1,52 +0,0 @@
1
-"""Compares vllm vs sparseml for compressed-tensors
2
-
3
-Note: vllm and sparseml do not have bitwise correctness,
4
-so in this test, we just confirm that the top selected
5
-tokens of the are in the top 5 selections of each other.
6
-"""
7
-
8
-import pytest
9
-
10
-from tests.quantization.utils import is_quant_method_supported
11
-
12
-from .utils import check_logprobs_close
13
-
14
-MODELS = [
15
- # No bias
16
- "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test",
17
- # Bias
18
- "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
19
-]
20
-
21
-MAX_TOKENS = 32
22
-NUM_LOGPROBS = 5
23
-
24
-
25
-@pytest.mark.skipif(
26
- not is_quant_method_supported("compressed-tensors"),
27
- reason="compressed-tensors is not supported on this machine type.")
28
-@pytest.mark.parametrize("model_name", MODELS)
29
-def test_models(
30
- vllm_runner,
31
- hf_runner,
32
- example_prompts,
33
- model_name,
34
-) -> None:
35
- # Run sparseml.
36
- with hf_runner(model_name=model_name,
37
- is_sparseml_model=True) as sparseml_model:
38
-
39
- sparseml_outputs = sparseml_model.generate_greedy_logprobs_limit(
40
- example_prompts, MAX_TOKENS, NUM_LOGPROBS)
41
-
42
- # Run vllm.
43
- with vllm_runner(model_name=model_name) as vllm_model:
44
- vllm_outputs = vllm_model.generate_greedy_logprobs(
45
- example_prompts, MAX_TOKENS, NUM_LOGPROBS)
46
-
47
- check_logprobs_close(
48
- outputs_0_lst=sparseml_outputs,
49
- outputs_1_lst=vllm_outputs,
50
- name_0="sparseml",
51
- name_1="vllm",
52
- )
tests/quantization/test_compressed_tensors.py
+1
-1
@@ -1,4 +1,4 @@
1
-"""Test model set-up and weight loading for sparseml-quantized models.
1
+"""Test model set-up and weight loading for llmcompressor-quantized models.
2
3
Run `pytest tests/quantization/test_compressed_tensors.py`.
4
"""