@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb3db61688
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build] Remove sparseml requirement from testing (#7037)

Michael Goin committed Aug 1, 2024 at 15:00 UTC fb3db616881d7225c4bbe64bb709ea6bcd6157f7
4 files changed +1 -58
requirements-test.txt
-1
@@ -14,7 +14,6 @@ peft
14 requests
15 ray
16 sentence-transformers # required for embedding
17 -sparseml==1.8.0 # required for compressed-tensors
17 compressed-tensors==0.4.0 # required for compressed-tensors
18 timm # required for internvl test
19
tests/conftest.py
-4
@@ -152,7 +152,6 @@ class HfRunner:
152 model_kwargs: Optional[Dict[str, Any]] = None,
153 is_embedding_model: bool = False,
154 is_vision_model: bool = False,
155 - is_sparseml_model: bool = False,
155 ) -> None:
156 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
157
@@ -169,9 +168,6 @@ class HfRunner:
168 else:
169 if is_vision_model:
170 auto_cls = AutoModelForVision2Seq
172 - elif is_sparseml_model:
173 - from sparseml.transformers import SparseAutoModelForCausalLM
174 - auto_cls = SparseAutoModelForCausalLM
171 else:
172 auto_cls = AutoModelForCausalLM
173
tests/models/test_compressed_tensors.py deleted
-52
@@ -1,52 +0,0 @@
1 -"""Compares vllm vs sparseml for compressed-tensors
2 -
3 -Note: vllm and sparseml do not have bitwise correctness,
4 -so in this test, we just confirm that the top selected
5 -tokens of the are in the top 5 selections of each other.
6 -"""
7 -
8 -import pytest
9 -
10 -from tests.quantization.utils import is_quant_method_supported
11 -
12 -from .utils import check_logprobs_close
13 -
14 -MODELS = [
15 - # No bias
16 - "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test",
17 - # Bias
18 - "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
19 -]
20 -
21 -MAX_TOKENS = 32
22 -NUM_LOGPROBS = 5
23 -
24 -
25 -@pytest.mark.skipif(
26 - not is_quant_method_supported("compressed-tensors"),
27 - reason="compressed-tensors is not supported on this machine type.")
28 -@pytest.mark.parametrize("model_name", MODELS)
29 -def test_models(
30 - vllm_runner,
31 - hf_runner,
32 - example_prompts,
33 - model_name,
34 -) -> None:
35 - # Run sparseml.
36 - with hf_runner(model_name=model_name,
37 - is_sparseml_model=True) as sparseml_model:
38 -
39 - sparseml_outputs = sparseml_model.generate_greedy_logprobs_limit(
40 - example_prompts, MAX_TOKENS, NUM_LOGPROBS)
41 -
42 - # Run vllm.
43 - with vllm_runner(model_name=model_name) as vllm_model:
44 - vllm_outputs = vllm_model.generate_greedy_logprobs(
45 - example_prompts, MAX_TOKENS, NUM_LOGPROBS)
46 -
47 - check_logprobs_close(
48 - outputs_0_lst=sparseml_outputs,
49 - outputs_1_lst=vllm_outputs,
50 - name_0="sparseml",
51 - name_1="vllm",
52 - )
tests/quantization/test_compressed_tensors.py
+1 -1
@@ -1,4 +1,4 @@
1 -"""Test model set-up and weight loading for sparseml-quantized models.
1 +"""Test model set-up and weight loading for llmcompressor-quantized models.
2
3 Run `pytest tests/quantization/test_compressed_tensors.py`.
4 """