f6aa122698 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/quantization/test_compressed_tensors.py

+2 -2

@@ -141,7 +141,7 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):

141 "neuralmagic/Llama-3.2-1B-quantized.w8a8",

142 ],

143 )

144 -@pytest.mark.parametrize("max_tokens", [8])

144 +@pytest.mark.parametrize("max_tokens", [4])

145 @pytest.mark.parametrize("num_logprobs", [10])

146 @pytest.mark.parametrize(

147 "use_aiter", [True, False] if current_platform.is_rocm() else [False]

@@ -182,7 +182,7 @@ def test_compressed_tensors_w8a8_logprobs(

182 example_prompts, max_tokens, num_logprobs

183 )

184

185 - with vllm_runner(model_path, dtype=dtype) as vllm_model:

185 + with vllm_runner(model_path, dtype=dtype, enforce_eager=True) as vllm_model:

186 vllm_outputs = vllm_model.generate_greedy_logprobs(

187 example_prompts, max_tokens, num_logprobs

188 )

tests/quantization/test_cpu_offload.py

+8 -8

@@ -19,8 +19,8 @@ def test_cpu_offload_fp8():

19 # Test loading a quantized checkpoint

20 compare_two_settings(

21 "neuralmagic/Qwen2-1.5B-Instruct-FP8",

22 - [],

23 - ["--cpu-offload-gb", "1"],

22 + ["--enforce_eager"],

23 + ["--enforce_eager", "--cpu-offload-gb", "1"],

24 max_wait_seconds=480,

25 )

26

@@ -35,8 +35,8 @@ def test_cpu_offload_gptq(monkeypatch):

35 # Test GPTQ Marlin

36 compare_two_settings(

37 "Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",

38 - [],

39 - ["--cpu-offload-gb", "1"],

38 + ["--enforce_eager"],

39 + ["--enforce_eager", "--cpu-offload-gb", "1"],

40 max_wait_seconds=480,

41 )

42

@@ -51,8 +51,8 @@ def test_cpu_offload_awq(monkeypatch):

51 # Test AWQ Marlin

52 compare_two_settings(

53 "Qwen/Qwen2-1.5B-Instruct-AWQ",

54 - [],

55 - ["--cpu-offload-gb", "1"],

54 + ["--enforce_eager"],

55 + ["--enforce_eager", "--cpu-offload-gb", "1"],

56 max_wait_seconds=480,

57 )

58

@@ -67,7 +67,7 @@ def test_cpu_offload_compressed_tensors(monkeypatch):

67 # Test wNa16

68 compare_two_settings(

69 "nm-testing/tinyllama-oneshot-w4a16-channel-v2",

70 - [],

71 - ["--cpu-offload-gb", "1"],

70 + ["--enforce_eager"],

71 + ["--enforce_eager", "--cpu-offload-gb", "1"],

72 max_wait_seconds=480,

73 )

tests/quantization/test_experts_int8.py

+4 -2

@@ -21,7 +21,7 @@ MODELS = ["ai21labs/Jamba-tiny-random", "pfnet/plamo-2-1b"]

21 )

22 @pytest.mark.parametrize("model", MODELS)

23 @pytest.mark.parametrize("dtype", ["bfloat16"])

24 -@pytest.mark.parametrize("max_tokens", [10])

24 +@pytest.mark.parametrize("max_tokens", [4])

25 def test_model_experts_int8_startup(

26 hf_runner,

27 vllm_runner,

@@ -33,5 +33,7 @@ def test_model_experts_int8_startup(

33 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

34 model_info.check_transformers_version(on_fail="skip")

35

36 - with vllm_runner(model, dtype=dtype, quantization="experts_int8") as vllm_model:

36 + with vllm_runner(

37 + model, dtype=dtype, enforce_eager=True, quantization="experts_int8"

38 + ) as vllm_model:

39 vllm_model.generate_greedy(example_prompts, max_tokens)

tests/quantization/test_fp8.py

+8 -5

@@ -45,10 +45,10 @@ def test_model_load_and_run(

45 if force_marlin:

46 monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")

47

48 - with vllm_runner(model_id) as llm:

48 + with vllm_runner(model_id, enforce_eager=True) as llm:

49 # note: this does not test accuracy, just that we can run through

50 # see lm-eval tests for accuracy

51 - outputs = llm.generate_greedy(["Hello my name is"], max_tokens=10)

51 + outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)

52 print(outputs[0][1])

53

54

@@ -85,7 +85,7 @@ def test_kv_cache_model_load_and_run(

85

86 # `LLM.apply_model` requires pickling a function.

87 monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

88 - with vllm_runner(model_id, kv_cache_dtype="fp8") as llm:

88 + with vllm_runner(model_id, kv_cache_dtype="fp8", enforce_eager=True) as llm:

89

90 def check_model(model):

91 attn = model.model.layers[0].self_attn.attn

@@ -112,7 +112,7 @@ def test_kv_cache_model_load_and_run(

112

113 # note: this does not test accuracy, just that we can run through

114 # see lm-eval tests for accuracy

115 - outputs = llm.generate_greedy(["Hello my name is"], max_tokens=10)

115 + outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)

116 print(outputs[0][1])

117

118

@@ -142,7 +142,10 @@ def test_load_fp16_model(

142 monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")

143

144 with vllm_runner(

145 - "facebook/opt-125m", quantization="fp8", kv_cache_dtype=kv_cache_dtype

145 + "facebook/opt-125m",

146 + quantization="fp8",

147 + enforce_eager=True,

148 + kv_cache_dtype=kv_cache_dtype,

149 ) as llm:

150

151 def check_model(model):

tests/quantization/test_ipex_quant.py

+2 -2

@@ -26,7 +26,7 @@ DTYPE = ["bfloat16"]

26 @pytest.mark.parametrize("model", MODELS)

27 @pytest.mark.parametrize("dtype", DTYPE)

28 def test_ipex_quant(vllm_runner, model, dtype):

29 - with vllm_runner(model, dtype=dtype) as llm:

30 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

29 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as llm:

30 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

31 assert output

32 print(output)

tests/quantization/test_lm_head.py

+1 -1

@@ -49,4 +49,4 @@ def test_lm_head(

49

50 vllm_model.apply_model(check_model)

51

52 - print(vllm_model.generate_greedy(["Hello my name is"], max_tokens=10)[0][1])

52 + print(vllm_model.generate_greedy(["Hello my name is"], max_tokens=4)[0][1])

tests/quantization/test_modelopt.py

+1 -1

@@ -88,6 +88,6 @@ def test_modelopt_fp8_checkpoint_setup(vllm_runner):

88 llm.apply_model(check_model)

89

90 # Run a simple generation test to ensure the model works

91 - output = llm.generate_greedy(["Hello my name is"], max_tokens=20)

91 + output = llm.generate_greedy(["Hello my name is"], max_tokens=4)

92 assert output

93 print(f"ModelOpt FP8 output: {output}")

tests/quantization/test_ptpc_fp8.py

+2 -1

@@ -38,6 +38,7 @@ def test_ptpc_fp8_rocm(vllm_runner, dtype: str, kv_cache_dtype: str) -> None:

38 "facebook/opt-125m",

39 dtype=dtype,

40 quantization="ptpc_fp8",

41 + enforce_eager=True,

42 kv_cache_dtype=kv_cache_dtype,

43 )

44 except AssertionError as e:

@@ -65,5 +66,5 @@ def test_ptpc_fp8_rocm(vllm_runner, dtype: str, kv_cache_dtype: str) -> None:

66

67 llm.apply_model(check_model)

68

68 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

69 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

70 assert output

tests/quantization/test_register_quantization_config.py

+3 -3

@@ -23,8 +23,8 @@ from vllm.model_executor.layers.quantization import (

23 get_quantization_config,

24 register_quantization_config,

25 )

26 -from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501

27 - QuantizationConfig,

26 +from vllm.model_executor.layers.quantization.base_config import (

27 + QuantizationConfig, # noqa: E501

28 )

29

30

@@ -142,5 +142,5 @@ def test_custom_quant(vllm_runner, model, monkeypatch):

142

143 llm.apply_model(check_model)

144

145 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

145 + output = llm.generate_greedy("Hello my name is", max_tokens=1)

146 assert output

tests/quantization/test_torchao.py

+1 -1

@@ -392,7 +392,7 @@ def test_opt_125m_int4wo_model_running_preshuffled_kernel_online_quant(

392 assert not has_int4_preshuffled_tensor

393

394 assert weight_attrs == [False, 1, 0, True]

395 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

395 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

396

397 assert output

398

[CI Sprint] Quantization CI Cleanup (#24130)