A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Fix cpu offload testing for gptq/awq/ct (#15648)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Mar 28, 2025 at 10:29 UTC
47e9038d2386d31b8493ac995094bdc1aec710ce
2 files changed
+42
-3
tests/quantization/test_cpu_offload.py
+9
-3
@@ -33,7 +33,9 @@ def test_cpu_offload_fp8():
33
34
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
35
reason="gptq_marlin is not supported on this GPU type.")
36
-def test_cpu_offload_gptq():
36
+def test_cpu_offload_gptq(monkeypatch):
37
+ # This quant method is sensitive to dummy weights, so we force real weights
38
+ monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto')
39
# Test GPTQ Marlin
40
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
41
["--cpu-offload-gb", "1"],
@@ -47,7 +49,9 @@ def test_cpu_offload_gptq():
49
50
@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
51
reason="awq_marlin is not supported on this GPU type.")
50
-def test_cpu_offload_awq():
52
+def test_cpu_offload_awq(monkeypatch):
53
+ # This quant method is sensitive to dummy weights, so we force real weights
54
+ monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto')
55
# Test AWQ Marlin
56
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
57
["--cpu-offload-gb", "1"],
@@ -61,7 +65,9 @@ def test_cpu_offload_awq():
65
66
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
67
reason="gptq_marlin is not supported on this GPU type.")
64
-def test_cpu_offload_compressed_tensors():
68
+def test_cpu_offload_compressed_tensors(monkeypatch):
69
+ # This quant method is sensitive to dummy weights, so we force real weights
70
+ monkeypatch.setenv('VLLM_TEST_FORCE_LOAD_FORMAT', 'auto')
71
# Test wNa16
72
compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
73
["--cpu-offload-gb", "1"],
tests/utils.py
+33
@@ -317,6 +317,37 @@ def _test_completion_close(
317
return results
318
319
320
+def _test_chat(
321
+ client: openai.OpenAI,
322
+ model: str,
323
+ prompt: str,
324
+):
325
+ results = []
326
+
327
+ messages = [{
328
+ "role": "user",
329
+ "content": [{
330
+ "type": "text",
331
+ "text": prompt
332
+ }]
333
+ }]
334
+
335
+ # test with text prompt
336
+ chat_response = client.chat.completions.create(model=model,
337
+ messages=messages,
338
+ max_tokens=5,
339
+ temperature=0.0)
340
+
341
+ results.append({
342
+ "test": "completion_close",
343
+ "text": chat_response.choices[0].message.content,
344
+ "finish_reason": chat_response.choices[0].finish_reason,
345
+ "usage": chat_response.usage,
346
+ })
347
+
348
+ return results
349
+
350
+
351
def _test_embeddings(
352
client: openai.OpenAI,
353
model: str,
@@ -512,6 +543,8 @@ def compare_all_settings(model: str,
543
results += _test_completion(client, model, prompt, token_ids)
544
elif method == "generate_close":
545
results += _test_completion_close(client, model, prompt)
546
+ elif method == "generate_chat":
547
+ results += _test_chat(client, model, prompt)
548
elif method == "generate_with_image":
549
results += _test_image_text(
550
client, model,