@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01c977e96d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[CI] Prune Quantization Tests and skip compilation (#27038)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Oct 16, 2025 at 17:26 UTC 01c977e96d7c1d8e483efb8eecf84d4cee2d9ce9

9 files changed +62 -134

tests/quantization/test_auto_round.py

+1 -1

@@ -26,7 +26,7 @@ MODELS = [

26 )

27 @pytest.mark.parametrize("model", MODELS)

28 def test_auto_round(vllm_runner, model):

29 - with vllm_runner(model) as llm:

29 + with vllm_runner(model, enforce_eager=True) as llm:

30 output = llm.generate_greedy(["The capital of France is"], max_tokens=8)

31 assert output

32 print(f"{output[0][1]}")

tests/quantization/test_compressed_tensors.py

+29 -76

@@ -66,13 +66,6 @@ def enable_pickle(monkeypatch):

66 2560,

67 True,

68 ),

69 - (

70 - "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",

71 - "channel",

72 - QuantizationType.INT,

73 - 2560,

74 - True,

75 - ),

69 (

70 "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",

71 "tensor",

@@ -138,7 +131,7 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):

131

132 llm.apply_model(check_model)

133

141 - output = llm.generate_greedy(["Hello my name is"], max_tokens=20)

134 + output = llm.generate_greedy(["Hello my name is"], max_tokens=4)

135 assert output

136

137

@@ -146,12 +139,9 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):

139 "model_path",

140 [

141 "neuralmagic/Llama-3.2-1B-quantized.w8a8",

149 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",

150 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",

151 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",

142 ],

143 )

154 -@pytest.mark.parametrize("max_tokens", [32])

144 +@pytest.mark.parametrize("max_tokens", [8])

145 @pytest.mark.parametrize("num_logprobs", [10])

146 @pytest.mark.parametrize(

147 "use_aiter", [True, False] if current_platform.is_rocm() else [False]

@@ -211,7 +201,7 @@ def test_compressed_tensors_w8a8_logprobs(

201 def test_compressed_tensors_no_enforce_eager(vllm_runner):

202 model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"

203 with vllm_runner(model_path) as llm:

214 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

204 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

205 assert output

206

207

@@ -219,15 +209,10 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):

209 "model_args",

210 [

211 ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),

222 - ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),

212 (

213 "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",

214 "channel",

215 ),

227 - (

228 - "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",

229 - "channel",

230 - ),

216 ],

217 )

218 @pytest.mark.parametrize(

@@ -253,7 +238,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(

238 # this will enable VLLM_ROCM_USE_AITER_LINEAR

239 monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")

240

256 - with vllm_runner(model_path, dtype=torch.float16) as llm:

241 + with vllm_runner(model_path, enforce_eager=True, dtype=torch.float16) as llm:

242

243 def check_model(model):

244 layer = model.model.layers[0]

@@ -268,7 +253,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(

253

254 llm.apply_model(check_model)

255

271 - output = llm.generate_greedy(["Hello my name is"], max_tokens=20)

256 + output = llm.generate_greedy(["Hello my name is"], max_tokens=4)

257 assert output

258

259

@@ -283,38 +268,6 @@ def test_compressed_tensors_w8a8_dynamic_per_token(

268 True,

269 False,

270 ),

286 - (

287 - "nm-testing/tinyllama-oneshot-w4a16-group128-v2",

288 - "group",

289 - 128,

290 - 8,

291 - True,

292 - False,

293 - ),

294 - (

295 - "nm-testing/tinyllama-oneshot-w8a16-per-channel",

296 - "channel",

297 - None,

298 - 4,

299 - True,

300 - False,

301 - ),

302 - (

303 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-awq-group128-asym256",

304 - "group",

305 - 128,

306 - 8,

307 - False,

308 - False,

309 - ),

310 - (

311 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-Channel",

312 - "channel",

313 - None,

314 - 8,

315 - False,

316 - False,

317 - ),

271 (

272 "nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",

273 "group",

@@ -330,7 +283,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(

283 )

284 def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):

285 model, strategy, group, pack_factor, symmetric, has_g_idx = wNa16_args

333 - with vllm_runner(model) as llm:

286 + with vllm_runner(model, enforce_eager=True) as llm:

287

288 def check_model(model):

289 layer = model.model.layers[0]

@@ -348,7 +301,7 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):

301

302 llm.apply_model(check_model)

303

351 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

304 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

305 assert output

306

307

@@ -357,7 +310,7 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):

310 )

311 def test_compressed_tensors_w4a16_marlin24(vllm_runner):

312 model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"

360 - with vllm_runner(model_path) as llm:

313 + with vllm_runner(model_path, enforce_eager=True) as llm:

314

315 def check_model(model):

316 layer = model.model.layers[0]

@@ -370,13 +323,13 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):

323

324 llm.apply_model(check_model)

325

373 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

326 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

327 assert output

328

329

330 def test_compressed_tensors_fp8(vllm_runner):

331 model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"

379 - with vllm_runner(model_path) as llm:

332 + with vllm_runner(model_path, enforce_eager=True) as llm:

333

334 def check_model(model):

335 layer = model.model.layers[0]

@@ -399,7 +352,7 @@ def test_compressed_tensors_fp8(vllm_runner):

352

353 llm.apply_model(check_model)

354

402 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

355 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

356 assert output

357

358

@@ -412,8 +365,8 @@ def test_compressed_tensors_fp8(vllm_runner):

365 )

366 def test_compressed_tensors_kv_cache(vllm_runner):

367 model_path = "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"

415 - with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:

416 - output = llm.generate_greedy("Hello world!", max_tokens=20)

368 + with vllm_runner(model_path, enforce_eager=True, kv_cache_dtype="fp8") as llm:

369 + output = llm.generate_greedy("Hello world!", max_tokens=4)

370 assert output

371

372

@@ -465,7 +418,7 @@ def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy, format="d

418 )

419 def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):

420 model, weight_strategy, input_strategy = args_2of4

468 - with vllm_runner(model) as llm:

421 + with vllm_runner(model, enforce_eager=True) as llm:

422

423 def check_model(model):

424 layer = model.model.layers[0]

@@ -476,7 +429,7 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):

429

430 llm.apply_model(check_model)

431

479 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

432 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

433 print(output)

434 assert output

435

@@ -512,7 +465,7 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):

465 )

466 def test_compressed_tensors_2of4_quant_fp8_compressed(vllm_runner, args_2of4):

467 model, weight_strategy, input_strategy = args_2of4

515 - with vllm_runner(model) as llm:

468 + with vllm_runner(model, enforce_eager=True) as llm:

469

470 def check_model(model):

471 layer = model.model.layers[0]

@@ -528,7 +481,7 @@ def test_compressed_tensors_2of4_quant_fp8_compressed(vllm_runner, args_2of4):

481

482 llm.apply_model(check_model)

483

531 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

484 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

485 print(output)

486 assert output

487

@@ -564,7 +517,7 @@ def test_compressed_tensors_2of4_quant_fp8_compressed(vllm_runner, args_2of4):

517 )

518 def test_compressed_tensors_2of4_quant_int8_compressed(vllm_runner, args_2of4):

519 model, weight_strategy, input_strategy = args_2of4

567 - with vllm_runner(model) as llm:

520 + with vllm_runner(model, enforce_eager=True) as llm:

521

522 def check_model(model):

523 layer = model.model.layers[0]

@@ -580,7 +533,7 @@ def test_compressed_tensors_2of4_quant_int8_compressed(vllm_runner, args_2of4):

533

534 llm.apply_model(check_model)

535

583 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

536 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

537 print(output)

538 assert output

539

@@ -611,7 +564,7 @@ def test_compressed_tensors_2of4_quant_int8_compressed(vllm_runner, args_2of4):

564 )

565 def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):

566 model, weight_strategy, input_strategy = args_2of4

614 - with vllm_runner(model) as llm:

567 + with vllm_runner(model, enforce_eager=True) as llm:

568

569 def check_model(model):

570 layer = model.model.layers[0]

@@ -622,7 +575,7 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):

575

576 llm.apply_model(check_model)

577

625 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

578 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

579 print(output)

580 assert output

581

@@ -637,7 +590,7 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):

590 )

591 def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):

592 model = args_2of4

640 - with vllm_runner(model) as llm:

593 + with vllm_runner(model, enforce_eager=True) as llm:

594

595 def check_model(model):

596 layer = model.model.layers[0]

@@ -656,7 +609,7 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):

609

610 llm.apply_model(check_model)

611

659 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

612 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

613 print(output)

614 assert output

615

@@ -670,7 +623,7 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):

623 )

624 def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):

625 model = args_2of4

673 - with vllm_runner(model) as llm:

626 + with vllm_runner(model, enforce_eager=True) as llm:

627

628 def check_model(model):

629 layer = model.model.layers[0]

@@ -689,7 +642,7 @@ def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):

642

643 llm.apply_model(check_model)

644

692 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

645 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

646 print(output)

647 assert output

648

@@ -723,7 +676,7 @@ def test_compressed_tensors_nvfp4(vllm_runner, args):

676 assert qkv_proj.scheme.group_size == 16

677

678 llm.apply_model(check_model)

726 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

679 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

680 print(output)

681 assert output

682

@@ -758,7 +711,7 @@ def test_compressed_tensors_w4a8_fp8(vllm_runner, args):

711 assert proj.scheme.group_size == 128

712

713 llm.apply_model(check_model)

761 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

714 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

715 print(output)

716 assert output

717

@@ -792,7 +745,7 @@ def test_compressed_tensors_transforms_perplexity(

745

746 def test_compressed_tensors_fp8_block_enabled(vllm_runner):

747 model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"

795 - with vllm_runner(model_path) as llm:

748 + with vllm_runner(model_path, enforce_eager=True) as llm:

749 fp8_dtype = current_platform.fp8_dtype()

750

751 def check_model(model):

@@ -816,5 +769,5 @@ def test_compressed_tensors_fp8_block_enabled(vllm_runner):

769

770 llm.apply_model(check_model)

771

819 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

772 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

773 assert output

tests/quantization/test_cpu_offload.py

-35

@@ -16,13 +16,6 @@ from ..utils import compare_two_settings

16 reason="fp8 is not supported on this GPU type.",

17 )

18 def test_cpu_offload_fp8():

19 - # Test quantization of an unquantized checkpoint

20 - compare_two_settings(

21 - "meta-llama/Llama-3.2-1B-Instruct",

22 - ["--quantization", "fp8"],

23 - ["--quantization", "fp8", "--cpu-offload-gb", "1"],

24 - max_wait_seconds=480,

25 - )

19 # Test loading a quantized checkpoint

20 compare_two_settings(

21 "neuralmagic/Qwen2-1.5B-Instruct-FP8",

@@ -46,13 +39,6 @@ def test_cpu_offload_gptq(monkeypatch):

39 ["--cpu-offload-gb", "1"],

40 max_wait_seconds=480,

41 )

49 - # Test GPTQ

50 - compare_two_settings(

51 - "Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",

52 - ["--quantization", "gptq"],

53 - ["--quantization", "gptq", "--cpu-offload-gb", "1"],

54 - max_wait_seconds=480,

55 - )

44 @pytest.mark.skipif(

@@ -69,13 +55,6 @@ def test_cpu_offload_awq(monkeypatch):

55 ["--cpu-offload-gb", "1"],

56 max_wait_seconds=480,

57 )

72 - # Test AWQ

73 - compare_two_settings(

74 - "Qwen/Qwen2-1.5B-Instruct-AWQ",

75 - ["--quantization", "awq"],

76 - ["--quantization", "awq", "--cpu-offload-gb", "1"],

77 - max_wait_seconds=480,

78 - )

60 @pytest.mark.skipif(

@@ -92,17 +71,3 @@ def test_cpu_offload_compressed_tensors(monkeypatch):

71 ["--cpu-offload-gb", "1"],

72 max_wait_seconds=480,

73 )

95 - # Test w4a16_marlin24

96 - compare_two_settings(

97 - "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",

98 - [],

99 - ["--cpu-offload-gb", "1"],

100 - max_wait_seconds=480,

101 - )

102 - # Test w8a8

103 - compare_two_settings(

104 - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",

105 - [],

106 - ["--cpu-offload-gb", "1"],

107 - max_wait_seconds=480,

108 - )

tests/quantization/test_fp8.py

-3

@@ -18,7 +18,6 @@ from vllm.platforms import current_platform

19 MODELS = [

20 "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",

21 - "nm-testing/Phi-3-mini-128k-instruct-FP8",

21 "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",

22 ]

@@ -49,8 +48,6 @@ def test_model_load_and_run(

50 KV_CACHE_MODELS = [

52 - # Deprecated AutoFP8 format using .kv_scale

53 - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",

51 # AutoFP8 format using separate .k_scale and .v_scale

52 "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",

53 ]

tests/quantization/test_gptq_dynamic.py

+3 -1

@@ -40,7 +40,9 @@ def test_gptq_with_dynamic(

40 GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)

41 )

43 - with vllm_runner(model_id, dtype=torch.float16, max_model_len=2048) as llm:

43 + with vllm_runner(

44 + model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True

45 + ) as llm:

47 def check_model(model):

48 for name, submodule in model.named_modules():

tests/quantization/test_lm_head.py

+3 -1

@@ -31,7 +31,9 @@ def test_lm_head(

31 ) -> None:

32 # `LLM.apply_model` requires pickling a function.

33 monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

34 - with vllm_runner(model_id, dtype=torch.float16, max_model_len=2048) as vllm_model:

34 + with vllm_runner(

35 + model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True

36 + ) as vllm_model:

38 def check_model(model):

39 lm_head_layer = model.lm_head

tests/quantization/test_quark.py

+9 -6

@@ -56,7 +56,10 @@ def enable_pickle(monkeypatch):

56 def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):

57 model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"

58 with vllm_runner(

59 - model_path, kv_cache_dtype=kv_cache_dtype, tensor_parallel_size=tp

59 + model_path,

60 + enforce_eager=True,

61 + kv_cache_dtype=kv_cache_dtype,

62 + tensor_parallel_size=tp,

63 ) as llm:

65 def check_model(model):

@@ -74,14 +77,14 @@ def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):

78 llm.apply_model(check_model)

77 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

80 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

81 assert output

84 @pytest.mark.parametrize("tp", [1])

85 def test_quark_fp8_w_per_channel_a_per_token(vllm_runner, tp):

86 model_path = "amd/Qwen2.5-1.5B-Instruct-ptpc-Quark-ts"

84 - with vllm_runner(model_path, tensor_parallel_size=tp) as llm:

87 + with vllm_runner(model_path, enforce_eager=True, tensor_parallel_size=tp) as llm:

89 def check_model(model):

90 layer = model.model.layers[0]

@@ -98,14 +101,14 @@ def test_quark_fp8_w_per_channel_a_per_token(vllm_runner, tp):

101

102 llm.apply_model(check_model)

103

101 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

104 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

105 assert output

106

107

108 @pytest.mark.parametrize("tp", [1])

109 def test_quark_int8_w_per_tensor_a_per_tensor(vllm_runner, tp):

110 model_path = "amd/Llama-3.1-8B-Instruct-w-int8-a-int8-sym-test"

108 - with vllm_runner(model_path, tensor_parallel_size=tp) as llm:

111 + with vllm_runner(model_path, enforce_eager=True, tensor_parallel_size=tp) as llm:

112

113 def check_model(model):

114 layer = model.model.layers[0]

@@ -117,7 +120,7 @@ def test_quark_int8_w_per_tensor_a_per_tensor(vllm_runner, tp):

120

121 llm.apply_model(check_model)

122

120 - output = llm.generate_greedy("Hello my name is", max_tokens=20)

123 + output = llm.generate_greedy("Hello my name is", max_tokens=4)

124 assert output

125

126

tests/quantization/test_rtn.py

+3 -2

@@ -10,7 +10,6 @@ import pytest

10 from tests.quantization.utils import is_quant_method_supported

12 MODELS = [

13 - "microsoft/Phi-3-mini-4k-instruct", # dense model

13 "ai21labs/Jamba-tiny-dev", # MoE model

14 ]

@@ -30,5 +29,7 @@ def test_model_rtn_startup(

29 dtype: str,

30 max_tokens: int,

31 ) -> None:

33 - with vllm_runner(model, dtype=dtype, quantization="rtn") as vllm_model:

32 + with vllm_runner(

33 + model, enforce_eager=True, dtype=dtype, quantization="rtn"

34 + ) as vllm_model:

35 vllm_model.generate_greedy(example_prompts, max_tokens)

tests/quantization/test_torchao.py

+14 -9

@@ -19,7 +19,7 @@ def test_pre_quantized_model(vllm_runner):

19 dtype="bfloat16",

20 enforce_eager=True,

21 ) as llm:

22 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

22 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

23 assert output

@@ -39,8 +39,9 @@ def test_opt_125m_int8wo_model_loading_with_params(vllm_runner, pt_load_map_loca

39 quantization="torchao",

40 dtype="bfloat16",

41 pt_load_map_location=pt_load_map_location,

42 + enforce_eager=True,

43 ) as llm:

43 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

44 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

46 assert output

@@ -54,8 +55,9 @@ def test_opt_125m_int4wo_model_per_module_quant(vllm_runner):

55 quantization="torchao",

56 dtype="bfloat16",

57 pt_load_map_location="cuda:0",

58 + enforce_eager=True,

59 ) as llm:

58 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

60 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

62 assert output

@@ -69,8 +71,9 @@ def test_qwenvl_int8wo_model_loading_with_params(vllm_runner):

71 quantization="torchao",

72 dtype="bfloat16",

73 pt_load_map_location="cuda:0",

74 + enforce_eager=True,

75 ) as llm:

73 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

76 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

78 assert output

@@ -90,7 +93,7 @@ def test_opt_125m_awq_int4wo_model_loading_with_params(vllm_runner):

93 dtype="bfloat16",

94 pt_load_map_location="cuda:0",

95 ) as llm:

93 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

96 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

98 assert output

@@ -122,8 +125,9 @@ def test_on_the_fly_quant_config_dict_json(vllm_runner):

125 pt_load_map_location="cuda:0",

126 quantization="torchao",

127 hf_overrides=hf_overrides,

128 + enforce_eager=True,

129 ) as llm:

126 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

130 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

131

132 assert output

133

@@ -156,8 +160,9 @@ def test_on_the_fly_quant_config_file(vllm_runner):

160 pt_load_map_location="cuda:0",

161 quantization="torchao",

162 hf_overrides=hf_overrides,

163 + enforce_eager=True,

164 ) as llm:

160 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

165 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

166

167 assert output

168

@@ -228,7 +233,7 @@ def test_opt_125m_float8_weight_only_safetensors_model_loading_with_params(vllm_

233 "torchao-testing/opt-125m-Float8WeightOnlyConfig-v2-0.14.0.dev-safetensors"

234 )

235 with vllm_runner(model_name=model_name, dtype="bfloat16") as llm:

231 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

236 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

237

238 assert output

239

@@ -245,7 +250,7 @@ def test_opt_125m_module_fqn_to_config_regex_model(vllm_runner):

250 with vllm_runner(

251 model_name=model_name, dtype="bfloat16", pt_load_map_location="cuda:0"

252 ) as llm:

248 - output = llm.generate_greedy(["The capital of France is"], max_tokens=32)

253 + output = llm.generate_greedy(["The capital of France is"], max_tokens=4)

254

255 assert output

256