A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI Failure] Fix torchao dep failure for Quantization Test (#26824)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Oct 14, 2025 at 19:41 UTC
7e0ef4084affa9de84904ba7726c46f53f4f6379
4 files changed
+8
-4
.buildkite/test-amd.yaml
+2
-1
@@ -603,7 +603,8 @@ steps:
603
# since torchao nightly is only compatible with torch nightly currently
604
# https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now
605
# we can only upgrade after this is resolved
606
- - pip install --pre torchao==0.13.0.dev20250814 --index-url https://download.pytorch.org/whl/nightly/cu128
606
+ # TODO(jerryzh168): resolve the above comment
607
+ - uv pip install --system torchao==0.13.0
608
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/
609
610
- label: LM Eval Small Models # 53min
.buildkite/test-pipeline.yaml
+2
-1
@@ -527,7 +527,8 @@ steps:
527
# since torchao nightly is only compatible with torch nightly currently
528
# https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now
529
# we can only upgrade after this is resolved
530
- - pip install --pre torchao==0.13.0.dev20250814 --index-url https://download.pytorch.org/whl/nightly/cu128
530
+ # TODO(jerryzh168): resolve the above comment
531
+ - uv pip install --system torchao==0.13.0
532
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/
533
534
- label: LM Eval Small Models # 53min
tests/quantization/test_compressed_tensors.py
+2
-1
@@ -697,7 +697,8 @@ def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):
697
@pytest.mark.parametrize(
698
"args",
699
[
700
- ("nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4A16", CompressedTensorsW4A16Fp4),
700
+ # TODO: Enable once model is available again
701
+ # ("nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4A16", CompressedTensorsW4A16Fp4),
702
("nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4", CompressedTensorsW4A4Fp4),
703
],
704
)
vllm/model_executor/layers/quantization/rtn.py
+2
-1
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe.config import (
15
FusedMoEConfig,
16
FusedMoEQuantConfig,
17
)
18
+from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe
19
from vllm.model_executor.layers.fused_moe.layer import FusedMoE, FusedMoEMethodBase
20
from vllm.model_executor.layers.linear import (
21
LinearBase,
@@ -396,7 +397,7 @@ class RTNMoEMethod(FusedMoEMethodBase):
397
indices_type=self.topk_indices_dtype,
398
)
399
399
- return torch.ops.vllm.fused_marlin_moe(
400
+ return fused_marlin_moe(
401
x,
402
layer.w13_weight,
403
layer.w2_weight,