A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] Fullgraph and opcheck tests (#8479)
bnellnm committed
Sep 25, 2024 at 10:35 UTC
300da09177477d0a4d2b55790addefd971f52ae0
26 files changed
+744
-116
.buildkite/test-pipeline.yaml
+17
-2
@@ -70,7 +70,7 @@ steps:
70
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
71
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
72
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
73
-
73
+
74
- label: Core Test # 10min
75
mirror_hardwares: [amd]
76
fast_check: true
@@ -210,6 +210,21 @@ steps:
210
command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
211
parallelism: 4
212
213
+- label: "PyTorch Fullgraph Smoke Test"
214
+ fast_check: true
215
+ source_file_dependencies:
216
+ - vllm/
217
+ - tests/compile
218
+ commands:
219
+ - pytest -v -s compile/test_full_graph_smoke.py
220
+
221
+- label: "PyTorch Fullgraph Test"
222
+ source_file_dependencies:
223
+ - vllm/
224
+ - tests/compile
225
+ commands:
226
+ - pytest -v -s compile/test_full_graph.py
227
+
228
- label: Kernels Test %N # 30min each
229
mirror_hardwares: [amd]
230
source_file_dependencies:
@@ -355,7 +370,7 @@ steps:
370
- tests/distributed/
371
- vllm/compilation
372
commands:
358
- - pytest -v -s ./compile/test_full_graph.py
373
+ - pytest -v -s ./compile/test_full_graph_multi_gpu.py
374
- pytest -v -s ./compile/test_wrapper.py
375
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
376
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+1
-1
@@ -586,7 +586,7 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
586
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), "selective_scan_fwd", [&] {
587
selective_scan_fwd_cuda<input_t, weight_t>(params, stream);
588
});
589
- std::vector<at::Tensor> result = {out, x.value()};
589
+ std::vector<at::Tensor> result = {out};
590
if (has_z) { result.push_back(out_z); }
591
return result;
592
}
csrc/torch_bindings.cpp
+2
-2
@@ -275,7 +275,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
275
"Tensor! A, Tensor! B, Tensor! C,"
276
"Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
277
"bool delta_softplus,"
278
- "Tensor? index_, Tensor(a! -> *)? x) -> Tensor(a)[]");
278
+ "Tensor? index_, Tensor!? x) -> Tensor[]");
279
ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
280
281
ops.def(
@@ -292,7 +292,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
292
"Tensor? bias_,"
293
"Tensor? seq_idx_,"
294
"Tensor? initial_states_,"
295
- "Tensor? final_states_out_,"
295
+ "Tensor!? final_states_out_,"
296
"bool silu_activation) -> Tensor");
297
ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
298
#endif
tests/compile/test_full_graph.py
+8
-37
@@ -1,42 +1,13 @@
1
-import os
2
-
1
import pytest
2
5
-from vllm.utils import cuda_device_count_stateless
6
-
7
-from ..utils import fork_new_process_for_each_test
8
-
9
-
10
-@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
11
-@pytest.mark.parametrize("tp_size", [1, 2])
12
-@fork_new_process_for_each_test
13
-def test_full_graph(model, tp_size):
14
-
15
- # Skip the test if there are not enough CUDA devices.
16
- if cuda_device_count_stateless() < tp_size:
17
- pytest.skip("Not enough CUDA devices for the test.")
18
-
19
- # make sure these models can be captured in full graph mode
20
- if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
21
- os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
3
+from vllm.compilation.backends import vllm_backend
4
23
- from vllm import LLM, SamplingParams
24
- prompts = [
25
- "Hello, my name is",
26
- "The president of the United States is",
27
- "The capital of France is",
28
- "The future of AI is",
29
- ]
30
- sampling_params = SamplingParams(temperature=0)
31
- llm = LLM(model=model,
32
- enforce_eager=True,
33
- tensor_parallel_size=tp_size,
34
- disable_custom_all_reduce=True)
5
+from .utils import TEST_MODELS, check_full_graph_support
6
36
- outputs = llm.generate(prompts, sampling_params)
7
38
- # Print the outputs.
39
- for output in outputs:
40
- prompt = output.prompt
41
- generated_text = output.outputs[0].text
42
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
8
+@pytest.mark.parametrize("model_info", TEST_MODELS)
9
+@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10
+def test_full_graph(model_info, backend):
11
+ model = model_info[0]
12
+ model_kwargs = model_info[1]
13
+ check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/test_full_graph_multi_gpu.py
new
+22
@@ -0,0 +1,22 @@
1
+import pytest
2
+
3
+from vllm.compilation.backends import vllm_backend
4
+from vllm.utils import cuda_device_count_stateless
5
+
6
+from ..utils import fork_new_process_for_each_test
7
+from .utils import TEST_MODELS_SMOKE, check_full_graph_support
8
+
9
+
10
+@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
11
+@pytest.mark.parametrize("tp_size", [2])
12
+@pytest.mark.parametrize("backend", ["eager", vllm_backend])
13
+@fork_new_process_for_each_test
14
+def test_full_graph_multi_gpu(model_info, tp_size, backend):
15
+ model = model_info[0]
16
+ model_kwargs = model_info[1]
17
+
18
+ # Skip the test if there are not enough CUDA devices.
19
+ if cuda_device_count_stateless() < tp_size:
20
+ pytest.skip("Not enough CUDA devices for the test.")
21
+
22
+ check_full_graph_support(model, model_kwargs, backend, tp_size=tp_size)
tests/compile/test_full_graph_smoke.py
new
+13
@@ -0,0 +1,13 @@
1
+import pytest
2
+
3
+from vllm.compilation.backends import vllm_backend
4
+
5
+from .utils import TEST_MODELS_SMOKE, check_full_graph_support
6
+
7
+
8
+@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
9
+@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10
+def test_full_graph(model_info, backend):
11
+ model = model_info[0]
12
+ model_kwargs = model_info[1]
13
+ check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/utils.py
new
+104
@@ -0,0 +1,104 @@
1
+import os
2
+
3
+import torch
4
+
5
+from tests.quantization.utils import is_quant_method_supported
6
+from vllm import LLM, SamplingParams
7
+from vllm.plugins import set_torch_compile_backend
8
+from vllm.utils import is_hip
9
+
10
+TEST_MODELS_SMOKE = [
11
+ ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
12
+ "quantization": "compressed-tensors"
13
+ }),
14
+ ("meta-llama/Meta-Llama-3-8B", {}),
15
+]
16
+
17
+TEST_MODELS = [
18
+ ("facebook/opt-125m", {}),
19
+ ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
20
+ "dtype": torch.float16,
21
+ "quantization": "compressed-tensors"
22
+ }),
23
+ ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
24
+ "dtype": torch.float16,
25
+ "quantization": "fp8"
26
+ }),
27
+ ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
28
+ "quantization": "compressed-tensors"
29
+ }),
30
+ ("meta-llama/Meta-Llama-3-8B", {}),
31
+]
32
+
33
+# TODO: enable in pytorch 2.5
34
+if False and is_quant_method_supported("aqlm"): # noqa: SIM223
35
+ TEST_MODELS.append(("ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf", {
36
+ "quantization": "aqlm"
37
+ }))
38
+
39
+# TODO: enable in pytorch 2.5
40
+if False and is_quant_method_supported("gguf"): # noqa: SIM223
41
+ TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {
42
+ "quantization": "gguf"
43
+ }))
44
+
45
+if is_quant_method_supported("gptq"):
46
+ TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {
47
+ "quantization": "gptq"
48
+ }))
49
+
50
+if is_quant_method_supported("gptq_marlin"):
51
+ TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", {
52
+ "quantization": "gptq_marlin"
53
+ }))
54
+
55
+if is_quant_method_supported("gptq_marlin_24"):
56
+ TEST_MODELS.append(("alexm-nm/tinyllama-24-marlin24-4bit-g128", {
57
+ "quantization": "gptq_marlin_24"
58
+ }))
59
+
60
+if is_quant_method_supported("marlin"):
61
+ TEST_MODELS.append(("robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-marlin", {
62
+ "quantization": "marlin"
63
+ }))
64
+
65
+if not is_hip() and is_quant_method_supported("awq"):
66
+ TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {
67
+ "quantization": "AWQ"
68
+ }))
69
+
70
+
71
+def check_full_graph_support(model, model_kwargs, backend, tp_size=1):
72
+ # make sure these models can be captured in full graph mode
73
+ if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
74
+ os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
75
+ os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
76
+
77
+ # Inductor doesn't support fp8/gptq_marlin_24 yet.
78
+ quantization = model_kwargs.get("quantization")
79
+ if (quantization == "fp8" or quantization == "gptq_marlin"
80
+ or quantization == "gptq_marlin_24") and backend != "eager":
81
+ return
82
+
83
+ set_torch_compile_backend(backend)
84
+
85
+ prompts = [
86
+ "Hello, my name is",
87
+ "The president of the United States is",
88
+ "The capital of France is",
89
+ "The future of AI is",
90
+ ]
91
+ sampling_params = SamplingParams(temperature=0)
92
+ llm = LLM(model=model,
93
+ enforce_eager=True,
94
+ tensor_parallel_size=tp_size,
95
+ disable_custom_all_reduce=True,
96
+ **model_kwargs)
97
+
98
+ outputs = llm.generate(prompts, sampling_params)
99
+
100
+ # Print the outputs.
101
+ for output in outputs:
102
+ prompt = output.prompt
103
+ generated_text = output.outputs[0].text
104
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/conftest.py
+6
@@ -169,6 +169,12 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
169
cleanup()
170
171
172
+@pytest.fixture(autouse=True)
173
+def dynamo_reset():
174
+ yield
175
+ torch._dynamo.reset()
176
+
177
+
178
@pytest.fixture
179
def example_prompts() -> List[str]:
180
prompts = []
tests/kernels/test_aqlm.py
new
+37
@@ -0,0 +1,37 @@
1
+import torch
2
+
3
+from tests.kernels.utils import opcheck
4
+from vllm import _custom_ops as ops # noqa: F401
5
+
6
+
7
+def test_aqlm_dequant_opcheck():
8
+ codes = torch.randint(-32768,
9
+ 32767, (22016, 512, 1),
10
+ device='cuda',
11
+ dtype=torch.int16)
12
+ codebooks = torch.rand((2, 65536, 1, 8),
13
+ device='cuda',
14
+ dtype=torch.float16)
15
+ codebook_partition_sizes = [11008, 11008]
16
+
17
+ opcheck(torch.ops._C.aqlm_dequant,
18
+ (codes, codebooks, codebook_partition_sizes))
19
+
20
+
21
+def test_aqlm_gemm_opcheck():
22
+ input = torch.rand((4, 4096), device='cuda', dtype=torch.float16)
23
+ codes = torch.randint(-32768,
24
+ 32767, (12288, 512, 1),
25
+ device='cuda',
26
+ dtype=torch.int16)
27
+ codebooks = torch.rand((3, 65536, 1, 8),
28
+ device='cuda',
29
+ dtype=torch.float16)
30
+ scales = torch.rand((12288, 1, 1, 1), device='cuda', dtype=torch.float16)
31
+ codebook_partition_sizes = [4096, 4096, 4096]
32
+ bias = None
33
+
34
+ opcheck(torch.ops._C.aqlm_gemm,
35
+ (input, codes, codebooks, scales, codebook_partition_sizes, None))
36
+ opcheck(torch.ops._C.aqlm_gemm,
37
+ (input, codes, codebooks, scales, codebook_partition_sizes, bias))
tests/kernels/test_attention.py
+6
-3
@@ -205,7 +205,8 @@ def test_paged_attention(
205
(output, query, key_cache, value_cache, num_kv_heads, scale,
206
block_tables, seq_lens, block_size, max_seq_len, alibi_slopes,
207
kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
208
- cond=(head_size == HEAD_SIZES[0]))
208
+ cond=(head_size == HEAD_SIZES[0]
209
+ and block_size == BLOCK_SIZES[0]))
210
211
elif version in ("v2", "rocm"):
212
num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
@@ -246,7 +247,8 @@ def test_paged_attention(
247
key_cache, value_cache, num_kv_heads, scale, block_tables,
248
seq_lens, block_size, max_seq_len, alibi_slopes,
249
kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
249
- cond=(head_size == HEAD_SIZES[0]))
250
+ cond=(head_size == HEAD_SIZES[0]
251
+ and block_size == BLOCK_SIZES[0]))
252
253
else:
254
ops.paged_attention_rocm(
@@ -274,7 +276,8 @@ def test_paged_attention(
276
key_cache, value_cache, num_kv_heads, scale, block_tables,
277
seq_lens, block_size, max_seq_len, alibi_slopes,
278
kv_cache_dtype, k_scale, v_scale),
277
- cond=(head_size == HEAD_SIZES[0]))
279
+ cond=(head_size == HEAD_SIZES[0]
280
+ and block_size == BLOCK_SIZES[0]))
281
282
else:
283
raise AssertionError(f"Unknown version: {version}")
tests/kernels/test_awq.py
new
+38
@@ -0,0 +1,38 @@
1
+import os
2
+
3
+import torch
4
+
5
+from tests.kernels.utils import opcheck
6
+from vllm import _custom_ops as ops # noqa: F401
7
+
8
+
9
+def test_awq_dequantize_opcheck():
10
+ os.environ["VLLM_USE_TRITON_AWQ"] = "0"
11
+ qweight = torch.randint(-2000000000,
12
+ 2000000000, (8192, 256),
13
+ device='cuda',
14
+ dtype=torch.int32)
15
+ scales = torch.rand((64, 2048), device='cuda', dtype=torch.float16)
16
+ zeros = torch.empty((64, 256), device='cuda', dtype=torch.int32)
17
+ split_k_iters = 0
18
+ thx = 0
19
+ thy = 0
20
+ opcheck(torch.ops._C.awq_dequantize,
21
+ (qweight, scales, zeros, split_k_iters, thx, thy))
22
+
23
+
24
+def test_awq_gemm_opcheck():
25
+ os.environ["VLLM_USE_TRITON_AWQ"] = "0"
26
+ input = torch.rand((2, 8192), device='cuda', dtype=torch.float16)
27
+ qweight = torch.randint(-2000000000,
28
+ 2000000000, (8192, 256),
29
+ device='cuda',
30
+ dtype=torch.int32)
31
+ scales = torch.randint(-2000000000,
32
+ 2000000000, (64, 256),
33
+ device='cuda',
34
+ dtype=torch.int32)
35
+ qzeros = torch.empty((64, 2048), device='cuda', dtype=torch.float16)
36
+ split_k_iters = 8
37
+ opcheck(torch.ops._C.awq_gemm,
38
+ (input, qweight, qzeros, scales, split_k_iters))
tests/kernels/test_causal_conv1d.py
+72
-2
@@ -5,6 +5,8 @@ import torch
5
import torch.nn.functional as F
6
from einops import rearrange
7
8
+from tests.kernels.utils import opcheck
9
+from vllm import _custom_ops as ops # noqa: F401
10
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
11
causal_conv1d_fn, causal_conv1d_update)
12
from vllm.utils import seed_everything
@@ -84,6 +86,64 @@ def causal_conv1d_update_ref(x: torch.Tensor,
86
return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
87
88
89
+def causal_conv1d_opcheck_fn(
90
+ x: torch.Tensor,
91
+ weight: torch.Tensor,
92
+ bias: Optional[torch.Tensor] = None,
93
+ seq_idx: Optional[torch.Tensor] = None,
94
+ initial_states: Optional[torch.Tensor] = None,
95
+ return_final_states: bool = False,
96
+ final_states_out=None,
97
+ activation: Optional[str] = "silu",
98
+):
99
+ """
100
+ x: (batch, dim, seqlen)
101
+ weight: (dim, width)
102
+ bias: (dim,)
103
+ seq_idx: (batch, seqlen)
104
+ initial_states: (batch, dim, width - 1)
105
+ final_states_out: (batch, dim, width - 1), to be written to
106
+ activation: either None or "silu" or "swish"
107
+
108
+ out: (batch, dim, seqlen)
109
+ """
110
+ if activation not in [None, "silu", "swish"]:
111
+ raise NotImplementedError("activation must be None, silu, or swish")
112
+ if x.stride(2) != 1 and x.stride(1) != 1:
113
+ x = x.contiguous()
114
+ bias = bias.contiguous() if bias is not None else None
115
+ if seq_idx is not None:
116
+ assert (initial_states is
117
+ None), "initial_states must be None if seq_idx is not None"
118
+ assert (not return_final_states
119
+ ), "If seq_idx is not None, we don't return final_states_out"
120
+ seq_idx = seq_idx.contiguous() if seq_idx is not None else None
121
+ if initial_states is not None and (initial_states.stride(2) != 1
122
+ and initial_states.stride(1) != 1):
123
+ initial_states = initial_states.contiguous()
124
+ if return_final_states:
125
+ assert (
126
+ x.stride(1) == 1
127
+ ), "Only channel-last layout support returning final_states_out"
128
+ if final_states_out is not None:
129
+ assert (final_states_out.stride(2) == 1
130
+ or final_states_out.stride(1) == 1)
131
+ else:
132
+ batch, dim, seqlen = x.shape
133
+ width = weight.shape[1]
134
+ final_states_out = torch.empty(batch,
135
+ width - 1,
136
+ dim,
137
+ device=x.device,
138
+ dtype=x.dtype).transpose(1, 2)
139
+ else:
140
+ final_states_out = None
141
+
142
+ opcheck(torch.ops._C.causal_conv1d_fwd,
143
+ (x, weight, bias, seq_idx, initial_states, final_states_out,
144
+ activation in ["silu", "swish"]))
145
+
146
+
147
@pytest.mark.parametrize("return_final_states", [False, True])
148
@pytest.mark.parametrize("has_initial_states", [False, True])
149
@pytest.mark.parametrize("channel_last", [False, True])
@@ -149,6 +209,14 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
209
initial_states=initial_states_ref,
210
return_final_states=return_final_states,
211
activation=activation)
212
+
213
+ causal_conv1d_opcheck_fn(x_ref,
214
+ weight_ref,
215
+ bias_ref,
216
+ initial_states=initial_states_ref,
217
+ return_final_states=return_final_states,
218
+ activation=activation)
219
+
220
if return_final_states:
221
assert final_states is not None and final_states_ref is not None
222
assert torch.allclose(final_states,
@@ -205,6 +273,10 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
273
assert torch.equal(conv_state, conv_state_ref)
274
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
275
276
+ opcheck(
277
+ torch.ops._C.causal_conv1d_update,
278
+ (x, conv_state, weight, bias, activation in ["silu", "swish"], None))
279
+
280
281
@pytest.mark.parametrize("itype",
282
[torch.float32, torch.float16, torch.bfloat16])
@@ -258,7 +330,5 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
330
bias,
331
activation=activation)
332
261
- print(f"Output max diff: {(out - out_ref).abs().max().item()}")
262
- print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
333
assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
334
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
tests/kernels/test_cutlass.py
+10
@@ -15,6 +15,9 @@ CUDA_DEVICES = [
15
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
16
]
17
18
+capability = current_platform.get_device_capability()
19
+capability = capability[0] * 10 + capability[1]
20
+
21
22
def to_fp8(tensor: torch.Tensor):
23
finfo = torch.finfo(torch.float8_e4m3fn)
@@ -74,6 +77,9 @@ def cutlass_fp8_gemm_helper(m: int,
77
78
torch.testing.assert_close(out, baseline, rtol=1e-2, atol=5e-2)
79
80
+ opcheck(torch.ops._C.cutlass_scaled_mm,
81
+ (out, a, b, scale_a, scale_b, bias))
82
+
83
84
def cutlass_int8_gemm_helper(m: int,
85
n: int,
@@ -425,3 +431,7 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
431
baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
432
scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
433
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
434
+
435
+
436
+def test_cutlass_support_opcheck():
437
+ opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability, ))
tests/kernels/test_flash_attn.py
+31
-30
@@ -4,6 +4,7 @@ import pytest
4
import torch
5
6
import vllm.attention.backends.flash_attn # noqa: F401
7
+from tests.kernels.utils import opcheck
8
from vllm.utils import seed_everything
9
10
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
@@ -127,19 +128,19 @@ def test_flash_attn_with_paged_kv(
128
else:
129
test_utils = ["test_faketensor"]
130
130
- torch.library.opcheck(torch.ops.vllm.flash_attn_with_kvcache,
131
- args=tuple(),
132
- kwargs=dict(
133
- decode_query=query.unsqueeze(1),
134
- key_cache=key_cache,
135
- value_cache=value_cache,
136
- softmax_scale=scale,
137
- causal=True,
138
- block_table=block_tables,
139
- cache_seqlens=kv_lens_tensor,
140
- softcap=soft_cap if soft_cap is not None else 0,
141
- ),
142
- test_utils=test_utils)
131
+ opcheck(torch.ops.vllm.flash_attn_with_kvcache,
132
+ args=tuple(),
133
+ kwargs=dict(
134
+ decode_query=query.unsqueeze(1),
135
+ key_cache=key_cache,
136
+ value_cache=value_cache,
137
+ softmax_scale=scale,
138
+ causal=True,
139
+ block_table=block_tables,
140
+ cache_seqlens=kv_lens_tensor,
141
+ softcap=soft_cap if soft_cap is not None else 0,
142
+ ),
143
+ test_utils=test_utils)
144
145
ref_output = ref_paged_attn(
146
query=query,
@@ -232,23 +233,23 @@ def test_varlen_with_paged_kv(
233
else:
234
test_utils = ["test_faketensor"]
235
235
- torch.library.opcheck(torch.ops.vllm.flash_attn_varlen_func,
236
- args=tuple(),
237
- kwargs=dict(
238
- q=query,
239
- k=key_cache,
240
- v=value_cache,
241
- cu_seqlens_q=cu_query_lens,
242
- cu_seqlens_k=cu_kv_lens,
243
- max_seqlen_q=max_query_len,
244
- max_seqlen_k=max_kv_len,
245
- softmax_scale=scale,
246
- causal=True,
247
- window_size=window_size,
248
- block_table=block_tables,
249
- softcap=soft_cap if soft_cap is not None else 0,
250
- ),
251
- test_utils=test_utils)
236
+ opcheck(torch.ops.vllm.flash_attn_varlen_func,
237
+ args=tuple(),
238
+ kwargs=dict(
239
+ q=query,
240
+ k=key_cache,
241
+ v=value_cache,
242
+ cu_seqlens_q=cu_query_lens,
243
+ cu_seqlens_k=cu_kv_lens,
244
+ max_seqlen_q=max_query_len,
245
+ max_seqlen_k=max_kv_len,
246
+ softmax_scale=scale,
247
+ causal=True,
248
+ window_size=window_size,
249
+ block_table=block_tables,
250
+ softcap=soft_cap if soft_cap is not None else 0,
251
+ ),
252
+ test_utils=test_utils)
253
254
ref_output = ref_paged_attn(
255
query=query,
tests/kernels/test_fp8_quant.py
+29
@@ -5,6 +5,7 @@ import vllm._custom_ops as ops
5
from tests.kernels.quant_utils import (FP8_DTYPE,
6
ref_dynamic_per_tensor_fp8_quant,
7
ref_dynamic_per_token_quant)
8
+from tests.kernels.utils import opcheck
9
from vllm.utils import seed_everything
10
11
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -16,6 +17,26 @@ SCALE_UBS = [True, False]
17
SEEDS = [0]
18
19
20
+def opcheck_fp8_quant(output,
21
+ input,
22
+ scale=None,
23
+ scale_ub=None,
24
+ use_per_token_if_dynamic=False):
25
+ if scale is not None:
26
+ opcheck(torch.ops._C.static_scaled_fp8_quant, (output, input, scale))
27
+ elif use_per_token_if_dynamic:
28
+ scale = torch.empty((input.shape[0], 1),
29
+ device=input.device,
30
+ dtype=torch.float32)
31
+ opcheck(torch.ops._C.dynamic_per_token_scaled_fp8_quant,
32
+ (output, input, scale, scale_ub))
33
+ else:
34
+ scale = torch.empty((input.numel() // input.shape[-1], 1),
35
+ device=input.device,
36
+ dtype=torch.float32)
37
+ opcheck(torch.ops._C.dynamic_scaled_fp8_quant, (output, input, scale))
38
+
39
+
40
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
41
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
42
@pytest.mark.parametrize("dtype", DTYPES)
@@ -41,6 +62,12 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
62
torch.testing.assert_close(ref_out.to(dtype=torch.float32),
63
ops_out.to(dtype=torch.float32))
64
65
+ opcheck_fp8_quant(ops_out,
66
+ x,
67
+ None,
68
+ scale_ub,
69
+ use_per_token_if_dynamic=True)
70
+
71
72
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
73
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@@ -60,6 +87,8 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
87
torch.testing.assert_close(ref_out.to(dtype=torch.float32),
88
ops_out.to(dtype=torch.float32))
89
90
+ opcheck_fp8_quant(ops_out, x)
91
+
92
93
# Regression test for a case with large activations where an int32 index cannot
94
# represent the number of elements.
tests/kernels/test_ggml.py
new
+22
@@ -0,0 +1,22 @@
1
+import gguf
2
+import pytest
3
+import torch
4
+
5
+from tests.kernels.utils import opcheck
6
+from vllm import _custom_ops as ops # noqa: F401
7
+
8
+
9
+@pytest.mark.parametrize("quant_type", [12])
10
+def test_ggml_opcheck(quant_type):
11
+ block_size, type_size = gguf.GGML_QUANT_SIZES[quant_type]
12
+ shape = [256, 1152]
13
+ qweight = torch.randint(0, 100, shape, device='cuda', dtype=torch.uint8)
14
+ m = qweight.shape[0]
15
+ n = qweight.shape[1] // type_size * block_size
16
+ opcheck(torch.ops._C.ggml_dequantize, (qweight, quant_type, m, n))
17
+
18
+ x = torch.rand((m, 512), device='cuda', dtype=torch.float16)
19
+ opcheck(torch.ops._C.ggml_mul_mat_a8,
20
+ (qweight, x, quant_type, qweight.shape[0]))
21
+ opcheck(torch.ops._C.ggml_mul_mat_vec_a8,
22
+ (qweight, x, quant_type, qweight.shape[0]))
tests/kernels/test_gptq.py
new
+29
@@ -0,0 +1,29 @@
1
+import torch
2
+
3
+from tests.kernels.utils import opcheck
4
+from vllm import _custom_ops as ops # noqa: F401
5
+
6
+
7
+def test_gptq_shuffle_opcheck():
8
+ weight = torch.randint(-2000000,
9
+ 2000000, (1792, 4096),
10
+ device='cuda',
11
+ dtype=torch.int32)
12
+ perm = torch.empty((0, ), device='cuda', dtype=torch.int32)
13
+ bit = 4
14
+ opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
15
+
16
+
17
+def test_gptq_gemm_opcheck():
18
+ a = torch.rand((240, 4096), device='cuda', dtype=torch.float16)
19
+ weight = torch.randint(-2000000,
20
+ 2000000, (512, 6144),
21
+ device='cuda',
22
+ dtype=torch.int32)
23
+ zeros = torch.zeros((32, 768), device='cuda', dtype=torch.int32)
24
+ scales = torch.rand((32, 6144), device='cuda', dtype=torch.float16)
25
+ idx = torch.empty((0, ), device='cuda', dtype=torch.int32)
26
+ use_exllama = True
27
+ bit = 4
28
+ opcheck(torch.ops._C.gptq_gemm,
29
+ (a, weight, zeros, scales, idx, use_exllama, bit))
tests/kernels/test_mamba_ssm.py
+66
@@ -3,6 +3,8 @@ import torch
3
import torch.nn.functional as F
4
from einops import rearrange, repeat
5
6
+from tests.kernels.utils import opcheck
7
+from vllm import _custom_ops as ops # noqa: F401
8
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
9
selective_scan_fn, selective_state_update)
10
from vllm.utils import seed_everything
@@ -161,6 +163,59 @@ def selective_scan_ref(u,
163
return out if not return_last_state else (out, last_state)
164
165
166
+def selective_scan_opcheck_fn(u,
167
+ delta,
168
+ A,
169
+ B,
170
+ C,
171
+ D=None,
172
+ z=None,
173
+ delta_bias=None,
174
+ delta_softplus=False,
175
+ return_last_state=False,
176
+ position_indices=None,
177
+ prev_state=None):
178
+ """if return_last_state is True, returns (out, last_state)
179
+ last_state has shape (batch, dim, dstate).
180
+ """
181
+ if u.stride(-1) != 1:
182
+ u = u.contiguous()
183
+ if delta.stride(-1) != 1:
184
+ delta = delta.contiguous()
185
+ if D is not None:
186
+ D = D.contiguous()
187
+ if B.stride(-1) != 1:
188
+ B = B.contiguous()
189
+ if C.stride(-1) != 1:
190
+ C = C.contiguous()
191
+ if z is not None and z.stride(-1) != 1:
192
+ z = z.contiguous()
193
+ if B.dim() == 3:
194
+ B = B.unsqueeze(1)
195
+ if C.dim() == 3:
196
+ C = C.unsqueeze(1)
197
+ n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
198
+ x = torch.zeros((
199
+ u.shape[0],
200
+ u.shape[1],
201
+ n_chunks,
202
+ int(A.shape[1] * 2),
203
+ ),
204
+ device=u.device,
205
+ dtype=torch.float32,
206
+ requires_grad=False)
207
+ x[:, :, 0, 0::2] = 1
208
+ if prev_state is not None:
209
+ x[:, :, 0, 1::2].copy_(prev_state)
210
+
211
+ # Disable test_autograd_registration for now as it seems to trigger
212
+ # a bogus error.
213
+ opcheck(torch.ops._C.selective_scan_fwd,
214
+ (u, delta, A, B, C, D, z, delta_bias, delta_softplus,
215
+ position_indices, x),
216
+ test_utils=["test_schema", "test_faketensor"])
217
+
218
+
219
@pytest.mark.parametrize('wtype', [torch.float32])
220
@pytest.mark.parametrize('itype', [torch.float32])
221
@pytest.mark.parametrize('seqlen', [128, 256, 512, 1024, 2048, 4096])
@@ -274,6 +329,17 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
329
assert state is not None and state_ref is not None
330
assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
331
332
+ selective_scan_opcheck_fn(u,
333
+ delta,
334
+ A,
335
+ B,
336
+ C,
337
+ D,
338
+ z=z,
339
+ delta_bias=delta_bias,
340
+ delta_softplus=delta_softplus,
341
+ return_last_state=return_last_state)
342
+
343
344
@pytest.mark.parametrize("itype",
345
[torch.float32, torch.float16, torch.bfloat16])
tests/kernels/test_marlin_gemm.py
+15
@@ -501,3 +501,18 @@ def test_marlin_qqq_gemm(
501
max_diff = compute_max_diff(output, output_ref)
502
503
assert max_diff < 0.04
504
+
505
+
506
+def test_marlin_gemm_opcheck():
507
+ size_m = 2048
508
+ size_n = 4096
509
+ size_k = 4096
510
+ a = torch.rand((size_m, size_n), device='cuda', dtype=torch.float16)
511
+ w = torch.randint(-5, 5, (256, 8192), device='cuda', dtype=torch.int32)
512
+ s = torch.full((32, size_k), 0.125, device='cuda', dtype=torch.float16)
513
+ wk = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
514
+ GPTQ_MARLIN_MAX_PARALLEL).scratch
515
+ x = torch.ops._C.marlin_gemm(a, w, s, wk, size_m, size_n, size_k)
516
+ y = torch.ops._C.marlin_gemm(a, w, s, wk, size_m, size_n, size_k)
517
+ torch.testing.assert_close(x, y)
518
+ opcheck(torch.ops._C.marlin_gemm, (a, w, s, wk, size_m, size_n, size_k))
tests/kernels/test_moe.py
+59
-1
@@ -9,11 +9,14 @@ import torch
9
from transformers import MixtralConfig
10
from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
11
12
+from tests.kernels.utils import opcheck
13
+from vllm import _custom_ops as ops
14
from vllm.model_executor.layers.activation import SiluAndMul
15
from vllm.model_executor.layers.fused_moe import fused_moe
16
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
17
fused_marlin_moe, single_marlin_moe)
16
-from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
18
+from vllm.model_executor.layers.fused_moe.fused_moe import (
19
+ fused_topk, moe_align_block_size)
20
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
21
marlin_quantize)
22
from vllm.model_executor.models.mixtral import MixtralMoE
@@ -247,6 +250,35 @@ def test_fused_marlin_moe(
250
251
assert compute_max_diff(marlin_output, triton_output) < 4e-2
252
253
+ if ops.supports_moe_ops:
254
+ token_expert_indicies = torch.empty(m,
255
+ topk,
256
+ dtype=torch.int32,
257
+ device=a.device)
258
+
259
+ opcheck(torch.ops._moe_C.topk_softmax, (
260
+ topk_weights,
261
+ topk_ids,
262
+ token_expert_indicies,
263
+ score.float(),
264
+ ))
265
+
266
+ block_size_m = 4
267
+
268
+ sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m,
269
+ e)
270
+
271
+ max_workspace_size = ((m + 255) // 256) * (max(2 * n, k) // 64) * 16
272
+ workspace = torch.zeros(max_workspace_size,
273
+ dtype=torch.int,
274
+ device="cuda",
275
+ requires_grad=False)
276
+
277
+ opcheck(torch.ops._moe_C.marlin_gemm_moe,
278
+ (a, qweight1, sorted_token_ids, topk_weights, topk_ids,
279
+ scales1, g_idx1, sort_indices1, workspace, quant_type, m,
280
+ 2 * n, k, True, e, topk, block_size_m, True, False))
281
+
282
283
@pytest.mark.skip("This test is here for the sake of debugging, "
284
"don't run it in automated tests.")
@@ -319,3 +351,29 @@ def test_single_marlin_moe_multiply(
351
torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
352
353
assert compute_max_diff(marlin_output, torch_output) < 1e-2
354
+
355
+
356
+def test_moe_align_block_size_opcheck():
357
+ num_experts = 4
358
+ block_size = 4
359
+ topk_ids = torch.randint(0,
360
+ num_experts, (3, 4),
361
+ dtype=torch.int32,
362
+ device='cuda')
363
+
364
+ max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
365
+ sorted_ids = torch.empty((max_num_tokens_padded, ),
366
+ dtype=torch.int32,
367
+ device=topk_ids.device)
368
+ sorted_ids.fill_(topk_ids.numel())
369
+ max_num_m_blocks = max_num_tokens_padded // block_size
370
+ expert_ids = torch.empty((max_num_m_blocks, ),
371
+ dtype=torch.int32,
372
+ device=topk_ids.device)
373
+ num_tokens_post_pad = torch.empty((1),
374
+ dtype=torch.int32,
375
+ device=topk_ids.device)
376
+
377
+ opcheck(torch.ops._C.moe_align_block_size,
378
+ (topk_ids, num_experts, block_size, sorted_ids, expert_ids,
379
+ num_tokens_post_pad))
tests/kernels/test_rotary_embedding.py
new
+62
@@ -0,0 +1,62 @@
1
+"""
2
+Tests for miscellaneous utilities
3
+"""
4
+
5
+from typing import Optional
6
+
7
+import pytest
8
+import torch
9
+
10
+from tests.kernels.utils import opcheck
11
+from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
12
+
13
+
14
+def rotary_embedding_opcheck(rot,
15
+ positions: torch.Tensor,
16
+ query: torch.Tensor,
17
+ key: torch.Tensor,
18
+ offsets: Optional[torch.Tensor] = None):
19
+ cos_sin_cache = rot.cos_sin_cache.to(query.device, dtype=query.dtype)
20
+
21
+ # ops.rotary_embedding()/batched_rotary_embedding()
22
+ # are in-place operations that update the query and key tensors.
23
+ if offsets is not None:
24
+ opcheck(torch.ops._C.batched_rotary_embedding,
25
+ (positions, query, key, rot.head_size, cos_sin_cache,
26
+ rot.is_neox_style, rot.rotary_dim, offsets))
27
+ else:
28
+ opcheck(torch.ops._C.rotary_embedding,
29
+ (positions, query, key, rot.head_size, cos_sin_cache,
30
+ rot.is_neox_style))
31
+
32
+
33
+@pytest.mark.parametrize("device", ["cuda"])
34
+@pytest.mark.parametrize("max_position", [11, 4096, 32768])
35
+@pytest.mark.parametrize("is_neox_style", [True, False])
36
+@pytest.mark.parametrize("rotary_dim", [32])
37
+@pytest.mark.parametrize("head_size", [32, 108])
38
+@pytest.mark.parametrize("seq_len", [11, 1024])
39
+def test_rotary_embedding_opcheck(dist_init, device, max_position,
40
+ is_neox_style, rotary_dim, head_size,
41
+ seq_len):
42
+ batch_size = 1
43
+ base = 0
44
+ num_heads = 7
45
+ rot = RotaryEmbedding(head_size, rotary_dim, max_position, base,
46
+ is_neox_style, torch.float32)
47
+
48
+ positions = torch.randint(0,
49
+ max_position, (batch_size, seq_len),
50
+ device=device)
51
+ query = torch.randn(batch_size,
52
+ seq_len,
53
+ num_heads * head_size,
54
+ dtype=torch.float32,
55
+ device=device)
56
+ key = torch.randn_like(query)
57
+
58
+ rotary_embedding_opcheck(rot, positions, query, key)
59
+ offsets = torch.zeros(batch_size * seq_len,
60
+ device=device,
61
+ dtype=torch.long)
62
+ rotary_embedding_opcheck(rot, positions, query, key, offsets)
tests/kernels/test_utils.py
new
+24
@@ -0,0 +1,24 @@
1
+"""
2
+Tests for miscellaneous utilities
3
+"""
4
+
5
+import pytest
6
+import torch
7
+
8
+from tests.kernels.utils import opcheck
9
+from vllm.platforms import current_platform
10
+
11
+
12
+def test_convert_fp8_opcheck():
13
+ data = torch.randn((256, 256), dtype=torch.float32, device="cuda")
14
+ result = torch.empty_like(data, dtype=torch.float8_e4m3fn)
15
+ opcheck(torch.ops._C_cache_ops.convert_fp8, (result, data, 1.0, "fp8"))
16
+
17
+
18
+@pytest.mark.skipif(not current_platform.is_cuda(),
19
+ reason="Only supported for CUDA")
20
+def test_cuda_utils_opcheck():
21
+ opcheck(torch.ops._C_cuda_utils.get_device_attribute, (0, 0))
22
+ opcheck(
23
+ torch.ops._C_cuda_utils.
24
+ get_max_shared_memory_per_block_device_attribute, (0, ))
tests/kernels/utils.py
+37
-6
@@ -2,12 +2,14 @@
2
3
import itertools
4
import random
5
+import unittest
6
from numbers import Number
7
from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
8
Union)
9
10
import pytest
11
import torch
12
+from torch._prims_common import TensorLikeType
13
14
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
@@ -946,6 +948,34 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
948
output_under_test.view_as(ideal_output))
949
950
951
+# Copied/modified from torch._refs.__init__.py
952
+def fp8_allclose(
953
+ a: TensorLikeType,
954
+ b: TensorLikeType,
955
+ rtol: float = 1e-05,
956
+ atol: float = 1e-08,
957
+ equal_nan: bool = False,
958
+) -> bool:
959
+ """
960
+ Reference implementation of torch.allclose
961
+ """
962
+ torch._refs._check_close_args(name="torch.allclose",
963
+ a=a,
964
+ b=b,
965
+ rtol=rtol,
966
+ atol=atol)
967
+
968
+ return bool(
969
+ torch.all(
970
+ torch.isclose(a.double(),
971
+ b.double(),
972
+ rtol=rtol,
973
+ atol=atol,
974
+ equal_nan=equal_nan)).item())
975
+
976
+
977
+# A special version of op check that has a restricted default set of test_utils
978
+# and a patched version of allclose that supports fp8 types.
979
def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
980
torch._library.custom_ops.CustomOpDef],
981
args: Tuple[Any, ...],
@@ -954,9 +984,10 @@ def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
984
test_utils: Union[str, Sequence[str]] = ALL_OPCHECK_TEST_UTILS,
985
raise_exception: bool = True,
986
cond: bool = True) -> Dict[str, str]:
957
- return torch.library.opcheck(
958
- op,
959
- args,
960
- kwargs,
961
- test_utils=test_utils,
962
- raise_exception=raise_exception) if cond else {}
987
+ with unittest.mock.patch('torch.allclose', new=fp8_allclose):
988
+ return torch.library.opcheck(
989
+ op,
990
+ args,
991
+ kwargs,
992
+ test_utils=test_utils,
993
+ raise_exception=raise_exception) if cond else {}
vllm/_custom_ops.py
+31
-30
@@ -20,8 +20,10 @@ if not current_platform.is_tpu():
20
if current_platform.is_rocm():
21
import vllm._rocm_C # noqa: F401
22
23
+supports_moe_ops = False
24
with contextlib.suppress(ImportError):
25
import vllm._moe_C # noqa: F401
26
+ supports_moe_ops = True
27
28
29
def hint_on_error(fn):
@@ -253,9 +255,7 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
255
b_g_idx, use_exllama, bit)
256
257
256
-# TODO: has to be a better way to do this
257
-try:
258
- torch.ops._C.gptq_gemm # noqa B018
258
+if hasattr(torch.ops._C, "gptq_gemm"):
259
260
@torch.library.register_fake("_C::gptq_gemm")
261
def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -265,8 +265,6 @@ try:
265
return torch.empty((a.size(0), b_q_weight.size(1)),
266
dtype=a.dtype,
267
device=a.device)
268
-except Exception:
269
- pass
268
269
270
def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
@@ -292,9 +290,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
290
size_n, size_k)
291
292
295
-# TODO: has to be a better way to do this
296
-try:
297
- torch.ops._C.gptq_marlin_24_gemm # noqa B018
293
+if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
294
295
@torch.library.register_fake("_C::gptq_marlin_24_gemm")
296
def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -420,8 +416,8 @@ try:
416
@torch.library.register_fake("_C::machete_gemm")
417
def machete_gemm_fake(
418
a: torch.Tensor,
423
- b_q: torch.
424
- Tensor, # Should be the tensor returned by machete_prepack_B
419
+ # Should be the tensor returned by machete_prepack_B
420
+ b_q: torch.Tensor,
421
b_type: ScalarType,
422
b_scales: Optional[torch.Tensor] = None,
423
b_zeros: Optional[torch.Tensor] = None,
@@ -451,10 +447,10 @@ try:
447
return torch.empty_like(x)
448
449
@torch.library.register_fake("_C::causal_conv1d_update")
454
- def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
455
- weight: torch.Tensor,
456
- bias_: Optional[torch.Tensor],
457
- silu_activation: bool) -> torch.Tensor:
450
+ def causal_conv1d_update_fake(
451
+ x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
452
+ bias_: Optional[torch.Tensor], silu_activation: bool,
453
+ conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
454
return torch.empty_like(x)
455
456
@torch.library.register_fake("_C::selective_scan_fwd")
@@ -465,20 +461,11 @@ try:
461
delta_softplus: bool, index_: Optional[torch.Tensor],
462
x: Optional[torch.Tensor]) -> List[torch.Tensor]:
463
a = torch.empty_like(u)
468
- if x is not None:
469
- b = x
470
- else:
471
- b = torch.empty((u.size(0), u.size(1), A.size(1)),
472
- dtype=u.dtype,
473
- device=u.device)
464
if z_ is not None:
465
c = torch.empty_like(z_)
476
- return [a, b, c]
466
+ return [a, c]
467
else:
478
- return [a, b]
479
-
480
-except Exception:
481
- pass
468
+ return [a]
469
470
471
# cutlass
@@ -626,16 +613,12 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
613
return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
614
615
629
-# TODO: has to be a better way to do this
630
-try:
631
- torch.ops._C.permute_cols # noqa B018
616
+if hasattr(torch.ops._C, "permute_cols"):
617
618
@torch.library.register_fake("_C::permute_cols")
619
def _permute_cols_fake(a: torch.Tensor,
620
perm: torch.Tensor) -> torch.Tensor:
621
return torch.empty_like(a)
637
-except Exception:
638
- pass
622
623
624
def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
@@ -828,6 +811,24 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
811
token_expert_indicies, gating_output)
812
813
814
+if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
815
+
816
+ @torch.library.register_fake("_moe_C::marlin_gemm_moe")
817
+ def marlin_gemm_moe_fake(a: torch.Tensor, b_q_weights: torch.Tensor,
818
+ sorted_ids: torch.Tensor,
819
+ topk_weights: torch.Tensor,
820
+ topk_ids: torch.Tensor, b_scales: torch.Tensor,
821
+ g_idx: torch.Tensor, perm: torch.Tensor,
822
+ workspace: torch.Tensor, b_q_type: ScalarType,
823
+ size_m: int, size_n: int, size_k: int,
824
+ is_k_full: bool, num_experts: int, topk: int,
825
+ moe_block_size: int, replicate_input: bool,
826
+ apply_weights: bool) -> torch.Tensor:
827
+ return torch.empty((size_m, topk, size_n),
828
+ dtype=a.dtype,
829
+ device=a.device)
830
+
831
+
832
def reshape_and_cache(
833
key: torch.Tensor,
834
value: torch.Tensor,
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+2
-2
@@ -361,8 +361,8 @@ def selective_scan_fn(u,
361
x[:, :, 0, 0::2] = 1
362
if prev_state is not None:
363
x[:, :, 0, 1::2].copy_(prev_state)
364
- out, x, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
365
- delta_softplus, position_indices, x)
364
+ out, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
365
+ delta_softplus, position_indices, x)
366
last_state = x[:, :, -1, 1::2] # (batch, dim, dstate)
367
if z is None:
368
return out if not return_last_state else (out, last_state)
vllm/model_executor/layers/quantization/gptq.py
+1
@@ -217,6 +217,7 @@ class GPTQLinearMethod(LinearMethodBase):
217
layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
218
layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
219
layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
220
+ layer.scales = Parameter(layer.scales.data, requires_grad=False)
221
222
# exllama needs to shuffle the weight after the weight is loaded
223
# here we do the shuffle on first forward pass