@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 300da09177
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel] Fullgraph and opcheck tests (#8479)

bnellnm committed Sep 25, 2024 at 10:35 UTC 300da09177477d0a4d2b55790addefd971f52ae0
26 files changed +744 -116
.buildkite/test-pipeline.yaml
+17 -2
@@ -70,7 +70,7 @@ steps:
70 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
71 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
72 - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
73 -
73 +
74 - label: Core Test # 10min
75 mirror_hardwares: [amd]
76 fast_check: true
@@ -210,6 +210,21 @@ steps:
210 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
211 parallelism: 4
212
213 +- label: "PyTorch Fullgraph Smoke Test"
214 + fast_check: true
215 + source_file_dependencies:
216 + - vllm/
217 + - tests/compile
218 + commands:
219 + - pytest -v -s compile/test_full_graph_smoke.py
220 +
221 +- label: "PyTorch Fullgraph Test"
222 + source_file_dependencies:
223 + - vllm/
224 + - tests/compile
225 + commands:
226 + - pytest -v -s compile/test_full_graph.py
227 +
228 - label: Kernels Test %N # 30min each
229 mirror_hardwares: [amd]
230 source_file_dependencies:
@@ -355,7 +370,7 @@ steps:
370 - tests/distributed/
371 - vllm/compilation
372 commands:
358 - - pytest -v -s ./compile/test_full_graph.py
373 + - pytest -v -s ./compile/test_full_graph_multi_gpu.py
374 - pytest -v -s ./compile/test_wrapper.py
375 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
376 - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+1 -1
@@ -586,7 +586,7 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
586 DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), "selective_scan_fwd", [&] {
587 selective_scan_fwd_cuda<input_t, weight_t>(params, stream);
588 });
589 - std::vector<at::Tensor> result = {out, x.value()};
589 + std::vector<at::Tensor> result = {out};
590 if (has_z) { result.push_back(out_z); }
591 return result;
592 }
csrc/torch_bindings.cpp
+2 -2
@@ -275,7 +275,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
275 "Tensor! A, Tensor! B, Tensor! C,"
276 "Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
277 "bool delta_softplus,"
278 - "Tensor? index_, Tensor(a! -> *)? x) -> Tensor(a)[]");
278 + "Tensor? index_, Tensor!? x) -> Tensor[]");
279 ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
280
281 ops.def(
@@ -292,7 +292,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
292 "Tensor? bias_,"
293 "Tensor? seq_idx_,"
294 "Tensor? initial_states_,"
295 - "Tensor? final_states_out_,"
295 + "Tensor!? final_states_out_,"
296 "bool silu_activation) -> Tensor");
297 ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
298 #endif
tests/compile/test_full_graph.py
+8 -37
@@ -1,42 +1,13 @@
1 -import os
2 -
1 import pytest
2
5 -from vllm.utils import cuda_device_count_stateless
6 -
7 -from ..utils import fork_new_process_for_each_test
8 -
9 -
10 -@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
11 -@pytest.mark.parametrize("tp_size", [1, 2])
12 -@fork_new_process_for_each_test
13 -def test_full_graph(model, tp_size):
14 -
15 - # Skip the test if there are not enough CUDA devices.
16 - if cuda_device_count_stateless() < tp_size:
17 - pytest.skip("Not enough CUDA devices for the test.")
18 -
19 - # make sure these models can be captured in full graph mode
20 - if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
21 - os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
3 +from vllm.compilation.backends import vllm_backend
4
23 - from vllm import LLM, SamplingParams
24 - prompts = [
25 - "Hello, my name is",
26 - "The president of the United States is",
27 - "The capital of France is",
28 - "The future of AI is",
29 - ]
30 - sampling_params = SamplingParams(temperature=0)
31 - llm = LLM(model=model,
32 - enforce_eager=True,
33 - tensor_parallel_size=tp_size,
34 - disable_custom_all_reduce=True)
5 +from .utils import TEST_MODELS, check_full_graph_support
6
36 - outputs = llm.generate(prompts, sampling_params)
7
38 - # Print the outputs.
39 - for output in outputs:
40 - prompt = output.prompt
41 - generated_text = output.outputs[0].text
42 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
8 +@pytest.mark.parametrize("model_info", TEST_MODELS)
9 +@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10 +def test_full_graph(model_info, backend):
11 + model = model_info[0]
12 + model_kwargs = model_info[1]
13 + check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/test_full_graph_multi_gpu.py new
+22
@@ -0,0 +1,22 @@
1 +import pytest
2 +
3 +from vllm.compilation.backends import vllm_backend
4 +from vllm.utils import cuda_device_count_stateless
5 +
6 +from ..utils import fork_new_process_for_each_test
7 +from .utils import TEST_MODELS_SMOKE, check_full_graph_support
8 +
9 +
10 +@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
11 +@pytest.mark.parametrize("tp_size", [2])
12 +@pytest.mark.parametrize("backend", ["eager", vllm_backend])
13 +@fork_new_process_for_each_test
14 +def test_full_graph_multi_gpu(model_info, tp_size, backend):
15 + model = model_info[0]
16 + model_kwargs = model_info[1]
17 +
18 + # Skip the test if there are not enough CUDA devices.
19 + if cuda_device_count_stateless() < tp_size:
20 + pytest.skip("Not enough CUDA devices for the test.")
21 +
22 + check_full_graph_support(model, model_kwargs, backend, tp_size=tp_size)
tests/compile/test_full_graph_smoke.py new
+13
@@ -0,0 +1,13 @@
1 +import pytest
2 +
3 +from vllm.compilation.backends import vllm_backend
4 +
5 +from .utils import TEST_MODELS_SMOKE, check_full_graph_support
6 +
7 +
8 +@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
9 +@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10 +def test_full_graph(model_info, backend):
11 + model = model_info[0]
12 + model_kwargs = model_info[1]
13 + check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/utils.py new
+104
@@ -0,0 +1,104 @@
1 +import os
2 +
3 +import torch
4 +
5 +from tests.quantization.utils import is_quant_method_supported
6 +from vllm import LLM, SamplingParams
7 +from vllm.plugins import set_torch_compile_backend
8 +from vllm.utils import is_hip
9 +
10 +TEST_MODELS_SMOKE = [
11 + ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
12 + "quantization": "compressed-tensors"
13 + }),
14 + ("meta-llama/Meta-Llama-3-8B", {}),
15 +]
16 +
17 +TEST_MODELS = [
18 + ("facebook/opt-125m", {}),
19 + ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
20 + "dtype": torch.float16,
21 + "quantization": "compressed-tensors"
22 + }),
23 + ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
24 + "dtype": torch.float16,
25 + "quantization": "fp8"
26 + }),
27 + ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
28 + "quantization": "compressed-tensors"
29 + }),
30 + ("meta-llama/Meta-Llama-3-8B", {}),
31 +]
32 +
33 +# TODO: enable in pytorch 2.5
34 +if False and is_quant_method_supported("aqlm"): # noqa: SIM223
35 + TEST_MODELS.append(("ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf", {
36 + "quantization": "aqlm"
37 + }))
38 +
39 +# TODO: enable in pytorch 2.5
40 +if False and is_quant_method_supported("gguf"): # noqa: SIM223
41 + TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {
42 + "quantization": "gguf"
43 + }))
44 +
45 +if is_quant_method_supported("gptq"):
46 + TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {
47 + "quantization": "gptq"
48 + }))
49 +
50 +if is_quant_method_supported("gptq_marlin"):
51 + TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", {
52 + "quantization": "gptq_marlin"
53 + }))
54 +
55 +if is_quant_method_supported("gptq_marlin_24"):
56 + TEST_MODELS.append(("alexm-nm/tinyllama-24-marlin24-4bit-g128", {
57 + "quantization": "gptq_marlin_24"
58 + }))
59 +
60 +if is_quant_method_supported("marlin"):
61 + TEST_MODELS.append(("robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-marlin", {
62 + "quantization": "marlin"
63 + }))
64 +
65 +if not is_hip() and is_quant_method_supported("awq"):
66 + TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {
67 + "quantization": "AWQ"
68 + }))
69 +
70 +
71 +def check_full_graph_support(model, model_kwargs, backend, tp_size=1):
72 + # make sure these models can be captured in full graph mode
73 + if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
74 + os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
75 + os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
76 +
77 + # Inductor doesn't support fp8/gptq_marlin_24 yet.
78 + quantization = model_kwargs.get("quantization")
79 + if (quantization == "fp8" or quantization == "gptq_marlin"
80 + or quantization == "gptq_marlin_24") and backend != "eager":
81 + return
82 +
83 + set_torch_compile_backend(backend)
84 +
85 + prompts = [
86 + "Hello, my name is",
87 + "The president of the United States is",
88 + "The capital of France is",
89 + "The future of AI is",
90 + ]
91 + sampling_params = SamplingParams(temperature=0)
92 + llm = LLM(model=model,
93 + enforce_eager=True,
94 + tensor_parallel_size=tp_size,
95 + disable_custom_all_reduce=True,
96 + **model_kwargs)
97 +
98 + outputs = llm.generate(prompts, sampling_params)
99 +
100 + # Print the outputs.
101 + for output in outputs:
102 + prompt = output.prompt
103 + generated_text = output.outputs[0].text
104 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/conftest.py
+6
@@ -169,6 +169,12 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
169 cleanup()
170
171
172 +@pytest.fixture(autouse=True)
173 +def dynamo_reset():
174 + yield
175 + torch._dynamo.reset()
176 +
177 +
178 @pytest.fixture
179 def example_prompts() -> List[str]:
180 prompts = []
tests/kernels/test_aqlm.py new
+37
@@ -0,0 +1,37 @@
1 +import torch
2 +
3 +from tests.kernels.utils import opcheck
4 +from vllm import _custom_ops as ops # noqa: F401
5 +
6 +
7 +def test_aqlm_dequant_opcheck():
8 + codes = torch.randint(-32768,
9 + 32767, (22016, 512, 1),
10 + device='cuda',
11 + dtype=torch.int16)
12 + codebooks = torch.rand((2, 65536, 1, 8),
13 + device='cuda',
14 + dtype=torch.float16)
15 + codebook_partition_sizes = [11008, 11008]
16 +
17 + opcheck(torch.ops._C.aqlm_dequant,
18 + (codes, codebooks, codebook_partition_sizes))
19 +
20 +
21 +def test_aqlm_gemm_opcheck():
22 + input = torch.rand((4, 4096), device='cuda', dtype=torch.float16)
23 + codes = torch.randint(-32768,
24 + 32767, (12288, 512, 1),
25 + device='cuda',
26 + dtype=torch.int16)
27 + codebooks = torch.rand((3, 65536, 1, 8),
28 + device='cuda',
29 + dtype=torch.float16)
30 + scales = torch.rand((12288, 1, 1, 1), device='cuda', dtype=torch.float16)
31 + codebook_partition_sizes = [4096, 4096, 4096]
32 + bias = None
33 +
34 + opcheck(torch.ops._C.aqlm_gemm,
35 + (input, codes, codebooks, scales, codebook_partition_sizes, None))
36 + opcheck(torch.ops._C.aqlm_gemm,
37 + (input, codes, codebooks, scales, codebook_partition_sizes, bias))
tests/kernels/test_attention.py
+6 -3
@@ -205,7 +205,8 @@ def test_paged_attention(
205 (output, query, key_cache, value_cache, num_kv_heads, scale,
206 block_tables, seq_lens, block_size, max_seq_len, alibi_slopes,
207 kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
208 - cond=(head_size == HEAD_SIZES[0]))
208 + cond=(head_size == HEAD_SIZES[0]
209 + and block_size == BLOCK_SIZES[0]))
210
211 elif version in ("v2", "rocm"):
212 num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
@@ -246,7 +247,8 @@ def test_paged_attention(
247 key_cache, value_cache, num_kv_heads, scale, block_tables,
248 seq_lens, block_size, max_seq_len, alibi_slopes,
249 kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
249 - cond=(head_size == HEAD_SIZES[0]))
250 + cond=(head_size == HEAD_SIZES[0]
251 + and block_size == BLOCK_SIZES[0]))
252
253 else:
254 ops.paged_attention_rocm(
@@ -274,7 +276,8 @@ def test_paged_attention(
276 key_cache, value_cache, num_kv_heads, scale, block_tables,
277 seq_lens, block_size, max_seq_len, alibi_slopes,
278 kv_cache_dtype, k_scale, v_scale),
277 - cond=(head_size == HEAD_SIZES[0]))
279 + cond=(head_size == HEAD_SIZES[0]
280 + and block_size == BLOCK_SIZES[0]))
281
282 else:
283 raise AssertionError(f"Unknown version: {version}")
tests/kernels/test_awq.py new
+38
@@ -0,0 +1,38 @@
1 +import os
2 +
3 +import torch
4 +
5 +from tests.kernels.utils import opcheck
6 +from vllm import _custom_ops as ops # noqa: F401
7 +
8 +
9 +def test_awq_dequantize_opcheck():
10 + os.environ["VLLM_USE_TRITON_AWQ"] = "0"
11 + qweight = torch.randint(-2000000000,
12 + 2000000000, (8192, 256),
13 + device='cuda',
14 + dtype=torch.int32)
15 + scales = torch.rand((64, 2048), device='cuda', dtype=torch.float16)
16 + zeros = torch.empty((64, 256), device='cuda', dtype=torch.int32)
17 + split_k_iters = 0
18 + thx = 0
19 + thy = 0
20 + opcheck(torch.ops._C.awq_dequantize,
21 + (qweight, scales, zeros, split_k_iters, thx, thy))
22 +
23 +
24 +def test_awq_gemm_opcheck():
25 + os.environ["VLLM_USE_TRITON_AWQ"] = "0"
26 + input = torch.rand((2, 8192), device='cuda', dtype=torch.float16)
27 + qweight = torch.randint(-2000000000,
28 + 2000000000, (8192, 256),
29 + device='cuda',
30 + dtype=torch.int32)
31 + scales = torch.randint(-2000000000,
32 + 2000000000, (64, 256),
33 + device='cuda',
34 + dtype=torch.int32)
35 + qzeros = torch.empty((64, 2048), device='cuda', dtype=torch.float16)
36 + split_k_iters = 8
37 + opcheck(torch.ops._C.awq_gemm,
38 + (input, qweight, qzeros, scales, split_k_iters))
tests/kernels/test_causal_conv1d.py
+72 -2
@@ -5,6 +5,8 @@ import torch
5 import torch.nn.functional as F
6 from einops import rearrange
7
8 +from tests.kernels.utils import opcheck
9 +from vllm import _custom_ops as ops # noqa: F401
10 from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
11 causal_conv1d_fn, causal_conv1d_update)
12 from vllm.utils import seed_everything
@@ -84,6 +86,64 @@ def causal_conv1d_update_ref(x: torch.Tensor,
86 return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
87
88
89 +def causal_conv1d_opcheck_fn(
90 + x: torch.Tensor,
91 + weight: torch.Tensor,
92 + bias: Optional[torch.Tensor] = None,
93 + seq_idx: Optional[torch.Tensor] = None,
94 + initial_states: Optional[torch.Tensor] = None,
95 + return_final_states: bool = False,
96 + final_states_out=None,
97 + activation: Optional[str] = "silu",
98 +):
99 + """
100 + x: (batch, dim, seqlen)
101 + weight: (dim, width)
102 + bias: (dim,)
103 + seq_idx: (batch, seqlen)
104 + initial_states: (batch, dim, width - 1)
105 + final_states_out: (batch, dim, width - 1), to be written to
106 + activation: either None or "silu" or "swish"
107 +
108 + out: (batch, dim, seqlen)
109 + """
110 + if activation not in [None, "silu", "swish"]:
111 + raise NotImplementedError("activation must be None, silu, or swish")
112 + if x.stride(2) != 1 and x.stride(1) != 1:
113 + x = x.contiguous()
114 + bias = bias.contiguous() if bias is not None else None
115 + if seq_idx is not None:
116 + assert (initial_states is
117 + None), "initial_states must be None if seq_idx is not None"
118 + assert (not return_final_states
119 + ), "If seq_idx is not None, we don't return final_states_out"
120 + seq_idx = seq_idx.contiguous() if seq_idx is not None else None
121 + if initial_states is not None and (initial_states.stride(2) != 1
122 + and initial_states.stride(1) != 1):
123 + initial_states = initial_states.contiguous()
124 + if return_final_states:
125 + assert (
126 + x.stride(1) == 1
127 + ), "Only channel-last layout support returning final_states_out"
128 + if final_states_out is not None:
129 + assert (final_states_out.stride(2) == 1
130 + or final_states_out.stride(1) == 1)
131 + else:
132 + batch, dim, seqlen = x.shape
133 + width = weight.shape[1]
134 + final_states_out = torch.empty(batch,
135 + width - 1,
136 + dim,
137 + device=x.device,
138 + dtype=x.dtype).transpose(1, 2)
139 + else:
140 + final_states_out = None
141 +
142 + opcheck(torch.ops._C.causal_conv1d_fwd,
143 + (x, weight, bias, seq_idx, initial_states, final_states_out,
144 + activation in ["silu", "swish"]))
145 +
146 +
147 @pytest.mark.parametrize("return_final_states", [False, True])
148 @pytest.mark.parametrize("has_initial_states", [False, True])
149 @pytest.mark.parametrize("channel_last", [False, True])
@@ -149,6 +209,14 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
209 initial_states=initial_states_ref,
210 return_final_states=return_final_states,
211 activation=activation)
212 +
213 + causal_conv1d_opcheck_fn(x_ref,
214 + weight_ref,
215 + bias_ref,
216 + initial_states=initial_states_ref,
217 + return_final_states=return_final_states,
218 + activation=activation)
219 +
220 if return_final_states:
221 assert final_states is not None and final_states_ref is not None
222 assert torch.allclose(final_states,
@@ -205,6 +273,10 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
273 assert torch.equal(conv_state, conv_state_ref)
274 assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
275
276 + opcheck(
277 + torch.ops._C.causal_conv1d_update,
278 + (x, conv_state, weight, bias, activation in ["silu", "swish"], None))
279 +
280
281 @pytest.mark.parametrize("itype",
282 [torch.float32, torch.float16, torch.bfloat16])
@@ -258,7 +330,5 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
330 bias,
331 activation=activation)
332
261 - print(f"Output max diff: {(out - out_ref).abs().max().item()}")
262 - print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
333 assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
334 assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
tests/kernels/test_cutlass.py
+10
@@ -15,6 +15,9 @@ CUDA_DEVICES = [
15 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
16 ]
17
18 +capability = current_platform.get_device_capability()
19 +capability = capability[0] * 10 + capability[1]
20 +
21
22 def to_fp8(tensor: torch.Tensor):
23 finfo = torch.finfo(torch.float8_e4m3fn)
@@ -74,6 +77,9 @@ def cutlass_fp8_gemm_helper(m: int,
77
78 torch.testing.assert_close(out, baseline, rtol=1e-2, atol=5e-2)
79
80 + opcheck(torch.ops._C.cutlass_scaled_mm,
81 + (out, a, b, scale_a, scale_b, bias))
82 +
83
84 def cutlass_int8_gemm_helper(m: int,
85 n: int,
@@ -425,3 +431,7 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
431 baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
432 scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
433 torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
434 +
435 +
436 +def test_cutlass_support_opcheck():
437 + opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability, ))
tests/kernels/test_flash_attn.py
+31 -30
@@ -4,6 +4,7 @@ import pytest
4 import torch
5
6 import vllm.attention.backends.flash_attn # noqa: F401
7 +from tests.kernels.utils import opcheck
8 from vllm.utils import seed_everything
9
10 NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
@@ -127,19 +128,19 @@ def test_flash_attn_with_paged_kv(
128 else:
129 test_utils = ["test_faketensor"]
130
130 - torch.library.opcheck(torch.ops.vllm.flash_attn_with_kvcache,
131 - args=tuple(),
132 - kwargs=dict(
133 - decode_query=query.unsqueeze(1),
134 - key_cache=key_cache,
135 - value_cache=value_cache,
136 - softmax_scale=scale,
137 - causal=True,
138 - block_table=block_tables,
139 - cache_seqlens=kv_lens_tensor,
140 - softcap=soft_cap if soft_cap is not None else 0,
141 - ),
142 - test_utils=test_utils)
131 + opcheck(torch.ops.vllm.flash_attn_with_kvcache,
132 + args=tuple(),
133 + kwargs=dict(
134 + decode_query=query.unsqueeze(1),
135 + key_cache=key_cache,
136 + value_cache=value_cache,
137 + softmax_scale=scale,
138 + causal=True,
139 + block_table=block_tables,
140 + cache_seqlens=kv_lens_tensor,
141 + softcap=soft_cap if soft_cap is not None else 0,
142 + ),
143 + test_utils=test_utils)
144
145 ref_output = ref_paged_attn(
146 query=query,
@@ -232,23 +233,23 @@ def test_varlen_with_paged_kv(
233 else:
234 test_utils = ["test_faketensor"]
235
235 - torch.library.opcheck(torch.ops.vllm.flash_attn_varlen_func,
236 - args=tuple(),
237 - kwargs=dict(
238 - q=query,
239 - k=key_cache,
240 - v=value_cache,
241 - cu_seqlens_q=cu_query_lens,
242 - cu_seqlens_k=cu_kv_lens,
243 - max_seqlen_q=max_query_len,
244 - max_seqlen_k=max_kv_len,
245 - softmax_scale=scale,
246 - causal=True,
247 - window_size=window_size,
248 - block_table=block_tables,
249 - softcap=soft_cap if soft_cap is not None else 0,
250 - ),
251 - test_utils=test_utils)
236 + opcheck(torch.ops.vllm.flash_attn_varlen_func,
237 + args=tuple(),
238 + kwargs=dict(
239 + q=query,
240 + k=key_cache,
241 + v=value_cache,
242 + cu_seqlens_q=cu_query_lens,
243 + cu_seqlens_k=cu_kv_lens,
244 + max_seqlen_q=max_query_len,
245 + max_seqlen_k=max_kv_len,
246 + softmax_scale=scale,
247 + causal=True,
248 + window_size=window_size,
249 + block_table=block_tables,
250 + softcap=soft_cap if soft_cap is not None else 0,
251 + ),
252 + test_utils=test_utils)
253
254 ref_output = ref_paged_attn(
255 query=query,
tests/kernels/test_fp8_quant.py
+29
@@ -5,6 +5,7 @@ import vllm._custom_ops as ops
5 from tests.kernels.quant_utils import (FP8_DTYPE,
6 ref_dynamic_per_tensor_fp8_quant,
7 ref_dynamic_per_token_quant)
8 +from tests.kernels.utils import opcheck
9 from vllm.utils import seed_everything
10
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -16,6 +17,26 @@ SCALE_UBS = [True, False]
17 SEEDS = [0]
18
19
20 +def opcheck_fp8_quant(output,
21 + input,
22 + scale=None,
23 + scale_ub=None,
24 + use_per_token_if_dynamic=False):
25 + if scale is not None:
26 + opcheck(torch.ops._C.static_scaled_fp8_quant, (output, input, scale))
27 + elif use_per_token_if_dynamic:
28 + scale = torch.empty((input.shape[0], 1),
29 + device=input.device,
30 + dtype=torch.float32)
31 + opcheck(torch.ops._C.dynamic_per_token_scaled_fp8_quant,
32 + (output, input, scale, scale_ub))
33 + else:
34 + scale = torch.empty((input.numel() // input.shape[-1], 1),
35 + device=input.device,
36 + dtype=torch.float32)
37 + opcheck(torch.ops._C.dynamic_scaled_fp8_quant, (output, input, scale))
38 +
39 +
40 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
41 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
42 @pytest.mark.parametrize("dtype", DTYPES)
@@ -41,6 +62,12 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
62 torch.testing.assert_close(ref_out.to(dtype=torch.float32),
63 ops_out.to(dtype=torch.float32))
64
65 + opcheck_fp8_quant(ops_out,
66 + x,
67 + None,
68 + scale_ub,
69 + use_per_token_if_dynamic=True)
70 +
71
72 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
73 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@@ -60,6 +87,8 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
87 torch.testing.assert_close(ref_out.to(dtype=torch.float32),
88 ops_out.to(dtype=torch.float32))
89
90 + opcheck_fp8_quant(ops_out, x)
91 +
92
93 # Regression test for a case with large activations where an int32 index cannot
94 # represent the number of elements.
tests/kernels/test_ggml.py new
+22
@@ -0,0 +1,22 @@
1 +import gguf
2 +import pytest
3 +import torch
4 +
5 +from tests.kernels.utils import opcheck
6 +from vllm import _custom_ops as ops # noqa: F401
7 +
8 +
9 +@pytest.mark.parametrize("quant_type", [12])
10 +def test_ggml_opcheck(quant_type):
11 + block_size, type_size = gguf.GGML_QUANT_SIZES[quant_type]
12 + shape = [256, 1152]
13 + qweight = torch.randint(0, 100, shape, device='cuda', dtype=torch.uint8)
14 + m = qweight.shape[0]
15 + n = qweight.shape[1] // type_size * block_size
16 + opcheck(torch.ops._C.ggml_dequantize, (qweight, quant_type, m, n))
17 +
18 + x = torch.rand((m, 512), device='cuda', dtype=torch.float16)
19 + opcheck(torch.ops._C.ggml_mul_mat_a8,
20 + (qweight, x, quant_type, qweight.shape[0]))
21 + opcheck(torch.ops._C.ggml_mul_mat_vec_a8,
22 + (qweight, x, quant_type, qweight.shape[0]))
tests/kernels/test_gptq.py new
+29
@@ -0,0 +1,29 @@
1 +import torch
2 +
3 +from tests.kernels.utils import opcheck
4 +from vllm import _custom_ops as ops # noqa: F401
5 +
6 +
7 +def test_gptq_shuffle_opcheck():
8 + weight = torch.randint(-2000000,
9 + 2000000, (1792, 4096),
10 + device='cuda',
11 + dtype=torch.int32)
12 + perm = torch.empty((0, ), device='cuda', dtype=torch.int32)
13 + bit = 4
14 + opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
15 +
16 +
17 +def test_gptq_gemm_opcheck():
18 + a = torch.rand((240, 4096), device='cuda', dtype=torch.float16)
19 + weight = torch.randint(-2000000,
20 + 2000000, (512, 6144),
21 + device='cuda',
22 + dtype=torch.int32)
23 + zeros = torch.zeros((32, 768), device='cuda', dtype=torch.int32)
24 + scales = torch.rand((32, 6144), device='cuda', dtype=torch.float16)
25 + idx = torch.empty((0, ), device='cuda', dtype=torch.int32)
26 + use_exllama = True
27 + bit = 4
28 + opcheck(torch.ops._C.gptq_gemm,
29 + (a, weight, zeros, scales, idx, use_exllama, bit))
tests/kernels/test_mamba_ssm.py
+66
@@ -3,6 +3,8 @@ import torch
3 import torch.nn.functional as F
4 from einops import rearrange, repeat
5
6 +from tests.kernels.utils import opcheck
7 +from vllm import _custom_ops as ops # noqa: F401
8 from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
9 selective_scan_fn, selective_state_update)
10 from vllm.utils import seed_everything
@@ -161,6 +163,59 @@ def selective_scan_ref(u,
163 return out if not return_last_state else (out, last_state)
164
165
166 +def selective_scan_opcheck_fn(u,
167 + delta,
168 + A,
169 + B,
170 + C,
171 + D=None,
172 + z=None,
173 + delta_bias=None,
174 + delta_softplus=False,
175 + return_last_state=False,
176 + position_indices=None,
177 + prev_state=None):
178 + """if return_last_state is True, returns (out, last_state)
179 + last_state has shape (batch, dim, dstate).
180 + """
181 + if u.stride(-1) != 1:
182 + u = u.contiguous()
183 + if delta.stride(-1) != 1:
184 + delta = delta.contiguous()
185 + if D is not None:
186 + D = D.contiguous()
187 + if B.stride(-1) != 1:
188 + B = B.contiguous()
189 + if C.stride(-1) != 1:
190 + C = C.contiguous()
191 + if z is not None and z.stride(-1) != 1:
192 + z = z.contiguous()
193 + if B.dim() == 3:
194 + B = B.unsqueeze(1)
195 + if C.dim() == 3:
196 + C = C.unsqueeze(1)
197 + n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
198 + x = torch.zeros((
199 + u.shape[0],
200 + u.shape[1],
201 + n_chunks,
202 + int(A.shape[1] * 2),
203 + ),
204 + device=u.device,
205 + dtype=torch.float32,
206 + requires_grad=False)
207 + x[:, :, 0, 0::2] = 1
208 + if prev_state is not None:
209 + x[:, :, 0, 1::2].copy_(prev_state)
210 +
211 + # Disable test_autograd_registration for now as it seems to trigger
212 + # a bogus error.
213 + opcheck(torch.ops._C.selective_scan_fwd,
214 + (u, delta, A, B, C, D, z, delta_bias, delta_softplus,
215 + position_indices, x),
216 + test_utils=["test_schema", "test_faketensor"])
217 +
218 +
219 @pytest.mark.parametrize('wtype', [torch.float32])
220 @pytest.mark.parametrize('itype', [torch.float32])
221 @pytest.mark.parametrize('seqlen', [128, 256, 512, 1024, 2048, 4096])
@@ -274,6 +329,17 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
329 assert state is not None and state_ref is not None
330 assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
331
332 + selective_scan_opcheck_fn(u,
333 + delta,
334 + A,
335 + B,
336 + C,
337 + D,
338 + z=z,
339 + delta_bias=delta_bias,
340 + delta_softplus=delta_softplus,
341 + return_last_state=return_last_state)
342 +
343
344 @pytest.mark.parametrize("itype",
345 [torch.float32, torch.float16, torch.bfloat16])
tests/kernels/test_marlin_gemm.py
+15
@@ -501,3 +501,18 @@ def test_marlin_qqq_gemm(
501 max_diff = compute_max_diff(output, output_ref)
502
503 assert max_diff < 0.04
504 +
505 +
506 +def test_marlin_gemm_opcheck():
507 + size_m = 2048
508 + size_n = 4096
509 + size_k = 4096
510 + a = torch.rand((size_m, size_n), device='cuda', dtype=torch.float16)
511 + w = torch.randint(-5, 5, (256, 8192), device='cuda', dtype=torch.int32)
512 + s = torch.full((32, size_k), 0.125, device='cuda', dtype=torch.float16)
513 + wk = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
514 + GPTQ_MARLIN_MAX_PARALLEL).scratch
515 + x = torch.ops._C.marlin_gemm(a, w, s, wk, size_m, size_n, size_k)
516 + y = torch.ops._C.marlin_gemm(a, w, s, wk, size_m, size_n, size_k)
517 + torch.testing.assert_close(x, y)
518 + opcheck(torch.ops._C.marlin_gemm, (a, w, s, wk, size_m, size_n, size_k))
tests/kernels/test_moe.py
+59 -1
@@ -9,11 +9,14 @@ import torch
9 from transformers import MixtralConfig
10 from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
11
12 +from tests.kernels.utils import opcheck
13 +from vllm import _custom_ops as ops
14 from vllm.model_executor.layers.activation import SiluAndMul
15 from vllm.model_executor.layers.fused_moe import fused_moe
16 from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
17 fused_marlin_moe, single_marlin_moe)
16 -from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
18 +from vllm.model_executor.layers.fused_moe.fused_moe import (
19 + fused_topk, moe_align_block_size)
20 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
21 marlin_quantize)
22 from vllm.model_executor.models.mixtral import MixtralMoE
@@ -247,6 +250,35 @@ def test_fused_marlin_moe(
250
251 assert compute_max_diff(marlin_output, triton_output) < 4e-2
252
253 + if ops.supports_moe_ops:
254 + token_expert_indicies = torch.empty(m,
255 + topk,
256 + dtype=torch.int32,
257 + device=a.device)
258 +
259 + opcheck(torch.ops._moe_C.topk_softmax, (
260 + topk_weights,
261 + topk_ids,
262 + token_expert_indicies,
263 + score.float(),
264 + ))
265 +
266 + block_size_m = 4
267 +
268 + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m,
269 + e)
270 +
271 + max_workspace_size = ((m + 255) // 256) * (max(2 * n, k) // 64) * 16
272 + workspace = torch.zeros(max_workspace_size,
273 + dtype=torch.int,
274 + device="cuda",
275 + requires_grad=False)
276 +
277 + opcheck(torch.ops._moe_C.marlin_gemm_moe,
278 + (a, qweight1, sorted_token_ids, topk_weights, topk_ids,
279 + scales1, g_idx1, sort_indices1, workspace, quant_type, m,
280 + 2 * n, k, True, e, topk, block_size_m, True, False))
281 +
282
283 @pytest.mark.skip("This test is here for the sake of debugging, "
284 "don't run it in automated tests.")
@@ -319,3 +351,29 @@ def test_single_marlin_moe_multiply(
351 torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
352
353 assert compute_max_diff(marlin_output, torch_output) < 1e-2
354 +
355 +
356 +def test_moe_align_block_size_opcheck():
357 + num_experts = 4
358 + block_size = 4
359 + topk_ids = torch.randint(0,
360 + num_experts, (3, 4),
361 + dtype=torch.int32,
362 + device='cuda')
363 +
364 + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
365 + sorted_ids = torch.empty((max_num_tokens_padded, ),
366 + dtype=torch.int32,
367 + device=topk_ids.device)
368 + sorted_ids.fill_(topk_ids.numel())
369 + max_num_m_blocks = max_num_tokens_padded // block_size
370 + expert_ids = torch.empty((max_num_m_blocks, ),
371 + dtype=torch.int32,
372 + device=topk_ids.device)
373 + num_tokens_post_pad = torch.empty((1),
374 + dtype=torch.int32,
375 + device=topk_ids.device)
376 +
377 + opcheck(torch.ops._C.moe_align_block_size,
378 + (topk_ids, num_experts, block_size, sorted_ids, expert_ids,
379 + num_tokens_post_pad))
tests/kernels/test_rotary_embedding.py new
+62
@@ -0,0 +1,62 @@
1 +"""
2 +Tests for miscellaneous utilities
3 +"""
4 +
5 +from typing import Optional
6 +
7 +import pytest
8 +import torch
9 +
10 +from tests.kernels.utils import opcheck
11 +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
12 +
13 +
14 +def rotary_embedding_opcheck(rot,
15 + positions: torch.Tensor,
16 + query: torch.Tensor,
17 + key: torch.Tensor,
18 + offsets: Optional[torch.Tensor] = None):
19 + cos_sin_cache = rot.cos_sin_cache.to(query.device, dtype=query.dtype)
20 +
21 + # ops.rotary_embedding()/batched_rotary_embedding()
22 + # are in-place operations that update the query and key tensors.
23 + if offsets is not None:
24 + opcheck(torch.ops._C.batched_rotary_embedding,
25 + (positions, query, key, rot.head_size, cos_sin_cache,
26 + rot.is_neox_style, rot.rotary_dim, offsets))
27 + else:
28 + opcheck(torch.ops._C.rotary_embedding,
29 + (positions, query, key, rot.head_size, cos_sin_cache,
30 + rot.is_neox_style))
31 +
32 +
33 +@pytest.mark.parametrize("device", ["cuda"])
34 +@pytest.mark.parametrize("max_position", [11, 4096, 32768])
35 +@pytest.mark.parametrize("is_neox_style", [True, False])
36 +@pytest.mark.parametrize("rotary_dim", [32])
37 +@pytest.mark.parametrize("head_size", [32, 108])
38 +@pytest.mark.parametrize("seq_len", [11, 1024])
39 +def test_rotary_embedding_opcheck(dist_init, device, max_position,
40 + is_neox_style, rotary_dim, head_size,
41 + seq_len):
42 + batch_size = 1
43 + base = 0
44 + num_heads = 7
45 + rot = RotaryEmbedding(head_size, rotary_dim, max_position, base,
46 + is_neox_style, torch.float32)
47 +
48 + positions = torch.randint(0,
49 + max_position, (batch_size, seq_len),
50 + device=device)
51 + query = torch.randn(batch_size,
52 + seq_len,
53 + num_heads * head_size,
54 + dtype=torch.float32,
55 + device=device)
56 + key = torch.randn_like(query)
57 +
58 + rotary_embedding_opcheck(rot, positions, query, key)
59 + offsets = torch.zeros(batch_size * seq_len,
60 + device=device,
61 + dtype=torch.long)
62 + rotary_embedding_opcheck(rot, positions, query, key, offsets)
tests/kernels/test_utils.py new
+24
@@ -0,0 +1,24 @@
1 +"""
2 +Tests for miscellaneous utilities
3 +"""
4 +
5 +import pytest
6 +import torch
7 +
8 +from tests.kernels.utils import opcheck
9 +from vllm.platforms import current_platform
10 +
11 +
12 +def test_convert_fp8_opcheck():
13 + data = torch.randn((256, 256), dtype=torch.float32, device="cuda")
14 + result = torch.empty_like(data, dtype=torch.float8_e4m3fn)
15 + opcheck(torch.ops._C_cache_ops.convert_fp8, (result, data, 1.0, "fp8"))
16 +
17 +
18 +@pytest.mark.skipif(not current_platform.is_cuda(),
19 + reason="Only supported for CUDA")
20 +def test_cuda_utils_opcheck():
21 + opcheck(torch.ops._C_cuda_utils.get_device_attribute, (0, 0))
22 + opcheck(
23 + torch.ops._C_cuda_utils.
24 + get_max_shared_memory_per_block_device_attribute, (0, ))
tests/kernels/utils.py
+37 -6
@@ -2,12 +2,14 @@
2
3 import itertools
4 import random
5 +import unittest
6 from numbers import Number
7 from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
8 Union)
9
10 import pytest
11 import torch
12 +from torch._prims_common import TensorLikeType
13
14 from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15 from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
@@ -946,6 +948,34 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
948 output_under_test.view_as(ideal_output))
949
950
951 +# Copied/modified from torch._refs.__init__.py
952 +def fp8_allclose(
953 + a: TensorLikeType,
954 + b: TensorLikeType,
955 + rtol: float = 1e-05,
956 + atol: float = 1e-08,
957 + equal_nan: bool = False,
958 +) -> bool:
959 + """
960 + Reference implementation of torch.allclose
961 + """
962 + torch._refs._check_close_args(name="torch.allclose",
963 + a=a,
964 + b=b,
965 + rtol=rtol,
966 + atol=atol)
967 +
968 + return bool(
969 + torch.all(
970 + torch.isclose(a.double(),
971 + b.double(),
972 + rtol=rtol,
973 + atol=atol,
974 + equal_nan=equal_nan)).item())
975 +
976 +
977 +# A special version of op check that has a restricted default set of test_utils
978 +# and a patched version of allclose that supports fp8 types.
979 def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
980 torch._library.custom_ops.CustomOpDef],
981 args: Tuple[Any, ...],
@@ -954,9 +984,10 @@ def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
984 test_utils: Union[str, Sequence[str]] = ALL_OPCHECK_TEST_UTILS,
985 raise_exception: bool = True,
986 cond: bool = True) -> Dict[str, str]:
957 - return torch.library.opcheck(
958 - op,
959 - args,
960 - kwargs,
961 - test_utils=test_utils,
962 - raise_exception=raise_exception) if cond else {}
987 + with unittest.mock.patch('torch.allclose', new=fp8_allclose):
988 + return torch.library.opcheck(
989 + op,
990 + args,
991 + kwargs,
992 + test_utils=test_utils,
993 + raise_exception=raise_exception) if cond else {}
vllm/_custom_ops.py
+31 -30
@@ -20,8 +20,10 @@ if not current_platform.is_tpu():
20 if current_platform.is_rocm():
21 import vllm._rocm_C # noqa: F401
22
23 +supports_moe_ops = False
24 with contextlib.suppress(ImportError):
25 import vllm._moe_C # noqa: F401
26 + supports_moe_ops = True
27
28
29 def hint_on_error(fn):
@@ -253,9 +255,7 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
255 b_g_idx, use_exllama, bit)
256
257
256 -# TODO: has to be a better way to do this
257 -try:
258 - torch.ops._C.gptq_gemm # noqa B018
258 +if hasattr(torch.ops._C, "gptq_gemm"):
259
260 @torch.library.register_fake("_C::gptq_gemm")
261 def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -265,8 +265,6 @@ try:
265 return torch.empty((a.size(0), b_q_weight.size(1)),
266 dtype=a.dtype,
267 device=a.device)
268 -except Exception:
269 - pass
268
269
270 def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
@@ -292,9 +290,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
290 size_n, size_k)
291
292
295 -# TODO: has to be a better way to do this
296 -try:
297 - torch.ops._C.gptq_marlin_24_gemm # noqa B018
293 +if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
294
295 @torch.library.register_fake("_C::gptq_marlin_24_gemm")
296 def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -420,8 +416,8 @@ try:
416 @torch.library.register_fake("_C::machete_gemm")
417 def machete_gemm_fake(
418 a: torch.Tensor,
423 - b_q: torch.
424 - Tensor, # Should be the tensor returned by machete_prepack_B
419 + # Should be the tensor returned by machete_prepack_B
420 + b_q: torch.Tensor,
421 b_type: ScalarType,
422 b_scales: Optional[torch.Tensor] = None,
423 b_zeros: Optional[torch.Tensor] = None,
@@ -451,10 +447,10 @@ try:
447 return torch.empty_like(x)
448
449 @torch.library.register_fake("_C::causal_conv1d_update")
454 - def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
455 - weight: torch.Tensor,
456 - bias_: Optional[torch.Tensor],
457 - silu_activation: bool) -> torch.Tensor:
450 + def causal_conv1d_update_fake(
451 + x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
452 + bias_: Optional[torch.Tensor], silu_activation: bool,
453 + conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
454 return torch.empty_like(x)
455
456 @torch.library.register_fake("_C::selective_scan_fwd")
@@ -465,20 +461,11 @@ try:
461 delta_softplus: bool, index_: Optional[torch.Tensor],
462 x: Optional[torch.Tensor]) -> List[torch.Tensor]:
463 a = torch.empty_like(u)
468 - if x is not None:
469 - b = x
470 - else:
471 - b = torch.empty((u.size(0), u.size(1), A.size(1)),
472 - dtype=u.dtype,
473 - device=u.device)
464 if z_ is not None:
465 c = torch.empty_like(z_)
476 - return [a, b, c]
466 + return [a, c]
467 else:
478 - return [a, b]
479 -
480 -except Exception:
481 - pass
468 + return [a]
469
470
471 # cutlass
@@ -626,16 +613,12 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
613 return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
614
615
629 -# TODO: has to be a better way to do this
630 -try:
631 - torch.ops._C.permute_cols # noqa B018
616 +if hasattr(torch.ops._C, "permute_cols"):
617
618 @torch.library.register_fake("_C::permute_cols")
619 def _permute_cols_fake(a: torch.Tensor,
620 perm: torch.Tensor) -> torch.Tensor:
621 return torch.empty_like(a)
637 -except Exception:
638 - pass
622
623
624 def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
@@ -828,6 +811,24 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
811 token_expert_indicies, gating_output)
812
813
814 +if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
815 +
816 + @torch.library.register_fake("_moe_C::marlin_gemm_moe")
817 + def marlin_gemm_moe_fake(a: torch.Tensor, b_q_weights: torch.Tensor,
818 + sorted_ids: torch.Tensor,
819 + topk_weights: torch.Tensor,
820 + topk_ids: torch.Tensor, b_scales: torch.Tensor,
821 + g_idx: torch.Tensor, perm: torch.Tensor,
822 + workspace: torch.Tensor, b_q_type: ScalarType,
823 + size_m: int, size_n: int, size_k: int,
824 + is_k_full: bool, num_experts: int, topk: int,
825 + moe_block_size: int, replicate_input: bool,
826 + apply_weights: bool) -> torch.Tensor:
827 + return torch.empty((size_m, topk, size_n),
828 + dtype=a.dtype,
829 + device=a.device)
830 +
831 +
832 def reshape_and_cache(
833 key: torch.Tensor,
834 value: torch.Tensor,
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+2 -2
@@ -361,8 +361,8 @@ def selective_scan_fn(u,
361 x[:, :, 0, 0::2] = 1
362 if prev_state is not None:
363 x[:, :, 0, 1::2].copy_(prev_state)
364 - out, x, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
365 - delta_softplus, position_indices, x)
364 + out, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
365 + delta_softplus, position_indices, x)
366 last_state = x[:, :, -1, 1::2] # (batch, dim, dstate)
367 if z is None:
368 return out if not return_last_state else (out, last_state)
vllm/model_executor/layers/quantization/gptq.py
+1
@@ -217,6 +217,7 @@ class GPTQLinearMethod(LinearMethodBase):
217 layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
218 layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
219 layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
220 + layer.scales = Parameter(layer.scales.data, requires_grad=False)
221
222 # exllama needs to shuffle the weight after the weight is loaded
223 # here we do the shuffle on first forward pass