A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Kernel][Misc] register ops to prevent graph breaks (#6917)
Co-authored-by: Sage Moore <sage@neuralmagic.com>
bnellnm committed
Sep 11, 2024 at 15:52 UTC
73202dbe77913df9cf520bf18172ac40e0b9951f
22 files changed
+528
-102
.github/PULL_REQUEST_TEMPLATE.md
+10
@@ -39,6 +39,16 @@ FIX #xxxx (*link existing issues this PR will resolve*)
39
<li>Please add documentation to <code>docs/source/</code> if the PR modifies the user-facing behaviors of vLLM. It helps vLLM user understand and utilize the new features or changes.</li>
40
</ul>
41
42
+<h3>Adding or changing kernels</h3>
43
+<p>Each custom kernel needs a schema and one or more implementations to be registered with PyTorch.</p>
44
+<ul>
45
+ <li>Make sure custom ops are registered following PyTorch guidelines: <a href="https://pytorch.org/tutorials/advanced/cpp_custom_ops.html#cpp-custom-ops-tutorial">Custom C++ and CUDA Operators</a> and <a href="https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU">The Custom Operators Manual</a></li>
46
+ <li>Custom operations that return <code>Tensors</code> require meta-functions. Meta-functions should be implemented and registered in python so that dynamic dims can be handled automatically. See above documents for a description of meta-functions.</li>
47
+ <li>Use <a href="https://pytorch.org/docs/stable/library.html#torch.library.opcheck"><code>torch.libary.opcheck()</code></a> to test the function registration and meta-function for any registered ops. See <code>tests/kernels</code> for examples.</li>
48
+ <li>When changing the C++ signature of an existing op, the schema must be updated to reflect the changes.</li>
49
+ <li>If a new custom type is needed, see the following document: <a href="https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA">Custom Class Support in PT2</a>.
50
+</ul>
51
+
52
<h3>Notes for Large Changes</h3>
53
<p>Please keep the changes as concise as possible. For major architectural changes (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue (RFC) discussing the technical design and justification. Otherwise, we will tag it with <code>rfc-required</code> and might not go through the PR.</p>
54
cmake/utils.cmake
+1
@@ -350,6 +350,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
350
target_include_directories(${GPU_MOD_NAME} PRIVATE csrc
351
${GPU_INCLUDE_DIRECTORIES})
352
353
+ # TODO: is torch_python_LIBRARY needed?
354
target_link_libraries(${GPU_MOD_NAME} PRIVATE torch ${torch_python_LIBRARY}
355
${GPU_LIBRARIES})
356
csrc/cpu/torch_bindings.cpp
+4
-4
@@ -32,8 +32,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
32
// PagedAttention V2.
33
ops.def(
34
"paged_attention_v2("
35
- " Tensor! out, Tensor exp_sums, Tensor max_logits,"
36
- " Tensor tmp_out, Tensor query, Tensor key_cache,"
35
+ " Tensor! out, Tensor! exp_sums, Tensor! max_logits,"
36
+ " Tensor! tmp_out, Tensor query, Tensor key_cache,"
37
" Tensor value_cache, int num_kv_heads, float scale,"
38
" Tensor block_tables, Tensor seq_lens, int block_size,"
39
" int max_seq_len, Tensor? alibi_slopes,"
@@ -122,8 +122,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
122
123
// Copy the cache blocks from src to dst.
124
cache_ops.def(
125
- "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
126
- "block_mapping) -> ()");
125
+ "copy_blocks(Tensor(a!)[] key_caches, Tensor[](b!) value_caches, "
126
+ "Tensor block_mapping) -> ()");
127
cache_ops.impl("copy_blocks", torch::kCPU, ©_blocks);
128
129
// Reshape the key and value tensors and cache them.
csrc/ops.h
+8
@@ -123,9 +123,17 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
123
int64_t size_k, int64_t size_n,
124
int64_t num_bits);
125
126
+torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
127
+ torch::Tensor& perm, c10::SymInt size_k,
128
+ c10::SymInt size_n, int64_t num_bits);
129
+
130
torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
131
int64_t size_n, int64_t num_bits);
132
133
+torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
134
+ c10::SymInt size_k, c10::SymInt size_n,
135
+ int64_t num_bits);
136
+
137
torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
138
int64_t n);
139
csrc/quantization/gptq_marlin/awq_marlin_repack.cu
+12
@@ -267,3 +267,15 @@ torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
267
}
268
269
#endif
270
+
271
+torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
272
+ c10::SymInt size_k, c10::SymInt size_n,
273
+ int64_t num_bits) {
274
+ int const pack_factor = 32 / num_bits;
275
+ auto options = torch::TensorOptions()
276
+ .dtype(b_q_weight.dtype())
277
+ .device(b_q_weight.device());
278
+ return torch::empty_symint(
279
+ {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
280
+ options);
281
+}
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+12
@@ -342,3 +342,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
342
}
343
344
#endif
345
+
346
+torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
347
+ torch::Tensor& perm, c10::SymInt size_k,
348
+ c10::SymInt size_n, int64_t num_bits) {
349
+ int const pack_factor = 32 / num_bits;
350
+ auto options = torch::TensorOptions()
351
+ .dtype(b_q_weight.dtype())
352
+ .device(b_q_weight.device());
353
+ return torch::empty_symint(
354
+ {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
355
+ options);
356
+}
csrc/torch_bindings.cpp
+100
-44
@@ -36,8 +36,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
36
// PagedAttention V2.
37
ops.def(
38
"paged_attention_v2("
39
- " Tensor! out, Tensor exp_sums, Tensor max_logits,"
40
- " Tensor tmp_out, Tensor query, Tensor key_cache,"
39
+ " Tensor! out, Tensor! exp_sums, Tensor! max_logits,"
40
+ " Tensor! tmp_out, Tensor query, Tensor key_cache,"
41
" Tensor value_cache, int num_kv_heads, float scale,"
42
" Tensor block_tables, Tensor seq_lens, int block_size,"
43
" int max_seq_len, Tensor? alibi_slopes,"
@@ -73,7 +73,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
73
ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74
75
// prepare_inputs advance_step
76
- ops.def("advance_step", &advance_step);
76
+ ops.def(
77
+ "advance_step(int num_seqs, int num_queries, int block_size, "
78
+ "Tensor! input_tokens, Tensor sampled_token_ids, "
79
+ "Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping, "
80
+ "Tensor block_tables) -> ()");
81
ops.impl("advance_step", torch::kCUDA, &advance_step);
82
83
// Layernorm
@@ -110,27 +114,56 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
114
// Quantization ops
115
#ifndef USE_ROCM
116
// Quantized GEMM for AQLM.
113
- ops.def("aqlm_gemm", &aqlm_gemm);
117
+ ops.def(
118
+ "aqlm_gemm(Tensor input, Tensor codes, Tensor codebooks, "
119
+ "Tensor scales, int[] codebook_partition_sizes, Tensor? bias) "
120
+ "-> Tensor");
121
ops.impl("aqlm_gemm", torch::kCUDA, &aqlm_gemm);
122
123
// Decompression method for AQLM.
117
- ops.def("aqlm_dequant", &aqlm_dequant);
124
+ ops.def(
125
+ "aqlm_dequant(Tensor codes, Tensor codebooks, "
126
+ "int[] codebook_partition_sizes) -> Tensor");
127
ops.impl("aqlm_dequant", torch::kCUDA, &aqlm_dequant);
128
129
// Quantized GEMM for AWQ.
121
- ops.def("awq_gemm", &awq_gemm);
130
+ ops.def(
131
+ "awq_gemm(Tensor _in_feats, Tensor _kernel, Tensor _scaling_factors, "
132
+ "Tensor _zeros, int split_k_iters) -> Tensor");
133
ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
134
135
// Dequantization for AWQ.
125
- ops.def("awq_dequantize", &awq_dequantize);
136
+ ops.def(
137
+ "awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
138
+ "Tensor _zeros, int split_k_iters, int thx, int thy) -> Tensor");
139
ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
140
141
+ // Note about marlin kernel 'workspace' arguments:
142
+ // Technically these should be mutable since they are modified by the kernel.
143
+ // But since they are set back to zero once the kernel is finished we can
144
+ // hand wave and say that they have no net effect.
145
+ //
146
+ // The reason to mark 'workspace' as immutable is so that they don't interfere
147
+ // with using ScalarType arguments in the ops. If they are marked as mutable,
148
+ // pytorch throws an assert in
149
+ // 'torch._higher_order_ops._register_effectful_op' that prevents these
150
+ // kernels from being torch.compile'd.
151
+ // See the following document for more info on custom types and ops that use
152
+ // custom types:
153
+ // https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
154
+
155
// Marlin (Dense) Optimized Quantized GEMM for GPTQ.
129
- ops.def("marlin_gemm", &marlin_gemm);
156
+ ops.def(
157
+ "marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
158
+ "Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
159
ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
160
161
// Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
133
- ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
162
+ ops.def(
163
+ "gptq_marlin_24_gemm(Tensor a, Tensor b_q_weight, Tensor b_meta, "
164
+ "Tensor b_scales, Tensor workspace, "
165
+ "__torch__.torch.classes._core_C.ScalarType b_q_type, "
166
+ "int size_m, int size_n, int size_k) -> Tensor");
167
ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
168
169
// Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
@@ -149,35 +182,55 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
182
ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
183
184
// gptq_marlin Optimized Quantized GEMM for GPTQ.
152
- ops.def("gptq_marlin_gemm", &gptq_marlin_gemm);
185
+ ops.def(
186
+ "gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
187
+ "Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
188
+ "__torch__.torch.classes._core_C.ScalarType b_q_type, "
189
+ "int size_m, int size_n, int size_k, bool is_k_full, "
190
+ "bool has_zp, bool use_fp32_reduce) -> Tensor");
191
ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
192
193
// gptq_marlin repack from GPTQ.
156
- ops.def("gptq_marlin_repack", &gptq_marlin_repack);
194
+ ops.def(
195
+ "gptq_marlin_repack(Tensor b_q_weight, Tensor perm, "
196
+ "SymInt size_k, SymInt size_n, int num_bits) -> Tensor");
197
ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
198
+ ops.impl("gptq_marlin_repack", torch::kMeta, &gptq_marlin_repack_meta);
199
200
// awq_marlin repack from AWQ.
160
- ops.def("awq_marlin_repack", &awq_marlin_repack);
201
+ ops.def(
202
+ "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
203
+ "SymInt size_n, int num_bits) -> Tensor");
204
ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
205
+ ops.impl("awq_marlin_repack", torch::kMeta, &awq_marlin_repack_meta);
206
207
// Dequantization for GGML.
164
- ops.def("ggml_dequantize", &ggml_dequantize);
208
+ ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
209
ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
210
211
// mmvq kernel for GGML.
168
- ops.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8);
212
+ ops.def(
213
+ "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, int row) "
214
+ "-> Tensor");
215
ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
216
217
// mmq kernel for GGML.
172
- ops.def("ggml_mul_mat_a8", &ggml_mul_mat_a8);
218
+ ops.def("ggml_mul_mat_a8(Tensor W, Tensor X, int type, int row) -> Tensor");
219
ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
220
221
// fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
176
- ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
222
+ ops.def(
223
+ "fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
224
+ "Tensor! workspace, int num_bits, int size_m, int size_n, "
225
+ "int size_k) -> Tensor");
226
ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
227
228
// marlin_qqq_gemm for QQQ.
180
- ops.def("marlin_qqq_gemm", &marlin_qqq_gemm);
229
+ ops.def(
230
+ "marlin_qqq_gemm(Tensor a, Tensor b_q_weight, "
231
+ "Tensor s_tok, Tensor s_ch, Tensor s_group, "
232
+ "Tensor! workspace, int size_m, int size_n, "
233
+ "int size_k) -> Tensor");
234
ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
235
236
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
@@ -199,16 +252,16 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
252
253
// Check if cutlass scaled_mm is supported for CUDA devices of the given
254
// capability
202
- ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
203
- ops.impl("cutlass_scaled_mm_supports_fp8", torch::kCUDA,
204
- &cutlass_scaled_mm_supports_fp8);
255
+ ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
256
+ ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
257
+
258
// Mamba selective scan kernel
259
ops.def(
260
"selective_scan_fwd(Tensor! u, Tensor! delta,"
261
"Tensor! A, Tensor! B, Tensor! C,"
262
"Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
263
"bool delta_softplus,"
211
- "Tensor? index_, Tensor? x) -> Tensor[]");
264
+ "Tensor? index_, Tensor(a! -> *)? x) -> Tensor(a)[]");
265
ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
266
267
ops.def(
@@ -230,7 +283,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
283
#endif
284
285
// Quantized GEMM for GPTQ.
233
- ops.def("gptq_gemm", &gptq_gemm);
286
+ // Note: even though the C++ inferred schema is correct for this op, it seems
287
+ // to prevent the meta function registry.
288
+ ops.def(
289
+ "gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, "
290
+ "Tensor b_gptq_scales, Tensor b_g_idx, bool use_exllama, int bit) "
291
+ "-> Tensor");
292
ops.impl("gptq_gemm", torch::kCUDA, &gptq_gemm);
293
294
// Post processing for GPTQ.
@@ -250,8 +308,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
308
309
// Compute dynamic-per-token FP8 quantized tensor and scaling factor.
310
ops.def(
253
- "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! "
254
- "scale, Tensor? scale_ub) -> "
311
+ "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, "
312
+ "Tensor! scale, Tensor? scale_ub) -> "
313
"()");
314
ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
315
&dynamic_per_token_scaled_fp8_quant);
@@ -288,8 +346,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
346
347
// Copy the cache blocks from src to dst.
348
cache_ops.def(
291
- "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
292
- "block_mapping) -> ()");
349
+ "copy_blocks(Tensor(a!)[] key_caches, Tensor[](b!) value_caches, "
350
+ "Tensor block_mapping) -> ()");
351
cache_ops.impl("copy_blocks", torch::kCUDA, ©_blocks);
352
353
// Reshape the key and value tensors and cache them.
@@ -314,8 +372,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
372
373
// Convert the key and value cache to fp8 data type.
374
cache_ops.def(
317
- "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, str "
318
- "kv_cache_dtype) -> ()");
375
+ "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, "
376
+ "str kv_cache_dtype) -> ()");
377
cache_ops.impl("convert_fp8", torch::kCUDA, &convert_fp8);
378
}
379
@@ -323,24 +381,28 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
381
// Cuda utils
382
383
// Gets the specified device attribute.
326
- cuda_utils.def("get_device_attribute", &get_device_attribute);
327
- cuda_utils.impl("get_device_attribute", torch::kCUDA, &get_device_attribute);
384
+ cuda_utils.def("get_device_attribute(int attribute, int device_id) -> int");
385
+ cuda_utils.impl("get_device_attribute", &get_device_attribute);
386
387
// Gets the maximum shared memory per block device attribute.
330
- cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
331
- &get_max_shared_memory_per_block_device_attribute);
388
+ cuda_utils.def(
389
+ "get_max_shared_memory_per_block_device_attribute(int device_id) -> int");
390
cuda_utils.impl("get_max_shared_memory_per_block_device_attribute",
333
- torch::kCUDA,
391
&get_max_shared_memory_per_block_device_attribute);
392
}
393
394
#ifndef USE_ROCM
395
TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
396
// Custom all-reduce kernels
340
- custom_ar.def("init_custom_ar", &init_custom_ar);
397
+ custom_ar.def(
398
+ "init_custom_ar(Tensor meta, Tensor rank_data, "
399
+ "str[] handles, int[] offsets, int rank, "
400
+ "bool full_nvlink) -> int");
401
custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
402
343
- custom_ar.def("should_custom_ar", &should_custom_ar);
403
+ custom_ar.def(
404
+ "should_custom_ar(Tensor inp, int max_size, int world_size, "
405
+ "bool full_nvlink) -> bool");
406
custom_ar.impl("should_custom_ar", torch::kCUDA, &should_custom_ar);
407
408
custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
@@ -352,21 +414,15 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
414
custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
415
416
custom_ar.def("dispose", &dispose);
355
- custom_ar.impl("dispose", torch::kCPU, &dispose);
356
-
417
custom_ar.def("meta_size", &meta_size);
358
- custom_ar.impl("meta_size", torch::kCPU, &meta_size);
418
360
- custom_ar.def("register_buffer", ®ister_buffer);
419
+ custom_ar.def(
420
+ "register_buffer(int fa, Tensor t, str[] handles, "
421
+ "int[] offsets) -> ()");
422
custom_ar.impl("register_buffer", torch::kCUDA, ®ister_buffer);
423
424
custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
364
- custom_ar.impl("get_graph_buffer_ipc_meta", torch::kCPU,
365
- &get_graph_buffer_ipc_meta);
366
-
425
custom_ar.def("register_graph_buffers", ®ister_graph_buffers);
368
- custom_ar.impl("register_graph_buffers", torch::kCPU,
369
- ®ister_graph_buffers);
426
}
427
#endif
428
tests/kernels/test_activation.py
+19
-3
@@ -3,8 +3,10 @@ from typing import Type
3
import pytest
4
import torch
5
6
+from tests.kernels.utils import opcheck
7
from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
7
- NewGELU, SiluAndMul)
8
+ NewGELU, QuickGELU,
9
+ SiluAndMul)
10
11
from .allclose_default import get_default_atol, get_default_rtol
12
@@ -39,18 +41,28 @@ def test_act_and_mul(
41
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
42
if activation == "silu":
43
layer = SiluAndMul()
44
+ fn = torch.ops._C.silu_and_mul
45
elif activation == "gelu":
46
layer = GeluAndMul(approximate="none")
47
+ fn = torch.ops._C.gelu_and_mul
48
elif activation == "gelu_tanh":
49
layer = GeluAndMul(approximate="tanh")
50
+ fn = torch.ops._C.gelu_tanh_and_mul
51
out = layer(x)
52
ref_out = layer.forward_native(x)
53
# The SiLU and GELU implementations are equivalent to the native PyTorch
54
# implementations, so we can do exact comparison.
55
torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
56
57
+ d = x.shape[-1] // 2
58
+ output_shape = (x.shape[:-1] + (d, ))
59
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
60
+ opcheck(fn, (out, x))
61
53
-@pytest.mark.parametrize("activation", [FastGELU, NewGELU])
62
+
63
+@pytest.mark.parametrize("activation", [(FastGELU, torch.ops._C.gelu_fast),
64
+ (NewGELU, torch.ops._C.gelu_new),
65
+ (QuickGELU, torch.ops._C.gelu_quick)])
66
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
67
@pytest.mark.parametrize("d", D)
68
@pytest.mark.parametrize("dtype", DTYPES)
@@ -70,10 +82,14 @@ def test_activation(
82
torch.cuda.manual_seed(seed)
83
torch.set_default_device(device)
84
x = torch.randn(num_tokens, d, dtype=dtype)
73
- layer = activation()
85
+ layer = activation[0]()
86
+ fn = activation[1]
87
out = layer(x)
88
ref_out = layer.forward_native(x)
89
torch.testing.assert_close(out,
90
ref_out,
91
atol=get_default_atol(out),
92
rtol=get_default_rtol(out))
93
+
94
+ out = torch.empty_like(x)
95
+ opcheck(fn, (out, x))
tests/kernels/test_attention.py
+16
@@ -6,6 +6,7 @@ import torch
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9
+from tests.kernels.utils import opcheck
10
from vllm import _custom_ops as ops
11
from vllm.utils import get_max_shared_memory_bytes, is_hip
12
@@ -198,6 +199,13 @@ def test_paged_attention(
199
k_scale,
200
v_scale,
201
)
202
+
203
+ opcheck(torch.ops._C.paged_attention_v1,
204
+ (output, query, key_cache, value_cache, num_kv_heads, scale,
205
+ block_tables, seq_lens, block_size, max_seq_len, alibi_slopes,
206
+ kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
207
+ cond=(head_size == HEAD_SIZES[0]))
208
+
209
elif version == "v2":
210
num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
211
assert PARTITION_SIZE % block_size == 0
@@ -230,6 +238,14 @@ def test_paged_attention(
238
k_scale,
239
v_scale,
240
)
241
+
242
+ opcheck(torch.ops._C.paged_attention_v2,
243
+ (output, exp_sums, max_logits, tmp_output, query, key_cache,
244
+ value_cache, num_kv_heads, scale, block_tables, seq_lens,
245
+ block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
246
+ k_scale, v_scale, 0, 0, 0, 64, 0),
247
+ cond=(head_size == HEAD_SIZES[0]))
248
+
249
else:
250
raise AssertionError(f"Unknown version: {version}")
251
tests/kernels/test_cache.py
+22
@@ -4,6 +4,7 @@ from typing import List, Tuple
4
import pytest
5
import torch
6
7
+from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
8
from vllm import _custom_ops as ops
9
10
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
@@ -87,6 +88,11 @@ def test_copy_blocks(
88
block_mapping_tensor = torch.tensor(block_mapping,
89
dtype=torch.int64,
90
device=device).view(-1, 2)
91
+
92
+ opcheck(torch.ops._C_cache_ops.copy_blocks,
93
+ (key_caches, value_caches, block_mapping_tensor),
94
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS,
95
+ cond=(head_size == HEAD_SIZES[0]))
96
ops.copy_blocks(key_caches, value_caches, block_mapping_tensor)
97
98
# Run the reference implementation.
@@ -162,6 +168,10 @@ def test_reshape_and_cache(
168
k_scale = v_scale = 1.0
169
170
# Call the reshape_and_cache kernel.
171
+ opcheck(torch.ops._C_cache_ops.reshape_and_cache,
172
+ (key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
173
+ k_scale, v_scale),
174
+ cond=(head_size == HEAD_SIZES[0]))
175
ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping,
176
kv_cache_dtype, k_scale, v_scale)
177
@@ -269,6 +279,10 @@ def test_reshape_and_cache_flash(
279
k_scale = v_scale = 1.0
280
281
# Call the reshape_and_cache kernel.
282
+ opcheck(torch.ops._C_cache_ops.reshape_and_cache_flash,
283
+ (key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
284
+ k_scale, v_scale),
285
+ cond=(head_size == HEAD_SIZES[0]))
286
ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
287
slot_mapping, kv_cache_dtype, k_scale, v_scale)
288
@@ -366,6 +380,14 @@ def test_swap_blocks(
380
src_value_caches_clone = src_value_caches[0].clone()
381
382
# Call the swap_blocks kernel.
383
+ do_opcheck = (head_size == HEAD_SIZES[0])
384
+ opcheck(torch.ops._C_cache_ops.swap_blocks,
385
+ (src_key_caches[0], dist_key_caches[0], block_mapping_tensor),
386
+ cond=do_opcheck)
387
+ opcheck(torch.ops._C_cache_ops.swap_blocks,
388
+ (src_value_caches[0], dist_value_caches[0], block_mapping_tensor),
389
+ cond=do_opcheck)
390
+
391
ops.swap_blocks(src_key_caches[0], dist_key_caches[0],
392
block_mapping_tensor)
393
ops.swap_blocks(src_value_caches[0], dist_value_caches[0],
tests/kernels/test_cutlass.py
+13
@@ -7,6 +7,7 @@ from typing import Optional, Type
7
import pytest
8
import torch
9
10
+from tests.kernels.utils import opcheck
11
from vllm import _custom_ops as ops
12
from vllm.platforms import current_platform
13
@@ -108,6 +109,9 @@ def cutlass_int8_gemm_helper(m: int,
109
110
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
111
112
+ opcheck(torch.ops._C.cutlass_scaled_mm,
113
+ (out, a, b, scale_a, scale_b, bias))
114
+
115
116
@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
117
@pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
@@ -341,6 +345,15 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
345
torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
346
torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
347
348
+ if azp_per_token:
349
+ opcheck(torch.ops._C.cutlass_scaled_mm_azp,
350
+ (out, aq_i8, bq_i8, scale_a, scale_b, azp_adj_i32, azp_i32,
351
+ func_bias))
352
+ else:
353
+ opcheck(torch.ops._C.cutlass_scaled_mm_azp,
354
+ (out, aq_i8, bq_i8, scale_a, scale_b, azp_with_adj_i32, None,
355
+ func_bias))
356
+
357
358
# Test working with a subset of A and B
359
def test_cutlass_subset():
tests/kernels/test_int8_quant.py
+15
@@ -2,6 +2,7 @@ import pytest
2
import torch
3
4
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5
+from tests.kernels.utils import opcheck
6
from vllm._custom_ops import scaled_int8_quant
7
8
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -12,6 +13,16 @@ SEEDS = [0]
13
SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
14
15
16
+def opcheck_int8_quant(output, input, scale=None):
17
+ if scale is not None:
18
+ opcheck(torch.ops._C.static_scaled_int8_quant, (output, input, scale))
19
+ else:
20
+ scale = torch.empty((input.numel() // input.shape[-1], 1),
21
+ device=input.device,
22
+ dtype=torch.float32)
23
+ opcheck(torch.ops._C.dynamic_scaled_int8_quant, (output, input, scale))
24
+
25
+
26
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
27
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
28
@pytest.mark.parametrize("dtype", DTYPES)
@@ -34,6 +45,8 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
45
ops_out, ref_out, atol=1,
46
rtol=0.0) # big atol to account for rounding errors
47
48
+ opcheck_int8_quant(ops_out, x)
49
+
50
51
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
52
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@@ -58,3 +71,5 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
71
torch.testing.assert_close(
72
out1, out2, atol=1,
73
rtol=0.0) # big atol to account for rounding errors
74
+
75
+ opcheck_int8_quant(out2, x, scale)
tests/kernels/test_layernorm.py
+8
@@ -1,6 +1,7 @@
1
import pytest
2
import torch
3
4
+from tests.kernels.utils import opcheck
5
from vllm.model_executor.layers.layernorm import RMSNorm
6
7
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -52,3 +53,10 @@ def test_rms_norm(
53
torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
54
else:
55
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
56
+
57
+ if residual is not None:
58
+ opcheck(torch.ops._C.fused_add_rms_norm,
59
+ (x, residual, layer.weight.data, layer.variance_epsilon))
60
+ else:
61
+ opcheck(torch.ops._C.rms_norm,
62
+ (out, x, layer.weight.data, layer.variance_epsilon))
tests/kernels/test_machete_gemm.py
+7
@@ -9,6 +9,7 @@ from typing import Optional, Tuple
9
import pytest
10
import torch
11
12
+from tests.kernels.utils import opcheck
13
from vllm import _custom_ops as ops
14
from vllm.model_executor.layers.quantization.utils.quant_utils import (
15
pack_rows, quantize_weights)
@@ -76,6 +77,8 @@ def machete_quantize_and_pack(w: torch.Tensor,
77
w_q = w_q.t().contiguous().t() # convert to col major
78
w_q_machete = ops.machete_prepack_B(w_q, wtype)
79
80
+ opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype))
81
+
82
return w_ref, w_q_machete, w_s, w_zp
83
84
@@ -146,6 +149,10 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
149
schedule=schedule,
150
)
151
152
+ opcheck(torch.ops._C.machete_gemm,
153
+ (a, w_q_machete, wtype, w_s, maybe_convert_zeropoints(
154
+ w_zp, w_s), group_size, None, None, None, schedule))
155
+
156
# Relax atol as our reduction dim becomes larger (more rounding error)
157
# Relax atol when we have zeropoints since the way machete applies
158
# zeropoints (after scales) causes noise around 0
tests/kernels/test_marlin_gemm.py
+29
-26
@@ -5,6 +5,7 @@ Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
5
import pytest
6
import torch
7
8
+from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
9
from tests.quantization.utils import is_quant_method_supported
10
from vllm import _custom_ops as ops
11
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
@@ -73,12 +74,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
74
act_order, mnk_factors):
75
m_factor, n_factor, k_factor = mnk_factors
76
76
- size_m = m_factor
77
size_k = k_chunk * k_factor
78
size_n = n_chunk * n_factor
79
80
- print(f"MNK = {size_m} {size_n} {size_k}")
81
-
80
# Filter act_order
81
if act_order:
82
if group_size == -1:
@@ -112,6 +110,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
110
marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
111
weight_perm)
112
113
+ opcheck(torch.ops._C.gptq_marlin_repack,
114
+ (q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits))
115
+
116
# Run Marlin repack GPU kernel
117
marlin_q_w_2 = ops.gptq_marlin_repack(
118
q_w_gptq,
@@ -137,12 +138,9 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
138
mnk_factors):
139
m_factor, n_factor, k_factor = mnk_factors
140
140
- size_m = m_factor
141
size_k = k_chunk * k_factor
142
size_n = n_chunk * n_factor
143
144
- print(f"MNK = {size_m} {size_n} {size_k}")
145
-
144
# Normalize group_size
145
if group_size == -1:
146
group_size = size_k
@@ -165,6 +163,9 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
163
marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
164
weight_perm)
165
166
+ opcheck(torch.ops._C.awq_marlin_repack,
167
+ (q_w_awq, size_k, size_n, quant_type.size_bits))
168
+
169
# Run Marlin repack GPU kernel
170
marlin_q_w_2 = ops.awq_marlin_repack(
171
q_w_awq,
@@ -204,9 +205,6 @@ def test_gptq_marlin_gemm(
205
size_k = k_chunk * k_factor
206
size_n = n_chunk * n_factor
207
207
- print(f"MNK = {size_m} {size_n} {size_k}")
208
- print(f"groupsize = {group_size}")
209
-
208
if act_order:
209
if group_size == -1:
210
return
@@ -224,6 +222,13 @@ def test_gptq_marlin_gemm(
222
workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
223
GPTQ_MARLIN_MAX_PARALLEL)
224
225
+ opcheck(
226
+ torch.ops._C.gptq_marlin_gemm,
227
+ (a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
228
+ workspace.scratch, quant_type, a_input.shape[0], b_weight.shape[1],
229
+ a_input.shape[1], is_k_full, False, use_fp32_reduce),
230
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS)
231
+
232
output = ops.gptq_marlin_gemm(
233
a_input,
234
marlin_q_w,
@@ -245,7 +250,6 @@ def test_gptq_marlin_gemm(
250
torch.cuda.synchronize()
251
252
max_diff = compute_max_diff(output, output_ref)
248
- print("max_diff = {}".format(max_diff))
253
254
assert max_diff < 0.04
255
@@ -265,9 +269,6 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
269
size_k = k_chunk * k_factor
270
size_n = n_chunk * n_factor
271
268
- print(f"MNK = {size_m} {size_n} {size_k}")
269
- print(f"groupsize = {group_size}")
270
-
272
a_input = rand_data((size_m, size_k))
273
b_weight = rand_data((size_k, size_n))
274
@@ -279,6 +280,12 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
280
281
output_ref = torch.matmul(a_input, w_24_ref)
282
283
+ opcheck(torch.ops._C.gptq_marlin_24_gemm,
284
+ (a_input, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s,
285
+ workspace_24.scratch, quant_type, a_input.shape[0],
286
+ b_weight.shape[1], a_input.shape[1]),
287
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS)
288
+
289
output = ops.gptq_marlin_24_gemm(
290
a_input,
291
marlin_24_q_w_comp,
@@ -294,7 +301,6 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
301
torch.cuda.synchronize()
302
303
max_diff = compute_max_diff(output, output_ref)
297
- print("max_diff = {}".format(max_diff))
304
305
assert max_diff < 0.04
306
@@ -321,9 +327,6 @@ def test_fp8_marlin_gemm(
327
size_k = k_chunk * k_factor
328
size_n = n_chunk * n_factor
329
324
- print(f"MNK = {size_m} {size_n} {size_k}")
325
- print(f"groupsize = {group_size}")
326
-
330
a_input = rand_data((size_m, size_k), dtype=dtype)
331
b_weight = rand_data((size_k, size_n), dtype=dtype)
332
@@ -353,6 +356,10 @@ def test_fp8_marlin_gemm(
356
workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
357
GPTQ_MARLIN_MAX_PARALLEL)
358
359
+ opcheck(torch.ops._C.fp8_marlin_gemm,
360
+ (a_input, marlin_qweight, marlin_scales, workspace.scratch,
361
+ num_bits, a_input.shape[0], b_weight.shape[1], a_input.shape[1]))
362
+
363
output = ops.fp8_marlin_gemm(
364
a=a_input,
365
b_q_weight=marlin_qweight,
@@ -368,7 +375,6 @@ def test_fp8_marlin_gemm(
375
torch.cuda.synchronize()
376
377
max_diff = compute_max_diff(output, output_ref)
371
- print("max_diff = {}".format(max_diff))
378
379
assert max_diff < 0.04
380
@@ -396,9 +402,6 @@ def test_awq_marlin_gemm(
402
size_k = k_chunk * k_factor
403
size_n = n_chunk * n_factor
404
399
- print(f"MNK = {size_m} {size_n} {size_k}")
400
- print(f"groupsize = {group_size}")
401
-
405
a_input = rand_data((size_m, size_k))
406
b_weight = rand_data((size_k, size_n))
407
@@ -434,7 +437,6 @@ def test_awq_marlin_gemm(
437
torch.cuda.synchronize()
438
439
max_diff = compute_max_diff(output, output_ref)
437
- print("max_diff = {}".format(max_diff))
440
441
assert max_diff < 0.04
442
@@ -460,9 +462,6 @@ def test_marlin_qqq_gemm(
462
size_k = k_chunk * k_factor
463
size_n = n_chunk * n_factor
464
463
- print(f"MNK = {size_m} {size_n} {size_k}")
464
- print(f"groupsize = {group_size}")
465
-
465
a_input = rand_data((size_m, size_k))
466
b_weight = rand_data((size_k, size_n))
467
@@ -479,6 +478,11 @@ def test_marlin_qqq_gemm(
478
workspace = MarlinWorkspace(size_n, MARLIN_QQQ_MIN_THREAD_N,
479
MARLIN_QQQ_MAX_PARALLEL)
480
481
+ opcheck(torch.ops._C.marlin_qqq_gemm,
482
+ (q_a, marlin_qqq_q_w, s_a, marlin_qqq_s_channel,
483
+ marlin_qqq_s_group, workspace.scratch, a_input.shape[0],
484
+ b_weight.shape[1], a_input.shape[1]))
485
+
486
output = ops.marlin_qqq_gemm(
487
q_a,
488
marlin_qqq_q_w,
@@ -495,6 +499,5 @@ def test_marlin_qqq_gemm(
499
torch.cuda.synchronize()
500
501
max_diff = compute_max_diff(output, output_ref)
498
- print("max_diff = {}".format(max_diff))
502
503
assert max_diff < 0.04
tests/kernels/utils.py
+33
-1
@@ -3,7 +3,8 @@
3
import itertools
4
import random
5
from numbers import Number
6
-from typing import Any, List, NamedTuple, Optional, Tuple, Union
6
+from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
7
+ Union)
8
9
import pytest
10
import torch
@@ -13,6 +14,21 @@ from vllm.attention.backends.xformers import XFormersBackend
14
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
15
make_tensor_with_pad)
16
17
+# For now, disable "test_aot_dispatch_dynamic" since there are some
18
+# bugs related to this test in PyTorch 2.4.
19
+DEFAULT_OPCHECK_TEST_UTILS: Tuple[str, ...] = (
20
+ "test_schema",
21
+ "test_autograd_registration",
22
+ "test_faketensor",
23
+)
24
+
25
+ALL_OPCHECK_TEST_UTILS: Tuple[str, ...] = (
26
+ "test_schema",
27
+ "test_autograd_registration",
28
+ "test_faketensor",
29
+ "test_aot_dispatch_dynamic",
30
+)
31
+
32
33
class QKVInputs(NamedTuple):
34
'''
@@ -926,3 +942,19 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
942
ideal_output = test_params.packed_qkvo.ideal_output
943
torch.testing.assert_close(ideal_output,
944
output_under_test.view_as(ideal_output))
945
+
946
+
947
+def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
948
+ torch._library.custom_ops.CustomOpDef],
949
+ args: Tuple[Any, ...],
950
+ kwargs: Optional[Dict[str, Any]] = None,
951
+ *,
952
+ test_utils: Union[str, Sequence[str]] = ALL_OPCHECK_TEST_UTILS,
953
+ raise_exception: bool = True,
954
+ cond: bool = True) -> Dict[str, str]:
955
+ return torch.library.opcheck(
956
+ op,
957
+ args,
958
+ kwargs,
959
+ test_utils=test_utils,
960
+ raise_exception=raise_exception) if cond else {}
tests/models/test_aqlm.py
-20
@@ -7,26 +7,6 @@ import pytest
7
8
from tests.quantization.utils import is_quant_method_supported
9
10
-# In this test we hardcode prompts and generations for the model so we don't
11
-# need to require the AQLM package as a dependency
12
-example_prompts = [
13
- 'vLLM is a high-throughput and memory-efficient inference and serving '
14
- 'engine for LLMs.\n',
15
- 'Briefly describe the major milestones in the development of artificial '
16
- 'intelligence from 1950 to 2020.\n',
17
- 'Compare and contrast artificial intelligence with human intelligence in '
18
- 'terms of processing information.\n',
19
- 'Describe the basic components of a neural network and how it can be '
20
- 'trained.\n',
21
- 'Write a short story about a robot that dreams for the first time.\n',
22
- 'Analyze the impact of the COVID-19 pandemic on global economic structures '
23
- 'and future business models.\n',
24
- 'Explain the cultural significance of the Mona Lisa painting, and how its '
25
- 'perception might vary in Western versus Eastern societies.\n',
26
- "Translate the following English sentence into Japanese, French, and "
27
- "Swahili: 'The early bird catches the worm.'\n"
28
-]
29
-
10
# These ground truth generations were generated using `transformers==4.38.1
11
# aqlm==1.1.0 torch==2.2.0`
12
# and the below code:
vllm/_custom_ops.py
+204
@@ -204,6 +204,22 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
204
b_g_idx, use_exllama, bit)
205
206
207
+# TODO: has to be a better way to do this
208
+try:
209
+ torch.ops._C.gptq_gemm # noqa B018
210
+
211
+ @torch.library.register_fake("_C::gptq_gemm")
212
+ def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
213
+ b_gptq_qzeros: torch.Tensor,
214
+ b_gptq_scales: torch.Tensor, b_g_idx: torch.Tensor,
215
+ use_exllama: bool, bit: int) -> torch.Tensor:
216
+ return torch.empty((a.size(0), b_q_weight.size(1)),
217
+ dtype=a.dtype,
218
+ device=a.device)
219
+except Exception:
220
+ pass
221
+
222
+
223
def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
224
bit: int) -> None:
225
torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
@@ -227,6 +243,194 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
243
size_n, size_k)
244
245
246
+# TODO: has to be a better way to do this
247
+try:
248
+ torch.ops._C.gptq_marlin_24_gemm # noqa B018
249
+
250
+ @torch.library.register_fake("_C::gptq_marlin_24_gemm")
251
+ def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
252
+ b_meta: torch.Tensor, b_scales: torch.Tensor,
253
+ workspace: torch.Tensor,
254
+ b_q_type: ScalarType, size_m: int,
255
+ size_n: int, size_k: int) -> torch.Tensor:
256
+ return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
257
+
258
+ @torch.library.register_fake("_C::gptq_marlin_gemm")
259
+ def _gptq_marlin_gemm_fake(a: torch.Tensor,
260
+ b_q_weight: torch.Tensor,
261
+ b_scales: torch.Tensor,
262
+ b_zeros: torch.Tensor,
263
+ g_idx: torch.Tensor,
264
+ perm: torch.Tensor,
265
+ workspace: torch.Tensor,
266
+ b_q_type: ScalarType,
267
+ size_m: int,
268
+ size_n: int,
269
+ size_k: int,
270
+ is_k_full: bool,
271
+ has_zp: bool = False,
272
+ use_fp32_reduce: bool = False) -> torch.Tensor:
273
+ return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
274
+
275
+ @torch.library.register_fake("_C::ggml_dequantize")
276
+ def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
277
+ n: int) -> torch.Tensor:
278
+ return torch.empty((m, n), dtype=torch.float16, device=W.device)
279
+
280
+ @torch.library.register_fake("_C::ggml_mul_mat_vec_a8")
281
+ def _ggml_mul_mat_vec_a8_fake(
282
+ W: torch.Tensor,
283
+ X: torch.Tensor,
284
+ quant_type: int,
285
+ row: int,
286
+ ) -> torch.Tensor:
287
+ return torch.empty((1, row), dtype=torch.float16, device=W.device)
288
+
289
+ @torch.library.register_fake("_C::ggml_mul_mat_a8")
290
+ def _ggml_mul_mat_a8_fake(
291
+ W: torch.Tensor,
292
+ X: torch.Tensor,
293
+ quant_type: int,
294
+ row: int,
295
+ ) -> torch.Tensor:
296
+ batch = X.size(0)
297
+ return torch.empty((batch, row), dtype=torch.float16, device=W.device)
298
+
299
+ @torch.library.register_fake("_C::marlin_qqq_gemm")
300
+ def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
301
+ s_tok: torch.Tensor, s_ch: torch.Tensor,
302
+ s_group: torch.Tensor, workspace: torch.Tensor,
303
+ size_m: int, size_n: int,
304
+ size_k: int) -> torch.Tensor:
305
+ return torch.empty((size_m, size_n),
306
+ dtype=torch.float16,
307
+ device=a.device)
308
+
309
+ @torch.library.register_fake("_C::marlin_gemm")
310
+ def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
311
+ b_scales: torch.Tensor, workspace: torch.Tensor,
312
+ size_m: int, size_n: int,
313
+ size_k: int) -> torch.Tensor:
314
+ return torch.empty((size_m, size_n),
315
+ dtype=torch.float16,
316
+ device=a.device)
317
+
318
+ @torch.library.register_fake("_C::awq_dequantize")
319
+ def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
320
+ zeros: torch.Tensor, split_k_iters: int, thx: int,
321
+ thy: int) -> torch.Tensor:
322
+ in_c = qweight.size(0)
323
+ qout_c = qweight.size(1)
324
+ out_c = qout_c * 8
325
+ return torch.empty((in_c, out_c),
326
+ dtype=scales.dtype,
327
+ device=scales.device)
328
+
329
+ @torch.library.register_fake("_C::awq_gemm")
330
+ def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
331
+ qzeros: torch.Tensor, scales: torch.Tensor,
332
+ split_k_iters: int) -> torch.Tensor:
333
+ num_in_feats = input.size(0)
334
+ return torch.empty((split_k_iters, num_in_feats, qweight.size(1) * 8),
335
+ dtype=input.dtype,
336
+ device=input.device).sum(0)
337
+
338
+ @torch.library.register_fake("_C::aqlm_gemm")
339
+ def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor,
340
+ codebooks: torch.Tensor, scales: torch.Tensor,
341
+ codebook_partition_sizes: List[int],
342
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
343
+ out_features = codes.size(0) * codebooks.size(2)
344
+ flat_input = input.reshape((-1, input.size(-1)))
345
+ flat_output = torch.empty((flat_input.size(0), out_features),
346
+ dtype=input.dtype,
347
+ device=input.device)
348
+
349
+ output_sizes = list(input.shape)
350
+ output_sizes.pop()
351
+ output_sizes.append(-1)
352
+ return flat_output.reshape(tuple(output_sizes))
353
+
354
+ @torch.library.register_fake("_C::aqlm_dequant")
355
+ def _aqlm_dequant_fake(
356
+ codes: torch.Tensor, codebooks: torch.Tensor,
357
+ codebook_partition_sizes: List[int]) -> torch.Tensor:
358
+ in_features = codes.size(1) * 8
359
+ out_features = codes.size(0)
360
+ return torch.empty((out_features, in_features),
361
+ dtype=codebooks.dtype,
362
+ device=codebooks.device)
363
+
364
+ @torch.library.register_fake("_C::fp8_marlin_gemm")
365
+ def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
366
+ b_scales: torch.Tensor, workspace: torch.Tensor,
367
+ num_bits: int, size_m: int, size_n: int,
368
+ size_k: int) -> torch.Tensor:
369
+ return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
370
+
371
+ @torch.library.register_fake("_C::machete_gemm")
372
+ def machete_gemm_fake(
373
+ a: torch.Tensor,
374
+ b_q: torch.
375
+ Tensor, # Should be the tensor returned by machete_prepack_B
376
+ b_type: ScalarType,
377
+ b_scales: Optional[torch.Tensor] = None,
378
+ b_zeros: Optional[torch.Tensor] = None,
379
+ b_group_size: Optional[int] = None,
380
+ c: Optional[torch.Tensor] = None,
381
+ alpha: Optional[float] = None,
382
+ beta: Optional[float] = None,
383
+ schedule: Optional[str] = None,
384
+ ) -> torch.Tensor:
385
+ m = a.size(0)
386
+ n = b_q.size(1)
387
+ return torch.empty((m, n), device=a.device, dtype=a.dtype)
388
+
389
+ @torch.library.register_fake("_C::machete_prepack_B")
390
+ def machete_prepack_B_fake(b_q_weight: torch.Tensor,
391
+ b_type: ScalarType) -> torch.Tensor:
392
+ return torch.empty_like(b_q_weight)
393
+
394
+ @torch.library.register_fake("_C::causal_conv1d_fwd")
395
+ def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
396
+ bias_: Optional[torch.Tensor],
397
+ seq_idx_: Optional[torch.Tensor],
398
+ initial_states_: Optional[torch.Tensor],
399
+ final_states_out_: Optional[torch.Tensor],
400
+ silu_activation: bool) -> torch.Tensor:
401
+ return torch.empty_like(x)
402
+
403
+ @torch.library.register_fake("_C::causal_conv1d_update")
404
+ def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
405
+ weight: torch.Tensor,
406
+ bias_: Optional[torch.Tensor],
407
+ silu_activation: bool) -> torch.Tensor:
408
+ return torch.empty_like(x)
409
+
410
+ @torch.library.register_fake("_C::selective_scan_fwd")
411
+ def selective_scan_fwd_fake(
412
+ u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
413
+ B: torch.Tensor, C: torch.Tensor, D_: Optional[torch.Tensor],
414
+ z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
415
+ delta_softplus: bool, index_: Optional[torch.Tensor],
416
+ x: Optional[torch.Tensor]) -> List[torch.Tensor]:
417
+ a = torch.empty_like(u)
418
+ if x is not None:
419
+ b = x
420
+ else:
421
+ b = torch.empty((u.size(0), u.size(1), A.size(1)),
422
+ dtype=u.dtype,
423
+ device=u.device)
424
+ if z_ is not None:
425
+ c = torch.empty_like(z_)
426
+ return [a, b, c]
427
+ else:
428
+ return [a, b]
429
+
430
+except Exception:
431
+ pass
432
+
433
+
434
# cutlass
435
def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
436
return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
vllm/envs.py
+5
@@ -203,6 +203,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
203
(os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
204
("true", "1")),
205
206
+ # Internal flag to enable Dynamo fullgraph capture
207
+ "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
208
+ lambda: bool(
209
+ os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
210
+
211
# local rank of the process in the distributed setting, used to determine
212
# the GPU device id
213
"LOCAL_RANK":
vllm/model_executor/models/jamba.py
+1
-1
@@ -733,7 +733,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
733
indices_for_current_run: List[int]):
734
# move out all of the occupied but currently not running blocks
735
# outside of the first n blocks
736
- destination_indices = set([range(batch_size)])
736
+ destination_indices = range(batch_size)
737
max_possible_batch_size = self.mamba_cache[0].shape[1]
738
for destination_index in destination_indices:
739
if destination_index in self._get_all_occupied_indices() and \
vllm/worker/model_runner.py
+8
-3
@@ -75,6 +75,10 @@ _NUM_WARMUP_ITERS = 2
75
76
TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
77
78
+# For now, bump up cache limits for recompilations during CUDA graph warmups.
79
+torch._dynamo.config.cache_size_limit = 128
80
+torch._dynamo.config.accumulated_cache_size_limit = 128
81
+
82
83
@dataclass(frozen=True)
84
class ModelInputForGPU(ModelRunnerInputBase):
@@ -1060,9 +1064,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1064
"This may lead to less accurate results!")
1065
1066
if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1063
- self.model = torch.compile(self.model,
1064
- fullgraph=True,
1065
- backend="eager")
1067
+ self.model = torch.compile(
1068
+ self.model,
1069
+ fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1070
+ backend="eager")
1071
1072
def save_sharded_state(
1073
self,
vllm/worker/worker.py
+1
@@ -166,6 +166,7 @@ class Worker(LocalOrDistributedWorkerBase):
166
torch.cuda.set_device(self.device)
167
168
_check_if_gpu_supports_dtype(self.model_config.dtype)
169
+ gc.collect()
170
torch.cuda.empty_cache()
171
self.init_gpu_memory = torch.cuda.mem_get_info()[0]
172
else: