@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73202dbe77
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Kernel][Misc] register ops to prevent graph breaks (#6917)

Co-authored-by: Sage Moore <sage@neuralmagic.com>

bnellnm committed Sep 11, 2024 at 15:52 UTC 73202dbe77913df9cf520bf18172ac40e0b9951f
22 files changed +528 -102
.github/PULL_REQUEST_TEMPLATE.md
+10
@@ -39,6 +39,16 @@ FIX #xxxx (*link existing issues this PR will resolve*)
39 <li>Please add documentation to <code>docs/source/</code> if the PR modifies the user-facing behaviors of vLLM. It helps vLLM user understand and utilize the new features or changes.</li>
40 </ul>
41
42 +<h3>Adding or changing kernels</h3>
43 +<p>Each custom kernel needs a schema and one or more implementations to be registered with PyTorch.</p>
44 +<ul>
45 + <li>Make sure custom ops are registered following PyTorch guidelines: <a href="https://pytorch.org/tutorials/advanced/cpp_custom_ops.html#cpp-custom-ops-tutorial">Custom C++ and CUDA Operators</a> and <a href="https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU">The Custom Operators Manual</a></li>
46 + <li>Custom operations that return <code>Tensors</code> require meta-functions. Meta-functions should be implemented and registered in python so that dynamic dims can be handled automatically. See above documents for a description of meta-functions.</li>
47 + <li>Use <a href="https://pytorch.org/docs/stable/library.html#torch.library.opcheck"><code>torch.libary.opcheck()</code></a> to test the function registration and meta-function for any registered ops. See <code>tests/kernels</code> for examples.</li>
48 + <li>When changing the C++ signature of an existing op, the schema must be updated to reflect the changes.</li>
49 + <li>If a new custom type is needed, see the following document: <a href="https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA">Custom Class Support in PT2</a>.
50 +</ul>
51 +
52 <h3>Notes for Large Changes</h3>
53 <p>Please keep the changes as concise as possible. For major architectural changes (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue (RFC) discussing the technical design and justification. Otherwise, we will tag it with <code>rfc-required</code> and might not go through the PR.</p>
54
cmake/utils.cmake
+1
@@ -350,6 +350,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
350 target_include_directories(${GPU_MOD_NAME} PRIVATE csrc
351 ${GPU_INCLUDE_DIRECTORIES})
352
353 + # TODO: is torch_python_LIBRARY needed?
354 target_link_libraries(${GPU_MOD_NAME} PRIVATE torch ${torch_python_LIBRARY}
355 ${GPU_LIBRARIES})
356
csrc/cpu/torch_bindings.cpp
+4 -4
@@ -32,8 +32,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
32 // PagedAttention V2.
33 ops.def(
34 "paged_attention_v2("
35 - " Tensor! out, Tensor exp_sums, Tensor max_logits,"
36 - " Tensor tmp_out, Tensor query, Tensor key_cache,"
35 + " Tensor! out, Tensor! exp_sums, Tensor! max_logits,"
36 + " Tensor! tmp_out, Tensor query, Tensor key_cache,"
37 " Tensor value_cache, int num_kv_heads, float scale,"
38 " Tensor block_tables, Tensor seq_lens, int block_size,"
39 " int max_seq_len, Tensor? alibi_slopes,"
@@ -122,8 +122,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
122
123 // Copy the cache blocks from src to dst.
124 cache_ops.def(
125 - "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
126 - "block_mapping) -> ()");
125 + "copy_blocks(Tensor(a!)[] key_caches, Tensor[](b!) value_caches, "
126 + "Tensor block_mapping) -> ()");
127 cache_ops.impl("copy_blocks", torch::kCPU, &copy_blocks);
128
129 // Reshape the key and value tensors and cache them.
csrc/ops.h
+8
@@ -123,9 +123,17 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
123 int64_t size_k, int64_t size_n,
124 int64_t num_bits);
125
126 +torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
127 + torch::Tensor& perm, c10::SymInt size_k,
128 + c10::SymInt size_n, int64_t num_bits);
129 +
130 torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
131 int64_t size_n, int64_t num_bits);
132
133 +torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
134 + c10::SymInt size_k, c10::SymInt size_n,
135 + int64_t num_bits);
136 +
137 torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
138 int64_t n);
139
csrc/quantization/gptq_marlin/awq_marlin_repack.cu
+12
@@ -267,3 +267,15 @@ torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
267 }
268
269 #endif
270 +
271 +torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
272 + c10::SymInt size_k, c10::SymInt size_n,
273 + int64_t num_bits) {
274 + int const pack_factor = 32 / num_bits;
275 + auto options = torch::TensorOptions()
276 + .dtype(b_q_weight.dtype())
277 + .device(b_q_weight.device());
278 + return torch::empty_symint(
279 + {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
280 + options);
281 +}
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+12
@@ -342,3 +342,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
342 }
343
344 #endif
345 +
346 +torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
347 + torch::Tensor& perm, c10::SymInt size_k,
348 + c10::SymInt size_n, int64_t num_bits) {
349 + int const pack_factor = 32 / num_bits;
350 + auto options = torch::TensorOptions()
351 + .dtype(b_q_weight.dtype())
352 + .device(b_q_weight.device());
353 + return torch::empty_symint(
354 + {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
355 + options);
356 +}
csrc/torch_bindings.cpp
+100 -44
@@ -36,8 +36,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
36 // PagedAttention V2.
37 ops.def(
38 "paged_attention_v2("
39 - " Tensor! out, Tensor exp_sums, Tensor max_logits,"
40 - " Tensor tmp_out, Tensor query, Tensor key_cache,"
39 + " Tensor! out, Tensor! exp_sums, Tensor! max_logits,"
40 + " Tensor! tmp_out, Tensor query, Tensor key_cache,"
41 " Tensor value_cache, int num_kv_heads, float scale,"
42 " Tensor block_tables, Tensor seq_lens, int block_size,"
43 " int max_seq_len, Tensor? alibi_slopes,"
@@ -73,7 +73,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
73 ops.impl("gelu_quick", torch::kCUDA, &gelu_quick);
74
75 // prepare_inputs advance_step
76 - ops.def("advance_step", &advance_step);
76 + ops.def(
77 + "advance_step(int num_seqs, int num_queries, int block_size, "
78 + "Tensor! input_tokens, Tensor sampled_token_ids, "
79 + "Tensor! input_positions, Tensor! seq_lens, Tensor! slot_mapping, "
80 + "Tensor block_tables) -> ()");
81 ops.impl("advance_step", torch::kCUDA, &advance_step);
82
83 // Layernorm
@@ -110,27 +114,56 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
114 // Quantization ops
115 #ifndef USE_ROCM
116 // Quantized GEMM for AQLM.
113 - ops.def("aqlm_gemm", &aqlm_gemm);
117 + ops.def(
118 + "aqlm_gemm(Tensor input, Tensor codes, Tensor codebooks, "
119 + "Tensor scales, int[] codebook_partition_sizes, Tensor? bias) "
120 + "-> Tensor");
121 ops.impl("aqlm_gemm", torch::kCUDA, &aqlm_gemm);
122
123 // Decompression method for AQLM.
117 - ops.def("aqlm_dequant", &aqlm_dequant);
124 + ops.def(
125 + "aqlm_dequant(Tensor codes, Tensor codebooks, "
126 + "int[] codebook_partition_sizes) -> Tensor");
127 ops.impl("aqlm_dequant", torch::kCUDA, &aqlm_dequant);
128
129 // Quantized GEMM for AWQ.
121 - ops.def("awq_gemm", &awq_gemm);
130 + ops.def(
131 + "awq_gemm(Tensor _in_feats, Tensor _kernel, Tensor _scaling_factors, "
132 + "Tensor _zeros, int split_k_iters) -> Tensor");
133 ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
134
135 // Dequantization for AWQ.
125 - ops.def("awq_dequantize", &awq_dequantize);
136 + ops.def(
137 + "awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
138 + "Tensor _zeros, int split_k_iters, int thx, int thy) -> Tensor");
139 ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
140
141 + // Note about marlin kernel 'workspace' arguments:
142 + // Technically these should be mutable since they are modified by the kernel.
143 + // But since they are set back to zero once the kernel is finished we can
144 + // hand wave and say that they have no net effect.
145 + //
146 + // The reason to mark 'workspace' as immutable is so that they don't interfere
147 + // with using ScalarType arguments in the ops. If they are marked as mutable,
148 + // pytorch throws an assert in
149 + // 'torch._higher_order_ops._register_effectful_op' that prevents these
150 + // kernels from being torch.compile'd.
151 + // See the following document for more info on custom types and ops that use
152 + // custom types:
153 + // https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
154 +
155 // Marlin (Dense) Optimized Quantized GEMM for GPTQ.
129 - ops.def("marlin_gemm", &marlin_gemm);
156 + ops.def(
157 + "marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
158 + "Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
159 ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
160
161 // Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
133 - ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
162 + ops.def(
163 + "gptq_marlin_24_gemm(Tensor a, Tensor b_q_weight, Tensor b_meta, "
164 + "Tensor b_scales, Tensor workspace, "
165 + "__torch__.torch.classes._core_C.ScalarType b_q_type, "
166 + "int size_m, int size_n, int size_k) -> Tensor");
167 ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
168
169 // Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
@@ -149,35 +182,55 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
182 ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
183
184 // gptq_marlin Optimized Quantized GEMM for GPTQ.
152 - ops.def("gptq_marlin_gemm", &gptq_marlin_gemm);
185 + ops.def(
186 + "gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
187 + "Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
188 + "__torch__.torch.classes._core_C.ScalarType b_q_type, "
189 + "int size_m, int size_n, int size_k, bool is_k_full, "
190 + "bool has_zp, bool use_fp32_reduce) -> Tensor");
191 ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
192
193 // gptq_marlin repack from GPTQ.
156 - ops.def("gptq_marlin_repack", &gptq_marlin_repack);
194 + ops.def(
195 + "gptq_marlin_repack(Tensor b_q_weight, Tensor perm, "
196 + "SymInt size_k, SymInt size_n, int num_bits) -> Tensor");
197 ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
198 + ops.impl("gptq_marlin_repack", torch::kMeta, &gptq_marlin_repack_meta);
199
200 // awq_marlin repack from AWQ.
160 - ops.def("awq_marlin_repack", &awq_marlin_repack);
201 + ops.def(
202 + "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
203 + "SymInt size_n, int num_bits) -> Tensor");
204 ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
205 + ops.impl("awq_marlin_repack", torch::kMeta, &awq_marlin_repack_meta);
206
207 // Dequantization for GGML.
164 - ops.def("ggml_dequantize", &ggml_dequantize);
208 + ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
209 ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
210
211 // mmvq kernel for GGML.
168 - ops.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8);
212 + ops.def(
213 + "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, int row) "
214 + "-> Tensor");
215 ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
216
217 // mmq kernel for GGML.
172 - ops.def("ggml_mul_mat_a8", &ggml_mul_mat_a8);
218 + ops.def("ggml_mul_mat_a8(Tensor W, Tensor X, int type, int row) -> Tensor");
219 ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
220
221 // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
176 - ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
222 + ops.def(
223 + "fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
224 + "Tensor! workspace, int num_bits, int size_m, int size_n, "
225 + "int size_k) -> Tensor");
226 ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
227
228 // marlin_qqq_gemm for QQQ.
180 - ops.def("marlin_qqq_gemm", &marlin_qqq_gemm);
229 + ops.def(
230 + "marlin_qqq_gemm(Tensor a, Tensor b_q_weight, "
231 + "Tensor s_tok, Tensor s_ch, Tensor s_group, "
232 + "Tensor! workspace, int size_m, int size_n, "
233 + "int size_k) -> Tensor");
234 ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
235
236 // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
@@ -199,16 +252,16 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
252
253 // Check if cutlass scaled_mm is supported for CUDA devices of the given
254 // capability
202 - ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
203 - ops.impl("cutlass_scaled_mm_supports_fp8", torch::kCUDA,
204 - &cutlass_scaled_mm_supports_fp8);
255 + ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
256 + ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
257 +
258 // Mamba selective scan kernel
259 ops.def(
260 "selective_scan_fwd(Tensor! u, Tensor! delta,"
261 "Tensor! A, Tensor! B, Tensor! C,"
262 "Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
263 "bool delta_softplus,"
211 - "Tensor? index_, Tensor? x) -> Tensor[]");
264 + "Tensor? index_, Tensor(a! -> *)? x) -> Tensor(a)[]");
265 ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
266
267 ops.def(
@@ -230,7 +283,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
283 #endif
284
285 // Quantized GEMM for GPTQ.
233 - ops.def("gptq_gemm", &gptq_gemm);
286 + // Note: even though the C++ inferred schema is correct for this op, it seems
287 + // to prevent the meta function registry.
288 + ops.def(
289 + "gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, "
290 + "Tensor b_gptq_scales, Tensor b_g_idx, bool use_exllama, int bit) "
291 + "-> Tensor");
292 ops.impl("gptq_gemm", torch::kCUDA, &gptq_gemm);
293
294 // Post processing for GPTQ.
@@ -250,8 +308,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
308
309 // Compute dynamic-per-token FP8 quantized tensor and scaling factor.
310 ops.def(
253 - "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! "
254 - "scale, Tensor? scale_ub) -> "
311 + "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, "
312 + "Tensor! scale, Tensor? scale_ub) -> "
313 "()");
314 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
315 &dynamic_per_token_scaled_fp8_quant);
@@ -288,8 +346,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
346
347 // Copy the cache blocks from src to dst.
348 cache_ops.def(
291 - "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
292 - "block_mapping) -> ()");
349 + "copy_blocks(Tensor(a!)[] key_caches, Tensor[](b!) value_caches, "
350 + "Tensor block_mapping) -> ()");
351 cache_ops.impl("copy_blocks", torch::kCUDA, &copy_blocks);
352
353 // Reshape the key and value tensors and cache them.
@@ -314,8 +372,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
372
373 // Convert the key and value cache to fp8 data type.
374 cache_ops.def(
317 - "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, str "
318 - "kv_cache_dtype) -> ()");
375 + "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, "
376 + "str kv_cache_dtype) -> ()");
377 cache_ops.impl("convert_fp8", torch::kCUDA, &convert_fp8);
378 }
379
@@ -323,24 +381,28 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
381 // Cuda utils
382
383 // Gets the specified device attribute.
326 - cuda_utils.def("get_device_attribute", &get_device_attribute);
327 - cuda_utils.impl("get_device_attribute", torch::kCUDA, &get_device_attribute);
384 + cuda_utils.def("get_device_attribute(int attribute, int device_id) -> int");
385 + cuda_utils.impl("get_device_attribute", &get_device_attribute);
386
387 // Gets the maximum shared memory per block device attribute.
330 - cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
331 - &get_max_shared_memory_per_block_device_attribute);
388 + cuda_utils.def(
389 + "get_max_shared_memory_per_block_device_attribute(int device_id) -> int");
390 cuda_utils.impl("get_max_shared_memory_per_block_device_attribute",
333 - torch::kCUDA,
391 &get_max_shared_memory_per_block_device_attribute);
392 }
393
394 #ifndef USE_ROCM
395 TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
396 // Custom all-reduce kernels
340 - custom_ar.def("init_custom_ar", &init_custom_ar);
397 + custom_ar.def(
398 + "init_custom_ar(Tensor meta, Tensor rank_data, "
399 + "str[] handles, int[] offsets, int rank, "
400 + "bool full_nvlink) -> int");
401 custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
402
343 - custom_ar.def("should_custom_ar", &should_custom_ar);
403 + custom_ar.def(
404 + "should_custom_ar(Tensor inp, int max_size, int world_size, "
405 + "bool full_nvlink) -> bool");
406 custom_ar.impl("should_custom_ar", torch::kCUDA, &should_custom_ar);
407
408 custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
@@ -352,21 +414,15 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
414 custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
415
416 custom_ar.def("dispose", &dispose);
355 - custom_ar.impl("dispose", torch::kCPU, &dispose);
356 -
417 custom_ar.def("meta_size", &meta_size);
358 - custom_ar.impl("meta_size", torch::kCPU, &meta_size);
418
360 - custom_ar.def("register_buffer", &register_buffer);
419 + custom_ar.def(
420 + "register_buffer(int fa, Tensor t, str[] handles, "
421 + "int[] offsets) -> ()");
422 custom_ar.impl("register_buffer", torch::kCUDA, &register_buffer);
423
424 custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
364 - custom_ar.impl("get_graph_buffer_ipc_meta", torch::kCPU,
365 - &get_graph_buffer_ipc_meta);
366 -
425 custom_ar.def("register_graph_buffers", &register_graph_buffers);
368 - custom_ar.impl("register_graph_buffers", torch::kCPU,
369 - &register_graph_buffers);
426 }
427 #endif
428
tests/kernels/test_activation.py
+19 -3
@@ -3,8 +3,10 @@ from typing import Type
3 import pytest
4 import torch
5
6 +from tests.kernels.utils import opcheck
7 from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
7 - NewGELU, SiluAndMul)
8 + NewGELU, QuickGELU,
9 + SiluAndMul)
10
11 from .allclose_default import get_default_atol, get_default_rtol
12
@@ -39,18 +41,28 @@ def test_act_and_mul(
41 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
42 if activation == "silu":
43 layer = SiluAndMul()
44 + fn = torch.ops._C.silu_and_mul
45 elif activation == "gelu":
46 layer = GeluAndMul(approximate="none")
47 + fn = torch.ops._C.gelu_and_mul
48 elif activation == "gelu_tanh":
49 layer = GeluAndMul(approximate="tanh")
50 + fn = torch.ops._C.gelu_tanh_and_mul
51 out = layer(x)
52 ref_out = layer.forward_native(x)
53 # The SiLU and GELU implementations are equivalent to the native PyTorch
54 # implementations, so we can do exact comparison.
55 torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
56
57 + d = x.shape[-1] // 2
58 + output_shape = (x.shape[:-1] + (d, ))
59 + out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
60 + opcheck(fn, (out, x))
61
53 -@pytest.mark.parametrize("activation", [FastGELU, NewGELU])
62 +
63 +@pytest.mark.parametrize("activation", [(FastGELU, torch.ops._C.gelu_fast),
64 + (NewGELU, torch.ops._C.gelu_new),
65 + (QuickGELU, torch.ops._C.gelu_quick)])
66 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
67 @pytest.mark.parametrize("d", D)
68 @pytest.mark.parametrize("dtype", DTYPES)
@@ -70,10 +82,14 @@ def test_activation(
82 torch.cuda.manual_seed(seed)
83 torch.set_default_device(device)
84 x = torch.randn(num_tokens, d, dtype=dtype)
73 - layer = activation()
85 + layer = activation[0]()
86 + fn = activation[1]
87 out = layer(x)
88 ref_out = layer.forward_native(x)
89 torch.testing.assert_close(out,
90 ref_out,
91 atol=get_default_atol(out),
92 rtol=get_default_rtol(out))
93 +
94 + out = torch.empty_like(x)
95 + opcheck(fn, (out, x))
tests/kernels/test_attention.py
+16
@@ -6,6 +6,7 @@ import torch
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 +from tests.kernels.utils import opcheck
10 from vllm import _custom_ops as ops
11 from vllm.utils import get_max_shared_memory_bytes, is_hip
12
@@ -198,6 +199,13 @@ def test_paged_attention(
199 k_scale,
200 v_scale,
201 )
202 +
203 + opcheck(torch.ops._C.paged_attention_v1,
204 + (output, query, key_cache, value_cache, num_kv_heads, scale,
205 + block_tables, seq_lens, block_size, max_seq_len, alibi_slopes,
206 + kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
207 + cond=(head_size == HEAD_SIZES[0]))
208 +
209 elif version == "v2":
210 num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
211 assert PARTITION_SIZE % block_size == 0
@@ -230,6 +238,14 @@ def test_paged_attention(
238 k_scale,
239 v_scale,
240 )
241 +
242 + opcheck(torch.ops._C.paged_attention_v2,
243 + (output, exp_sums, max_logits, tmp_output, query, key_cache,
244 + value_cache, num_kv_heads, scale, block_tables, seq_lens,
245 + block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
246 + k_scale, v_scale, 0, 0, 0, 64, 0),
247 + cond=(head_size == HEAD_SIZES[0]))
248 +
249 else:
250 raise AssertionError(f"Unknown version: {version}")
251
tests/kernels/test_cache.py
+22
@@ -4,6 +4,7 @@ from typing import List, Tuple
4 import pytest
5 import torch
6
7 +from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
8 from vllm import _custom_ops as ops
9
10 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
@@ -87,6 +88,11 @@ def test_copy_blocks(
88 block_mapping_tensor = torch.tensor(block_mapping,
89 dtype=torch.int64,
90 device=device).view(-1, 2)
91 +
92 + opcheck(torch.ops._C_cache_ops.copy_blocks,
93 + (key_caches, value_caches, block_mapping_tensor),
94 + test_utils=DEFAULT_OPCHECK_TEST_UTILS,
95 + cond=(head_size == HEAD_SIZES[0]))
96 ops.copy_blocks(key_caches, value_caches, block_mapping_tensor)
97
98 # Run the reference implementation.
@@ -162,6 +168,10 @@ def test_reshape_and_cache(
168 k_scale = v_scale = 1.0
169
170 # Call the reshape_and_cache kernel.
171 + opcheck(torch.ops._C_cache_ops.reshape_and_cache,
172 + (key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
173 + k_scale, v_scale),
174 + cond=(head_size == HEAD_SIZES[0]))
175 ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping,
176 kv_cache_dtype, k_scale, v_scale)
177
@@ -269,6 +279,10 @@ def test_reshape_and_cache_flash(
279 k_scale = v_scale = 1.0
280
281 # Call the reshape_and_cache kernel.
282 + opcheck(torch.ops._C_cache_ops.reshape_and_cache_flash,
283 + (key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
284 + k_scale, v_scale),
285 + cond=(head_size == HEAD_SIZES[0]))
286 ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
287 slot_mapping, kv_cache_dtype, k_scale, v_scale)
288
@@ -366,6 +380,14 @@ def test_swap_blocks(
380 src_value_caches_clone = src_value_caches[0].clone()
381
382 # Call the swap_blocks kernel.
383 + do_opcheck = (head_size == HEAD_SIZES[0])
384 + opcheck(torch.ops._C_cache_ops.swap_blocks,
385 + (src_key_caches[0], dist_key_caches[0], block_mapping_tensor),
386 + cond=do_opcheck)
387 + opcheck(torch.ops._C_cache_ops.swap_blocks,
388 + (src_value_caches[0], dist_value_caches[0], block_mapping_tensor),
389 + cond=do_opcheck)
390 +
391 ops.swap_blocks(src_key_caches[0], dist_key_caches[0],
392 block_mapping_tensor)
393 ops.swap_blocks(src_value_caches[0], dist_value_caches[0],
tests/kernels/test_cutlass.py
+13
@@ -7,6 +7,7 @@ from typing import Optional, Type
7 import pytest
8 import torch
9
10 +from tests.kernels.utils import opcheck
11 from vllm import _custom_ops as ops
12 from vllm.platforms import current_platform
13
@@ -108,6 +109,9 @@ def cutlass_int8_gemm_helper(m: int,
109
110 torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
111
112 + opcheck(torch.ops._C.cutlass_scaled_mm,
113 + (out, a, b, scale_a, scale_b, bias))
114 +
115
116 @pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
117 @pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
@@ -341,6 +345,15 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
345 torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
346 torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
347
348 + if azp_per_token:
349 + opcheck(torch.ops._C.cutlass_scaled_mm_azp,
350 + (out, aq_i8, bq_i8, scale_a, scale_b, azp_adj_i32, azp_i32,
351 + func_bias))
352 + else:
353 + opcheck(torch.ops._C.cutlass_scaled_mm_azp,
354 + (out, aq_i8, bq_i8, scale_a, scale_b, azp_with_adj_i32, None,
355 + func_bias))
356 +
357
358 # Test working with a subset of A and B
359 def test_cutlass_subset():
tests/kernels/test_int8_quant.py
+15
@@ -2,6 +2,7 @@ import pytest
2 import torch
3
4 from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5 +from tests.kernels.utils import opcheck
6 from vllm._custom_ops import scaled_int8_quant
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -12,6 +13,16 @@ SEEDS = [0]
13 SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
14
15
16 +def opcheck_int8_quant(output, input, scale=None):
17 + if scale is not None:
18 + opcheck(torch.ops._C.static_scaled_int8_quant, (output, input, scale))
19 + else:
20 + scale = torch.empty((input.numel() // input.shape[-1], 1),
21 + device=input.device,
22 + dtype=torch.float32)
23 + opcheck(torch.ops._C.dynamic_scaled_int8_quant, (output, input, scale))
24 +
25 +
26 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
27 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
28 @pytest.mark.parametrize("dtype", DTYPES)
@@ -34,6 +45,8 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
45 ops_out, ref_out, atol=1,
46 rtol=0.0) # big atol to account for rounding errors
47
48 + opcheck_int8_quant(ops_out, x)
49 +
50
51 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
52 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@@ -58,3 +71,5 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
71 torch.testing.assert_close(
72 out1, out2, atol=1,
73 rtol=0.0) # big atol to account for rounding errors
74 +
75 + opcheck_int8_quant(out2, x, scale)
tests/kernels/test_layernorm.py
+8
@@ -1,6 +1,7 @@
1 import pytest
2 import torch
3
4 +from tests.kernels.utils import opcheck
5 from vllm.model_executor.layers.layernorm import RMSNorm
6
7 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -52,3 +53,10 @@ def test_rms_norm(
53 torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
54 else:
55 torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
56 +
57 + if residual is not None:
58 + opcheck(torch.ops._C.fused_add_rms_norm,
59 + (x, residual, layer.weight.data, layer.variance_epsilon))
60 + else:
61 + opcheck(torch.ops._C.rms_norm,
62 + (out, x, layer.weight.data, layer.variance_epsilon))
tests/kernels/test_machete_gemm.py
+7
@@ -9,6 +9,7 @@ from typing import Optional, Tuple
9 import pytest
10 import torch
11
12 +from tests.kernels.utils import opcheck
13 from vllm import _custom_ops as ops
14 from vllm.model_executor.layers.quantization.utils.quant_utils import (
15 pack_rows, quantize_weights)
@@ -76,6 +77,8 @@ def machete_quantize_and_pack(w: torch.Tensor,
77 w_q = w_q.t().contiguous().t() # convert to col major
78 w_q_machete = ops.machete_prepack_B(w_q, wtype)
79
80 + opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype))
81 +
82 return w_ref, w_q_machete, w_s, w_zp
83
84
@@ -146,6 +149,10 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
149 schedule=schedule,
150 )
151
152 + opcheck(torch.ops._C.machete_gemm,
153 + (a, w_q_machete, wtype, w_s, maybe_convert_zeropoints(
154 + w_zp, w_s), group_size, None, None, None, schedule))
155 +
156 # Relax atol as our reduction dim becomes larger (more rounding error)
157 # Relax atol when we have zeropoints since the way machete applies
158 # zeropoints (after scales) causes noise around 0
tests/kernels/test_marlin_gemm.py
+29 -26
@@ -5,6 +5,7 @@ Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
5 import pytest
6 import torch
7
8 +from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
9 from tests.quantization.utils import is_quant_method_supported
10 from vllm import _custom_ops as ops
11 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
@@ -73,12 +74,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
74 act_order, mnk_factors):
75 m_factor, n_factor, k_factor = mnk_factors
76
76 - size_m = m_factor
77 size_k = k_chunk * k_factor
78 size_n = n_chunk * n_factor
79
80 - print(f"MNK = {size_m} {size_n} {size_k}")
81 -
80 # Filter act_order
81 if act_order:
82 if group_size == -1:
@@ -112,6 +110,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
110 marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
111 weight_perm)
112
113 + opcheck(torch.ops._C.gptq_marlin_repack,
114 + (q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits))
115 +
116 # Run Marlin repack GPU kernel
117 marlin_q_w_2 = ops.gptq_marlin_repack(
118 q_w_gptq,
@@ -137,12 +138,9 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
138 mnk_factors):
139 m_factor, n_factor, k_factor = mnk_factors
140
140 - size_m = m_factor
141 size_k = k_chunk * k_factor
142 size_n = n_chunk * n_factor
143
144 - print(f"MNK = {size_m} {size_n} {size_k}")
145 -
144 # Normalize group_size
145 if group_size == -1:
146 group_size = size_k
@@ -165,6 +163,9 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
163 marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
164 weight_perm)
165
166 + opcheck(torch.ops._C.awq_marlin_repack,
167 + (q_w_awq, size_k, size_n, quant_type.size_bits))
168 +
169 # Run Marlin repack GPU kernel
170 marlin_q_w_2 = ops.awq_marlin_repack(
171 q_w_awq,
@@ -204,9 +205,6 @@ def test_gptq_marlin_gemm(
205 size_k = k_chunk * k_factor
206 size_n = n_chunk * n_factor
207
207 - print(f"MNK = {size_m} {size_n} {size_k}")
208 - print(f"groupsize = {group_size}")
209 -
208 if act_order:
209 if group_size == -1:
210 return
@@ -224,6 +222,13 @@ def test_gptq_marlin_gemm(
222 workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
223 GPTQ_MARLIN_MAX_PARALLEL)
224
225 + opcheck(
226 + torch.ops._C.gptq_marlin_gemm,
227 + (a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
228 + workspace.scratch, quant_type, a_input.shape[0], b_weight.shape[1],
229 + a_input.shape[1], is_k_full, False, use_fp32_reduce),
230 + test_utils=DEFAULT_OPCHECK_TEST_UTILS)
231 +
232 output = ops.gptq_marlin_gemm(
233 a_input,
234 marlin_q_w,
@@ -245,7 +250,6 @@ def test_gptq_marlin_gemm(
250 torch.cuda.synchronize()
251
252 max_diff = compute_max_diff(output, output_ref)
248 - print("max_diff = {}".format(max_diff))
253
254 assert max_diff < 0.04
255
@@ -265,9 +269,6 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
269 size_k = k_chunk * k_factor
270 size_n = n_chunk * n_factor
271
268 - print(f"MNK = {size_m} {size_n} {size_k}")
269 - print(f"groupsize = {group_size}")
270 -
272 a_input = rand_data((size_m, size_k))
273 b_weight = rand_data((size_k, size_n))
274
@@ -279,6 +280,12 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
280
281 output_ref = torch.matmul(a_input, w_24_ref)
282
283 + opcheck(torch.ops._C.gptq_marlin_24_gemm,
284 + (a_input, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s,
285 + workspace_24.scratch, quant_type, a_input.shape[0],
286 + b_weight.shape[1], a_input.shape[1]),
287 + test_utils=DEFAULT_OPCHECK_TEST_UTILS)
288 +
289 output = ops.gptq_marlin_24_gemm(
290 a_input,
291 marlin_24_q_w_comp,
@@ -294,7 +301,6 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
301 torch.cuda.synchronize()
302
303 max_diff = compute_max_diff(output, output_ref)
297 - print("max_diff = {}".format(max_diff))
304
305 assert max_diff < 0.04
306
@@ -321,9 +327,6 @@ def test_fp8_marlin_gemm(
327 size_k = k_chunk * k_factor
328 size_n = n_chunk * n_factor
329
324 - print(f"MNK = {size_m} {size_n} {size_k}")
325 - print(f"groupsize = {group_size}")
326 -
330 a_input = rand_data((size_m, size_k), dtype=dtype)
331 b_weight = rand_data((size_k, size_n), dtype=dtype)
332
@@ -353,6 +356,10 @@ def test_fp8_marlin_gemm(
356 workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
357 GPTQ_MARLIN_MAX_PARALLEL)
358
359 + opcheck(torch.ops._C.fp8_marlin_gemm,
360 + (a_input, marlin_qweight, marlin_scales, workspace.scratch,
361 + num_bits, a_input.shape[0], b_weight.shape[1], a_input.shape[1]))
362 +
363 output = ops.fp8_marlin_gemm(
364 a=a_input,
365 b_q_weight=marlin_qweight,
@@ -368,7 +375,6 @@ def test_fp8_marlin_gemm(
375 torch.cuda.synchronize()
376
377 max_diff = compute_max_diff(output, output_ref)
371 - print("max_diff = {}".format(max_diff))
378
379 assert max_diff < 0.04
380
@@ -396,9 +402,6 @@ def test_awq_marlin_gemm(
402 size_k = k_chunk * k_factor
403 size_n = n_chunk * n_factor
404
399 - print(f"MNK = {size_m} {size_n} {size_k}")
400 - print(f"groupsize = {group_size}")
401 -
405 a_input = rand_data((size_m, size_k))
406 b_weight = rand_data((size_k, size_n))
407
@@ -434,7 +437,6 @@ def test_awq_marlin_gemm(
437 torch.cuda.synchronize()
438
439 max_diff = compute_max_diff(output, output_ref)
437 - print("max_diff = {}".format(max_diff))
440
441 assert max_diff < 0.04
442
@@ -460,9 +462,6 @@ def test_marlin_qqq_gemm(
462 size_k = k_chunk * k_factor
463 size_n = n_chunk * n_factor
464
463 - print(f"MNK = {size_m} {size_n} {size_k}")
464 - print(f"groupsize = {group_size}")
465 -
465 a_input = rand_data((size_m, size_k))
466 b_weight = rand_data((size_k, size_n))
467
@@ -479,6 +478,11 @@ def test_marlin_qqq_gemm(
478 workspace = MarlinWorkspace(size_n, MARLIN_QQQ_MIN_THREAD_N,
479 MARLIN_QQQ_MAX_PARALLEL)
480
481 + opcheck(torch.ops._C.marlin_qqq_gemm,
482 + (q_a, marlin_qqq_q_w, s_a, marlin_qqq_s_channel,
483 + marlin_qqq_s_group, workspace.scratch, a_input.shape[0],
484 + b_weight.shape[1], a_input.shape[1]))
485 +
486 output = ops.marlin_qqq_gemm(
487 q_a,
488 marlin_qqq_q_w,
@@ -495,6 +499,5 @@ def test_marlin_qqq_gemm(
499 torch.cuda.synchronize()
500
501 max_diff = compute_max_diff(output, output_ref)
498 - print("max_diff = {}".format(max_diff))
502
503 assert max_diff < 0.04
tests/kernels/utils.py
+33 -1
@@ -3,7 +3,8 @@
3 import itertools
4 import random
5 from numbers import Number
6 -from typing import Any, List, NamedTuple, Optional, Tuple, Union
6 +from typing import (Any, Dict, List, NamedTuple, Optional, Sequence, Tuple,
7 + Union)
8
9 import pytest
10 import torch
@@ -13,6 +14,21 @@ from vllm.attention.backends.xformers import XFormersBackend
14 from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
15 make_tensor_with_pad)
16
17 +# For now, disable "test_aot_dispatch_dynamic" since there are some
18 +# bugs related to this test in PyTorch 2.4.
19 +DEFAULT_OPCHECK_TEST_UTILS: Tuple[str, ...] = (
20 + "test_schema",
21 + "test_autograd_registration",
22 + "test_faketensor",
23 +)
24 +
25 +ALL_OPCHECK_TEST_UTILS: Tuple[str, ...] = (
26 + "test_schema",
27 + "test_autograd_registration",
28 + "test_faketensor",
29 + "test_aot_dispatch_dynamic",
30 +)
31 +
32
33 class QKVInputs(NamedTuple):
34 '''
@@ -926,3 +942,19 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
942 ideal_output = test_params.packed_qkvo.ideal_output
943 torch.testing.assert_close(ideal_output,
944 output_under_test.view_as(ideal_output))
945 +
946 +
947 +def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
948 + torch._library.custom_ops.CustomOpDef],
949 + args: Tuple[Any, ...],
950 + kwargs: Optional[Dict[str, Any]] = None,
951 + *,
952 + test_utils: Union[str, Sequence[str]] = ALL_OPCHECK_TEST_UTILS,
953 + raise_exception: bool = True,
954 + cond: bool = True) -> Dict[str, str]:
955 + return torch.library.opcheck(
956 + op,
957 + args,
958 + kwargs,
959 + test_utils=test_utils,
960 + raise_exception=raise_exception) if cond else {}
tests/models/test_aqlm.py
-20
@@ -7,26 +7,6 @@ import pytest
7
8 from tests.quantization.utils import is_quant_method_supported
9
10 -# In this test we hardcode prompts and generations for the model so we don't
11 -# need to require the AQLM package as a dependency
12 -example_prompts = [
13 - 'vLLM is a high-throughput and memory-efficient inference and serving '
14 - 'engine for LLMs.\n',
15 - 'Briefly describe the major milestones in the development of artificial '
16 - 'intelligence from 1950 to 2020.\n',
17 - 'Compare and contrast artificial intelligence with human intelligence in '
18 - 'terms of processing information.\n',
19 - 'Describe the basic components of a neural network and how it can be '
20 - 'trained.\n',
21 - 'Write a short story about a robot that dreams for the first time.\n',
22 - 'Analyze the impact of the COVID-19 pandemic on global economic structures '
23 - 'and future business models.\n',
24 - 'Explain the cultural significance of the Mona Lisa painting, and how its '
25 - 'perception might vary in Western versus Eastern societies.\n',
26 - "Translate the following English sentence into Japanese, French, and "
27 - "Swahili: 'The early bird catches the worm.'\n"
28 -]
29 -
10 # These ground truth generations were generated using `transformers==4.38.1
11 # aqlm==1.1.0 torch==2.2.0`
12 # and the below code:
vllm/_custom_ops.py
+204
@@ -204,6 +204,22 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
204 b_g_idx, use_exllama, bit)
205
206
207 +# TODO: has to be a better way to do this
208 +try:
209 + torch.ops._C.gptq_gemm # noqa B018
210 +
211 + @torch.library.register_fake("_C::gptq_gemm")
212 + def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
213 + b_gptq_qzeros: torch.Tensor,
214 + b_gptq_scales: torch.Tensor, b_g_idx: torch.Tensor,
215 + use_exllama: bool, bit: int) -> torch.Tensor:
216 + return torch.empty((a.size(0), b_q_weight.size(1)),
217 + dtype=a.dtype,
218 + device=a.device)
219 +except Exception:
220 + pass
221 +
222 +
223 def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
224 bit: int) -> None:
225 torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
@@ -227,6 +243,194 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
243 size_n, size_k)
244
245
246 +# TODO: has to be a better way to do this
247 +try:
248 + torch.ops._C.gptq_marlin_24_gemm # noqa B018
249 +
250 + @torch.library.register_fake("_C::gptq_marlin_24_gemm")
251 + def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
252 + b_meta: torch.Tensor, b_scales: torch.Tensor,
253 + workspace: torch.Tensor,
254 + b_q_type: ScalarType, size_m: int,
255 + size_n: int, size_k: int) -> torch.Tensor:
256 + return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
257 +
258 + @torch.library.register_fake("_C::gptq_marlin_gemm")
259 + def _gptq_marlin_gemm_fake(a: torch.Tensor,
260 + b_q_weight: torch.Tensor,
261 + b_scales: torch.Tensor,
262 + b_zeros: torch.Tensor,
263 + g_idx: torch.Tensor,
264 + perm: torch.Tensor,
265 + workspace: torch.Tensor,
266 + b_q_type: ScalarType,
267 + size_m: int,
268 + size_n: int,
269 + size_k: int,
270 + is_k_full: bool,
271 + has_zp: bool = False,
272 + use_fp32_reduce: bool = False) -> torch.Tensor:
273 + return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
274 +
275 + @torch.library.register_fake("_C::ggml_dequantize")
276 + def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
277 + n: int) -> torch.Tensor:
278 + return torch.empty((m, n), dtype=torch.float16, device=W.device)
279 +
280 + @torch.library.register_fake("_C::ggml_mul_mat_vec_a8")
281 + def _ggml_mul_mat_vec_a8_fake(
282 + W: torch.Tensor,
283 + X: torch.Tensor,
284 + quant_type: int,
285 + row: int,
286 + ) -> torch.Tensor:
287 + return torch.empty((1, row), dtype=torch.float16, device=W.device)
288 +
289 + @torch.library.register_fake("_C::ggml_mul_mat_a8")
290 + def _ggml_mul_mat_a8_fake(
291 + W: torch.Tensor,
292 + X: torch.Tensor,
293 + quant_type: int,
294 + row: int,
295 + ) -> torch.Tensor:
296 + batch = X.size(0)
297 + return torch.empty((batch, row), dtype=torch.float16, device=W.device)
298 +
299 + @torch.library.register_fake("_C::marlin_qqq_gemm")
300 + def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
301 + s_tok: torch.Tensor, s_ch: torch.Tensor,
302 + s_group: torch.Tensor, workspace: torch.Tensor,
303 + size_m: int, size_n: int,
304 + size_k: int) -> torch.Tensor:
305 + return torch.empty((size_m, size_n),
306 + dtype=torch.float16,
307 + device=a.device)
308 +
309 + @torch.library.register_fake("_C::marlin_gemm")
310 + def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
311 + b_scales: torch.Tensor, workspace: torch.Tensor,
312 + size_m: int, size_n: int,
313 + size_k: int) -> torch.Tensor:
314 + return torch.empty((size_m, size_n),
315 + dtype=torch.float16,
316 + device=a.device)
317 +
318 + @torch.library.register_fake("_C::awq_dequantize")
319 + def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
320 + zeros: torch.Tensor, split_k_iters: int, thx: int,
321 + thy: int) -> torch.Tensor:
322 + in_c = qweight.size(0)
323 + qout_c = qweight.size(1)
324 + out_c = qout_c * 8
325 + return torch.empty((in_c, out_c),
326 + dtype=scales.dtype,
327 + device=scales.device)
328 +
329 + @torch.library.register_fake("_C::awq_gemm")
330 + def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
331 + qzeros: torch.Tensor, scales: torch.Tensor,
332 + split_k_iters: int) -> torch.Tensor:
333 + num_in_feats = input.size(0)
334 + return torch.empty((split_k_iters, num_in_feats, qweight.size(1) * 8),
335 + dtype=input.dtype,
336 + device=input.device).sum(0)
337 +
338 + @torch.library.register_fake("_C::aqlm_gemm")
339 + def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor,
340 + codebooks: torch.Tensor, scales: torch.Tensor,
341 + codebook_partition_sizes: List[int],
342 + bias: Optional[torch.Tensor]) -> torch.Tensor:
343 + out_features = codes.size(0) * codebooks.size(2)
344 + flat_input = input.reshape((-1, input.size(-1)))
345 + flat_output = torch.empty((flat_input.size(0), out_features),
346 + dtype=input.dtype,
347 + device=input.device)
348 +
349 + output_sizes = list(input.shape)
350 + output_sizes.pop()
351 + output_sizes.append(-1)
352 + return flat_output.reshape(tuple(output_sizes))
353 +
354 + @torch.library.register_fake("_C::aqlm_dequant")
355 + def _aqlm_dequant_fake(
356 + codes: torch.Tensor, codebooks: torch.Tensor,
357 + codebook_partition_sizes: List[int]) -> torch.Tensor:
358 + in_features = codes.size(1) * 8
359 + out_features = codes.size(0)
360 + return torch.empty((out_features, in_features),
361 + dtype=codebooks.dtype,
362 + device=codebooks.device)
363 +
364 + @torch.library.register_fake("_C::fp8_marlin_gemm")
365 + def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
366 + b_scales: torch.Tensor, workspace: torch.Tensor,
367 + num_bits: int, size_m: int, size_n: int,
368 + size_k: int) -> torch.Tensor:
369 + return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
370 +
371 + @torch.library.register_fake("_C::machete_gemm")
372 + def machete_gemm_fake(
373 + a: torch.Tensor,
374 + b_q: torch.
375 + Tensor, # Should be the tensor returned by machete_prepack_B
376 + b_type: ScalarType,
377 + b_scales: Optional[torch.Tensor] = None,
378 + b_zeros: Optional[torch.Tensor] = None,
379 + b_group_size: Optional[int] = None,
380 + c: Optional[torch.Tensor] = None,
381 + alpha: Optional[float] = None,
382 + beta: Optional[float] = None,
383 + schedule: Optional[str] = None,
384 + ) -> torch.Tensor:
385 + m = a.size(0)
386 + n = b_q.size(1)
387 + return torch.empty((m, n), device=a.device, dtype=a.dtype)
388 +
389 + @torch.library.register_fake("_C::machete_prepack_B")
390 + def machete_prepack_B_fake(b_q_weight: torch.Tensor,
391 + b_type: ScalarType) -> torch.Tensor:
392 + return torch.empty_like(b_q_weight)
393 +
394 + @torch.library.register_fake("_C::causal_conv1d_fwd")
395 + def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
396 + bias_: Optional[torch.Tensor],
397 + seq_idx_: Optional[torch.Tensor],
398 + initial_states_: Optional[torch.Tensor],
399 + final_states_out_: Optional[torch.Tensor],
400 + silu_activation: bool) -> torch.Tensor:
401 + return torch.empty_like(x)
402 +
403 + @torch.library.register_fake("_C::causal_conv1d_update")
404 + def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
405 + weight: torch.Tensor,
406 + bias_: Optional[torch.Tensor],
407 + silu_activation: bool) -> torch.Tensor:
408 + return torch.empty_like(x)
409 +
410 + @torch.library.register_fake("_C::selective_scan_fwd")
411 + def selective_scan_fwd_fake(
412 + u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
413 + B: torch.Tensor, C: torch.Tensor, D_: Optional[torch.Tensor],
414 + z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
415 + delta_softplus: bool, index_: Optional[torch.Tensor],
416 + x: Optional[torch.Tensor]) -> List[torch.Tensor]:
417 + a = torch.empty_like(u)
418 + if x is not None:
419 + b = x
420 + else:
421 + b = torch.empty((u.size(0), u.size(1), A.size(1)),
422 + dtype=u.dtype,
423 + device=u.device)
424 + if z_ is not None:
425 + c = torch.empty_like(z_)
426 + return [a, b, c]
427 + else:
428 + return [a, b]
429 +
430 +except Exception:
431 + pass
432 +
433 +
434 # cutlass
435 def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
436 return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
vllm/envs.py
+5
@@ -203,6 +203,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
203 (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
204 ("true", "1")),
205
206 + # Internal flag to enable Dynamo fullgraph capture
207 + "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
208 + lambda: bool(
209 + os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
210 +
211 # local rank of the process in the distributed setting, used to determine
212 # the GPU device id
213 "LOCAL_RANK":
vllm/model_executor/models/jamba.py
+1 -1
@@ -733,7 +733,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
733 indices_for_current_run: List[int]):
734 # move out all of the occupied but currently not running blocks
735 # outside of the first n blocks
736 - destination_indices = set([range(batch_size)])
736 + destination_indices = range(batch_size)
737 max_possible_batch_size = self.mamba_cache[0].shape[1]
738 for destination_index in destination_indices:
739 if destination_index in self._get_all_occupied_indices() and \
vllm/worker/model_runner.py
+8 -3
@@ -75,6 +75,10 @@ _NUM_WARMUP_ITERS = 2
75
76 TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
77
78 +# For now, bump up cache limits for recompilations during CUDA graph warmups.
79 +torch._dynamo.config.cache_size_limit = 128
80 +torch._dynamo.config.accumulated_cache_size_limit = 128
81 +
82
83 @dataclass(frozen=True)
84 class ModelInputForGPU(ModelRunnerInputBase):
@@ -1060,9 +1064,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1064 "This may lead to less accurate results!")
1065
1066 if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1063 - self.model = torch.compile(self.model,
1064 - fullgraph=True,
1065 - backend="eager")
1067 + self.model = torch.compile(
1068 + self.model,
1069 + fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1070 + backend="eager")
1071
1072 def save_sharded_state(
1073 self,
vllm/worker/worker.py
+1
@@ -166,6 +166,7 @@ class Worker(LocalOrDistributedWorkerBase):
166 torch.cuda.set_device(self.device)
167
168 _check_if_gpu_supports_dtype(self.model_config.dtype)
169 + gc.collect()
170 torch.cuda.empty_cache()
171 self.init_gpu_memory = torch.cuda.mem_get_info()[0]
172 else: