@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd37b9fbe2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Try to handle older versions of pytorch (#9086)

bnellnm committed Oct 8, 2024 at 17:28 UTC bd37b9fbe274e28e12c0687cb9a8111dda270936
3 files changed +41 -21
tests/kernels/test_awq.py
+5
@@ -1,11 +1,14 @@
1 import os
2
3 +import pytest
4 import torch
5
6 from tests.kernels.utils import opcheck
7 from vllm import _custom_ops as ops # noqa: F401
8
9
10 +@pytest.mark.skipif(not hasattr(torch.ops._C, "awq_dequantize"),
11 + reason="AWQ is not supported on this GPU type.")
12 def test_awq_dequantize_opcheck():
13 os.environ["VLLM_USE_TRITON_AWQ"] = "0"
14 qweight = torch.randint(-2000000000,
@@ -21,6 +24,8 @@ def test_awq_dequantize_opcheck():
24 (qweight, scales, zeros, split_k_iters, thx, thy))
25
26
27 +@pytest.mark.skipif(not hasattr(torch.ops._C, "awq_gemm"),
28 + reason="AWQ is not supported on this GPU type.")
29 def test_awq_gemm_opcheck():
30 os.environ["VLLM_USE_TRITON_AWQ"] = "0"
31 input = torch.rand((2, 8192), device='cuda', dtype=torch.float16)
tests/kernels/test_awq_marlin.py
+4
@@ -7,6 +7,7 @@ import torch
7
8 from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
9 torch_moe_single)
10 +from vllm import _custom_ops as ops
11 from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
12 fused_marlin_moe, single_marlin_moe)
13 from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
@@ -21,6 +22,9 @@ from vllm.scalar_type import scalar_types
22 @pytest.mark.parametrize("e", [8, 64])
23 @pytest.mark.parametrize("topk", [2, 6])
24 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
25 +@pytest.mark.skipif(not (ops.supports_moe_ops
26 + and hasattr(torch.ops._moe_C, "marlin_gemm_moe")),
27 + reason="Marlin is not supported on this GPU type.")
28 def test_fused_marlin_moe_awq(
29 m: int,
30 n: int,
vllm/_custom_ops.py
+32 -21
@@ -1,8 +1,9 @@
1 import contextlib
2 import functools
3 -from typing import List, Optional, Tuple, Union
3 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
4
5 import torch
6 +import torch.library
7
8 import vllm.envs as envs
9 from vllm._core_ext import ScalarType
@@ -25,6 +26,16 @@ with contextlib.suppress(ImportError):
26 import vllm._moe_C # noqa: F401
27 supports_moe_ops = True
28
29 +if TYPE_CHECKING:
30 +
31 + def register_fake(fn):
32 + return lambda name: fn
33 +else:
34 + try:
35 + from torch.library import register_fake
36 + except ImportError:
37 + from torch.library import impl_abstract as register_fake
38 +
39
40 def hint_on_error(fn):
41
@@ -266,7 +277,7 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
277
278 if hasattr(torch.ops._C, "gptq_gemm"):
279
269 - @torch.library.register_fake("_C::gptq_gemm")
280 + @register_fake("_C::gptq_gemm")
281 def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
282 b_gptq_qzeros: torch.Tensor,
283 b_gptq_scales: torch.Tensor, b_g_idx: torch.Tensor,
@@ -301,7 +312,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
312
313 if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
314
304 - @torch.library.register_fake("_C::gptq_marlin_24_gemm")
315 + @register_fake("_C::gptq_marlin_24_gemm")
316 def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
317 b_meta: torch.Tensor, b_scales: torch.Tensor,
318 workspace: torch.Tensor,
@@ -309,7 +320,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
320 size_n: int, size_k: int) -> torch.Tensor:
321 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
322
312 - @torch.library.register_fake("_C::gptq_marlin_gemm")
323 + @register_fake("_C::gptq_marlin_gemm")
324 def _gptq_marlin_gemm_fake(a: torch.Tensor,
325 b_q_weight: torch.Tensor,
326 b_scales: torch.Tensor,
@@ -326,12 +337,12 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
337 use_fp32_reduce: bool = False) -> torch.Tensor:
338 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
339
329 - @torch.library.register_fake("_C::ggml_dequantize")
340 + @register_fake("_C::ggml_dequantize")
341 def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
342 n: int) -> torch.Tensor:
343 return torch.empty((m, n), dtype=torch.float16, device=W.device)
344
334 - @torch.library.register_fake("_C::ggml_mul_mat_vec_a8")
345 + @register_fake("_C::ggml_mul_mat_vec_a8")
346 def _ggml_mul_mat_vec_a8_fake(
347 W: torch.Tensor,
348 X: torch.Tensor,
@@ -340,7 +351,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
351 ) -> torch.Tensor:
352 return torch.empty((1, row), dtype=torch.float16, device=W.device)
353
343 - @torch.library.register_fake("_C::ggml_mul_mat_a8")
354 + @register_fake("_C::ggml_mul_mat_a8")
355 def _ggml_mul_mat_a8_fake(
356 W: torch.Tensor,
357 X: torch.Tensor,
@@ -350,7 +361,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
361 batch = X.size(0)
362 return torch.empty((batch, row), dtype=torch.float16, device=W.device)
363
353 - @torch.library.register_fake("_C::marlin_qqq_gemm")
364 + @register_fake("_C::marlin_qqq_gemm")
365 def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
366 s_tok: torch.Tensor, s_ch: torch.Tensor,
367 s_group: torch.Tensor, workspace: torch.Tensor,
@@ -360,7 +371,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
371 dtype=torch.float16,
372 device=a.device)
373
363 - @torch.library.register_fake("_C::marlin_gemm")
374 + @register_fake("_C::marlin_gemm")
375 def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
376 b_scales: torch.Tensor, workspace: torch.Tensor,
377 size_m: int, size_n: int,
@@ -369,7 +380,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
380 dtype=torch.float16,
381 device=a.device)
382
372 - @torch.library.register_fake("_C::awq_dequantize")
383 + @register_fake("_C::awq_dequantize")
384 def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
385 zeros: torch.Tensor, split_k_iters: int, thx: int,
386 thy: int) -> torch.Tensor:
@@ -380,7 +391,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
391 dtype=scales.dtype,
392 device=scales.device)
393
383 - @torch.library.register_fake("_C::awq_gemm")
394 + @register_fake("_C::awq_gemm")
395 def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
396 qzeros: torch.Tensor, scales: torch.Tensor,
397 split_k_iters: int) -> torch.Tensor:
@@ -389,7 +400,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
400 dtype=input.dtype,
401 device=input.device).sum(0)
402
392 - @torch.library.register_fake("_C::aqlm_gemm")
403 + @register_fake("_C::aqlm_gemm")
404 def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor,
405 codebooks: torch.Tensor, scales: torch.Tensor,
406 codebook_partition_sizes: List[int],
@@ -405,7 +416,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
416 output_sizes.append(-1)
417 return flat_output.reshape(tuple(output_sizes))
418
408 - @torch.library.register_fake("_C::aqlm_dequant")
419 + @register_fake("_C::aqlm_dequant")
420 def _aqlm_dequant_fake(
421 codes: torch.Tensor, codebooks: torch.Tensor,
422 codebook_partition_sizes: List[int]) -> torch.Tensor:
@@ -415,14 +426,14 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
426 dtype=codebooks.dtype,
427 device=codebooks.device)
428
418 - @torch.library.register_fake("_C::fp8_marlin_gemm")
429 + @register_fake("_C::fp8_marlin_gemm")
430 def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
431 b_scales: torch.Tensor, workspace: torch.Tensor,
432 num_bits: int, size_m: int, size_n: int,
433 size_k: int) -> torch.Tensor:
434 return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
435
425 - @torch.library.register_fake("_C::machete_gemm")
436 + @register_fake("_C::machete_gemm")
437 def machete_gemm_fake(
438 a: torch.Tensor,
439 # Should be the tensor returned by machete_prepack_B
@@ -440,13 +451,13 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
451 n = b_q.size(1)
452 return torch.empty((m, n), device=a.device, dtype=a.dtype)
453
443 - @torch.library.register_fake("_C::machete_prepack_B")
454 + @register_fake("_C::machete_prepack_B")
455 def machete_prepack_B_fake(b_q_weight: torch.Tensor,
456 b_type: ScalarType) -> torch.Tensor:
457 return torch.empty_like(b_q_weight,
458 memory_format=torch.contiguous_format)
459
449 - @torch.library.register_fake("_C::causal_conv1d_fwd")
460 + @register_fake("_C::causal_conv1d_fwd")
461 def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
462 bias_: Optional[torch.Tensor],
463 conv_states: Optional[torch.Tensor],
@@ -456,7 +467,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
467 silu_activation: bool) -> torch.Tensor:
468 return torch.empty_like(x)
469
459 - @torch.library.register_fake("_C::causal_conv1d_update")
470 + @register_fake("_C::causal_conv1d_update")
471 def causal_conv1d_update_fake(
472 x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
473 bias_: Optional[torch.Tensor], silu_activation: bool,
@@ -464,7 +475,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
475 conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
476 return torch.empty_like(x)
477
467 - @torch.library.register_fake("_C::selective_scan_fwd")
478 + @register_fake("_C::selective_scan_fwd")
479 def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
480 A: torch.Tensor, B: torch.Tensor,
481 C: torch.Tensor, D_: Optional[torch.Tensor],
@@ -639,7 +650,7 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
650
651 if hasattr(torch.ops._C, "permute_cols"):
652
642 - @torch.library.register_fake("_C::permute_cols")
653 + @register_fake("_C::permute_cols")
654 def _permute_cols_fake(a: torch.Tensor,
655 perm: torch.Tensor) -> torch.Tensor:
656 return torch.empty_like(a)
@@ -837,7 +848,7 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
848
849 if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
850
840 - @torch.library.register_fake("_moe_C::marlin_gemm_moe")
851 + @register_fake("_moe_C::marlin_gemm_moe")
852 def marlin_gemm_moe_fake(a: torch.Tensor, b_q_weights: torch.Tensor,
853 sorted_ids: torch.Tensor,
854 topk_weights: torch.Tensor,