A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Try to handle older versions of pytorch (#9086)
bnellnm committed
Oct 8, 2024 at 17:28 UTC
bd37b9fbe274e28e12c0687cb9a8111dda270936
3 files changed
+41
-21
tests/kernels/test_awq.py
+5
@@ -1,11 +1,14 @@
1
import os
2
3
+import pytest
4
import torch
5
6
from tests.kernels.utils import opcheck
7
from vllm import _custom_ops as ops # noqa: F401
8
9
10
+@pytest.mark.skipif(not hasattr(torch.ops._C, "awq_dequantize"),
11
+ reason="AWQ is not supported on this GPU type.")
12
def test_awq_dequantize_opcheck():
13
os.environ["VLLM_USE_TRITON_AWQ"] = "0"
14
qweight = torch.randint(-2000000000,
@@ -21,6 +24,8 @@ def test_awq_dequantize_opcheck():
24
(qweight, scales, zeros, split_k_iters, thx, thy))
25
26
27
+@pytest.mark.skipif(not hasattr(torch.ops._C, "awq_gemm"),
28
+ reason="AWQ is not supported on this GPU type.")
29
def test_awq_gemm_opcheck():
30
os.environ["VLLM_USE_TRITON_AWQ"] = "0"
31
input = torch.rand((2, 8192), device='cuda', dtype=torch.float16)
tests/kernels/test_awq_marlin.py
+4
@@ -7,6 +7,7 @@ import torch
7
8
from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
9
torch_moe_single)
10
+from vllm import _custom_ops as ops
11
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
12
fused_marlin_moe, single_marlin_moe)
13
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
@@ -21,6 +22,9 @@ from vllm.scalar_type import scalar_types
22
@pytest.mark.parametrize("e", [8, 64])
23
@pytest.mark.parametrize("topk", [2, 6])
24
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
25
+@pytest.mark.skipif(not (ops.supports_moe_ops
26
+ and hasattr(torch.ops._moe_C, "marlin_gemm_moe")),
27
+ reason="Marlin is not supported on this GPU type.")
28
def test_fused_marlin_moe_awq(
29
m: int,
30
n: int,
vllm/_custom_ops.py
+32
-21
@@ -1,8 +1,9 @@
1
import contextlib
2
import functools
3
-from typing import List, Optional, Tuple, Union
3
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
4
5
import torch
6
+import torch.library
7
8
import vllm.envs as envs
9
from vllm._core_ext import ScalarType
@@ -25,6 +26,16 @@ with contextlib.suppress(ImportError):
26
import vllm._moe_C # noqa: F401
27
supports_moe_ops = True
28
29
+if TYPE_CHECKING:
30
+
31
+ def register_fake(fn):
32
+ return lambda name: fn
33
+else:
34
+ try:
35
+ from torch.library import register_fake
36
+ except ImportError:
37
+ from torch.library import impl_abstract as register_fake
38
+
39
40
def hint_on_error(fn):
41
@@ -266,7 +277,7 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
277
278
if hasattr(torch.ops._C, "gptq_gemm"):
279
269
- @torch.library.register_fake("_C::gptq_gemm")
280
+ @register_fake("_C::gptq_gemm")
281
def _gptq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
282
b_gptq_qzeros: torch.Tensor,
283
b_gptq_scales: torch.Tensor, b_g_idx: torch.Tensor,
@@ -301,7 +312,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
312
313
if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
314
304
- @torch.library.register_fake("_C::gptq_marlin_24_gemm")
315
+ @register_fake("_C::gptq_marlin_24_gemm")
316
def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
317
b_meta: torch.Tensor, b_scales: torch.Tensor,
318
workspace: torch.Tensor,
@@ -309,7 +320,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
320
size_n: int, size_k: int) -> torch.Tensor:
321
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
322
312
- @torch.library.register_fake("_C::gptq_marlin_gemm")
323
+ @register_fake("_C::gptq_marlin_gemm")
324
def _gptq_marlin_gemm_fake(a: torch.Tensor,
325
b_q_weight: torch.Tensor,
326
b_scales: torch.Tensor,
@@ -326,12 +337,12 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
337
use_fp32_reduce: bool = False) -> torch.Tensor:
338
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
339
329
- @torch.library.register_fake("_C::ggml_dequantize")
340
+ @register_fake("_C::ggml_dequantize")
341
def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
342
n: int) -> torch.Tensor:
343
return torch.empty((m, n), dtype=torch.float16, device=W.device)
344
334
- @torch.library.register_fake("_C::ggml_mul_mat_vec_a8")
345
+ @register_fake("_C::ggml_mul_mat_vec_a8")
346
def _ggml_mul_mat_vec_a8_fake(
347
W: torch.Tensor,
348
X: torch.Tensor,
@@ -340,7 +351,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
351
) -> torch.Tensor:
352
return torch.empty((1, row), dtype=torch.float16, device=W.device)
353
343
- @torch.library.register_fake("_C::ggml_mul_mat_a8")
354
+ @register_fake("_C::ggml_mul_mat_a8")
355
def _ggml_mul_mat_a8_fake(
356
W: torch.Tensor,
357
X: torch.Tensor,
@@ -350,7 +361,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
361
batch = X.size(0)
362
return torch.empty((batch, row), dtype=torch.float16, device=W.device)
363
353
- @torch.library.register_fake("_C::marlin_qqq_gemm")
364
+ @register_fake("_C::marlin_qqq_gemm")
365
def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
366
s_tok: torch.Tensor, s_ch: torch.Tensor,
367
s_group: torch.Tensor, workspace: torch.Tensor,
@@ -360,7 +371,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
371
dtype=torch.float16,
372
device=a.device)
373
363
- @torch.library.register_fake("_C::marlin_gemm")
374
+ @register_fake("_C::marlin_gemm")
375
def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
376
b_scales: torch.Tensor, workspace: torch.Tensor,
377
size_m: int, size_n: int,
@@ -369,7 +380,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
380
dtype=torch.float16,
381
device=a.device)
382
372
- @torch.library.register_fake("_C::awq_dequantize")
383
+ @register_fake("_C::awq_dequantize")
384
def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
385
zeros: torch.Tensor, split_k_iters: int, thx: int,
386
thy: int) -> torch.Tensor:
@@ -380,7 +391,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
391
dtype=scales.dtype,
392
device=scales.device)
393
383
- @torch.library.register_fake("_C::awq_gemm")
394
+ @register_fake("_C::awq_gemm")
395
def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
396
qzeros: torch.Tensor, scales: torch.Tensor,
397
split_k_iters: int) -> torch.Tensor:
@@ -389,7 +400,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
400
dtype=input.dtype,
401
device=input.device).sum(0)
402
392
- @torch.library.register_fake("_C::aqlm_gemm")
403
+ @register_fake("_C::aqlm_gemm")
404
def _aqlm_gemm_fake(input: torch.Tensor, codes: torch.Tensor,
405
codebooks: torch.Tensor, scales: torch.Tensor,
406
codebook_partition_sizes: List[int],
@@ -405,7 +416,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
416
output_sizes.append(-1)
417
return flat_output.reshape(tuple(output_sizes))
418
408
- @torch.library.register_fake("_C::aqlm_dequant")
419
+ @register_fake("_C::aqlm_dequant")
420
def _aqlm_dequant_fake(
421
codes: torch.Tensor, codebooks: torch.Tensor,
422
codebook_partition_sizes: List[int]) -> torch.Tensor:
@@ -415,14 +426,14 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
426
dtype=codebooks.dtype,
427
device=codebooks.device)
428
418
- @torch.library.register_fake("_C::fp8_marlin_gemm")
429
+ @register_fake("_C::fp8_marlin_gemm")
430
def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
431
b_scales: torch.Tensor, workspace: torch.Tensor,
432
num_bits: int, size_m: int, size_n: int,
433
size_k: int) -> torch.Tensor:
434
return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
435
425
- @torch.library.register_fake("_C::machete_gemm")
436
+ @register_fake("_C::machete_gemm")
437
def machete_gemm_fake(
438
a: torch.Tensor,
439
# Should be the tensor returned by machete_prepack_B
@@ -440,13 +451,13 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
451
n = b_q.size(1)
452
return torch.empty((m, n), device=a.device, dtype=a.dtype)
453
443
- @torch.library.register_fake("_C::machete_prepack_B")
454
+ @register_fake("_C::machete_prepack_B")
455
def machete_prepack_B_fake(b_q_weight: torch.Tensor,
456
b_type: ScalarType) -> torch.Tensor:
457
return torch.empty_like(b_q_weight,
458
memory_format=torch.contiguous_format)
459
449
- @torch.library.register_fake("_C::causal_conv1d_fwd")
460
+ @register_fake("_C::causal_conv1d_fwd")
461
def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
462
bias_: Optional[torch.Tensor],
463
conv_states: Optional[torch.Tensor],
@@ -456,7 +467,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
467
silu_activation: bool) -> torch.Tensor:
468
return torch.empty_like(x)
469
459
- @torch.library.register_fake("_C::causal_conv1d_update")
470
+ @register_fake("_C::causal_conv1d_update")
471
def causal_conv1d_update_fake(
472
x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
473
bias_: Optional[torch.Tensor], silu_activation: bool,
@@ -464,7 +475,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
475
conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
476
return torch.empty_like(x)
477
467
- @torch.library.register_fake("_C::selective_scan_fwd")
478
+ @register_fake("_C::selective_scan_fwd")
479
def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
480
A: torch.Tensor, B: torch.Tensor,
481
C: torch.Tensor, D_: Optional[torch.Tensor],
@@ -639,7 +650,7 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
650
651
if hasattr(torch.ops._C, "permute_cols"):
652
642
- @torch.library.register_fake("_C::permute_cols")
653
+ @register_fake("_C::permute_cols")
654
def _permute_cols_fake(a: torch.Tensor,
655
perm: torch.Tensor) -> torch.Tensor:
656
return torch.empty_like(a)
@@ -837,7 +848,7 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
848
849
if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
850
840
- @torch.library.register_fake("_moe_C::marlin_gemm_moe")
851
+ @register_fake("_moe_C::marlin_gemm_moe")
852
def marlin_gemm_moe_fake(a: torch.Tensor, b_q_weights: torch.Tensor,
853
sorted_ids: torch.Tensor,
854
topk_weights: torch.Tensor,