A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix chunked prefill with model dtype float32 on Turing Devices (#9850)
Signed-off-by: Wallas Santos <wallashss@ibm.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Wallas Henrique committed
Nov 25, 2024 at 14:23 UTC
c27df94e1ff98551b987b40bb2049bf4640e202a
6 files changed
+122
-13
pyproject.toml
+1
@@ -98,4 +98,5 @@ markers = [
98
"quant_model: run this model test under Quantized category",
99
"distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100
"skip_v1: do not run this test with v1",
101
+ "optional: optional tests that are automatically skipped, include --optional to run them",
102
]
tests/conftest.py
+19
@@ -1030,3 +1030,22 @@ def dummy_gemma2_embedding_path():
1030
with open(json_path, "w") as f:
1031
json.dump(config, f)
1032
return _dummy_gemma2_embedding_path
1033
+
1034
+
1035
+# Add the flag `--optional` to allow run tests
1036
+# that are marked with @pytest.mark.optional
1037
+def pytest_addoption(parser):
1038
+ parser.addoption("--optional",
1039
+ action="store_true",
1040
+ default=False,
1041
+ help="run optional test")
1042
+
1043
+
1044
+def pytest_collection_modifyitems(config, items):
1045
+ if config.getoption("--optional"):
1046
+ # --optional given in cli: do not skip optional tests
1047
+ return
1048
+ skip_optional = pytest.mark.skip(reason="need --optional option to run")
1049
+ for item in items:
1050
+ if "optional" in item.keywords:
1051
+ item.add_marker(skip_optional)
tests/kernels/test_prefix_prefill.py
+63
@@ -40,6 +40,13 @@ def test_contexted_kv_attention(
40
kv_cache_dtype: str,
41
device: str,
42
) -> None:
43
+
44
+ if 'fp8' in kv_cache_dtype and not current_platform.has_device_capability(
45
+ 89):
46
+ pytest.skip(
47
+ 'Triton limitation: fp8e4nv data type is not supported on CUDA'
48
+ ' arch < 89')
49
+
50
current_platform.seed_everything(0)
51
torch.set_default_device(device)
52
@@ -235,6 +242,13 @@ def test_contexted_kv_attention_alibi(
242
kv_cache_dtype: str,
243
device: str,
244
) -> None:
245
+
246
+ if 'fp8' in kv_cache_dtype and not current_platform.has_device_capability(
247
+ 89):
248
+ pytest.skip(
249
+ 'Triton limitation: fp8e4nv data type is not supported on CUDA'
250
+ ' arch < 89')
251
+
252
current_platform.seed_everything(0)
253
torch.set_default_device(device)
254
@@ -462,3 +476,52 @@ def test_contexted_kv_attention_alibi(
476
print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
477
atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
478
torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
479
+
480
+
481
+# These tests are optional to only run when explicitly invoked
482
+#
483
+# pytest -v -s --optional \
484
+# tests/kernels/test_prefix_prefill.py::test_contexted_kv_attention_f32
485
+#
486
+# These tests are useful to test model dtype float32 on Turing devices.
487
+# We skip them to not increase the time when running tests on CI
488
+@pytest.mark.optional
489
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
490
+@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
491
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
492
+@pytest.mark.parametrize("dtype", [torch.float32])
493
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
494
+@pytest.mark.parametrize("device", CUDA_DEVICES)
495
+@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
496
+@torch.inference_mode()
497
+def test_contexted_kv_attention_f32(
498
+ num_heads: int,
499
+ num_queries_per_kv: int,
500
+ head_size: int,
501
+ sliding_window: int,
502
+ dtype: torch.dtype,
503
+ kv_cache_dtype: str,
504
+ device: str,
505
+) -> None:
506
+ test_contexted_kv_attention(num_heads, num_queries_per_kv, head_size,
507
+ sliding_window, dtype, kv_cache_dtype, device)
508
+
509
+
510
+@pytest.mark.optional
511
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
512
+@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
513
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
514
+@pytest.mark.parametrize("dtype", [torch.float32])
515
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
516
+@pytest.mark.parametrize("device", CUDA_DEVICES)
517
+@torch.inference_mode()
518
+def test_contexted_kv_attention_alibi_f32(
519
+ num_heads: int,
520
+ num_queries_per_kv: int,
521
+ head_size: int,
522
+ dtype: torch.dtype,
523
+ kv_cache_dtype: str,
524
+ device: str,
525
+) -> None:
526
+ test_contexted_kv_attention_alibi(num_heads, num_queries_per_kv, head_size,
527
+ dtype, kv_cache_dtype, device)
vllm/attention/ops/prefix_prefill.py
+28
-13
@@ -7,6 +7,13 @@ import triton.language as tl
7
8
from vllm.platforms import current_platform
9
10
+# Static kernels parameters
11
+BASE_BLOCK = 128 if current_platform.has_device_capability(80) else 64
12
+NUM_WARPS = 8
13
+
14
+# To check compatibility
15
+IS_TURING = current_platform.get_device_capability() == (7, 5)
16
+
17
if triton.__version__ >= "2.1.0":
18
19
@triton.jit
@@ -50,6 +57,7 @@ if triton.__version__ >= "2.1.0":
57
stride_v_cache_d,
58
stride_v_cache_bl,
59
num_queries_per_kv: int,
60
+ IN_PRECISION: tl.constexpr,
61
BLOCK_M: tl.constexpr,
62
BLOCK_DMODEL: tl.constexpr, # head size
63
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
@@ -130,7 +138,7 @@ if triton.__version__ >= "2.1.0":
138
k = k_load
139
140
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
133
- qk += tl.dot(q, k)
141
+ qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
142
qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
143
float("-inf"))
144
qk *= sm_scale
@@ -178,7 +186,7 @@ if triton.__version__ >= "2.1.0":
186
v = v_load
187
p = p.to(v.dtype)
188
181
- acc += tl.dot(p, v)
189
+ acc = tl.dot(p, v, acc=acc, input_precision=IN_PRECISION)
190
# # update m_i and l_i
191
l_i = l_i_new
192
m_i = m_i_new
@@ -204,7 +212,7 @@ if triton.__version__ >= "2.1.0":
212
other=0.0)
213
214
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
207
- qk += tl.dot(q, k)
215
+ qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
216
qk *= sm_scale
217
# apply causal mask
218
qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
@@ -238,7 +246,7 @@ if triton.__version__ >= "2.1.0":
246
other=0.0)
247
p = p.to(v.dtype)
248
241
- acc += tl.dot(p, v)
249
+ acc = tl.dot(p, v, acc=acc, input_precision=IN_PRECISION)
250
# update m_i and l_i
251
l_i = l_i_new
252
m_i = m_i_new
@@ -485,6 +493,7 @@ if triton.__version__ >= "2.1.0":
493
stride_v_cache_d,
494
stride_v_cache_bl,
495
num_queries_per_kv: int,
496
+ IN_PRECISION: tl.constexpr,
497
BLOCK_M: tl.constexpr,
498
BLOCK_DMODEL: tl.constexpr, # head size
499
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
@@ -560,7 +569,7 @@ if triton.__version__ >= "2.1.0":
569
k = k_load
570
571
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
563
- qk += tl.dot(q, k)
572
+ qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
573
qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
574
float("-inf"))
575
qk *= sm_scale
@@ -600,7 +609,7 @@ if triton.__version__ >= "2.1.0":
609
v = v_load
610
p = p.to(v.dtype)
611
603
- acc += tl.dot(p, v, allow_tf32=False)
612
+ acc = tl.dot(p, v, acc=acc, input_precision='ieee')
613
# update m_i and l_i
614
l_i = l_i_new
615
m_i = m_i_new
@@ -635,7 +644,7 @@ if triton.__version__ >= "2.1.0":
644
other=0.0)
645
646
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
638
- qk += tl.dot(q, k, allow_tf32=False)
647
+ qk = tl.dot(q, k, acc=qk, input_precision='ieee')
648
qk *= sm_scale
649
qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
650
float("-inf"))
@@ -673,7 +682,7 @@ if triton.__version__ >= "2.1.0":
682
other=0.0)
683
p = p.to(v.dtype)
684
676
- acc += tl.dot(p, v, allow_tf32=False)
685
+ acc = tl.dot(p, v, acc=acc, input_precision='ieee')
686
# update m_i and l_i
687
l_i = l_i_new
688
m_i = m_i_new
@@ -709,13 +718,17 @@ if triton.__version__ >= "2.1.0":
718
alibi_slopes=None,
719
sliding_window=None):
720
712
- BLOCK = 128 if current_platform.has_device_capability(80) else 64
713
- NUM_WARPS = 8
714
-
721
+ q_dtype_is_f32 = q.dtype is torch.float32
722
# need to reduce num. blocks when using fp32
723
# due to increased use of GPU shared memory
717
- if q.dtype is torch.float32:
718
- BLOCK = BLOCK // 2
724
+ # if q.dtype is torch.float32:
725
+ BLOCK = BASE_BLOCK // 2 if q_dtype_is_f32 else BASE_BLOCK
726
+
727
+ # Turing does have tensor core for float32 multiplication
728
+ # use ieee as fallback for triton kernels work. There is also
729
+ # warning on vllm/config.py to inform users this fallback
730
+ # implementation
731
+ IN_PRECISION = 'ieee' if IS_TURING and q_dtype_is_f32 else None
732
733
# Conversion of FP8 Tensor from uint8 storage to
734
# appropriate torch.dtype for interpretation by Triton
@@ -799,6 +812,7 @@ if triton.__version__ >= "2.1.0":
812
v_cache.stride(
813
3), #[num_blocks, num_kv_heads, head_size, block_size]
814
num_queries_per_kv=num_queries_per_kv,
815
+ IN_PRECISION=IN_PRECISION,
816
BLOCK_M=BLOCK,
817
BLOCK_DMODEL=Lk,
818
BLOCK_DMODEL_PADDED=Lk_padded,
@@ -850,6 +864,7 @@ if triton.__version__ >= "2.1.0":
864
v_cache.stride(
865
3), #[num_blocks, num_kv_heads, head_size, block_size]
866
num_queries_per_kv=num_queries_per_kv,
867
+ IN_PRECISION=IN_PRECISION,
868
BLOCK_M=BLOCK,
869
BLOCK_DMODEL=Lk,
870
BLOCK_DMODEL_PADDED=Lk_padded,
vllm/config.py
+10
@@ -2388,6 +2388,16 @@ class VllmConfig:
2388
self.quant_config = VllmConfig._get_quantization_config(
2389
self.model_config, self.load_config)
2390
2391
+ if self.scheduler_config is not None and \
2392
+ self.model_config is not None and \
2393
+ self.scheduler_config.chunked_prefill_enabled and \
2394
+ self.model_config.dtype == torch.float32 and \
2395
+ current_platform.get_device_capability() == (7, 5):
2396
+ print_warning_once(
2397
+ "Turing devices tensor cores do not support float32 matmul. "
2398
+ "To workaround this limitation, vLLM will set 'ieee' input "
2399
+ "precision for chunked prefill triton kernels.")
2400
+
2401
if self.compilation_config is None:
2402
self.compilation_config = CompilationConfig()
2403
if envs.VLLM_USE_V1 and not self.model_config.enforce_eager:
vllm/engine/arg_utils.py
+1
@@ -1055,6 +1055,7 @@ class EngineArgs:
1055
msg = "Chunked prefill is not supported for embedding models"
1056
raise ValueError(msg)
1057
1058
+
1059
speculative_config = SpeculativeConfig.maybe_create_spec_config(
1060
target_model_config=model_config,
1061
target_parallel_config=parallel_config,