@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c27df94e1f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix chunked prefill with model dtype float32 on Turing Devices (#9850)

Signed-off-by: Wallas Santos <wallashss@ibm.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Wallas Henrique committed Nov 25, 2024 at 14:23 UTC c27df94e1ff98551b987b40bb2049bf4640e202a
6 files changed +122 -13
pyproject.toml
+1
@@ -98,4 +98,5 @@ markers = [
98 "quant_model: run this model test under Quantized category",
99 "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100 "skip_v1: do not run this test with v1",
101 + "optional: optional tests that are automatically skipped, include --optional to run them",
102 ]
tests/conftest.py
+19
@@ -1030,3 +1030,22 @@ def dummy_gemma2_embedding_path():
1030 with open(json_path, "w") as f:
1031 json.dump(config, f)
1032 return _dummy_gemma2_embedding_path
1033 +
1034 +
1035 +# Add the flag `--optional` to allow run tests
1036 +# that are marked with @pytest.mark.optional
1037 +def pytest_addoption(parser):
1038 + parser.addoption("--optional",
1039 + action="store_true",
1040 + default=False,
1041 + help="run optional test")
1042 +
1043 +
1044 +def pytest_collection_modifyitems(config, items):
1045 + if config.getoption("--optional"):
1046 + # --optional given in cli: do not skip optional tests
1047 + return
1048 + skip_optional = pytest.mark.skip(reason="need --optional option to run")
1049 + for item in items:
1050 + if "optional" in item.keywords:
1051 + item.add_marker(skip_optional)
tests/kernels/test_prefix_prefill.py
+63
@@ -40,6 +40,13 @@ def test_contexted_kv_attention(
40 kv_cache_dtype: str,
41 device: str,
42 ) -> None:
43 +
44 + if 'fp8' in kv_cache_dtype and not current_platform.has_device_capability(
45 + 89):
46 + pytest.skip(
47 + 'Triton limitation: fp8e4nv data type is not supported on CUDA'
48 + ' arch < 89')
49 +
50 current_platform.seed_everything(0)
51 torch.set_default_device(device)
52
@@ -235,6 +242,13 @@ def test_contexted_kv_attention_alibi(
242 kv_cache_dtype: str,
243 device: str,
244 ) -> None:
245 +
246 + if 'fp8' in kv_cache_dtype and not current_platform.has_device_capability(
247 + 89):
248 + pytest.skip(
249 + 'Triton limitation: fp8e4nv data type is not supported on CUDA'
250 + ' arch < 89')
251 +
252 current_platform.seed_everything(0)
253 torch.set_default_device(device)
254
@@ -462,3 +476,52 @@ def test_contexted_kv_attention_alibi(
476 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
477 atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
478 torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
479 +
480 +
481 +# These tests are optional to only run when explicitly invoked
482 +#
483 +# pytest -v -s --optional \
484 +# tests/kernels/test_prefix_prefill.py::test_contexted_kv_attention_f32
485 +#
486 +# These tests are useful to test model dtype float32 on Turing devices.
487 +# We skip them to not increase the time when running tests on CI
488 +@pytest.mark.optional
489 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
490 +@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
491 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
492 +@pytest.mark.parametrize("dtype", [torch.float32])
493 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
494 +@pytest.mark.parametrize("device", CUDA_DEVICES)
495 +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
496 +@torch.inference_mode()
497 +def test_contexted_kv_attention_f32(
498 + num_heads: int,
499 + num_queries_per_kv: int,
500 + head_size: int,
501 + sliding_window: int,
502 + dtype: torch.dtype,
503 + kv_cache_dtype: str,
504 + device: str,
505 +) -> None:
506 + test_contexted_kv_attention(num_heads, num_queries_per_kv, head_size,
507 + sliding_window, dtype, kv_cache_dtype, device)
508 +
509 +
510 +@pytest.mark.optional
511 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
512 +@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
513 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
514 +@pytest.mark.parametrize("dtype", [torch.float32])
515 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
516 +@pytest.mark.parametrize("device", CUDA_DEVICES)
517 +@torch.inference_mode()
518 +def test_contexted_kv_attention_alibi_f32(
519 + num_heads: int,
520 + num_queries_per_kv: int,
521 + head_size: int,
522 + dtype: torch.dtype,
523 + kv_cache_dtype: str,
524 + device: str,
525 +) -> None:
526 + test_contexted_kv_attention_alibi(num_heads, num_queries_per_kv, head_size,
527 + dtype, kv_cache_dtype, device)
vllm/attention/ops/prefix_prefill.py
+28 -13
@@ -7,6 +7,13 @@ import triton.language as tl
7
8 from vllm.platforms import current_platform
9
10 +# Static kernels parameters
11 +BASE_BLOCK = 128 if current_platform.has_device_capability(80) else 64
12 +NUM_WARPS = 8
13 +
14 +# To check compatibility
15 +IS_TURING = current_platform.get_device_capability() == (7, 5)
16 +
17 if triton.__version__ >= "2.1.0":
18
19 @triton.jit
@@ -50,6 +57,7 @@ if triton.__version__ >= "2.1.0":
57 stride_v_cache_d,
58 stride_v_cache_bl,
59 num_queries_per_kv: int,
60 + IN_PRECISION: tl.constexpr,
61 BLOCK_M: tl.constexpr,
62 BLOCK_DMODEL: tl.constexpr, # head size
63 BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
@@ -130,7 +138,7 @@ if triton.__version__ >= "2.1.0":
138 k = k_load
139
140 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
133 - qk += tl.dot(q, k)
141 + qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
142 qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
143 float("-inf"))
144 qk *= sm_scale
@@ -178,7 +186,7 @@ if triton.__version__ >= "2.1.0":
186 v = v_load
187 p = p.to(v.dtype)
188
181 - acc += tl.dot(p, v)
189 + acc = tl.dot(p, v, acc=acc, input_precision=IN_PRECISION)
190 # # update m_i and l_i
191 l_i = l_i_new
192 m_i = m_i_new
@@ -204,7 +212,7 @@ if triton.__version__ >= "2.1.0":
212 other=0.0)
213
214 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
207 - qk += tl.dot(q, k)
215 + qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
216 qk *= sm_scale
217 # apply causal mask
218 qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
@@ -238,7 +246,7 @@ if triton.__version__ >= "2.1.0":
246 other=0.0)
247 p = p.to(v.dtype)
248
241 - acc += tl.dot(p, v)
249 + acc = tl.dot(p, v, acc=acc, input_precision=IN_PRECISION)
250 # update m_i and l_i
251 l_i = l_i_new
252 m_i = m_i_new
@@ -485,6 +493,7 @@ if triton.__version__ >= "2.1.0":
493 stride_v_cache_d,
494 stride_v_cache_bl,
495 num_queries_per_kv: int,
496 + IN_PRECISION: tl.constexpr,
497 BLOCK_M: tl.constexpr,
498 BLOCK_DMODEL: tl.constexpr, # head size
499 BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
@@ -560,7 +569,7 @@ if triton.__version__ >= "2.1.0":
569 k = k_load
570
571 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
563 - qk += tl.dot(q, k)
572 + qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION)
573 qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
574 float("-inf"))
575 qk *= sm_scale
@@ -600,7 +609,7 @@ if triton.__version__ >= "2.1.0":
609 v = v_load
610 p = p.to(v.dtype)
611
603 - acc += tl.dot(p, v, allow_tf32=False)
612 + acc = tl.dot(p, v, acc=acc, input_precision='ieee')
613 # update m_i and l_i
614 l_i = l_i_new
615 m_i = m_i_new
@@ -635,7 +644,7 @@ if triton.__version__ >= "2.1.0":
644 other=0.0)
645
646 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
638 - qk += tl.dot(q, k, allow_tf32=False)
647 + qk = tl.dot(q, k, acc=qk, input_precision='ieee')
648 qk *= sm_scale
649 qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
650 float("-inf"))
@@ -673,7 +682,7 @@ if triton.__version__ >= "2.1.0":
682 other=0.0)
683 p = p.to(v.dtype)
684
676 - acc += tl.dot(p, v, allow_tf32=False)
685 + acc = tl.dot(p, v, acc=acc, input_precision='ieee')
686 # update m_i and l_i
687 l_i = l_i_new
688 m_i = m_i_new
@@ -709,13 +718,17 @@ if triton.__version__ >= "2.1.0":
718 alibi_slopes=None,
719 sliding_window=None):
720
712 - BLOCK = 128 if current_platform.has_device_capability(80) else 64
713 - NUM_WARPS = 8
714 -
721 + q_dtype_is_f32 = q.dtype is torch.float32
722 # need to reduce num. blocks when using fp32
723 # due to increased use of GPU shared memory
717 - if q.dtype is torch.float32:
718 - BLOCK = BLOCK // 2
724 + # if q.dtype is torch.float32:
725 + BLOCK = BASE_BLOCK // 2 if q_dtype_is_f32 else BASE_BLOCK
726 +
727 + # Turing does have tensor core for float32 multiplication
728 + # use ieee as fallback for triton kernels work. There is also
729 + # warning on vllm/config.py to inform users this fallback
730 + # implementation
731 + IN_PRECISION = 'ieee' if IS_TURING and q_dtype_is_f32 else None
732
733 # Conversion of FP8 Tensor from uint8 storage to
734 # appropriate torch.dtype for interpretation by Triton
@@ -799,6 +812,7 @@ if triton.__version__ >= "2.1.0":
812 v_cache.stride(
813 3), #[num_blocks, num_kv_heads, head_size, block_size]
814 num_queries_per_kv=num_queries_per_kv,
815 + IN_PRECISION=IN_PRECISION,
816 BLOCK_M=BLOCK,
817 BLOCK_DMODEL=Lk,
818 BLOCK_DMODEL_PADDED=Lk_padded,
@@ -850,6 +864,7 @@ if triton.__version__ >= "2.1.0":
864 v_cache.stride(
865 3), #[num_blocks, num_kv_heads, head_size, block_size]
866 num_queries_per_kv=num_queries_per_kv,
867 + IN_PRECISION=IN_PRECISION,
868 BLOCK_M=BLOCK,
869 BLOCK_DMODEL=Lk,
870 BLOCK_DMODEL_PADDED=Lk_padded,
vllm/config.py
+10
@@ -2388,6 +2388,16 @@ class VllmConfig:
2388 self.quant_config = VllmConfig._get_quantization_config(
2389 self.model_config, self.load_config)
2390
2391 + if self.scheduler_config is not None and \
2392 + self.model_config is not None and \
2393 + self.scheduler_config.chunked_prefill_enabled and \
2394 + self.model_config.dtype == torch.float32 and \
2395 + current_platform.get_device_capability() == (7, 5):
2396 + print_warning_once(
2397 + "Turing devices tensor cores do not support float32 matmul. "
2398 + "To workaround this limitation, vLLM will set 'ieee' input "
2399 + "precision for chunked prefill triton kernels.")
2400 +
2401 if self.compilation_config is None:
2402 self.compilation_config = CompilationConfig()
2403 if envs.VLLM_USE_V1 and not self.model_config.enforce_eager:
vllm/engine/arg_utils.py
+1
@@ -1055,6 +1055,7 @@ class EngineArgs:
1055 msg = "Chunked prefill is not supported for embedding models"
1056 raise ValueError(msg)
1057
1058 +
1059 speculative_config = SpeculativeConfig.maybe_create_spec_config(
1060 target_model_config=model_config,
1061 target_parallel_config=parallel_config,