@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9dad5cc859
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Kernel] Turn off CUTLASS scaled_mm for Ada Lovelace (#6384)

Tyler Michael Smith committed Jul 14, 2024 at 09:37 UTC 9dad5cc85902f419aea5320dd49a827621ed5668
3 files changed +15 -9
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
+4 -4
@@ -1,11 +1,11 @@
1 -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 250 -f 5 -t 1
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 1000 -f 5 -t 1
2 model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
3 tasks:
4 - name: "gsm8k"
5 metrics:
6 - name: "exact_match,strict-match"
7 - value: 0.752
7 + value: 0.755
8 - name: "exact_match,flexible-extract"
9 - value: 0.752
10 -limit: 250
9 + value: 0.755
10 +limit: 1000
11 num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml
+3 -3
@@ -4,8 +4,8 @@ tasks:
4 - name: "gsm8k"
5 metrics:
6 - name: "exact_match,strict-match"
7 - value: 0.756
7 + value: 0.753
8 - name: "exact_match,flexible-extract"
9 - value: 0.752
10 -limit: 250
9 + value: 0.753
10 +limit: 1000
11 num_fewshot: 5
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+8 -2
@@ -38,7 +38,13 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
38 if (cuda_device_capability >= 90) {
39 return CUDA_VERSION >= 12000;
40 } else if (cuda_device_capability >= 89) {
41 - return CUDA_VERSION >= 12040;
41 + // CUTLASS Kernels have not been tuned for Ada Lovelace systems
42 + // and are slower than torch.mm. Return false unconditionally in this case.
43 + return false;
44 +
45 + // Once the CUTLASS kernels have been optimized for Lovelace systems,
46 + // use the following check:
47 + // return CUDA_VERSION >= 12040;
48 }
49 #endif
50
@@ -98,4 +104,4 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
104 TORCH_CHECK(version_num >= 75);
105 cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
106 }
101 -}
\ No newline at end of file
107 +}