A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Kernel] Turn off CUTLASS scaled_mm for Ada Lovelace (#6384)
Tyler Michael Smith committed
Jul 14, 2024 at 09:37 UTC
9dad5cc85902f419aea5320dd49a827621ed5668
3 files changed
+15
-9
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
+4
-4
@@ -1,11 +1,11 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 250 -f 5 -t 1
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 1000 -f 5 -t 1
2
model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
3
tasks:
4
- name: "gsm8k"
5
metrics:
6
- name: "exact_match,strict-match"
7
- value: 0.752
7
+ value: 0.755
8
- name: "exact_match,flexible-extract"
9
- value: 0.752
10
-limit: 250
9
+ value: 0.755
10
+limit: 1000
11
num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml
+3
-3
@@ -4,8 +4,8 @@ tasks:
4
- name: "gsm8k"
5
metrics:
6
- name: "exact_match,strict-match"
7
- value: 0.756
7
+ value: 0.753
8
- name: "exact_match,flexible-extract"
9
- value: 0.752
10
-limit: 250
9
+ value: 0.753
10
+limit: 1000
11
num_fewshot: 5
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+8
-2
@@ -38,7 +38,13 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
38
if (cuda_device_capability >= 90) {
39
return CUDA_VERSION >= 12000;
40
} else if (cuda_device_capability >= 89) {
41
- return CUDA_VERSION >= 12040;
41
+ // CUTLASS Kernels have not been tuned for Ada Lovelace systems
42
+ // and are slower than torch.mm. Return false unconditionally in this case.
43
+ return false;
44
+
45
+ // Once the CUTLASS kernels have been optimized for Lovelace systems,
46
+ // use the following check:
47
+ // return CUDA_VERSION >= 12040;
48
}
49
#endif
50
@@ -98,4 +104,4 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
104
TORCH_CHECK(version_num >= 75);
105
cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
106
}
101
-}
\ No newline at end of file
107
+}