A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Speed up Punica compilation (#2632)
Woosuk Kwon committed
Jan 27, 2024 at 17:46 UTC
f8ecb84c0283a7f1ba02ee732c9f044f8f9d36ee
21 files changed
+100
-22
.buildkite/test-template.j2
+1
-1
@@ -5,7 +5,7 @@
5
steps:
6
- label: ":docker: build image"
7
commands:
8
- - "docker build --tag {{ docker_image }} --target test --progress plain ."
8
+ - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
9
- "docker push {{ docker_image }}"
10
env:
11
DOCKER_BUILDKIT: "1"
csrc/punica/bgmv/bgmv_all.cu
deleted
-21
@@ -1,21 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
6
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
7
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
8
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
9
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
10
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
11
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
12
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
13
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
14
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
15
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
16
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
17
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
18
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
19
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
20
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
21
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu
new
+4
@@ -0,0 +1,4 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
csrc/punica/bgmv/generator.py
new
+27
@@ -0,0 +1,27 @@
1
+DTYPES = ["fp16", "bf16", "fp32"]
2
+DTYPE_MAP = {
3
+ "fp16": "nv_half",
4
+ "bf16": "nv_bfloat16",
5
+ "fp32": "float",
6
+}
7
+
8
+TEMPLATE = """
9
+#include "bgmv_config.h"
10
+#include "bgmv_impl.cuh"
11
+
12
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13
+""".lstrip()
14
+
15
+for input_dtype in DTYPES:
16
+ for output_dtype in DTYPES:
17
+ for weight_dtype in DTYPES:
18
+ if weight_dtype == "fp32":
19
+ # FP32 weights are not supported.
20
+ continue
21
+ kernel_definition = TEMPLATE.format(
22
+ input_dtype=DTYPE_MAP[input_dtype],
23
+ output_dtype=DTYPE_MAP[output_dtype],
24
+ weight_dtype=DTYPE_MAP[weight_dtype])
25
+ filename = f"bgmv_{input_dtype}_{output_dtype}_{weight_dtype}.cu"
26
+ with open(filename, "w") as f:
27
+ f.write(kernel_definition)