@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f8ecb84c02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Speed up Punica compilation (#2632)

Woosuk Kwon committed Jan 27, 2024 at 17:46 UTC f8ecb84c0283a7f1ba02ee732c9f044f8f9d36ee
21 files changed +100 -22
.buildkite/test-template.j2
+1 -1
@@ -5,7 +5,7 @@
5 steps:
6 - label: ":docker: build image"
7 commands:
8 - - "docker build --tag {{ docker_image }} --target test --progress plain ."
8 + - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
9 - "docker push {{ docker_image }}"
10 env:
11 DOCKER_BUILDKIT: "1"
csrc/punica/bgmv/bgmv_all.cu deleted
-21
@@ -1,21 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
6 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
7 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
8 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
9 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
10 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
11 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
12 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
13 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
14 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
15 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
16 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
17 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
18 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
19 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
20 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
21 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu new
+4
@@ -0,0 +1,4 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
csrc/punica/bgmv/generator.py new
+27
@@ -0,0 +1,27 @@
1 +DTYPES = ["fp16", "bf16", "fp32"]
2 +DTYPE_MAP = {
3 + "fp16": "nv_half",
4 + "bf16": "nv_bfloat16",
5 + "fp32": "float",
6 +}
7 +
8 +TEMPLATE = """
9 +#include "bgmv_config.h"
10 +#include "bgmv_impl.cuh"
11 +
12 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13 +""".lstrip()
14 +
15 +for input_dtype in DTYPES:
16 + for output_dtype in DTYPES:
17 + for weight_dtype in DTYPES:
18 + if weight_dtype == "fp32":
19 + # FP32 weights are not supported.
20 + continue
21 + kernel_definition = TEMPLATE.format(
22 + input_dtype=DTYPE_MAP[input_dtype],
23 + output_dtype=DTYPE_MAP[output_dtype],
24 + weight_dtype=DTYPE_MAP[weight_dtype])
25 + filename = f"bgmv_{input_dtype}_{output_dtype}_{weight_dtype}.cu"
26 + with open(filename, "w") as f:
27 + f.write(kernel_definition)