@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae0ccb4017
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add missing kernel for CodeLlama-34B on A/H100 (no tensor parallelism) when using Multi-LoRA. (#3350)

Or Sharir committed Mar 13, 2024 at 21:18 UTC ae0ccb40170d140ded8de99fc905fd8cb0bd409c
2 files changed +2 -1
csrc/punica/bgmv/bgmv_config.h
+1
@@ -43,6 +43,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
43 f(in_T, out_T, W_T, narrow, 14336) \
44 f(in_T, out_T, W_T, narrow, 16384) \
45 f(in_T, out_T, W_T, narrow, 20480) \
46 + f(in_T, out_T, W_T, narrow, 22016) \
47 f(in_T, out_T, W_T, narrow, 24576) \
48 f(in_T, out_T, W_T, narrow, 28672) \
49 f(in_T, out_T, W_T, narrow, 32000) \
tests/lora/test_punica.py
+1 -1
@@ -45,7 +45,7 @@ def _lora_ref_impl(
45 H1 = H2 = [
46 128, 256, 512, 1024, 1280, 2048, 2560, 2752, 3072, 3456, 3584, 4096, 5120,
47 5504, 5632, 6144, 6912, 7168, 8192, 9216, 10240, 11008, 13824, 14336,
48 - 24576, 32000, 32256, 32512, 32768, 33024
48 + 22016, 24576, 32000, 32256, 32512, 32768, 33024
49 ]
50 SEED = [0xabcdabcd987]
51