A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
Clean up remaining Punica C information (#7027)
Jee Jee Li committed
Aug 5, 2024 at 06:37 UTC
f80ab3521ca2aa74e121e26a27b87da7a1065939
5 files changed
+3
-15
.github/workflows/clang-format.yml
-6
@@ -30,12 +30,6 @@ jobs:
30
run: |
31
EXCLUDES=(
32
'csrc/moe/topk_softmax_kernels.cu'
33
- 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
34
- 'csrc/punica/bgmv/bgmv_config.h'
35
- 'csrc/punica/bgmv/bgmv_impl.cuh'
36
- 'csrc/punica/bgmv/vec_dtypes.cuh'
37
- 'csrc/punica/punica_ops.cu'
38
- 'csrc/punica/type_convert.h'
33
)
34
find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
35
| grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
cmake/utils.cmake
+1
-1
@@ -181,7 +181,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
181
#
182
# The torch cmake setup hardcodes the detected architecture flags in
183
# `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
184
- # can't modified on a per-target basis, e.g. for the `punica` extension.
184
+ # can't modified on a per-target basis.
185
# So, all the `-gencode` flags need to be extracted and removed from
186
# `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
187
# Since it's not possible to use `target_compiler_options` for adding target
format.sh
-6
@@ -242,12 +242,6 @@ echo 'vLLM isort: Done'
242
# NOTE: Keep up to date with .github/workflows/clang-format.yml
243
CLANG_FORMAT_EXCLUDES=(
244
'csrc/moe/topk_softmax_kernels.cu'
245
- 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
246
- 'csrc/punica/bgmv/bgmv_config.h'
247
- 'csrc/punica/bgmv/bgmv_impl.cuh'
248
- 'csrc/punica/bgmv/vec_dtypes.cuh'
249
- 'csrc/punica/punica_ops.cu'
250
- 'csrc/punica/type_convert.h'
245
)
246
247
# Format specified files with clang-format
vllm/config.py
+1
-1
@@ -1304,7 +1304,7 @@ class LoRAConfig:
1304
long_lora_scaling_factors: Optional[Tuple[float]] = None
1305
1306
def __post_init__(self):
1307
- # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
1307
+ # TODO: Increase the range of rank
1308
possible_max_ranks = (8, 16, 32, 64)
1309
possible_lora_extra_vocab_size = (0, 256, 512)
1310
if self.max_lora_rank not in possible_max_ranks:
vllm/lora/layers.py
+1
-1
@@ -1073,7 +1073,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1073
lora_config: LoRAConfig,
1074
model_config: Optional[PretrainedConfig] = None,
1075
) -> None:
1076
- # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
1076
+ # TODO: Verify if this condition can be relaxed
1077
if 32000 < self.base_layer.vocab_size > 128512:
1078
raise ValueError("When using LoRA, vocab size must be "
1079
"32000 >= vocab_size <= 128512")