@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f80ab3521c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

Clean up remaining Punica C information (#7027)

Jee Jee Li committed Aug 5, 2024 at 06:37 UTC f80ab3521ca2aa74e121e26a27b87da7a1065939
5 files changed +3 -15
.github/workflows/clang-format.yml
-6
@@ -30,12 +30,6 @@ jobs:
30 run: |
31 EXCLUDES=(
32 'csrc/moe/topk_softmax_kernels.cu'
33 - 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
34 - 'csrc/punica/bgmv/bgmv_config.h'
35 - 'csrc/punica/bgmv/bgmv_impl.cuh'
36 - 'csrc/punica/bgmv/vec_dtypes.cuh'
37 - 'csrc/punica/punica_ops.cu'
38 - 'csrc/punica/type_convert.h'
33 )
34 find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
35 | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
cmake/utils.cmake
+1 -1
@@ -181,7 +181,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
181 #
182 # The torch cmake setup hardcodes the detected architecture flags in
183 # `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
184 - # can't modified on a per-target basis, e.g. for the `punica` extension.
184 + # can't modified on a per-target basis.
185 # So, all the `-gencode` flags need to be extracted and removed from
186 # `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
187 # Since it's not possible to use `target_compiler_options` for adding target
format.sh
-6
@@ -242,12 +242,6 @@ echo 'vLLM isort: Done'
242 # NOTE: Keep up to date with .github/workflows/clang-format.yml
243 CLANG_FORMAT_EXCLUDES=(
244 'csrc/moe/topk_softmax_kernels.cu'
245 - 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
246 - 'csrc/punica/bgmv/bgmv_config.h'
247 - 'csrc/punica/bgmv/bgmv_impl.cuh'
248 - 'csrc/punica/bgmv/vec_dtypes.cuh'
249 - 'csrc/punica/punica_ops.cu'
250 - 'csrc/punica/type_convert.h'
245 )
246
247 # Format specified files with clang-format
vllm/config.py
+1 -1
@@ -1304,7 +1304,7 @@ class LoRAConfig:
1304 long_lora_scaling_factors: Optional[Tuple[float]] = None
1305
1306 def __post_init__(self):
1307 - # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
1307 + # TODO: Increase the range of rank
1308 possible_max_ranks = (8, 16, 32, 64)
1309 possible_lora_extra_vocab_size = (0, 256, 512)
1310 if self.max_lora_rank not in possible_max_ranks:
vllm/lora/layers.py
+1 -1
@@ -1073,7 +1073,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1073 lora_config: LoRAConfig,
1074 model_config: Optional[PretrainedConfig] = None,
1075 ) -> None:
1076 - # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
1076 + # TODO: Verify if this condition can be relaxed
1077 if 32000 < self.base_layer.vocab_size > 128512:
1078 raise ValueError("When using LoRA, vocab size must be "
1079 "32000 >= vocab_size <= 128512")