@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f0ec3935a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Remove `_get_cache_block_size` (#12214)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Jan 20, 2025 at 21:54 UTC 5f0ec3935a0118fee8cf2764728f765c8cc53d2a
1 file changed +1 -23
vllm/v1/worker/gpu_worker.py
+1 -23
@@ -8,14 +8,13 @@ import torch.distributed
8 import torch.nn as nn
9
10 import vllm.envs as envs
11 -from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
11 +from vllm.config import ParallelConfig, VllmConfig
12 from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment,
14 set_custom_all_reduce)
15 from vllm.logger import init_logger
16 from vllm.model_executor import set_random_seed
17 from vllm.platforms import current_platform
18 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType, get_dtype_size
18 from vllm.v1.core.scheduler import SchedulerOutput
19 from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
20 from vllm.v1.outputs import ModelRunnerOutput
@@ -235,24 +234,3 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
234 f"of at least 8.0. Your {gpu_name} GPU {compute_str}. "
235 "You can use float16 instead by explicitly setting the"
236 "`dtype` flag in CLI, for example: --dtype=half.")
238 -
239 -
240 -def _get_cache_block_size(
241 - cache_config: CacheConfig,
242 - model_config: ModelConfig,
243 - parallel_config: ParallelConfig,
244 -) -> int:
245 - head_size = model_config.get_head_size()
246 - num_heads = model_config.get_num_kv_heads(parallel_config)
247 - num_attention_layers = model_config.get_num_layers_by_block_type(
248 - parallel_config, LayerBlockType.attention)
249 -
250 - key_cache_block = cache_config.block_size * num_heads * head_size
251 - value_cache_block = key_cache_block
252 - total = num_attention_layers * (key_cache_block + value_cache_block)
253 - if cache_config.cache_dtype == "auto":
254 - dtype = model_config.dtype
255 - else:
256 - dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
257 - dtype_size = get_dtype_size(dtype)
258 - return dtype_size * total