A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
fix: change GB to GiB in logging close #14979 (#15807)
Signed-off-by: yihong0618 <zouzou0208@gmail.com>
yihong committed
Apr 1, 2025 at 01:00 UTC
2de411824316bfd44a3fc27786f40ef3781eeb9d
4 files changed
+11
-11
vllm/v1/core/kv_cache_utils.py
+2
-2
@@ -488,9 +488,9 @@ def check_enough_kv_cache_memory(vllm_config: VllmConfig,
488
if needed_memory > available_memory:
489
raise ValueError(
490
f"To serve at least one request with the models's max seq len "
491
- f"({max_model_len}), ({needed_memory/1024/1024/1024:.2f} GB KV "
491
+ f"({max_model_len}), ({needed_memory/1024/1024/1024:.2f} GiB KV "
492
f"cache is needed, which is larger than the available KV cache "
493
- f"memory ({available_memory/1024/1024/1024:.2f} GB). Try "
493
+ f"memory ({available_memory/1024/1024/1024:.2f} GiB). Try "
494
f"increasing `gpu_memory_utilization` or decreasing "
495
f"`max_model_len` when initializing the engine.")
496
vllm/v1/worker/gpu_model_runner.py
+4
-4
@@ -24,8 +24,8 @@ from vllm.multimodal.utils import group_mm_inputs_by_modality
24
from vllm.sampling_params import SamplingType
25
from vllm.sequence import IntermediateTensors
26
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
27
- LayerBlockType, LazyLoader, cdiv, check_use_alibi,
28
- is_pin_memory_available)
27
+ GiB_bytes, LayerBlockType, LazyLoader, cdiv,
28
+ check_use_alibi, is_pin_memory_available)
29
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
30
from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
31
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
@@ -1206,8 +1206,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1206
self.device)
1207
time_after_load = time.perf_counter()
1208
self.model_memory_usage = m.consumed_memory
1209
- logger.info("Model loading took %.4f GB and %.6f seconds",
1210
- self.model_memory_usage / float(2**30),
1209
+ logger.info("Model loading took %.4f GiB and %.6f seconds",
1210
+ self.model_memory_usage / GiB_bytes,
1211
time_after_load - time_before_load)
1212
1213
def _get_prompt_logprobs_dict(
vllm/worker/model_runner.py
+2
-2
@@ -1143,8 +1143,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1143
time_after_load = time.perf_counter()
1144
1145
self.model_memory_usage = m.consumed_memory
1146
- logger.info("Model loading took %.4f GB and %.6f seconds",
1147
- self.model_memory_usage / float(2**30),
1146
+ logger.info("Model loading took %.4f GiB and %.6f seconds",
1147
+ self.model_memory_usage / GiB_bytes,
1148
time_after_load - time_before_load)
1149
if self.prompt_adapter_config:
1150
self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager(
vllm/worker/xpu_model_runner.py
+3
-3
@@ -25,7 +25,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
25
MultiModalRegistry)
26
from vllm.sampling_params import SamplingParams
27
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
28
-from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad
28
+from vllm.utils import DeviceMemoryProfiler, GiB_bytes, make_tensor_with_pad
29
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
30
from vllm.worker.model_runner_base import (
31
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -422,8 +422,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
422
self.model = get_model(vllm_config=self.vllm_config)
423
424
self.model_memory_usage = m.consumed_memory
425
- logger.info("Loading model weights took %.4f GB",
426
- self.model_memory_usage / float(2**30))
425
+ logger.info("Loading model weights took %.4f GiB",
426
+ self.model_memory_usage / GiB_bytes)
427
428
def get_model(self) -> nn.Module:
429
return self.model