@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca2b628b3c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[MISC] rename CudaMemoryProfiler to DeviceMemoryProfiler (#8703)

Huazhong Ji committed Sep 23, 2024 at 01:44 UTC ca2b628b3c25b014b9951731c0331b75262a59e0
3 files changed +5 -5
vllm/utils.py
+1 -1
@@ -757,7 +757,7 @@ def is_pin_memory_available() -> bool:
757 return True
758
759
760 -class CudaMemoryProfiler:
760 +class DeviceMemoryProfiler:
761
762 def __init__(self, device: Optional[torch.types.Device] = None):
763 self.device = device
vllm/worker/model_runner.py
+2 -2
@@ -45,7 +45,7 @@ from vllm.prompt_adapter.worker_manager import (
45 LRUCacheWorkerPromptAdapterManager)
46 from vllm.sampling_params import SamplingParams
47 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
48 -from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
48 +from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
49 flatten_2d_lists, is_hip, is_pin_memory_available,
50 supports_dynamo)
51 from vllm.worker.model_runner_base import (
@@ -1012,7 +1012,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1012
1013 def load_model(self) -> None:
1014 logger.info("Starting to load model %s...", self.model_config.model)
1015 - with CudaMemoryProfiler() as m:
1015 + with DeviceMemoryProfiler() as m:
1016 self.model = get_model(model_config=self.model_config,
1017 device_config=self.device_config,
1018 load_config=self.load_config,
vllm/worker/xpu_model_runner.py
+2 -2
@@ -21,7 +21,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
21 MultiModalInputs, MultiModalRegistry)
22 from vllm.sampling_params import SamplingParams
23 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
24 -from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
24 +from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad
25 from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
26 from vllm.worker.model_runner_base import (
27 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -391,7 +391,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
391 self.model: nn.Module # Set after init_Model
392
393 def load_model(self) -> None:
394 - with CudaMemoryProfiler() as m:
394 + with DeviceMemoryProfiler() as m:
395 self.model = get_model(
396 model_config=self.model_config,
397 device_config=self.device_config,