A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[MISC] rename CudaMemoryProfiler to DeviceMemoryProfiler (#8703)
Huazhong Ji committed
Sep 23, 2024 at 01:44 UTC
ca2b628b3c25b014b9951731c0331b75262a59e0
3 files changed
+5
-5
vllm/utils.py
+1
-1
@@ -757,7 +757,7 @@ def is_pin_memory_available() -> bool:
757
return True
758
759
760
-class CudaMemoryProfiler:
760
+class DeviceMemoryProfiler:
761
762
def __init__(self, device: Optional[torch.types.Device] = None):
763
self.device = device
vllm/worker/model_runner.py
+2
-2
@@ -45,7 +45,7 @@ from vllm.prompt_adapter.worker_manager import (
45
LRUCacheWorkerPromptAdapterManager)
46
from vllm.sampling_params import SamplingParams
47
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
48
-from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
48
+from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
49
flatten_2d_lists, is_hip, is_pin_memory_available,
50
supports_dynamo)
51
from vllm.worker.model_runner_base import (
@@ -1012,7 +1012,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1012
1013
def load_model(self) -> None:
1014
logger.info("Starting to load model %s...", self.model_config.model)
1015
- with CudaMemoryProfiler() as m:
1015
+ with DeviceMemoryProfiler() as m:
1016
self.model = get_model(model_config=self.model_config,
1017
device_config=self.device_config,
1018
load_config=self.load_config,
vllm/worker/xpu_model_runner.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
21
MultiModalInputs, MultiModalRegistry)
22
from vllm.sampling_params import SamplingParams
23
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
24
-from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
24
+from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad
25
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
26
from vllm.worker.model_runner_base import (
27
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -391,7 +391,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
391
self.model: nn.Module # Set after init_Model
392
393
def load_model(self) -> None:
394
- with CudaMemoryProfiler() as m:
394
+ with DeviceMemoryProfiler() as m:
395
self.model = get_model(
396
model_config=self.model_config,
397
device_config=self.device_config,