@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da7512215f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[misc] add cuda runtime version to usage data (#12190)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

youkaichao committed Jan 21, 2025 at 08:31 UTC da7512215f0b5c589c2747303b171357940c0614
1 file changed +3
vllm/usage/usage_lib.py
+3
@@ -130,6 +130,7 @@ class UsageMessage:
130 self.total_memory: Optional[int] = None
131 self.architecture: Optional[str] = None
132 self.platform: Optional[str] = None
133 + self.cuda_runtime: Optional[str] = None
134 self.gpu_count: Optional[int] = None
135 self.gpu_type: Optional[str] = None
136 self.gpu_memory_per_device: Optional[int] = None
@@ -169,6 +170,8 @@ class UsageMessage:
170 self.gpu_count = torch.cuda.device_count()
171 self.gpu_type = device_property.name
172 self.gpu_memory_per_device = device_property.total_memory
173 + if current_platform.is_cuda():
174 + self.cuda_runtime = torch.version.cuda
175 self.provider = _detect_cloud_provider()
176 self.architecture = platform.machine()
177 self.platform = platform.platform()