A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[misc] add cuda runtime version to usage data (#12190)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
youkaichao committed
Jan 21, 2025 at 08:31 UTC
da7512215f0b5c589c2747303b171357940c0614
1 file changed
+3
vllm/usage/usage_lib.py
+3
@@ -130,6 +130,7 @@ class UsageMessage:
130
self.total_memory: Optional[int] = None
131
self.architecture: Optional[str] = None
132
self.platform: Optional[str] = None
133
+ self.cuda_runtime: Optional[str] = None
134
self.gpu_count: Optional[int] = None
135
self.gpu_type: Optional[str] = None
136
self.gpu_memory_per_device: Optional[int] = None
@@ -169,6 +170,8 @@ class UsageMessage:
170
self.gpu_count = torch.cuda.device_count()
171
self.gpu_type = device_property.name
172
self.gpu_memory_per_device = device_property.total_memory
173
+ if current_platform.is_cuda():
174
+ self.cuda_runtime = torch.version.cuda
175
self.provider = _detect_cloud_provider()
176
self.architecture = platform.machine()
177
self.platform = platform.platform()