@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a8a48d51e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Collect env var for usage stats (#12115)

Simon Mo committed Jan 18, 2025 at 19:07 UTC 7a8a48d51e51554645b233f870b71ef43bc70177
1 file changed +18
vllm/usage/usage_lib.py
+18
@@ -27,6 +27,17 @@ _USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
27
28 _GLOBAL_RUNTIME_DATA: Dict[str, Union[str, int, bool]] = {}
29
30 +_USAGE_ENV_VARS_TO_COLLECT = [
31 + "VLLM_USE_MODELSCOPE",
32 + "VLLM_USE_TRITON_FLASH_ATTN",
33 + "VLLM_ATTENTION_BACKEND",
34 + "VLLM_USE_FLASHINFER_SAMPLER",
35 + "VLLM_PP_LAYER_PARTITION",
36 + "VLLM_USE_TRITON_AWQ",
37 + "VLLM_USE_V1",
38 + "VLLM_ENABLE_V1_MULTIPROCESSING",
39 +]
40 +
41
42 def set_runtime_usage_data(key: str, value: Union[str, int, bool]) -> None:
43 """Set global usage data that will be sent with every usage heartbeat."""
@@ -122,6 +133,7 @@ class UsageMessage:
133 self.gpu_count: Optional[int] = None
134 self.gpu_type: Optional[str] = None
135 self.gpu_memory_per_device: Optional[int] = None
136 + self.env_var_json: Optional[str] = None
137
138 # vLLM Information
139 self.model_architecture: Optional[str] = None
@@ -176,6 +188,12 @@ class UsageMessage:
188 self.vllm_version = VLLM_VERSION
189 self.model_architecture = model_architecture
190
191 + # Environment variables
192 + self.env_var_json = json.dumps({
193 + env_var: getattr(envs, env_var)
194 + for env_var in _USAGE_ENV_VARS_TO_COLLECT
195 + })
196 +
197 # Metadata
198 self.log_time = _get_current_timestamp_ns()
199 self.source = envs.VLLM_USAGE_SOURCE