A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1] Collect env var for usage stats (#12115)
Simon Mo committed
Jan 18, 2025 at 19:07 UTC
7a8a48d51e51554645b233f870b71ef43bc70177
1 file changed
+18
vllm/usage/usage_lib.py
+18
@@ -27,6 +27,17 @@ _USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
27
28
_GLOBAL_RUNTIME_DATA: Dict[str, Union[str, int, bool]] = {}
29
30
+_USAGE_ENV_VARS_TO_COLLECT = [
31
+ "VLLM_USE_MODELSCOPE",
32
+ "VLLM_USE_TRITON_FLASH_ATTN",
33
+ "VLLM_ATTENTION_BACKEND",
34
+ "VLLM_USE_FLASHINFER_SAMPLER",
35
+ "VLLM_PP_LAYER_PARTITION",
36
+ "VLLM_USE_TRITON_AWQ",
37
+ "VLLM_USE_V1",
38
+ "VLLM_ENABLE_V1_MULTIPROCESSING",
39
+]
40
+
41
42
def set_runtime_usage_data(key: str, value: Union[str, int, bool]) -> None:
43
"""Set global usage data that will be sent with every usage heartbeat."""
@@ -122,6 +133,7 @@ class UsageMessage:
133
self.gpu_count: Optional[int] = None
134
self.gpu_type: Optional[str] = None
135
self.gpu_memory_per_device: Optional[int] = None
136
+ self.env_var_json: Optional[str] = None
137
138
# vLLM Information
139
self.model_architecture: Optional[str] = None
@@ -176,6 +188,12 @@ class UsageMessage:
188
self.vllm_version = VLLM_VERSION
189
self.model_architecture = model_architecture
190
191
+ # Environment variables
192
+ self.env_var_json = json.dumps({
193
+ env_var: getattr(envs, env_var)
194
+ for env_var in _USAGE_ENV_VARS_TO_COLLECT
195
+ })
196
+
197
# Metadata
198
self.log_time = _get_current_timestamp_ns()
199
self.source = envs.VLLM_USAGE_SOURCE