A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
Report usage for beam search (#6404)
Simon Mo committed
Jul 14, 2024 at 19:37 UTC
32c9d7f7650842cc20b2e66a4125ffe126619c50
2 files changed
+18
-2
vllm/sampling_params.py
+5
@@ -189,6 +189,10 @@ class SamplingParams:
189
190
self._verify_args()
191
if self.use_beam_search:
192
+ # Lazy import to avoid circular imports.
193
+ from vllm.usage.usage_lib import set_runtime_usage_data
194
+ set_runtime_usage_data("use_beam_search", True)
195
+
196
if not envs.VLLM_NO_DEPRECATION_WARNING:
197
logger.warning(
198
"[IMPORTANT] We plan to discontinue the support for beam "
@@ -196,6 +200,7 @@ class SamplingParams:
200
"https://github.com/vllm-project/vllm/issues/6226 for "
201
"more information. Set VLLM_NO_DEPRECATION_WARNING=1 to "
202
"suppress this warning.")
203
+
204
self._verify_beam_search()
205
else:
206
self._verify_non_beam_search()
vllm/usage/usage_lib.py
+13
-2
@@ -7,7 +7,7 @@ import time
7
from enum import Enum
8
from pathlib import Path
9
from threading import Thread
10
-from typing import Any, Dict, Optional
10
+from typing import Any, Dict, Optional, Union
11
from uuid import uuid4
12
13
import cpuinfo
@@ -25,6 +25,13 @@ _USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
25
_USAGE_STATS_ENABLED = None
26
_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
27
28
+_GLOBAL_RUNTIME_DATA: Dict[str, Union[str, int, bool]] = {}
29
+
30
+
31
+def set_runtime_usage_data(key: str, value: Union[str, int, bool]) -> None:
32
+ """Set global usage data that will be sent with every usage heartbeat."""
33
+ _GLOBAL_RUNTIME_DATA[key] = value
34
+
35
36
def is_usage_stats_enabled():
37
"""Determine whether or not we can send usage stats to the server.
@@ -187,7 +194,11 @@ class UsageMessage:
194
"""
195
while True:
196
time.sleep(600)
190
- data = {"uuid": self.uuid, "log_time": _get_current_timestamp_ns()}
197
+ data = {
198
+ "uuid": self.uuid,
199
+ "log_time": _get_current_timestamp_ns(),
200
+ }
201
+ data.update(_GLOBAL_RUNTIME_DATA)
202
203
self._write_to_file(data)
204
self._send_to_server(data)