@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32c9d7f765
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

Report usage for beam search (#6404)

Simon Mo committed Jul 14, 2024 at 19:37 UTC 32c9d7f7650842cc20b2e66a4125ffe126619c50
2 files changed +18 -2
vllm/sampling_params.py
+5
@@ -189,6 +189,10 @@ class SamplingParams:
189
190 self._verify_args()
191 if self.use_beam_search:
192 + # Lazy import to avoid circular imports.
193 + from vllm.usage.usage_lib import set_runtime_usage_data
194 + set_runtime_usage_data("use_beam_search", True)
195 +
196 if not envs.VLLM_NO_DEPRECATION_WARNING:
197 logger.warning(
198 "[IMPORTANT] We plan to discontinue the support for beam "
@@ -196,6 +200,7 @@ class SamplingParams:
200 "https://github.com/vllm-project/vllm/issues/6226 for "
201 "more information. Set VLLM_NO_DEPRECATION_WARNING=1 to "
202 "suppress this warning.")
203 +
204 self._verify_beam_search()
205 else:
206 self._verify_non_beam_search()
vllm/usage/usage_lib.py
+13 -2
@@ -7,7 +7,7 @@ import time
7 from enum import Enum
8 from pathlib import Path
9 from threading import Thread
10 -from typing import Any, Dict, Optional
10 +from typing import Any, Dict, Optional, Union
11 from uuid import uuid4
12
13 import cpuinfo
@@ -25,6 +25,13 @@ _USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
25 _USAGE_STATS_ENABLED = None
26 _USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
27
28 +_GLOBAL_RUNTIME_DATA: Dict[str, Union[str, int, bool]] = {}
29 +
30 +
31 +def set_runtime_usage_data(key: str, value: Union[str, int, bool]) -> None:
32 + """Set global usage data that will be sent with every usage heartbeat."""
33 + _GLOBAL_RUNTIME_DATA[key] = value
34 +
35
36 def is_usage_stats_enabled():
37 """Determine whether or not we can send usage stats to the server.
@@ -187,7 +194,11 @@ class UsageMessage:
194 """
195 while True:
196 time.sleep(600)
190 - data = {"uuid": self.uuid, "log_time": _get_current_timestamp_ns()}
197 + data = {
198 + "uuid": self.uuid,
199 + "log_time": _get_current_timestamp_ns(),
200 + }
201 + data.update(_GLOBAL_RUNTIME_DATA)
202
203 self._write_to_file(data)
204 self._send_to_server(data)