A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Bugfix] Flush TunableOp results before worker processes are destroyed. (#13623)
Signed-off-by: Nichols A. Romero <nick.romero@amd.com>
Nichols A. Romero committed
Feb 25, 2025 at 05:08 UTC
fa82074167b171018acb856ec0b26e54f6fca0cb
1 file changed
+9
vllm/executor/multiproc_worker_utils.py
+9
@@ -250,6 +250,15 @@ def _run_worker_process(
250
except Exception:
251
logger.exception("Worker failed")
252
253
+ # Flush TunableOp results when TunableOp is enabled and
254
+ # online (in situ) tuning is enabled.
255
+ # Offline tuning API (record_untuned_is_enabled()) only
256
+ # available in PyTorch 2.6 or later.
257
+ import torch.cuda.tunable as tunable
258
+ if (tunable.is_enabled() and tunable.tuning_is_enabled()
259
+ and not tunable.record_untuned_is_enabled()):
260
+ tunable.write_file()
261
+
262
logger.info("Worker exiting")
263
264