A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[V1] Guard Against Main Thread Usage (#14972)
Signed-off-by: rshaw@neuralmagic.com <robertgshaw2@gmail.com>
Robert Shaw committed
Mar 17, 2025 at 16:23 UTC
e41e1602630f49a4f1678a976985b6a9355b58da
1 file changed
+6
vllm/engine/arg_utils.py
+6
@@ -3,6 +3,7 @@
3
import argparse
4
import dataclasses
5
import json
6
+import threading
7
from dataclasses import dataclass
8
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
9
Tuple, Type, Union, cast, get_args)
@@ -1576,6 +1577,11 @@ class EngineArgs:
1577
#############################################################
1578
# Experimental Features - allow users to opt in.
1579
1580
+ # Signal Handlers requires running in main thread.
1581
+ if (threading.current_thread() != threading.main_thread()
1582
+ and _warn_or_fallback("Engine in background thread")):
1583
+ return False
1584
+
1585
# LoRA is supported on V1, but off by default for now.
1586
if self.enable_lora and _warn_or_fallback("LORA"):
1587
return False