A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] Update outlines and increase its threadpool size (#11140)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Dec 14, 2024 at 02:46 UTC
48259264a4012e756215adc87e3682bf1e7dfee9
2 files changed
+11
-2
requirements-common.txt
+1
-1
@@ -19,7 +19,7 @@ prometheus_client >= 0.18.0
19
prometheus-fastapi-instrumentator >= 7.0.0
20
tiktoken >= 0.6.0 # Required for DBRX tokenizer
21
lm-format-enforcer >= 0.10.9, < 0.11
22
-outlines == 0.1.9
22
+outlines == 0.1.11
23
xgrammar >= 0.1.6; platform_machine == "x86_64"
24
typing_extensions >= 4.10
25
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
vllm/model_executor/guided_decoding/outlines_decoding.py
+10
-1
@@ -1,5 +1,6 @@
1
import asyncio
2
import concurrent.futures
3
+import os
4
from enum import Enum
5
from json import dumps as json_dumps
6
from re import escape as regex_escape
@@ -48,6 +49,11 @@ pair : UNESCAPED_STRING ":" value
49
50
global_thread_pool = None # used for generating logits processor fsm
51
52
+# It's not yet clear that using more provides a benefit, and it could
53
+# potentially starve other processes on the machine. We'll cap this for now and
54
+# adjust later if testing proves it to help overcome a bottleneck.
55
+_MAX_THREADPOOL_WORKERS = 16
56
+
57
58
async def get_outlines_guided_decoding_logits_processor(
59
guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
@@ -65,8 +71,11 @@ async def get_outlines_guided_decoding_logits_processor(
71
return None
72
73
if global_thread_pool is None:
74
+ max_workers = os.cpu_count() or 2
75
+ if max_workers > _MAX_THREADPOOL_WORKERS:
76
+ max_workers = _MAX_THREADPOOL_WORKERS
77
global_thread_pool = concurrent.futures.ThreadPoolExecutor(
69
- max_workers=2)
78
+ max_workers=max_workers)
79
loop = asyncio.get_running_loop()
80
81
return await loop.run_in_executor(global_thread_pool,