@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48259264a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Core] Update outlines and increase its threadpool size (#11140)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Dec 14, 2024 at 02:46 UTC 48259264a4012e756215adc87e3682bf1e7dfee9
2 files changed +11 -2
requirements-common.txt
+1 -1
@@ -19,7 +19,7 @@ prometheus_client >= 0.18.0
19 prometheus-fastapi-instrumentator >= 7.0.0
20 tiktoken >= 0.6.0 # Required for DBRX tokenizer
21 lm-format-enforcer >= 0.10.9, < 0.11
22 -outlines == 0.1.9
22 +outlines == 0.1.11
23 xgrammar >= 0.1.6; platform_machine == "x86_64"
24 typing_extensions >= 4.10
25 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
vllm/model_executor/guided_decoding/outlines_decoding.py
+10 -1
@@ -1,5 +1,6 @@
1 import asyncio
2 import concurrent.futures
3 +import os
4 from enum import Enum
5 from json import dumps as json_dumps
6 from re import escape as regex_escape
@@ -48,6 +49,11 @@ pair : UNESCAPED_STRING ":" value
49
50 global_thread_pool = None # used for generating logits processor fsm
51
52 +# It's not yet clear that using more provides a benefit, and it could
53 +# potentially starve other processes on the machine. We'll cap this for now and
54 +# adjust later if testing proves it to help overcome a bottleneck.
55 +_MAX_THREADPOOL_WORKERS = 16
56 +
57
58 async def get_outlines_guided_decoding_logits_processor(
59 guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
@@ -65,8 +71,11 @@ async def get_outlines_guided_decoding_logits_processor(
71 return None
72
73 if global_thread_pool is None:
74 + max_workers = os.cpu_count() or 2
75 + if max_workers > _MAX_THREADPOOL_WORKERS:
76 + max_workers = _MAX_THREADPOOL_WORKERS
77 global_thread_pool = concurrent.futures.ThreadPoolExecutor(
69 - max_workers=2)
78 + max_workers=max_workers)
79 loop = asyncio.get_running_loop()
80
81 return await loop.run_in_executor(global_thread_pool,