@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55fb97f7bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[2/N] API Server: Avoid ulimit footgun (#11530)

Robert Shaw committed Dec 26, 2024 at 18:43 UTC 55fb97f7bd61273fe8464a72866a72eaa88b5759
3 files changed +26 -2
vllm/entrypoints/api_server.py
+3 -1
@@ -21,7 +21,7 @@ from vllm.entrypoints.utils import with_cancellation
21 from vllm.logger import init_logger
22 from vllm.sampling_params import SamplingParams
23 from vllm.usage.usage_lib import UsageContext
24 -from vllm.utils import FlexibleArgumentParser, random_uuid
24 +from vllm.utils import FlexibleArgumentParser, random_uuid, set_ulimit
25 from vllm.version import __version__ as VLLM_VERSION
26
27 logger = init_logger("vllm.entrypoints.api_server")
@@ -119,6 +119,8 @@ async def run_server(args: Namespace,
119 logger.info("vLLM API server version %s", VLLM_VERSION)
120 logger.info("args: %s", args)
121
122 + set_ulimit()
123 +
124 app = await init_app(args, llm_engine)
125 assert engine is not None
126
vllm/entrypoints/openai/api_server.py
+5 -1
@@ -68,7 +68,7 @@ from vllm.entrypoints.utils import with_cancellation
68 from vllm.logger import init_logger
69 from vllm.usage.usage_lib import UsageContext
70 from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
71 - is_valid_ipv6_address)
71 + is_valid_ipv6_address, set_ulimit)
72 from vllm.version import __version__ as VLLM_VERSION
73
74 TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -727,6 +727,10 @@ async def run_server(args, **uvicorn_kwargs) -> None:
727 sock_addr = (args.host or "", args.port)
728 sock = create_server_socket(sock_addr)
729
730 + # workaround to avoid footguns where uvicorn drops requests with too
731 + # many concurrent requests active
732 + set_ulimit()
733 +
734 def signal_handler(*_) -> None:
735 # Interrupt server on sigterm while initializing
736 raise KeyboardInterrupt("terminated")
vllm/utils.py
+18
@@ -12,6 +12,7 @@ import inspect
12 import ipaddress
13 import os
14 import re
15 +import resource
16 import signal
17 import socket
18 import subprocess
@@ -1818,3 +1819,20 @@ def memory_profiling(
1819 result.non_torch_increase_in_bytes = current_cuda_memory_bytes - baseline_memory_in_bytes - weights_memory_in_bytes - diff.torch_memory_in_bytes # noqa
1820 result.profile_time = diff.timestamp
1821 result.non_kv_cache_memory_in_bytes = result.non_torch_increase_in_bytes + result.torch_peak_increase_in_bytes + result.weights_memory_in_bytes # noqa
1822 +
1823 +
1824 +# Adapted from: https://github.com/sgl-project/sglang/blob/f46f394f4d4dbe4aae85403dec006199b34d2840/python/sglang/srt/utils.py#L630 # noqa: E501Curre
1825 +def set_ulimit(target_soft_limit=65535):
1826 + resource_type = resource.RLIMIT_NOFILE
1827 + current_soft, current_hard = resource.getrlimit(resource_type)
1828 +
1829 + if current_soft < target_soft_limit:
1830 + try:
1831 + resource.setrlimit(resource_type,
1832 + (target_soft_limit, current_hard))
1833 + except ValueError as e:
1834 + logger.warning(
1835 + "Found ulimit of %s and failed to automatically increase"
1836 + "with error %s. This can cause fd limit errors like"
1837 + "`OSError: [Errno 24] Too many open files`. Consider "
1838 + "increasing with ulimit -n", current_soft, e)