A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Bugfix] set VLLM_WORKER_MULTIPROC_METHOD=spawn for vllm.entrypoionts.openai.api_server (#15700)
Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com>
Jinzhen Lin committed
Mar 29, 2025 at 12:12 UTC
5b800f0932c0d6661cb3aa85a62d89265197a034
3 files changed
+31
-27
vllm/entrypoints/cli/main.py
+2
-26
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
# The CLI entrypoint to vLLM.
4
-import os
4
import signal
5
import sys
6
@@ -9,11 +8,9 @@ import vllm.entrypoints.cli.benchmark.main
8
import vllm.entrypoints.cli.openai
9
import vllm.entrypoints.cli.serve
10
import vllm.version
12
-from vllm.logger import init_logger
11
+from vllm.entrypoints.utils import cli_env_setup
12
from vllm.utils import FlexibleArgumentParser
13
15
-logger = init_logger(__name__)
16
-
14
CMD_MODULES = [
15
vllm.entrypoints.cli.openai,
16
vllm.entrypoints.cli.serve,
@@ -30,29 +27,8 @@ def register_signal_handlers():
27
signal.signal(signal.SIGTSTP, signal_handler)
28
29
33
-def env_setup():
34
- # The safest multiprocessing method is `spawn`, as the default `fork` method
35
- # is not compatible with some accelerators. The default method will be
36
- # changing in future versions of Python, so we should use it explicitly when
37
- # possible.
38
- #
39
- # We only set it here in the CLI entrypoint, because changing to `spawn`
40
- # could break some existing code using vLLM as a library. `spawn` will cause
41
- # unexpected behavior if the code is not protected by
42
- # `if __name__ == "__main__":`.
43
- #
44
- # References:
45
- # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
46
- # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
47
- # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
48
- # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
49
- if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
50
- logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
51
- os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
52
-
53
-
30
def main():
55
- env_setup()
31
+ cli_env_setup()
32
33
parser = FlexibleArgumentParser(description="vLLM CLI")
34
parser.add_argument('-v',
vllm/entrypoints/openai/api_server.py
+3
-1
@@ -82,7 +82,8 @@ from vllm.entrypoints.openai.serving_tokenization import (
82
from vllm.entrypoints.openai.serving_transcription import (
83
OpenAIServingTranscription)
84
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
85
-from vllm.entrypoints.utils import load_aware_call, with_cancellation
85
+from vllm.entrypoints.utils import (cli_env_setup, load_aware_call,
86
+ with_cancellation)
87
from vllm.logger import init_logger
88
from vllm.reasoning import ReasoningParserManager
89
from vllm.transformers_utils.config import (
@@ -1106,6 +1107,7 @@ if __name__ == "__main__":
1107
# NOTE(simon):
1108
# This section should be in sync with vllm/entrypoints/cli/main.py for CLI
1109
# entrypoints.
1110
+ cli_env_setup()
1111
parser = FlexibleArgumentParser(
1112
description="vLLM OpenAI-Compatible RESTful API server.")
1113
parser = make_arg_parser(parser)
vllm/entrypoints/utils.py
+26
@@ -2,11 +2,16 @@
2
3
import asyncio
4
import functools
5
+import os
6
7
from fastapi import Request
8
from fastapi.responses import JSONResponse, StreamingResponse
9
from starlette.background import BackgroundTask, BackgroundTasks
10
11
+from vllm.logger import init_logger
12
+
13
+logger = init_logger(__name__)
14
+
15
16
async def listen_for_disconnect(request: Request) -> None:
17
"""Returns if a disconnect message is received"""
@@ -108,3 +113,24 @@ def load_aware_call(func):
113
return response
114
115
return wrapper
116
+
117
+
118
+def cli_env_setup():
119
+ # The safest multiprocessing method is `spawn`, as the default `fork` method
120
+ # is not compatible with some accelerators. The default method will be
121
+ # changing in future versions of Python, so we should use it explicitly when
122
+ # possible.
123
+ #
124
+ # We only set it here in the CLI entrypoint, because changing to `spawn`
125
+ # could break some existing code using vLLM as a library. `spawn` will cause
126
+ # unexpected behavior if the code is not protected by
127
+ # `if __name__ == "__main__":`.
128
+ #
129
+ # References:
130
+ # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
131
+ # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
132
+ # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
133
+ # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
134
+ if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
135
+ logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
136
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"