A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[BugFix] Fix clean shutdown issues (#8492)
Nick Hill committed
Sep 16, 2024 at 17:33 UTC
acd5511b6d0e196b273b6250201115b5c5cfd1ca
11 files changed
+215
-136
tests/async_engine/test_async_llm_engine.py
+8
-2
@@ -26,6 +26,11 @@ class RequestOutput:
26
finished: bool = False
27
28
29
+@dataclass
30
+class MockModelConfig:
31
+ use_async_output_proc = True
32
+
33
+
34
class MockEngine:
35
36
def __init__(self):
@@ -35,6 +40,7 @@ class MockEngine:
40
self.request_id = None
41
# Ugly, remove dependency when possible
42
self.parallel_config = ParallelConfig(1, 1, False)
43
+ self.model_config = MockModelConfig()
44
45
async def step_async(self, virtual_engine):
46
# PP size is 1, ignore virtual engine
@@ -80,7 +86,7 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87
@pytest.mark.asyncio
88
async def test_new_requests_event():
83
- engine = MockAsyncLLMEngine(worker_use_ray=False)
89
+ engine = MockAsyncLLMEngine()
90
engine.start_background_loop()
91
await asyncio.sleep(0.01)
92
assert engine.engine.step_calls == 0
@@ -113,7 +119,7 @@ async def test_new_requests_event():
119
assert engine.engine.add_request_calls == 3
120
assert engine.engine.step_calls == old_step_calls + 1
121
116
- engine = MockAsyncLLMEngine(worker_use_ray=True)
122
+ engine = MockAsyncLLMEngine()
123
assert engine.get_model_config() is not None
124
assert engine.get_tokenizer() is not None
125
assert engine.get_decoding_config() is not None
vllm/engine/async_llm_engine.py
+45
-25
@@ -1,8 +1,10 @@
1
import asyncio
2
import time
3
+import weakref
4
from functools import partial
5
from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6
Mapping, Optional, Set, Tuple, Type, Union)
7
+from weakref import ReferenceType
8
9
import vllm.envs as envs
10
from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
@@ -26,6 +28,7 @@ from vllm.sampling_params import SamplingParams
28
from vllm.sequence import ExecuteModelRequest
29
from vllm.transformers_utils.tokenizer import AnyTokenizer
30
from vllm.usage.usage_lib import UsageContext
31
+from vllm.utils import weak_bind
32
33
logger = init_logger(__name__)
34
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -450,9 +453,6 @@ class AsyncLLMEngine:
453
method yields the outputs from the :class:`LLMEngine` to the caller.
454
455
Args:
453
- worker_use_ray: Whether to use Ray for model workers. Required for
454
- distributed execution. Should be the same as
455
- `parallel_config.worker_use_ray`.
456
log_requests: Whether to log the requests.
457
start_engine_loop: If True, the background task to run the engine
458
will be automatically started in the generate call.
@@ -463,23 +463,22 @@ class AsyncLLMEngine:
463
_engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
464
465
def __init__(self,
466
- worker_use_ray: bool,
466
*args,
467
log_requests: bool = True,
468
start_engine_loop: bool = True,
469
**kwargs) -> None:
471
- self.worker_use_ray = worker_use_ray
470
self.log_requests = log_requests
471
self.engine = self._engine_class(*args, **kwargs)
472
473
# This ensures quick processing of request outputs
474
# so the append to asyncio queues is not delayed,
475
# especially for multi-step.
478
- #
479
- self.use_process_request_outputs_callback = True
476
+ self.use_process_request_outputs_callback = (
477
+ self.engine.model_config.use_async_output_proc)
478
+
479
if self.use_process_request_outputs_callback:
480
self.engine.process_request_outputs_callback = \
482
- self.process_request_outputs
481
+ weak_bind(self.process_request_outputs)
482
483
self.background_loop: Optional[asyncio.Future] = None
484
# We need to keep a reference to unshielded
@@ -492,6 +491,11 @@ class AsyncLLMEngine:
491
# Lazy initialized fields
492
self._request_tracker: RequestTracker
493
494
+ def __del__(self):
495
+ if rt := getattr(self, "request_tracker", None):
496
+ # Wake up engine loop so that it will exit cleanly
497
+ rt.new_requests_event.set()
498
+
499
@classmethod
500
def _get_executor_cls(
501
cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
@@ -502,15 +506,12 @@ class AsyncLLMEngine:
506
raise TypeError(
507
"distributed_executor_backend must be a subclass of "
508
f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
505
- if distributed_executor_backend.uses_ray: # type: ignore
506
- initialize_ray_cluster(engine_config.parallel_config)
509
executor_class = distributed_executor_backend
510
elif engine_config.device_config.device_type == "neuron":
511
from vllm.executor.neuron_executor import NeuronExecutorAsync
512
executor_class = NeuronExecutorAsync
513
elif engine_config.device_config.device_type == "tpu":
514
if distributed_executor_backend == "ray":
513
- initialize_ray_cluster(engine_config.parallel_config)
515
from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
516
executor_class = RayTPUExecutorAsync
517
else:
@@ -531,11 +532,9 @@ class AsyncLLMEngine:
532
from vllm.executor.xpu_executor import XPUExecutorAsync
533
executor_class = XPUExecutorAsync
534
elif distributed_executor_backend == "ray":
534
- initialize_ray_cluster(engine_config.parallel_config)
535
from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
536
executor_class = RayXPUExecutorAsync
537
elif distributed_executor_backend == "mp":
538
- initialize_ray_cluster(engine_config.parallel_config)
538
from vllm.executor.multiproc_xpu_executor import (
539
MultiprocessingXPUExecutorAsync)
540
executor_class = MultiprocessingXPUExecutorAsync
@@ -543,7 +542,6 @@ class AsyncLLMEngine:
542
raise RuntimeError(
543
"Not supported distributed execution model on XPU device.")
544
elif distributed_executor_backend == "ray":
546
- initialize_ray_cluster(engine_config.parallel_config)
545
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
546
executor_class = RayGPUExecutorAsync
547
elif distributed_executor_backend == "mp":
@@ -559,19 +557,23 @@ class AsyncLLMEngine:
557
def from_engine_args(
558
cls,
559
engine_args: AsyncEngineArgs,
560
+ engine_config: Optional[EngineConfig] = None,
561
start_engine_loop: bool = True,
562
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
563
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
564
) -> "AsyncLLMEngine":
565
"""Creates an async LLM engine from the engine arguments."""
566
# Create the engine configs.
568
- engine_config = engine_args.create_engine_config()
567
+ if engine_config is None:
568
+ engine_config = engine_args.create_engine_config()
569
570
executor_class = cls._get_executor_cls(engine_config)
571
572
+ if executor_class.uses_ray:
573
+ initialize_ray_cluster(engine_config.parallel_config)
574
+
575
# Create the async LLM engine.
576
engine = cls(
574
- executor_class.uses_ray,
577
**engine_config.to_dict(),
578
executor_class=executor_class,
579
log_requests=not engine_args.disable_log_requests,
@@ -628,7 +630,7 @@ class AsyncLLMEngine:
630
self._request_tracker = RequestTracker()
631
632
self._background_loop_unshielded = asyncio.get_event_loop(
631
- ).create_task(self.run_engine_loop())
633
+ ).create_task(self.run_engine_loop(weakref.ref(self)))
634
self._background_loop_unshielded.add_done_callback(
635
partial(_log_task_completion, error_callback=self._error_callback))
636
self.background_loop = asyncio.shield(self._background_loop_unshielded)
@@ -698,9 +700,16 @@ class AsyncLLMEngine:
700
async def _engine_abort(self, request_ids: Iterable[str]):
701
self.engine.abort_request(request_ids)
702
701
- async def run_engine_loop(self):
703
+ @staticmethod
704
+ async def run_engine_loop(engine_ref: ReferenceType):
705
+ """We use a weakref to the engine so that the running loop
706
+ doesn't prevent the engine being garbage collected."""
707
+ engine: Optional["AsyncLLMEngine"] = engine_ref()
708
+ if not engine:
709
+ return
710
+
711
pipeline_parallel_size = \
703
- self.engine.parallel_config.pipeline_parallel_size
712
+ engine.engine.parallel_config.pipeline_parallel_size
713
has_requests_in_progress = [False] * pipeline_parallel_size
714
while True:
715
if not any(has_requests_in_progress):
@@ -711,11 +720,21 @@ class AsyncLLMEngine:
720
# timeout, and unblocks the RPC thread in the workers so that
721
# they can process any other queued control plane messages,
722
# such as add/remove lora adapters.
714
- await self.engine.stop_remote_worker_execution_loop_async()
715
- await self._request_tracker.wait_for_new_requests()
723
+ await engine.engine.stop_remote_worker_execution_loop_async()
724
+ request_tracker = engine._request_tracker
725
+ # Allow engine to be garbage collected while
726
+ # waiting for new requests
727
+ del engine
728
+ await asyncio.sleep(0)
729
+ if engine_ref() is None:
730
+ return
731
+ await request_tracker.wait_for_new_requests()
732
+ engine = engine_ref()
733
+ if not engine:
734
+ return
735
logger.debug("Got new requests!")
736
requests_in_progress = [
718
- asyncio.create_task(self.engine_step(ve))
737
+ asyncio.create_task(engine.engine_step(ve))
738
for ve in range(pipeline_parallel_size)
739
]
740
has_requests_in_progress = [True] * pipeline_parallel_size
@@ -733,19 +752,20 @@ class AsyncLLMEngine:
752
result = task.result()
753
virtual_engine = requests_in_progress.index(task)
754
has_unfinished_requests = (
736
- self.engine.has_unfinished_requests_for_virtual_engine(
755
+ engine.engine.
756
+ has_unfinished_requests_for_virtual_engine(
757
virtual_engine))
758
if result or has_unfinished_requests:
759
requests_in_progress[virtual_engine] = (
760
asyncio.create_task(
741
- self.engine_step(virtual_engine)))
761
+ engine.engine_step(virtual_engine)))
762
has_requests_in_progress[virtual_engine] = True
763
else:
764
has_requests_in_progress[virtual_engine] = False
765
except asyncio.TimeoutError as exc:
766
logger.error(
767
"Engine iteration timed out. This should never happen!")
748
- self.set_errored(exc)
768
+ engine.set_errored(exc)
769
raise
770
await asyncio.sleep(0)
771
vllm/engine/llm_engine.py
+13
-8
@@ -1,8 +1,8 @@
1
-import functools
1
import time
2
from collections import deque
3
from contextlib import contextmanager
4
from dataclasses import dataclass
5
+from functools import partial
6
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51
BaseTokenizerGroup, init_tokenizer_from_configs)
52
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53
usage_message)
54
-from vllm.utils import Counter, Device
54
+from vllm.utils import Counter, Device, weak_bind
55
from vllm.version import __version__ as VLLM_VERSION
56
57
logger = init_logger(__name__)
@@ -382,11 +382,16 @@ class LLMEngine:
382
for _ in range(self.parallel_config.pipeline_parallel_size)
383
]
384
385
- self.async_callbacks = [
386
- functools.partial(self._process_model_outputs,
387
- ctx=self.scheduler_contexts[v_id])
388
- for v_id in range(self.parallel_config.pipeline_parallel_size)
389
- ]
385
+ if model_config.use_async_output_proc:
386
+ process_model_outputs = weak_bind(self._process_model_outputs)
387
+
388
+ self.async_callbacks = [
389
+ partial(process_model_outputs,
390
+ ctx=self.scheduler_contexts[v_id])
391
+ for v_id in range(self.parallel_config.pipeline_parallel_size)
392
+ ]
393
+ else:
394
+ self.async_callbacks = []
395
396
# Currently used by AsyncLLMEngine to ensure quick append
397
# of request outputs to asyncio queues
@@ -869,8 +874,8 @@ class LLMEngine:
874
"""
875
return self.scheduler[virtual_engine].has_unfinished_seqs()
876
877
+ @staticmethod
878
def _process_sequence_group_outputs(
873
- self,
879
seq_group: SequenceGroup,
880
outputs: List[EmbeddingSequenceGroupOutput],
881
) -> None:
vllm/entrypoints/launcher.py
+10
-11
@@ -1,21 +1,20 @@
1
import asyncio
2
import signal
3
from http import HTTPStatus
4
-from typing import Any
4
+from typing import Any, Optional
5
6
import uvicorn
7
-from fastapi import FastAPI, Response
7
+from fastapi import FastAPI, Request, Response
8
9
from vllm import envs
10
from vllm.engine.async_llm_engine import AsyncEngineDeadError
11
-from vllm.engine.protocol import AsyncEngineClient
11
from vllm.logger import init_logger
12
from vllm.utils import find_process_using_port
13
14
logger = init_logger(__name__)
15
16
18
-async def serve_http(app: FastAPI, engine: AsyncEngineClient,
17
+async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
18
**uvicorn_kwargs: Any):
19
logger.info("Available routes are:")
20
for route in app.routes:
@@ -29,16 +28,16 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
28
29
# Set concurrency limits in uvicorn if running in multiprocessing mode
30
# since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
32
- if engine.limit_concurrency is not None:
31
+ if limit_concurrency is not None:
32
logger.info(
33
"Launching Uvicorn with --limit_concurrency %s. To avoid this "
34
"limit at the expense of performance run with "
36
- "--disable-frontend-multiprocessing", engine.limit_concurrency)
37
- uvicorn_kwargs["limit_concurrency"] = engine.limit_concurrency
35
+ "--disable-frontend-multiprocessing", limit_concurrency)
36
+ uvicorn_kwargs["limit_concurrency"] = limit_concurrency
37
38
config = uvicorn.Config(app, **uvicorn_kwargs)
39
server = uvicorn.Server(config)
41
- _add_shutdown_handlers(app, server, engine)
40
+ _add_shutdown_handlers(app, server)
41
42
loop = asyncio.get_running_loop()
43
@@ -68,15 +67,15 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
67
return server.shutdown()
68
69
71
-def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server,
72
- engine: AsyncEngineClient) -> None:
70
+def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
71
"""Adds handlers for fatal errors that should crash the server"""
72
73
@app.exception_handler(RuntimeError)
76
- async def runtime_error_handler(_, __):
74
+ async def runtime_error_handler(request: Request, __):
75
"""On generic runtime error, check to see if the engine has died.
76
It probably has, in which case the server will no longer be able to
77
handle requests. Trigger a graceful shutdown with a SIGTERM."""
78
+ engine = request.app.state.engine_client
79
if (not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine.errored
80
and not engine.is_running):
81
logger.fatal("AsyncLLMEngine has failed, terminating server "
vllm/entrypoints/openai/api_server.py
+109
-72
@@ -4,16 +4,20 @@ import inspect
4
import multiprocessing
5
import os
6
import re
7
+import signal
8
import tempfile
9
from argparse import Namespace
10
from contextlib import asynccontextmanager
11
+from functools import partial
12
from http import HTTPStatus
13
from typing import AsyncIterator, Optional, Set
14
15
+import uvloop
16
from fastapi import APIRouter, FastAPI, Request
17
from fastapi.exceptions import RequestValidationError
18
from fastapi.middleware.cors import CORSMiddleware
19
from fastapi.responses import JSONResponse, Response, StreamingResponse
20
+from starlette.datastructures import State
21
from starlette.routing import Mount
22
from typing_extensions import assert_never
23
@@ -54,12 +58,6 @@ from vllm.version import __version__ as VLLM_VERSION
58
59
TIMEOUT_KEEP_ALIVE = 5 # seconds
60
57
-async_engine_client: AsyncEngineClient
58
-engine_args: AsyncEngineArgs
59
-openai_serving_chat: OpenAIServingChat
60
-openai_serving_completion: OpenAIServingCompletion
61
-openai_serving_embedding: OpenAIServingEmbedding
62
-openai_serving_tokenization: OpenAIServingTokenization
61
prometheus_multiproc_dir: tempfile.TemporaryDirectory
62
63
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
@@ -83,18 +81,28 @@ def model_is_embedding(model_name: str, trust_remote_code: bool,
81
82
@asynccontextmanager
83
async def lifespan(app: FastAPI):
86
-
87
- async def _force_log():
88
- while True:
89
- await asyncio.sleep(10)
90
- await async_engine_client.do_log_stats()
91
-
92
- if not engine_args.disable_log_stats:
93
- task = asyncio.create_task(_force_log())
94
- _running_tasks.add(task)
95
- task.add_done_callback(_running_tasks.remove)
96
-
97
- yield
84
+ try:
85
+ if app.state.log_stats:
86
+ async_engine_client = app.state.engine_client
87
+
88
+ async def _force_log():
89
+ while True:
90
+ await asyncio.sleep(10)
91
+ await async_engine_client.do_log_stats()
92
+
93
+ task = asyncio.create_task(_force_log())
94
+ _running_tasks.add(task)
95
+ task.add_done_callback(_running_tasks.remove)
96
+ else:
97
+ task = None
98
+ try:
99
+ yield
100
+ finally:
101
+ if task is not None:
102
+ task.cancel()
103
+ finally:
104
+ # Ensure app state including engine ref is gc'd
105
+ del app.state
106
107
108
@asynccontextmanager
@@ -103,16 +111,10 @@ async def build_async_engine_client(
111
112
# Context manager to handle async_engine_client lifecycle
113
# Ensures everything is shutdown and cleaned up on error/exit
106
- global engine_args
114
engine_args = AsyncEngineArgs.from_cli_args(args)
115
109
- # Backend itself still global for the silly lil' health handler
110
- global async_engine_client
111
-
116
async with build_async_engine_client_from_engine_args(
117
engine_args, args.disable_frontend_multiprocessing) as engine:
114
-
115
- async_engine_client = engine # type: ignore[assignment]
118
yield engine
119
120
@@ -134,12 +136,22 @@ async def build_async_engine_client_from_engine_args(
136
if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
137
engine_args.quantization, engine_args.revision)
138
or disable_frontend_multiprocessing):
137
- engine_client = AsyncLLMEngine.from_engine_args(
138
- engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
139
- try:
140
- yield engine_client
141
- finally:
142
- engine_client.shutdown_background_loop()
139
+ engine_config = engine_args.create_engine_config()
140
+ uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
141
+ "uses_ray", False)
142
+
143
+ build_engine = partial(AsyncLLMEngine.from_engine_args,
144
+ engine_args=engine_args,
145
+ engine_config=engine_config,
146
+ usage_context=UsageContext.OPENAI_API_SERVER)
147
+ if uses_ray:
148
+ # Must run in main thread with ray for its signal handlers to work
149
+ engine_client = build_engine()
150
+ else:
151
+ engine_client = await asyncio.get_running_loop().run_in_executor(
152
+ None, build_engine)
153
+
154
+ yield engine_client
155
return
156
157
# Otherwise, use the multiprocessing AsyncLLMEngine.
@@ -241,16 +253,36 @@ def mount_metrics(app: FastAPI):
253
app.routes.append(metrics_route)
254
255
256
+def chat(request: Request) -> OpenAIServingChat:
257
+ return request.app.state.openai_serving_chat
258
+
259
+
260
+def completion(request: Request) -> OpenAIServingCompletion:
261
+ return request.app.state.openai_serving_completion
262
+
263
+
264
+def tokenization(request: Request) -> OpenAIServingTokenization:
265
+ return request.app.state.openai_serving_tokenization
266
+
267
+
268
+def embedding(request: Request) -> OpenAIServingEmbedding:
269
+ return request.app.state.openai_serving_embedding
270
+
271
+
272
+def engine_client(request: Request) -> AsyncEngineClient:
273
+ return request.app.state.engine_client
274
+
275
+
276
@router.get("/health")
245
-async def health() -> Response:
277
+async def health(raw_request: Request) -> Response:
278
"""Health check."""
247
- await async_engine_client.check_health()
279
+ await engine_client(raw_request).check_health()
280
return Response(status_code=200)
281
282
283
@router.post("/tokenize")
252
-async def tokenize(request: TokenizeRequest):
253
- generator = await openai_serving_tokenization.create_tokenize(request)
284
+async def tokenize(request: TokenizeRequest, raw_request: Request):
285
+ generator = await tokenization(raw_request).create_tokenize(request)
286
if isinstance(generator, ErrorResponse):
287
return JSONResponse(content=generator.model_dump(),
288
status_code=generator.code)
@@ -261,8 +293,8 @@ async def tokenize(request: TokenizeRequest):
293
294
295
@router.post("/detokenize")
264
-async def detokenize(request: DetokenizeRequest):
265
- generator = await openai_serving_tokenization.create_detokenize(request)
296
+async def detokenize(request: DetokenizeRequest, raw_request: Request):
297
+ generator = await tokenization(raw_request).create_detokenize(request)
298
if isinstance(generator, ErrorResponse):
299
return JSONResponse(content=generator.model_dump(),
300
status_code=generator.code)
@@ -273,8 +305,8 @@ async def detokenize(request: DetokenizeRequest):
305
306
307
@router.get("/v1/models")
276
-async def show_available_models():
277
- models = await openai_serving_completion.show_available_models()
308
+async def show_available_models(raw_request: Request):
309
+ models = await completion(raw_request).show_available_models()
310
return JSONResponse(content=models.model_dump())
311
312
@@ -288,7 +320,7 @@ async def show_version():
320
async def create_chat_completion(request: ChatCompletionRequest,
321
raw_request: Request):
322
291
- generator = await openai_serving_chat.create_chat_completion(
323
+ generator = await chat(raw_request).create_chat_completion(
324
request, raw_request)
325
326
if isinstance(generator, ErrorResponse):
@@ -303,7 +335,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
335
336
@router.post("/v1/completions")
337
async def create_completion(request: CompletionRequest, raw_request: Request):
306
- generator = await openai_serving_completion.create_completion(
338
+ generator = await completion(raw_request).create_completion(
339
request, raw_request)
340
if isinstance(generator, ErrorResponse):
341
return JSONResponse(content=generator.model_dump(),
@@ -316,7 +348,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
348
349
@router.post("/v1/embeddings")
350
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
319
- generator = await openai_serving_embedding.create_embedding(
351
+ generator = await embedding(raw_request).create_embedding(
352
request, raw_request)
353
if isinstance(generator, ErrorResponse):
354
return JSONResponse(content=generator.model_dump(),
@@ -333,16 +365,16 @@ if envs.VLLM_TORCH_PROFILER_DIR:
365
"used for local development!")
366
367
@router.post("/start_profile")
336
- async def start_profile():
368
+ async def start_profile(raw_request: Request):
369
logger.info("Starting profiler...")
338
- await async_engine_client.start_profile()
370
+ await engine_client(raw_request).start_profile()
371
logger.info("Profiler started.")
372
return Response(status_code=200)
373
374
@router.post("/stop_profile")
343
- async def stop_profile():
375
+ async def stop_profile(raw_request: Request):
376
logger.info("Stopping profiler...")
345
- await async_engine_client.stop_profile()
377
+ await engine_client(raw_request).stop_profile()
378
logger.info("Profiler stopped.")
379
return Response(status_code=200)
380
@@ -353,13 +385,14 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
385
"This should ONLY be used for local development!")
386
387
@router.post("/v1/load_lora_adapter")
356
- async def load_lora_adapter(request: LoadLoraAdapterRequest):
357
- response = await openai_serving_chat.load_lora_adapter(request)
388
+ async def load_lora_adapter(request: LoadLoraAdapterRequest,
389
+ raw_request: Request):
390
+ response = await chat(raw_request).load_lora_adapter(request)
391
if isinstance(response, ErrorResponse):
392
return JSONResponse(content=response.model_dump(),
393
status_code=response.code)
394
362
- response = await openai_serving_completion.load_lora_adapter(request)
395
+ response = await completion(raw_request).load_lora_adapter(request)
396
if isinstance(response, ErrorResponse):
397
return JSONResponse(content=response.model_dump(),
398
status_code=response.code)
@@ -367,13 +400,14 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
400
return Response(status_code=200, content=response)
401
402
@router.post("/v1/unload_lora_adapter")
370
- async def unload_lora_adapter(request: UnloadLoraAdapterRequest):
371
- response = await openai_serving_chat.unload_lora_adapter(request)
403
+ async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
404
+ raw_request: Request):
405
+ response = await chat(raw_request).unload_lora_adapter(request)
406
if isinstance(response, ErrorResponse):
407
return JSONResponse(content=response.model_dump(),
408
status_code=response.code)
409
376
- response = await openai_serving_completion.unload_lora_adapter(request)
410
+ response = await completion(raw_request).unload_lora_adapter(request)
411
if isinstance(response, ErrorResponse):
412
return JSONResponse(content=response.model_dump(),
413
status_code=response.code)
@@ -398,7 +432,8 @@ def build_app(args: Namespace) -> FastAPI:
432
433
@app.exception_handler(RequestValidationError)
434
async def validation_exception_handler(_, exc):
401
- err = openai_serving_chat.create_error_response(message=str(exc))
435
+ chat = app.state.openai_serving_chat
436
+ err = chat.create_error_response(message=str(exc))
437
return JSONResponse(err.model_dump(),
438
status_code=HTTPStatus.BAD_REQUEST)
439
@@ -430,30 +465,26 @@ def build_app(args: Namespace) -> FastAPI:
465
return app
466
467
433
-async def init_app(
468
+def init_app_state(
469
async_engine_client: AsyncEngineClient,
470
+ model_config: ModelConfig,
471
+ state: State,
472
args: Namespace,
436
-) -> FastAPI:
437
- app = build_app(args)
438
-
473
+) -> None:
474
if args.served_model_name is not None:
475
served_model_names = args.served_model_name
476
else:
477
served_model_names = [args.model]
478
444
- model_config = await async_engine_client.get_model_config()
445
-
479
if args.disable_log_requests:
480
request_logger = None
481
else:
482
request_logger = RequestLogger(max_log_len=args.max_log_len)
483
451
- global openai_serving_chat
452
- global openai_serving_completion
453
- global openai_serving_embedding
454
- global openai_serving_tokenization
484
+ state.engine_client = async_engine_client
485
+ state.log_stats = not args.disable_log_stats
486
456
- openai_serving_chat = OpenAIServingChat(
487
+ state.openai_serving_chat = OpenAIServingChat(
488
async_engine_client,
489
model_config,
490
served_model_names,
@@ -465,7 +496,7 @@ async def init_app(
496
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
497
enable_auto_tools=args.enable_auto_tool_choice,
498
tool_parser=args.tool_call_parser)
468
- openai_serving_completion = OpenAIServingCompletion(
499
+ state.openai_serving_completion = OpenAIServingCompletion(
500
async_engine_client,
501
model_config,
502
served_model_names,
@@ -474,13 +505,13 @@ async def init_app(
505
request_logger=request_logger,
506
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
507
)
477
- openai_serving_embedding = OpenAIServingEmbedding(
508
+ state.openai_serving_embedding = OpenAIServingEmbedding(
509
async_engine_client,
510
model_config,
511
served_model_names,
512
request_logger=request_logger,
513
)
483
- openai_serving_tokenization = OpenAIServingTokenization(
514
+ state.openai_serving_tokenization = OpenAIServingTokenization(
515
async_engine_client,
516
model_config,
517
served_model_names,
@@ -488,25 +519,31 @@ async def init_app(
519
request_logger=request_logger,
520
chat_template=args.chat_template,
521
)
491
- app.root_path = args.root_path
492
-
493
- return app
522
523
524
async def run_server(args, **uvicorn_kwargs) -> None:
525
logger.info("vLLM API server version %s", VLLM_VERSION)
526
logger.info("args: %s", args)
527
528
+ def signal_handler(*_) -> None:
529
+ # Interrupt server on sigterm while initializing
530
+ raise KeyboardInterrupt("terminated")
531
+
532
+ signal.signal(signal.SIGTERM, signal_handler)
533
+
534
async with build_async_engine_client(args) as async_engine_client:
535
# If None, creation of the client failed and we exit.
536
if async_engine_client is None:
537
return
538
505
- app = await init_app(async_engine_client, args)
539
+ app = build_app(args)
540
+
541
+ model_config = await async_engine_client.get_model_config()
542
+ init_app_state(async_engine_client, model_config, app.state, args)
543
544
shutdown_task = await serve_http(
545
app,
509
- engine=async_engine_client,
546
+ limit_concurrency=async_engine_client.limit_concurrency,
547
host=args.host,
548
port=args.port,
549
log_level=args.uvicorn_log_level,
@@ -530,4 +567,4 @@ if __name__ == "__main__":
567
parser = make_arg_parser(parser)
568
args = parser.parse_args()
569
533
- asyncio.run(run_server(args))
570
+ uvloop.run(run_server(args))
vllm/entrypoints/openai/rpc/server.py
+7
-1
@@ -46,7 +46,6 @@ class AsyncEngineRPCServer:
46
"""Cleanup all resources."""
47
self.socket.close()
48
self.context.destroy()
49
- self.engine.shutdown_background_loop()
49
# Clear the engine reference so that it can be GC'ed.
50
del self.engine
51
@@ -233,5 +232,12 @@ async def run_server(server: AsyncEngineRPCServer):
232
233
def run_rpc_server(async_engine_args: AsyncEngineArgs,
234
usage_context: UsageContext, rpc_path: str):
235
+
236
+ def signal_handler(*_) -> None:
237
+ # Interrupt server on sigterm while initializing
238
+ raise KeyboardInterrupt("AsyncEngineRPCServer terminated")
239
+
240
+ signal.signal(signal.SIGTERM, signal_handler)
241
+
242
server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
243
uvloop.run(run_server(server))
vllm/executor/multiproc_gpu_executor.py
-14
@@ -1,8 +1,5 @@
1
import asyncio
2
import os
3
-import signal
4
-import threading
5
-import weakref
3
from functools import partial
4
from typing import Any, List, Optional
5
@@ -108,17 +105,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
105
# Set up signal handlers to shutdown the executor cleanly
106
# sometimes gc does not work well
107
111
- # Use weakref to avoid holding a reference to self
112
- ref = weakref.ref(self)
113
-
114
- def shutdown(signum, frame):
115
- if executor := ref():
116
- executor.shutdown()
117
-
118
- if threading.current_thread() is threading.main_thread():
119
- signal.signal(signal.SIGINT, shutdown)
120
- signal.signal(signal.SIGTERM, shutdown)
121
-
108
self.driver_worker = self._create_worker(
109
distributed_init_method=distributed_init_method)
110
self._run_workers("init_device")
vllm/executor/multiproc_worker_utils.py
+4
-1
@@ -120,7 +120,8 @@ class WorkerMonitor(threading.Thread):
120
logger.error("Worker %s pid %s died, exit code: %s",
121
process.name, process.pid, process.exitcode)
122
# Cleanup any remaining workers
123
- logger.info("Killing local vLLM worker processes")
123
+ if logger:
124
+ logger.info("Killing local vLLM worker processes")
125
for worker in self.workers:
126
worker.kill_worker()
127
# Must be done after worker task queues are all closed
@@ -221,6 +222,8 @@ def _run_worker_process(
222
try:
223
executor = getattr(worker, method)
224
output = executor(*args, **kwargs)
225
+ except KeyboardInterrupt:
226
+ break
227
except BaseException as e:
228
tb = traceback.format_exc()
229
logger.error(
vllm/executor/ray_tpu_executor.py
+2
@@ -26,6 +26,8 @@ logger = init_logger(__name__)
26
27
class RayTPUExecutor(TPUExecutor):
28
29
+ uses_ray: bool = True
30
+
31
def __init__(self, *args, **kwargs):
32
# This is non-None when the execute model loop is running
33
# in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
vllm/scripts.py
+2
-2
@@ -1,11 +1,11 @@
1
# The CLI entrypoint to vLLM.
2
import argparse
3
-import asyncio
3
import os
4
import signal
5
import sys
6
from typing import List, Optional
7
8
+import uvloop
9
from openai import OpenAI
10
from openai.types.chat import ChatCompletionMessageParam
11
@@ -34,7 +34,7 @@ def serve(args: argparse.Namespace) -> None:
34
# EngineArgs expects the model name to be passed as --model.
35
args.model = args.model_tag
36
37
- asyncio.run(run_server(args))
37
+ uvloop.run(run_server(args))
38
39
40
def interactive_cli(args: argparse.Namespace) -> None:
vllm/utils.py
+15
@@ -12,6 +12,7 @@ import tempfile
12
import threading
13
import uuid
14
import warnings
15
+import weakref
16
from asyncio import FIRST_COMPLETED, ensure_future
17
from functools import lru_cache, partial, wraps
18
from platform import uname
@@ -1079,6 +1080,20 @@ def cuda_device_count_stateless() -> int:
1080
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
1081
1082
1083
+def weak_bind(bound_method: Callable[..., Any], ) -> Callable[..., None]:
1084
+ """Make an instance method that weakly references
1085
+ its associated instance and no-ops once that
1086
+ instance is collected."""
1087
+ ref = weakref.ref(bound_method.__self__) # type: ignore[attr-defined]
1088
+ unbound = bound_method.__func__ # type: ignore[attr-defined]
1089
+
1090
+ def weak_bound(*args, **kwargs) -> None:
1091
+ if inst := ref():
1092
+ unbound(inst, *args, **kwargs)
1093
+
1094
+ return weak_bound
1095
+
1096
+
1097
#From: https://stackoverflow.com/a/4104188/2749989
1098
def run_once(f: Callable[P, None]) -> Callable[P, None]:
1099