@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / acd5511b6d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[BugFix] Fix clean shutdown issues (#8492)

Nick Hill committed Sep 16, 2024 at 17:33 UTC acd5511b6d0e196b273b6250201115b5c5cfd1ca
11 files changed +215 -136
tests/async_engine/test_async_llm_engine.py
+8 -2
@@ -26,6 +26,11 @@ class RequestOutput:
26 finished: bool = False
27
28
29 +@dataclass
30 +class MockModelConfig:
31 + use_async_output_proc = True
32 +
33 +
34 class MockEngine:
35
36 def __init__(self):
@@ -35,6 +40,7 @@ class MockEngine:
40 self.request_id = None
41 # Ugly, remove dependency when possible
42 self.parallel_config = ParallelConfig(1, 1, False)
43 + self.model_config = MockModelConfig()
44
45 async def step_async(self, virtual_engine):
46 # PP size is 1, ignore virtual engine
@@ -80,7 +86,7 @@ class MockAsyncLLMEngine(AsyncLLMEngine):
86
87 @pytest.mark.asyncio
88 async def test_new_requests_event():
83 - engine = MockAsyncLLMEngine(worker_use_ray=False)
89 + engine = MockAsyncLLMEngine()
90 engine.start_background_loop()
91 await asyncio.sleep(0.01)
92 assert engine.engine.step_calls == 0
@@ -113,7 +119,7 @@ async def test_new_requests_event():
119 assert engine.engine.add_request_calls == 3
120 assert engine.engine.step_calls == old_step_calls + 1
121
116 - engine = MockAsyncLLMEngine(worker_use_ray=True)
122 + engine = MockAsyncLLMEngine()
123 assert engine.get_model_config() is not None
124 assert engine.get_tokenizer() is not None
125 assert engine.get_decoding_config() is not None
vllm/engine/async_llm_engine.py
+45 -25
@@ -1,8 +1,10 @@
1 import asyncio
2 import time
3 +import weakref
4 from functools import partial
5 from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6 Mapping, Optional, Set, Tuple, Type, Union)
7 +from weakref import ReferenceType
8
9 import vllm.envs as envs
10 from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
@@ -26,6 +28,7 @@ from vllm.sampling_params import SamplingParams
28 from vllm.sequence import ExecuteModelRequest
29 from vllm.transformers_utils.tokenizer import AnyTokenizer
30 from vllm.usage.usage_lib import UsageContext
31 +from vllm.utils import weak_bind
32
33 logger = init_logger(__name__)
34 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -450,9 +453,6 @@ class AsyncLLMEngine:
453 method yields the outputs from the :class:`LLMEngine` to the caller.
454
455 Args:
453 - worker_use_ray: Whether to use Ray for model workers. Required for
454 - distributed execution. Should be the same as
455 - `parallel_config.worker_use_ray`.
456 log_requests: Whether to log the requests.
457 start_engine_loop: If True, the background task to run the engine
458 will be automatically started in the generate call.
@@ -463,23 +463,22 @@ class AsyncLLMEngine:
463 _engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
464
465 def __init__(self,
466 - worker_use_ray: bool,
466 *args,
467 log_requests: bool = True,
468 start_engine_loop: bool = True,
469 **kwargs) -> None:
471 - self.worker_use_ray = worker_use_ray
470 self.log_requests = log_requests
471 self.engine = self._engine_class(*args, **kwargs)
472
473 # This ensures quick processing of request outputs
474 # so the append to asyncio queues is not delayed,
475 # especially for multi-step.
478 - #
479 - self.use_process_request_outputs_callback = True
476 + self.use_process_request_outputs_callback = (
477 + self.engine.model_config.use_async_output_proc)
478 +
479 if self.use_process_request_outputs_callback:
480 self.engine.process_request_outputs_callback = \
482 - self.process_request_outputs
481 + weak_bind(self.process_request_outputs)
482
483 self.background_loop: Optional[asyncio.Future] = None
484 # We need to keep a reference to unshielded
@@ -492,6 +491,11 @@ class AsyncLLMEngine:
491 # Lazy initialized fields
492 self._request_tracker: RequestTracker
493
494 + def __del__(self):
495 + if rt := getattr(self, "request_tracker", None):
496 + # Wake up engine loop so that it will exit cleanly
497 + rt.new_requests_event.set()
498 +
499 @classmethod
500 def _get_executor_cls(
501 cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
@@ -502,15 +506,12 @@ class AsyncLLMEngine:
506 raise TypeError(
507 "distributed_executor_backend must be a subclass of "
508 f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
505 - if distributed_executor_backend.uses_ray: # type: ignore
506 - initialize_ray_cluster(engine_config.parallel_config)
509 executor_class = distributed_executor_backend
510 elif engine_config.device_config.device_type == "neuron":
511 from vllm.executor.neuron_executor import NeuronExecutorAsync
512 executor_class = NeuronExecutorAsync
513 elif engine_config.device_config.device_type == "tpu":
514 if distributed_executor_backend == "ray":
513 - initialize_ray_cluster(engine_config.parallel_config)
515 from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
516 executor_class = RayTPUExecutorAsync
517 else:
@@ -531,11 +532,9 @@ class AsyncLLMEngine:
532 from vllm.executor.xpu_executor import XPUExecutorAsync
533 executor_class = XPUExecutorAsync
534 elif distributed_executor_backend == "ray":
534 - initialize_ray_cluster(engine_config.parallel_config)
535 from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
536 executor_class = RayXPUExecutorAsync
537 elif distributed_executor_backend == "mp":
538 - initialize_ray_cluster(engine_config.parallel_config)
538 from vllm.executor.multiproc_xpu_executor import (
539 MultiprocessingXPUExecutorAsync)
540 executor_class = MultiprocessingXPUExecutorAsync
@@ -543,7 +542,6 @@ class AsyncLLMEngine:
542 raise RuntimeError(
543 "Not supported distributed execution model on XPU device.")
544 elif distributed_executor_backend == "ray":
546 - initialize_ray_cluster(engine_config.parallel_config)
545 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
546 executor_class = RayGPUExecutorAsync
547 elif distributed_executor_backend == "mp":
@@ -559,19 +557,23 @@ class AsyncLLMEngine:
557 def from_engine_args(
558 cls,
559 engine_args: AsyncEngineArgs,
560 + engine_config: Optional[EngineConfig] = None,
561 start_engine_loop: bool = True,
562 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
563 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
564 ) -> "AsyncLLMEngine":
565 """Creates an async LLM engine from the engine arguments."""
566 # Create the engine configs.
568 - engine_config = engine_args.create_engine_config()
567 + if engine_config is None:
568 + engine_config = engine_args.create_engine_config()
569
570 executor_class = cls._get_executor_cls(engine_config)
571
572 + if executor_class.uses_ray:
573 + initialize_ray_cluster(engine_config.parallel_config)
574 +
575 # Create the async LLM engine.
576 engine = cls(
574 - executor_class.uses_ray,
577 **engine_config.to_dict(),
578 executor_class=executor_class,
579 log_requests=not engine_args.disable_log_requests,
@@ -628,7 +630,7 @@ class AsyncLLMEngine:
630 self._request_tracker = RequestTracker()
631
632 self._background_loop_unshielded = asyncio.get_event_loop(
631 - ).create_task(self.run_engine_loop())
633 + ).create_task(self.run_engine_loop(weakref.ref(self)))
634 self._background_loop_unshielded.add_done_callback(
635 partial(_log_task_completion, error_callback=self._error_callback))
636 self.background_loop = asyncio.shield(self._background_loop_unshielded)
@@ -698,9 +700,16 @@ class AsyncLLMEngine:
700 async def _engine_abort(self, request_ids: Iterable[str]):
701 self.engine.abort_request(request_ids)
702
701 - async def run_engine_loop(self):
703 + @staticmethod
704 + async def run_engine_loop(engine_ref: ReferenceType):
705 + """We use a weakref to the engine so that the running loop
706 + doesn't prevent the engine being garbage collected."""
707 + engine: Optional["AsyncLLMEngine"] = engine_ref()
708 + if not engine:
709 + return
710 +
711 pipeline_parallel_size = \
703 - self.engine.parallel_config.pipeline_parallel_size
712 + engine.engine.parallel_config.pipeline_parallel_size
713 has_requests_in_progress = [False] * pipeline_parallel_size
714 while True:
715 if not any(has_requests_in_progress):
@@ -711,11 +720,21 @@ class AsyncLLMEngine:
720 # timeout, and unblocks the RPC thread in the workers so that
721 # they can process any other queued control plane messages,
722 # such as add/remove lora adapters.
714 - await self.engine.stop_remote_worker_execution_loop_async()
715 - await self._request_tracker.wait_for_new_requests()
723 + await engine.engine.stop_remote_worker_execution_loop_async()
724 + request_tracker = engine._request_tracker
725 + # Allow engine to be garbage collected while
726 + # waiting for new requests
727 + del engine
728 + await asyncio.sleep(0)
729 + if engine_ref() is None:
730 + return
731 + await request_tracker.wait_for_new_requests()
732 + engine = engine_ref()
733 + if not engine:
734 + return
735 logger.debug("Got new requests!")
736 requests_in_progress = [
718 - asyncio.create_task(self.engine_step(ve))
737 + asyncio.create_task(engine.engine_step(ve))
738 for ve in range(pipeline_parallel_size)
739 ]
740 has_requests_in_progress = [True] * pipeline_parallel_size
@@ -733,19 +752,20 @@ class AsyncLLMEngine:
752 result = task.result()
753 virtual_engine = requests_in_progress.index(task)
754 has_unfinished_requests = (
736 - self.engine.has_unfinished_requests_for_virtual_engine(
755 + engine.engine.
756 + has_unfinished_requests_for_virtual_engine(
757 virtual_engine))
758 if result or has_unfinished_requests:
759 requests_in_progress[virtual_engine] = (
760 asyncio.create_task(
741 - self.engine_step(virtual_engine)))
761 + engine.engine_step(virtual_engine)))
762 has_requests_in_progress[virtual_engine] = True
763 else:
764 has_requests_in_progress[virtual_engine] = False
765 except asyncio.TimeoutError as exc:
766 logger.error(
767 "Engine iteration timed out. This should never happen!")
748 - self.set_errored(exc)
768 + engine.set_errored(exc)
769 raise
770 await asyncio.sleep(0)
771
vllm/engine/llm_engine.py
+13 -8
@@ -1,8 +1,8 @@
1 -import functools
1 import time
2 from collections import deque
3 from contextlib import contextmanager
4 from dataclasses import dataclass
5 +from functools import partial
6 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
@@ -51,7 +51,7 @@ from vllm.transformers_utils.tokenizer_group import (
51 BaseTokenizerGroup, init_tokenizer_from_configs)
52 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
53 usage_message)
54 -from vllm.utils import Counter, Device
54 +from vllm.utils import Counter, Device, weak_bind
55 from vllm.version import __version__ as VLLM_VERSION
56
57 logger = init_logger(__name__)
@@ -382,11 +382,16 @@ class LLMEngine:
382 for _ in range(self.parallel_config.pipeline_parallel_size)
383 ]
384
385 - self.async_callbacks = [
386 - functools.partial(self._process_model_outputs,
387 - ctx=self.scheduler_contexts[v_id])
388 - for v_id in range(self.parallel_config.pipeline_parallel_size)
389 - ]
385 + if model_config.use_async_output_proc:
386 + process_model_outputs = weak_bind(self._process_model_outputs)
387 +
388 + self.async_callbacks = [
389 + partial(process_model_outputs,
390 + ctx=self.scheduler_contexts[v_id])
391 + for v_id in range(self.parallel_config.pipeline_parallel_size)
392 + ]
393 + else:
394 + self.async_callbacks = []
395
396 # Currently used by AsyncLLMEngine to ensure quick append
397 # of request outputs to asyncio queues
@@ -869,8 +874,8 @@ class LLMEngine:
874 """
875 return self.scheduler[virtual_engine].has_unfinished_seqs()
876
877 + @staticmethod
878 def _process_sequence_group_outputs(
873 - self,
879 seq_group: SequenceGroup,
880 outputs: List[EmbeddingSequenceGroupOutput],
881 ) -> None:
vllm/entrypoints/launcher.py
+10 -11
@@ -1,21 +1,20 @@
1 import asyncio
2 import signal
3 from http import HTTPStatus
4 -from typing import Any
4 +from typing import Any, Optional
5
6 import uvicorn
7 -from fastapi import FastAPI, Response
7 +from fastapi import FastAPI, Request, Response
8
9 from vllm import envs
10 from vllm.engine.async_llm_engine import AsyncEngineDeadError
11 -from vllm.engine.protocol import AsyncEngineClient
11 from vllm.logger import init_logger
12 from vllm.utils import find_process_using_port
13
14 logger = init_logger(__name__)
15
16
18 -async def serve_http(app: FastAPI, engine: AsyncEngineClient,
17 +async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
18 **uvicorn_kwargs: Any):
19 logger.info("Available routes are:")
20 for route in app.routes:
@@ -29,16 +28,16 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
28
29 # Set concurrency limits in uvicorn if running in multiprocessing mode
30 # since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
32 - if engine.limit_concurrency is not None:
31 + if limit_concurrency is not None:
32 logger.info(
33 "Launching Uvicorn with --limit_concurrency %s. To avoid this "
34 "limit at the expense of performance run with "
36 - "--disable-frontend-multiprocessing", engine.limit_concurrency)
37 - uvicorn_kwargs["limit_concurrency"] = engine.limit_concurrency
35 + "--disable-frontend-multiprocessing", limit_concurrency)
36 + uvicorn_kwargs["limit_concurrency"] = limit_concurrency
37
38 config = uvicorn.Config(app, **uvicorn_kwargs)
39 server = uvicorn.Server(config)
41 - _add_shutdown_handlers(app, server, engine)
40 + _add_shutdown_handlers(app, server)
41
42 loop = asyncio.get_running_loop()
43
@@ -68,15 +67,15 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
67 return server.shutdown()
68
69
71 -def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server,
72 - engine: AsyncEngineClient) -> None:
70 +def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
71 """Adds handlers for fatal errors that should crash the server"""
72
73 @app.exception_handler(RuntimeError)
76 - async def runtime_error_handler(_, __):
74 + async def runtime_error_handler(request: Request, __):
75 """On generic runtime error, check to see if the engine has died.
76 It probably has, in which case the server will no longer be able to
77 handle requests. Trigger a graceful shutdown with a SIGTERM."""
78 + engine = request.app.state.engine_client
79 if (not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine.errored
80 and not engine.is_running):
81 logger.fatal("AsyncLLMEngine has failed, terminating server "
vllm/entrypoints/openai/api_server.py
+109 -72
@@ -4,16 +4,20 @@ import inspect
4 import multiprocessing
5 import os
6 import re
7 +import signal
8 import tempfile
9 from argparse import Namespace
10 from contextlib import asynccontextmanager
11 +from functools import partial
12 from http import HTTPStatus
13 from typing import AsyncIterator, Optional, Set
14
15 +import uvloop
16 from fastapi import APIRouter, FastAPI, Request
17 from fastapi.exceptions import RequestValidationError
18 from fastapi.middleware.cors import CORSMiddleware
19 from fastapi.responses import JSONResponse, Response, StreamingResponse
20 +from starlette.datastructures import State
21 from starlette.routing import Mount
22 from typing_extensions import assert_never
23
@@ -54,12 +58,6 @@ from vllm.version import __version__ as VLLM_VERSION
58
59 TIMEOUT_KEEP_ALIVE = 5 # seconds
60
57 -async_engine_client: AsyncEngineClient
58 -engine_args: AsyncEngineArgs
59 -openai_serving_chat: OpenAIServingChat
60 -openai_serving_completion: OpenAIServingCompletion
61 -openai_serving_embedding: OpenAIServingEmbedding
62 -openai_serving_tokenization: OpenAIServingTokenization
61 prometheus_multiproc_dir: tempfile.TemporaryDirectory
62
63 # Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
@@ -83,18 +81,28 @@ def model_is_embedding(model_name: str, trust_remote_code: bool,
81
82 @asynccontextmanager
83 async def lifespan(app: FastAPI):
86 -
87 - async def _force_log():
88 - while True:
89 - await asyncio.sleep(10)
90 - await async_engine_client.do_log_stats()
91 -
92 - if not engine_args.disable_log_stats:
93 - task = asyncio.create_task(_force_log())
94 - _running_tasks.add(task)
95 - task.add_done_callback(_running_tasks.remove)
96 -
97 - yield
84 + try:
85 + if app.state.log_stats:
86 + async_engine_client = app.state.engine_client
87 +
88 + async def _force_log():
89 + while True:
90 + await asyncio.sleep(10)
91 + await async_engine_client.do_log_stats()
92 +
93 + task = asyncio.create_task(_force_log())
94 + _running_tasks.add(task)
95 + task.add_done_callback(_running_tasks.remove)
96 + else:
97 + task = None
98 + try:
99 + yield
100 + finally:
101 + if task is not None:
102 + task.cancel()
103 + finally:
104 + # Ensure app state including engine ref is gc'd
105 + del app.state
106
107
108 @asynccontextmanager
@@ -103,16 +111,10 @@ async def build_async_engine_client(
111
112 # Context manager to handle async_engine_client lifecycle
113 # Ensures everything is shutdown and cleaned up on error/exit
106 - global engine_args
114 engine_args = AsyncEngineArgs.from_cli_args(args)
115
109 - # Backend itself still global for the silly lil' health handler
110 - global async_engine_client
111 -
116 async with build_async_engine_client_from_engine_args(
117 engine_args, args.disable_frontend_multiprocessing) as engine:
114 -
115 - async_engine_client = engine # type: ignore[assignment]
118 yield engine
119
120
@@ -134,12 +136,22 @@ async def build_async_engine_client_from_engine_args(
136 if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
137 engine_args.quantization, engine_args.revision)
138 or disable_frontend_multiprocessing):
137 - engine_client = AsyncLLMEngine.from_engine_args(
138 - engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
139 - try:
140 - yield engine_client
141 - finally:
142 - engine_client.shutdown_background_loop()
139 + engine_config = engine_args.create_engine_config()
140 + uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
141 + "uses_ray", False)
142 +
143 + build_engine = partial(AsyncLLMEngine.from_engine_args,
144 + engine_args=engine_args,
145 + engine_config=engine_config,
146 + usage_context=UsageContext.OPENAI_API_SERVER)
147 + if uses_ray:
148 + # Must run in main thread with ray for its signal handlers to work
149 + engine_client = build_engine()
150 + else:
151 + engine_client = await asyncio.get_running_loop().run_in_executor(
152 + None, build_engine)
153 +
154 + yield engine_client
155 return
156
157 # Otherwise, use the multiprocessing AsyncLLMEngine.
@@ -241,16 +253,36 @@ def mount_metrics(app: FastAPI):
253 app.routes.append(metrics_route)
254
255
256 +def chat(request: Request) -> OpenAIServingChat:
257 + return request.app.state.openai_serving_chat
258 +
259 +
260 +def completion(request: Request) -> OpenAIServingCompletion:
261 + return request.app.state.openai_serving_completion
262 +
263 +
264 +def tokenization(request: Request) -> OpenAIServingTokenization:
265 + return request.app.state.openai_serving_tokenization
266 +
267 +
268 +def embedding(request: Request) -> OpenAIServingEmbedding:
269 + return request.app.state.openai_serving_embedding
270 +
271 +
272 +def engine_client(request: Request) -> AsyncEngineClient:
273 + return request.app.state.engine_client
274 +
275 +
276 @router.get("/health")
245 -async def health() -> Response:
277 +async def health(raw_request: Request) -> Response:
278 """Health check."""
247 - await async_engine_client.check_health()
279 + await engine_client(raw_request).check_health()
280 return Response(status_code=200)
281
282
283 @router.post("/tokenize")
252 -async def tokenize(request: TokenizeRequest):
253 - generator = await openai_serving_tokenization.create_tokenize(request)
284 +async def tokenize(request: TokenizeRequest, raw_request: Request):
285 + generator = await tokenization(raw_request).create_tokenize(request)
286 if isinstance(generator, ErrorResponse):
287 return JSONResponse(content=generator.model_dump(),
288 status_code=generator.code)
@@ -261,8 +293,8 @@ async def tokenize(request: TokenizeRequest):
293
294
295 @router.post("/detokenize")
264 -async def detokenize(request: DetokenizeRequest):
265 - generator = await openai_serving_tokenization.create_detokenize(request)
296 +async def detokenize(request: DetokenizeRequest, raw_request: Request):
297 + generator = await tokenization(raw_request).create_detokenize(request)
298 if isinstance(generator, ErrorResponse):
299 return JSONResponse(content=generator.model_dump(),
300 status_code=generator.code)
@@ -273,8 +305,8 @@ async def detokenize(request: DetokenizeRequest):
305
306
307 @router.get("/v1/models")
276 -async def show_available_models():
277 - models = await openai_serving_completion.show_available_models()
308 +async def show_available_models(raw_request: Request):
309 + models = await completion(raw_request).show_available_models()
310 return JSONResponse(content=models.model_dump())
311
312
@@ -288,7 +320,7 @@ async def show_version():
320 async def create_chat_completion(request: ChatCompletionRequest,
321 raw_request: Request):
322
291 - generator = await openai_serving_chat.create_chat_completion(
323 + generator = await chat(raw_request).create_chat_completion(
324 request, raw_request)
325
326 if isinstance(generator, ErrorResponse):
@@ -303,7 +335,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
335
336 @router.post("/v1/completions")
337 async def create_completion(request: CompletionRequest, raw_request: Request):
306 - generator = await openai_serving_completion.create_completion(
338 + generator = await completion(raw_request).create_completion(
339 request, raw_request)
340 if isinstance(generator, ErrorResponse):
341 return JSONResponse(content=generator.model_dump(),
@@ -316,7 +348,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
348
349 @router.post("/v1/embeddings")
350 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
319 - generator = await openai_serving_embedding.create_embedding(
351 + generator = await embedding(raw_request).create_embedding(
352 request, raw_request)
353 if isinstance(generator, ErrorResponse):
354 return JSONResponse(content=generator.model_dump(),
@@ -333,16 +365,16 @@ if envs.VLLM_TORCH_PROFILER_DIR:
365 "used for local development!")
366
367 @router.post("/start_profile")
336 - async def start_profile():
368 + async def start_profile(raw_request: Request):
369 logger.info("Starting profiler...")
338 - await async_engine_client.start_profile()
370 + await engine_client(raw_request).start_profile()
371 logger.info("Profiler started.")
372 return Response(status_code=200)
373
374 @router.post("/stop_profile")
343 - async def stop_profile():
375 + async def stop_profile(raw_request: Request):
376 logger.info("Stopping profiler...")
345 - await async_engine_client.stop_profile()
377 + await engine_client(raw_request).stop_profile()
378 logger.info("Profiler stopped.")
379 return Response(status_code=200)
380
@@ -353,13 +385,14 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
385 "This should ONLY be used for local development!")
386
387 @router.post("/v1/load_lora_adapter")
356 - async def load_lora_adapter(request: LoadLoraAdapterRequest):
357 - response = await openai_serving_chat.load_lora_adapter(request)
388 + async def load_lora_adapter(request: LoadLoraAdapterRequest,
389 + raw_request: Request):
390 + response = await chat(raw_request).load_lora_adapter(request)
391 if isinstance(response, ErrorResponse):
392 return JSONResponse(content=response.model_dump(),
393 status_code=response.code)
394
362 - response = await openai_serving_completion.load_lora_adapter(request)
395 + response = await completion(raw_request).load_lora_adapter(request)
396 if isinstance(response, ErrorResponse):
397 return JSONResponse(content=response.model_dump(),
398 status_code=response.code)
@@ -367,13 +400,14 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
400 return Response(status_code=200, content=response)
401
402 @router.post("/v1/unload_lora_adapter")
370 - async def unload_lora_adapter(request: UnloadLoraAdapterRequest):
371 - response = await openai_serving_chat.unload_lora_adapter(request)
403 + async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
404 + raw_request: Request):
405 + response = await chat(raw_request).unload_lora_adapter(request)
406 if isinstance(response, ErrorResponse):
407 return JSONResponse(content=response.model_dump(),
408 status_code=response.code)
409
376 - response = await openai_serving_completion.unload_lora_adapter(request)
410 + response = await completion(raw_request).unload_lora_adapter(request)
411 if isinstance(response, ErrorResponse):
412 return JSONResponse(content=response.model_dump(),
413 status_code=response.code)
@@ -398,7 +432,8 @@ def build_app(args: Namespace) -> FastAPI:
432
433 @app.exception_handler(RequestValidationError)
434 async def validation_exception_handler(_, exc):
401 - err = openai_serving_chat.create_error_response(message=str(exc))
435 + chat = app.state.openai_serving_chat
436 + err = chat.create_error_response(message=str(exc))
437 return JSONResponse(err.model_dump(),
438 status_code=HTTPStatus.BAD_REQUEST)
439
@@ -430,30 +465,26 @@ def build_app(args: Namespace) -> FastAPI:
465 return app
466
467
433 -async def init_app(
468 +def init_app_state(
469 async_engine_client: AsyncEngineClient,
470 + model_config: ModelConfig,
471 + state: State,
472 args: Namespace,
436 -) -> FastAPI:
437 - app = build_app(args)
438 -
473 +) -> None:
474 if args.served_model_name is not None:
475 served_model_names = args.served_model_name
476 else:
477 served_model_names = [args.model]
478
444 - model_config = await async_engine_client.get_model_config()
445 -
479 if args.disable_log_requests:
480 request_logger = None
481 else:
482 request_logger = RequestLogger(max_log_len=args.max_log_len)
483
451 - global openai_serving_chat
452 - global openai_serving_completion
453 - global openai_serving_embedding
454 - global openai_serving_tokenization
484 + state.engine_client = async_engine_client
485 + state.log_stats = not args.disable_log_stats
486
456 - openai_serving_chat = OpenAIServingChat(
487 + state.openai_serving_chat = OpenAIServingChat(
488 async_engine_client,
489 model_config,
490 served_model_names,
@@ -465,7 +496,7 @@ async def init_app(
496 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
497 enable_auto_tools=args.enable_auto_tool_choice,
498 tool_parser=args.tool_call_parser)
468 - openai_serving_completion = OpenAIServingCompletion(
499 + state.openai_serving_completion = OpenAIServingCompletion(
500 async_engine_client,
501 model_config,
502 served_model_names,
@@ -474,13 +505,13 @@ async def init_app(
505 request_logger=request_logger,
506 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
507 )
477 - openai_serving_embedding = OpenAIServingEmbedding(
508 + state.openai_serving_embedding = OpenAIServingEmbedding(
509 async_engine_client,
510 model_config,
511 served_model_names,
512 request_logger=request_logger,
513 )
483 - openai_serving_tokenization = OpenAIServingTokenization(
514 + state.openai_serving_tokenization = OpenAIServingTokenization(
515 async_engine_client,
516 model_config,
517 served_model_names,
@@ -488,25 +519,31 @@ async def init_app(
519 request_logger=request_logger,
520 chat_template=args.chat_template,
521 )
491 - app.root_path = args.root_path
492 -
493 - return app
522
523
524 async def run_server(args, **uvicorn_kwargs) -> None:
525 logger.info("vLLM API server version %s", VLLM_VERSION)
526 logger.info("args: %s", args)
527
528 + def signal_handler(*_) -> None:
529 + # Interrupt server on sigterm while initializing
530 + raise KeyboardInterrupt("terminated")
531 +
532 + signal.signal(signal.SIGTERM, signal_handler)
533 +
534 async with build_async_engine_client(args) as async_engine_client:
535 # If None, creation of the client failed and we exit.
536 if async_engine_client is None:
537 return
538
505 - app = await init_app(async_engine_client, args)
539 + app = build_app(args)
540 +
541 + model_config = await async_engine_client.get_model_config()
542 + init_app_state(async_engine_client, model_config, app.state, args)
543
544 shutdown_task = await serve_http(
545 app,
509 - engine=async_engine_client,
546 + limit_concurrency=async_engine_client.limit_concurrency,
547 host=args.host,
548 port=args.port,
549 log_level=args.uvicorn_log_level,
@@ -530,4 +567,4 @@ if __name__ == "__main__":
567 parser = make_arg_parser(parser)
568 args = parser.parse_args()
569
533 - asyncio.run(run_server(args))
570 + uvloop.run(run_server(args))
vllm/entrypoints/openai/rpc/server.py
+7 -1
@@ -46,7 +46,6 @@ class AsyncEngineRPCServer:
46 """Cleanup all resources."""
47 self.socket.close()
48 self.context.destroy()
49 - self.engine.shutdown_background_loop()
49 # Clear the engine reference so that it can be GC'ed.
50 del self.engine
51
@@ -233,5 +232,12 @@ async def run_server(server: AsyncEngineRPCServer):
232
233 def run_rpc_server(async_engine_args: AsyncEngineArgs,
234 usage_context: UsageContext, rpc_path: str):
235 +
236 + def signal_handler(*_) -> None:
237 + # Interrupt server on sigterm while initializing
238 + raise KeyboardInterrupt("AsyncEngineRPCServer terminated")
239 +
240 + signal.signal(signal.SIGTERM, signal_handler)
241 +
242 server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
243 uvloop.run(run_server(server))
vllm/executor/multiproc_gpu_executor.py
-14
@@ -1,8 +1,5 @@
1 import asyncio
2 import os
3 -import signal
4 -import threading
5 -import weakref
3 from functools import partial
4 from typing import Any, List, Optional
5
@@ -108,17 +105,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
105 # Set up signal handlers to shutdown the executor cleanly
106 # sometimes gc does not work well
107
111 - # Use weakref to avoid holding a reference to self
112 - ref = weakref.ref(self)
113 -
114 - def shutdown(signum, frame):
115 - if executor := ref():
116 - executor.shutdown()
117 -
118 - if threading.current_thread() is threading.main_thread():
119 - signal.signal(signal.SIGINT, shutdown)
120 - signal.signal(signal.SIGTERM, shutdown)
121 -
108 self.driver_worker = self._create_worker(
109 distributed_init_method=distributed_init_method)
110 self._run_workers("init_device")
vllm/executor/multiproc_worker_utils.py
+4 -1
@@ -120,7 +120,8 @@ class WorkerMonitor(threading.Thread):
120 logger.error("Worker %s pid %s died, exit code: %s",
121 process.name, process.pid, process.exitcode)
122 # Cleanup any remaining workers
123 - logger.info("Killing local vLLM worker processes")
123 + if logger:
124 + logger.info("Killing local vLLM worker processes")
125 for worker in self.workers:
126 worker.kill_worker()
127 # Must be done after worker task queues are all closed
@@ -221,6 +222,8 @@ def _run_worker_process(
222 try:
223 executor = getattr(worker, method)
224 output = executor(*args, **kwargs)
225 + except KeyboardInterrupt:
226 + break
227 except BaseException as e:
228 tb = traceback.format_exc()
229 logger.error(
vllm/executor/ray_tpu_executor.py
+2
@@ -26,6 +26,8 @@ logger = init_logger(__name__)
26
27 class RayTPUExecutor(TPUExecutor):
28
29 + uses_ray: bool = True
30 +
31 def __init__(self, *args, **kwargs):
32 # This is non-None when the execute model loop is running
33 # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
vllm/scripts.py
+2 -2
@@ -1,11 +1,11 @@
1 # The CLI entrypoint to vLLM.
2 import argparse
3 -import asyncio
3 import os
4 import signal
5 import sys
6 from typing import List, Optional
7
8 +import uvloop
9 from openai import OpenAI
10 from openai.types.chat import ChatCompletionMessageParam
11
@@ -34,7 +34,7 @@ def serve(args: argparse.Namespace) -> None:
34 # EngineArgs expects the model name to be passed as --model.
35 args.model = args.model_tag
36
37 - asyncio.run(run_server(args))
37 + uvloop.run(run_server(args))
38
39
40 def interactive_cli(args: argparse.Namespace) -> None:
vllm/utils.py
+15
@@ -12,6 +12,7 @@ import tempfile
12 import threading
13 import uuid
14 import warnings
15 +import weakref
16 from asyncio import FIRST_COMPLETED, ensure_future
17 from functools import lru_cache, partial, wraps
18 from platform import uname
@@ -1079,6 +1080,20 @@ def cuda_device_count_stateless() -> int:
1080 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
1081
1082
1083 +def weak_bind(bound_method: Callable[..., Any], ) -> Callable[..., None]:
1084 + """Make an instance method that weakly references
1085 + its associated instance and no-ops once that
1086 + instance is collected."""
1087 + ref = weakref.ref(bound_method.__self__) # type: ignore[attr-defined]
1088 + unbound = bound_method.__func__ # type: ignore[attr-defined]
1089 +
1090 + def weak_bound(*args, **kwargs) -> None:
1091 + if inst := ref():
1092 + unbound(inst, *args, **kwargs)
1093 +
1094 + return weak_bound
1095 +
1096 +
1097 #From: https://stackoverflow.com/a/4104188/2749989
1098 def run_once(f: Callable[P, None]) -> Callable[P, None]:
1099