A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Frontend][V1] Online serving performance improvements (#12287)
Nick Hill committed
Jan 22, 2025 at 14:22 UTC
aea94362c9bdd08ed2b346701bdc09d278e85f66
7 files changed
+101
-45
vllm/entrypoints/openai/api_server.py
+6
@@ -1,5 +1,6 @@
1
import asyncio
2
import atexit
3
+import gc
4
import importlib
5
import inspect
6
import multiprocessing
@@ -104,6 +105,11 @@ async def lifespan(app: FastAPI):
105
task.add_done_callback(_running_tasks.remove)
106
else:
107
task = None
108
+
109
+ # Mark the startup heap as static so that it's ignored by GC.
110
+ # Reduces pause times of oldest generation collections.
111
+ gc.collect()
112
+ gc.freeze()
113
try:
114
yield
115
finally:
vllm/entrypoints/openai/protocol.py
+19
-11
@@ -3,7 +3,7 @@
3
import re
4
import time
5
from argparse import Namespace
6
-from typing import Any, Dict, List, Literal, Optional, Union
6
+from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union
7
8
import torch
9
from pydantic import BaseModel, ConfigDict, Field, model_validator
@@ -42,23 +42,31 @@ class OpenAIBaseModel(BaseModel):
42
# OpenAI API does allow extra fields
43
model_config = ConfigDict(extra="allow")
44
45
+ # Cache class field names
46
+ field_names: ClassVar[Optional[Set[str]]] = None
47
+
48
@model_validator(mode="before")
49
@classmethod
50
def __log_extra_fields__(cls, data):
48
- if isinstance(data, dict):
51
+
52
+ field_names = cls.field_names
53
+ if field_names is None:
54
+ if not isinstance(data, dict):
55
+ return data
56
# Get all class field names and their potential aliases
57
field_names = set()
58
for field_name, field in cls.model_fields.items():
59
field_names.add(field_name)
53
- if hasattr(field, 'alias') and field.alias:
54
- field_names.add(field.alias)
55
-
56
- # Compare against both field names and aliases
57
- extra_fields = data.keys() - field_names
58
- if extra_fields:
59
- logger.warning(
60
- "The following fields were present in the request "
61
- "but ignored: %s", extra_fields)
60
+ if alias := getattr(field, 'alias', None):
61
+ field_names.add(alias)
62
+ cls.field_names = field_names
63
+
64
+ # Compare against both field names and aliases
65
+ if any(k not in field_names for k in data):
66
+ logger.warning(
67
+ "The following fields were present in the request "
68
+ "but ignored: %s",
69
+ data.keys() - field_names)
70
return data
71
72
vllm/envs.py
+11
@@ -73,6 +73,7 @@ if TYPE_CHECKING:
73
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74
VLLM_DISABLE_COMPILE_CACHE: bool = False
75
VLLM_SERVER_DEV_MODE: bool = False
76
+ VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
77
78
79
def get_default_cache_root():
@@ -474,6 +475,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
475
# e.g. `/reset_prefix_cache`
476
"VLLM_SERVER_DEV_MODE":
477
lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
478
+
479
+ # Controls the maximum number of requests to handle in a
480
+ # single asyncio task when processing per-token outputs in the
481
+ # V1 AsyncLLM interface. It is applicable when handling a high
482
+ # concurrency of streaming requests.
483
+ # Setting this too high can result in a higher variance of
484
+ # inter-message latencies. Setting it too low can negatively impact
485
+ # TTFT and overall throughput.
486
+ "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE":
487
+ lambda: int(os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")),
488
}
489
490
# end-env-vars-definition
vllm/v1/engine/async_llm.py
+37
-17
@@ -2,9 +2,12 @@ import asyncio
2
import os
3
from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
4
5
+import numpy as np
6
+
7
from vllm.config import ModelConfig, VllmConfig
8
from vllm.engine.arg_utils import AsyncEngineArgs
9
from vllm.engine.protocol import EngineClient
10
+from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
11
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
12
from vllm.inputs.preprocess import InputPreprocessor
13
from vllm.logger import init_logger
@@ -16,7 +19,7 @@ from vllm.sampling_params import SamplingParams
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21
from vllm.usage.usage_lib import UsageContext
19
-from vllm.utils import kill_process_tree
22
+from vllm.utils import cdiv, kill_process_tree
23
from vllm.v1.engine.core_client import EngineCoreClient
24
from vllm.v1.engine.output_processor import OutputProcessor
25
from vllm.v1.engine.processor import Processor
@@ -205,17 +208,15 @@ class AsyncLLM(EngineClient):
208
209
# The output_handler task pushes items into the queue.
210
# This task pulls from the queue and yields to caller.
208
- while True:
211
+ finished = False
212
+ while not finished:
213
# Note: drain queue without await if possible (avoids
214
# task switching under load which helps performance).
211
- out = q.get_nowait() if q.qsize() > 0 else await q.get()
215
+ out = q.get_nowait() if not q.empty() else await q.get()
216
217
# Note: both OutputProcessor and EngineCore handle their
218
# own request cleanup based on finished.
215
- if out.finished:
216
- yield out
217
- break
218
-
219
+ finished = out.finished
220
yield out
221
222
# If the request is disconnected by the client, the
@@ -233,22 +234,41 @@ class AsyncLLM(EngineClient):
234
# 1) Pull EngineCoreOutputs from the EngineCore.
235
outputs = await self.engine_core.get_output_async()
236
236
- # 2) Process EngineCoreOutputs.
237
- processed_outputs = self.output_processor.process_outputs(
238
- outputs.outputs)
239
- # NOTE: RequestOutputs are pushed to their queues.
240
- assert len(processed_outputs.request_outputs) == 0
241
-
242
- # 3) Abort any reqs that finished due to stop strings.
243
- await self.engine_core.abort_requests_async(
244
- processed_outputs.reqs_to_abort)
237
+ # Split outputs into chunks of at most
238
+ # VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
239
+ # event loop for too long.
240
+ num_outputs = len(outputs.outputs)
241
+ if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
242
+ slices = (outputs.outputs, )
243
+ else:
244
+ slices = np.array_split(
245
+ outputs.outputs,
246
+ cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
247
+
248
+ iteration_stats = None
249
+ for i, outputs_slice in enumerate(slices):
250
+ # 2) Process EngineCoreOutputs.
251
+ processed_outputs = self.output_processor.process_outputs(
252
+ outputs_slice, iteration_stats)
253
+ # NOTE: RequestOutputs are pushed to their queues.
254
+ assert not processed_outputs.request_outputs
255
+ iteration_stats = processed_outputs.iteration_stats
256
+
257
+ # Allow other asyncio tasks to run between chunks
258
+ if i + 1 < len(slices):
259
+ await asyncio.sleep(0)
260
+
261
+ # 3) Abort any reqs that finished due to stop strings.
262
+ await self.engine_core.abort_requests_async(
263
+ processed_outputs.reqs_to_abort)
264
265
# 4) Logging.
266
# TODO(rob): make into a coroutine and launch it in
267
# background thread once we add Prometheus.
268
+ assert iteration_stats is not None
269
self._log_stats(
270
scheduler_stats=outputs.scheduler_stats,
251
- iteration_stats=processed_outputs.iteration_stats,
271
+ iteration_stats=iteration_stats,
272
)
273
274
except Exception as e:
vllm/v1/engine/core_client.py
+18
-3
@@ -1,8 +1,9 @@
1
+import asyncio
2
import os
3
import signal
4
import weakref
5
from abc import ABC, abstractmethod
5
-from typing import List, Type
6
+from typing import List, Optional, Type
7
8
import msgspec
9
import zmq
@@ -255,10 +256,24 @@ class AsyncMPClient(MPClient):
256
log_stats=True,
257
)
258
259
+ self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
260
+ self.queue_task: Optional[asyncio.Task] = None
261
+
262
async def get_output_async(self) -> EngineCoreOutputs:
263
+ if self.outputs_queue is None:
264
+ # Perform IO in separate task to parallelize as much as possible
265
+ self.outputs_queue = asyncio.Queue()
266
+
267
+ async def process_outputs_socket():
268
+ assert self.outputs_queue is not None
269
+ while True:
270
+ (frame, ) = await self.output_socket.recv_multipart(
271
+ copy=False)
272
+ self.outputs_queue.put_nowait(frame.buffer)
273
+
274
+ self.queue_task = asyncio.create_task(process_outputs_socket())
275
260
- frames = await self.output_socket.recv_multipart(copy=False)
261
- return self.decoder.decode(frames[0].buffer)
276
+ return self.decoder.decode(await self.outputs_queue.get())
277
278
async def _send_input(self, request_type: EngineCoreRequestType,
279
request: EngineCoreRequestUnion) -> None:
vllm/v1/engine/output_processor.py
+4
-2
@@ -101,6 +101,7 @@ class OutputProcessor:
101
def process_outputs(
102
self,
103
engine_core_outputs: List[EngineCoreOutput],
104
+ iteration_stats: Optional[IterationStats] = None,
105
) -> OutputProcessorOutput:
106
"""
107
Process the EngineCoreOutputs:
@@ -133,7 +134,8 @@ class OutputProcessor:
134
135
request_outputs: List[RequestOutput] = []
136
reqs_to_abort: List[str] = []
136
- iteration_stats = IterationStats(self.log_stats)
137
+ if not iteration_stats:
138
+ iteration_stats = IterationStats(self.log_stats)
139
for engine_core_output in engine_core_outputs:
140
req_id = engine_core_output.request_id
141
req_state = self.request_states.get(req_id)
@@ -175,8 +177,8 @@ class OutputProcessor:
177
iteration_stats=iteration_stats,
178
)
179
180
+ @staticmethod
181
def _make_request_output(
179
- self,
182
request_state: RequestState,
183
detokenizer_output: Optional[DetokenizerOutput],
184
) -> Optional[RequestOutput]:
vllm/v1/request.py
+6
-12
@@ -64,6 +64,12 @@ class Request:
64
# recomputing.
65
self._kv_block_hashes: List[BlockHashType] = []
66
67
+ # Read-only views
68
+ # Prevent directly appending to the these lists since
69
+ # they should also be updated simultaneously.
70
+ self.output_token_ids = ConstantList(self._output_token_ids)
71
+ self.all_token_ids = ConstantList(self._all_token_ids)
72
+
73
@classmethod
74
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
75
return cls(
@@ -79,18 +85,6 @@ class Request:
85
lora_request=request.lora_request,
86
)
87
82
- @property
83
- def output_token_ids(self) -> ConstantList[int]:
84
- # Prevent directly appending to the output_token_ids since
85
- # all_token_ids should also be updated simultaneously.
86
- return ConstantList(self._output_token_ids)
87
-
88
- @property
89
- def all_token_ids(self) -> ConstantList[int]:
90
- # Prevent directly appending to the all_token_ids since
91
- # output_token_ids should also be updated simultaneously
92
- return ConstantList(self._all_token_ids)
93
-
88
def append_output_token_ids(
89
self,
90
token_ids: Union[int, List[int]],