@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aea94362c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Frontend][V1] Online serving performance improvements (#12287)

Nick Hill committed Jan 22, 2025 at 14:22 UTC aea94362c9bdd08ed2b346701bdc09d278e85f66
7 files changed +101 -45
vllm/entrypoints/openai/api_server.py
+6
@@ -1,5 +1,6 @@
1 import asyncio
2 import atexit
3 +import gc
4 import importlib
5 import inspect
6 import multiprocessing
@@ -104,6 +105,11 @@ async def lifespan(app: FastAPI):
105 task.add_done_callback(_running_tasks.remove)
106 else:
107 task = None
108 +
109 + # Mark the startup heap as static so that it's ignored by GC.
110 + # Reduces pause times of oldest generation collections.
111 + gc.collect()
112 + gc.freeze()
113 try:
114 yield
115 finally:
vllm/entrypoints/openai/protocol.py
+19 -11
@@ -3,7 +3,7 @@
3 import re
4 import time
5 from argparse import Namespace
6 -from typing import Any, Dict, List, Literal, Optional, Union
6 +from typing import Any, ClassVar, Dict, List, Literal, Optional, Set, Union
7
8 import torch
9 from pydantic import BaseModel, ConfigDict, Field, model_validator
@@ -42,23 +42,31 @@ class OpenAIBaseModel(BaseModel):
42 # OpenAI API does allow extra fields
43 model_config = ConfigDict(extra="allow")
44
45 + # Cache class field names
46 + field_names: ClassVar[Optional[Set[str]]] = None
47 +
48 @model_validator(mode="before")
49 @classmethod
50 def __log_extra_fields__(cls, data):
48 - if isinstance(data, dict):
51 +
52 + field_names = cls.field_names
53 + if field_names is None:
54 + if not isinstance(data, dict):
55 + return data
56 # Get all class field names and their potential aliases
57 field_names = set()
58 for field_name, field in cls.model_fields.items():
59 field_names.add(field_name)
53 - if hasattr(field, 'alias') and field.alias:
54 - field_names.add(field.alias)
55 -
56 - # Compare against both field names and aliases
57 - extra_fields = data.keys() - field_names
58 - if extra_fields:
59 - logger.warning(
60 - "The following fields were present in the request "
61 - "but ignored: %s", extra_fields)
60 + if alias := getattr(field, 'alias', None):
61 + field_names.add(alias)
62 + cls.field_names = field_names
63 +
64 + # Compare against both field names and aliases
65 + if any(k not in field_names for k in data):
66 + logger.warning(
67 + "The following fields were present in the request "
68 + "but ignored: %s",
69 + data.keys() - field_names)
70 return data
71
72
vllm/envs.py
+11
@@ -73,6 +73,7 @@ if TYPE_CHECKING:
73 VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74 VLLM_DISABLE_COMPILE_CACHE: bool = False
75 VLLM_SERVER_DEV_MODE: bool = False
76 + VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
77
78
79 def get_default_cache_root():
@@ -474,6 +475,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
475 # e.g. `/reset_prefix_cache`
476 "VLLM_SERVER_DEV_MODE":
477 lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
478 +
479 + # Controls the maximum number of requests to handle in a
480 + # single asyncio task when processing per-token outputs in the
481 + # V1 AsyncLLM interface. It is applicable when handling a high
482 + # concurrency of streaming requests.
483 + # Setting this too high can result in a higher variance of
484 + # inter-message latencies. Setting it too low can negatively impact
485 + # TTFT and overall throughput.
486 + "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE":
487 + lambda: int(os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")),
488 }
489
490 # end-env-vars-definition
vllm/v1/engine/async_llm.py
+37 -17
@@ -2,9 +2,12 @@ import asyncio
2 import os
3 from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
4
5 +import numpy as np
6 +
7 from vllm.config import ModelConfig, VllmConfig
8 from vllm.engine.arg_utils import AsyncEngineArgs
9 from vllm.engine.protocol import EngineClient
10 +from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
11 from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
12 from vllm.inputs.preprocess import InputPreprocessor
13 from vllm.logger import init_logger
@@ -16,7 +19,7 @@ from vllm.sampling_params import SamplingParams
19 from vllm.transformers_utils.tokenizer import AnyTokenizer
20 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21 from vllm.usage.usage_lib import UsageContext
19 -from vllm.utils import kill_process_tree
22 +from vllm.utils import cdiv, kill_process_tree
23 from vllm.v1.engine.core_client import EngineCoreClient
24 from vllm.v1.engine.output_processor import OutputProcessor
25 from vllm.v1.engine.processor import Processor
@@ -205,17 +208,15 @@ class AsyncLLM(EngineClient):
208
209 # The output_handler task pushes items into the queue.
210 # This task pulls from the queue and yields to caller.
208 - while True:
211 + finished = False
212 + while not finished:
213 # Note: drain queue without await if possible (avoids
214 # task switching under load which helps performance).
211 - out = q.get_nowait() if q.qsize() > 0 else await q.get()
215 + out = q.get_nowait() if not q.empty() else await q.get()
216
217 # Note: both OutputProcessor and EngineCore handle their
218 # own request cleanup based on finished.
215 - if out.finished:
216 - yield out
217 - break
218 -
219 + finished = out.finished
220 yield out
221
222 # If the request is disconnected by the client, the
@@ -233,22 +234,41 @@ class AsyncLLM(EngineClient):
234 # 1) Pull EngineCoreOutputs from the EngineCore.
235 outputs = await self.engine_core.get_output_async()
236
236 - # 2) Process EngineCoreOutputs.
237 - processed_outputs = self.output_processor.process_outputs(
238 - outputs.outputs)
239 - # NOTE: RequestOutputs are pushed to their queues.
240 - assert len(processed_outputs.request_outputs) == 0
241 -
242 - # 3) Abort any reqs that finished due to stop strings.
243 - await self.engine_core.abort_requests_async(
244 - processed_outputs.reqs_to_abort)
237 + # Split outputs into chunks of at most
238 + # VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
239 + # event loop for too long.
240 + num_outputs = len(outputs.outputs)
241 + if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
242 + slices = (outputs.outputs, )
243 + else:
244 + slices = np.array_split(
245 + outputs.outputs,
246 + cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
247 +
248 + iteration_stats = None
249 + for i, outputs_slice in enumerate(slices):
250 + # 2) Process EngineCoreOutputs.
251 + processed_outputs = self.output_processor.process_outputs(
252 + outputs_slice, iteration_stats)
253 + # NOTE: RequestOutputs are pushed to their queues.
254 + assert not processed_outputs.request_outputs
255 + iteration_stats = processed_outputs.iteration_stats
256 +
257 + # Allow other asyncio tasks to run between chunks
258 + if i + 1 < len(slices):
259 + await asyncio.sleep(0)
260 +
261 + # 3) Abort any reqs that finished due to stop strings.
262 + await self.engine_core.abort_requests_async(
263 + processed_outputs.reqs_to_abort)
264
265 # 4) Logging.
266 # TODO(rob): make into a coroutine and launch it in
267 # background thread once we add Prometheus.
268 + assert iteration_stats is not None
269 self._log_stats(
270 scheduler_stats=outputs.scheduler_stats,
251 - iteration_stats=processed_outputs.iteration_stats,
271 + iteration_stats=iteration_stats,
272 )
273
274 except Exception as e:
vllm/v1/engine/core_client.py
+18 -3
@@ -1,8 +1,9 @@
1 +import asyncio
2 import os
3 import signal
4 import weakref
5 from abc import ABC, abstractmethod
5 -from typing import List, Type
6 +from typing import List, Optional, Type
7
8 import msgspec
9 import zmq
@@ -255,10 +256,24 @@ class AsyncMPClient(MPClient):
256 log_stats=True,
257 )
258
259 + self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
260 + self.queue_task: Optional[asyncio.Task] = None
261 +
262 async def get_output_async(self) -> EngineCoreOutputs:
263 + if self.outputs_queue is None:
264 + # Perform IO in separate task to parallelize as much as possible
265 + self.outputs_queue = asyncio.Queue()
266 +
267 + async def process_outputs_socket():
268 + assert self.outputs_queue is not None
269 + while True:
270 + (frame, ) = await self.output_socket.recv_multipart(
271 + copy=False)
272 + self.outputs_queue.put_nowait(frame.buffer)
273 +
274 + self.queue_task = asyncio.create_task(process_outputs_socket())
275
260 - frames = await self.output_socket.recv_multipart(copy=False)
261 - return self.decoder.decode(frames[0].buffer)
276 + return self.decoder.decode(await self.outputs_queue.get())
277
278 async def _send_input(self, request_type: EngineCoreRequestType,
279 request: EngineCoreRequestUnion) -> None:
vllm/v1/engine/output_processor.py
+4 -2
@@ -101,6 +101,7 @@ class OutputProcessor:
101 def process_outputs(
102 self,
103 engine_core_outputs: List[EngineCoreOutput],
104 + iteration_stats: Optional[IterationStats] = None,
105 ) -> OutputProcessorOutput:
106 """
107 Process the EngineCoreOutputs:
@@ -133,7 +134,8 @@ class OutputProcessor:
134
135 request_outputs: List[RequestOutput] = []
136 reqs_to_abort: List[str] = []
136 - iteration_stats = IterationStats(self.log_stats)
137 + if not iteration_stats:
138 + iteration_stats = IterationStats(self.log_stats)
139 for engine_core_output in engine_core_outputs:
140 req_id = engine_core_output.request_id
141 req_state = self.request_states.get(req_id)
@@ -175,8 +177,8 @@ class OutputProcessor:
177 iteration_stats=iteration_stats,
178 )
179
180 + @staticmethod
181 def _make_request_output(
179 - self,
182 request_state: RequestState,
183 detokenizer_output: Optional[DetokenizerOutput],
184 ) -> Optional[RequestOutput]:
vllm/v1/request.py
+6 -12
@@ -64,6 +64,12 @@ class Request:
64 # recomputing.
65 self._kv_block_hashes: List[BlockHashType] = []
66
67 + # Read-only views
68 + # Prevent directly appending to the these lists since
69 + # they should also be updated simultaneously.
70 + self.output_token_ids = ConstantList(self._output_token_ids)
71 + self.all_token_ids = ConstantList(self._all_token_ids)
72 +
73 @classmethod
74 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
75 return cls(
@@ -79,18 +85,6 @@ class Request:
85 lora_request=request.lora_request,
86 )
87
82 - @property
83 - def output_token_ids(self) -> ConstantList[int]:
84 - # Prevent directly appending to the output_token_ids since
85 - # all_token_ids should also be updated simultaneously.
86 - return ConstantList(self._output_token_ids)
87 -
88 - @property
89 - def all_token_ids(self) -> ConstantList[int]:
90 - # Prevent directly appending to the all_token_ids since
91 - # output_token_ids should also be updated simultaneously
92 - return ConstantList(self._all_token_ids)
93 -
88 def append_output_token_ids(
89 self,
90 token_ids: Union[int, List[int]],