A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Core] add and implement `VLLM_LOGITS_PROCESSOR_THREADS` (#12368)
Signed-off-by: Aviv Keshet <akeshet@scaledcognition.com>
Aviv Keshet committed
Feb 4, 2025 at 18:46 UTC
b3a0d01e4551118c735ee905c4ddc800ec603f24
2 files changed
+44
-11
vllm/envs.py
+9
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
31
VLLM_LOGGING_LEVEL: str = "INFO"
32
VLLM_LOGGING_PREFIX: str = ""
33
VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
34
+ VLLM_LOGITS_PROCESSOR_THREADS: Optional[int] = None
35
VLLM_TRACE_FUNCTION: int = 0
36
VLLM_ATTENTION_BACKEND: Optional[str] = None
37
VLLM_USE_FLASHINFER_SAMPLER: Optional[bool] = None
@@ -282,6 +283,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
283
"VLLM_LOGGING_PREFIX":
284
lambda: os.getenv("VLLM_LOGGING_PREFIX", ""),
285
286
+ # if set, vllm will call logits processors in a thread pool with this many
287
+ # threads. This is useful when using custom logits processors that either
288
+ # (a) launch additional CUDA kernels or (b) do significant CPU-bound work
289
+ # while not holding the python GIL, or both.
290
+ "VLLM_LOGITS_PROCESSOR_THREADS":
291
+ lambda: int(os.getenv("VLLM_LOGITS_PROCESSOR_THREADS", "0"))
292
+ if "VLLM_LOGITS_PROCESSOR_THREADS" in os.environ else None,
293
+
294
# Trace function calls
295
# If set to 1, vllm will trace function calls
296
# Useful for debugging
vllm/model_executor/layers/logits_processor.py
+35
-11
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""A layer that compute logits from hidden_stats."""
3
import inspect
4
+from concurrent.futures import ThreadPoolExecutor
5
from typing import Optional
6
7
import torch
@@ -15,6 +16,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.platforms import current_platform
18
19
+_logits_processor_threadpool: Optional[ThreadPoolExecutor] = None
20
+if envs.VLLM_LOGITS_PROCESSOR_THREADS is not None:
21
+ _logits_processor_threadpool = ThreadPoolExecutor(
22
+ envs.VLLM_LOGITS_PROCESSOR_THREADS)
23
+
24
25
class LogitsProcessor(nn.Module):
26
"""Process logits and apply logits processors from sampling metadata.
@@ -135,6 +141,7 @@ def _apply_logits_processors(
141
) -> torch.Tensor:
142
found_logits_processors = False
143
logits_processed = 0
144
+ logits_row_ids_and_logits_row_futures = []
145
for seq_group in sampling_metadata.seq_groups:
146
seq_ids = seq_group.seq_ids
147
sampling_params = seq_group.sampling_params
@@ -148,22 +155,39 @@ def _apply_logits_processors(
155
past_tokens_ids = seq_group.seq_data[seq_id].output_token_ids
156
prompt_tokens_ids = seq_group.seq_data[seq_id].prompt_token_ids
157
151
- for logits_processor in logits_processors:
152
- parameters = inspect.signature(logits_processor).parameters
153
- if len(parameters) == 3:
154
- logits_row = logits_processor(prompt_tokens_ids,
155
- past_tokens_ids,
156
- logits_row)
157
- else:
158
- logits_row = logits_processor(past_tokens_ids,
159
- logits_row)
160
-
161
- logits[logits_row_idx] = logits_row
158
+ if _logits_processor_threadpool is not None:
159
+ logits_row_ids_and_logits_row_futures.append(
160
+ (logits_row_idx,
161
+ _logits_processor_threadpool.submit(
162
+ _apply_logits_processors_single_seq, logits_row,
163
+ logits_processors, past_tokens_ids,
164
+ prompt_tokens_ids)))
165
+ else:
166
+ logits[logits_row_idx] = \
167
+ _apply_logits_processors_single_seq(
168
+ logits_row, logits_processors, past_tokens_ids,
169
+ prompt_tokens_ids)
170
171
logits_processed += len(seq_group.sample_indices) + len(
172
seq_group.prompt_logprob_indices)
173
174
+ for logits_row_idx, future in logits_row_ids_and_logits_row_futures:
175
+ logits[logits_row_idx] = future.result()
176
+
177
if found_logits_processors:
178
# verifies that no rows in logits were missed unexpectedly
179
assert logits_processed == logits.shape[0]
180
return logits
181
+
182
+
183
+def _apply_logits_processors_single_seq(logits_row, logits_processors,
184
+ past_tokens_ids,
185
+ prompt_tokens_ids) -> torch.Tensor:
186
+ for logits_processor in logits_processors:
187
+ parameters = inspect.signature(logits_processor).parameters
188
+ if len(parameters) == 3:
189
+ logits_row = logits_processor(prompt_tokens_ids, past_tokens_ids,
190
+ logits_row)
191
+ else:
192
+ logits_row = logits_processor(past_tokens_ids, logits_row)
193
+ return logits_row