A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Minor] Add some clarifying comments to recent changes (#27130)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Oct 18, 2025 at 09:52 UTC
3b45075206f4847fdfb38d9a9139abad5fed8e31
2 files changed
+10
-2
vllm/distributed/device_communicators/shm_broadcast.py
+7
-1
@@ -236,7 +236,9 @@ class MessageQueue:
236
n_reader, # number of all readers
237
n_local_reader, # number of local readers through shared memory
238
local_reader_ranks: list[int] | None = None,
239
- max_chunk_bytes: int = 1024 * 1024 * 24, # 24MiB
239
+ # Default of 24MiB chosen to be large enough to accommodate grammar
240
+ # bitmask tensors for large batches (1024 requests).
241
+ max_chunk_bytes: int = 1024 * 1024 * 24,
242
max_chunks: int = 10,
243
connect_ip: str | None = None,
244
):
@@ -538,6 +540,10 @@ class MessageQueue:
540
buf[0] = 1 # overflow
541
self.local_socket.send_multipart(all_buffers, copy=False)
542
else:
543
+ # Byte 0: 0
544
+ # Bytes 1-2: Count of buffers
545
+ # Then each buffer follows, preceded by 4 bytes containing its length:
546
+ # [4 byte int L][L bytes of buffer content] ...
547
with self.acquire_write(timeout) as buf:
548
buf[0] = 0 # not overflow
549
offset = 3
vllm/v1/core/sched/output.py
+3
-1
@@ -165,7 +165,9 @@ class SchedulerOutput:
165
# freed from the encoder cache.
166
free_encoder_mm_hashes: list[str]
167
168
- # ids of structured outputs requests included in the bitmask, in order.
168
+ # ids of structured outputs requests included in the bitmask, in the
169
+ # same order as the corresponding stacked rows of the bitmask.
170
+ # There may be more than one row per request in the case of speculative decoding.
171
structured_output_request_ids: list[str]
172
# the bitmask for the whole batch
173
grammar_bitmask: "npt.NDArray[np.int32] | None"