@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 12dbd834cf

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Remove from_seq_group methods (#25330)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Sep 20, 2025 at 21:10 UTC 12dbd834cf33b539a50d65516e44327caff9d824

2 files changed +2 -315

vllm/multimodal/base.py

+1 -121

@@ -2,14 +2,8 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from abc import ABC, abstractmethod

5 -from collections.abc import Sequence

5 from pathlib import Path

7 -from typing import TYPE_CHECKING, Generic, NamedTuple, TypeVar

8 -

9 -if TYPE_CHECKING:

10 - from vllm.sequence import SequenceGroupMetadata

11 -

12 -from .inputs import MultiModalKwargs, PlaceholderRange

6 +from typing import Generic, NamedTuple, TypeVar

8 _T = TypeVar("_T")

@@ -53,120 +47,6 @@ class MultiModalPlaceholderMap:

47 self.dest_ranges = []

48 self.dest_len = 0

56 - @classmethod

57 - def from_seq_group(

58 - cls, seq_group: "SequenceGroupMetadata", positions: range

59 - ) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:

60 - """

61 - Returns the multi-modal items that intersect with the portion of a

62 - prompt (``seq_group``) represented by ``positions``, as well as a

63 - ``MultiModalPlaceholderMap`` that relates the multi-modal embedding

64 - vectors to their corresponding placeholders.

65 -

66 - Examples:

67 -

68 - ```

69 - Prompt: |AAAA BBBB What's in these images?|

70 - Positions: |.................................|

71 -

72 - images = [A, B]

73 - src_ranges = [(0, 4), (4, 8)]

74 - dest_ranges = [(0, 4), (5, 9)]

75 -

76 - Prompt: |AAAA BBBB What's in these images?|

77 - Positions: | ..... |

78 -

79 - images = [A, B]

80 - src_ranges = [(2, 4), (4, 6)]

81 - dest_ranges = [(0, 2), (3, 5)]

82 -

83 - Prompt: |AAAA BBBB What's in these images?|

84 - Positions: | ......... |

85 -

86 - images = [B]

87 - src_ranges = [(0, 4)]

88 - dest_ranges = [(0, 4)]

89 -

90 - Prompt: |AAAA BBBB What's in these images?|

91 - Positions: | .......................|

92 -

93 - images = []

94 - src_ranges = []

95 - dest_ranges = []

96 - ```

97 - """

98 - seq_mm_data = seq_group.multi_modal_data

99 - seq_mm_placeholders = seq_group.multi_modal_placeholders

100 -

101 - if not seq_mm_data or not seq_mm_placeholders:

102 - return MultiModalKwargs(), {}

103 -

104 - placeholder_maps = dict[str, MultiModalPlaceholderMap]()

105 -

106 - for modality, placeholders in seq_mm_placeholders.items():

107 - placeholder_map = MultiModalPlaceholderMap()

108 -

109 - if positions:

110 - placeholder_map.append_items_from_seq_group(

111 - positions,

112 - # Dummy, since we don't care about intersecting items

113 - [None] * len(placeholders),

114 - placeholders,

115 - )

116 -

117 - placeholder_maps[modality] = placeholder_map

118 -

119 - return seq_mm_data, placeholder_maps

120 -

121 - def append_items_from_seq_group(

122 - self,

123 - positions: range,

124 - multi_modal_items: list[_T],

125 - multi_modal_placeholders: Sequence[PlaceholderRange],

126 - ) -> list[_T]:

127 - """

128 - Adds the multi-modal items that intersect ```positions`` to this

129 - placeholder map and returns the intersecting items.

130 - """

131 - intersecting_items = []

132 -

133 - if len(multi_modal_items) != len(multi_modal_placeholders):

134 - raise ValueError(

135 - "Multi-modal placeholders and items must have the same length."

136 - )

137 - for placeholder_dict, mm_item in zip(multi_modal_placeholders,

138 - multi_modal_items):

139 - placeholder = range(

140 - placeholder_dict.offset,

141 - placeholder_dict.offset + placeholder_dict.length,

142 - )

143 - intersection = range(

144 - max(positions.start, placeholder.start),

145 - min(positions.stop, placeholder.stop),

146 - )

147 -

148 - if not intersection:

149 - # Skip this multi-modal item.

150 - continue

151 -

152 - token_embedding_range = range(

153 - intersection.start - positions.start,

154 - intersection.stop - positions.start,

155 - )

156 -

157 - multimodal_embedding_range = range(

158 - intersection.start - placeholder.start + self.src_len,

159 - intersection.stop - placeholder.start + self.src_len,

160 - )

161 -

162 - intersecting_items.append(mm_item)

163 - self.dest_ranges.append(token_embedding_range)

164 - self.src_ranges.append(multimodal_embedding_range)

165 - self.src_len += len(placeholder)

166 -

167 - self.dest_len += len(positions)

168 - return intersecting_items

169 -

50 def extend(self, other: "MultiModalPlaceholderMap"):

51 """

52 Adds the placeholders from another ``MultiModalPlaceholderMap`` to this

vllm/outputs.py

+1 -194

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -import time

4 from collections.abc import MutableSequence

5 from collections.abc import Sequence as GenericSequence

6 from dataclasses import dataclass

@@ -14,9 +13,7 @@ from vllm.logger import init_logger

13 from vllm.logprobs import PromptLogprobs, SampleLogprobs

14 from vllm.lora.request import LoRARequest

15 from vllm.multimodal.inputs import MultiModalPlaceholderDict

17 -from vllm.sampling_params import RequestOutputKind

18 -from vllm.sequence import (RequestMetrics, SequenceGroup, SequenceGroupBase,

19 - SequenceStatus)

16 +from vllm.sequence import RequestMetrics

18 logger = init_logger(__name__)

@@ -171,170 +168,6 @@ class RequestOutput:

168 else:

169 self.outputs.append(next_completion)

170

174 - @classmethod

175 - def from_seq_group(

176 - cls, seq_group: SequenceGroup, use_cache: bool,

177 - seq_id_to_seq_group: dict[str, SequenceGroupBase]

178 - ) -> Optional["RequestOutput"]:

179 - finished = seq_group.is_finished()

180 -

181 - if seq_group.request_id in seq_id_to_seq_group:

182 - group: SequenceGroupBase = seq_id_to_seq_group[

183 - seq_group.request_id]

184 - assembled_seq_group = group.maybe_assemble_group(seq_group)

185 - if finished:

186 - group.finish_seq(seq_group)

187 - if assembled_seq_group is None:

188 - return None

189 -

190 - # clear finished seq in seq_id_to_seq_group

191 - if len(group.to_be_finished) == 0:

192 - for sub_request_id in list(group.seq_id_to_index.keys()):

193 - if sub_request_id in seq_id_to_seq_group:

194 - del seq_id_to_seq_group[sub_request_id]

195 -

196 - return cls.from_seq_group(assembled_seq_group, use_cache,

197 - seq_id_to_seq_group)

198 -

199 - sampling_params = seq_group.sampling_params

200 - if sampling_params is None:

201 - raise ValueError(

202 - "Sampling parameters are missing for a CompletionRequest.")

203 -

204 - if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (

205 - not finished):

206 - return None

207 -

208 - # Init cache (if needed)

209 - if use_cache and seq_group.cached_request_output is None:

210 - seq_group.cached_request_output = RequestOutput( # type: ignore

211 - request_id="",

212 - prompt=None,

213 - prompt_token_ids=[],

214 - prompt_logprobs=None,

215 - outputs=[],

216 - finished=False)

217 -

218 - top_n_seqs = seq_group.get_seqs()

219 -

220 - # Create the outputs.

221 - # NOTE: We need omit logprobs here explicitly because the sequence

222 - # always has the logprobs of the sampled tokens even if the

223 - # logprobs are not requested.

224 - include_logprobs = sampling_params.logprobs is not None

225 - text_buffer_length = sampling_params.output_text_buffer_length

226 - delta = sampling_params.output_kind == RequestOutputKind.DELTA

227 -

228 - outputs = []

229 - include_prompt = True

230 - # num_cached_tokens should be the same for all the sequences

231 - num_cached_tokens = None

232 - for i, seq in enumerate(top_n_seqs):

233 - output_text = seq.get_output_text_to_return(

234 - text_buffer_length, delta)

235 -

236 - output_token_ids = seq.get_output_token_ids_to_return(delta)

237 - num_output_tokens = 1 if isinstance(output_token_ids,

238 - int) else len(output_token_ids)

239 - num_cached_tokens = seq.data.get_num_cached_tokens()

240 -

241 - output_logprobs = seq.output_logprobs if include_logprobs else None

242 -

243 - if delta:

244 - # Slice logprobs delta if applicable

245 - if output_logprobs:

246 - # num_output_tokens can be 0 when n > 1 and request finishes

247 - # before the others

248 - if num_output_tokens > 0:

249 - output_logprobs = output_logprobs[-num_output_tokens:]

250 - else:

251 - output_logprobs = None

252 - # Don't include prompt if this is after the first output

253 - # containing decode token ids

254 - if include_prompt and seq.get_output_len() > num_output_tokens:

255 - include_prompt = False

256 -

257 - if use_cache:

258 - # Get cached output object

259 - cached_outputs = seq_group.cached_request_output.outputs # type: ignore

260 - if i >= len(cached_outputs):

261 - cached_outputs.append(

262 - CompletionOutput(index=i,

263 - text="",

264 - token_ids=[],

265 - cumulative_logprob=None,

266 - logprobs=None,

267 - finish_reason=None,

268 - stop_reason=None))

269 - output = cached_outputs[i]

270 -

271 - # Init cached output object

272 - assert output.index == i

273 - output.text = output_text

274 -

275 - if isinstance(output_token_ids, int):

276 - output.token_ids.clear()

277 - output.token_ids.append(output_token_ids)

278 - else:

279 - output.token_ids = output_token_ids

280 -

281 - output.cumulative_logprob = seq.get_cumulative_logprob() \

282 - if include_logprobs else None

283 - output.logprobs = output_logprobs

284 - output.finish_reason = SequenceStatus.get_finished_reason(

285 - seq.status)

286 - output.stop_reason = seq.stop_reason

287 -

288 - else:

289 - output = CompletionOutput(

290 - top_n_seqs.index(seq), output_text, [output_token_ids]

291 - if isinstance(output_token_ids, int) else output_token_ids,

292 - seq.get_cumulative_logprob() if include_logprobs else None,

293 - output_logprobs,

294 - SequenceStatus.get_finished_reason(seq.status),

295 - seq.stop_reason)

296 -

297 - outputs.append(output)

298 -

299 - # Every sequence in the sequence group should have the same prompt.

300 - if include_prompt:

301 - prompt = seq_group.prompt

302 - prompt_token_ids = seq_group.prompt_token_ids

303 - encoder_prompt = seq_group.encoder_prompt

304 - encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids

305 - prompt_logprobs = seq_group.prompt_logprobs

306 - else:

307 - prompt = None

308 - prompt_token_ids = None

309 - encoder_prompt = None

310 - encoder_prompt_token_ids = None

311 - prompt_logprobs = None

312 - finished_time = time.time() if finished else None

313 - seq_group.set_finished_time(finished_time)

314 -

315 - init_kwargs = {

316 - "request_id": seq_group.request_id,

317 - "prompt": prompt,

318 - "prompt_token_ids": prompt_token_ids,

319 - "prompt_logprobs": prompt_logprobs,

320 - "outputs": outputs,

321 - "finished": finished,

322 - "metrics": seq_group.metrics,

323 - "lora_request": seq_group.lora_request,

324 - "encoder_prompt": encoder_prompt,

325 - "encoder_prompt_token_ids": encoder_prompt_token_ids,

326 - "num_cached_tokens": num_cached_tokens,

327 - "multi_modal_placeholders": seq_group.multi_modal_placeholders

328 - }

329 -

330 - if use_cache:

331 - request_output = seq_group.cached_request_output

332 - request_output.__init__(**init_kwargs) # type: ignore

333 - else:

334 - request_output = cls(**init_kwargs) # type: ignore

335 -

336 - return request_output

337 -

171 def __repr__(self) -> str:

172 return (f"RequestOutput(request_id={self.request_id}, "

173 f"prompt={self.prompt!r}, "

@@ -371,19 +204,6 @@ class PoolingRequestOutput(Generic[_O]):

204 self.finished = finished

205 self.outputs = outputs

206

374 - @staticmethod

375 - def from_seq_group(seq_group: SequenceGroup) -> "PoolingRequestOutput":

376 - pooled_data = seq_group.pooled_data

377 - assert pooled_data is not None

378 -

379 - data = pooled_data.to(dtype=torch.float32, device="cpu")

380 - output = PoolingOutput(data)

381 - prompt_token_ids = seq_group.prompt_token_ids

382 - finished = seq_group.is_finished()

383 -

384 - return PoolingRequestOutput(seq_group.request_id, output,

385 - prompt_token_ids, finished)

386 -

207 def __repr__(self):

208 return (f"{type(self).__name__}(request_id={self.request_id!r}, "

209 f"outputs={self.outputs!r}, "

@@ -391,19 +211,6 @@ class PoolingRequestOutput(Generic[_O]):

211 f"finished={self.finished})")

212

213

394 -class RequestOutputFactory:

395 -

396 - @staticmethod

397 - def create(seq_group: SequenceGroup,

398 - seq_id_to_seq_group: dict[str, SequenceGroupBase],

399 - use_cache: bool = False):

400 - if seq_group.pooled_data is not None:

401 - return PoolingRequestOutput.from_seq_group(seq_group)

402 - else:

403 - return RequestOutput.from_seq_group(seq_group, use_cache,

404 - seq_id_to_seq_group)

405 -

406 -

214 @dataclass

215 class EmbeddingOutput:

216 """The output data of one embedding output of a request.