@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 643ecf7b11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Refactor model executable interface for all text-only language models (#10374)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 16, 2024 at 21:18 UTC 643ecf7b11a3e74c838f438cfc1b3e59c018853b
43 files changed +483 -90
vllm/model_executor/models/arctic.py
+14 -2
@@ -389,6 +389,9 @@ class ArcticModel(nn.Module):
389 make_empty_intermediate_tensors_factory(["hidden_states"],
390 config.hidden_size))
391
392 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
393 + return self.embed_tokens(input_ids)
394 +
395 def forward(
396 self,
397 input_ids: torch.Tensor,
@@ -396,9 +399,13 @@ class ArcticModel(nn.Module):
399 kv_caches: List[torch.Tensor],
400 attn_metadata: AttentionMetadata,
401 intermediate_tensors: Optional[IntermediateTensors],
402 + inputs_embeds: Optional[torch.Tensor] = None,
403 ) -> Union[torch.Tensor, IntermediateTensors]:
404 if get_pp_group().is_first_rank:
401 - hidden_states = self.embed_tokens(input_ids)
405 + if inputs_embeds is not None:
406 + hidden_states = inputs_embeds
407 + else:
408 + hidden_states = self.get_input_embeddings(input_ids)
409 else:
410 assert intermediate_tensors is not None
411 hidden_states = intermediate_tensors["hidden_states"]
@@ -439,6 +446,9 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
446 self.make_empty_intermediate_tensors = (
447 self.model.make_empty_intermediate_tensors)
448
449 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
450 + return self.model.get_input_embeddings(input_ids)
451 +
452 def forward(
453 self,
454 input_ids: torch.Tensor,
@@ -446,9 +456,11 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
456 kv_caches: List[torch.Tensor],
457 attn_metadata: AttentionMetadata,
458 intermediate_tensors: Optional[IntermediateTensors] = None,
459 + inputs_embeds: Optional[torch.Tensor] = None,
460 ) -> Union[torch.Tensor, IntermediateTensors]:
461 hidden_states = self.model(input_ids, positions, kv_caches,
451 - attn_metadata, intermediate_tensors)
462 + attn_metadata, intermediate_tensors,
463 + inputs_embeds)
464 return hidden_states
465
466 def compute_logits(
vllm/model_executor/models/baichuan.py
+14 -2
@@ -284,6 +284,9 @@ class BaiChuanModel(nn.Module):
284 make_empty_intermediate_tensors_factory(
285 ["hidden_states", "residual"], config.hidden_size))
286
287 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
288 + return self.embed_tokens(input_ids)
289 +
290 def forward(
291 self,
292 input_ids: torch.Tensor,
@@ -291,9 +294,13 @@ class BaiChuanModel(nn.Module):
294 kv_caches: List[torch.Tensor],
295 attn_metadata: AttentionMetadata,
296 intermediate_tensors: Optional[IntermediateTensors],
297 + inputs_embeds: Optional[torch.Tensor] = None,
298 ) -> Union[torch.Tensor, IntermediateTensors]:
299 if get_pp_group().is_first_rank:
296 - hidden_states = self.embed_tokens(input_ids)
300 + if inputs_embeds is not None:
301 + hidden_states = inputs_embeds
302 + else:
303 + hidden_states = self.get_input_embeddings(input_ids)
304 residual = None
305 else:
306 assert intermediate_tensors is not None
@@ -363,6 +370,9 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370 self.make_empty_intermediate_tensors = (
371 self.model.make_empty_intermediate_tensors)
372
373 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
374 + return self.model.get_input_embeddings(input_ids)
375 +
376 def forward(
377 self,
378 input_ids: torch.Tensor,
@@ -370,9 +380,11 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
380 kv_caches: List[torch.Tensor],
381 attn_metadata: AttentionMetadata,
382 intermediate_tensors: Optional[IntermediateTensors] = None,
383 + inputs_embeds: Optional[torch.Tensor] = None,
384 ) -> Union[torch.Tensor, IntermediateTensors]:
385 hidden_states = self.model(input_ids, positions, kv_caches,
375 - attn_metadata, intermediate_tensors)
386 + attn_metadata, intermediate_tensors,
387 + inputs_embeds)
388 return hidden_states
389
390 def compute_logits(
vllm/model_executor/models/bloom.py
+14 -3
@@ -251,6 +251,9 @@ class BloomModel(nn.Module):
251 make_empty_intermediate_tensors_factory(["hidden_states"],
252 config.hidden_size))
253
254 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
255 + return self.word_embeddings_layernorm(self.word_embeddings(input_ids))
256 +
257 def forward(
258 self,
259 input_ids: torch.Tensor,
@@ -258,10 +261,13 @@ class BloomModel(nn.Module):
261 kv_caches: List[torch.Tensor],
262 attn_metadata: AttentionMetadata,
263 intermediate_tensors: Optional[IntermediateTensors],
264 + inputs_embeds: Optional[torch.Tensor] = None,
265 ) -> Union[torch.Tensor, IntermediateTensors]:
266 if get_pp_group().is_first_rank:
263 - hidden_states = self.word_embeddings(input_ids)
264 - hidden_states = self.word_embeddings_layernorm(hidden_states)
267 + if inputs_embeds is not None:
268 + hidden_states = inputs_embeds
269 + else:
270 + hidden_states = self.get_input_embeddings(input_ids)
271 else:
272 assert intermediate_tensors is not None
273 hidden_states = intermediate_tensors["hidden_states"]
@@ -301,6 +307,9 @@ class BloomForCausalLM(nn.Module, SupportsPP):
307 self.make_empty_intermediate_tensors = (
308 self.transformer.make_empty_intermediate_tensors)
309
310 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
311 + return self.transformer.get_input_embeddings(input_ids)
312 +
313 def forward(
314 self,
315 input_ids: torch.Tensor,
@@ -308,9 +317,11 @@ class BloomForCausalLM(nn.Module, SupportsPP):
317 kv_caches: List[torch.Tensor],
318 attn_metadata: AttentionMetadata,
319 intermediate_tensors: Optional[IntermediateTensors] = None,
320 + inputs_embeds: Optional[torch.Tensor] = None,
321 ) -> Union[torch.Tensor, IntermediateTensors]:
322 hidden_states = self.transformer(input_ids, positions, kv_caches,
313 - attn_metadata, intermediate_tensors)
323 + attn_metadata, intermediate_tensors,
324 + inputs_embeds)
325 return hidden_states
326
327 def compute_logits(
vllm/model_executor/models/commandr.py
+14 -2
@@ -280,6 +280,9 @@ class CohereModel(nn.Module):
280 make_empty_intermediate_tensors_factory(
281 ["hidden_states", "residual"], config.hidden_size))
282
283 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
284 + return self.embed_tokens(input_ids)
285 +
286 def forward(
287 self,
288 input_ids: torch.Tensor,
@@ -287,9 +290,13 @@ class CohereModel(nn.Module):
290 kv_caches: List[torch.Tensor],
291 attn_metadata: AttentionMetadata,
292 intermediate_tensors: Optional[IntermediateTensors],
293 + inputs_embeds: Optional[torch.Tensor] = None,
294 ) -> Union[torch.Tensor, IntermediateTensors]:
295 if get_pp_group().is_first_rank:
292 - hidden_states = self.embed_tokens(input_ids)
296 + if inputs_embeds is not None:
297 + hidden_states = inputs_embeds
298 + else:
299 + hidden_states = self.get_input_embeddings(input_ids)
300 residual = None
301 else:
302 assert intermediate_tensors is not None
@@ -354,6 +361,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
361 self.make_empty_intermediate_tensors = (
362 self.model.make_empty_intermediate_tensors)
363
364 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
365 + return self.model.get_input_embeddings(input_ids)
366 +
367 @torch.no_grad()
368 def forward(
369 self,
@@ -362,9 +372,11 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
372 kv_caches: List[torch.Tensor],
373 attn_metadata: AttentionMetadata,
374 intermediate_tensors: Optional[IntermediateTensors] = None,
375 + inputs_embeds: Optional[torch.Tensor] = None,
376 ) -> Union[torch.Tensor, IntermediateTensors]:
377 hidden_states = self.model(input_ids, positions, kv_caches,
367 - attn_metadata, intermediate_tensors)
378 + attn_metadata, intermediate_tensors,
379 + inputs_embeds)
380 return hidden_states
381
382 def compute_logits(
vllm/model_executor/models/dbrx.py
+14 -2
@@ -321,6 +321,9 @@ class DbrxModel(nn.Module):
321 make_empty_intermediate_tensors_factory(["hidden_states"],
322 config.d_model))
323
324 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
325 + return self.wte(input_ids)
326 +
327 def forward(
328 self,
329 input_ids: torch.Tensor,
@@ -328,9 +331,13 @@ class DbrxModel(nn.Module):
331 kv_caches: List[torch.Tensor],
332 attn_metadata: AttentionMetadata,
333 intermediate_tensors: Optional[IntermediateTensors],
334 + inputs_embeds: Optional[torch.Tensor] = None,
335 ) -> Union[torch.Tensor, IntermediateTensors]:
336 if get_pp_group().is_first_rank:
333 - hidden_states = self.wte(input_ids)
337 + if inputs_embeds is not None:
338 + hidden_states = inputs_embeds
339 + else:
340 + hidden_states = self.get_input_embeddings(input_ids)
341 else:
342 assert intermediate_tensors
343 hidden_states = intermediate_tensors["hidden_states"]
@@ -376,6 +383,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
383 self.make_empty_intermediate_tensors = (
384 self.transformer.make_empty_intermediate_tensors)
385
386 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
387 + return self.transformer.get_input_embeddings(input_ids)
388 +
389 def forward(
390 self,
391 input_ids: torch.Tensor,
@@ -383,9 +393,11 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
393 kv_caches: List[torch.Tensor],
394 attn_metadata: AttentionMetadata,
395 intermediate_tensors: Optional[IntermediateTensors] = None,
396 + inputs_embeds: Optional[torch.Tensor] = None,
397 ) -> Union[torch.Tensor, IntermediateTensors]:
398 hidden_states = self.transformer(input_ids, positions, kv_caches,
388 - attn_metadata, intermediate_tensors)
399 + attn_metadata, intermediate_tensors,
400 + inputs_embeds)
401 return hidden_states
402
403 def compute_logits(
vllm/model_executor/models/deepseek.py
+14 -2
@@ -353,6 +353,9 @@ class DeepseekModel(nn.Module):
353 make_empty_intermediate_tensors_factory(
354 ["hidden_states", "residual"], config.hidden_size))
355
356 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
357 + return self.embed_tokens(input_ids)
358 +
359 def forward(
360 self,
361 input_ids: torch.Tensor,
@@ -360,9 +363,13 @@ class DeepseekModel(nn.Module):
363 kv_caches: List[torch.Tensor],
364 attn_metadata: AttentionMetadata,
365 intermediate_tensors: Optional[IntermediateTensors],
366 + inputs_embeds: Optional[torch.Tensor] = None,
367 ) -> Union[torch.Tensor, IntermediateTensors]:
368 if get_pp_group().is_first_rank:
365 - hidden_states = self.embed_tokens(input_ids)
369 + if inputs_embeds is not None:
370 + hidden_states = inputs_embeds
371 + else:
372 + hidden_states = self.get_input_embeddings(input_ids)
373 residual = None
374 else:
375 hidden_states = intermediate_tensors["hidden_states"]
@@ -401,6 +408,9 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
408 self.make_empty_intermediate_tensors = (
409 self.model.make_empty_intermediate_tensors)
410
411 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
412 + return self.model.get_input_embeddings(input_ids)
413 +
414 def forward(
415 self,
416 input_ids: torch.Tensor,
@@ -408,9 +418,11 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
418 kv_caches: List[torch.Tensor],
419 attn_metadata: AttentionMetadata,
420 intermediate_tensors: Optional[IntermediateTensors] = None,
421 + inputs_embeds: Optional[torch.Tensor] = None,
422 ) -> Union[torch.Tensor, IntermediateTensors]:
423 hidden_states = self.model(input_ids, positions, kv_caches,
413 - attn_metadata, intermediate_tensors)
424 + attn_metadata, intermediate_tensors,
425 + inputs_embeds)
426 return hidden_states
427
428 def compute_logits(
vllm/model_executor/models/deepseek_v2.py
+14 -2
@@ -445,6 +445,9 @@ class DeepseekV2Model(nn.Module):
445 make_empty_intermediate_tensors_factory(
446 ["hidden_states", "residual"], config.hidden_size))
447
448 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
449 + return self.embed_tokens(input_ids)
450 +
451 def forward(
452 self,
453 input_ids: torch.Tensor,
@@ -452,9 +455,13 @@ class DeepseekV2Model(nn.Module):
455 kv_caches: List[torch.Tensor],
456 attn_metadata: AttentionMetadata,
457 intermediate_tensors: Optional[IntermediateTensors],
458 + inputs_embeds: Optional[torch.Tensor] = None,
459 ) -> Union[torch.Tensor, IntermediateTensors]:
460 if get_pp_group().is_first_rank:
457 - hidden_states = self.embed_tokens(input_ids)
461 + if inputs_embeds is not None:
462 + hidden_states = inputs_embeds
463 + else:
464 + hidden_states = self.get_input_embeddings(input_ids)
465 residual = None
466 else:
467 assert intermediate_tensors is not None
@@ -495,6 +502,9 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
502 self.make_empty_intermediate_tensors = (
503 self.model.make_empty_intermediate_tensors)
504
505 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
506 + return self.model.get_input_embeddings(input_ids)
507 +
508 def forward(
509 self,
510 input_ids: torch.Tensor,
@@ -502,9 +512,11 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
512 kv_caches: List[torch.Tensor],
513 attn_metadata: AttentionMetadata,
514 intermediate_tensors: Optional[IntermediateTensors] = None,
515 + inputs_embeds: Optional[torch.Tensor] = None,
516 ) -> Union[torch.Tensor, IntermediateTensors]:
517 hidden_states = self.model(input_ids, positions, kv_caches,
507 - attn_metadata, intermediate_tensors)
518 + attn_metadata, intermediate_tensors,
519 + inputs_embeds)
520 return hidden_states
521
522 def compute_logits(
vllm/model_executor/models/eagle.py
+10 -3
@@ -78,6 +78,9 @@ class EAGLE(nn.Module):
78 def sampler(self):
79 return self.model.sampler
80
81 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
82 + return self.model.model.get_input_embeddings(input_ids)
83 +
84 def forward(
85 self,
86 input_ids: torch.Tensor,
@@ -86,11 +89,14 @@ class EAGLE(nn.Module):
89 attn_metadata: AttentionMetadata,
90 previous_hidden_states: torch.Tensor,
91 intermediate_tensors: Optional[IntermediateTensors] = None,
92 + inputs_embeds: Optional[torch.Tensor] = None,
93 ) -> torch.Tensor:
94
91 - tok_embeds = self.model.model.embed_tokens(input_ids)
95 + if inputs_embeds is None:
96 + inputs_embeds = self.get_input_embeddings(input_ids)
97 +
98 inputs_embeds = self.fc(
93 - torch.cat([tok_embeds, previous_hidden_states], dim=-1))
99 + torch.cat([inputs_embeds, previous_hidden_states], dim=-1))
100
101 inputs_embeds[positions == 0] = 0 # masking inputs at position=0
102
@@ -100,7 +106,8 @@ class EAGLE(nn.Module):
106 positions=positions,
107 kv_caches=kv_caches,
108 attn_metadata=attn_metadata,
103 - intermediate_tensors=intermediate_tensors)
109 + intermediate_tensors=intermediate_tensors,
110 + )
111 return hidden_states
112
113 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/exaone.py
+6 -1
@@ -479,6 +479,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
479 self.make_empty_intermediate_tensors = (
480 self.transformer.make_empty_intermediate_tensors)
481
482 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
483 + return self.model.get_input_embeddings(input_ids)
484 +
485 def forward(
486 self,
487 input_ids: torch.Tensor,
@@ -486,9 +489,11 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
489 kv_caches: List[torch.Tensor],
490 attn_metadata: AttentionMetadata,
491 intermediate_tensors: Optional[IntermediateTensors] = None,
492 + inputs_embeds: Optional[torch.Tensor] = None,
493 ) -> Union[torch.Tensor, IntermediateTensors]:
494 model_output = self.transformer(input_ids, positions, kv_caches,
491 - attn_metadata, intermediate_tensors)
495 + attn_metadata, intermediate_tensors,
496 + inputs_embeds)
497 return model_output
498
499 def compute_logits(
vllm/model_executor/models/falcon.py
+14 -2
@@ -367,6 +367,9 @@ class FalconModel(nn.Module):
367 make_empty_intermediate_tensors_factory(["hidden_states"],
368 config.hidden_size))
369
370 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
371 + return self.word_embeddings(input_ids)
372 +
373 def forward(
374 self,
375 input_ids: torch.Tensor,
@@ -374,9 +377,13 @@ class FalconModel(nn.Module):
377 kv_caches: List[torch.Tensor],
378 attn_metadata: AttentionMetadata,
379 intermediate_tensors: Optional[IntermediateTensors],
380 + inputs_embeds: Optional[torch.Tensor] = None,
381 ) -> Union[torch.Tensor, IntermediateTensors]:
382 if get_pp_group().is_first_rank:
379 - hidden_states = self.word_embeddings(input_ids)
383 + if inputs_embeds is not None:
384 + hidden_states = inputs_embeds
385 + else:
386 + hidden_states = self.get_input_embeddings(input_ids)
387 else:
388 hidden_states = intermediate_tensors["hidden_states"]
389 for i in range(self.start_layer, self.end_layer):
@@ -432,6 +439,9 @@ class FalconForCausalLM(nn.Module, SupportsPP):
439 self.make_empty_intermediate_tensors = (
440 self.transformer.make_empty_intermediate_tensors)
441
442 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
443 + return self.transformer.get_input_embeddings(input_ids)
444 +
445 def forward(
446 self,
447 input_ids: torch.LongTensor,
@@ -439,9 +449,11 @@ class FalconForCausalLM(nn.Module, SupportsPP):
449 kv_caches: List[torch.Tensor],
450 attn_metadata: AttentionMetadata,
451 intermediate_tensors: Optional[IntermediateTensors] = None,
452 + inputs_embeds: Optional[torch.Tensor] = None,
453 ) -> torch.Tensor:
454 hidden_states = self.transformer(input_ids, positions, kv_caches,
444 - attn_metadata, intermediate_tensors)
455 + attn_metadata, intermediate_tensors,
456 + inputs_embeds)
457 return hidden_states
458
459 def compute_logits(
vllm/model_executor/models/gemma.py
+6 -1
@@ -390,6 +390,9 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
390 self.make_empty_intermediate_tensors = (
391 self.model.make_empty_intermediate_tensors)
392
393 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
394 + return self.model.get_input_embeddings(input_ids)
395 +
396 def forward(
397 self,
398 input_ids: torch.Tensor,
@@ -397,9 +400,11 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
400 kv_caches: List[torch.Tensor],
401 attn_metadata: AttentionMetadata,
402 intermediate_tensors: Optional[IntermediateTensors] = None,
403 + inputs_embeds: Optional[torch.Tensor] = None,
404 ) -> Union[torch.Tensor, IntermediateTensors]:
405 hidden_states = self.model(input_ids, positions, kv_caches,
402 - attn_metadata, intermediate_tensors)
406 + attn_metadata, intermediate_tensors,
407 + inputs_embeds)
408 return hidden_states
409
410 def compute_logits(
vllm/model_executor/models/gemma2.py
+10 -2
@@ -272,6 +272,9 @@ class Gemma2Model(nn.Module):
272 make_empty_intermediate_tensors_factory(
273 ["hidden_states", "residual"], config.hidden_size))
274
275 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
276 + return self.embed_tokens(input_ids)
277 +
278 def forward(
279 self,
280 input_ids: Optional[torch.Tensor],
@@ -285,7 +288,7 @@ class Gemma2Model(nn.Module):
288 if inputs_embeds is not None:
289 hidden_states = inputs_embeds
290 else:
288 - hidden_states = self.embed_tokens(input_ids)
291 + hidden_states = self.get_input_embeddings(input_ids)
292 hidden_states *= self.normalizer
293 residual = None
294 else:
@@ -414,6 +417,9 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
417 self.make_empty_intermediate_tensors = (
418 self.model.make_empty_intermediate_tensors)
419
420 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
421 + return self.model.get_input_embeddings(input_ids)
422 +
423 def forward(
424 self,
425 input_ids: torch.Tensor,
@@ -421,9 +427,11 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
427 kv_caches: List[torch.Tensor],
428 attn_metadata: AttentionMetadata,
429 intermediate_tensors: Optional[IntermediateTensors] = None,
430 + inputs_embeds: Optional[torch.Tensor] = None,
431 ) -> Union[torch.Tensor, IntermediateTensors]:
432 hidden_states = self.model(input_ids, positions, kv_caches,
426 - attn_metadata, intermediate_tensors)
433 + attn_metadata, intermediate_tensors,
434 + inputs_embeds)
435 return hidden_states
436
437 def compute_logits(
vllm/model_executor/models/gpt2.py
+5 -2
@@ -209,6 +209,9 @@ class GPT2Model(nn.Module):
209 make_empty_intermediate_tensors_factory(["hidden_states"],
210 config.n_embd))
211
212 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
213 + return self.wte(input_ids)
214 +
215 def forward(
216 self,
217 input_ids: torch.Tensor,
@@ -220,7 +223,7 @@ class GPT2Model(nn.Module):
223 ) -> Union[torch.Tensor, IntermediateTensors]:
224 if get_pp_group().is_first_rank:
225 if inputs_embeds is None:
223 - inputs_embeds = self.wte(input_ids)
226 + inputs_embeds = self.get_input_embeddings(input_ids)
227 position_embeds = self.wpe(position_ids)
228 hidden_states = inputs_embeds + position_embeds
229 else:
@@ -262,7 +265,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
265 self.transformer.make_empty_intermediate_tensors)
266
267 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
265 - return self.transformer.wte(input_ids)
268 + return self.transformer.get_input_embeddings(input_ids)
269
270 def forward(
271 self,
vllm/model_executor/models/gpt_bigcode.py
+13 -4
@@ -218,6 +218,9 @@ class GPTBigCodeModel(nn.Module):
218 make_empty_intermediate_tensors_factory(["hidden_states"],
219 config.n_embd))
220
221 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222 + return self.wte(input_ids)
223 +
224 def forward(
225 self,
226 input_ids: torch.Tensor,
@@ -225,11 +228,12 @@ class GPTBigCodeModel(nn.Module):
228 kv_caches: List[torch.Tensor],
229 attn_metadata: AttentionMetadata,
230 intermediate_tensors: Optional[IntermediateTensors],
231 + inputs_embeds: Optional[torch.Tensor] = None,
232 ) -> Union[torch.Tensor, IntermediateTensors]:
233 if get_pp_group().is_first_rank:
230 - inputs_embeds = self.wte(input_ids)
231 - position_embeds = self.wpe(position_ids)
232 - hidden_states = inputs_embeds + position_embeds
234 + if inputs_embeds is None:
235 + inputs_embeds = self.get_input_embeddings(input_ids)
236 + hidden_states = inputs_embeds + self.wpe(position_ids)
237 else:
238 hidden_states = intermediate_tensors["hidden_states"]
239
@@ -285,6 +289,9 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
289 self.make_empty_intermediate_tensors = (
290 self.transformer.make_empty_intermediate_tensors)
291
292 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
293 + return self.transformer.get_input_embeddings(input_ids)
294 +
295 def forward(
296 self,
297 input_ids: torch.Tensor,
@@ -292,9 +299,11 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
299 kv_caches: List[torch.Tensor],
300 attn_metadata: AttentionMetadata,
301 intermediate_tensors: Optional[IntermediateTensors] = None,
302 + inputs_embeds: Optional[torch.Tensor] = None,
303 ) -> Union[torch.Tensor, IntermediateTensors]:
304 hidden_states = self.transformer(input_ids, positions, kv_caches,
297 - attn_metadata, intermediate_tensors)
305 + attn_metadata, intermediate_tensors,
306 + inputs_embeds)
307 return hidden_states
308
309 def compute_logits(
vllm/model_executor/models/gpt_j.py
+14 -2
@@ -201,6 +201,9 @@ class GPTJModel(nn.Module):
201 make_empty_intermediate_tensors_factory(["hidden_states"],
202 config.n_embd))
203
204 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
205 + return self.wte(input_ids)
206 +
207 def forward(
208 self,
209 input_ids: torch.Tensor,
@@ -208,9 +211,13 @@ class GPTJModel(nn.Module):
211 kv_caches: List[torch.Tensor],
212 attn_metadata: AttentionMetadata,
213 intermediate_tensors: Optional[IntermediateTensors],
214 + inputs_embeds: Optional[torch.Tensor] = None,
215 ) -> Union[torch.Tensor, IntermediateTensors]:
216 if get_pp_group().is_first_rank:
213 - hidden_states = self.wte(input_ids)
217 + if inputs_embeds is not None:
218 + hidden_states = inputs_embeds
219 + else:
220 + hidden_states = self.get_input_embeddings(input_ids)
221 else:
222 hidden_states = intermediate_tensors["hidden_states"]
223 for i in range(self.start_layer, self.end_layer):
@@ -250,6 +257,9 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
257 self.make_empty_intermediate_tensors = (
258 self.transformer.make_empty_intermediate_tensors)
259
260 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
261 + return self.transformer.get_input_embeddings(input_ids)
262 +
263 def forward(
264 self,
265 input_ids: torch.Tensor,
@@ -257,9 +267,11 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
267 kv_caches: List[torch.Tensor],
268 attn_metadata: AttentionMetadata,
269 intermediate_tensors: Optional[IntermediateTensors] = None,
270 + inputs_embeds: Optional[torch.Tensor] = None,
271 ) -> Union[torch.Tensor, IntermediateTensors]:
272 hidden_states = self.transformer(input_ids, positions, kv_caches,
262 - attn_metadata, intermediate_tensors)
273 + attn_metadata, intermediate_tensors,
274 + inputs_embeds)
275 return hidden_states
276
277 def compute_logits(
vllm/model_executor/models/gpt_neox.py
+14 -2
@@ -214,6 +214,9 @@ class GPTNeoXModel(nn.Module):
214 make_empty_intermediate_tensors_factory(["hidden_states"],
215 config.hidden_size))
216
217 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
218 + return self.embed_in(input_ids)
219 +
220 def forward(
221 self,
222 input_ids: torch.Tensor,
@@ -221,9 +224,13 @@ class GPTNeoXModel(nn.Module):
224 kv_caches: List[torch.Tensor],
225 attn_metadata: AttentionMetadata,
226 intermediate_tensors: Optional[IntermediateTensors],
227 + inputs_embeds: Optional[torch.Tensor] = None,
228 ) -> Union[torch.Tensor, IntermediateTensors]:
229 if get_pp_group().is_first_rank:
226 - hidden_states = self.embed_in(input_ids)
230 + if inputs_embeds is not None:
231 + hidden_states = inputs_embeds
232 + else:
233 + hidden_states = self.get_input_embeddings(input_ids)
234 else:
235 hidden_states = intermediate_tensors["hidden_states"]
236 for i in range(self.start_layer, self.end_layer):
@@ -262,6 +269,9 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
269 self.make_empty_intermediate_tensors = (
270 self.gpt_neox.make_empty_intermediate_tensors)
271
272 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
273 + return self.gpt_neox.get_input_embeddings(input_ids)
274 +
275 def forward(
276 self,
277 input_ids: torch.Tensor,
@@ -269,9 +279,11 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
279 kv_caches: List[torch.Tensor],
280 attn_metadata: AttentionMetadata,
281 intermediate_tensors: Optional[IntermediateTensors] = None,
282 + inputs_embeds: Optional[torch.Tensor] = None,
283 ) -> Union[torch.Tensor, IntermediateTensors]:
284 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
274 - attn_metadata, intermediate_tensors)
285 + attn_metadata, intermediate_tensors,
286 + inputs_embeds)
287 return hidden_states
288
289 def compute_logits(
vllm/model_executor/models/granite.py
+6 -1
@@ -409,6 +409,9 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
409 else:
410 self.lm_head = PPMissingLayer()
411
412 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
413 + return self.model.get_input_embeddings(input_ids)
414 +
415 def forward(
416 self,
417 input_ids: torch.Tensor,
@@ -416,9 +419,11 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419 kv_caches: List[torch.Tensor],
420 attn_metadata: AttentionMetadata,
421 intermediate_tensors: Optional[IntermediateTensors] = None,
422 + inputs_embeds: Optional[torch.Tensor] = None,
423 ) -> Union[torch.Tensor, IntermediateTensors]:
424 model_output = self.model(input_ids, positions, kv_caches,
421 - attn_metadata, intermediate_tensors)
425 + attn_metadata, intermediate_tensors,
426 + inputs_embeds)
427 return model_output
428
429 def compute_logits(
vllm/model_executor/models/granitemoe.py
+14 -2
@@ -277,6 +277,9 @@ class GraniteMoeModel(nn.Module):
277
278 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
279
280 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
281 + return self.embed_tokens(input_ids)
282 +
283 def forward(
284 self,
285 input_ids: torch.Tensor,
@@ -284,9 +287,13 @@ class GraniteMoeModel(nn.Module):
287 kv_caches: List[torch.Tensor],
288 attn_metadata: AttentionMetadata,
289 intermediate_tensors: Optional[IntermediateTensors],
290 + inputs_embeds: Optional[torch.Tensor] = None,
291 ) -> torch.Tensor:
292 if get_pp_group().is_first_rank:
289 - hidden_states = self.embed_tokens(input_ids)
293 + if inputs_embeds is not None:
294 + hidden_states = inputs_embeds
295 + else:
296 + hidden_states = self.get_input_embeddings(input_ids)
297 hidden_states *= self.embedding_multiplier
298 residual = None
299 else:
@@ -366,6 +373,9 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
373
374 self.sampler = get_sampler()
375
376 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
377 + return self.model.get_input_embeddings(input_ids)
378 +
379 def forward(
380 self,
381 input_ids: torch.Tensor,
@@ -373,9 +383,11 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
383 kv_caches: List[torch.Tensor],
384 attn_metadata: AttentionMetadata,
385 intermediate_tensors: Optional[IntermediateTensors] = None,
386 + inputs_embeds: Optional[torch.Tensor] = None,
387 ) -> torch.Tensor:
388 hidden_states = self.model(input_ids, positions, kv_caches,
378 - attn_metadata, intermediate_tensors)
389 + attn_metadata, intermediate_tensors,
390 + inputs_embeds)
391 return hidden_states
392
393 def compute_logits(
vllm/model_executor/models/internlm2.py
+7 -2
@@ -290,7 +290,7 @@ class InternLM2Model(nn.Module):
290 if inputs_embeds is not None:
291 hidden_states = inputs_embeds
292 else:
293 - hidden_states = self.tok_embeddings(input_ids)
293 + hidden_states = self.get_input_embeddings(input_ids)
294 residual = None
295 else:
296 assert intermediate_tensors is not None
@@ -335,6 +335,9 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
335 self.make_empty_intermediate_tensors = (
336 self.model.make_empty_intermediate_tensors)
337
338 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
339 + return self.model.get_input_embeddings(input_ids)
340 +
341 def forward(
342 self,
343 input_ids: torch.Tensor,
@@ -342,9 +345,11 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
345 kv_caches: List[torch.Tensor],
346 attn_metadata: AttentionMetadata,
347 intermediate_tensors: Optional[IntermediateTensors],
348 + inputs_embeds: Optional[torch.Tensor] = None,
349 ) -> torch.Tensor:
350 hidden_states = self.model(input_ids, positions, kv_caches,
347 - attn_metadata, intermediate_tensors)
351 + attn_metadata, intermediate_tensors,
352 + inputs_embeds)
353 return hidden_states
354
355 def compute_logits(
vllm/model_executor/models/jais.py
+12 -2
@@ -250,6 +250,9 @@ class JAISModel(nn.Module):
250 make_empty_intermediate_tensors_factory(["hidden_states"],
251 config.n_embd))
252
253 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
254 + return self.wte(input_ids)
255 +
256 def forward(
257 self,
258 input_ids: torch.Tensor,
@@ -257,9 +260,11 @@ class JAISModel(nn.Module):
260 kv_caches: List[torch.Tensor],
261 attn_metadata: AttentionMetadata,
262 intermediate_tensors: Optional[IntermediateTensors] = None,
263 + inputs_embeds: Optional[torch.Tensor] = None,
264 ) -> Union[IntermediateTensors, torch.Tensor]:
265 if get_pp_group().is_first_rank:
262 - inputs_embeds = self.wte(input_ids)
266 + if inputs_embeds is None:
267 + inputs_embeds = self.get_input_embeddings(input_ids)
268 if self.wpe is not None:
269 position_embeds = self.wpe(position_ids)
270 hidden_states = inputs_embeds + position_embeds
@@ -311,6 +316,9 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
316 self.make_empty_intermediate_tensors = (
317 self.transformer.make_empty_intermediate_tensors)
318
319 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
320 + return self.transformer.get_input_embeddings(input_ids)
321 +
322 def forward(
323 self,
324 input_ids: torch.Tensor,
@@ -318,9 +326,11 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
326 kv_caches: List[torch.Tensor],
327 attn_metadata: AttentionMetadata,
328 intermediate_tensors: Optional[IntermediateTensors] = None,
329 + inputs_embeds: Optional[torch.Tensor] = None,
330 ) -> Union[IntermediateTensors, torch.Tensor]:
331 hidden_states = self.transformer(input_ids, positions, kv_caches,
323 - attn_metadata, intermediate_tensors)
332 + attn_metadata, intermediate_tensors,
333 + inputs_embeds)
334 return hidden_states
335
336 def compute_logits(
vllm/model_executor/models/jamba.py
+14 -2
@@ -292,6 +292,9 @@ class JambaModel(nn.Module):
292 self.final_layernorm = RMSNorm(config.hidden_size,
293 eps=config.rms_norm_eps)
294
295 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
296 + return self.embed_tokens(input_ids)
297 +
298 def forward(
299 self,
300 input_ids: torch.Tensor,
@@ -299,8 +302,12 @@ class JambaModel(nn.Module):
302 kv_caches: List[torch.Tensor],
303 attn_metadata: AttentionMetadata,
304 mamba_cache_params: MambaCacheParams,
305 + inputs_embeds: Optional[torch.Tensor] = None,
306 ) -> torch.Tensor:
303 - hidden_states = self.embed_tokens(input_ids)
307 + if inputs_embeds is not None:
308 + hidden_states = inputs_embeds
309 + else:
310 + hidden_states = self.get_input_embeddings(input_ids)
311 residual = None
312 for i in range(len(self.layers)):
313 layer = self.layers[i]
@@ -381,12 +388,16 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
388 config.vocab_size)
389 self.sampler = get_sampler()
390
391 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
392 + return self.model.get_input_embeddings(input_ids)
393 +
394 def forward(self,
395 input_ids: torch.Tensor,
396 positions: torch.Tensor,
397 kv_caches: List[KVCache],
398 attn_metadata: AttentionMetadata,
399 intermediate_tensors: Optional[IntermediateTensors] = None,
400 + inputs_embeds: Optional[torch.Tensor] = None,
401 **kwargs):
402 if self.mamba_cache is None:
403 max_batch_size = (_get_graph_batch_size(
@@ -409,7 +420,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
420 mamba_cache_tensors[1],
421 state_indices_tensor)
422 hidden_states = self.model(input_ids, positions, kv_caches,
412 - attn_metadata, mamba_cache_params)
423 + attn_metadata, mamba_cache_params,
424 + inputs_embeds)
425 return hidden_states
426
427 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
vllm/model_executor/models/mamba.py
+13 -2
@@ -106,15 +106,22 @@ class MambaModel(nn.Module):
106 self.norm_f = RMSNorm(config.hidden_size,
107 eps=config.layer_norm_epsilon)
108
109 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
110 + return self.embeddings(input_ids)
111 +
112 def forward(
113 self,
114 input_ids: torch.Tensor,
115 positions: torch.Tensor,
116 attn_metadata: AttentionMetadata,
117 mamba_cache_params: MambaCacheParams,
118 + inputs_embeds: Optional[torch.Tensor] = None,
119 ) -> torch.Tensor:
120
117 - hidden_states = self.embeddings(input_ids)
121 + if inputs_embeds is not None:
122 + hidden_states = inputs_embeds
123 + else:
124 + hidden_states = self.get_input_embeddings(input_ids)
125 residual = None
126
127 for i in range(len(self.layers)):
@@ -168,12 +175,16 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
175 config.vocab_size)
176 self.sampler = get_sampler()
177
178 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
179 + return self.backbone.get_input_embeddings(input_ids)
180 +
181 def forward(self,
182 input_ids: torch.Tensor,
183 positions: torch.Tensor,
184 kv_caches: List[KVCache],
185 attn_metadata: AttentionMetadata,
186 intermediate_tensors: Optional[IntermediateTensors] = None,
187 + inputs_embeds: Optional[torch.Tensor] = None,
188 **kwargs):
189 if self.mamba_cache is None:
190 max_batch_size = (_get_graph_batch_size(
@@ -194,7 +205,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
205 state_indices_tensor)
206
207 hidden_states = self.backbone(input_ids, positions, attn_metadata,
197 - mamba_cache_params)
208 + mamba_cache_params, inputs_embeds)
209
210 return hidden_states
211
vllm/model_executor/models/minicpm.py
+6 -1
@@ -504,6 +504,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
504 self.model = MiniCPMModel(vllm_config=vllm_config,
505 prefix=maybe_prefix(prefix, "model"))
506
507 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
508 + return self.model.get_input_embeddings(input_ids)
509 +
510 def forward(
511 self,
512 input_ids: torch.Tensor,
@@ -511,9 +514,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
514 kv_caches: List[torch.Tensor],
515 attn_metadata: AttentionMetadata,
516 intermediate_tensors: Optional[IntermediateTensors] = None,
517 + inputs_embeds: Optional[torch.Tensor] = None,
518 ) -> Union[torch.Tensor, IntermediateTensors]:
519 hidden_states = self.model(input_ids, positions, kv_caches,
516 - attn_metadata, intermediate_tensors)
520 + attn_metadata, intermediate_tensors,
521 + inputs_embeds)
522 return hidden_states
523
524 def compute_logits(
vllm/model_executor/models/mixtral.py
+14 -2
@@ -281,6 +281,9 @@ class MixtralModel(nn.Module):
281 make_empty_intermediate_tensors_factory(
282 ["hidden_states", "residual"], config.hidden_size))
283
284 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
285 + return self.embed_tokens(input_ids)
286 +
287 def forward(
288 self,
289 input_ids: torch.Tensor,
@@ -288,9 +291,13 @@ class MixtralModel(nn.Module):
291 kv_caches: List[torch.Tensor],
292 attn_metadata: AttentionMetadata,
293 intermediate_tensors: Optional[IntermediateTensors],
294 + inputs_embeds: Optional[torch.Tensor] = None,
295 ) -> Union[torch.Tensor, IntermediateTensors]:
296 if get_pp_group().is_first_rank:
293 - hidden_states = self.embed_tokens(input_ids)
297 + if inputs_embeds is not None:
298 + hidden_states = inputs_embeds
299 + else:
300 + hidden_states = self.get_input_embeddings(input_ids)
301 residual = None
302 else:
303 assert intermediate_tensors is not None
@@ -363,6 +370,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370 self.make_empty_intermediate_tensors = (
371 self.model.make_empty_intermediate_tensors)
372
373 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
374 + return self.model.get_input_embeddings(input_ids)
375 +
376 def forward(
377 self,
378 input_ids: torch.Tensor,
@@ -370,9 +380,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
380 kv_caches: List[torch.Tensor],
381 attn_metadata: AttentionMetadata,
382 intermediate_tensors: Optional[IntermediateTensors] = None,
383 + inputs_embeds: Optional[torch.Tensor] = None,
384 ) -> Union[torch.Tensor, IntermediateTensors]:
385 hidden_states = self.model(input_ids, positions, kv_caches,
375 - attn_metadata, intermediate_tensors)
386 + attn_metadata, intermediate_tensors,
387 + inputs_embeds)
388 return hidden_states
389
390 def compute_logits(
vllm/model_executor/models/mixtral_quant.py
+14 -2
@@ -318,6 +318,9 @@ class MixtralModel(nn.Module):
318 make_empty_intermediate_tensors_factory(
319 ["hidden_states", "residual"], config.hidden_size))
320
321 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
322 + return self.embed_tokens(input_ids)
323 +
324 def forward(
325 self,
326 input_ids: torch.Tensor,
@@ -325,9 +328,13 @@ class MixtralModel(nn.Module):
328 kv_caches: List[torch.Tensor],
329 attn_metadata: AttentionMetadata,
330 intermediate_tensors: Optional[IntermediateTensors],
331 + inputs_embeds: Optional[torch.Tensor] = None,
332 ) -> Union[torch.Tensor, IntermediateTensors]:
333 if get_pp_group().is_first_rank:
330 - hidden_states = self.embed_tokens(input_ids)
334 + if inputs_embeds is not None:
335 + hidden_states = inputs_embeds
336 + else:
337 + hidden_states = self.get_input_embeddings(input_ids)
338 residual = None
339 else:
340 assert intermediate_tensors is not None
@@ -368,6 +375,9 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
375 self.make_empty_intermediate_tensors = (
376 self.model.make_empty_intermediate_tensors)
377
378 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
379 + return self.model.get_input_embeddings(input_ids)
380 +
381 def forward(
382 self,
383 input_ids: torch.Tensor,
@@ -375,9 +385,11 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
385 kv_caches: List[torch.Tensor],
386 attn_metadata: AttentionMetadata,
387 intermediate_tensors: Optional[IntermediateTensors] = None,
388 + inputs_embeds: Optional[torch.Tensor] = None,
389 ) -> Union[torch.Tensor, IntermediateTensors]:
390 hidden_states = self.model(input_ids, positions, kv_caches,
380 - attn_metadata, intermediate_tensors)
391 + attn_metadata, intermediate_tensors,
392 + inputs_embeds)
393 return hidden_states
394
395 def compute_logits(
vllm/model_executor/models/mpt.py
+14 -2
@@ -237,6 +237,9 @@ class MPTModel(nn.Module):
237 make_empty_intermediate_tensors_factory(["hidden_states"],
238 config.d_model))
239
240 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241 + return self.wte(input_ids)
242 +
243 def forward(
244 self,
245 input_ids: torch.Tensor,
@@ -244,9 +247,13 @@ class MPTModel(nn.Module):
247 kv_caches: List[torch.Tensor],
248 attn_metadata: AttentionMetadata,
249 intermediate_tensors: Optional[IntermediateTensors],
250 + inputs_embeds: Optional[torch.Tensor] = None,
251 ) -> Union[torch.Tensor, IntermediateTensors]:
252 if get_pp_group().is_first_rank:
249 - hidden_states = self.wte(input_ids)
253 + if inputs_embeds is not None:
254 + hidden_states = inputs_embeds
255 + else:
256 + hidden_states = self.get_input_embeddings(input_ids)
257 else:
258 assert intermediate_tensors is not None
259 hidden_states = intermediate_tensors["hidden_states"]
@@ -283,6 +290,9 @@ class MPTForCausalLM(nn.Module, SupportsPP):
290 self.make_empty_intermediate_tensors = (
291 self.transformer.make_empty_intermediate_tensors)
292
293 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
294 + return self.transformer.get_input_embeddings(input_ids)
295 +
296 def forward(
297 self,
298 input_ids: torch.Tensor,
@@ -290,9 +300,11 @@ class MPTForCausalLM(nn.Module, SupportsPP):
300 kv_caches: List[torch.Tensor],
301 attn_metadata: AttentionMetadata,
302 intermediate_tensors: Optional[IntermediateTensors] = None,
303 + inputs_embeds: Optional[torch.Tensor] = None,
304 ) -> Union[torch.Tensor, IntermediateTensors]:
305 hidden_states = self.transformer(input_ids, positions, kv_caches,
295 - attn_metadata, intermediate_tensors)
306 + attn_metadata, intermediate_tensors,
307 + inputs_embeds)
308 return hidden_states
309
310 def compute_logits(
vllm/model_executor/models/nemotron.py
+6 -1
@@ -440,6 +440,9 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440 self.make_empty_intermediate_tensors = (
441 self.model.make_empty_intermediate_tensors)
442
443 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
444 + return self.model.get_input_embeddings(input_ids)
445 +
446 def forward(
447 self,
448 input_ids: torch.Tensor,
@@ -447,9 +450,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
450 kv_caches: List[torch.Tensor],
451 attn_metadata: AttentionMetadata,
452 intermediate_tensors: Optional[IntermediateTensors] = None,
453 + inputs_embeds: Optional[torch.Tensor] = None,
454 ) -> Union[torch.Tensor, IntermediateTensors]:
455 model_output = self.model(input_ids, positions, kv_caches,
452 - attn_metadata, intermediate_tensors)
456 + attn_metadata, intermediate_tensors,
457 + inputs_embeds)
458 return model_output
459
460 def compute_logits(
vllm/model_executor/models/olmo.py
+13 -6
@@ -248,6 +248,9 @@ class OlmoModel(nn.Module):
248 make_empty_intermediate_tensors_factory(["hidden_states"],
249 config.hidden_size))
250
251 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
252 + return self.embed_tokens(input_ids)
253 +
254 def forward(
255 self,
256 input_ids: torch.Tensor,
@@ -255,17 +258,16 @@ class OlmoModel(nn.Module):
258 kv_caches: List[torch.Tensor],
259 attn_metadata: AttentionMetadata,
260 intermediate_tensors: Optional[IntermediateTensors],
261 + inputs_embeds: Optional[torch.Tensor] = None,
262 ) -> Union[torch.Tensor, IntermediateTensors]:
263 """
264 :param input_ids: A tensor of shape `(batch_size, seq_len)`.
265 """
266 if get_pp_group().is_first_rank:
263 - # Get embeddings of input.
264 - # shape: (batch_size, seq_len, d_model)
265 - inputs_embeds = self.embed_tokens(input_ids)
266 -
267 - # embed positions
268 - hidden_states = inputs_embeds
267 + if inputs_embeds is not None:
268 + hidden_states = inputs_embeds
269 + else:
270 + hidden_states = self.get_input_embeddings(input_ids)
271 else:
272 assert intermediate_tensors is not None
273 hidden_states = intermediate_tensors["hidden_states"]
@@ -315,6 +317,9 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
317 self.make_empty_intermediate_tensors = (
318 self.model.make_empty_intermediate_tensors)
319
320 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
321 + return self.model.get_input_embeddings(input_ids)
322 +
323 def forward(
324 self,
325 input_ids: torch.Tensor,
@@ -322,6 +327,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
327 kv_caches: List[torch.Tensor],
328 attn_metadata: AttentionMetadata,
329 intermediate_tensors: Optional[IntermediateTensors] = None,
330 + inputs_embeds: Optional[torch.Tensor] = None,
331 ) -> Union[torch.Tensor, IntermediateTensors]:
332 hidden_states = self.model(
333 input_ids=input_ids,
@@ -329,6 +335,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
335 kv_caches=kv_caches,
336 attn_metadata=attn_metadata,
337 intermediate_tensors=intermediate_tensors,
338 + inputs_embeds=inputs_embeds,
339 )
340 return hidden_states
341
vllm/model_executor/models/olmoe.py
+14 -2
@@ -269,6 +269,9 @@ class OlmoeModel(nn.Module):
269 make_empty_intermediate_tensors_factory(
270 ["hidden_states", "residual"], config.hidden_size))
271
272 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
273 + return self.embed_tokens(input_ids)
274 +
275 def forward(
276 self,
277 input_ids: torch.Tensor,
@@ -276,9 +279,13 @@ class OlmoeModel(nn.Module):
279 kv_caches: List[torch.Tensor],
280 attn_metadata: AttentionMetadata,
281 intermediate_tensors: Optional[IntermediateTensors],
282 + inputs_embeds: Optional[torch.Tensor] = None,
283 ) -> Union[torch.Tensor, IntermediateTensors]:
284 if get_pp_group().is_first_rank:
281 - hidden_states = self.embed_tokens(input_ids)
285 + if inputs_embeds is not None:
286 + hidden_states = inputs_embeds
287 + else:
288 + hidden_states = self.get_input_embeddings(input_ids)
289 residual = None
290 else:
291 assert intermediate_tensors is not None
@@ -326,6 +333,9 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
333 self.make_empty_intermediate_tensors = (
334 self.model.make_empty_intermediate_tensors)
335
336 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
337 + return self.model.get_input_embeddings(input_ids)
338 +
339 def forward(
340 self,
341 input_ids: torch.Tensor,
@@ -333,9 +343,11 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
343 kv_caches: List[torch.Tensor],
344 attn_metadata: AttentionMetadata,
345 intermediate_tensors: Optional[IntermediateTensors] = None,
346 + inputs_embeds: Optional[torch.Tensor] = None,
347 ) -> Union[torch.Tensor, IntermediateTensors]:
348 hidden_states = self.model(input_ids, positions, kv_caches,
338 - attn_metadata, intermediate_tensors)
349 + attn_metadata, intermediate_tensors,
350 + inputs_embeds)
351 return hidden_states
352
353 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/orion.py
+14 -2
@@ -237,6 +237,9 @@ class OrionModel(nn.Module):
237 "hidden_states",
238 ], config.hidden_size))
239
240 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241 + return self.embed_tokens(input_ids)
242 +
243 def forward(
244 self,
245 input_ids: torch.Tensor,
@@ -244,9 +247,13 @@ class OrionModel(nn.Module):
247 kv_caches: List[torch.Tensor],
248 attn_metadata: AttentionMetadata,
249 intermediate_tensors: Optional[IntermediateTensors],
250 + inputs_embeds: Optional[torch.Tensor] = None,
251 ) -> Union[torch.Tensor, IntermediateTensors]:
252 if get_pp_group().is_first_rank:
249 - hidden_states = self.embed_tokens(input_ids)
253 + if inputs_embeds is not None:
254 + hidden_states = inputs_embeds
255 + else:
256 + hidden_states = self.get_input_embeddings(input_ids)
257 else:
258 assert intermediate_tensors is not None
259 hidden_states = intermediate_tensors["hidden_states"]
@@ -286,6 +293,9 @@ class OrionForCausalLM(nn.Module, SupportsPP):
293 self.make_empty_intermediate_tensors = (
294 self.model.make_empty_intermediate_tensors)
295
296 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
297 + return self.model.get_input_embeddings(input_ids)
298 +
299 def forward(
300 self,
301 input_ids: torch.Tensor,
@@ -293,9 +303,11 @@ class OrionForCausalLM(nn.Module, SupportsPP):
303 kv_caches: List[torch.Tensor],
304 attn_metadata: AttentionMetadata,
305 intermediate_tensors: Optional[IntermediateTensors] = None,
306 + inputs_embeds: Optional[torch.Tensor] = None,
307 ) -> Union[torch.Tensor, IntermediateTensors]:
308 hidden_states = self.model(input_ids, positions, kv_caches,
298 - attn_metadata, intermediate_tensors)
309 + attn_metadata, intermediate_tensors,
310 + inputs_embeds)
311 return hidden_states
312
313 def compute_logits(
vllm/model_executor/models/persimmon.py
+7 -1
@@ -235,6 +235,9 @@ class PersimmonModel(nn.Module):
235 make_empty_intermediate_tensors_factory(["hidden_states"],
236 config.hidden_size))
237
238 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
239 + return self.embed_tokens(input_ids)
240 +
241 def forward(
242 self,
243 input_ids: torch.Tensor,
@@ -248,7 +251,7 @@ class PersimmonModel(nn.Module):
251 if inputs_embeds is not None:
252 hidden_states = inputs_embeds
253 else:
251 - hidden_states = self.embed_tokens(input_ids)
254 + hidden_states = self.get_input_embeddings(input_ids)
255 else:
256 assert intermediate_tensors is not None
257 hidden_states = intermediate_tensors["hidden_states"]
@@ -282,6 +285,9 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
285 self.make_empty_intermediate_tensors = (
286 self.model.make_empty_intermediate_tensors)
287
288 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
289 + return self.model.get_input_embeddings(input_ids)
290 +
291 def forward(
292 self,
293 input_ids: torch.Tensor,
vllm/model_executor/models/phi.py
+14 -2
@@ -218,6 +218,9 @@ class PhiModel(nn.Module):
218 make_empty_intermediate_tensors_factory(["hidden_states"],
219 config.hidden_size))
220
221 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222 + return self.embed_tokens(input_ids)
223 +
224 def forward(
225 self,
226 input_ids: torch.Tensor,
@@ -225,9 +228,13 @@ class PhiModel(nn.Module):
228 kv_caches: List[torch.Tensor],
229 attn_metadata: AttentionMetadata,
230 intermediate_tensors: Optional[IntermediateTensors],
231 + inputs_embeds: Optional[torch.Tensor] = None,
232 ) -> Union[torch.Tensor, IntermediateTensors]:
233 if get_pp_group().is_first_rank:
230 - hidden_states = self.embed_tokens(input_ids)
234 + if inputs_embeds is not None:
235 + hidden_states = inputs_embeds
236 + else:
237 + hidden_states = self.get_input_embeddings(input_ids)
238 else:
239 assert intermediate_tensors is not None
240 hidden_states = intermediate_tensors["hidden_states"]
@@ -303,6 +310,9 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
310 self.make_empty_intermediate_tensors = (
311 self.model.make_empty_intermediate_tensors)
312
313 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
314 + return self.model.get_input_embeddings(input_ids)
315 +
316 def forward(
317 self,
318 input_ids: torch.Tensor,
@@ -310,9 +320,11 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
320 kv_caches: List[torch.Tensor],
321 attn_metadata: AttentionMetadata,
322 intermediate_tensors: Optional[IntermediateTensors] = None,
323 + inputs_embeds: Optional[torch.Tensor] = None,
324 ) -> Union[torch.Tensor, IntermediateTensors]:
325 hidden_states = self.model(input_ids, positions, kv_caches,
315 - attn_metadata, intermediate_tensors)
326 + attn_metadata, intermediate_tensors,
327 + inputs_embeds)
328
329 return hidden_states
330
vllm/model_executor/models/phi3_small.py
+11 -8
@@ -324,11 +324,8 @@ class Phi3SmallModel(nn.Module):
324 make_empty_intermediate_tensors_factory(["hidden_states"],
325 config.hidden_size))
326
327 - def get_input_embeddings(self):
328 - return self.embed_tokens
329 -
330 - def set_input_embeddings(self, value):
331 - self.embed_tokens = value
327 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
328 + return self.embed_tokens(input_ids)
329
330 def forward(
331 self,
@@ -337,9 +334,13 @@ class Phi3SmallModel(nn.Module):
334 kv_caches: List[torch.Tensor],
335 attn_metadata: AttentionMetadata,
336 intermediate_tensors: Optional[IntermediateTensors],
337 + inputs_embeds: Optional[torch.Tensor],
338 ) -> Union[torch.Tensor, IntermediateTensors]:
339 if get_pp_group().is_first_rank:
342 - hidden_states = self.embed_tokens(input_ids)
340 + if inputs_embeds is not None:
341 + hidden_states = inputs_embeds
342 + else:
343 + hidden_states = self.get_input_embeddings(input_ids)
344 if (self.mup_embedding_multiplier is not None
345 and self.mup_embedding_multiplier > 0.0):
346 hidden_states = hidden_states * self.mup_embedding_multiplier
@@ -397,8 +398,8 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
398 else:
399 self.dummy_token_indices = None
400
400 - def get_input_embeddings(self):
401 - return self.model.embed_tokens
401 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
402 + return self.model.get_input_embeddings(input_ids)
403
404 def set_input_embeddings(self, value):
405 self.model.embed_tokens = value
@@ -433,6 +434,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
434 kv_caches: List[torch.Tensor],
435 attn_metadata: AttentionMetadata,
436 intermediate_tensors: Optional[IntermediateTensors] = None,
437 + inputs_embeds: Optional[torch.Tensor] = None,
438 ) -> Union[torch.Tensor, IntermediateTensors]:
439 output_hidden_states = self.model(
440 input_ids=input_ids,
@@ -440,6 +442,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
442 kv_caches=kv_caches,
443 attn_metadata=attn_metadata,
444 intermediate_tensors=intermediate_tensors,
445 + inputs_embeds=inputs_embeds,
446 )
447 output_hidden_states = output_hidden_states
448 return output_hidden_states
vllm/model_executor/models/phimoe.py
+14 -2
@@ -465,6 +465,9 @@ class PhiMoEModel(nn.Module):
465 make_empty_intermediate_tensors_factory(
466 ["hidden_states", "residual"], config.hidden_size))
467
468 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
469 + return self.embed_tokens(input_ids)
470 +
471 def forward(
472 self,
473 input_ids: torch.Tensor,
@@ -472,9 +475,13 @@ class PhiMoEModel(nn.Module):
475 kv_caches: List[torch.Tensor],
476 attn_metadata: AttentionMetadata,
477 intermediate_tensors: Optional[IntermediateTensors],
478 + inputs_embeds: Optional[torch.Tensor] = None,
479 ) -> Union[torch.Tensor, IntermediateTensors]:
480 if get_pp_group().is_first_rank:
477 - hidden_states = self.embed_tokens(input_ids)
481 + if inputs_embeds is not None:
482 + hidden_states = inputs_embeds
483 + else:
484 + hidden_states = self.get_input_embeddings(input_ids)
485 residual = None
486 else:
487 assert intermediate_tensors is not None
@@ -560,6 +567,9 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
567 self.make_empty_intermediate_tensors = (
568 self.model.make_empty_intermediate_tensors)
569
570 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
571 + return self.model.get_input_embeddings(input_ids)
572 +
573 def forward(
574 self,
575 input_ids: torch.Tensor,
@@ -567,9 +577,11 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
577 kv_caches: List[torch.Tensor],
578 attn_metadata: AttentionMetadata,
579 intermediate_tensors: Optional[IntermediateTensors] = None,
580 + inputs_embeds: Optional[torch.Tensor] = None,
581 ) -> Union[torch.Tensor, IntermediateTensors]:
582 hidden_states = self.model(input_ids, positions, kv_caches,
572 - attn_metadata, intermediate_tensors)
583 + attn_metadata, intermediate_tensors,
584 + inputs_embeds)
585 return hidden_states
586
587 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen.py
+14 -2
@@ -578,6 +578,9 @@ class QWenModel(nn.Module):
578 quant_config=quant_config) if hasattr(
579 config, "visual") else None
580
581 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
582 + return self.wte(input_ids)
583 +
584 def forward(
585 self,
586 input_ids: torch.Tensor,
@@ -586,6 +589,7 @@ class QWenModel(nn.Module):
589 attn_metadata: AttentionMetadata,
590 intermediate_tensors: Optional[IntermediateTensors],
591 pixel_values: Optional[QwenImageInputs],
592 + inputs_embeds: Optional[torch.Tensor] = None,
593 ) -> Union[torch.Tensor, IntermediateTensors]:
594 img_pos = None
595 # If pixel / visual embeddings are provided, this is a visual model
@@ -606,6 +610,10 @@ class QWenModel(nn.Module):
610 )
611
612 if get_pp_group().is_first_rank:
613 + if inputs_embeds is not None:
614 + hidden_states = inputs_embeds
615 + else:
616 + hidden_states = self.get_input_embeddings(input_ids)
617 hidden_states = self.wte(input_ids)
618 # Merge the image embeddings into the hidden states if actually have
619 # visual features and the corresponding image tokens
@@ -915,6 +923,9 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
923 )
924 return None
925
926 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
927 + return self.transformer.get_input_embeddings(input_ids)
928 +
929 def forward(
930 self,
931 input_ids: torch.Tensor,
@@ -922,7 +933,8 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
933 kv_caches: List[torch.Tensor],
934 attn_metadata: AttentionMetadata,
935 intermediate_tensors: Optional[IntermediateTensors] = None,
925 - pixel_values: Optional[torch.Tensor] = None
936 + pixel_values: Optional[torch.Tensor] = None,
937 + inputs_embeds: Optional[torch.Tensor] = None,
938 ) -> Union[torch.Tensor, IntermediateTensors]:
939 if intermediate_tensors is not None:
940 input_ids = None
@@ -932,7 +944,7 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
944
945 hidden_states = self.transformer(input_ids, positions, kv_caches,
946 attn_metadata, intermediate_tensors,
935 - pixel_values)
947 + pixel_values, inputs_embeds)
948 return hidden_states
949
950 def compute_logits(
vllm/model_executor/models/qwen2.py
+1 -1
@@ -309,7 +309,7 @@ class Qwen2Model(nn.Module):
309 if inputs_embeds is not None:
310 hidden_states = inputs_embeds
311 else:
312 - hidden_states = self.embed_tokens(input_ids)
312 + hidden_states = self.get_input_embeddings(input_ids)
313 residual = None
314 else:
315 assert intermediate_tensors is not None
vllm/model_executor/models/qwen2_cls.py
+6 -1
@@ -72,6 +72,9 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
72 normalize=False,
73 softmax=True)
74
75 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
76 + return self.model.get_input_embeddings(input_ids)
77 +
78 def forward(
79 self,
80 input_ids: torch.Tensor,
@@ -79,9 +82,11 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
82 kv_caches: List[torch.Tensor],
83 attn_metadata: AttentionMetadata,
84 intermediate_tensors: Optional[IntermediateTensors] = None,
85 + inputs_embeds: Optional[torch.Tensor] = None,
86 ) -> torch.Tensor:
87 hidden_states = self.model(input_ids, positions, kv_caches,
84 - attn_metadata, intermediate_tensors)
88 + attn_metadata, intermediate_tensors,
89 + inputs_embeds)
90 logits, _ = self.score(hidden_states)
91 return logits
92
vllm/model_executor/models/qwen2_moe.py
+14 -2
@@ -344,6 +344,9 @@ class Qwen2MoeModel(nn.Module):
344 make_empty_intermediate_tensors_factory(
345 ["hidden_states", "residual"], config.hidden_size))
346
347 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
348 + return self.embed_tokens(input_ids)
349 +
350 def forward(
351 self,
352 input_ids: torch.Tensor,
@@ -351,9 +354,13 @@ class Qwen2MoeModel(nn.Module):
354 kv_caches: List[torch.Tensor],
355 attn_metadata: AttentionMetadata,
356 intermediate_tensors: Optional[IntermediateTensors] = None,
357 + inputs_embeds: Optional[torch.Tensor] = None,
358 ) -> Union[torch.Tensor, IntermediateTensors]:
359 if get_pp_group().is_first_rank:
356 - hidden_states = self.embed_tokens(input_ids)
360 + if inputs_embeds is not None:
361 + hidden_states = inputs_embeds
362 + else:
363 + hidden_states = self.get_input_embeddings(input_ids)
364 residual = None
365 else:
366 assert intermediate_tensors is not None
@@ -395,6 +402,9 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
402 self.make_empty_intermediate_tensors = (
403 self.model.make_empty_intermediate_tensors)
404
405 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
406 + return self.model.get_input_embeddings(input_ids)
407 +
408 def forward(
409 self,
410 input_ids: torch.Tensor,
@@ -402,9 +412,11 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
412 kv_caches: List[torch.Tensor],
413 attn_metadata: AttentionMetadata,
414 intermediate_tensors: Optional[IntermediateTensors] = None,
415 + inputs_embeds: Optional[torch.Tensor] = None,
416 ) -> Union[torch.Tensor, IntermediateTensors]:
417 hidden_states = self.model(input_ids, positions, kv_caches,
407 - attn_metadata, intermediate_tensors)
418 + attn_metadata, intermediate_tensors,
419 + inputs_embeds)
420 return hidden_states
421
422 def compute_logits(
vllm/model_executor/models/qwen2_rm.py
+6 -1
@@ -85,6 +85,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
85 self.make_empty_intermediate_tensors = (
86 self.model.make_empty_intermediate_tensors)
87
88 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
89 + return self.model.get_input_embeddings(input_ids)
90 +
91 def forward(
92 self,
93 input_ids: torch.Tensor,
@@ -92,9 +95,11 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
95 kv_caches: List[torch.Tensor],
96 attn_metadata: AttentionMetadata,
97 intermediate_tensors: Optional[IntermediateTensors] = None,
98 + inputs_embeds: Optional[torch.Tensor] = None,
99 ) -> Union[torch.Tensor, IntermediateTensors]:
100 hidden_states = self.model(input_ids, positions, kv_caches,
97 - attn_metadata, intermediate_tensors)
101 + attn_metadata, intermediate_tensors,
102 + inputs_embeds)
103 logits, _ = self.score(hidden_states)
104 return logits
105
vllm/model_executor/models/solar.py
+3 -1
@@ -456,9 +456,11 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
456 kv_caches: List[torch.Tensor],
457 attn_metadata: AttentionMetadata,
458 intermediate_tensors: Optional[IntermediateTensors] = None,
459 + inputs_embeds: Optional[torch.Tensor] = None,
460 ) -> Union[torch.Tensor, IntermediateTensors]:
461 model_output = self.model(input_ids, positions, kv_caches,
461 - attn_metadata, intermediate_tensors)
462 + attn_metadata, intermediate_tensors,
463 + inputs_embeds)
464 return model_output
465
466 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/stablelm.py
+14 -2
@@ -218,6 +218,9 @@ class StableLMEpochModel(nn.Module):
218 make_empty_intermediate_tensors_factory(["hidden_states"],
219 config.hidden_size))
220
221 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222 + return self.embed_tokens(input_ids)
223 +
224 def forward(
225 self,
226 input_ids: torch.Tensor,
@@ -225,9 +228,13 @@ class StableLMEpochModel(nn.Module):
228 kv_caches: List[torch.Tensor],
229 attn_metadata: AttentionMetadata,
230 intermediate_tensors: Optional[IntermediateTensors],
231 + inputs_embeds: Optional[torch.Tensor] = None,
232 ) -> Union[torch.Tensor, IntermediateTensors]:
233 if get_pp_group().is_first_rank:
230 - hidden_states = self.embed_tokens(input_ids)
234 + if inputs_embeds is not None:
235 + hidden_states = inputs_embeds
236 + else:
237 + hidden_states = self.get_input_embeddings(input_ids)
238 else:
239 assert intermediate_tensors is not None
240 hidden_states = intermediate_tensors["hidden_states"]
@@ -265,6 +272,9 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
272 self.make_empty_intermediate_tensors = (
273 self.model.make_empty_intermediate_tensors)
274
275 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
276 + return self.model.get_input_embeddings(input_ids)
277 +
278 def forward(
279 self,
280 input_ids: torch.Tensor,
@@ -272,9 +282,11 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
282 kv_caches: List[torch.Tensor],
283 attn_metadata: AttentionMetadata,
284 intermediate_tensors: Optional[IntermediateTensors] = None,
285 + inputs_embeds: Optional[torch.Tensor] = None,
286 ) -> Union[torch.Tensor, IntermediateTensors]:
287 hidden_states = self.model(input_ids, positions, kv_caches,
277 - attn_metadata, intermediate_tensors)
288 + attn_metadata, intermediate_tensors,
289 + inputs_embeds)
290 return hidden_states
291
292 def compute_logits(
vllm/model_executor/models/starcoder2.py
+14 -2
@@ -221,6 +221,9 @@ class Starcoder2Model(nn.Module):
221 make_empty_intermediate_tensors_factory(["hidden_states"],
222 config.hidden_size))
223
224 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
225 + return self.embed_tokens(input_ids)
226 +
227 def forward(
228 self,
229 input_ids: torch.Tensor,
@@ -228,9 +231,13 @@ class Starcoder2Model(nn.Module):
231 kv_caches: List[torch.Tensor],
232 attn_metadata: AttentionMetadata,
233 intermediate_tensors: Optional[IntermediateTensors],
234 + inputs_embeds: Optional[torch.Tensor] = None,
235 ) -> Union[torch.Tensor, IntermediateTensors]:
236 if get_pp_group().is_first_rank:
233 - hidden_states = self.embed_tokens(input_ids)
237 + if inputs_embeds is not None:
238 + hidden_states = inputs_embeds
239 + else:
240 + hidden_states = self.get_input_embeddings(input_ids)
241 else:
242 assert intermediate_tensors is not None
243 hidden_states = intermediate_tensors["hidden_states"]
@@ -273,6 +280,9 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
280 self.make_empty_intermediate_tensors = (
281 self.model.make_empty_intermediate_tensors)
282
283 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
284 + return self.model.get_input_embeddings(input_ids)
285 +
286 def forward(
287 self,
288 input_ids: torch.Tensor,
@@ -280,9 +290,11 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
290 kv_caches: List[torch.Tensor],
291 attn_metadata: AttentionMetadata,
292 intermediate_tensors: Optional[IntermediateTensors] = None,
293 + inputs_embeds: Optional[torch.Tensor] = None,
294 ) -> Union[torch.Tensor, IntermediateTensors]:
295 hidden_states = self.model(input_ids, positions, kv_caches,
285 - attn_metadata, intermediate_tensors)
296 + attn_metadata, intermediate_tensors,
297 + inputs_embeds)
298 return hidden_states
299
300 def compute_logits(
vllm/model_executor/models/xverse.py
+14 -2
@@ -252,6 +252,9 @@ class XverseModel(nn.Module):
252 make_empty_intermediate_tensors_factory(
253 ["hidden_states", "residual"], config.hidden_size))
254
255 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
256 + return self.embed_tokens(input_ids)
257 +
258 def forward(
259 self,
260 input_ids: torch.Tensor,
@@ -259,9 +262,13 @@ class XverseModel(nn.Module):
262 kv_caches: List[torch.Tensor],
263 attn_metadata: AttentionMetadata,
264 intermediate_tensors: Optional[IntermediateTensors],
265 + inputs_embeds: Optional[torch.Tensor] = None,
266 ) -> Union[torch.Tensor, IntermediateTensors]:
267 if get_pp_group().is_first_rank:
264 - hidden_states = self.embed_tokens(input_ids)
268 + if inputs_embeds is not None:
269 + hidden_states = inputs_embeds
270 + else:
271 + hidden_states = self.get_input_embeddings(input_ids)
272 residual = None
273 else:
274 hidden_states = intermediate_tensors["hidden_states"]
@@ -335,6 +342,9 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
342 self.make_empty_intermediate_tensors = (
343 self.model.make_empty_intermediate_tensors)
344
345 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
346 + return self.model.get_input_embeddings(input_ids)
347 +
348 def forward(
349 self,
350 input_ids: torch.Tensor,
@@ -342,9 +352,11 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
352 kv_caches: List[torch.Tensor],
353 attn_metadata: AttentionMetadata,
354 intermediate_tensors: Optional[IntermediateTensors] = None,
355 + inputs_embeds: Optional[torch.Tensor] = None,
356 ) -> Union[torch.Tensor, IntermediateTensors]:
357 hidden_states = self.model(input_ids, positions, kv_caches,
347 - attn_metadata, intermediate_tensors)
358 + attn_metadata, intermediate_tensors,
359 + inputs_embeds)
360 return hidden_states
361
362 def compute_logits(