A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Refactor model executable interface for all text-only language models (#10374)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 16, 2024 at 21:18 UTC
643ecf7b11a3e74c838f438cfc1b3e59c018853b
43 files changed
+483
-90
vllm/model_executor/models/arctic.py
+14
-2
@@ -389,6 +389,9 @@ class ArcticModel(nn.Module):
389
make_empty_intermediate_tensors_factory(["hidden_states"],
390
config.hidden_size))
391
392
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
393
+ return self.embed_tokens(input_ids)
394
+
395
def forward(
396
self,
397
input_ids: torch.Tensor,
@@ -396,9 +399,13 @@ class ArcticModel(nn.Module):
399
kv_caches: List[torch.Tensor],
400
attn_metadata: AttentionMetadata,
401
intermediate_tensors: Optional[IntermediateTensors],
402
+ inputs_embeds: Optional[torch.Tensor] = None,
403
) -> Union[torch.Tensor, IntermediateTensors]:
404
if get_pp_group().is_first_rank:
401
- hidden_states = self.embed_tokens(input_ids)
405
+ if inputs_embeds is not None:
406
+ hidden_states = inputs_embeds
407
+ else:
408
+ hidden_states = self.get_input_embeddings(input_ids)
409
else:
410
assert intermediate_tensors is not None
411
hidden_states = intermediate_tensors["hidden_states"]
@@ -439,6 +446,9 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
446
self.make_empty_intermediate_tensors = (
447
self.model.make_empty_intermediate_tensors)
448
449
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
450
+ return self.model.get_input_embeddings(input_ids)
451
+
452
def forward(
453
self,
454
input_ids: torch.Tensor,
@@ -446,9 +456,11 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
456
kv_caches: List[torch.Tensor],
457
attn_metadata: AttentionMetadata,
458
intermediate_tensors: Optional[IntermediateTensors] = None,
459
+ inputs_embeds: Optional[torch.Tensor] = None,
460
) -> Union[torch.Tensor, IntermediateTensors]:
461
hidden_states = self.model(input_ids, positions, kv_caches,
451
- attn_metadata, intermediate_tensors)
462
+ attn_metadata, intermediate_tensors,
463
+ inputs_embeds)
464
return hidden_states
465
466
def compute_logits(
vllm/model_executor/models/baichuan.py
+14
-2
@@ -284,6 +284,9 @@ class BaiChuanModel(nn.Module):
284
make_empty_intermediate_tensors_factory(
285
["hidden_states", "residual"], config.hidden_size))
286
287
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
288
+ return self.embed_tokens(input_ids)
289
+
290
def forward(
291
self,
292
input_ids: torch.Tensor,
@@ -291,9 +294,13 @@ class BaiChuanModel(nn.Module):
294
kv_caches: List[torch.Tensor],
295
attn_metadata: AttentionMetadata,
296
intermediate_tensors: Optional[IntermediateTensors],
297
+ inputs_embeds: Optional[torch.Tensor] = None,
298
) -> Union[torch.Tensor, IntermediateTensors]:
299
if get_pp_group().is_first_rank:
296
- hidden_states = self.embed_tokens(input_ids)
300
+ if inputs_embeds is not None:
301
+ hidden_states = inputs_embeds
302
+ else:
303
+ hidden_states = self.get_input_embeddings(input_ids)
304
residual = None
305
else:
306
assert intermediate_tensors is not None
@@ -363,6 +370,9 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370
self.make_empty_intermediate_tensors = (
371
self.model.make_empty_intermediate_tensors)
372
373
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
374
+ return self.model.get_input_embeddings(input_ids)
375
+
376
def forward(
377
self,
378
input_ids: torch.Tensor,
@@ -370,9 +380,11 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
380
kv_caches: List[torch.Tensor],
381
attn_metadata: AttentionMetadata,
382
intermediate_tensors: Optional[IntermediateTensors] = None,
383
+ inputs_embeds: Optional[torch.Tensor] = None,
384
) -> Union[torch.Tensor, IntermediateTensors]:
385
hidden_states = self.model(input_ids, positions, kv_caches,
375
- attn_metadata, intermediate_tensors)
386
+ attn_metadata, intermediate_tensors,
387
+ inputs_embeds)
388
return hidden_states
389
390
def compute_logits(
vllm/model_executor/models/bloom.py
+14
-3
@@ -251,6 +251,9 @@ class BloomModel(nn.Module):
251
make_empty_intermediate_tensors_factory(["hidden_states"],
252
config.hidden_size))
253
254
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
255
+ return self.word_embeddings_layernorm(self.word_embeddings(input_ids))
256
+
257
def forward(
258
self,
259
input_ids: torch.Tensor,
@@ -258,10 +261,13 @@ class BloomModel(nn.Module):
261
kv_caches: List[torch.Tensor],
262
attn_metadata: AttentionMetadata,
263
intermediate_tensors: Optional[IntermediateTensors],
264
+ inputs_embeds: Optional[torch.Tensor] = None,
265
) -> Union[torch.Tensor, IntermediateTensors]:
266
if get_pp_group().is_first_rank:
263
- hidden_states = self.word_embeddings(input_ids)
264
- hidden_states = self.word_embeddings_layernorm(hidden_states)
267
+ if inputs_embeds is not None:
268
+ hidden_states = inputs_embeds
269
+ else:
270
+ hidden_states = self.get_input_embeddings(input_ids)
271
else:
272
assert intermediate_tensors is not None
273
hidden_states = intermediate_tensors["hidden_states"]
@@ -301,6 +307,9 @@ class BloomForCausalLM(nn.Module, SupportsPP):
307
self.make_empty_intermediate_tensors = (
308
self.transformer.make_empty_intermediate_tensors)
309
310
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
311
+ return self.transformer.get_input_embeddings(input_ids)
312
+
313
def forward(
314
self,
315
input_ids: torch.Tensor,
@@ -308,9 +317,11 @@ class BloomForCausalLM(nn.Module, SupportsPP):
317
kv_caches: List[torch.Tensor],
318
attn_metadata: AttentionMetadata,
319
intermediate_tensors: Optional[IntermediateTensors] = None,
320
+ inputs_embeds: Optional[torch.Tensor] = None,
321
) -> Union[torch.Tensor, IntermediateTensors]:
322
hidden_states = self.transformer(input_ids, positions, kv_caches,
313
- attn_metadata, intermediate_tensors)
323
+ attn_metadata, intermediate_tensors,
324
+ inputs_embeds)
325
return hidden_states
326
327
def compute_logits(
vllm/model_executor/models/commandr.py
+14
-2
@@ -280,6 +280,9 @@ class CohereModel(nn.Module):
280
make_empty_intermediate_tensors_factory(
281
["hidden_states", "residual"], config.hidden_size))
282
283
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
284
+ return self.embed_tokens(input_ids)
285
+
286
def forward(
287
self,
288
input_ids: torch.Tensor,
@@ -287,9 +290,13 @@ class CohereModel(nn.Module):
290
kv_caches: List[torch.Tensor],
291
attn_metadata: AttentionMetadata,
292
intermediate_tensors: Optional[IntermediateTensors],
293
+ inputs_embeds: Optional[torch.Tensor] = None,
294
) -> Union[torch.Tensor, IntermediateTensors]:
295
if get_pp_group().is_first_rank:
292
- hidden_states = self.embed_tokens(input_ids)
296
+ if inputs_embeds is not None:
297
+ hidden_states = inputs_embeds
298
+ else:
299
+ hidden_states = self.get_input_embeddings(input_ids)
300
residual = None
301
else:
302
assert intermediate_tensors is not None
@@ -354,6 +361,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
361
self.make_empty_intermediate_tensors = (
362
self.model.make_empty_intermediate_tensors)
363
364
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
365
+ return self.model.get_input_embeddings(input_ids)
366
+
367
@torch.no_grad()
368
def forward(
369
self,
@@ -362,9 +372,11 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
372
kv_caches: List[torch.Tensor],
373
attn_metadata: AttentionMetadata,
374
intermediate_tensors: Optional[IntermediateTensors] = None,
375
+ inputs_embeds: Optional[torch.Tensor] = None,
376
) -> Union[torch.Tensor, IntermediateTensors]:
377
hidden_states = self.model(input_ids, positions, kv_caches,
367
- attn_metadata, intermediate_tensors)
378
+ attn_metadata, intermediate_tensors,
379
+ inputs_embeds)
380
return hidden_states
381
382
def compute_logits(
vllm/model_executor/models/dbrx.py
+14
-2
@@ -321,6 +321,9 @@ class DbrxModel(nn.Module):
321
make_empty_intermediate_tensors_factory(["hidden_states"],
322
config.d_model))
323
324
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
325
+ return self.wte(input_ids)
326
+
327
def forward(
328
self,
329
input_ids: torch.Tensor,
@@ -328,9 +331,13 @@ class DbrxModel(nn.Module):
331
kv_caches: List[torch.Tensor],
332
attn_metadata: AttentionMetadata,
333
intermediate_tensors: Optional[IntermediateTensors],
334
+ inputs_embeds: Optional[torch.Tensor] = None,
335
) -> Union[torch.Tensor, IntermediateTensors]:
336
if get_pp_group().is_first_rank:
333
- hidden_states = self.wte(input_ids)
337
+ if inputs_embeds is not None:
338
+ hidden_states = inputs_embeds
339
+ else:
340
+ hidden_states = self.get_input_embeddings(input_ids)
341
else:
342
assert intermediate_tensors
343
hidden_states = intermediate_tensors["hidden_states"]
@@ -376,6 +383,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
383
self.make_empty_intermediate_tensors = (
384
self.transformer.make_empty_intermediate_tensors)
385
386
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
387
+ return self.transformer.get_input_embeddings(input_ids)
388
+
389
def forward(
390
self,
391
input_ids: torch.Tensor,
@@ -383,9 +393,11 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
393
kv_caches: List[torch.Tensor],
394
attn_metadata: AttentionMetadata,
395
intermediate_tensors: Optional[IntermediateTensors] = None,
396
+ inputs_embeds: Optional[torch.Tensor] = None,
397
) -> Union[torch.Tensor, IntermediateTensors]:
398
hidden_states = self.transformer(input_ids, positions, kv_caches,
388
- attn_metadata, intermediate_tensors)
399
+ attn_metadata, intermediate_tensors,
400
+ inputs_embeds)
401
return hidden_states
402
403
def compute_logits(
vllm/model_executor/models/deepseek.py
+14
-2
@@ -353,6 +353,9 @@ class DeepseekModel(nn.Module):
353
make_empty_intermediate_tensors_factory(
354
["hidden_states", "residual"], config.hidden_size))
355
356
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
357
+ return self.embed_tokens(input_ids)
358
+
359
def forward(
360
self,
361
input_ids: torch.Tensor,
@@ -360,9 +363,13 @@ class DeepseekModel(nn.Module):
363
kv_caches: List[torch.Tensor],
364
attn_metadata: AttentionMetadata,
365
intermediate_tensors: Optional[IntermediateTensors],
366
+ inputs_embeds: Optional[torch.Tensor] = None,
367
) -> Union[torch.Tensor, IntermediateTensors]:
368
if get_pp_group().is_first_rank:
365
- hidden_states = self.embed_tokens(input_ids)
369
+ if inputs_embeds is not None:
370
+ hidden_states = inputs_embeds
371
+ else:
372
+ hidden_states = self.get_input_embeddings(input_ids)
373
residual = None
374
else:
375
hidden_states = intermediate_tensors["hidden_states"]
@@ -401,6 +408,9 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
408
self.make_empty_intermediate_tensors = (
409
self.model.make_empty_intermediate_tensors)
410
411
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
412
+ return self.model.get_input_embeddings(input_ids)
413
+
414
def forward(
415
self,
416
input_ids: torch.Tensor,
@@ -408,9 +418,11 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
418
kv_caches: List[torch.Tensor],
419
attn_metadata: AttentionMetadata,
420
intermediate_tensors: Optional[IntermediateTensors] = None,
421
+ inputs_embeds: Optional[torch.Tensor] = None,
422
) -> Union[torch.Tensor, IntermediateTensors]:
423
hidden_states = self.model(input_ids, positions, kv_caches,
413
- attn_metadata, intermediate_tensors)
424
+ attn_metadata, intermediate_tensors,
425
+ inputs_embeds)
426
return hidden_states
427
428
def compute_logits(
vllm/model_executor/models/deepseek_v2.py
+14
-2
@@ -445,6 +445,9 @@ class DeepseekV2Model(nn.Module):
445
make_empty_intermediate_tensors_factory(
446
["hidden_states", "residual"], config.hidden_size))
447
448
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
449
+ return self.embed_tokens(input_ids)
450
+
451
def forward(
452
self,
453
input_ids: torch.Tensor,
@@ -452,9 +455,13 @@ class DeepseekV2Model(nn.Module):
455
kv_caches: List[torch.Tensor],
456
attn_metadata: AttentionMetadata,
457
intermediate_tensors: Optional[IntermediateTensors],
458
+ inputs_embeds: Optional[torch.Tensor] = None,
459
) -> Union[torch.Tensor, IntermediateTensors]:
460
if get_pp_group().is_first_rank:
457
- hidden_states = self.embed_tokens(input_ids)
461
+ if inputs_embeds is not None:
462
+ hidden_states = inputs_embeds
463
+ else:
464
+ hidden_states = self.get_input_embeddings(input_ids)
465
residual = None
466
else:
467
assert intermediate_tensors is not None
@@ -495,6 +502,9 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
502
self.make_empty_intermediate_tensors = (
503
self.model.make_empty_intermediate_tensors)
504
505
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
506
+ return self.model.get_input_embeddings(input_ids)
507
+
508
def forward(
509
self,
510
input_ids: torch.Tensor,
@@ -502,9 +512,11 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
512
kv_caches: List[torch.Tensor],
513
attn_metadata: AttentionMetadata,
514
intermediate_tensors: Optional[IntermediateTensors] = None,
515
+ inputs_embeds: Optional[torch.Tensor] = None,
516
) -> Union[torch.Tensor, IntermediateTensors]:
517
hidden_states = self.model(input_ids, positions, kv_caches,
507
- attn_metadata, intermediate_tensors)
518
+ attn_metadata, intermediate_tensors,
519
+ inputs_embeds)
520
return hidden_states
521
522
def compute_logits(
vllm/model_executor/models/eagle.py
+10
-3
@@ -78,6 +78,9 @@ class EAGLE(nn.Module):
78
def sampler(self):
79
return self.model.sampler
80
81
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
82
+ return self.model.model.get_input_embeddings(input_ids)
83
+
84
def forward(
85
self,
86
input_ids: torch.Tensor,
@@ -86,11 +89,14 @@ class EAGLE(nn.Module):
89
attn_metadata: AttentionMetadata,
90
previous_hidden_states: torch.Tensor,
91
intermediate_tensors: Optional[IntermediateTensors] = None,
92
+ inputs_embeds: Optional[torch.Tensor] = None,
93
) -> torch.Tensor:
94
91
- tok_embeds = self.model.model.embed_tokens(input_ids)
95
+ if inputs_embeds is None:
96
+ inputs_embeds = self.get_input_embeddings(input_ids)
97
+
98
inputs_embeds = self.fc(
93
- torch.cat([tok_embeds, previous_hidden_states], dim=-1))
99
+ torch.cat([inputs_embeds, previous_hidden_states], dim=-1))
100
101
inputs_embeds[positions == 0] = 0 # masking inputs at position=0
102
@@ -100,7 +106,8 @@ class EAGLE(nn.Module):
106
positions=positions,
107
kv_caches=kv_caches,
108
attn_metadata=attn_metadata,
103
- intermediate_tensors=intermediate_tensors)
109
+ intermediate_tensors=intermediate_tensors,
110
+ )
111
return hidden_states
112
113
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/exaone.py
+6
-1
@@ -479,6 +479,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
479
self.make_empty_intermediate_tensors = (
480
self.transformer.make_empty_intermediate_tensors)
481
482
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
483
+ return self.model.get_input_embeddings(input_ids)
484
+
485
def forward(
486
self,
487
input_ids: torch.Tensor,
@@ -486,9 +489,11 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
489
kv_caches: List[torch.Tensor],
490
attn_metadata: AttentionMetadata,
491
intermediate_tensors: Optional[IntermediateTensors] = None,
492
+ inputs_embeds: Optional[torch.Tensor] = None,
493
) -> Union[torch.Tensor, IntermediateTensors]:
494
model_output = self.transformer(input_ids, positions, kv_caches,
491
- attn_metadata, intermediate_tensors)
495
+ attn_metadata, intermediate_tensors,
496
+ inputs_embeds)
497
return model_output
498
499
def compute_logits(
vllm/model_executor/models/falcon.py
+14
-2
@@ -367,6 +367,9 @@ class FalconModel(nn.Module):
367
make_empty_intermediate_tensors_factory(["hidden_states"],
368
config.hidden_size))
369
370
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
371
+ return self.word_embeddings(input_ids)
372
+
373
def forward(
374
self,
375
input_ids: torch.Tensor,
@@ -374,9 +377,13 @@ class FalconModel(nn.Module):
377
kv_caches: List[torch.Tensor],
378
attn_metadata: AttentionMetadata,
379
intermediate_tensors: Optional[IntermediateTensors],
380
+ inputs_embeds: Optional[torch.Tensor] = None,
381
) -> Union[torch.Tensor, IntermediateTensors]:
382
if get_pp_group().is_first_rank:
379
- hidden_states = self.word_embeddings(input_ids)
383
+ if inputs_embeds is not None:
384
+ hidden_states = inputs_embeds
385
+ else:
386
+ hidden_states = self.get_input_embeddings(input_ids)
387
else:
388
hidden_states = intermediate_tensors["hidden_states"]
389
for i in range(self.start_layer, self.end_layer):
@@ -432,6 +439,9 @@ class FalconForCausalLM(nn.Module, SupportsPP):
439
self.make_empty_intermediate_tensors = (
440
self.transformer.make_empty_intermediate_tensors)
441
442
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
443
+ return self.transformer.get_input_embeddings(input_ids)
444
+
445
def forward(
446
self,
447
input_ids: torch.LongTensor,
@@ -439,9 +449,11 @@ class FalconForCausalLM(nn.Module, SupportsPP):
449
kv_caches: List[torch.Tensor],
450
attn_metadata: AttentionMetadata,
451
intermediate_tensors: Optional[IntermediateTensors] = None,
452
+ inputs_embeds: Optional[torch.Tensor] = None,
453
) -> torch.Tensor:
454
hidden_states = self.transformer(input_ids, positions, kv_caches,
444
- attn_metadata, intermediate_tensors)
455
+ attn_metadata, intermediate_tensors,
456
+ inputs_embeds)
457
return hidden_states
458
459
def compute_logits(
vllm/model_executor/models/gemma.py
+6
-1
@@ -390,6 +390,9 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
390
self.make_empty_intermediate_tensors = (
391
self.model.make_empty_intermediate_tensors)
392
393
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
394
+ return self.model.get_input_embeddings(input_ids)
395
+
396
def forward(
397
self,
398
input_ids: torch.Tensor,
@@ -397,9 +400,11 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
400
kv_caches: List[torch.Tensor],
401
attn_metadata: AttentionMetadata,
402
intermediate_tensors: Optional[IntermediateTensors] = None,
403
+ inputs_embeds: Optional[torch.Tensor] = None,
404
) -> Union[torch.Tensor, IntermediateTensors]:
405
hidden_states = self.model(input_ids, positions, kv_caches,
402
- attn_metadata, intermediate_tensors)
406
+ attn_metadata, intermediate_tensors,
407
+ inputs_embeds)
408
return hidden_states
409
410
def compute_logits(
vllm/model_executor/models/gemma2.py
+10
-2
@@ -272,6 +272,9 @@ class Gemma2Model(nn.Module):
272
make_empty_intermediate_tensors_factory(
273
["hidden_states", "residual"], config.hidden_size))
274
275
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
276
+ return self.embed_tokens(input_ids)
277
+
278
def forward(
279
self,
280
input_ids: Optional[torch.Tensor],
@@ -285,7 +288,7 @@ class Gemma2Model(nn.Module):
288
if inputs_embeds is not None:
289
hidden_states = inputs_embeds
290
else:
288
- hidden_states = self.embed_tokens(input_ids)
291
+ hidden_states = self.get_input_embeddings(input_ids)
292
hidden_states *= self.normalizer
293
residual = None
294
else:
@@ -414,6 +417,9 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
417
self.make_empty_intermediate_tensors = (
418
self.model.make_empty_intermediate_tensors)
419
420
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
421
+ return self.model.get_input_embeddings(input_ids)
422
+
423
def forward(
424
self,
425
input_ids: torch.Tensor,
@@ -421,9 +427,11 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
427
kv_caches: List[torch.Tensor],
428
attn_metadata: AttentionMetadata,
429
intermediate_tensors: Optional[IntermediateTensors] = None,
430
+ inputs_embeds: Optional[torch.Tensor] = None,
431
) -> Union[torch.Tensor, IntermediateTensors]:
432
hidden_states = self.model(input_ids, positions, kv_caches,
426
- attn_metadata, intermediate_tensors)
433
+ attn_metadata, intermediate_tensors,
434
+ inputs_embeds)
435
return hidden_states
436
437
def compute_logits(
vllm/model_executor/models/gpt2.py
+5
-2
@@ -209,6 +209,9 @@ class GPT2Model(nn.Module):
209
make_empty_intermediate_tensors_factory(["hidden_states"],
210
config.n_embd))
211
212
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
213
+ return self.wte(input_ids)
214
+
215
def forward(
216
self,
217
input_ids: torch.Tensor,
@@ -220,7 +223,7 @@ class GPT2Model(nn.Module):
223
) -> Union[torch.Tensor, IntermediateTensors]:
224
if get_pp_group().is_first_rank:
225
if inputs_embeds is None:
223
- inputs_embeds = self.wte(input_ids)
226
+ inputs_embeds = self.get_input_embeddings(input_ids)
227
position_embeds = self.wpe(position_ids)
228
hidden_states = inputs_embeds + position_embeds
229
else:
@@ -262,7 +265,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
265
self.transformer.make_empty_intermediate_tensors)
266
267
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
265
- return self.transformer.wte(input_ids)
268
+ return self.transformer.get_input_embeddings(input_ids)
269
270
def forward(
271
self,
vllm/model_executor/models/gpt_bigcode.py
+13
-4
@@ -218,6 +218,9 @@ class GPTBigCodeModel(nn.Module):
218
make_empty_intermediate_tensors_factory(["hidden_states"],
219
config.n_embd))
220
221
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222
+ return self.wte(input_ids)
223
+
224
def forward(
225
self,
226
input_ids: torch.Tensor,
@@ -225,11 +228,12 @@ class GPTBigCodeModel(nn.Module):
228
kv_caches: List[torch.Tensor],
229
attn_metadata: AttentionMetadata,
230
intermediate_tensors: Optional[IntermediateTensors],
231
+ inputs_embeds: Optional[torch.Tensor] = None,
232
) -> Union[torch.Tensor, IntermediateTensors]:
233
if get_pp_group().is_first_rank:
230
- inputs_embeds = self.wte(input_ids)
231
- position_embeds = self.wpe(position_ids)
232
- hidden_states = inputs_embeds + position_embeds
234
+ if inputs_embeds is None:
235
+ inputs_embeds = self.get_input_embeddings(input_ids)
236
+ hidden_states = inputs_embeds + self.wpe(position_ids)
237
else:
238
hidden_states = intermediate_tensors["hidden_states"]
239
@@ -285,6 +289,9 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
289
self.make_empty_intermediate_tensors = (
290
self.transformer.make_empty_intermediate_tensors)
291
292
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
293
+ return self.transformer.get_input_embeddings(input_ids)
294
+
295
def forward(
296
self,
297
input_ids: torch.Tensor,
@@ -292,9 +299,11 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
299
kv_caches: List[torch.Tensor],
300
attn_metadata: AttentionMetadata,
301
intermediate_tensors: Optional[IntermediateTensors] = None,
302
+ inputs_embeds: Optional[torch.Tensor] = None,
303
) -> Union[torch.Tensor, IntermediateTensors]:
304
hidden_states = self.transformer(input_ids, positions, kv_caches,
297
- attn_metadata, intermediate_tensors)
305
+ attn_metadata, intermediate_tensors,
306
+ inputs_embeds)
307
return hidden_states
308
309
def compute_logits(
vllm/model_executor/models/gpt_j.py
+14
-2
@@ -201,6 +201,9 @@ class GPTJModel(nn.Module):
201
make_empty_intermediate_tensors_factory(["hidden_states"],
202
config.n_embd))
203
204
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
205
+ return self.wte(input_ids)
206
+
207
def forward(
208
self,
209
input_ids: torch.Tensor,
@@ -208,9 +211,13 @@ class GPTJModel(nn.Module):
211
kv_caches: List[torch.Tensor],
212
attn_metadata: AttentionMetadata,
213
intermediate_tensors: Optional[IntermediateTensors],
214
+ inputs_embeds: Optional[torch.Tensor] = None,
215
) -> Union[torch.Tensor, IntermediateTensors]:
216
if get_pp_group().is_first_rank:
213
- hidden_states = self.wte(input_ids)
217
+ if inputs_embeds is not None:
218
+ hidden_states = inputs_embeds
219
+ else:
220
+ hidden_states = self.get_input_embeddings(input_ids)
221
else:
222
hidden_states = intermediate_tensors["hidden_states"]
223
for i in range(self.start_layer, self.end_layer):
@@ -250,6 +257,9 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
257
self.make_empty_intermediate_tensors = (
258
self.transformer.make_empty_intermediate_tensors)
259
260
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
261
+ return self.transformer.get_input_embeddings(input_ids)
262
+
263
def forward(
264
self,
265
input_ids: torch.Tensor,
@@ -257,9 +267,11 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
267
kv_caches: List[torch.Tensor],
268
attn_metadata: AttentionMetadata,
269
intermediate_tensors: Optional[IntermediateTensors] = None,
270
+ inputs_embeds: Optional[torch.Tensor] = None,
271
) -> Union[torch.Tensor, IntermediateTensors]:
272
hidden_states = self.transformer(input_ids, positions, kv_caches,
262
- attn_metadata, intermediate_tensors)
273
+ attn_metadata, intermediate_tensors,
274
+ inputs_embeds)
275
return hidden_states
276
277
def compute_logits(
vllm/model_executor/models/gpt_neox.py
+14
-2
@@ -214,6 +214,9 @@ class GPTNeoXModel(nn.Module):
214
make_empty_intermediate_tensors_factory(["hidden_states"],
215
config.hidden_size))
216
217
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
218
+ return self.embed_in(input_ids)
219
+
220
def forward(
221
self,
222
input_ids: torch.Tensor,
@@ -221,9 +224,13 @@ class GPTNeoXModel(nn.Module):
224
kv_caches: List[torch.Tensor],
225
attn_metadata: AttentionMetadata,
226
intermediate_tensors: Optional[IntermediateTensors],
227
+ inputs_embeds: Optional[torch.Tensor] = None,
228
) -> Union[torch.Tensor, IntermediateTensors]:
229
if get_pp_group().is_first_rank:
226
- hidden_states = self.embed_in(input_ids)
230
+ if inputs_embeds is not None:
231
+ hidden_states = inputs_embeds
232
+ else:
233
+ hidden_states = self.get_input_embeddings(input_ids)
234
else:
235
hidden_states = intermediate_tensors["hidden_states"]
236
for i in range(self.start_layer, self.end_layer):
@@ -262,6 +269,9 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
269
self.make_empty_intermediate_tensors = (
270
self.gpt_neox.make_empty_intermediate_tensors)
271
272
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
273
+ return self.gpt_neox.get_input_embeddings(input_ids)
274
+
275
def forward(
276
self,
277
input_ids: torch.Tensor,
@@ -269,9 +279,11 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
279
kv_caches: List[torch.Tensor],
280
attn_metadata: AttentionMetadata,
281
intermediate_tensors: Optional[IntermediateTensors] = None,
282
+ inputs_embeds: Optional[torch.Tensor] = None,
283
) -> Union[torch.Tensor, IntermediateTensors]:
284
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
274
- attn_metadata, intermediate_tensors)
285
+ attn_metadata, intermediate_tensors,
286
+ inputs_embeds)
287
return hidden_states
288
289
def compute_logits(
vllm/model_executor/models/granite.py
+6
-1
@@ -409,6 +409,9 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
409
else:
410
self.lm_head = PPMissingLayer()
411
412
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
413
+ return self.model.get_input_embeddings(input_ids)
414
+
415
def forward(
416
self,
417
input_ids: torch.Tensor,
@@ -416,9 +419,11 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
419
kv_caches: List[torch.Tensor],
420
attn_metadata: AttentionMetadata,
421
intermediate_tensors: Optional[IntermediateTensors] = None,
422
+ inputs_embeds: Optional[torch.Tensor] = None,
423
) -> Union[torch.Tensor, IntermediateTensors]:
424
model_output = self.model(input_ids, positions, kv_caches,
421
- attn_metadata, intermediate_tensors)
425
+ attn_metadata, intermediate_tensors,
426
+ inputs_embeds)
427
return model_output
428
429
def compute_logits(
vllm/model_executor/models/granitemoe.py
+14
-2
@@ -277,6 +277,9 @@ class GraniteMoeModel(nn.Module):
277
278
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
279
280
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
281
+ return self.embed_tokens(input_ids)
282
+
283
def forward(
284
self,
285
input_ids: torch.Tensor,
@@ -284,9 +287,13 @@ class GraniteMoeModel(nn.Module):
287
kv_caches: List[torch.Tensor],
288
attn_metadata: AttentionMetadata,
289
intermediate_tensors: Optional[IntermediateTensors],
290
+ inputs_embeds: Optional[torch.Tensor] = None,
291
) -> torch.Tensor:
292
if get_pp_group().is_first_rank:
289
- hidden_states = self.embed_tokens(input_ids)
293
+ if inputs_embeds is not None:
294
+ hidden_states = inputs_embeds
295
+ else:
296
+ hidden_states = self.get_input_embeddings(input_ids)
297
hidden_states *= self.embedding_multiplier
298
residual = None
299
else:
@@ -366,6 +373,9 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
373
374
self.sampler = get_sampler()
375
376
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
377
+ return self.model.get_input_embeddings(input_ids)
378
+
379
def forward(
380
self,
381
input_ids: torch.Tensor,
@@ -373,9 +383,11 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
383
kv_caches: List[torch.Tensor],
384
attn_metadata: AttentionMetadata,
385
intermediate_tensors: Optional[IntermediateTensors] = None,
386
+ inputs_embeds: Optional[torch.Tensor] = None,
387
) -> torch.Tensor:
388
hidden_states = self.model(input_ids, positions, kv_caches,
378
- attn_metadata, intermediate_tensors)
389
+ attn_metadata, intermediate_tensors,
390
+ inputs_embeds)
391
return hidden_states
392
393
def compute_logits(
vllm/model_executor/models/internlm2.py
+7
-2
@@ -290,7 +290,7 @@ class InternLM2Model(nn.Module):
290
if inputs_embeds is not None:
291
hidden_states = inputs_embeds
292
else:
293
- hidden_states = self.tok_embeddings(input_ids)
293
+ hidden_states = self.get_input_embeddings(input_ids)
294
residual = None
295
else:
296
assert intermediate_tensors is not None
@@ -335,6 +335,9 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
335
self.make_empty_intermediate_tensors = (
336
self.model.make_empty_intermediate_tensors)
337
338
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
339
+ return self.model.get_input_embeddings(input_ids)
340
+
341
def forward(
342
self,
343
input_ids: torch.Tensor,
@@ -342,9 +345,11 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
345
kv_caches: List[torch.Tensor],
346
attn_metadata: AttentionMetadata,
347
intermediate_tensors: Optional[IntermediateTensors],
348
+ inputs_embeds: Optional[torch.Tensor] = None,
349
) -> torch.Tensor:
350
hidden_states = self.model(input_ids, positions, kv_caches,
347
- attn_metadata, intermediate_tensors)
351
+ attn_metadata, intermediate_tensors,
352
+ inputs_embeds)
353
return hidden_states
354
355
def compute_logits(
vllm/model_executor/models/jais.py
+12
-2
@@ -250,6 +250,9 @@ class JAISModel(nn.Module):
250
make_empty_intermediate_tensors_factory(["hidden_states"],
251
config.n_embd))
252
253
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
254
+ return self.wte(input_ids)
255
+
256
def forward(
257
self,
258
input_ids: torch.Tensor,
@@ -257,9 +260,11 @@ class JAISModel(nn.Module):
260
kv_caches: List[torch.Tensor],
261
attn_metadata: AttentionMetadata,
262
intermediate_tensors: Optional[IntermediateTensors] = None,
263
+ inputs_embeds: Optional[torch.Tensor] = None,
264
) -> Union[IntermediateTensors, torch.Tensor]:
265
if get_pp_group().is_first_rank:
262
- inputs_embeds = self.wte(input_ids)
266
+ if inputs_embeds is None:
267
+ inputs_embeds = self.get_input_embeddings(input_ids)
268
if self.wpe is not None:
269
position_embeds = self.wpe(position_ids)
270
hidden_states = inputs_embeds + position_embeds
@@ -311,6 +316,9 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
316
self.make_empty_intermediate_tensors = (
317
self.transformer.make_empty_intermediate_tensors)
318
319
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
320
+ return self.transformer.get_input_embeddings(input_ids)
321
+
322
def forward(
323
self,
324
input_ids: torch.Tensor,
@@ -318,9 +326,11 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
326
kv_caches: List[torch.Tensor],
327
attn_metadata: AttentionMetadata,
328
intermediate_tensors: Optional[IntermediateTensors] = None,
329
+ inputs_embeds: Optional[torch.Tensor] = None,
330
) -> Union[IntermediateTensors, torch.Tensor]:
331
hidden_states = self.transformer(input_ids, positions, kv_caches,
323
- attn_metadata, intermediate_tensors)
332
+ attn_metadata, intermediate_tensors,
333
+ inputs_embeds)
334
return hidden_states
335
336
def compute_logits(
vllm/model_executor/models/jamba.py
+14
-2
@@ -292,6 +292,9 @@ class JambaModel(nn.Module):
292
self.final_layernorm = RMSNorm(config.hidden_size,
293
eps=config.rms_norm_eps)
294
295
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
296
+ return self.embed_tokens(input_ids)
297
+
298
def forward(
299
self,
300
input_ids: torch.Tensor,
@@ -299,8 +302,12 @@ class JambaModel(nn.Module):
302
kv_caches: List[torch.Tensor],
303
attn_metadata: AttentionMetadata,
304
mamba_cache_params: MambaCacheParams,
305
+ inputs_embeds: Optional[torch.Tensor] = None,
306
) -> torch.Tensor:
303
- hidden_states = self.embed_tokens(input_ids)
307
+ if inputs_embeds is not None:
308
+ hidden_states = inputs_embeds
309
+ else:
310
+ hidden_states = self.get_input_embeddings(input_ids)
311
residual = None
312
for i in range(len(self.layers)):
313
layer = self.layers[i]
@@ -381,12 +388,16 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
388
config.vocab_size)
389
self.sampler = get_sampler()
390
391
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
392
+ return self.model.get_input_embeddings(input_ids)
393
+
394
def forward(self,
395
input_ids: torch.Tensor,
396
positions: torch.Tensor,
397
kv_caches: List[KVCache],
398
attn_metadata: AttentionMetadata,
399
intermediate_tensors: Optional[IntermediateTensors] = None,
400
+ inputs_embeds: Optional[torch.Tensor] = None,
401
**kwargs):
402
if self.mamba_cache is None:
403
max_batch_size = (_get_graph_batch_size(
@@ -409,7 +420,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
420
mamba_cache_tensors[1],
421
state_indices_tensor)
422
hidden_states = self.model(input_ids, positions, kv_caches,
412
- attn_metadata, mamba_cache_params)
423
+ attn_metadata, mamba_cache_params,
424
+ inputs_embeds)
425
return hidden_states
426
427
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
vllm/model_executor/models/mamba.py
+13
-2
@@ -106,15 +106,22 @@ class MambaModel(nn.Module):
106
self.norm_f = RMSNorm(config.hidden_size,
107
eps=config.layer_norm_epsilon)
108
109
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
110
+ return self.embeddings(input_ids)
111
+
112
def forward(
113
self,
114
input_ids: torch.Tensor,
115
positions: torch.Tensor,
116
attn_metadata: AttentionMetadata,
117
mamba_cache_params: MambaCacheParams,
118
+ inputs_embeds: Optional[torch.Tensor] = None,
119
) -> torch.Tensor:
120
117
- hidden_states = self.embeddings(input_ids)
121
+ if inputs_embeds is not None:
122
+ hidden_states = inputs_embeds
123
+ else:
124
+ hidden_states = self.get_input_embeddings(input_ids)
125
residual = None
126
127
for i in range(len(self.layers)):
@@ -168,12 +175,16 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
175
config.vocab_size)
176
self.sampler = get_sampler()
177
178
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
179
+ return self.backbone.get_input_embeddings(input_ids)
180
+
181
def forward(self,
182
input_ids: torch.Tensor,
183
positions: torch.Tensor,
184
kv_caches: List[KVCache],
185
attn_metadata: AttentionMetadata,
186
intermediate_tensors: Optional[IntermediateTensors] = None,
187
+ inputs_embeds: Optional[torch.Tensor] = None,
188
**kwargs):
189
if self.mamba_cache is None:
190
max_batch_size = (_get_graph_batch_size(
@@ -194,7 +205,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
205
state_indices_tensor)
206
207
hidden_states = self.backbone(input_ids, positions, attn_metadata,
197
- mamba_cache_params)
208
+ mamba_cache_params, inputs_embeds)
209
210
return hidden_states
211
vllm/model_executor/models/minicpm.py
+6
-1
@@ -504,6 +504,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
504
self.model = MiniCPMModel(vllm_config=vllm_config,
505
prefix=maybe_prefix(prefix, "model"))
506
507
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
508
+ return self.model.get_input_embeddings(input_ids)
509
+
510
def forward(
511
self,
512
input_ids: torch.Tensor,
@@ -511,9 +514,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
514
kv_caches: List[torch.Tensor],
515
attn_metadata: AttentionMetadata,
516
intermediate_tensors: Optional[IntermediateTensors] = None,
517
+ inputs_embeds: Optional[torch.Tensor] = None,
518
) -> Union[torch.Tensor, IntermediateTensors]:
519
hidden_states = self.model(input_ids, positions, kv_caches,
516
- attn_metadata, intermediate_tensors)
520
+ attn_metadata, intermediate_tensors,
521
+ inputs_embeds)
522
return hidden_states
523
524
def compute_logits(
vllm/model_executor/models/mixtral.py
+14
-2
@@ -281,6 +281,9 @@ class MixtralModel(nn.Module):
281
make_empty_intermediate_tensors_factory(
282
["hidden_states", "residual"], config.hidden_size))
283
284
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
285
+ return self.embed_tokens(input_ids)
286
+
287
def forward(
288
self,
289
input_ids: torch.Tensor,
@@ -288,9 +291,13 @@ class MixtralModel(nn.Module):
291
kv_caches: List[torch.Tensor],
292
attn_metadata: AttentionMetadata,
293
intermediate_tensors: Optional[IntermediateTensors],
294
+ inputs_embeds: Optional[torch.Tensor] = None,
295
) -> Union[torch.Tensor, IntermediateTensors]:
296
if get_pp_group().is_first_rank:
293
- hidden_states = self.embed_tokens(input_ids)
297
+ if inputs_embeds is not None:
298
+ hidden_states = inputs_embeds
299
+ else:
300
+ hidden_states = self.get_input_embeddings(input_ids)
301
residual = None
302
else:
303
assert intermediate_tensors is not None
@@ -363,6 +370,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
370
self.make_empty_intermediate_tensors = (
371
self.model.make_empty_intermediate_tensors)
372
373
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
374
+ return self.model.get_input_embeddings(input_ids)
375
+
376
def forward(
377
self,
378
input_ids: torch.Tensor,
@@ -370,9 +380,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
380
kv_caches: List[torch.Tensor],
381
attn_metadata: AttentionMetadata,
382
intermediate_tensors: Optional[IntermediateTensors] = None,
383
+ inputs_embeds: Optional[torch.Tensor] = None,
384
) -> Union[torch.Tensor, IntermediateTensors]:
385
hidden_states = self.model(input_ids, positions, kv_caches,
375
- attn_metadata, intermediate_tensors)
386
+ attn_metadata, intermediate_tensors,
387
+ inputs_embeds)
388
return hidden_states
389
390
def compute_logits(
vllm/model_executor/models/mixtral_quant.py
+14
-2
@@ -318,6 +318,9 @@ class MixtralModel(nn.Module):
318
make_empty_intermediate_tensors_factory(
319
["hidden_states", "residual"], config.hidden_size))
320
321
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
322
+ return self.embed_tokens(input_ids)
323
+
324
def forward(
325
self,
326
input_ids: torch.Tensor,
@@ -325,9 +328,13 @@ class MixtralModel(nn.Module):
328
kv_caches: List[torch.Tensor],
329
attn_metadata: AttentionMetadata,
330
intermediate_tensors: Optional[IntermediateTensors],
331
+ inputs_embeds: Optional[torch.Tensor] = None,
332
) -> Union[torch.Tensor, IntermediateTensors]:
333
if get_pp_group().is_first_rank:
330
- hidden_states = self.embed_tokens(input_ids)
334
+ if inputs_embeds is not None:
335
+ hidden_states = inputs_embeds
336
+ else:
337
+ hidden_states = self.get_input_embeddings(input_ids)
338
residual = None
339
else:
340
assert intermediate_tensors is not None
@@ -368,6 +375,9 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
375
self.make_empty_intermediate_tensors = (
376
self.model.make_empty_intermediate_tensors)
377
378
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
379
+ return self.model.get_input_embeddings(input_ids)
380
+
381
def forward(
382
self,
383
input_ids: torch.Tensor,
@@ -375,9 +385,11 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
385
kv_caches: List[torch.Tensor],
386
attn_metadata: AttentionMetadata,
387
intermediate_tensors: Optional[IntermediateTensors] = None,
388
+ inputs_embeds: Optional[torch.Tensor] = None,
389
) -> Union[torch.Tensor, IntermediateTensors]:
390
hidden_states = self.model(input_ids, positions, kv_caches,
380
- attn_metadata, intermediate_tensors)
391
+ attn_metadata, intermediate_tensors,
392
+ inputs_embeds)
393
return hidden_states
394
395
def compute_logits(
vllm/model_executor/models/mpt.py
+14
-2
@@ -237,6 +237,9 @@ class MPTModel(nn.Module):
237
make_empty_intermediate_tensors_factory(["hidden_states"],
238
config.d_model))
239
240
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241
+ return self.wte(input_ids)
242
+
243
def forward(
244
self,
245
input_ids: torch.Tensor,
@@ -244,9 +247,13 @@ class MPTModel(nn.Module):
247
kv_caches: List[torch.Tensor],
248
attn_metadata: AttentionMetadata,
249
intermediate_tensors: Optional[IntermediateTensors],
250
+ inputs_embeds: Optional[torch.Tensor] = None,
251
) -> Union[torch.Tensor, IntermediateTensors]:
252
if get_pp_group().is_first_rank:
249
- hidden_states = self.wte(input_ids)
253
+ if inputs_embeds is not None:
254
+ hidden_states = inputs_embeds
255
+ else:
256
+ hidden_states = self.get_input_embeddings(input_ids)
257
else:
258
assert intermediate_tensors is not None
259
hidden_states = intermediate_tensors["hidden_states"]
@@ -283,6 +290,9 @@ class MPTForCausalLM(nn.Module, SupportsPP):
290
self.make_empty_intermediate_tensors = (
291
self.transformer.make_empty_intermediate_tensors)
292
293
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
294
+ return self.transformer.get_input_embeddings(input_ids)
295
+
296
def forward(
297
self,
298
input_ids: torch.Tensor,
@@ -290,9 +300,11 @@ class MPTForCausalLM(nn.Module, SupportsPP):
300
kv_caches: List[torch.Tensor],
301
attn_metadata: AttentionMetadata,
302
intermediate_tensors: Optional[IntermediateTensors] = None,
303
+ inputs_embeds: Optional[torch.Tensor] = None,
304
) -> Union[torch.Tensor, IntermediateTensors]:
305
hidden_states = self.transformer(input_ids, positions, kv_caches,
295
- attn_metadata, intermediate_tensors)
306
+ attn_metadata, intermediate_tensors,
307
+ inputs_embeds)
308
return hidden_states
309
310
def compute_logits(
vllm/model_executor/models/nemotron.py
+6
-1
@@ -440,6 +440,9 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
440
self.make_empty_intermediate_tensors = (
441
self.model.make_empty_intermediate_tensors)
442
443
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
444
+ return self.model.get_input_embeddings(input_ids)
445
+
446
def forward(
447
self,
448
input_ids: torch.Tensor,
@@ -447,9 +450,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
450
kv_caches: List[torch.Tensor],
451
attn_metadata: AttentionMetadata,
452
intermediate_tensors: Optional[IntermediateTensors] = None,
453
+ inputs_embeds: Optional[torch.Tensor] = None,
454
) -> Union[torch.Tensor, IntermediateTensors]:
455
model_output = self.model(input_ids, positions, kv_caches,
452
- attn_metadata, intermediate_tensors)
456
+ attn_metadata, intermediate_tensors,
457
+ inputs_embeds)
458
return model_output
459
460
def compute_logits(
vllm/model_executor/models/olmo.py
+13
-6
@@ -248,6 +248,9 @@ class OlmoModel(nn.Module):
248
make_empty_intermediate_tensors_factory(["hidden_states"],
249
config.hidden_size))
250
251
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
252
+ return self.embed_tokens(input_ids)
253
+
254
def forward(
255
self,
256
input_ids: torch.Tensor,
@@ -255,17 +258,16 @@ class OlmoModel(nn.Module):
258
kv_caches: List[torch.Tensor],
259
attn_metadata: AttentionMetadata,
260
intermediate_tensors: Optional[IntermediateTensors],
261
+ inputs_embeds: Optional[torch.Tensor] = None,
262
) -> Union[torch.Tensor, IntermediateTensors]:
263
"""
264
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
265
"""
266
if get_pp_group().is_first_rank:
263
- # Get embeddings of input.
264
- # shape: (batch_size, seq_len, d_model)
265
- inputs_embeds = self.embed_tokens(input_ids)
266
-
267
- # embed positions
268
- hidden_states = inputs_embeds
267
+ if inputs_embeds is not None:
268
+ hidden_states = inputs_embeds
269
+ else:
270
+ hidden_states = self.get_input_embeddings(input_ids)
271
else:
272
assert intermediate_tensors is not None
273
hidden_states = intermediate_tensors["hidden_states"]
@@ -315,6 +317,9 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
317
self.make_empty_intermediate_tensors = (
318
self.model.make_empty_intermediate_tensors)
319
320
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
321
+ return self.model.get_input_embeddings(input_ids)
322
+
323
def forward(
324
self,
325
input_ids: torch.Tensor,
@@ -322,6 +327,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
327
kv_caches: List[torch.Tensor],
328
attn_metadata: AttentionMetadata,
329
intermediate_tensors: Optional[IntermediateTensors] = None,
330
+ inputs_embeds: Optional[torch.Tensor] = None,
331
) -> Union[torch.Tensor, IntermediateTensors]:
332
hidden_states = self.model(
333
input_ids=input_ids,
@@ -329,6 +335,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
335
kv_caches=kv_caches,
336
attn_metadata=attn_metadata,
337
intermediate_tensors=intermediate_tensors,
338
+ inputs_embeds=inputs_embeds,
339
)
340
return hidden_states
341
vllm/model_executor/models/olmoe.py
+14
-2
@@ -269,6 +269,9 @@ class OlmoeModel(nn.Module):
269
make_empty_intermediate_tensors_factory(
270
["hidden_states", "residual"], config.hidden_size))
271
272
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
273
+ return self.embed_tokens(input_ids)
274
+
275
def forward(
276
self,
277
input_ids: torch.Tensor,
@@ -276,9 +279,13 @@ class OlmoeModel(nn.Module):
279
kv_caches: List[torch.Tensor],
280
attn_metadata: AttentionMetadata,
281
intermediate_tensors: Optional[IntermediateTensors],
282
+ inputs_embeds: Optional[torch.Tensor] = None,
283
) -> Union[torch.Tensor, IntermediateTensors]:
284
if get_pp_group().is_first_rank:
281
- hidden_states = self.embed_tokens(input_ids)
285
+ if inputs_embeds is not None:
286
+ hidden_states = inputs_embeds
287
+ else:
288
+ hidden_states = self.get_input_embeddings(input_ids)
289
residual = None
290
else:
291
assert intermediate_tensors is not None
@@ -326,6 +333,9 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
333
self.make_empty_intermediate_tensors = (
334
self.model.make_empty_intermediate_tensors)
335
336
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
337
+ return self.model.get_input_embeddings(input_ids)
338
+
339
def forward(
340
self,
341
input_ids: torch.Tensor,
@@ -333,9 +343,11 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
343
kv_caches: List[torch.Tensor],
344
attn_metadata: AttentionMetadata,
345
intermediate_tensors: Optional[IntermediateTensors] = None,
346
+ inputs_embeds: Optional[torch.Tensor] = None,
347
) -> Union[torch.Tensor, IntermediateTensors]:
348
hidden_states = self.model(input_ids, positions, kv_caches,
338
- attn_metadata, intermediate_tensors)
349
+ attn_metadata, intermediate_tensors,
350
+ inputs_embeds)
351
return hidden_states
352
353
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/orion.py
+14
-2
@@ -237,6 +237,9 @@ class OrionModel(nn.Module):
237
"hidden_states",
238
], config.hidden_size))
239
240
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241
+ return self.embed_tokens(input_ids)
242
+
243
def forward(
244
self,
245
input_ids: torch.Tensor,
@@ -244,9 +247,13 @@ class OrionModel(nn.Module):
247
kv_caches: List[torch.Tensor],
248
attn_metadata: AttentionMetadata,
249
intermediate_tensors: Optional[IntermediateTensors],
250
+ inputs_embeds: Optional[torch.Tensor] = None,
251
) -> Union[torch.Tensor, IntermediateTensors]:
252
if get_pp_group().is_first_rank:
249
- hidden_states = self.embed_tokens(input_ids)
253
+ if inputs_embeds is not None:
254
+ hidden_states = inputs_embeds
255
+ else:
256
+ hidden_states = self.get_input_embeddings(input_ids)
257
else:
258
assert intermediate_tensors is not None
259
hidden_states = intermediate_tensors["hidden_states"]
@@ -286,6 +293,9 @@ class OrionForCausalLM(nn.Module, SupportsPP):
293
self.make_empty_intermediate_tensors = (
294
self.model.make_empty_intermediate_tensors)
295
296
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
297
+ return self.model.get_input_embeddings(input_ids)
298
+
299
def forward(
300
self,
301
input_ids: torch.Tensor,
@@ -293,9 +303,11 @@ class OrionForCausalLM(nn.Module, SupportsPP):
303
kv_caches: List[torch.Tensor],
304
attn_metadata: AttentionMetadata,
305
intermediate_tensors: Optional[IntermediateTensors] = None,
306
+ inputs_embeds: Optional[torch.Tensor] = None,
307
) -> Union[torch.Tensor, IntermediateTensors]:
308
hidden_states = self.model(input_ids, positions, kv_caches,
298
- attn_metadata, intermediate_tensors)
309
+ attn_metadata, intermediate_tensors,
310
+ inputs_embeds)
311
return hidden_states
312
313
def compute_logits(
vllm/model_executor/models/persimmon.py
+7
-1
@@ -235,6 +235,9 @@ class PersimmonModel(nn.Module):
235
make_empty_intermediate_tensors_factory(["hidden_states"],
236
config.hidden_size))
237
238
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
239
+ return self.embed_tokens(input_ids)
240
+
241
def forward(
242
self,
243
input_ids: torch.Tensor,
@@ -248,7 +251,7 @@ class PersimmonModel(nn.Module):
251
if inputs_embeds is not None:
252
hidden_states = inputs_embeds
253
else:
251
- hidden_states = self.embed_tokens(input_ids)
254
+ hidden_states = self.get_input_embeddings(input_ids)
255
else:
256
assert intermediate_tensors is not None
257
hidden_states = intermediate_tensors["hidden_states"]
@@ -282,6 +285,9 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
285
self.make_empty_intermediate_tensors = (
286
self.model.make_empty_intermediate_tensors)
287
288
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
289
+ return self.model.get_input_embeddings(input_ids)
290
+
291
def forward(
292
self,
293
input_ids: torch.Tensor,
vllm/model_executor/models/phi.py
+14
-2
@@ -218,6 +218,9 @@ class PhiModel(nn.Module):
218
make_empty_intermediate_tensors_factory(["hidden_states"],
219
config.hidden_size))
220
221
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222
+ return self.embed_tokens(input_ids)
223
+
224
def forward(
225
self,
226
input_ids: torch.Tensor,
@@ -225,9 +228,13 @@ class PhiModel(nn.Module):
228
kv_caches: List[torch.Tensor],
229
attn_metadata: AttentionMetadata,
230
intermediate_tensors: Optional[IntermediateTensors],
231
+ inputs_embeds: Optional[torch.Tensor] = None,
232
) -> Union[torch.Tensor, IntermediateTensors]:
233
if get_pp_group().is_first_rank:
230
- hidden_states = self.embed_tokens(input_ids)
234
+ if inputs_embeds is not None:
235
+ hidden_states = inputs_embeds
236
+ else:
237
+ hidden_states = self.get_input_embeddings(input_ids)
238
else:
239
assert intermediate_tensors is not None
240
hidden_states = intermediate_tensors["hidden_states"]
@@ -303,6 +310,9 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
310
self.make_empty_intermediate_tensors = (
311
self.model.make_empty_intermediate_tensors)
312
313
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
314
+ return self.model.get_input_embeddings(input_ids)
315
+
316
def forward(
317
self,
318
input_ids: torch.Tensor,
@@ -310,9 +320,11 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
320
kv_caches: List[torch.Tensor],
321
attn_metadata: AttentionMetadata,
322
intermediate_tensors: Optional[IntermediateTensors] = None,
323
+ inputs_embeds: Optional[torch.Tensor] = None,
324
) -> Union[torch.Tensor, IntermediateTensors]:
325
hidden_states = self.model(input_ids, positions, kv_caches,
315
- attn_metadata, intermediate_tensors)
326
+ attn_metadata, intermediate_tensors,
327
+ inputs_embeds)
328
329
return hidden_states
330
vllm/model_executor/models/phi3_small.py
+11
-8
@@ -324,11 +324,8 @@ class Phi3SmallModel(nn.Module):
324
make_empty_intermediate_tensors_factory(["hidden_states"],
325
config.hidden_size))
326
327
- def get_input_embeddings(self):
328
- return self.embed_tokens
329
-
330
- def set_input_embeddings(self, value):
331
- self.embed_tokens = value
327
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
328
+ return self.embed_tokens(input_ids)
329
330
def forward(
331
self,
@@ -337,9 +334,13 @@ class Phi3SmallModel(nn.Module):
334
kv_caches: List[torch.Tensor],
335
attn_metadata: AttentionMetadata,
336
intermediate_tensors: Optional[IntermediateTensors],
337
+ inputs_embeds: Optional[torch.Tensor],
338
) -> Union[torch.Tensor, IntermediateTensors]:
339
if get_pp_group().is_first_rank:
342
- hidden_states = self.embed_tokens(input_ids)
340
+ if inputs_embeds is not None:
341
+ hidden_states = inputs_embeds
342
+ else:
343
+ hidden_states = self.get_input_embeddings(input_ids)
344
if (self.mup_embedding_multiplier is not None
345
and self.mup_embedding_multiplier > 0.0):
346
hidden_states = hidden_states * self.mup_embedding_multiplier
@@ -397,8 +398,8 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
398
else:
399
self.dummy_token_indices = None
400
400
- def get_input_embeddings(self):
401
- return self.model.embed_tokens
401
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
402
+ return self.model.get_input_embeddings(input_ids)
403
404
def set_input_embeddings(self, value):
405
self.model.embed_tokens = value
@@ -433,6 +434,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
434
kv_caches: List[torch.Tensor],
435
attn_metadata: AttentionMetadata,
436
intermediate_tensors: Optional[IntermediateTensors] = None,
437
+ inputs_embeds: Optional[torch.Tensor] = None,
438
) -> Union[torch.Tensor, IntermediateTensors]:
439
output_hidden_states = self.model(
440
input_ids=input_ids,
@@ -440,6 +442,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
442
kv_caches=kv_caches,
443
attn_metadata=attn_metadata,
444
intermediate_tensors=intermediate_tensors,
445
+ inputs_embeds=inputs_embeds,
446
)
447
output_hidden_states = output_hidden_states
448
return output_hidden_states
vllm/model_executor/models/phimoe.py
+14
-2
@@ -465,6 +465,9 @@ class PhiMoEModel(nn.Module):
465
make_empty_intermediate_tensors_factory(
466
["hidden_states", "residual"], config.hidden_size))
467
468
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
469
+ return self.embed_tokens(input_ids)
470
+
471
def forward(
472
self,
473
input_ids: torch.Tensor,
@@ -472,9 +475,13 @@ class PhiMoEModel(nn.Module):
475
kv_caches: List[torch.Tensor],
476
attn_metadata: AttentionMetadata,
477
intermediate_tensors: Optional[IntermediateTensors],
478
+ inputs_embeds: Optional[torch.Tensor] = None,
479
) -> Union[torch.Tensor, IntermediateTensors]:
480
if get_pp_group().is_first_rank:
477
- hidden_states = self.embed_tokens(input_ids)
481
+ if inputs_embeds is not None:
482
+ hidden_states = inputs_embeds
483
+ else:
484
+ hidden_states = self.get_input_embeddings(input_ids)
485
residual = None
486
else:
487
assert intermediate_tensors is not None
@@ -560,6 +567,9 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
567
self.make_empty_intermediate_tensors = (
568
self.model.make_empty_intermediate_tensors)
569
570
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
571
+ return self.model.get_input_embeddings(input_ids)
572
+
573
def forward(
574
self,
575
input_ids: torch.Tensor,
@@ -567,9 +577,11 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
577
kv_caches: List[torch.Tensor],
578
attn_metadata: AttentionMetadata,
579
intermediate_tensors: Optional[IntermediateTensors] = None,
580
+ inputs_embeds: Optional[torch.Tensor] = None,
581
) -> Union[torch.Tensor, IntermediateTensors]:
582
hidden_states = self.model(input_ids, positions, kv_caches,
572
- attn_metadata, intermediate_tensors)
583
+ attn_metadata, intermediate_tensors,
584
+ inputs_embeds)
585
return hidden_states
586
587
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen.py
+14
-2
@@ -578,6 +578,9 @@ class QWenModel(nn.Module):
578
quant_config=quant_config) if hasattr(
579
config, "visual") else None
580
581
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
582
+ return self.wte(input_ids)
583
+
584
def forward(
585
self,
586
input_ids: torch.Tensor,
@@ -586,6 +589,7 @@ class QWenModel(nn.Module):
589
attn_metadata: AttentionMetadata,
590
intermediate_tensors: Optional[IntermediateTensors],
591
pixel_values: Optional[QwenImageInputs],
592
+ inputs_embeds: Optional[torch.Tensor] = None,
593
) -> Union[torch.Tensor, IntermediateTensors]:
594
img_pos = None
595
# If pixel / visual embeddings are provided, this is a visual model
@@ -606,6 +610,10 @@ class QWenModel(nn.Module):
610
)
611
612
if get_pp_group().is_first_rank:
613
+ if inputs_embeds is not None:
614
+ hidden_states = inputs_embeds
615
+ else:
616
+ hidden_states = self.get_input_embeddings(input_ids)
617
hidden_states = self.wte(input_ids)
618
# Merge the image embeddings into the hidden states if actually have
619
# visual features and the corresponding image tokens
@@ -915,6 +923,9 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
923
)
924
return None
925
926
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
927
+ return self.transformer.get_input_embeddings(input_ids)
928
+
929
def forward(
930
self,
931
input_ids: torch.Tensor,
@@ -922,7 +933,8 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
933
kv_caches: List[torch.Tensor],
934
attn_metadata: AttentionMetadata,
935
intermediate_tensors: Optional[IntermediateTensors] = None,
925
- pixel_values: Optional[torch.Tensor] = None
936
+ pixel_values: Optional[torch.Tensor] = None,
937
+ inputs_embeds: Optional[torch.Tensor] = None,
938
) -> Union[torch.Tensor, IntermediateTensors]:
939
if intermediate_tensors is not None:
940
input_ids = None
@@ -932,7 +944,7 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
944
945
hidden_states = self.transformer(input_ids, positions, kv_caches,
946
attn_metadata, intermediate_tensors,
935
- pixel_values)
947
+ pixel_values, inputs_embeds)
948
return hidden_states
949
950
def compute_logits(
vllm/model_executor/models/qwen2.py
+1
-1
@@ -309,7 +309,7 @@ class Qwen2Model(nn.Module):
309
if inputs_embeds is not None:
310
hidden_states = inputs_embeds
311
else:
312
- hidden_states = self.embed_tokens(input_ids)
312
+ hidden_states = self.get_input_embeddings(input_ids)
313
residual = None
314
else:
315
assert intermediate_tensors is not None
vllm/model_executor/models/qwen2_cls.py
+6
-1
@@ -72,6 +72,9 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
72
normalize=False,
73
softmax=True)
74
75
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
76
+ return self.model.get_input_embeddings(input_ids)
77
+
78
def forward(
79
self,
80
input_ids: torch.Tensor,
@@ -79,9 +82,11 @@ class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
82
kv_caches: List[torch.Tensor],
83
attn_metadata: AttentionMetadata,
84
intermediate_tensors: Optional[IntermediateTensors] = None,
85
+ inputs_embeds: Optional[torch.Tensor] = None,
86
) -> torch.Tensor:
87
hidden_states = self.model(input_ids, positions, kv_caches,
84
- attn_metadata, intermediate_tensors)
88
+ attn_metadata, intermediate_tensors,
89
+ inputs_embeds)
90
logits, _ = self.score(hidden_states)
91
return logits
92
vllm/model_executor/models/qwen2_moe.py
+14
-2
@@ -344,6 +344,9 @@ class Qwen2MoeModel(nn.Module):
344
make_empty_intermediate_tensors_factory(
345
["hidden_states", "residual"], config.hidden_size))
346
347
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
348
+ return self.embed_tokens(input_ids)
349
+
350
def forward(
351
self,
352
input_ids: torch.Tensor,
@@ -351,9 +354,13 @@ class Qwen2MoeModel(nn.Module):
354
kv_caches: List[torch.Tensor],
355
attn_metadata: AttentionMetadata,
356
intermediate_tensors: Optional[IntermediateTensors] = None,
357
+ inputs_embeds: Optional[torch.Tensor] = None,
358
) -> Union[torch.Tensor, IntermediateTensors]:
359
if get_pp_group().is_first_rank:
356
- hidden_states = self.embed_tokens(input_ids)
360
+ if inputs_embeds is not None:
361
+ hidden_states = inputs_embeds
362
+ else:
363
+ hidden_states = self.get_input_embeddings(input_ids)
364
residual = None
365
else:
366
assert intermediate_tensors is not None
@@ -395,6 +402,9 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
402
self.make_empty_intermediate_tensors = (
403
self.model.make_empty_intermediate_tensors)
404
405
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
406
+ return self.model.get_input_embeddings(input_ids)
407
+
408
def forward(
409
self,
410
input_ids: torch.Tensor,
@@ -402,9 +412,11 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
412
kv_caches: List[torch.Tensor],
413
attn_metadata: AttentionMetadata,
414
intermediate_tensors: Optional[IntermediateTensors] = None,
415
+ inputs_embeds: Optional[torch.Tensor] = None,
416
) -> Union[torch.Tensor, IntermediateTensors]:
417
hidden_states = self.model(input_ids, positions, kv_caches,
407
- attn_metadata, intermediate_tensors)
418
+ attn_metadata, intermediate_tensors,
419
+ inputs_embeds)
420
return hidden_states
421
422
def compute_logits(
vllm/model_executor/models/qwen2_rm.py
+6
-1
@@ -85,6 +85,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
85
self.make_empty_intermediate_tensors = (
86
self.model.make_empty_intermediate_tensors)
87
88
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
89
+ return self.model.get_input_embeddings(input_ids)
90
+
91
def forward(
92
self,
93
input_ids: torch.Tensor,
@@ -92,9 +95,11 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
95
kv_caches: List[torch.Tensor],
96
attn_metadata: AttentionMetadata,
97
intermediate_tensors: Optional[IntermediateTensors] = None,
98
+ inputs_embeds: Optional[torch.Tensor] = None,
99
) -> Union[torch.Tensor, IntermediateTensors]:
100
hidden_states = self.model(input_ids, positions, kv_caches,
97
- attn_metadata, intermediate_tensors)
101
+ attn_metadata, intermediate_tensors,
102
+ inputs_embeds)
103
logits, _ = self.score(hidden_states)
104
return logits
105
vllm/model_executor/models/solar.py
+3
-1
@@ -456,9 +456,11 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
456
kv_caches: List[torch.Tensor],
457
attn_metadata: AttentionMetadata,
458
intermediate_tensors: Optional[IntermediateTensors] = None,
459
+ inputs_embeds: Optional[torch.Tensor] = None,
460
) -> Union[torch.Tensor, IntermediateTensors]:
461
model_output = self.model(input_ids, positions, kv_caches,
461
- attn_metadata, intermediate_tensors)
462
+ attn_metadata, intermediate_tensors,
463
+ inputs_embeds)
464
return model_output
465
466
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/stablelm.py
+14
-2
@@ -218,6 +218,9 @@ class StableLMEpochModel(nn.Module):
218
make_empty_intermediate_tensors_factory(["hidden_states"],
219
config.hidden_size))
220
221
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222
+ return self.embed_tokens(input_ids)
223
+
224
def forward(
225
self,
226
input_ids: torch.Tensor,
@@ -225,9 +228,13 @@ class StableLMEpochModel(nn.Module):
228
kv_caches: List[torch.Tensor],
229
attn_metadata: AttentionMetadata,
230
intermediate_tensors: Optional[IntermediateTensors],
231
+ inputs_embeds: Optional[torch.Tensor] = None,
232
) -> Union[torch.Tensor, IntermediateTensors]:
233
if get_pp_group().is_first_rank:
230
- hidden_states = self.embed_tokens(input_ids)
234
+ if inputs_embeds is not None:
235
+ hidden_states = inputs_embeds
236
+ else:
237
+ hidden_states = self.get_input_embeddings(input_ids)
238
else:
239
assert intermediate_tensors is not None
240
hidden_states = intermediate_tensors["hidden_states"]
@@ -265,6 +272,9 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
272
self.make_empty_intermediate_tensors = (
273
self.model.make_empty_intermediate_tensors)
274
275
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
276
+ return self.model.get_input_embeddings(input_ids)
277
+
278
def forward(
279
self,
280
input_ids: torch.Tensor,
@@ -272,9 +282,11 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
282
kv_caches: List[torch.Tensor],
283
attn_metadata: AttentionMetadata,
284
intermediate_tensors: Optional[IntermediateTensors] = None,
285
+ inputs_embeds: Optional[torch.Tensor] = None,
286
) -> Union[torch.Tensor, IntermediateTensors]:
287
hidden_states = self.model(input_ids, positions, kv_caches,
277
- attn_metadata, intermediate_tensors)
288
+ attn_metadata, intermediate_tensors,
289
+ inputs_embeds)
290
return hidden_states
291
292
def compute_logits(
vllm/model_executor/models/starcoder2.py
+14
-2
@@ -221,6 +221,9 @@ class Starcoder2Model(nn.Module):
221
make_empty_intermediate_tensors_factory(["hidden_states"],
222
config.hidden_size))
223
224
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
225
+ return self.embed_tokens(input_ids)
226
+
227
def forward(
228
self,
229
input_ids: torch.Tensor,
@@ -228,9 +231,13 @@ class Starcoder2Model(nn.Module):
231
kv_caches: List[torch.Tensor],
232
attn_metadata: AttentionMetadata,
233
intermediate_tensors: Optional[IntermediateTensors],
234
+ inputs_embeds: Optional[torch.Tensor] = None,
235
) -> Union[torch.Tensor, IntermediateTensors]:
236
if get_pp_group().is_first_rank:
233
- hidden_states = self.embed_tokens(input_ids)
237
+ if inputs_embeds is not None:
238
+ hidden_states = inputs_embeds
239
+ else:
240
+ hidden_states = self.get_input_embeddings(input_ids)
241
else:
242
assert intermediate_tensors is not None
243
hidden_states = intermediate_tensors["hidden_states"]
@@ -273,6 +280,9 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
280
self.make_empty_intermediate_tensors = (
281
self.model.make_empty_intermediate_tensors)
282
283
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
284
+ return self.model.get_input_embeddings(input_ids)
285
+
286
def forward(
287
self,
288
input_ids: torch.Tensor,
@@ -280,9 +290,11 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
290
kv_caches: List[torch.Tensor],
291
attn_metadata: AttentionMetadata,
292
intermediate_tensors: Optional[IntermediateTensors] = None,
293
+ inputs_embeds: Optional[torch.Tensor] = None,
294
) -> Union[torch.Tensor, IntermediateTensors]:
295
hidden_states = self.model(input_ids, positions, kv_caches,
285
- attn_metadata, intermediate_tensors)
296
+ attn_metadata, intermediate_tensors,
297
+ inputs_embeds)
298
return hidden_states
299
300
def compute_logits(
vllm/model_executor/models/xverse.py
+14
-2
@@ -252,6 +252,9 @@ class XverseModel(nn.Module):
252
make_empty_intermediate_tensors_factory(
253
["hidden_states", "residual"], config.hidden_size))
254
255
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
256
+ return self.embed_tokens(input_ids)
257
+
258
def forward(
259
self,
260
input_ids: torch.Tensor,
@@ -259,9 +262,13 @@ class XverseModel(nn.Module):
262
kv_caches: List[torch.Tensor],
263
attn_metadata: AttentionMetadata,
264
intermediate_tensors: Optional[IntermediateTensors],
265
+ inputs_embeds: Optional[torch.Tensor] = None,
266
) -> Union[torch.Tensor, IntermediateTensors]:
267
if get_pp_group().is_first_rank:
264
- hidden_states = self.embed_tokens(input_ids)
268
+ if inputs_embeds is not None:
269
+ hidden_states = inputs_embeds
270
+ else:
271
+ hidden_states = self.get_input_embeddings(input_ids)
272
residual = None
273
else:
274
hidden_states = intermediate_tensors["hidden_states"]
@@ -335,6 +342,9 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
342
self.make_empty_intermediate_tensors = (
343
self.model.make_empty_intermediate_tensors)
344
345
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
346
+ return self.model.get_input_embeddings(input_ids)
347
+
348
def forward(
349
self,
350
input_ids: torch.Tensor,
@@ -342,9 +352,11 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
352
kv_caches: List[torch.Tensor],
353
attn_metadata: AttentionMetadata,
354
intermediate_tensors: Optional[IntermediateTensors] = None,
355
+ inputs_embeds: Optional[torch.Tensor] = None,
356
) -> Union[torch.Tensor, IntermediateTensors]:
357
hidden_states = self.model(input_ids, positions, kv_caches,
347
- attn_metadata, intermediate_tensors)
358
+ attn_metadata, intermediate_tensors,
359
+ inputs_embeds)
360
return hidden_states
361
362
def compute_logits(