A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Model] Adds Phi-3 support (#4298)
Caio Mendes committed
Apr 25, 2024 at 00:06 UTC
96e90fdeb3c4ebacfe24513556afccb918722b7c
6 files changed
+149
-9
README.md
+1
@@ -78,6 +78,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
78
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
79
- Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
80
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
81
+- Phi3 (`microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, etc.)
82
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
83
- Qwen2 (`Qwen/Qwen1.5-7B`, `Qwen/Qwen1.5-7B-Chat`, etc.)
84
- Qwen2MoE (`Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc.)
docs/source/models/supported_models.rst
+4
@@ -115,6 +115,10 @@ Alongside each architecture, we include some popular models that use it.
115
- Phi
116
- :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
117
-
118
+ * - :code:`Phi3ForCausalLM`
119
+ - Phi-3
120
+ - :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
121
+ -
122
* - :code:`QWenLMHeadModel`
123
- Qwen
124
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
vllm/config.py
+1
-1
@@ -1056,7 +1056,7 @@ def _get_and_verify_max_len(
1056
derived_max_model_len = default_max_len
1057
1058
rope_scaling = getattr(hf_config, "rope_scaling", None)
1059
- if rope_scaling is not None:
1059
+ if rope_scaling is not None and rope_scaling["type"] != "su":
1060
assert "factor" in rope_scaling
1061
scaling_factor = rope_scaling["factor"]
1062
if rope_scaling["type"] == "yarn":
vllm/model_executor/layers/rotary_embedding.py
+133
-3
@@ -338,6 +338,114 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
338
return cache
339
340
341
+class Phi3SuScaledRotaryEmbedding(nn.Module):
342
+ """Phi3 family of models scaled rotary embedding.
343
+
344
+ Based on the original RotaryEmbedding implementation.
345
+ """
346
+
347
+ def __init__(
348
+ self,
349
+ head_size: int,
350
+ rotary_dim: int,
351
+ max_position_embeddings: int,
352
+ original_max_position_embeddings: int,
353
+ base: int,
354
+ is_neox_style: bool,
355
+ short_factor: List[float],
356
+ long_factor: List[float],
357
+ short_mscale: float = 1.1,
358
+ long_mscale: float = 1.225,
359
+ ):
360
+ super().__init__()
361
+
362
+ if rotary_dim != head_size:
363
+ raise ValueError(
364
+ f"`Phi3SuScaledRotaryEmbedding` does not support rotary_dim != \
365
+ head_size ({rotary_dim}!={head_size}).")
366
+ if is_neox_style is False:
367
+ raise ValueError(
368
+ "`Phi3SuScaledRotaryEmbedding` only supports neox_style.")
369
+
370
+ self.head_size = head_size
371
+ self.max_position_embeddings = max_position_embeddings
372
+ self.original_max_position_embeddings = original_max_position_embeddings
373
+ self.base = base
374
+ self.short_factor = short_factor
375
+ self.long_factor = long_factor
376
+ self.short_mscale = short_mscale
377
+ self.long_mscale = long_mscale
378
+
379
+ short_cache = self._compute_cos_sin_cache(
380
+ original_max_position_embeddings, short_factor, short_mscale)
381
+ short_cache = short_cache.to(torch.get_default_dtype())
382
+ self.register_buffer("short_cos_sin_cache",
383
+ short_cache,
384
+ persistent=False)
385
+
386
+ long_cache = self._compute_cos_sin_cache(max_position_embeddings,
387
+ long_factor, long_mscale)
388
+ long_cache = long_cache.to(torch.get_default_dtype())
389
+ self.register_buffer("long_cos_sin_cache",
390
+ long_cache,
391
+ persistent=False)
392
+
393
+ long_short_cache = torch.cat(
394
+ [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0)
395
+ self.register_buffer("long_short_cos_sin_cache",
396
+ long_short_cache,
397
+ persistent=False)
398
+
399
+ def _compute_inv_freq(self, rescale_factors: List[float]) -> torch.Tensor:
400
+ rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32)
401
+ inv_freq = 1.0 / (rescale_factors * (self.base**(torch.arange(
402
+ 0, self.head_size, 2, dtype=torch.float) / self.head_size)))
403
+ return inv_freq
404
+
405
+ def _compute_cos_sin_cache(
406
+ self,
407
+ max_position_embeddings: int,
408
+ rescale_factors: List[float],
409
+ mscale: float,
410
+ ) -> torch.Tensor:
411
+ inv_freq = self._compute_inv_freq(rescale_factors)
412
+ t = torch.arange(max_position_embeddings, dtype=torch.float)
413
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
414
+ cos = freqs.cos() * mscale
415
+ sin = freqs.sin() * mscale
416
+ cache = torch.cat((cos, sin), dim=-1)
417
+ return cache
418
+
419
+ def forward(
420
+ self,
421
+ positions: torch.Tensor,
422
+ query: torch.Tensor,
423
+ key: torch.Tensor,
424
+ offsets: Optional[torch.Tensor] = None,
425
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
426
+ query = query.view(*query.shape[:-1], -1, self.head_size)
427
+ key = key.view(*key.shape[:-1], -1, self.head_size)
428
+
429
+ k = self.original_max_position_embeddings
430
+ long_prompt_offset = (torch.any(positions > k).float() *
431
+ torch.full_like(positions, k)).long()
432
+ idx = (torch.add(positions, long_prompt_offset)
433
+ if long_prompt_offset is not None else positions)
434
+ self.long_short_cos_sin_cache = self.long_short_cos_sin_cache.to(
435
+ idx.device)
436
+ idx = torch.add(idx, offsets) if offsets is not None else idx
437
+ cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
438
+
439
+ cos, sin = cos_sin.chunk(2, dim=-1)
440
+ cos = cos.repeat(1, 2).unsqueeze(-2)
441
+ sin = sin.repeat(1, 2).unsqueeze(-2)
442
+
443
+ query = query * cos + _rotate_neox(query) * sin
444
+ key = key * cos + _rotate_neox(key) * sin
445
+
446
+ return query.flatten(-2), key.flatten(-2)
447
+
448
+
449
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
450
451
@@ -349,17 +457,26 @@ def get_rope(
457
is_neox_style: bool = True,
458
rope_scaling: Optional[Dict[str, Any]] = None,
459
) -> RotaryEmbedding:
460
+ if rope_scaling is not None:
461
+ # Transforms every value that is a list into a tuple for caching calls
462
+ rope_scaling_tuple = {
463
+ k: tuple(v) if isinstance(v, list) else v
464
+ for k, v in rope_scaling.items()
465
+ }
466
+ rope_scaling_args = tuple(rope_scaling_tuple.items())
467
+ else:
468
+ rope_scaling_args = None
469
key = (head_size, rotary_dim, max_position, base, is_neox_style,
353
- tuple(rope_scaling.items()) if rope_scaling is not None else None)
470
+ rope_scaling_args)
471
if key in _ROPE_DICT:
472
return _ROPE_DICT[key]
356
-
473
if rope_scaling is None:
474
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
475
is_neox_style)
476
else:
477
scaling_type = rope_scaling["type"]
362
- scaling_factor = rope_scaling["factor"]
478
+ if scaling_type != "su":
479
+ scaling_factor = rope_scaling["factor"]
480
if scaling_type == "linear":
481
rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
482
max_position, base,
@@ -383,6 +500,19 @@ def get_rope(
500
base, is_neox_style,
501
scaling_factor,
502
**extra_kwargs)
503
+ elif scaling_type == "su":
504
+ short_factor = rope_scaling["short_factor"]
505
+ long_factor = rope_scaling["long_factor"]
506
+ original_max_position = rope_scaling[
507
+ "original_max_position_embeddings"]
508
+ extra_kwargs = {
509
+ k: v
510
+ for k, v in rope_scaling.items()
511
+ if k in ("short_mscale", "long_mscale")
512
+ }
513
+ rotary_emb = Phi3SuScaledRotaryEmbedding(
514
+ head_size, rotary_dim, max_position, original_max_position,
515
+ base, is_neox_style, short_factor, long_factor, **extra_kwargs)
516
else:
517
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
518
_ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/__init__.py
+1
@@ -46,6 +46,7 @@ _MODELS = {
46
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
47
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
48
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
49
+ "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
50
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
51
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
52
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/llama.py
+9
-5
@@ -180,6 +180,10 @@ class LlamaDecoderLayer(nn.Module):
180
self.hidden_size = config.hidden_size
181
rope_theta = getattr(config, "rope_theta", 10000)
182
rope_scaling = getattr(config, "rope_scaling", None)
183
+ if rope_scaling is not None and getattr(
184
+ config, "original_max_position_embeddings", None):
185
+ rope_scaling["original_max_position_embeddings"] = (
186
+ config.original_max_position_embeddings)
187
max_position_embeddings = getattr(config, "max_position_embeddings",
188
8192)
189
sliding_window = getattr(config, "sliding_window", None)
@@ -378,11 +382,11 @@ class LlamaForCausalLM(nn.Module):
382
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
383
stacked_params_mapping = [
384
# (param_name, shard_name, shard_id)
381
- ("qkv_proj", "q_proj", "q"),
382
- ("qkv_proj", "k_proj", "k"),
383
- ("qkv_proj", "v_proj", "v"),
384
- ("gate_up_proj", "gate_proj", 0),
385
- ("gate_up_proj", "up_proj", 1),
385
+ (".qkv_proj", ".q_proj", "q"),
386
+ (".qkv_proj", ".k_proj", "k"),
387
+ (".qkv_proj", ".v_proj", "v"),
388
+ (".gate_up_proj", ".gate_proj", 0),
389
+ (".gate_up_proj", ".up_proj", 1),
390
]
391
params_dict = dict(self.named_parameters())
392
for name, loaded_weight in weights: