@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96e90fdeb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Model] Adds Phi-3 support (#4298)

Caio Mendes committed Apr 25, 2024 at 00:06 UTC 96e90fdeb3c4ebacfe24513556afccb918722b7c
6 files changed +149 -9
README.md
+1
@@ -78,6 +78,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
78 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
79 - Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
80 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
81 +- Phi3 (`microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, etc.)
82 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
83 - Qwen2 (`Qwen/Qwen1.5-7B`, `Qwen/Qwen1.5-7B-Chat`, etc.)
84 - Qwen2MoE (`Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc.)
docs/source/models/supported_models.rst
+4
@@ -115,6 +115,10 @@ Alongside each architecture, we include some popular models that use it.
115 - Phi
116 - :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
117 -
118 + * - :code:`Phi3ForCausalLM`
119 + - Phi-3
120 + - :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
121 + -
122 * - :code:`QWenLMHeadModel`
123 - Qwen
124 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
vllm/config.py
+1 -1
@@ -1056,7 +1056,7 @@ def _get_and_verify_max_len(
1056 derived_max_model_len = default_max_len
1057
1058 rope_scaling = getattr(hf_config, "rope_scaling", None)
1059 - if rope_scaling is not None:
1059 + if rope_scaling is not None and rope_scaling["type"] != "su":
1060 assert "factor" in rope_scaling
1061 scaling_factor = rope_scaling["factor"]
1062 if rope_scaling["type"] == "yarn":
vllm/model_executor/layers/rotary_embedding.py
+133 -3
@@ -338,6 +338,114 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
338 return cache
339
340
341 +class Phi3SuScaledRotaryEmbedding(nn.Module):
342 + """Phi3 family of models scaled rotary embedding.
343 +
344 + Based on the original RotaryEmbedding implementation.
345 + """
346 +
347 + def __init__(
348 + self,
349 + head_size: int,
350 + rotary_dim: int,
351 + max_position_embeddings: int,
352 + original_max_position_embeddings: int,
353 + base: int,
354 + is_neox_style: bool,
355 + short_factor: List[float],
356 + long_factor: List[float],
357 + short_mscale: float = 1.1,
358 + long_mscale: float = 1.225,
359 + ):
360 + super().__init__()
361 +
362 + if rotary_dim != head_size:
363 + raise ValueError(
364 + f"`Phi3SuScaledRotaryEmbedding` does not support rotary_dim != \
365 + head_size ({rotary_dim}!={head_size}).")
366 + if is_neox_style is False:
367 + raise ValueError(
368 + "`Phi3SuScaledRotaryEmbedding` only supports neox_style.")
369 +
370 + self.head_size = head_size
371 + self.max_position_embeddings = max_position_embeddings
372 + self.original_max_position_embeddings = original_max_position_embeddings
373 + self.base = base
374 + self.short_factor = short_factor
375 + self.long_factor = long_factor
376 + self.short_mscale = short_mscale
377 + self.long_mscale = long_mscale
378 +
379 + short_cache = self._compute_cos_sin_cache(
380 + original_max_position_embeddings, short_factor, short_mscale)
381 + short_cache = short_cache.to(torch.get_default_dtype())
382 + self.register_buffer("short_cos_sin_cache",
383 + short_cache,
384 + persistent=False)
385 +
386 + long_cache = self._compute_cos_sin_cache(max_position_embeddings,
387 + long_factor, long_mscale)
388 + long_cache = long_cache.to(torch.get_default_dtype())
389 + self.register_buffer("long_cos_sin_cache",
390 + long_cache,
391 + persistent=False)
392 +
393 + long_short_cache = torch.cat(
394 + [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0)
395 + self.register_buffer("long_short_cos_sin_cache",
396 + long_short_cache,
397 + persistent=False)
398 +
399 + def _compute_inv_freq(self, rescale_factors: List[float]) -> torch.Tensor:
400 + rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32)
401 + inv_freq = 1.0 / (rescale_factors * (self.base**(torch.arange(
402 + 0, self.head_size, 2, dtype=torch.float) / self.head_size)))
403 + return inv_freq
404 +
405 + def _compute_cos_sin_cache(
406 + self,
407 + max_position_embeddings: int,
408 + rescale_factors: List[float],
409 + mscale: float,
410 + ) -> torch.Tensor:
411 + inv_freq = self._compute_inv_freq(rescale_factors)
412 + t = torch.arange(max_position_embeddings, dtype=torch.float)
413 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
414 + cos = freqs.cos() * mscale
415 + sin = freqs.sin() * mscale
416 + cache = torch.cat((cos, sin), dim=-1)
417 + return cache
418 +
419 + def forward(
420 + self,
421 + positions: torch.Tensor,
422 + query: torch.Tensor,
423 + key: torch.Tensor,
424 + offsets: Optional[torch.Tensor] = None,
425 + ) -> Tuple[torch.Tensor, torch.Tensor]:
426 + query = query.view(*query.shape[:-1], -1, self.head_size)
427 + key = key.view(*key.shape[:-1], -1, self.head_size)
428 +
429 + k = self.original_max_position_embeddings
430 + long_prompt_offset = (torch.any(positions > k).float() *
431 + torch.full_like(positions, k)).long()
432 + idx = (torch.add(positions, long_prompt_offset)
433 + if long_prompt_offset is not None else positions)
434 + self.long_short_cos_sin_cache = self.long_short_cos_sin_cache.to(
435 + idx.device)
436 + idx = torch.add(idx, offsets) if offsets is not None else idx
437 + cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
438 +
439 + cos, sin = cos_sin.chunk(2, dim=-1)
440 + cos = cos.repeat(1, 2).unsqueeze(-2)
441 + sin = sin.repeat(1, 2).unsqueeze(-2)
442 +
443 + query = query * cos + _rotate_neox(query) * sin
444 + key = key * cos + _rotate_neox(key) * sin
445 +
446 + return query.flatten(-2), key.flatten(-2)
447 +
448 +
449 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
450
451
@@ -349,17 +457,26 @@ def get_rope(
457 is_neox_style: bool = True,
458 rope_scaling: Optional[Dict[str, Any]] = None,
459 ) -> RotaryEmbedding:
460 + if rope_scaling is not None:
461 + # Transforms every value that is a list into a tuple for caching calls
462 + rope_scaling_tuple = {
463 + k: tuple(v) if isinstance(v, list) else v
464 + for k, v in rope_scaling.items()
465 + }
466 + rope_scaling_args = tuple(rope_scaling_tuple.items())
467 + else:
468 + rope_scaling_args = None
469 key = (head_size, rotary_dim, max_position, base, is_neox_style,
353 - tuple(rope_scaling.items()) if rope_scaling is not None else None)
470 + rope_scaling_args)
471 if key in _ROPE_DICT:
472 return _ROPE_DICT[key]
356 -
473 if rope_scaling is None:
474 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
475 is_neox_style)
476 else:
477 scaling_type = rope_scaling["type"]
362 - scaling_factor = rope_scaling["factor"]
478 + if scaling_type != "su":
479 + scaling_factor = rope_scaling["factor"]
480 if scaling_type == "linear":
481 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
482 max_position, base,
@@ -383,6 +500,19 @@ def get_rope(
500 base, is_neox_style,
501 scaling_factor,
502 **extra_kwargs)
503 + elif scaling_type == "su":
504 + short_factor = rope_scaling["short_factor"]
505 + long_factor = rope_scaling["long_factor"]
506 + original_max_position = rope_scaling[
507 + "original_max_position_embeddings"]
508 + extra_kwargs = {
509 + k: v
510 + for k, v in rope_scaling.items()
511 + if k in ("short_mscale", "long_mscale")
512 + }
513 + rotary_emb = Phi3SuScaledRotaryEmbedding(
514 + head_size, rotary_dim, max_position, original_max_position,
515 + base, is_neox_style, short_factor, long_factor, **extra_kwargs)
516 else:
517 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
518 _ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/__init__.py
+1
@@ -46,6 +46,7 @@ _MODELS = {
46 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
47 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
48 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
49 + "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
50 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
51 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
52 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/llama.py
+9 -5
@@ -180,6 +180,10 @@ class LlamaDecoderLayer(nn.Module):
180 self.hidden_size = config.hidden_size
181 rope_theta = getattr(config, "rope_theta", 10000)
182 rope_scaling = getattr(config, "rope_scaling", None)
183 + if rope_scaling is not None and getattr(
184 + config, "original_max_position_embeddings", None):
185 + rope_scaling["original_max_position_embeddings"] = (
186 + config.original_max_position_embeddings)
187 max_position_embeddings = getattr(config, "max_position_embeddings",
188 8192)
189 sliding_window = getattr(config, "sliding_window", None)
@@ -378,11 +382,11 @@ class LlamaForCausalLM(nn.Module):
382 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
383 stacked_params_mapping = [
384 # (param_name, shard_name, shard_id)
381 - ("qkv_proj", "q_proj", "q"),
382 - ("qkv_proj", "k_proj", "k"),
383 - ("qkv_proj", "v_proj", "v"),
384 - ("gate_up_proj", "gate_proj", 0),
385 - ("gate_up_proj", "up_proj", 1),
385 + (".qkv_proj", ".q_proj", "q"),
386 + (".qkv_proj", ".k_proj", "k"),
387 + (".qkv_proj", ".v_proj", "v"),
388 + (".gate_up_proj", ".gate_proj", 0),
389 + (".gate_up_proj", ".up_proj", 1),
390 ]
391 params_dict = dict(self.named_parameters())
392 for name, loaded_weight in weights: