@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 654865e21d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Support Mistral Model Inference with transformers-neuronx (#3153)

DAIZHENWEI committed Mar 11, 2024 at 13:19 UTC 654865e21df8ac6fe95de926625306e5756c2c0d
3 files changed +93 -6
examples/offline_inference_neuron.py
+6 -4
@@ -14,14 +14,16 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
14 llm = LLM(
15 model="openlm-research/open_llama_3b",
16 max_num_seqs=8,
17 - # The max_model_len and block_size arguments are required to be same as max sequence length,
18 - # when targeting neuron device. Currently, this is a known limitation in continuous batching
19 - # support in transformers-neuronx.
17 + # The max_model_len and block_size arguments are required to be same as
18 + # max sequence length when targeting neuron device.
19 + # Currently, this is a known limitation in continuous batching support
20 + # in transformers-neuronx.
21 # TODO(liangfu): Support paged-attention in transformers-neuronx.
22 max_model_len=128,
23 block_size=128,
24 # The device can be automatically detected when AWS Neuron SDK is installed.
24 - # The device argument can be either unspecified for automated detection, or explicitly assigned.
25 + # The device argument can be either unspecified for automated detection,
26 + # or explicitly assigned.
27 device="neuron")
28 # Generate texts from the prompts. The output is a list of RequestOutput objects
29 # that contain the prompt, generated text, and other information.
vllm/model_executor/models/__init__.py
+5 -2
@@ -62,8 +62,11 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
62 "Sliding window attention is not yet supported in ROCm's flash attention",
63 }
64
65 -# Models not supported by Neuron.
66 -_NEURON_SUPPORTED_MODELS = {"LlamaForCausalLM": "neuron.llama"}
65 +# Models supported by Neuron.
66 +_NEURON_SUPPORTED_MODELS = {
67 + "LlamaForCausalLM": "neuron.llama",
68 + "MistralForCausalLM": "neuron.mistral"
69 +}
70
71
72 class ModelRegistry:
vllm/model_executor/models/neuron/mistral.py new
+82
@@ -0,0 +1,82 @@
1 +"""Inference-only Mistral model compatible with HuggingFace weights."""
2 +from typing import List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import MistralConfig
7 +
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.sampler import Sampler
10 +from vllm.model_executor.sampling_metadata import SamplingMetadata
11 +from vllm.sequence import SamplerOutput
12 +import os
13 +
14 +KVCache = Tuple[torch.Tensor, torch.Tensor]
15 +
16 +
17 +class MistralForCausalLM(nn.Module):
18 +
19 + def __init__(
20 + self,
21 + config: MistralConfig,
22 + linear_method=None,
23 + ) -> None:
24 + super().__init__()
25 + self.config = config
26 + self.linear_method = linear_method
27 + self.model = None
28 + self.lm_head = None
29 + self.sampler = Sampler(config.vocab_size)
30 +
31 + def forward(
32 + self,
33 + input_ids: torch.Tensor,
34 + positions: torch.Tensor,
35 + kv_caches: List[KVCache],
36 + input_metadata: InputMetadata,
37 + ) -> SamplerOutput:
38 + with torch.inference_mode():
39 + seq_ids = []
40 + block_size = self.model.context_buckets[-1]
41 + if input_metadata.is_prompt:
42 + seq_ids = input_metadata.slot_mapping[:, 0] // block_size
43 + else:
44 + seq_ids = input_metadata.block_tables
45 +
46 + logits = self.model(input_ids,
47 + cache_ids=positions,
48 + start_ids=seq_ids)
49 + return logits
50 +
51 + def sample(
52 + self,
53 + hidden_states: torch.Tensor,
54 + sampling_metadata: SamplingMetadata,
55 + ) -> Optional[SamplerOutput]:
56 + next_tokens = self.sampler(self.model.chkpt_model.lm_head,
57 + hidden_states, sampling_metadata)
58 + return next_tokens
59 +
60 + def load_weights(self,
61 + model_name_or_path: str,
62 + cache_dir: Optional[str] = None,
63 + load_format: str = "auto",
64 + revision: Optional[str] = None,
65 + **kwargs):
66 + from transformers_neuronx.mistral.model import MistralForSampling
67 +
68 + split_model_dir = f"{model_name_or_path}-split"
69 + if os.path.isdir(os.path.join(model_name_or_path,
70 + "pytorch_model.bin")):
71 + split_model_dir = model_name_or_path
72 + elif not os.path.exists(f"{model_name_or_path}-split"):
73 + from transformers import MistralForCausalLM
74 + from transformers_neuronx.module import save_pretrained_split
75 +
76 + hf_model = MistralForCausalLM.from_pretrained(
77 + model_name_or_path, low_cpu_mem_usage=True)
78 + save_pretrained_split(hf_model, f"{model_name_or_path}-split")
79 +
80 + self.model = MistralForSampling.from_pretrained(
81 + split_model_dir, **kwargs)
82 + self.model.to_neuron()