A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support Mistral Model Inference with transformers-neuronx (#3153)
DAIZHENWEI committed
Mar 11, 2024 at 13:19 UTC
654865e21df8ac6fe95de926625306e5756c2c0d
3 files changed
+93
-6
examples/offline_inference_neuron.py
+6
-4
@@ -14,14 +14,16 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
14
llm = LLM(
15
model="openlm-research/open_llama_3b",
16
max_num_seqs=8,
17
- # The max_model_len and block_size arguments are required to be same as max sequence length,
18
- # when targeting neuron device. Currently, this is a known limitation in continuous batching
19
- # support in transformers-neuronx.
17
+ # The max_model_len and block_size arguments are required to be same as
18
+ # max sequence length when targeting neuron device.
19
+ # Currently, this is a known limitation in continuous batching support
20
+ # in transformers-neuronx.
21
# TODO(liangfu): Support paged-attention in transformers-neuronx.
22
max_model_len=128,
23
block_size=128,
24
# The device can be automatically detected when AWS Neuron SDK is installed.
24
- # The device argument can be either unspecified for automated detection, or explicitly assigned.
25
+ # The device argument can be either unspecified for automated detection,
26
+ # or explicitly assigned.
27
device="neuron")
28
# Generate texts from the prompts. The output is a list of RequestOutput objects
29
# that contain the prompt, generated text, and other information.
vllm/model_executor/models/__init__.py
+5
-2
@@ -62,8 +62,11 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
62
"Sliding window attention is not yet supported in ROCm's flash attention",
63
}
64
65
-# Models not supported by Neuron.
66
-_NEURON_SUPPORTED_MODELS = {"LlamaForCausalLM": "neuron.llama"}
65
+# Models supported by Neuron.
66
+_NEURON_SUPPORTED_MODELS = {
67
+ "LlamaForCausalLM": "neuron.llama",
68
+ "MistralForCausalLM": "neuron.mistral"
69
+}
70
71
72
class ModelRegistry:
vllm/model_executor/models/neuron/mistral.py
new
+82
@@ -0,0 +1,82 @@
1
+"""Inference-only Mistral model compatible with HuggingFace weights."""
2
+from typing import List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+from transformers import MistralConfig
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.sampler import Sampler
10
+from vllm.model_executor.sampling_metadata import SamplingMetadata
11
+from vllm.sequence import SamplerOutput
12
+import os
13
+
14
+KVCache = Tuple[torch.Tensor, torch.Tensor]
15
+
16
+
17
+class MistralForCausalLM(nn.Module):
18
+
19
+ def __init__(
20
+ self,
21
+ config: MistralConfig,
22
+ linear_method=None,
23
+ ) -> None:
24
+ super().__init__()
25
+ self.config = config
26
+ self.linear_method = linear_method
27
+ self.model = None
28
+ self.lm_head = None
29
+ self.sampler = Sampler(config.vocab_size)
30
+
31
+ def forward(
32
+ self,
33
+ input_ids: torch.Tensor,
34
+ positions: torch.Tensor,
35
+ kv_caches: List[KVCache],
36
+ input_metadata: InputMetadata,
37
+ ) -> SamplerOutput:
38
+ with torch.inference_mode():
39
+ seq_ids = []
40
+ block_size = self.model.context_buckets[-1]
41
+ if input_metadata.is_prompt:
42
+ seq_ids = input_metadata.slot_mapping[:, 0] // block_size
43
+ else:
44
+ seq_ids = input_metadata.block_tables
45
+
46
+ logits = self.model(input_ids,
47
+ cache_ids=positions,
48
+ start_ids=seq_ids)
49
+ return logits
50
+
51
+ def sample(
52
+ self,
53
+ hidden_states: torch.Tensor,
54
+ sampling_metadata: SamplingMetadata,
55
+ ) -> Optional[SamplerOutput]:
56
+ next_tokens = self.sampler(self.model.chkpt_model.lm_head,
57
+ hidden_states, sampling_metadata)
58
+ return next_tokens
59
+
60
+ def load_weights(self,
61
+ model_name_or_path: str,
62
+ cache_dir: Optional[str] = None,
63
+ load_format: str = "auto",
64
+ revision: Optional[str] = None,
65
+ **kwargs):
66
+ from transformers_neuronx.mistral.model import MistralForSampling
67
+
68
+ split_model_dir = f"{model_name_or_path}-split"
69
+ if os.path.isdir(os.path.join(model_name_or_path,
70
+ "pytorch_model.bin")):
71
+ split_model_dir = model_name_or_path
72
+ elif not os.path.exists(f"{model_name_or_path}-split"):
73
+ from transformers import MistralForCausalLM
74
+ from transformers_neuronx.module import save_pretrained_split
75
+
76
+ hf_model = MistralForCausalLM.from_pretrained(
77
+ model_name_or_path, low_cpu_mem_usage=True)
78
+ save_pretrained_split(hf_model, f"{model_name_or_path}-split")
79
+
80
+ self.model = MistralForSampling.from_pretrained(
81
+ split_model_dir, **kwargs)
82
+ self.model.to_neuron()