A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix Mistral model (#1220)
Woosuk Kwon committed
Sep 28, 2023 at 10:44 UTC
a8e98aee0c1625312b5477aa15a7cf1dd58aa576
4 files changed
+27
-14
vllm/model_executor/models/mistral.py
+1
-1
@@ -29,7 +29,6 @@ from typing import List, Optional, Tuple
29
30
import torch
31
from torch import nn
32
-from vllm.transformers_utils.configs.mistral import MistralConfig
32
33
from vllm.model_executor.input_metadata import InputMetadata
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -46,6 +45,7 @@ from vllm.model_executor.weight_utils import (
45
convert_pyslice_to_tensor, hf_model_weights_iterator,
46
load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47
from vllm.sequence import SamplerOutput
48
+from vllm.transformers_utils.configs.mistral import MistralConfig
49
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
51
vllm/transformers_utils/config.py
+9
@@ -17,6 +17,15 @@ _CONFIG_REGISTRY = {
17
def get_config(model: str,
18
trust_remote_code: bool,
19
revision: Optional[str] = None) -> PretrainedConfig:
20
+ # NOTE: Because the Mistral model in HF hub does not have
21
+ # `configuration_mistral.py`, we cannot use `AutoConfig` to load the
22
+ # config. Instead, we use `MistralConfig` directly.
23
+ # NOTE: This is a hack. This does not work for local models.
24
+ # FIXME: Remove this once the Mistral model is available in the stable
25
+ # version of HF transformers.
26
+ if "mistral" in model.lower():
27
+ return MistralConfig.from_pretrained(model, revision=revision)
28
+
29
try:
30
config = AutoConfig.from_pretrained(
31
model, trust_remote_code=trust_remote_code, revision=revision)
vllm/transformers_utils/configs/__init__.py
+2
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7
# `FalconConfig` class from the official HuggingFace transformers library.
8
from vllm.transformers_utils.configs.falcon import RWConfig
9
+from vllm.transformers_utils.configs.mistral import MistralConfig
10
11
__all__ = [
12
"MPTConfig",
@@ -13,4 +14,5 @@ __all__ = [
14
"AquilaConfig",
15
"QWenConfig",
16
"RWConfig",
17
+ "MistralConfig",
18
]
vllm/worker/worker.py
+15
-13
@@ -42,6 +42,7 @@ class Worker:
42
# self.init_cache_engine().
43
self.cache_config = None
44
self.block_size = None
45
+ self.sliding_window = None
46
self.cache_engine = None
47
self.cache_events = None
48
self.gpu_cache = None
@@ -136,10 +137,13 @@ class Worker:
137
def init_cache_engine(self, cache_config: CacheConfig) -> None:
138
self.cache_config = cache_config
139
self.block_size = cache_config.block_size
140
+ self.sliding_window = cache_config.sliding_window
141
140
- max_seq_len = min(self.scheduler_config.max_model_len,
141
- cache_config.sliding_window or float("inf"))
142
-
142
+ if self.sliding_window is None:
143
+ max_seq_len = self.scheduler_config.max_model_len
144
+ else:
145
+ max_seq_len = min(self.scheduler_config.max_model_len,
146
+ self.sliding_window)
147
_check_if_can_support_max_seq_len(max_seq_len, self.block_size)
148
149
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
@@ -151,6 +155,8 @@ class Worker:
155
self,
156
seq_group_metadata_list: List[SequenceGroupMetadata],
157
) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
158
+ assert self.block_size is not None
159
+
160
seq_groups: List[Tuple[List[int], SamplingParams]] = []
161
input_tokens: List[int] = []
162
input_positions: List[int] = []
@@ -193,9 +199,6 @@ class Worker:
199
slot = block_number * self.block_size + block_offset
200
slot_mapping.append(slot)
201
196
- sliding_window = getattr(self.model_config.hf_config, "sliding_window",
197
- float("inf"))
198
-
202
# Add generation tokens.
203
max_context_len = 0
204
max_num_blocks_per_seq = 0
@@ -216,8 +219,8 @@ class Worker:
219
220
context_len = seq_data.get_len()
221
position = context_len - 1
219
- if sliding_window:
220
- context_len = min(context_len, sliding_window)
222
+ if self.sliding_window is not None:
223
+ context_len = min(context_len, self.sliding_window)
224
input_positions.append(position)
225
226
block_table = seq_group_metadata.block_tables[seq_id]
@@ -232,10 +235,9 @@ class Worker:
235
slot = block_number * self.block_size + block_offset
236
slot_mapping.append(slot)
237
235
- if sliding_window:
236
- assert self.cache_config is not None
237
- sliding_window_blocks = (sliding_window //
238
- self.cache_config.block_size)
238
+ if self.sliding_window is not None:
239
+ sliding_window_blocks = (self.sliding_window //
240
+ self.block_size)
241
block_table = block_table[-sliding_window_blocks:]
242
generation_block_tables.append(block_table)
243
@@ -277,7 +279,7 @@ class Worker:
279
context_lens=context_lens_tensor,
280
max_context_len=max_context_len,
281
block_tables=block_tables_tensor,
280
- sliding_window=sliding_window,
282
+ sliding_window=self.sliding_window,
283
)
284
return tokens_tensor, positions_tensor, input_metadata
285