@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a8e98aee0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix Mistral model (#1220)

Woosuk Kwon committed Sep 28, 2023 at 10:44 UTC a8e98aee0c1625312b5477aa15a7cf1dd58aa576
4 files changed +27 -14
vllm/model_executor/models/mistral.py
+1 -1
@@ -29,7 +29,6 @@ from typing import List, Optional, Tuple
29
30 import torch
31 from torch import nn
32 -from vllm.transformers_utils.configs.mistral import MistralConfig
32
33 from vllm.model_executor.input_metadata import InputMetadata
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -46,6 +45,7 @@ from vllm.model_executor.weight_utils import (
45 convert_pyslice_to_tensor, hf_model_weights_iterator,
46 load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47 from vllm.sequence import SamplerOutput
48 +from vllm.transformers_utils.configs.mistral import MistralConfig
49
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
51
vllm/transformers_utils/config.py
+9
@@ -17,6 +17,15 @@ _CONFIG_REGISTRY = {
17 def get_config(model: str,
18 trust_remote_code: bool,
19 revision: Optional[str] = None) -> PretrainedConfig:
20 + # NOTE: Because the Mistral model in HF hub does not have
21 + # `configuration_mistral.py`, we cannot use `AutoConfig` to load the
22 + # config. Instead, we use `MistralConfig` directly.
23 + # NOTE: This is a hack. This does not work for local models.
24 + # FIXME: Remove this once the Mistral model is available in the stable
25 + # version of HF transformers.
26 + if "mistral" in model.lower():
27 + return MistralConfig.from_pretrained(model, revision=revision)
28 +
29 try:
30 config = AutoConfig.from_pretrained(
31 model, trust_remote_code=trust_remote_code, revision=revision)
vllm/transformers_utils/configs/__init__.py
+2
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7 # `FalconConfig` class from the official HuggingFace transformers library.
8 from vllm.transformers_utils.configs.falcon import RWConfig
9 +from vllm.transformers_utils.configs.mistral import MistralConfig
10
11 __all__ = [
12 "MPTConfig",
@@ -13,4 +14,5 @@ __all__ = [
14 "AquilaConfig",
15 "QWenConfig",
16 "RWConfig",
17 + "MistralConfig",
18 ]
vllm/worker/worker.py
+15 -13
@@ -42,6 +42,7 @@ class Worker:
42 # self.init_cache_engine().
43 self.cache_config = None
44 self.block_size = None
45 + self.sliding_window = None
46 self.cache_engine = None
47 self.cache_events = None
48 self.gpu_cache = None
@@ -136,10 +137,13 @@ class Worker:
137 def init_cache_engine(self, cache_config: CacheConfig) -> None:
138 self.cache_config = cache_config
139 self.block_size = cache_config.block_size
140 + self.sliding_window = cache_config.sliding_window
141
140 - max_seq_len = min(self.scheduler_config.max_model_len,
141 - cache_config.sliding_window or float("inf"))
142 -
142 + if self.sliding_window is None:
143 + max_seq_len = self.scheduler_config.max_model_len
144 + else:
145 + max_seq_len = min(self.scheduler_config.max_model_len,
146 + self.sliding_window)
147 _check_if_can_support_max_seq_len(max_seq_len, self.block_size)
148
149 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
@@ -151,6 +155,8 @@ class Worker:
155 self,
156 seq_group_metadata_list: List[SequenceGroupMetadata],
157 ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
158 + assert self.block_size is not None
159 +
160 seq_groups: List[Tuple[List[int], SamplingParams]] = []
161 input_tokens: List[int] = []
162 input_positions: List[int] = []
@@ -193,9 +199,6 @@ class Worker:
199 slot = block_number * self.block_size + block_offset
200 slot_mapping.append(slot)
201
196 - sliding_window = getattr(self.model_config.hf_config, "sliding_window",
197 - float("inf"))
198 -
202 # Add generation tokens.
203 max_context_len = 0
204 max_num_blocks_per_seq = 0
@@ -216,8 +219,8 @@ class Worker:
219
220 context_len = seq_data.get_len()
221 position = context_len - 1
219 - if sliding_window:
220 - context_len = min(context_len, sliding_window)
222 + if self.sliding_window is not None:
223 + context_len = min(context_len, self.sliding_window)
224 input_positions.append(position)
225
226 block_table = seq_group_metadata.block_tables[seq_id]
@@ -232,10 +235,9 @@ class Worker:
235 slot = block_number * self.block_size + block_offset
236 slot_mapping.append(slot)
237
235 - if sliding_window:
236 - assert self.cache_config is not None
237 - sliding_window_blocks = (sliding_window //
238 - self.cache_config.block_size)
238 + if self.sliding_window is not None:
239 + sliding_window_blocks = (self.sliding_window //
240 + self.block_size)
241 block_table = block_table[-sliding_window_blocks:]
242 generation_block_tables.append(block_table)
243
@@ -277,7 +279,7 @@ class Worker:
279 context_lens=context_lens_tensor,
280 max_context_len=max_context_len,
281 block_tables=block_tables_tensor,
280 - sliding_window=sliding_window,
282 + sliding_window=self.sliding_window,
283 )
284 return tokens_tensor, positions_tensor, input_metadata
285