A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Set block size at initialization & Fix test_model_runner (#4705)
Woosuk Kwon committed
May 9, 2024 at 09:04 UTC
0ee535b2945d042cbb1fc6e63fd3fddd94d491f2
5 files changed
+64
-104
tests/worker/test_model_runner.py
+32
-58
@@ -1,27 +1,38 @@
1
import pytest
2
import torch
3
4
-from vllm.config import ModelConfig, SchedulerConfig
4
from vllm.distributed.parallel_state import init_distributed_environment
5
+from vllm.engine.arg_utils import EngineArgs
6
from vllm.model_executor.sampling_metadata import SamplingMetadata
7
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8
from vllm.utils import get_open_port
9
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
10
11
12
+def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
13
+ engine_args = EngineArgs(model, *args, **kwargs)
14
+ engine_config = engine_args.create_engine_config()
15
+ model_runner = ModelRunner(
16
+ model_config=engine_config.model_config,
17
+ parallel_config=engine_config.parallel_config,
18
+ scheduler_config=engine_config.scheduler_config,
19
+ device_config=engine_config.device_config,
20
+ cache_config=engine_config.cache_config,
21
+ load_config=engine_config.load_config,
22
+ lora_config=engine_config.lora_config,
23
+ is_driver_worker=True,
24
+ )
25
+ return model_runner
26
+
27
+
28
@pytest.mark.parametrize("batch_size", list(range(1, 257)))
29
def test_prepare_prompt(batch_size):
14
- scheduler_config = SchedulerConfig(100000,
15
- 100000,
16
- 100000,
17
- enable_chunked_prefill=False)
18
- model_runner = ModelRunner(model_config=None,
19
- parallel_config=None,
20
- scheduler_config=scheduler_config,
21
- device_config=None,
22
- load_config=None,
23
- lora_config=None)
24
- model_runner.set_block_size(16)
30
+ model_runner = _create_model_runner(
31
+ "facebook/opt-125m",
32
+ max_num_batched_tokens=100000,
33
+ max_num_seqs=100000,
34
+ enable_chunked_prefill=False,
35
+ )
36
37
seq_lens = []
38
seq_group_metadata_list = []
@@ -123,27 +134,15 @@ def test_prepare_prompt(batch_size):
134
135
@pytest.mark.parametrize("batch_size", list(range(1, 257)))
136
def test_prepare_decode_cuda_graph(batch_size):
126
- model_config = ModelConfig(
137
+ model_runner = _create_model_runner(
138
"facebook/opt-125m",
128
- "facebook/opt-125m",
129
- tokenizer_mode="auto",
130
- trust_remote_code=False,
139
seed=0,
140
dtype="float16",
133
- revision=None,
141
enforce_eager=False,
142
+ max_num_batched_tokens=100000,
143
+ max_num_seqs=100000,
144
+ enable_chunked_prefill=False,
145
)
136
- scheduler_config = SchedulerConfig(100000,
137
- 100000,
138
- 100000,
139
- enable_chunked_prefill=False)
140
- model_runner = ModelRunner(model_config=model_config,
141
- parallel_config=None,
142
- scheduler_config=scheduler_config,
143
- device_config=None,
144
- load_config=None,
145
- lora_config=None)
146
- model_runner.set_block_size(16)
146
147
seq_lens = []
148
seq_group_metadata_list = []
@@ -214,23 +213,12 @@ def test_prepare_decode_cuda_graph(batch_size):
213
214
def test_empty_seq_group():
215
"""Verify prepare prompt and decode returns empty output."""
217
- model_config = ModelConfig(
218
- "facebook/opt-125m",
216
+ model_runner = _create_model_runner(
217
"facebook/opt-125m",
220
- tokenizer_mode="auto",
221
- trust_remote_code=False,
218
seed=0,
219
dtype="float16",
224
- revision=None,
220
enforce_eager=False,
221
)
227
- model_runner = ModelRunner(model_config=model_config,
228
- parallel_config=None,
229
- scheduler_config=None,
230
- device_config=None,
231
- load_config=None,
232
- lora_config=None)
233
- model_runner.set_block_size(16)
222
seq_group_metadata_list = []
223
input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
224
model_runner._prepare_decode(seq_group_metadata_list))
@@ -260,29 +248,15 @@ def distributed_init():
248
@pytest.mark.parametrize("batch_size", list(range(2, 128)))
249
@pytest.mark.parametrize("enforce_eager", [True, False])
250
def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
263
-
264
- model_config = ModelConfig(
265
- "facebook/opt-125m",
251
+ model_runner = _create_model_runner(
252
"facebook/opt-125m",
267
- tokenizer_mode="auto",
268
- trust_remote_code=False,
253
seed=0,
254
dtype="float16",
271
- revision=None,
255
enforce_eager=enforce_eager,
256
+ max_num_batched_tokens=100000,
257
+ max_num_seqs=100000,
258
+ enable_chunked_prefill=True,
259
)
274
- scheduler_config = SchedulerConfig(100000,
275
- 100000,
276
- 100000,
277
- enable_chunked_prefill=True)
278
- model_runner = ModelRunner(model_config=model_config,
279
- parallel_config=None,
280
- scheduler_config=scheduler_config,
281
- device_config=None,
282
- load_config=None,
283
- lora_config=None,
284
- is_driver_worker=True)
285
- model_runner.set_block_size(16)
260
261
# Add prefill requests.
262
seq_lens = []
vllm/worker/cpu_model_runner.py
+9
-12
@@ -4,8 +4,9 @@ import torch
4
from torch import nn
5
6
from vllm.attention import AttentionMetadata, get_attn_backend
7
-from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
7
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8
+ ModelConfig, ParallelConfig, SchedulerConfig,
9
+ VisionLanguageConfig)
10
from vllm.distributed import broadcast_tensor_dict
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
@@ -26,6 +27,7 @@ class CPUModelRunner:
27
parallel_config: ParallelConfig,
28
scheduler_config: SchedulerConfig,
29
device_config: DeviceConfig,
30
+ cache_config: CacheConfig,
31
load_config: LoadConfig,
32
lora_config: Optional[LoRAConfig],
33
vision_language_config: Optional[VisionLanguageConfig],
@@ -39,27 +41,22 @@ class CPUModelRunner:
41
self.scheduler_config = scheduler_config
42
# Currently, CPU worker doesn't support chunked prefill.
43
assert self.scheduler_config.chunked_prefill_enabled is False
44
+ self.device_config = device_config
45
+ self.cache_config = cache_config
46
self.lora_config = lora_config
47
self.vision_language_config = vision_language_config
48
self.load_config = load_config
49
self.is_driver_worker = is_driver_worker
50
47
- # model_config can be None in tests/samplers/test_sampler.py.
48
- # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
49
- self.sliding_window = (model_config.get_sliding_window()
50
- if model_config is not None else None)
51
- self.device_config = (device_config
52
- if device_config is not None else DeviceConfig())
51
self.device = self.device_config.device
52
53
self.kv_cache_dtype = kv_cache_dtype
56
-
57
- self.attn_backend = get_attn_backend(
58
- self.model_config.dtype if model_config is not None else None)
54
+ self.sliding_window = model_config.get_sliding_window()
55
+ self.block_size = cache_config.block_size
56
+ self.attn_backend = get_attn_backend(self.model_config.dtype)
57
58
# Lazy initialization.
59
self.model: nn.Module # Set after init_Model
62
- self.block_size: int # Set after initial profiling.
60
61
def load_model(self) -> None:
62
self.model = get_model(
vllm/worker/cpu_worker.py
+1
@@ -151,6 +151,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
151
parallel_config,
152
scheduler_config,
153
device_config,
154
+ cache_config,
155
load_config=self.load_config,
156
lora_config=self.lora_config,
157
vision_language_config=self.vision_language_config,
vllm/worker/model_runner.py
+21
-33
@@ -9,8 +9,9 @@ import torch.nn as nn
9
10
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
get_attn_backend)
12
-from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
12
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
+ ModelConfig, ParallelConfig, SchedulerConfig,
14
+ VisionLanguageConfig)
15
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16
from vllm.distributed.device_communicators import (custom_all_reduce,
17
pynccl_utils)
@@ -106,6 +107,7 @@ class ModelRunner:
107
parallel_config: ParallelConfig,
108
scheduler_config: SchedulerConfig,
109
device_config: DeviceConfig,
110
+ cache_config: CacheConfig,
111
load_config: LoadConfig,
112
lora_config: Optional[LoRAConfig],
113
kv_cache_dtype: Optional[str] = "auto",
@@ -115,48 +117,40 @@ class ModelRunner:
117
self.model_config = model_config
118
self.parallel_config = parallel_config
119
self.scheduler_config = scheduler_config
120
+ self.device_config = device_config
121
+ self.cache_config = cache_config
122
self.lora_config = lora_config
123
self.load_config = load_config
124
self.is_driver_worker = is_driver_worker
125
+ self.vision_language_config = vision_language_config
126
122
- # model_config can be None in tests/samplers/test_sampler.py.
123
- # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
124
- self.sliding_window = (model_config.get_sliding_window()
125
- if model_config is not None else None)
126
- self.device_config = (device_config
127
- if device_config is not None else DeviceConfig())
127
self.device = self.device_config.device
128
+ self.pin_memory = is_pin_memory_available()
129
130
- # Set after load_model.
131
- self.lora_manager: LRUCacheWorkerLoRAManager = None
132
-
130
+ self.kv_cache_dtype = kv_cache_dtype
131
+ self.sliding_window = model_config.get_sliding_window()
132
+ self.block_size = cache_config.block_size
133
+ self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
134
self.graph_runners: Dict[int, CUDAGraphRunner] = {}
135
self.graph_memory_pool: Optional[Tuple[
136
int, int]] = None # Set during graph capture.
136
-
137
- self.max_seq_len_to_capture = (self.model_config.max_seq_len_to_capture
138
- if self.model_config is not None else 0)
139
-
140
- self.pin_memory = is_pin_memory_available()
141
- self.kv_cache_dtype = kv_cache_dtype
142
- self.vision_language_config = vision_language_config
143
-
144
- self.attn_backend = get_attn_backend(
145
- self.model_config.dtype if model_config is not None else None)
146
-
147
- # Lazy initialization
148
- self.model: torch.nn.Module # Set after load_model
149
- self.block_size: int # Set after initial profiling.
137
# When using CUDA graph, the input block tables must be padded to
138
# max_seq_len_to_capture. However, creating the block table in
139
# Python can be expensive. To optimize this, we cache the block table
140
# in numpy and only copy the actual input content at every iteration.
141
# The shape of the cached block table will be
142
# (max batch size to capture, max context len to capture / block size).
156
- self.graph_block_tables: torch.Tensor # Set after initial profiling.
143
+ self.graph_block_tables = np.zeros(
144
+ (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
145
+ dtype=np.int32)
146
+ self.attn_backend = get_attn_backend(self.model_config.dtype)
147
148
+ # Lazy initialization
149
+ self.model: torch.nn.Module # Set after load_model
150
# Set if the backend is flashinfer.
151
self.flashinfer_workspace_buffer: torch.Tensor
152
+ # Set after load_model.
153
+ self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
154
155
def load_model(self) -> None:
156
with CudaMemoryProfiler() as m:
@@ -211,13 +205,6 @@ class ModelRunner:
205
"but the KV cache data type is not FP8. "
206
"KV cache scaling factors will not be used.")
207
214
- def set_block_size(self, block_size: int) -> None:
215
- self.block_size = block_size
216
-
217
- self.graph_block_tables = np.zeros(
218
- (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
219
- dtype=np.int32)
220
-
208
def get_max_block_per_batch(self) -> int:
209
block_size = self.block_size
210
return (self.max_seq_len_to_capture + block_size - 1) // block_size
@@ -835,6 +822,7 @@ class ModelRunner:
822
dummy_lora_requests = []
823
dummy_lora_requests_per_seq = []
824
if self.lora_config:
825
+ assert self.lora_manager is not None
826
with self.lora_manager.dummy_lora_cache():
827
for idx in range(self.lora_config.max_loras):
828
lora_id = idx + 1
vllm/worker/worker.py
+1
-1
@@ -75,6 +75,7 @@ class Worker(WorkerBase):
75
parallel_config,
76
scheduler_config,
77
device_config,
78
+ cache_config,
79
load_config=load_config,
80
lora_config=self.lora_config,
81
kv_cache_dtype=self.cache_config.cache_dtype,
@@ -184,7 +185,6 @@ class Worker(WorkerBase):
185
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
186
self.parallel_config)
187
self.gpu_cache = self.cache_engine.gpu_cache
187
- self.model_runner.set_block_size(self.cache_engine.block_size)
188
189
def _warm_up_model(self) -> None:
190
if not self.model_config.enforce_eager: