@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ee535b294
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Set block size at initialization & Fix test_model_runner (#4705)

Woosuk Kwon committed May 9, 2024 at 09:04 UTC 0ee535b2945d042cbb1fc6e63fd3fddd94d491f2
5 files changed +64 -104
tests/worker/test_model_runner.py
+32 -58
@@ -1,27 +1,38 @@
1 import pytest
2 import torch
3
4 -from vllm.config import ModelConfig, SchedulerConfig
4 from vllm.distributed.parallel_state import init_distributed_environment
5 +from vllm.engine.arg_utils import EngineArgs
6 from vllm.model_executor.sampling_metadata import SamplingMetadata
7 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8 from vllm.utils import get_open_port
9 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
10
11
12 +def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
13 + engine_args = EngineArgs(model, *args, **kwargs)
14 + engine_config = engine_args.create_engine_config()
15 + model_runner = ModelRunner(
16 + model_config=engine_config.model_config,
17 + parallel_config=engine_config.parallel_config,
18 + scheduler_config=engine_config.scheduler_config,
19 + device_config=engine_config.device_config,
20 + cache_config=engine_config.cache_config,
21 + load_config=engine_config.load_config,
22 + lora_config=engine_config.lora_config,
23 + is_driver_worker=True,
24 + )
25 + return model_runner
26 +
27 +
28 @pytest.mark.parametrize("batch_size", list(range(1, 257)))
29 def test_prepare_prompt(batch_size):
14 - scheduler_config = SchedulerConfig(100000,
15 - 100000,
16 - 100000,
17 - enable_chunked_prefill=False)
18 - model_runner = ModelRunner(model_config=None,
19 - parallel_config=None,
20 - scheduler_config=scheduler_config,
21 - device_config=None,
22 - load_config=None,
23 - lora_config=None)
24 - model_runner.set_block_size(16)
30 + model_runner = _create_model_runner(
31 + "facebook/opt-125m",
32 + max_num_batched_tokens=100000,
33 + max_num_seqs=100000,
34 + enable_chunked_prefill=False,
35 + )
36
37 seq_lens = []
38 seq_group_metadata_list = []
@@ -123,27 +134,15 @@ def test_prepare_prompt(batch_size):
134
135 @pytest.mark.parametrize("batch_size", list(range(1, 257)))
136 def test_prepare_decode_cuda_graph(batch_size):
126 - model_config = ModelConfig(
137 + model_runner = _create_model_runner(
138 "facebook/opt-125m",
128 - "facebook/opt-125m",
129 - tokenizer_mode="auto",
130 - trust_remote_code=False,
139 seed=0,
140 dtype="float16",
133 - revision=None,
141 enforce_eager=False,
142 + max_num_batched_tokens=100000,
143 + max_num_seqs=100000,
144 + enable_chunked_prefill=False,
145 )
136 - scheduler_config = SchedulerConfig(100000,
137 - 100000,
138 - 100000,
139 - enable_chunked_prefill=False)
140 - model_runner = ModelRunner(model_config=model_config,
141 - parallel_config=None,
142 - scheduler_config=scheduler_config,
143 - device_config=None,
144 - load_config=None,
145 - lora_config=None)
146 - model_runner.set_block_size(16)
146
147 seq_lens = []
148 seq_group_metadata_list = []
@@ -214,23 +213,12 @@ def test_prepare_decode_cuda_graph(batch_size):
213
214 def test_empty_seq_group():
215 """Verify prepare prompt and decode returns empty output."""
217 - model_config = ModelConfig(
218 - "facebook/opt-125m",
216 + model_runner = _create_model_runner(
217 "facebook/opt-125m",
220 - tokenizer_mode="auto",
221 - trust_remote_code=False,
218 seed=0,
219 dtype="float16",
224 - revision=None,
220 enforce_eager=False,
221 )
227 - model_runner = ModelRunner(model_config=model_config,
228 - parallel_config=None,
229 - scheduler_config=None,
230 - device_config=None,
231 - load_config=None,
232 - lora_config=None)
233 - model_runner.set_block_size(16)
222 seq_group_metadata_list = []
223 input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
224 model_runner._prepare_decode(seq_group_metadata_list))
@@ -260,29 +248,15 @@ def distributed_init():
248 @pytest.mark.parametrize("batch_size", list(range(2, 128)))
249 @pytest.mark.parametrize("enforce_eager", [True, False])
250 def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
263 -
264 - model_config = ModelConfig(
265 - "facebook/opt-125m",
251 + model_runner = _create_model_runner(
252 "facebook/opt-125m",
267 - tokenizer_mode="auto",
268 - trust_remote_code=False,
253 seed=0,
254 dtype="float16",
271 - revision=None,
255 enforce_eager=enforce_eager,
256 + max_num_batched_tokens=100000,
257 + max_num_seqs=100000,
258 + enable_chunked_prefill=True,
259 )
274 - scheduler_config = SchedulerConfig(100000,
275 - 100000,
276 - 100000,
277 - enable_chunked_prefill=True)
278 - model_runner = ModelRunner(model_config=model_config,
279 - parallel_config=None,
280 - scheduler_config=scheduler_config,
281 - device_config=None,
282 - load_config=None,
283 - lora_config=None,
284 - is_driver_worker=True)
285 - model_runner.set_block_size(16)
260
261 # Add prefill requests.
262 seq_lens = []
vllm/worker/cpu_model_runner.py
+9 -12
@@ -4,8 +4,9 @@ import torch
4 from torch import nn
5
6 from vllm.attention import AttentionMetadata, get_attn_backend
7 -from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
7 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
8 + ModelConfig, ParallelConfig, SchedulerConfig,
9 + VisionLanguageConfig)
10 from vllm.distributed import broadcast_tensor_dict
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
@@ -26,6 +27,7 @@ class CPUModelRunner:
27 parallel_config: ParallelConfig,
28 scheduler_config: SchedulerConfig,
29 device_config: DeviceConfig,
30 + cache_config: CacheConfig,
31 load_config: LoadConfig,
32 lora_config: Optional[LoRAConfig],
33 vision_language_config: Optional[VisionLanguageConfig],
@@ -39,27 +41,22 @@ class CPUModelRunner:
41 self.scheduler_config = scheduler_config
42 # Currently, CPU worker doesn't support chunked prefill.
43 assert self.scheduler_config.chunked_prefill_enabled is False
44 + self.device_config = device_config
45 + self.cache_config = cache_config
46 self.lora_config = lora_config
47 self.vision_language_config = vision_language_config
48 self.load_config = load_config
49 self.is_driver_worker = is_driver_worker
50
47 - # model_config can be None in tests/samplers/test_sampler.py.
48 - # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
49 - self.sliding_window = (model_config.get_sliding_window()
50 - if model_config is not None else None)
51 - self.device_config = (device_config
52 - if device_config is not None else DeviceConfig())
51 self.device = self.device_config.device
52
53 self.kv_cache_dtype = kv_cache_dtype
56 -
57 - self.attn_backend = get_attn_backend(
58 - self.model_config.dtype if model_config is not None else None)
54 + self.sliding_window = model_config.get_sliding_window()
55 + self.block_size = cache_config.block_size
56 + self.attn_backend = get_attn_backend(self.model_config.dtype)
57
58 # Lazy initialization.
59 self.model: nn.Module # Set after init_Model
62 - self.block_size: int # Set after initial profiling.
60
61 def load_model(self) -> None:
62 self.model = get_model(
vllm/worker/cpu_worker.py
+1
@@ -151,6 +151,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
151 parallel_config,
152 scheduler_config,
153 device_config,
154 + cache_config,
155 load_config=self.load_config,
156 lora_config=self.lora_config,
157 vision_language_config=self.vision_language_config,
vllm/worker/model_runner.py
+21 -33
@@ -9,8 +9,9 @@ import torch.nn as nn
9
10 from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 get_attn_backend)
12 -from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
12 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 + ModelConfig, ParallelConfig, SchedulerConfig,
14 + VisionLanguageConfig)
15 from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16 from vllm.distributed.device_communicators import (custom_all_reduce,
17 pynccl_utils)
@@ -106,6 +107,7 @@ class ModelRunner:
107 parallel_config: ParallelConfig,
108 scheduler_config: SchedulerConfig,
109 device_config: DeviceConfig,
110 + cache_config: CacheConfig,
111 load_config: LoadConfig,
112 lora_config: Optional[LoRAConfig],
113 kv_cache_dtype: Optional[str] = "auto",
@@ -115,48 +117,40 @@ class ModelRunner:
117 self.model_config = model_config
118 self.parallel_config = parallel_config
119 self.scheduler_config = scheduler_config
120 + self.device_config = device_config
121 + self.cache_config = cache_config
122 self.lora_config = lora_config
123 self.load_config = load_config
124 self.is_driver_worker = is_driver_worker
125 + self.vision_language_config = vision_language_config
126
122 - # model_config can be None in tests/samplers/test_sampler.py.
123 - # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
124 - self.sliding_window = (model_config.get_sliding_window()
125 - if model_config is not None else None)
126 - self.device_config = (device_config
127 - if device_config is not None else DeviceConfig())
127 self.device = self.device_config.device
128 + self.pin_memory = is_pin_memory_available()
129
130 - # Set after load_model.
131 - self.lora_manager: LRUCacheWorkerLoRAManager = None
132 -
130 + self.kv_cache_dtype = kv_cache_dtype
131 + self.sliding_window = model_config.get_sliding_window()
132 + self.block_size = cache_config.block_size
133 + self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
134 self.graph_runners: Dict[int, CUDAGraphRunner] = {}
135 self.graph_memory_pool: Optional[Tuple[
136 int, int]] = None # Set during graph capture.
136 -
137 - self.max_seq_len_to_capture = (self.model_config.max_seq_len_to_capture
138 - if self.model_config is not None else 0)
139 -
140 - self.pin_memory = is_pin_memory_available()
141 - self.kv_cache_dtype = kv_cache_dtype
142 - self.vision_language_config = vision_language_config
143 -
144 - self.attn_backend = get_attn_backend(
145 - self.model_config.dtype if model_config is not None else None)
146 -
147 - # Lazy initialization
148 - self.model: torch.nn.Module # Set after load_model
149 - self.block_size: int # Set after initial profiling.
137 # When using CUDA graph, the input block tables must be padded to
138 # max_seq_len_to_capture. However, creating the block table in
139 # Python can be expensive. To optimize this, we cache the block table
140 # in numpy and only copy the actual input content at every iteration.
141 # The shape of the cached block table will be
142 # (max batch size to capture, max context len to capture / block size).
156 - self.graph_block_tables: torch.Tensor # Set after initial profiling.
143 + self.graph_block_tables = np.zeros(
144 + (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
145 + dtype=np.int32)
146 + self.attn_backend = get_attn_backend(self.model_config.dtype)
147
148 + # Lazy initialization
149 + self.model: torch.nn.Module # Set after load_model
150 # Set if the backend is flashinfer.
151 self.flashinfer_workspace_buffer: torch.Tensor
152 + # Set after load_model.
153 + self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
154
155 def load_model(self) -> None:
156 with CudaMemoryProfiler() as m:
@@ -211,13 +205,6 @@ class ModelRunner:
205 "but the KV cache data type is not FP8. "
206 "KV cache scaling factors will not be used.")
207
214 - def set_block_size(self, block_size: int) -> None:
215 - self.block_size = block_size
216 -
217 - self.graph_block_tables = np.zeros(
218 - (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
219 - dtype=np.int32)
220 -
208 def get_max_block_per_batch(self) -> int:
209 block_size = self.block_size
210 return (self.max_seq_len_to_capture + block_size - 1) // block_size
@@ -835,6 +822,7 @@ class ModelRunner:
822 dummy_lora_requests = []
823 dummy_lora_requests_per_seq = []
824 if self.lora_config:
825 + assert self.lora_manager is not None
826 with self.lora_manager.dummy_lora_cache():
827 for idx in range(self.lora_config.max_loras):
828 lora_id = idx + 1
vllm/worker/worker.py
+1 -1
@@ -75,6 +75,7 @@ class Worker(WorkerBase):
75 parallel_config,
76 scheduler_config,
77 device_config,
78 + cache_config,
79 load_config=load_config,
80 lora_config=self.lora_config,
81 kv_cache_dtype=self.cache_config.cache_dtype,
@@ -184,7 +185,6 @@ class Worker(WorkerBase):
185 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
186 self.parallel_config)
187 self.gpu_cache = self.cache_engine.gpu_cache
187 - self.model_runner.set_block_size(self.cache_engine.block_size)
188
189 def _warm_up_model(self) -> None:
190 if not self.model_config.enforce_eager: