@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbc53b6b8d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware][TPU] Support parallel sampling & Swapping (#5855)

Woosuk Kwon committed Jun 26, 2024 at 11:07 UTC cbc53b6b8d87b29949ce13d504750f63714df532
3 files changed +147 -56
vllm/attention/backends/pallas.py
+22 -8
@@ -28,21 +28,35 @@ class PallasAttentionBackend(AttentionBackend):
28 ) -> Tuple[int, ...]:
29 return (num_kv_heads, num_blocks, block_size, head_size)
30
31 + @torch.compile(backend="openxla")
32 @staticmethod
33 def swap_blocks(
33 - src_kv_cache: torch.Tensor,
34 - dst_kv_cache: torch.Tensor,
35 - src_to_dst: Dict[int, int],
34 + src_kv_cache: Tuple[torch.Tensor, torch.Tensor],
35 + dst_kv_cache: Tuple[torch.Tensor, torch.Tensor],
36 + src_to_dst: Tuple[torch.Tensor, torch.Tensor],
37 ) -> None:
37 - raise NotImplementedError("swap_blocks is not implemented.")
38 + src_k_cache, src_v_cache = src_kv_cache
39 + dst_k_cache, dst_v_cache = dst_kv_cache
40 + torch.ops.xla.dynamo_set_buffer_donor_(dst_k_cache, True)
41 + torch.ops.xla.dynamo_set_buffer_donor_(dst_v_cache, True)
42
43 + device = dst_k_cache.device
44 + src_indices, dst_indices = src_to_dst
45 + dst_k_cache[:, dst_indices] = src_k_cache[:, src_indices].to(device)
46 + dst_v_cache[:, dst_indices] = src_v_cache[:, src_indices].to(device)
47 +
48 + @torch.compile(backend="openxla")
49 @staticmethod
50 def copy_blocks(
41 - kv_caches: List[torch.Tensor],
42 - src_to_dists: Dict[int, List[int]],
51 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
52 + src_to_dists: Tuple[torch.Tensor, torch.Tensor],
53 ) -> None:
44 - # TODO(woosuk): Implement this.
45 - raise NotImplementedError("copy_blocks is not implemented.")
54 + src_indices, dst_indices = src_to_dists
55 + for k_cache, v_cache in kv_caches:
56 + torch.ops.xla.dynamo_set_buffer_donor_(k_cache, True)
57 + k_cache[:, dst_indices] = k_cache[:, src_indices]
58 + torch.ops.xla.dynamo_set_buffer_donor_(v_cache, True)
59 + v_cache[:, dst_indices] = v_cache[:, src_indices]
60
61
62 @dataclass
vllm/worker/tpu_model_runner.py
+50 -26
@@ -22,6 +22,9 @@ logger = init_logger(__name__)
22 _PAD_SLOT_ID = 0 # FIXME(woosuk)
23 # FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow.
24 _ENABLE_TOP_P = False
25 +# FIXME(woosuk): A temporary hack to support `n > 1`.
26 +# This can significantly affect the performance if too large.
27 +_MAX_NUM_SAMPLES = 128
28
29
30 class TPUModelRunner:
@@ -143,8 +146,9 @@ class TPUModelRunner:
146 p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
147
148 # Dummy run.
149 + num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
150 self.model(token_ids, position_ids, kv_caches, attn_metadata,
147 - input_lens, t, p)
151 + input_lens, t, p, num_samples)
152
153 def warmup_model(
154 self,
@@ -268,14 +272,11 @@ class TPUModelRunner:
272 input_positions: List[List[int]] = []
273 slot_mapping: List[List[int]] = []
274 context_lens: List[int] = []
271 - num_seq_groups = len(seq_group_metadata_list)
272 - batch_size = _get_padded_batch_size(num_seq_groups)
275
274 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
276 + batch_idx = 0
277 + for seq_group_metadata in seq_group_metadata_list:
278 assert not seq_group_metadata.is_prompt
276 -
279 seq_ids = list(seq_group_metadata.seq_data.keys())
278 -
280 for seq_id in seq_ids:
281 seq_data = seq_group_metadata.seq_data[seq_id]
282 generation_token = seq_data.get_last_token_id()
@@ -288,14 +289,16 @@ class TPUModelRunner:
289
290 assert seq_group_metadata.block_tables is not None
291 block_table = seq_group_metadata.block_tables[seq_id]
291 - self.block_tables[i, :len(block_table)] = block_table
292 + self.block_tables[batch_idx, :len(block_table)] = block_table
293 + batch_idx += 1
294
295 block_number = block_table[position // self.block_size]
296 block_offset = position % self.block_size
297 slot = block_number * self.block_size + block_offset
298 slot_mapping.append([slot])
299
298 - num_paddings = batch_size - num_seq_groups
300 + batch_size = _get_padded_batch_size(batch_idx)
301 + num_paddings = batch_size - batch_idx
302 input_tokens = input_tokens + [[0]] * num_paddings
303 input_positions = input_positions + [[0]] * num_paddings
304 slot_mapping = slot_mapping + [[_PAD_SLOT_ID]] * num_paddings
@@ -333,14 +336,13 @@ class TPUModelRunner:
336 self,
337 seq_group_metadata_list: List[SequenceGroupMetadata],
338 padded_batch_size: int,
336 - ) -> Tuple[torch.Tensor, torch.Tensor]:
339 + ) -> Tuple[torch.Tensor, torch.Tensor, List[int]]:
340 assert len(seq_group_metadata_list) > 0
341 t = []
342 p = []
343 + best_of = []
344 for seq_group_metadata in seq_group_metadata_list:
341 - assert seq_group_metadata.sampling_params is not None
345 sampling_params = seq_group_metadata.sampling_params
343 -
346 # NOTE(woosuk): Here we mimic argmax sampling by applying a very
347 # low temperature. This is not accurate.
348 t.append(sampling_params.temperature
@@ -354,10 +356,11 @@ class TPUModelRunner:
356 raise NotImplementedError(
357 "Top-k sampling is currently disabled for the TPU backend "
358 "due to performance issues.")
357 - if sampling_params.best_of > 1:
359 + if sampling_params.best_of > _MAX_NUM_SAMPLES:
360 raise NotImplementedError(
359 - "best_of > 1 is not currently supported by the TPU "
361 + f"Best of > {_MAX_NUM_SAMPLES} is not supported by the TPU "
362 "backend.")
363 + best_of.append(sampling_params.best_of)
364 if sampling_params.use_beam_search:
365 raise NotImplementedError(
366 "Beam search is not supported by the TPU backend.")
@@ -369,13 +372,19 @@ class TPUModelRunner:
372 "prompt_logprobs is not currently supported by the TPU "
373 "backend.")
374
372 - num_paddings = padded_batch_size - len(seq_group_metadata_list)
375 + # Repeat the sampling params if the seq group has multiple seqs.
376 + num_seqs = len(seq_group_metadata.seq_data)
377 + t += [t[-1]] * (num_seqs - 1)
378 + p += [p[-1]] * (num_seqs - 1)
379 + best_of += [best_of[-1]] * (num_seqs - 1)
380 +
381 + num_paddings = padded_batch_size - len(t)
382 t += [1.0] * num_paddings
383 p += [1.0] * num_paddings
384
385 t = torch.tensor(t, dtype=torch.float32, device=self.device)
386 p = torch.tensor(p, dtype=torch.float32, device=self.device)
378 - return t, p
387 + return t, p, best_of
388
389 def _execute_model(
390 self,
@@ -392,28 +401,41 @@ class TPUModelRunner:
401 else:
402 inputs = self._prepare_decode(seq_group_metadata_list)
403 padded_batch_size = inputs[0].shape[0]
395 - t, p = self._prepare_sample(seq_group_metadata_list, padded_batch_size)
404 + t, p, best_of = self._prepare_sample(seq_group_metadata_list,
405 + padded_batch_size)
406 + num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
407
408 # Execute the model.
409 next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
399 - *inputs[2:], t, p)
410 + *inputs[2:], t, p, num_samples)
411 # Retrieve the outputs to CPU.
412 next_token_ids = next_token_ids.cpu().tolist()
413
414 # NOTE(woosuk): Minimal code to construct the sampler outputs.
415 # The TPU backend does not reuse the sampler, since the TPU backend
416 # does not support the advanced sampling parameters such as logprobs.
406 - i = 0
417 + zero_logprob = Logprob(0.0)
418 + batch_idx = 0
419 sampler_outputs = []
420 for seq_group_metadata in seq_group_metadata_list:
421 seq_outputs = []
422 seq_ids = list(seq_group_metadata.seq_data.keys())
411 - for seq_id in seq_ids:
412 - next_token_id = next_token_ids[i]
413 - seq_outputs.append(
414 - SequenceOutput(seq_id, next_token_id,
415 - {next_token_id: Logprob(0.0)}))
416 - i += 1
423 + if is_prompt:
424 + assert len(seq_ids) == 1
425 + seq_id = seq_ids[0]
426 + for i in range(best_of[batch_idx]):
427 + next_token_id = next_token_ids[batch_idx][i]
428 + seq_outputs.append(
429 + SequenceOutput(seq_id, next_token_id,
430 + {next_token_id: zero_logprob}))
431 + batch_idx += 1
432 + else:
433 + for seq_id in seq_ids:
434 + next_token_id = next_token_ids[batch_idx][0]
435 + seq_outputs.append(
436 + SequenceOutput(seq_id, next_token_id,
437 + {next_token_id: zero_logprob}))
438 + batch_idx += 1
439 sampler_outputs.append(
440 CompletionSequenceGroupOutput(seq_outputs, None))
441 return sampler_outputs
@@ -458,6 +480,7 @@ class ModelWrapper(nn.Module):
480 input_lens: torch.Tensor,
481 t: torch.Tensor,
482 p: torch.Tensor,
483 + num_samples: int,
484 ) -> torch.Tensor:
485 """Executes the forward pass of the model and samples the next token.
486
@@ -520,8 +543,9 @@ class ModelWrapper(nn.Module):
543 if _ENABLE_TOP_P:
544 logits = _apply_top_p(logits, p.unsqueeze(dim=1))
545 probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
523 - # FIXME(woosuk): best_of > 1 is not supported.
524 - next_token_ids = torch.multinomial(probs, num_samples=1).squeeze(dim=1)
546 + next_token_ids = torch.multinomial(probs,
547 + num_samples,
548 + replacement=True)
549 return next_token_ids
550
551
vllm/worker/tpu_worker.py
+75 -22
@@ -1,5 +1,5 @@
1 import os
2 -from typing import List, Optional, Tuple
2 +from typing import List, Optional, Tuple, Union
3
4 import torch
5 import torch_xla.core.xla_model as xm
@@ -117,19 +117,26 @@ class TPUWorker(LoraNotSupportedWorkerBase):
117 # Synchronize before measuring the memory usage.
118 xm.wait_device_ops()
119
120 + dtype_btyes = get_dtype_size(self.cache_dtype)
121 + block_size = self.cache_config.block_size
122 + block_size_bytes = (dtype_btyes * block_size * num_layers * 2 *
123 + head_size * num_kv_heads)
124 +
125 + # Calculate the TPU KV cache size based on profiling.
126 m = xm.get_memory_info(self.device)
127 total_memory_size = m["bytes_limit"]
128 usable_memory_size = int(total_memory_size *
129 self.cache_config.gpu_memory_utilization)
130 profiled = m["bytes_used"] # Weights + intermediate activations.
125 - kv_cache_bytes = max(usable_memory_size - profiled, 0)
126 - dtype_btyes = get_dtype_size(self.cache_dtype)
127 - block_size = self.cache_config.block_size
128 - num_tpu_blocks = (kv_cache_bytes //
129 - (dtype_btyes * block_size * num_layers * 2 *
130 - head_size * num_kv_heads))
131 + tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0)
132 + num_tpu_blocks = tpu_kv_cache_bytes // block_size_bytes
133 num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
132 - return num_tpu_blocks, 0
134 +
135 + # Calculate the CPU KV cache size based on the config.
136 + num_cpu_blocks = (self.cache_config.swap_space_bytes //
137 + block_size_bytes)
138 + num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
139 + return num_tpu_blocks, num_cpu_blocks
140
141 def initialize_cache(
142 self,
@@ -145,15 +152,19 @@ class TPUWorker(LoraNotSupportedWorkerBase):
152 num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
153 head_size = self.model_config.get_head_size()
154
155 + self.cpu_cache = []
156 self.tpu_cache = []
157 tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
158 num_gpu_blocks, self.block_size, num_kv_heads, head_size)
159 for _ in range(num_layers):
152 - key_cache = torch.zeros(tpu_cache_shape,
153 - dtype=dtype,
154 - device=self.device)
155 - value_cache = torch.zeros_like(key_cache)
156 - self.tpu_cache.append((key_cache, value_cache))
160 + tpu_k_cache = torch.zeros(tpu_cache_shape,
161 + dtype=dtype,
162 + device=self.device)
163 + tpu_v_cache = torch.zeros_like(tpu_k_cache)
164 + self.tpu_cache.append((tpu_k_cache, tpu_v_cache))
165 + cpu_k_cache = torch.zeros_like(tpu_k_cache, device="cpu")
166 + cpu_v_cache = torch.zeros_like(tpu_v_cache, device="cpu")
167 + self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
168 self._warmup_model()
169
170 def _warmup_model(self) -> None:
@@ -187,22 +198,48 @@ class TPUWorker(LoraNotSupportedWorkerBase):
198 if not self.is_driver_worker:
199 self._execute_model_non_driver()
200 return []
190 -
201 assert execute_model_req is not None
192 - # Currently, TPUWorker does not support swapping.
193 - # TODO(woosuk): Support block copying.
194 - assert len(execute_model_req.blocks_to_swap_in) == 0, (
195 - "Swapping is not supported for the TPU backend.")
196 - assert len(execute_model_req.blocks_to_swap_out) == 0, (
197 - "Swapping is not supported for the TPU backend.")
198 - assert len(execute_model_req.blocks_to_copy) == 0
199 -
202 + # Issue cache operations.
203 + self.cache_swap(
204 + execute_model_req.blocks_to_swap_in,
205 + execute_model_req.blocks_to_swap_out,
206 + execute_model_req.blocks_to_copy,
207 + )
208 + # Run the model.
209 seq_group_metadata_list = execute_model_req.seq_group_metadata_list
210 assert len(seq_group_metadata_list) > 0
211 output = self.model_runner.execute_model(seq_group_metadata_list,
212 self.tpu_cache)
213 return [output]
214
215 + def cache_swap(
216 + self,
217 + blocks_to_swap_in: List[Tuple[int, int]],
218 + blocks_to_swap_out: List[Tuple[int, int]],
219 + blocks_to_copy: List[Tuple[int, int]],
220 + ) -> None:
221 + attn_backend = self.model_runner.attn_backend
222 + num_layers = self.model_config.get_num_layers(self.parallel_config)
223 +
224 + if blocks_to_swap_in:
225 + # Swap from CPU to TPU.
226 + src_to_dst = _make_src_to_dst(blocks_to_swap_in, "cpu",
227 + self.device)
228 + for i in range(num_layers):
229 + attn_backend.swap_blocks(self.cpu_cache[i], self.tpu_cache[i],
230 + src_to_dst)
231 + if blocks_to_swap_out:
232 + # Swap from TPU to CPU.
233 + src_to_dst = _make_src_to_dst(blocks_to_swap_out, self.device,
234 + "cpu")
235 + for i in range(num_layers):
236 + attn_backend.swap_blocks(self.tpu_cache[i], self.cpu_cache[i],
237 + src_to_dst)
238 + if blocks_to_copy:
239 + src_to_dst = _make_src_to_dst(blocks_to_copy, self.device,
240 + self.device)
241 + attn_backend.copy_blocks(self.tpu_cache, src_to_dst)
242 +
243 def start_worker_execution_loop(self) -> None:
244 while self._execute_model_non_driver():
245 pass
@@ -210,3 +247,19 @@ class TPUWorker(LoraNotSupportedWorkerBase):
247 def _execute_model_non_driver(self) -> bool:
248 self.model_runner.execute_model(None, self.tpu_cache)
249 return True
250 +
251 +
252 +def _make_src_to_dst(
253 + mapping: List[Tuple[int, int]],
254 + src_device: Union[torch.device, str],
255 + dst_device: Union[torch.device, str],
256 +) -> Tuple[torch.Tensor, torch.Tensor]:
257 + src_indices = [i for i, _ in mapping]
258 + dst_indices = [i for _, i in mapping]
259 + src_indices = torch.tensor(src_indices,
260 + device=src_device,
261 + dtype=torch.int64)
262 + dst_indices = torch.tensor(dst_indices,
263 + device=dst_device,
264 + dtype=torch.int64)
265 + return src_indices, dst_indices