A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Small perf improvements (#6520)
Antoni Baum committed
Jul 19, 2024 at 12:10 UTC
9ed82e7074a18e25680ab106fc846364ad97bc00
7 files changed
+46
-23
tests/core/block/test_block_manager_v2.py
+7
-4
@@ -249,10 +249,13 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
249
250
# Expect consumed blocks to be new blocks required to support the new slots.
251
expected_consumed_blocks = len(
252
- chunk_list(
253
- list(
254
- range(prompt_len + num_slots_to_append + num_lookahead_slots)),
255
- block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
252
+ list(
253
+ chunk_list(
254
+ list(
255
+ range(prompt_len + num_slots_to_append +
256
+ num_lookahead_slots)),
257
+ block_size))) - len(
258
+ list(chunk_list(list(range(prompt_len)), block_size)))
259
assert num_consumed_blocks == expected_consumed_blocks
260
261
tests/core/block/test_cpu_gpu_block_allocator.py
+4
-4
@@ -58,10 +58,10 @@ def test_allocate_immutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
58
59
unique_token_ids = list(
60
range((num_cpu_blocks + num_gpu_blocks) * block_size))
61
- gpu_token_ids = chunk_list(unique_token_ids[:num_gpu_blocks * block_size],
62
- block_size)
63
- cpu_token_ids = chunk_list(unique_token_ids[num_gpu_blocks * block_size:],
64
- block_size)
61
+ gpu_token_ids = list(
62
+ chunk_list(unique_token_ids[:num_gpu_blocks * block_size], block_size))
63
+ cpu_token_ids = list(
64
+ chunk_list(unique_token_ids[num_gpu_blocks * block_size:], block_size))
65
66
assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
67
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
vllm/core/block/block_table.py
+14
-5
@@ -1,3 +1,4 @@
1
+import math
2
from typing import List, Optional
3
4
from vllm.core.block.common import BlockList
@@ -337,10 +338,17 @@ class BlockTable:
338
This is required for the scheduler to determine whether a sequence can
339
continue generation, or if it must be preempted.
340
"""
341
+ # Math below is equivalent to:
342
+ # all_token_ids = token_ids + [-1] * num_lookahead_slots
343
+ # token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
344
+ # return len(token_blocks)
345
341
- all_token_ids = token_ids + [-1] * num_lookahead_slots
342
- token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
343
- return len(token_blocks)
346
+ num_token_ids = len(token_ids) + num_lookahead_slots
347
+ first_chunk_size = self._block_size - (self._num_full_slots %
348
+ self._block_size)
349
+ num_token_blocks = (1 + math.ceil(
350
+ (num_token_ids - first_chunk_size) / self._block_size))
351
+ return num_token_blocks
352
353
def _chunk_token_blocks_for_append(
354
self, token_ids: List[int]) -> List[List[int]]:
@@ -351,6 +359,7 @@ class BlockTable:
359
"""
360
first_chunk_size = self._block_size - (self._num_full_slots %
361
self._block_size)
354
- token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
355
- token_ids[first_chunk_size:], self._block_size)
362
+ token_blocks = [token_ids[:first_chunk_size]]
363
+ token_blocks.extend(
364
+ chunk_list(token_ids[first_chunk_size:], self._block_size))
365
return token_blocks
vllm/core/block/prefix_caching_block.py
+4
-1
@@ -552,9 +552,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
552
# runner.
553
554
# It returns a list of int although type annotation says list of string.
555
+ if len(computed_seq_block_ids) == 1:
556
+ return computed_seq_block_ids[0]
557
+
558
return commonprefix([
559
ids for ids in computed_seq_block_ids # type: ignore
557
- if ids != []
560
+ if ids
561
])
562
563
def get_num_blocks_touched(self,
vllm/model_executor/models/__init__.py
+10
-4
@@ -1,3 +1,4 @@
1
+import functools
2
import importlib
3
from typing import Dict, List, Optional, Type
4
@@ -98,6 +99,14 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
99
100
class ModelRegistry:
101
102
+ @staticmethod
103
+ @functools.lru_cache(maxsize=128)
104
+ def _get_model(model_arch: str):
105
+ module_name, model_cls_name = _MODELS[model_arch]
106
+ module = importlib.import_module(
107
+ f"vllm.model_executor.models.{module_name}")
108
+ return getattr(module, model_cls_name, None)
109
+
110
@staticmethod
111
def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
112
if model_arch in _OOT_MODELS:
@@ -114,10 +123,7 @@ class ModelRegistry:
123
"Model architecture %s is partially supported by ROCm: %s",
124
model_arch, _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
125
117
- module_name, model_cls_name = _MODELS[model_arch]
118
- module = importlib.import_module(
119
- f"vllm.model_executor.models.{module_name}")
120
- return getattr(module, model_cls_name, None)
126
+ return ModelRegistry._get_model(model_arch)
127
128
@staticmethod
129
def get_supported_archs() -> List[str]:
vllm/sequence.py
+4
-3
@@ -457,24 +457,25 @@ class SequenceGroup:
457
self.prompt_adapter_request = prompt_adapter_request
458
self.encoder_seq = encoder_seq
459
self.trace_headers = trace_headers
460
+ self._first_seq = next(iter(self.seqs_dict.values()))
461
462
@property
463
def prompt(self) -> Optional[str]:
464
# All sequences in the group should have the same prompt.
465
# We use the prompt of an arbitrary sequence.
465
- return next(iter(self.seqs_dict.values())).prompt
466
+ return self._first_seq.prompt
467
468
@property
469
def prompt_token_ids(self) -> List[int]:
470
# All sequences in the group should have the same prompt.
471
# We use the prompt of an arbitrary sequence.
471
- return next(iter(self.seqs_dict.values())).prompt_token_ids
472
+ return self._first_seq.prompt_token_ids
473
474
@property
475
def multi_modal_data(self) -> "MultiModalDataDict":
476
# All sequences in the group should have the same multi-modal data.
477
# We use the multi-modal data of an arbitrary sequence.
477
- return next(iter(self.seqs_dict.values())).multi_modal_data
478
+ return self._first_seq.multi_modal_data
479
480
@property
481
def lora_int_id(self) -> int:
vllm/utils.py
+3
-2
@@ -415,9 +415,10 @@ def init_kmp_env():
415
os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
416
417
418
-def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
418
+def chunk_list(lst: List[T], chunk_size: int):
419
"""Yield successive chunk_size chunks from lst."""
420
- return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
420
+ for i in range(0, len(lst), chunk_size):
421
+ yield lst[i:i + chunk_size]
422
423
424
def cdiv(a: int, b: int) -> int: