@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ed82e7074
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Small perf improvements (#6520)

Antoni Baum committed Jul 19, 2024 at 12:10 UTC 9ed82e7074a18e25680ab106fc846364ad97bc00
7 files changed +46 -23
tests/core/block/test_block_manager_v2.py
+7 -4
@@ -249,10 +249,13 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
249
250 # Expect consumed blocks to be new blocks required to support the new slots.
251 expected_consumed_blocks = len(
252 - chunk_list(
253 - list(
254 - range(prompt_len + num_slots_to_append + num_lookahead_slots)),
255 - block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
252 + list(
253 + chunk_list(
254 + list(
255 + range(prompt_len + num_slots_to_append +
256 + num_lookahead_slots)),
257 + block_size))) - len(
258 + list(chunk_list(list(range(prompt_len)), block_size)))
259 assert num_consumed_blocks == expected_consumed_blocks
260
261
tests/core/block/test_cpu_gpu_block_allocator.py
+4 -4
@@ -58,10 +58,10 @@ def test_allocate_immutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
58
59 unique_token_ids = list(
60 range((num_cpu_blocks + num_gpu_blocks) * block_size))
61 - gpu_token_ids = chunk_list(unique_token_ids[:num_gpu_blocks * block_size],
62 - block_size)
63 - cpu_token_ids = chunk_list(unique_token_ids[num_gpu_blocks * block_size:],
64 - block_size)
61 + gpu_token_ids = list(
62 + chunk_list(unique_token_ids[:num_gpu_blocks * block_size], block_size))
63 + cpu_token_ids = list(
64 + chunk_list(unique_token_ids[num_gpu_blocks * block_size:], block_size))
65
66 assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
67 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
vllm/core/block/block_table.py
+14 -5
@@ -1,3 +1,4 @@
1 +import math
2 from typing import List, Optional
3
4 from vllm.core.block.common import BlockList
@@ -337,10 +338,17 @@ class BlockTable:
338 This is required for the scheduler to determine whether a sequence can
339 continue generation, or if it must be preempted.
340 """
341 + # Math below is equivalent to:
342 + # all_token_ids = token_ids + [-1] * num_lookahead_slots
343 + # token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
344 + # return len(token_blocks)
345
341 - all_token_ids = token_ids + [-1] * num_lookahead_slots
342 - token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
343 - return len(token_blocks)
346 + num_token_ids = len(token_ids) + num_lookahead_slots
347 + first_chunk_size = self._block_size - (self._num_full_slots %
348 + self._block_size)
349 + num_token_blocks = (1 + math.ceil(
350 + (num_token_ids - first_chunk_size) / self._block_size))
351 + return num_token_blocks
352
353 def _chunk_token_blocks_for_append(
354 self, token_ids: List[int]) -> List[List[int]]:
@@ -351,6 +359,7 @@ class BlockTable:
359 """
360 first_chunk_size = self._block_size - (self._num_full_slots %
361 self._block_size)
354 - token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
355 - token_ids[first_chunk_size:], self._block_size)
362 + token_blocks = [token_ids[:first_chunk_size]]
363 + token_blocks.extend(
364 + chunk_list(token_ids[first_chunk_size:], self._block_size))
365 return token_blocks
vllm/core/block/prefix_caching_block.py
+4 -1
@@ -552,9 +552,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
552 # runner.
553
554 # It returns a list of int although type annotation says list of string.
555 + if len(computed_seq_block_ids) == 1:
556 + return computed_seq_block_ids[0]
557 +
558 return commonprefix([
559 ids for ids in computed_seq_block_ids # type: ignore
557 - if ids != []
560 + if ids
561 ])
562
563 def get_num_blocks_touched(self,
vllm/model_executor/models/__init__.py
+10 -4
@@ -1,3 +1,4 @@
1 +import functools
2 import importlib
3 from typing import Dict, List, Optional, Type
4
@@ -98,6 +99,14 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
99
100 class ModelRegistry:
101
102 + @staticmethod
103 + @functools.lru_cache(maxsize=128)
104 + def _get_model(model_arch: str):
105 + module_name, model_cls_name = _MODELS[model_arch]
106 + module = importlib.import_module(
107 + f"vllm.model_executor.models.{module_name}")
108 + return getattr(module, model_cls_name, None)
109 +
110 @staticmethod
111 def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
112 if model_arch in _OOT_MODELS:
@@ -114,10 +123,7 @@ class ModelRegistry:
123 "Model architecture %s is partially supported by ROCm: %s",
124 model_arch, _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
125
117 - module_name, model_cls_name = _MODELS[model_arch]
118 - module = importlib.import_module(
119 - f"vllm.model_executor.models.{module_name}")
120 - return getattr(module, model_cls_name, None)
126 + return ModelRegistry._get_model(model_arch)
127
128 @staticmethod
129 def get_supported_archs() -> List[str]:
vllm/sequence.py
+4 -3
@@ -457,24 +457,25 @@ class SequenceGroup:
457 self.prompt_adapter_request = prompt_adapter_request
458 self.encoder_seq = encoder_seq
459 self.trace_headers = trace_headers
460 + self._first_seq = next(iter(self.seqs_dict.values()))
461
462 @property
463 def prompt(self) -> Optional[str]:
464 # All sequences in the group should have the same prompt.
465 # We use the prompt of an arbitrary sequence.
465 - return next(iter(self.seqs_dict.values())).prompt
466 + return self._first_seq.prompt
467
468 @property
469 def prompt_token_ids(self) -> List[int]:
470 # All sequences in the group should have the same prompt.
471 # We use the prompt of an arbitrary sequence.
471 - return next(iter(self.seqs_dict.values())).prompt_token_ids
472 + return self._first_seq.prompt_token_ids
473
474 @property
475 def multi_modal_data(self) -> "MultiModalDataDict":
476 # All sequences in the group should have the same multi-modal data.
477 # We use the multi-modal data of an arbitrary sequence.
477 - return next(iter(self.seqs_dict.values())).multi_modal_data
478 + return self._first_seq.multi_modal_data
479
480 @property
481 def lora_int_id(self) -> int:
vllm/utils.py
+3 -2
@@ -415,9 +415,10 @@ def init_kmp_env():
415 os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
416
417
418 -def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
418 +def chunk_list(lst: List[T], chunk_size: int):
419 """Yield successive chunk_size chunks from lst."""
420 - return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
420 + for i in range(0, len(lst), chunk_size):
421 + yield lst[i:i + chunk_size]
422
423
424 def cdiv(a: int, b: int) -> int: