@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2092a6fa7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Core] Add worker_base for v1 worker (#12816)

Signed-off-by: Aoyu <aoyuzhan@amazon.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Aoyu <aoyuzhan@amazon.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Aoyu committed Feb 13, 2025 at 20:35 UTC 2092a6fa7d58e398bc1a234edcef62ce555faafe
4 files changed +153 -52
vllm/utils.py
+43
@@ -2220,3 +2220,46 @@ def import_pynvml():
2220 """
2221 import vllm.third_party.pynvml as pynvml
2222 return pynvml
2223 +
2224 +
2225 +def warn_for_unimplemented_methods(cls: Type[T]) -> Type[T]:
2226 + """
2227 + A replacement for `abc.ABC`.
2228 + When we use `abc.ABC`, subclasses will fail to instantiate
2229 + if they do not implement all abstract methods.
2230 + Here, we only require `raise NotImplementedError` in the
2231 + base class, and log a warning if the method is not implemented
2232 + in the subclass.
2233 + """
2234 +
2235 + original_init = cls.__init__
2236 +
2237 + def find_unimplemented_methods(self: object):
2238 + unimplemented_methods = []
2239 + for attr_name in dir(self):
2240 + # bypass inner method
2241 + if attr_name.startswith('_'):
2242 + continue
2243 +
2244 + try:
2245 + attr = getattr(self, attr_name)
2246 + # get the func of callable method
2247 + if callable(attr):
2248 + attr_func = attr.__func__
2249 + except AttributeError:
2250 + continue
2251 + src = inspect.getsource(attr_func)
2252 + if "NotImplementedError" in src:
2253 + unimplemented_methods.append(attr_name)
2254 + if unimplemented_methods:
2255 + method_names = ','.join(unimplemented_methods)
2256 + msg = (f"Methods {method_names} not implemented in {self}")
2257 + logger.warning(msg)
2258 +
2259 + @wraps(original_init)
2260 + def wrapped_init(self, *args, **kwargs) -> None:
2261 + original_init(self, *args, **kwargs)
2262 + find_unimplemented_methods(self)
2263 +
2264 + type.__setattr__(cls, '__init__', wrapped_init)
2265 + return cls
vllm/v1/worker/gpu_worker.py
+9 -19
@@ -21,6 +21,7 @@ from vllm.utils import GiB_bytes
21 from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
22 from vllm.v1.outputs import ModelRunnerOutput
23 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
24 +from vllm.v1.worker.worker_base import WorkerBase
25
26 logger = init_logger(__name__)
27
@@ -28,7 +29,7 @@ if TYPE_CHECKING:
29 from vllm.v1.core.scheduler_output import SchedulerOutput
30
31
31 -class Worker:
32 +class Worker(WorkerBase):
33
34 def __init__(
35 self,
@@ -39,23 +40,11 @@ class Worker:
40 is_driver_worker: bool = False,
41 ):
42
42 - # TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
43 - self.vllm_config = vllm_config
44 - self.model_config = vllm_config.model_config
45 - self.cache_config = vllm_config.cache_config
46 - self.lora_config = vllm_config.lora_config
47 - self.load_config = vllm_config.load_config
48 - self.parallel_config = vllm_config.parallel_config
49 - self.scheduler_config = vllm_config.scheduler_config
50 - self.device_config = vllm_config.device_config
51 - self.speculative_config = vllm_config.speculative_config
52 - self.prompt_adapter_config = vllm_config.prompt_adapter_config
53 - self.observability_config = vllm_config.observability_config
54 -
55 - self.parallel_config.rank = rank
56 - self.local_rank = local_rank
57 - self.rank = rank
58 - self.distributed_init_method = distributed_init_method
43 + super().__init__(vllm_config=vllm_config,
44 + local_rank=local_rank,
45 + rank=rank,
46 + distributed_init_method=distributed_init_method,
47 + is_driver_worker=is_driver_worker)
48
49 if self.model_config.trust_remote_code:
50 # note: lazy import to avoid importing torch before initializing
@@ -126,7 +115,8 @@ class Worker:
115 set_random_seed(self.model_config.seed)
116
117 # Construct the model runner
129 - self.model_runner = GPUModelRunner(self.vllm_config, self.device)
118 + self.model_runner: GPUModelRunner = GPUModelRunner(
119 + self.vllm_config, self.device)
120
121 def load_model(self) -> None:
122 if self.vllm_config.model_config.enable_sleep_mode:
vllm/v1/worker/worker_base.py new
+63
@@ -0,0 +1,63 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from typing import Optional
4 +
5 +import torch
6 +import torch.nn as nn
7 +
8 +from vllm.config import VllmConfig
9 +from vllm.logger import init_logger
10 +from vllm.v1.kv_cache_interface import KVCacheSpec
11 +from vllm.worker.worker_base import WorkerBase as WorkerBaseV0
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class WorkerBase(WorkerBaseV0):
17 + """
18 + Abstract class for v1 worker, mainly define some methods for v1.
19 + For methods shared by v0 and v1, define them in v0 WorkerBase
20 + """
21 +
22 + def __init__(
23 + self,
24 + vllm_config: VllmConfig,
25 + local_rank: int,
26 + rank: int,
27 + distributed_init_method: str,
28 + is_driver_worker: bool = False,
29 + ):
30 + """
31 + Initialize common worker components.
32 +
33 + Args:
34 + vllm_config: Complete vLLM configuration
35 + local_rank: Local device index
36 + rank: Global rank in distributed setup
37 + distributed_init_method: Distributed initialization method
38 + is_driver_worker: Whether this worker handles driver
39 + responsibilities
40 + """
41 + # Configuration storage
42 + super().__init__(vllm_config=vllm_config)
43 +
44 + self.local_rank = local_rank
45 + self.rank = rank
46 + self.distributed_init_method = distributed_init_method
47 + self.is_driver_worker = is_driver_worker
48 +
49 + # Device and model state
50 + self.device: Optional[torch.device] = None
51 + self.model_runner: Optional[nn.Module] = None
52 +
53 + def get_kv_cache_spec(self) -> KVCacheSpec:
54 + """Get specifications for KV cache implementation."""
55 + raise NotImplementedError
56 +
57 + def compile_or_warm_up_model(self) -> None:
58 + """Prepare model for execution through compilation/warmup."""
59 + raise NotImplementedError
60 +
61 + def check_health(self) -> None:
62 + """Basic health check (override for device-specific checks)."""
63 + return
vllm/worker/worker_base.py
+38 -33
@@ -3,7 +3,7 @@
3 import dataclasses
4 import os
5 import time
6 -from abc import ABC, abstractmethod
6 +from abc import abstractmethod
7 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
8
9 import cloudpickle
@@ -19,7 +19,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
20 from vllm.utils import (enable_trace_function_call_for_thread,
21 resolve_obj_by_qualname, run_method,
22 - update_environment_variables)
22 + update_environment_variables,
23 + warn_for_unimplemented_methods)
24 from vllm.worker.model_runner_base import (BroadcastableModelInput,
25 ModelRunnerBase,
26 ModelRunnerInputBase)
@@ -27,7 +28,8 @@ from vllm.worker.model_runner_base import (BroadcastableModelInput,
28 logger = init_logger(__name__)
29
30
30 -class WorkerBase(ABC):
31 +@warn_for_unimplemented_methods
32 +class WorkerBase:
33 """Worker interface that allows vLLM to cleanly separate implementations for
34 different hardware. Also abstracts control plane communication, e.g., to
35 communicate request metadata to other workers.
@@ -53,35 +55,31 @@ class WorkerBase(ABC):
55 from vllm.platforms import current_platform
56 self.current_platform = current_platform
57
56 - @abstractmethod
58 def init_device(self) -> None:
59 """Initialize device state, such as loading the model or other on-device
60 memory allocations.
61 """
62 raise NotImplementedError
63
63 - @abstractmethod
64 - def determine_num_available_blocks(self) -> Tuple[int, int]:
65 - """Determine the number of available blocks for the GPU KV cache and
66 - swappable CPU KV cache.
67 -
68 - The implementation may run profiling or other heuristics to determine
69 - the size of caches.
70 -
71 - Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
72 - are blocks that are "active" on the device and can be appended to.
73 - num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
74 - appended to.
75 - """
76 - raise NotImplementedError
77 -
78 - @abstractmethod
64 def initialize_cache(self, num_gpu_blocks: int,
65 num_cpu_blocks: int) -> None:
66 """Initialize the KV cache with the given size in blocks.
67 """
68 raise NotImplementedError
69
70 + def get_model(self) -> nn.Module:
71 + raise NotImplementedError
72 +
73 + def load_model(self) -> None:
74 + """Load model onto target device."""
75 + raise NotImplementedError
76 +
77 + def execute_model(
78 + self,
79 + execute_model_req: Optional[ExecuteModelRequest] = None
80 + ) -> Optional[List[SamplerOutput]]:
81 + raise NotImplementedError
82 +
83 def start_worker_execution_loop(self) -> None:
84 """Execute model loop in parallel worker.
85
@@ -94,40 +92,43 @@ class WorkerBase(ABC):
92 if output is None:
93 return None
94
97 - @abstractmethod
98 - def get_model(self) -> nn.Module:
99 - raise NotImplementedError
95 + def determine_num_available_blocks(self) -> Tuple[int, int]:
96 + """Determine the number of available blocks for the GPU KV cache and
97 + swappable CPU KV cache.
98
101 - @abstractmethod
102 - def execute_model(
103 - self,
104 - execute_model_req: Optional[ExecuteModelRequest] = None
105 - ) -> Optional[List[SamplerOutput]]:
99 + The implementation may run profiling or other heuristics to determine
100 + the size of caches.
101 +
102 + Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
103 + are blocks that are "active" on the device and can be appended to.
104 + num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
105 + appended to.
106 + """
107 raise NotImplementedError
108
108 - @abstractmethod
109 def get_cache_block_size_bytes(self) -> int:
110 """Return the size of a single cache block, in bytes. Used in
111 speculative decoding.
112 """
113 raise NotImplementedError
114
115 - @abstractmethod
115 def add_lora(self, lora_request: LoRARequest) -> bool:
116 raise NotImplementedError
117
119 - @abstractmethod
118 def remove_lora(self, lora_id: int) -> bool:
119 raise NotImplementedError
120
123 - @abstractmethod
121 def pin_lora(self, lora_id: int) -> bool:
122 raise NotImplementedError
123
127 - @abstractmethod
124 def list_loras(self) -> Set[int]:
125 raise NotImplementedError
126
127 + @property
128 + def vocab_size(self) -> int:
129 + """Get vocabulary size from model configuration."""
130 + return self.model_config.get_vocab_size()
131 +
132
133 class DelegateWorkerBase(WorkerBase):
134 """
@@ -156,6 +157,10 @@ class DelegateWorkerBase(WorkerBase):
157 num_cpu_blocks: int) -> None:
158 self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
159
160 + def load_model(self) -> None:
161 + """Load model onto target device."""
162 + self.worker.load_model()
163 +
164 def get_model(self) -> nn.Module:
165 return self.worker.get_model()
166