A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Core] Add worker_base for v1 worker (#12816)
Signed-off-by: Aoyu <aoyuzhan@amazon.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Aoyu <aoyuzhan@amazon.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Aoyu committed
Feb 13, 2025 at 20:35 UTC
2092a6fa7d58e398bc1a234edcef62ce555faafe
4 files changed
+153
-52
vllm/utils.py
+43
@@ -2220,3 +2220,46 @@ def import_pynvml():
2220
"""
2221
import vllm.third_party.pynvml as pynvml
2222
return pynvml
2223
+
2224
+
2225
+def warn_for_unimplemented_methods(cls: Type[T]) -> Type[T]:
2226
+ """
2227
+ A replacement for `abc.ABC`.
2228
+ When we use `abc.ABC`, subclasses will fail to instantiate
2229
+ if they do not implement all abstract methods.
2230
+ Here, we only require `raise NotImplementedError` in the
2231
+ base class, and log a warning if the method is not implemented
2232
+ in the subclass.
2233
+ """
2234
+
2235
+ original_init = cls.__init__
2236
+
2237
+ def find_unimplemented_methods(self: object):
2238
+ unimplemented_methods = []
2239
+ for attr_name in dir(self):
2240
+ # bypass inner method
2241
+ if attr_name.startswith('_'):
2242
+ continue
2243
+
2244
+ try:
2245
+ attr = getattr(self, attr_name)
2246
+ # get the func of callable method
2247
+ if callable(attr):
2248
+ attr_func = attr.__func__
2249
+ except AttributeError:
2250
+ continue
2251
+ src = inspect.getsource(attr_func)
2252
+ if "NotImplementedError" in src:
2253
+ unimplemented_methods.append(attr_name)
2254
+ if unimplemented_methods:
2255
+ method_names = ','.join(unimplemented_methods)
2256
+ msg = (f"Methods {method_names} not implemented in {self}")
2257
+ logger.warning(msg)
2258
+
2259
+ @wraps(original_init)
2260
+ def wrapped_init(self, *args, **kwargs) -> None:
2261
+ original_init(self, *args, **kwargs)
2262
+ find_unimplemented_methods(self)
2263
+
2264
+ type.__setattr__(cls, '__init__', wrapped_init)
2265
+ return cls
vllm/v1/worker/gpu_worker.py
+9
-19
@@ -21,6 +21,7 @@ from vllm.utils import GiB_bytes
21
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
22
from vllm.v1.outputs import ModelRunnerOutput
23
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
24
+from vllm.v1.worker.worker_base import WorkerBase
25
26
logger = init_logger(__name__)
27
@@ -28,7 +29,7 @@ if TYPE_CHECKING:
29
from vllm.v1.core.scheduler_output import SchedulerOutput
30
31
31
-class Worker:
32
+class Worker(WorkerBase):
33
34
def __init__(
35
self,
@@ -39,23 +40,11 @@ class Worker:
40
is_driver_worker: bool = False,
41
):
42
42
- # TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
43
- self.vllm_config = vllm_config
44
- self.model_config = vllm_config.model_config
45
- self.cache_config = vllm_config.cache_config
46
- self.lora_config = vllm_config.lora_config
47
- self.load_config = vllm_config.load_config
48
- self.parallel_config = vllm_config.parallel_config
49
- self.scheduler_config = vllm_config.scheduler_config
50
- self.device_config = vllm_config.device_config
51
- self.speculative_config = vllm_config.speculative_config
52
- self.prompt_adapter_config = vllm_config.prompt_adapter_config
53
- self.observability_config = vllm_config.observability_config
54
-
55
- self.parallel_config.rank = rank
56
- self.local_rank = local_rank
57
- self.rank = rank
58
- self.distributed_init_method = distributed_init_method
43
+ super().__init__(vllm_config=vllm_config,
44
+ local_rank=local_rank,
45
+ rank=rank,
46
+ distributed_init_method=distributed_init_method,
47
+ is_driver_worker=is_driver_worker)
48
49
if self.model_config.trust_remote_code:
50
# note: lazy import to avoid importing torch before initializing
@@ -126,7 +115,8 @@ class Worker:
115
set_random_seed(self.model_config.seed)
116
117
# Construct the model runner
129
- self.model_runner = GPUModelRunner(self.vllm_config, self.device)
118
+ self.model_runner: GPUModelRunner = GPUModelRunner(
119
+ self.vllm_config, self.device)
120
121
def load_model(self) -> None:
122
if self.vllm_config.model_config.enable_sleep_mode:
vllm/v1/worker/worker_base.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from typing import Optional
4
+
5
+import torch
6
+import torch.nn as nn
7
+
8
+from vllm.config import VllmConfig
9
+from vllm.logger import init_logger
10
+from vllm.v1.kv_cache_interface import KVCacheSpec
11
+from vllm.worker.worker_base import WorkerBase as WorkerBaseV0
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+class WorkerBase(WorkerBaseV0):
17
+ """
18
+ Abstract class for v1 worker, mainly define some methods for v1.
19
+ For methods shared by v0 and v1, define them in v0 WorkerBase
20
+ """
21
+
22
+ def __init__(
23
+ self,
24
+ vllm_config: VllmConfig,
25
+ local_rank: int,
26
+ rank: int,
27
+ distributed_init_method: str,
28
+ is_driver_worker: bool = False,
29
+ ):
30
+ """
31
+ Initialize common worker components.
32
+
33
+ Args:
34
+ vllm_config: Complete vLLM configuration
35
+ local_rank: Local device index
36
+ rank: Global rank in distributed setup
37
+ distributed_init_method: Distributed initialization method
38
+ is_driver_worker: Whether this worker handles driver
39
+ responsibilities
40
+ """
41
+ # Configuration storage
42
+ super().__init__(vllm_config=vllm_config)
43
+
44
+ self.local_rank = local_rank
45
+ self.rank = rank
46
+ self.distributed_init_method = distributed_init_method
47
+ self.is_driver_worker = is_driver_worker
48
+
49
+ # Device and model state
50
+ self.device: Optional[torch.device] = None
51
+ self.model_runner: Optional[nn.Module] = None
52
+
53
+ def get_kv_cache_spec(self) -> KVCacheSpec:
54
+ """Get specifications for KV cache implementation."""
55
+ raise NotImplementedError
56
+
57
+ def compile_or_warm_up_model(self) -> None:
58
+ """Prepare model for execution through compilation/warmup."""
59
+ raise NotImplementedError
60
+
61
+ def check_health(self) -> None:
62
+ """Basic health check (override for device-specific checks)."""
63
+ return
vllm/worker/worker_base.py
+38
-33
@@ -3,7 +3,7 @@
3
import dataclasses
4
import os
5
import time
6
-from abc import ABC, abstractmethod
6
+from abc import abstractmethod
7
from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
8
9
import cloudpickle
@@ -19,7 +19,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput
19
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
20
from vllm.utils import (enable_trace_function_call_for_thread,
21
resolve_obj_by_qualname, run_method,
22
- update_environment_variables)
22
+ update_environment_variables,
23
+ warn_for_unimplemented_methods)
24
from vllm.worker.model_runner_base import (BroadcastableModelInput,
25
ModelRunnerBase,
26
ModelRunnerInputBase)
@@ -27,7 +28,8 @@ from vllm.worker.model_runner_base import (BroadcastableModelInput,
28
logger = init_logger(__name__)
29
30
30
-class WorkerBase(ABC):
31
+@warn_for_unimplemented_methods
32
+class WorkerBase:
33
"""Worker interface that allows vLLM to cleanly separate implementations for
34
different hardware. Also abstracts control plane communication, e.g., to
35
communicate request metadata to other workers.
@@ -53,35 +55,31 @@ class WorkerBase(ABC):
55
from vllm.platforms import current_platform
56
self.current_platform = current_platform
57
56
- @abstractmethod
58
def init_device(self) -> None:
59
"""Initialize device state, such as loading the model or other on-device
60
memory allocations.
61
"""
62
raise NotImplementedError
63
63
- @abstractmethod
64
- def determine_num_available_blocks(self) -> Tuple[int, int]:
65
- """Determine the number of available blocks for the GPU KV cache and
66
- swappable CPU KV cache.
67
-
68
- The implementation may run profiling or other heuristics to determine
69
- the size of caches.
70
-
71
- Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
72
- are blocks that are "active" on the device and can be appended to.
73
- num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
74
- appended to.
75
- """
76
- raise NotImplementedError
77
-
78
- @abstractmethod
64
def initialize_cache(self, num_gpu_blocks: int,
65
num_cpu_blocks: int) -> None:
66
"""Initialize the KV cache with the given size in blocks.
67
"""
68
raise NotImplementedError
69
70
+ def get_model(self) -> nn.Module:
71
+ raise NotImplementedError
72
+
73
+ def load_model(self) -> None:
74
+ """Load model onto target device."""
75
+ raise NotImplementedError
76
+
77
+ def execute_model(
78
+ self,
79
+ execute_model_req: Optional[ExecuteModelRequest] = None
80
+ ) -> Optional[List[SamplerOutput]]:
81
+ raise NotImplementedError
82
+
83
def start_worker_execution_loop(self) -> None:
84
"""Execute model loop in parallel worker.
85
@@ -94,40 +92,43 @@ class WorkerBase(ABC):
92
if output is None:
93
return None
94
97
- @abstractmethod
98
- def get_model(self) -> nn.Module:
99
- raise NotImplementedError
95
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
96
+ """Determine the number of available blocks for the GPU KV cache and
97
+ swappable CPU KV cache.
98
101
- @abstractmethod
102
- def execute_model(
103
- self,
104
- execute_model_req: Optional[ExecuteModelRequest] = None
105
- ) -> Optional[List[SamplerOutput]]:
99
+ The implementation may run profiling or other heuristics to determine
100
+ the size of caches.
101
+
102
+ Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
103
+ are blocks that are "active" on the device and can be appended to.
104
+ num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
105
+ appended to.
106
+ """
107
raise NotImplementedError
108
108
- @abstractmethod
109
def get_cache_block_size_bytes(self) -> int:
110
"""Return the size of a single cache block, in bytes. Used in
111
speculative decoding.
112
"""
113
raise NotImplementedError
114
115
- @abstractmethod
115
def add_lora(self, lora_request: LoRARequest) -> bool:
116
raise NotImplementedError
117
119
- @abstractmethod
118
def remove_lora(self, lora_id: int) -> bool:
119
raise NotImplementedError
120
123
- @abstractmethod
121
def pin_lora(self, lora_id: int) -> bool:
122
raise NotImplementedError
123
127
- @abstractmethod
124
def list_loras(self) -> Set[int]:
125
raise NotImplementedError
126
127
+ @property
128
+ def vocab_size(self) -> int:
129
+ """Get vocabulary size from model configuration."""
130
+ return self.model_config.get_vocab_size()
131
+
132
133
class DelegateWorkerBase(WorkerBase):
134
"""
@@ -156,6 +157,10 @@ class DelegateWorkerBase(WorkerBase):
157
num_cpu_blocks: int) -> None:
158
self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
159
160
+ def load_model(self) -> None:
161
+ """Load model onto target device."""
162
+ self.worker.load_model()
163
+
164
def get_model(self) -> nn.Module:
165
return self.worker.get_model()
166