A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Collect model support info in a single process per model (#9233)
Cyrus Leung committed
Oct 11, 2024 at 19:08 UTC
e808156f305ce2ecfbe87eefa19ce2ae11c83d00
4 files changed
+228
-159
docs/source/models/adding_model.rst
+1
-1
@@ -99,7 +99,7 @@ This method should load the weights from the HuggingFace's checkpoint file and a
99
5. Register your model
100
----------------------
101
102
-Finally, register your :code:`*ForCausalLM` class to the :code:`_MODELS` in `vllm/model_executor/models/registry.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py>`_.
102
+Finally, register your :code:`*ForCausalLM` class to the :code:`_VLLM_MODELS` in `vllm/model_executor/models/registry.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py>`_.
103
104
6. Out-of-Tree Model Integration
105
--------------------------------------------
vllm/engine/arg_utils.py
+2
@@ -183,6 +183,8 @@ class EngineArgs:
183
def __post_init__(self):
184
if self.tokenizer is None:
185
self.tokenizer = self.model
186
+
187
+ # Setup plugins
188
from vllm.plugins import load_general_plugins
189
load_general_plugins()
190
vllm/engine/multiprocessing/engine.py
+3
@@ -130,6 +130,9 @@ class MQLLMEngine:
130
def from_engine_args(cls, engine_args: AsyncEngineArgs,
131
usage_context: UsageContext, ipc_path: str):
132
"""Creates an MQLLMEngine from the engine arguments."""
133
+ # Setup plugins for each process
134
+ from vllm.plugins import load_general_plugins
135
+ load_general_plugins()
136
137
engine_config = engine_args.create_engine_config()
138
vllm/model_executor/models/registry.py
+222
-158
@@ -3,8 +3,10 @@ import pickle
3
import subprocess
4
import sys
5
import tempfile
6
-from functools import lru_cache, partial
7
-from typing import Callable, Dict, List, Optional, Tuple, Type, Union
6
+from abc import ABC, abstractmethod
7
+from dataclasses import dataclass, field
8
+from functools import lru_cache
9
+from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
10
11
import cloudpickle
12
import torch.nn as nn
@@ -116,18 +118,13 @@ _SPECULATIVE_DECODING_MODELS = {
118
}
119
# yapf: enable
120
119
-_MODELS = {
121
+_VLLM_MODELS = {
122
**_TEXT_GENERATION_MODELS,
123
**_EMBEDDING_MODELS,
124
**_MULTIMODAL_MODELS,
125
**_SPECULATIVE_DECODING_MODELS,
126
}
127
126
-# Architecture -> type or (module, class).
127
-# out of tree models
128
-_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
129
-_OOT_MODELS_LAZY: Dict[str, Tuple[str, str]] = {}
130
-
128
# Models not supported by ROCm.
129
_ROCM_UNSUPPORTED_MODELS: List[str] = []
130
@@ -154,79 +151,125 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
151
}
152
153
157
-class ModelRegistry:
154
+@dataclass(frozen=True)
155
+class _ModelInfo:
156
+ is_text_generation_model: bool
157
+ is_embedding_model: bool
158
+ supports_multimodal: bool
159
+ supports_pp: bool
160
161
@staticmethod
160
- def _get_module_cls_name(model_arch: str) -> Tuple[str, str]:
161
- if model_arch in _MODELS:
162
- module_relname, cls_name = _MODELS[model_arch]
163
- return f"vllm.model_executor.models.{module_relname}", cls_name
162
+ def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
163
+ return _ModelInfo(
164
+ is_text_generation_model=is_text_generation_model(model),
165
+ is_embedding_model=is_embedding_model(model),
166
+ supports_multimodal=supports_multimodal(model),
167
+ supports_pp=supports_pp(model),
168
+ )
169
165
- if model_arch in _OOT_MODELS_LAZY:
166
- return _OOT_MODELS_LAZY[model_arch]
170
168
- raise KeyError(model_arch)
171
+class _BaseRegisteredModel(ABC):
172
170
- @staticmethod
171
- @lru_cache(maxsize=128)
172
- def _try_get_model_stateful(model_arch: str) -> Optional[Type[nn.Module]]:
173
- try:
174
- mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
175
- except KeyError:
176
- return None
173
+ @abstractmethod
174
+ def inspect_model_cls(self) -> _ModelInfo:
175
+ raise NotImplementedError
176
178
- module = importlib.import_module(mod_name)
179
- return getattr(module, cls_name, None)
177
+ @abstractmethod
178
+ def load_model_cls(self) -> Type[nn.Module]:
179
+ raise NotImplementedError
180
181
- @staticmethod
182
- def _try_get_model_stateless(model_arch: str) -> Optional[Type[nn.Module]]:
183
- if model_arch in _OOT_MODELS:
184
- return _OOT_MODELS[model_arch]
185
-
186
- if is_hip():
187
- if model_arch in _ROCM_UNSUPPORTED_MODELS:
188
- raise ValueError(
189
- f"Model architecture {model_arch} is not supported by "
190
- "ROCm for now.")
191
- if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
192
- logger.warning(
193
- "Model architecture %s is partially supported by ROCm: %s",
194
- model_arch, _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
181
196
- return None
182
+@dataclass(frozen=True)
183
+class _RegisteredModel(_BaseRegisteredModel):
184
+ """
185
+ Represents a model that has already been imported in the main process.
186
+ """
187
+
188
+ interfaces: _ModelInfo
189
+ model_cls: Type[nn.Module]
190
191
@staticmethod
199
- def _try_load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
200
- model = ModelRegistry._try_get_model_stateless(model_arch)
201
- if model is not None:
202
- return model
192
+ def from_model_cls(model_cls: Type[nn.Module]):
193
+ return _RegisteredModel(
194
+ interfaces=_ModelInfo.from_model_cls(model_cls),
195
+ model_cls=model_cls,
196
+ )
197
+
198
+ def inspect_model_cls(self) -> _ModelInfo:
199
+ return self.interfaces
200
+
201
+ def load_model_cls(self) -> Type[nn.Module]:
202
+ return self.model_cls
203
+
204
+
205
+@dataclass(frozen=True)
206
+class _LazyRegisteredModel(_BaseRegisteredModel):
207
+ """
208
+ Represents a model that has not been imported in the main process.
209
+ """
210
+ module_name: str
211
+ class_name: str
212
+
213
+ # Performed in another process to avoid initializing CUDA
214
+ def inspect_model_cls(self) -> _ModelInfo:
215
+ return _run_in_subprocess(
216
+ lambda: _ModelInfo.from_model_cls(self.load_model_cls()))
217
+
218
+ def load_model_cls(self) -> Type[nn.Module]:
219
+ mod = importlib.import_module(self.module_name)
220
+ return getattr(mod, self.class_name)
221
+
222
+
223
+@lru_cache(maxsize=128)
224
+def _try_load_model_cls(
225
+ model_arch: str,
226
+ model: _BaseRegisteredModel,
227
+) -> Optional[Type[nn.Module]]:
228
+ if is_hip():
229
+ if model_arch in _ROCM_UNSUPPORTED_MODELS:
230
+ raise ValueError(f"Model architecture '{model_arch}' is not "
231
+ "supported by ROCm for now.")
232
+
233
+ if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
234
+ msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
235
+ logger.warning(
236
+ "Model architecture '%s' is partially "
237
+ "supported by ROCm: %s", model_arch, msg)
238
+
239
+ try:
240
+ return model.load_model_cls()
241
+ except Exception:
242
+ logger.exception("Error in loading model architecture '%s'",
243
+ model_arch)
244
+ return None
245
204
- return ModelRegistry._try_get_model_stateful(model_arch)
246
206
- @staticmethod
207
- def resolve_model_cls(
208
- architectures: Union[str, List[str]], ) -> Tuple[Type[nn.Module], str]:
209
- if isinstance(architectures, str):
210
- architectures = [architectures]
211
- if not architectures:
212
- logger.warning("No model architectures are specified")
247
+@lru_cache(maxsize=128)
248
+def _try_inspect_model_cls(
249
+ model_arch: str,
250
+ model: _BaseRegisteredModel,
251
+) -> Optional[_ModelInfo]:
252
+ try:
253
+ return model.inspect_model_cls()
254
+ except Exception:
255
+ logger.exception("Error in inspecting model architecture '%s'",
256
+ model_arch)
257
+ return None
258
214
- for arch in architectures:
215
- model_cls = ModelRegistry._try_load_model_cls(arch)
216
- if model_cls is not None:
217
- return (model_cls, arch)
259
219
- raise ValueError(
220
- f"Model architectures {architectures} are not supported for now. "
221
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
260
+@dataclass
261
+class _ModelRegistry:
262
+ # Keyed by model_arch
263
+ models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
264
223
- @staticmethod
224
- def get_supported_archs() -> List[str]:
225
- return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
265
+ def get_supported_archs(self) -> List[str]:
266
+ return list(self.models.keys())
267
227
- @staticmethod
228
- def register_model(model_arch: str, model_cls: Union[Type[nn.Module],
229
- str]):
268
+ def register_model(
269
+ self,
270
+ model_arch: str,
271
+ model_cls: Union[Type[nn.Module], str],
272
+ ) -> None:
273
"""
274
Register an external model to be used in vLLM.
275
@@ -238,7 +281,7 @@ class ModelRegistry:
281
when importing the model and thus the related error
282
:code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
283
"""
241
- if model_arch in _MODELS:
284
+ if model_arch in self.models:
285
logger.warning(
286
"Model architecture %s is already registered, and will be "
287
"overwritten by the new model class %s.", model_arch,
@@ -250,120 +293,141 @@ class ModelRegistry:
293
msg = "Expected a string in the format `<module>:<class>`"
294
raise ValueError(msg)
295
253
- module_name, cls_name = split_str
254
- _OOT_MODELS_LAZY[model_arch] = module_name, cls_name
296
+ model = _LazyRegisteredModel(*split_str)
297
else:
256
- _OOT_MODELS[model_arch] = model_cls
298
+ model = _RegisteredModel.from_model_cls(model_cls)
299
258
- @staticmethod
259
- @lru_cache(maxsize=128)
260
- def _check_stateless(
261
- func: Callable[[Type[nn.Module]], bool],
262
- model_arch: str,
263
- *,
264
- default: Optional[bool] = None,
265
- ) -> bool:
266
- """
267
- Run a boolean function against a model and return the result.
300
+ self.models[model_arch] = model
301
269
- If the model is not found, returns the provided default value.
302
+ def _raise_for_unsupported(self, architectures: List[str]):
303
+ all_supported_archs = self.get_supported_archs()
304
271
- If the model is not already imported, the function is run inside a
272
- subprocess to avoid initializing CUDA for the main program.
273
- """
274
- model = ModelRegistry._try_get_model_stateless(model_arch)
275
- if model is not None:
276
- return func(model)
305
+ raise ValueError(
306
+ f"Model architectures {architectures} are not supported for now. "
307
+ f"Supported architectures: {all_supported_archs}")
308
278
- try:
279
- mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
280
- except KeyError:
281
- if default is not None:
282
- return default
283
-
284
- raise
285
-
286
- with tempfile.NamedTemporaryFile() as output_file:
287
- # `cloudpickle` allows pickling lambda functions directly
288
- input_bytes = cloudpickle.dumps(
289
- (mod_name, cls_name, func, output_file.name))
290
- # cannot use `sys.executable __file__` here because the script
291
- # contains relative imports
292
- returned = subprocess.run(
293
- [sys.executable, "-m", "vllm.model_executor.models.registry"],
294
- input=input_bytes,
295
- capture_output=True)
296
-
297
- # check if the subprocess is successful
298
- try:
299
- returned.check_returncode()
300
- except Exception as e:
301
- # wrap raised exception to provide more information
302
- raise RuntimeError(f"Error happened when testing "
303
- f"model support for{mod_name}.{cls_name}:\n"
304
- f"{returned.stderr.decode()}") from e
305
- with open(output_file.name, "rb") as f:
306
- result = pickle.load(f)
307
- return result
309
+ def _try_load_model_cls(self,
310
+ model_arch: str) -> Optional[Type[nn.Module]]:
311
+ if model_arch not in self.models:
312
+ return None
313
309
- @staticmethod
310
- def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
311
- if isinstance(architectures, str):
312
- architectures = [architectures]
313
- if not architectures:
314
- logger.warning("No model architectures are specified")
314
+ return _try_load_model_cls(model_arch, self.models[model_arch])
315
316
- is_txt_gen = partial(ModelRegistry._check_stateless,
317
- is_text_generation_model,
318
- default=False)
316
+ def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
317
+ if model_arch not in self.models:
318
+ return None
319
320
- return any(is_txt_gen(arch) for arch in architectures)
320
+ return _try_inspect_model_cls(model_arch, self.models[model_arch])
321
322
- @staticmethod
323
- def is_embedding_model(architectures: Union[str, List[str]]) -> bool:
322
+ def _normalize_archs(
323
+ self,
324
+ architectures: Union[str, List[str]],
325
+ ) -> List[str]:
326
if isinstance(architectures, str):
327
architectures = [architectures]
328
if not architectures:
329
logger.warning("No model architectures are specified")
330
329
- is_emb = partial(ModelRegistry._check_stateless,
330
- is_embedding_model,
331
- default=False)
331
+ return architectures
332
333
- return any(is_emb(arch) for arch in architectures)
333
+ def inspect_model_cls(
334
+ self,
335
+ architectures: Union[str, List[str]],
336
+ ) -> _ModelInfo:
337
+ architectures = self._normalize_archs(architectures)
338
335
- @staticmethod
336
- def is_multimodal_model(architectures: Union[str, List[str]]) -> bool:
337
- if isinstance(architectures, str):
338
- architectures = [architectures]
339
- if not architectures:
340
- logger.warning("No model architectures are specified")
339
+ for arch in architectures:
340
+ model_info = self._try_inspect_model_cls(arch)
341
+ if model_info is not None:
342
+ return model_info
343
342
- is_mm = partial(ModelRegistry._check_stateless,
343
- supports_multimodal,
344
- default=False)
344
+ return self._raise_for_unsupported(architectures)
345
346
- return any(is_mm(arch) for arch in architectures)
346
+ def resolve_model_cls(
347
+ self,
348
+ architectures: Union[str, List[str]],
349
+ ) -> Tuple[Type[nn.Module], str]:
350
+ architectures = self._normalize_archs(architectures)
351
348
- @staticmethod
349
- def is_pp_supported_model(architectures: Union[str, List[str]]) -> bool:
350
- if isinstance(architectures, str):
351
- architectures = [architectures]
352
- if not architectures:
353
- logger.warning("No model architectures are specified")
352
+ for arch in architectures:
353
+ model_cls = self._try_load_model_cls(arch)
354
+ if model_cls is not None:
355
+ return (model_cls, arch)
356
355
- is_pp = partial(ModelRegistry._check_stateless,
356
- supports_pp,
357
- default=False)
357
+ return self._raise_for_unsupported(architectures)
358
359
- return any(is_pp(arch) for arch in architectures)
359
+ def is_text_generation_model(
360
+ self,
361
+ architectures: Union[str, List[str]],
362
+ ) -> bool:
363
+ return self.inspect_model_cls(architectures).is_text_generation_model
364
365
+ def is_embedding_model(
366
+ self,
367
+ architectures: Union[str, List[str]],
368
+ ) -> bool:
369
+ return self.inspect_model_cls(architectures).is_embedding_model
370
+
371
+ def is_multimodal_model(
372
+ self,
373
+ architectures: Union[str, List[str]],
374
+ ) -> bool:
375
+ return self.inspect_model_cls(architectures).supports_multimodal
376
+
377
+ def is_pp_supported_model(
378
+ self,
379
+ architectures: Union[str, List[str]],
380
+ ) -> bool:
381
+ return self.inspect_model_cls(architectures).supports_pp
382
+
383
+
384
+ModelRegistry = _ModelRegistry({
385
+ model_arch: _LazyRegisteredModel(
386
+ module_name=f"vllm.model_executor.models.{mod_relname}",
387
+ class_name=cls_name,
388
+ )
389
+ for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
390
+})
391
+
392
+_T = TypeVar("_T")
393
+
394
+
395
+def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
396
+ with tempfile.NamedTemporaryFile() as output_file:
397
+ # `cloudpickle` allows pickling lambda functions directly
398
+ input_bytes = cloudpickle.dumps((fn, output_file.name))
399
+
400
+ # cannot use `sys.executable __file__` here because the script
401
+ # contains relative imports
402
+ returned = subprocess.run(
403
+ [sys.executable, "-m", "vllm.model_executor.models.registry"],
404
+ input=input_bytes,
405
+ capture_output=True)
406
+
407
+ # check if the subprocess is successful
408
+ try:
409
+ returned.check_returncode()
410
+ except Exception as e:
411
+ # wrap raised exception to provide more information
412
+ raise RuntimeError(f"Error raised in subprocess:\n"
413
+ f"{returned.stderr.decode()}") from e
414
+
415
+ with open(output_file.name, "rb") as f:
416
+ return pickle.load(f)
417
+
418
+
419
+def _run() -> None:
420
+ # Setup plugins
421
+ from vllm.plugins import load_general_plugins
422
+ load_general_plugins()
423
+
424
+ fn, output_file = pickle.loads(sys.stdin.buffer.read())
425
+
426
+ result = fn()
427
362
-if __name__ == "__main__":
363
- (mod_name, cls_name, func,
364
- output_file) = pickle.loads(sys.stdin.buffer.read())
365
- mod = importlib.import_module(mod_name)
366
- klass = getattr(mod, cls_name)
367
- result = func(klass)
428
with open(output_file, "wb") as f:
429
f.write(pickle.dumps(result))
430
+
431
+
432
+if __name__ == "__main__":
433
+ _run()