@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e808156f30
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Collect model support info in a single process per model (#9233)

Cyrus Leung committed Oct 11, 2024 at 19:08 UTC e808156f305ce2ecfbe87eefa19ce2ae11c83d00
4 files changed +228 -159
docs/source/models/adding_model.rst
+1 -1
@@ -99,7 +99,7 @@ This method should load the weights from the HuggingFace's checkpoint file and a
99 5. Register your model
100 ----------------------
101
102 -Finally, register your :code:`*ForCausalLM` class to the :code:`_MODELS` in `vllm/model_executor/models/registry.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py>`_.
102 +Finally, register your :code:`*ForCausalLM` class to the :code:`_VLLM_MODELS` in `vllm/model_executor/models/registry.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py>`_.
103
104 6. Out-of-Tree Model Integration
105 --------------------------------------------
vllm/engine/arg_utils.py
+2
@@ -183,6 +183,8 @@ class EngineArgs:
183 def __post_init__(self):
184 if self.tokenizer is None:
185 self.tokenizer = self.model
186 +
187 + # Setup plugins
188 from vllm.plugins import load_general_plugins
189 load_general_plugins()
190
vllm/engine/multiprocessing/engine.py
+3
@@ -130,6 +130,9 @@ class MQLLMEngine:
130 def from_engine_args(cls, engine_args: AsyncEngineArgs,
131 usage_context: UsageContext, ipc_path: str):
132 """Creates an MQLLMEngine from the engine arguments."""
133 + # Setup plugins for each process
134 + from vllm.plugins import load_general_plugins
135 + load_general_plugins()
136
137 engine_config = engine_args.create_engine_config()
138
vllm/model_executor/models/registry.py
+222 -158
@@ -3,8 +3,10 @@ import pickle
3 import subprocess
4 import sys
5 import tempfile
6 -from functools import lru_cache, partial
7 -from typing import Callable, Dict, List, Optional, Tuple, Type, Union
6 +from abc import ABC, abstractmethod
7 +from dataclasses import dataclass, field
8 +from functools import lru_cache
9 +from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
10
11 import cloudpickle
12 import torch.nn as nn
@@ -116,18 +118,13 @@ _SPECULATIVE_DECODING_MODELS = {
118 }
119 # yapf: enable
120
119 -_MODELS = {
121 +_VLLM_MODELS = {
122 **_TEXT_GENERATION_MODELS,
123 **_EMBEDDING_MODELS,
124 **_MULTIMODAL_MODELS,
125 **_SPECULATIVE_DECODING_MODELS,
126 }
127
126 -# Architecture -> type or (module, class).
127 -# out of tree models
128 -_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
129 -_OOT_MODELS_LAZY: Dict[str, Tuple[str, str]] = {}
130 -
128 # Models not supported by ROCm.
129 _ROCM_UNSUPPORTED_MODELS: List[str] = []
130
@@ -154,79 +151,125 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
151 }
152
153
157 -class ModelRegistry:
154 +@dataclass(frozen=True)
155 +class _ModelInfo:
156 + is_text_generation_model: bool
157 + is_embedding_model: bool
158 + supports_multimodal: bool
159 + supports_pp: bool
160
161 @staticmethod
160 - def _get_module_cls_name(model_arch: str) -> Tuple[str, str]:
161 - if model_arch in _MODELS:
162 - module_relname, cls_name = _MODELS[model_arch]
163 - return f"vllm.model_executor.models.{module_relname}", cls_name
162 + def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
163 + return _ModelInfo(
164 + is_text_generation_model=is_text_generation_model(model),
165 + is_embedding_model=is_embedding_model(model),
166 + supports_multimodal=supports_multimodal(model),
167 + supports_pp=supports_pp(model),
168 + )
169
165 - if model_arch in _OOT_MODELS_LAZY:
166 - return _OOT_MODELS_LAZY[model_arch]
170
168 - raise KeyError(model_arch)
171 +class _BaseRegisteredModel(ABC):
172
170 - @staticmethod
171 - @lru_cache(maxsize=128)
172 - def _try_get_model_stateful(model_arch: str) -> Optional[Type[nn.Module]]:
173 - try:
174 - mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
175 - except KeyError:
176 - return None
173 + @abstractmethod
174 + def inspect_model_cls(self) -> _ModelInfo:
175 + raise NotImplementedError
176
178 - module = importlib.import_module(mod_name)
179 - return getattr(module, cls_name, None)
177 + @abstractmethod
178 + def load_model_cls(self) -> Type[nn.Module]:
179 + raise NotImplementedError
180
181 - @staticmethod
182 - def _try_get_model_stateless(model_arch: str) -> Optional[Type[nn.Module]]:
183 - if model_arch in _OOT_MODELS:
184 - return _OOT_MODELS[model_arch]
185 -
186 - if is_hip():
187 - if model_arch in _ROCM_UNSUPPORTED_MODELS:
188 - raise ValueError(
189 - f"Model architecture {model_arch} is not supported by "
190 - "ROCm for now.")
191 - if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
192 - logger.warning(
193 - "Model architecture %s is partially supported by ROCm: %s",
194 - model_arch, _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
181
196 - return None
182 +@dataclass(frozen=True)
183 +class _RegisteredModel(_BaseRegisteredModel):
184 + """
185 + Represents a model that has already been imported in the main process.
186 + """
187 +
188 + interfaces: _ModelInfo
189 + model_cls: Type[nn.Module]
190
191 @staticmethod
199 - def _try_load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
200 - model = ModelRegistry._try_get_model_stateless(model_arch)
201 - if model is not None:
202 - return model
192 + def from_model_cls(model_cls: Type[nn.Module]):
193 + return _RegisteredModel(
194 + interfaces=_ModelInfo.from_model_cls(model_cls),
195 + model_cls=model_cls,
196 + )
197 +
198 + def inspect_model_cls(self) -> _ModelInfo:
199 + return self.interfaces
200 +
201 + def load_model_cls(self) -> Type[nn.Module]:
202 + return self.model_cls
203 +
204 +
205 +@dataclass(frozen=True)
206 +class _LazyRegisteredModel(_BaseRegisteredModel):
207 + """
208 + Represents a model that has not been imported in the main process.
209 + """
210 + module_name: str
211 + class_name: str
212 +
213 + # Performed in another process to avoid initializing CUDA
214 + def inspect_model_cls(self) -> _ModelInfo:
215 + return _run_in_subprocess(
216 + lambda: _ModelInfo.from_model_cls(self.load_model_cls()))
217 +
218 + def load_model_cls(self) -> Type[nn.Module]:
219 + mod = importlib.import_module(self.module_name)
220 + return getattr(mod, self.class_name)
221 +
222 +
223 +@lru_cache(maxsize=128)
224 +def _try_load_model_cls(
225 + model_arch: str,
226 + model: _BaseRegisteredModel,
227 +) -> Optional[Type[nn.Module]]:
228 + if is_hip():
229 + if model_arch in _ROCM_UNSUPPORTED_MODELS:
230 + raise ValueError(f"Model architecture '{model_arch}' is not "
231 + "supported by ROCm for now.")
232 +
233 + if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
234 + msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
235 + logger.warning(
236 + "Model architecture '%s' is partially "
237 + "supported by ROCm: %s", model_arch, msg)
238 +
239 + try:
240 + return model.load_model_cls()
241 + except Exception:
242 + logger.exception("Error in loading model architecture '%s'",
243 + model_arch)
244 + return None
245
204 - return ModelRegistry._try_get_model_stateful(model_arch)
246
206 - @staticmethod
207 - def resolve_model_cls(
208 - architectures: Union[str, List[str]], ) -> Tuple[Type[nn.Module], str]:
209 - if isinstance(architectures, str):
210 - architectures = [architectures]
211 - if not architectures:
212 - logger.warning("No model architectures are specified")
247 +@lru_cache(maxsize=128)
248 +def _try_inspect_model_cls(
249 + model_arch: str,
250 + model: _BaseRegisteredModel,
251 +) -> Optional[_ModelInfo]:
252 + try:
253 + return model.inspect_model_cls()
254 + except Exception:
255 + logger.exception("Error in inspecting model architecture '%s'",
256 + model_arch)
257 + return None
258
214 - for arch in architectures:
215 - model_cls = ModelRegistry._try_load_model_cls(arch)
216 - if model_cls is not None:
217 - return (model_cls, arch)
259
219 - raise ValueError(
220 - f"Model architectures {architectures} are not supported for now. "
221 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
260 +@dataclass
261 +class _ModelRegistry:
262 + # Keyed by model_arch
263 + models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
264
223 - @staticmethod
224 - def get_supported_archs() -> List[str]:
225 - return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
265 + def get_supported_archs(self) -> List[str]:
266 + return list(self.models.keys())
267
227 - @staticmethod
228 - def register_model(model_arch: str, model_cls: Union[Type[nn.Module],
229 - str]):
268 + def register_model(
269 + self,
270 + model_arch: str,
271 + model_cls: Union[Type[nn.Module], str],
272 + ) -> None:
273 """
274 Register an external model to be used in vLLM.
275
@@ -238,7 +281,7 @@ class ModelRegistry:
281 when importing the model and thus the related error
282 :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
283 """
241 - if model_arch in _MODELS:
284 + if model_arch in self.models:
285 logger.warning(
286 "Model architecture %s is already registered, and will be "
287 "overwritten by the new model class %s.", model_arch,
@@ -250,120 +293,141 @@ class ModelRegistry:
293 msg = "Expected a string in the format `<module>:<class>`"
294 raise ValueError(msg)
295
253 - module_name, cls_name = split_str
254 - _OOT_MODELS_LAZY[model_arch] = module_name, cls_name
296 + model = _LazyRegisteredModel(*split_str)
297 else:
256 - _OOT_MODELS[model_arch] = model_cls
298 + model = _RegisteredModel.from_model_cls(model_cls)
299
258 - @staticmethod
259 - @lru_cache(maxsize=128)
260 - def _check_stateless(
261 - func: Callable[[Type[nn.Module]], bool],
262 - model_arch: str,
263 - *,
264 - default: Optional[bool] = None,
265 - ) -> bool:
266 - """
267 - Run a boolean function against a model and return the result.
300 + self.models[model_arch] = model
301
269 - If the model is not found, returns the provided default value.
302 + def _raise_for_unsupported(self, architectures: List[str]):
303 + all_supported_archs = self.get_supported_archs()
304
271 - If the model is not already imported, the function is run inside a
272 - subprocess to avoid initializing CUDA for the main program.
273 - """
274 - model = ModelRegistry._try_get_model_stateless(model_arch)
275 - if model is not None:
276 - return func(model)
305 + raise ValueError(
306 + f"Model architectures {architectures} are not supported for now. "
307 + f"Supported architectures: {all_supported_archs}")
308
278 - try:
279 - mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
280 - except KeyError:
281 - if default is not None:
282 - return default
283 -
284 - raise
285 -
286 - with tempfile.NamedTemporaryFile() as output_file:
287 - # `cloudpickle` allows pickling lambda functions directly
288 - input_bytes = cloudpickle.dumps(
289 - (mod_name, cls_name, func, output_file.name))
290 - # cannot use `sys.executable __file__` here because the script
291 - # contains relative imports
292 - returned = subprocess.run(
293 - [sys.executable, "-m", "vllm.model_executor.models.registry"],
294 - input=input_bytes,
295 - capture_output=True)
296 -
297 - # check if the subprocess is successful
298 - try:
299 - returned.check_returncode()
300 - except Exception as e:
301 - # wrap raised exception to provide more information
302 - raise RuntimeError(f"Error happened when testing "
303 - f"model support for{mod_name}.{cls_name}:\n"
304 - f"{returned.stderr.decode()}") from e
305 - with open(output_file.name, "rb") as f:
306 - result = pickle.load(f)
307 - return result
309 + def _try_load_model_cls(self,
310 + model_arch: str) -> Optional[Type[nn.Module]]:
311 + if model_arch not in self.models:
312 + return None
313
309 - @staticmethod
310 - def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
311 - if isinstance(architectures, str):
312 - architectures = [architectures]
313 - if not architectures:
314 - logger.warning("No model architectures are specified")
314 + return _try_load_model_cls(model_arch, self.models[model_arch])
315
316 - is_txt_gen = partial(ModelRegistry._check_stateless,
317 - is_text_generation_model,
318 - default=False)
316 + def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
317 + if model_arch not in self.models:
318 + return None
319
320 - return any(is_txt_gen(arch) for arch in architectures)
320 + return _try_inspect_model_cls(model_arch, self.models[model_arch])
321
322 - @staticmethod
323 - def is_embedding_model(architectures: Union[str, List[str]]) -> bool:
322 + def _normalize_archs(
323 + self,
324 + architectures: Union[str, List[str]],
325 + ) -> List[str]:
326 if isinstance(architectures, str):
327 architectures = [architectures]
328 if not architectures:
329 logger.warning("No model architectures are specified")
330
329 - is_emb = partial(ModelRegistry._check_stateless,
330 - is_embedding_model,
331 - default=False)
331 + return architectures
332
333 - return any(is_emb(arch) for arch in architectures)
333 + def inspect_model_cls(
334 + self,
335 + architectures: Union[str, List[str]],
336 + ) -> _ModelInfo:
337 + architectures = self._normalize_archs(architectures)
338
335 - @staticmethod
336 - def is_multimodal_model(architectures: Union[str, List[str]]) -> bool:
337 - if isinstance(architectures, str):
338 - architectures = [architectures]
339 - if not architectures:
340 - logger.warning("No model architectures are specified")
339 + for arch in architectures:
340 + model_info = self._try_inspect_model_cls(arch)
341 + if model_info is not None:
342 + return model_info
343
342 - is_mm = partial(ModelRegistry._check_stateless,
343 - supports_multimodal,
344 - default=False)
344 + return self._raise_for_unsupported(architectures)
345
346 - return any(is_mm(arch) for arch in architectures)
346 + def resolve_model_cls(
347 + self,
348 + architectures: Union[str, List[str]],
349 + ) -> Tuple[Type[nn.Module], str]:
350 + architectures = self._normalize_archs(architectures)
351
348 - @staticmethod
349 - def is_pp_supported_model(architectures: Union[str, List[str]]) -> bool:
350 - if isinstance(architectures, str):
351 - architectures = [architectures]
352 - if not architectures:
353 - logger.warning("No model architectures are specified")
352 + for arch in architectures:
353 + model_cls = self._try_load_model_cls(arch)
354 + if model_cls is not None:
355 + return (model_cls, arch)
356
355 - is_pp = partial(ModelRegistry._check_stateless,
356 - supports_pp,
357 - default=False)
357 + return self._raise_for_unsupported(architectures)
358
359 - return any(is_pp(arch) for arch in architectures)
359 + def is_text_generation_model(
360 + self,
361 + architectures: Union[str, List[str]],
362 + ) -> bool:
363 + return self.inspect_model_cls(architectures).is_text_generation_model
364
365 + def is_embedding_model(
366 + self,
367 + architectures: Union[str, List[str]],
368 + ) -> bool:
369 + return self.inspect_model_cls(architectures).is_embedding_model
370 +
371 + def is_multimodal_model(
372 + self,
373 + architectures: Union[str, List[str]],
374 + ) -> bool:
375 + return self.inspect_model_cls(architectures).supports_multimodal
376 +
377 + def is_pp_supported_model(
378 + self,
379 + architectures: Union[str, List[str]],
380 + ) -> bool:
381 + return self.inspect_model_cls(architectures).supports_pp
382 +
383 +
384 +ModelRegistry = _ModelRegistry({
385 + model_arch: _LazyRegisteredModel(
386 + module_name=f"vllm.model_executor.models.{mod_relname}",
387 + class_name=cls_name,
388 + )
389 + for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
390 +})
391 +
392 +_T = TypeVar("_T")
393 +
394 +
395 +def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
396 + with tempfile.NamedTemporaryFile() as output_file:
397 + # `cloudpickle` allows pickling lambda functions directly
398 + input_bytes = cloudpickle.dumps((fn, output_file.name))
399 +
400 + # cannot use `sys.executable __file__` here because the script
401 + # contains relative imports
402 + returned = subprocess.run(
403 + [sys.executable, "-m", "vllm.model_executor.models.registry"],
404 + input=input_bytes,
405 + capture_output=True)
406 +
407 + # check if the subprocess is successful
408 + try:
409 + returned.check_returncode()
410 + except Exception as e:
411 + # wrap raised exception to provide more information
412 + raise RuntimeError(f"Error raised in subprocess:\n"
413 + f"{returned.stderr.decode()}") from e
414 +
415 + with open(output_file.name, "rb") as f:
416 + return pickle.load(f)
417 +
418 +
419 +def _run() -> None:
420 + # Setup plugins
421 + from vllm.plugins import load_general_plugins
422 + load_general_plugins()
423 +
424 + fn, output_file = pickle.loads(sys.stdin.buffer.read())
425 +
426 + result = fn()
427
362 -if __name__ == "__main__":
363 - (mod_name, cls_name, func,
364 - output_file) = pickle.loads(sys.stdin.buffer.read())
365 - mod = importlib.import_module(mod_name)
366 - klass = getattr(mod, cls_name)
367 - result = func(klass)
428 with open(output_file, "wb") as f:
429 f.write(pickle.dumps(result))
430 +
431 +
432 +if __name__ == "__main__":
433 + _run()