@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de895f1697
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[misc] improve model support check in another process (#9208)

youkaichao committed Oct 9, 2024 at 21:58 UTC de895f1697d22ea19a5a4d4ab3dc17037a3e9af3
2 files changed +36 -32
docs/requirements-docs.txt
+1
@@ -4,6 +4,7 @@ sphinx-copybutton==0.5.2
4 myst-parser==2.0.0
5 sphinx-argparse==0.4.0
6 msgspec
7 +cloudpickle
8
9 # packages to install to build the documentation
10 pydantic >= 2.8
vllm/model_executor/models/registry.py
+35 -32
@@ -1,11 +1,12 @@
1 import importlib
2 -import string
2 +import pickle
3 import subprocess
4 import sys
5 -import uuid
5 +import tempfile
6 from functools import lru_cache, partial
7 from typing import Callable, Dict, List, Optional, Tuple, Type, Union
8
9 +import cloudpickle
10 import torch.nn as nn
11
12 from vllm.logger import init_logger
@@ -282,36 +283,28 @@ class ModelRegistry:
283
284 raise
285
285 - valid_name_characters = string.ascii_letters + string.digits + "._"
286 - if any(s not in valid_name_characters for s in mod_name):
287 - raise ValueError(f"Unsafe module name detected for {model_arch}")
288 - if any(s not in valid_name_characters for s in cls_name):
289 - raise ValueError(f"Unsafe class name detected for {model_arch}")
290 - if any(s not in valid_name_characters for s in func.__module__):
291 - raise ValueError(f"Unsafe module name detected for {func}")
292 - if any(s not in valid_name_characters for s in func.__name__):
293 - raise ValueError(f"Unsafe class name detected for {func}")
294 -
295 - err_id = uuid.uuid4()
296 -
297 - stmts = ";".join([
298 - f"from {mod_name} import {cls_name}",
299 - f"from {func.__module__} import {func.__name__}",
300 - f"assert {func.__name__}({cls_name}), '{err_id}'",
301 - ])
302 -
303 - result = subprocess.run([sys.executable, "-c", stmts],
304 - capture_output=True)
305 -
306 - if result.returncode != 0:
307 - err_lines = [line.decode() for line in result.stderr.splitlines()]
308 - if err_lines and err_lines[-1] != f"AssertionError: {err_id}":
309 - err_str = "\n".join(err_lines)
310 - raise RuntimeError(
311 - "An unexpected error occurred while importing the model in "
312 - f"another process. Error log:\n{err_str}")
313 -
314 - return result.returncode == 0
286 + with tempfile.NamedTemporaryFile() as output_file:
287 + # `cloudpickle` allows pickling lambda functions directly
288 + input_bytes = cloudpickle.dumps(
289 + (mod_name, cls_name, func, output_file.name))
290 + # cannot use `sys.executable __file__` here because the script
291 + # contains relative imports
292 + returned = subprocess.run(
293 + [sys.executable, "-m", "vllm.model_executor.models.registry"],
294 + input=input_bytes,
295 + capture_output=True)
296 +
297 + # check if the subprocess is successful
298 + try:
299 + returned.check_returncode()
300 + except Exception as e:
301 + # wrap raised exception to provide more information
302 + raise RuntimeError(f"Error happened when testing "
303 + f"model support for{mod_name}.{cls_name}:\n"
304 + f"{returned.stderr.decode()}") from e
305 + with open(output_file.name, "rb") as f:
306 + result = pickle.load(f)
307 + return result
308
309 @staticmethod
310 def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
@@ -364,3 +357,13 @@ class ModelRegistry:
357 default=False)
358
359 return any(is_pp(arch) for arch in architectures)
360 +
361 +
362 +if __name__ == "__main__":
363 + (mod_name, cls_name, func,
364 + output_file) = pickle.loads(sys.stdin.buffer.read())
365 + mod = importlib.import_module(mod_name)
366 + klass = getattr(mod, cls_name)
367 + result = func(klass)
368 + with open(output_file, "wb") as f:
369 + f.write(pickle.dumps(result))