@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16422ea76f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[misc][plugin] add plugin system implementation (#7426)

youkaichao committed Aug 13, 2024 at 16:24 UTC 16422ea76f213f5b1035513b441245b19ca5bdce
13 files changed +161 -101
.buildkite/test-pipeline.yaml
+4
@@ -77,11 +77,13 @@ steps:
77 - pytest -v -s core
78
79 - label: Entrypoints Test # 20min
80 + working_dir: "/vllm-workspace/tests"
81 fast_check: true
82 mirror_hardwares: [amd]
83 source_file_dependencies:
84 - vllm/
85 commands:
86 + - pip install -e ./plugins/vllm_add_dummy_model
87 - pytest -v -s entrypoints/llm
88 - pytest -v -s entrypoints/openai
89
@@ -154,6 +156,7 @@ steps:
156 - vllm/
157 - tests/models
158 commands:
159 + - pip install -e ./plugins/vllm_add_dummy_model
160 - pytest -v -s models -m \"not vlm\"
161
162 - label: Vision Language Models Test # 42min
@@ -289,6 +292,7 @@ steps:
292 - pytest -v -s distributed/test_chunked_prefill_distributed.py
293 - pytest -v -s distributed/test_multimodal_broadcast.py
294 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
295 + - pytest -v -s distributed/test_distributed_oot.py
296 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
297 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
298
requirements-common.txt
+1
@@ -23,4 +23,5 @@ pyzmq
23 librosa # Required for audio processing
24 soundfile # Required for audio processing
25 gguf == 0.9.1
26 +importlib_metadata
27 compressed-tensors == 0.5.0
tests/conftest.py
+26
@@ -1,7 +1,9 @@
1 import contextlib
2 import gc
3 +import json
4 import os
5 import sys
6 +import tempfile
7 from collections import UserList
8 from enum import Enum
9 from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
@@ -11,6 +13,7 @@ import pytest
13 import torch
14 import torch.nn as nn
15 import torch.nn.functional as F
16 +from huggingface_hub import snapshot_download
17 from PIL import Image
18 from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
19 AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
@@ -757,3 +760,26 @@ def num_gpus_available():
760 in current process."""
761
762 return cuda_device_count_stateless()
763 +
764 +
765 +temp_dir = tempfile.gettempdir()
766 +_dummy_path = os.path.join(temp_dir, "dummy_opt")
767 +
768 +
769 +@pytest.fixture
770 +def dummy_opt_path():
771 + json_path = os.path.join(_dummy_path, "config.json")
772 + if not os.path.exists(_dummy_path):
773 + snapshot_download(repo_id="facebook/opt-125m",
774 + local_dir=_dummy_path,
775 + ignore_patterns=[
776 + "*.bin", "*.bin.index.json", "*.pt", "*.h5",
777 + "*.msgpack"
778 + ])
779 + assert os.path.exists(json_path)
780 + with open(json_path, "r") as f:
781 + config = json.load(f)
782 + config["architectures"] = ["MyOPTForCausalLM"]
783 + with open(json_path, "w") as f:
784 + json.dump(config, f)
785 + return _dummy_path
tests/distributed/test_distributed_oot.py new
+6
@@ -0,0 +1,6 @@
1 +from ..entrypoints.openai.test_oot_registration import (
2 + run_and_test_dummy_opt_api_server)
3 +
4 +
5 +def test_distributed_oot(dummy_opt_path: str):
6 + run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
tests/entrypoints/openai/test_oot_registration.py
+27 -79
@@ -1,94 +1,42 @@
1 -import sys
2 -import time
3 -
4 -import torch
5 -from openai import OpenAI, OpenAIError
6 -
7 -from vllm import ModelRegistry
8 -from vllm.model_executor.models.opt import OPTForCausalLM
9 -from vllm.model_executor.sampling_metadata import SamplingMetadata
10 -from vllm.utils import get_open_port
11 -
1 from ...utils import VLLM_PATH, RemoteOpenAIServer
2
3 chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
4 assert chatml_jinja_path.exists()
5
6
18 -class MyOPTForCausalLM(OPTForCausalLM):
19 -
20 - def compute_logits(self, hidden_states: torch.Tensor,
21 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
22 - # this dummy model always predicts the first token
23 - logits = super().compute_logits(hidden_states, sampling_metadata)
24 - logits.zero_()
25 - logits[:, 0] += 1.0
26 - return logits
27 -
28 -
29 -def server_function(port: int):
30 - # register our dummy model
31 - ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
32 -
33 - sys.argv = ["placeholder.py"] + [
34 - "--model",
35 - "facebook/opt-125m",
7 +def run_and_test_dummy_opt_api_server(model, tp=1):
8 + # the model is registered through the plugin
9 + server_args = [
10 "--gpu-memory-utilization",
11 "0.10",
12 "--dtype",
13 "float32",
40 - "--api-key",
41 - "token-abc123",
42 - "--port",
43 - str(port),
14 "--chat-template",
15 str(chatml_jinja_path),
16 + "--load-format",
17 + "dummy",
18 + "-tp",
19 + f"{tp}",
20 ]
47 -
48 - import runpy
49 - runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
50 -
51 -
52 -def test_oot_registration_for_api_server():
53 - port = get_open_port()
54 - ctx = torch.multiprocessing.get_context()
55 - server = ctx.Process(target=server_function, args=(port, ))
56 - server.start()
57 -
58 - try:
59 - client = OpenAI(
60 - base_url=f"http://localhost:{port}/v1",
61 - api_key="token-abc123",
21 + with RemoteOpenAIServer(model, server_args) as server:
22 + client = server.get_client()
23 + completion = client.chat.completions.create(
24 + model=model,
25 + messages=[{
26 + "role": "system",
27 + "content": "You are a helpful assistant."
28 + }, {
29 + "role": "user",
30 + "content": "Hello!"
31 + }],
32 + temperature=0,
33 )
63 - now = time.time()
64 - while True:
65 - try:
66 - completion = client.chat.completions.create(
67 - model="facebook/opt-125m",
68 - messages=[{
69 - "role": "system",
70 - "content": "You are a helpful assistant."
71 - }, {
72 - "role": "user",
73 - "content": "Hello!"
74 - }],
75 - temperature=0,
76 - )
77 - break
78 - except OpenAIError as e:
79 - if "Connection error" in str(e):
80 - time.sleep(3)
81 - if time.time() - now > RemoteOpenAIServer.MAX_START_WAIT_S:
82 - msg = "Server did not start in time"
83 - raise RuntimeError(msg) from e
84 - else:
85 - raise e
86 - finally:
87 - server.terminate()
34 + generated_text = completion.choices[0].message.content
35 + assert generated_text is not None
36 + # make sure only the first token is generated
37 + rest = generated_text.replace("<s>", "")
38 + assert rest == ""
39 +
40
89 - generated_text = completion.choices[0].message.content
90 - assert generated_text is not None
91 - # make sure only the first token is generated
92 - # TODO(youkaichao): Fix the test with plugin
93 - rest = generated_text.replace("<s>", "") # noqa
94 - # assert rest == ""
41 +def test_oot_registration_for_api_server(dummy_opt_path: str):
42 + run_and_test_dummy_opt_api_server(dummy_opt_path)
tests/models/test_oot_registration.py
+15 -20
@@ -1,32 +1,27 @@
1 -from typing import Optional
1 +import os
2
3 -import torch
3 +import pytest
4
5 -from vllm import LLM, ModelRegistry, SamplingParams
6 -from vllm.model_executor.models.opt import OPTForCausalLM
7 -from vllm.model_executor.sampling_metadata import SamplingMetadata
5 +from vllm import LLM, SamplingParams
6
7 +# NOTE: the order of the tests is important
8 +# the first test does not load any plugins
9 +# the second test loads the plugin
10 +# they share the same process, so the plugin is loaded for the second test
11
10 -class MyOPTForCausalLM(OPTForCausalLM):
12
12 - def compute_logits(
13 - self,
14 - hidden_states: torch.Tensor,
15 - sampling_metadata: SamplingMetadata,
16 - ) -> Optional[torch.Tensor]:
17 - # this dummy model always predicts the first token
18 - logits = super().compute_logits(hidden_states, sampling_metadata)
19 - logits.zero_()
20 - logits[:, 0] += 1.0
21 - return logits
13 +def test_plugin(dummy_opt_path):
14 + os.environ["VLLM_PLUGINS"] = ""
15 + with pytest.raises(Exception) as excinfo:
16 + LLM(model=dummy_opt_path, load_format="dummy")
17 + assert "are not supported for now" in str(excinfo.value)
18
19
24 -def test_oot_registration():
25 - # register our dummy model
26 - ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
20 +def test_oot_registration(dummy_opt_path):
21 + os.environ["VLLM_PLUGINS"] = "register_dummy_model"
22 prompts = ["Hello, my name is", "The text does not matter"]
23 sampling_params = SamplingParams(temperature=0)
29 - llm = LLM(model="facebook/opt-125m")
24 + llm = LLM(model=dummy_opt_path, load_format="dummy")
25 first_token = llm.get_tokenizer().decode(0)
26 outputs = llm.generate(prompts, sampling_params)
27
tests/plugins/vllm_add_dummy_model/setup.py new
+9
@@ -0,0 +1,9 @@
1 +from setuptools import setup
2 +
3 +setup(name='vllm_add_dummy_model',
4 + version='0.1',
5 + packages=['vllm_add_dummy_model'],
6 + entry_points={
7 + 'vllm.general_plugins':
8 + ["register_dummy_model = vllm_add_dummy_model:register"]
9 + })
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/__init__.py new
+26
@@ -0,0 +1,26 @@
1 +from typing import Optional
2 +
3 +import torch
4 +
5 +from vllm import ModelRegistry
6 +from vllm.model_executor.models.opt import OPTForCausalLM
7 +from vllm.model_executor.sampling_metadata import SamplingMetadata
8 +
9 +
10 +class MyOPTForCausalLM(OPTForCausalLM):
11 +
12 + def compute_logits(
13 + self, hidden_states: torch.Tensor,
14 + sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
15 + # this dummy model always predicts the first token
16 + logits = super().compute_logits(hidden_states, sampling_metadata)
17 + if logits is not None:
18 + logits.zero_()
19 + logits[:, 0] += 1.0
20 + return logits
21 +
22 +
23 +def register():
24 + # register our dummy model
25 + if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
26 + ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
vllm/engine/llm_engine.py
+3
@@ -227,6 +227,9 @@ class LLMEngine:
227 )
228 # TODO(woosuk): Print more configs in debug mode.
229
230 + from vllm.plugins import load_general_plugins
231 + load_general_plugins()
232 +
233 self.model_config = model_config
234 self.cache_config = cache_config
235 self.lora_config = lora_config
vllm/envs.py
+9 -1
@@ -1,6 +1,6 @@
1 import os
2 import tempfile
3 -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
3 +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
4
5 if TYPE_CHECKING:
6 VLLM_HOST_IP: str = ""
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
55 VERBOSE: bool = False
56 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
57 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
58 + VLLM_PLUGINS: Optional[List[str]] = None
59
60
61 def get_default_cache_root():
@@ -362,6 +363,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
363 lambda:
364 (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
365 ("1", "true")),
366 +
367 + # a list of plugin names to load, separated by commas.
368 + # if this is not set, it means all plugins will be loaded
369 + # if this is set to an empty string, no plugins will be loaded
370 + "VLLM_PLUGINS":
371 + lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
372 + "VLLM_PLUGINS"].split(","),
373 }
374
375 # end-env-vars-definition
vllm/model_executor/models/__init__.py
+1 -1
@@ -166,7 +166,7 @@ class ModelRegistry:
166
167 @staticmethod
168 def get_supported_archs() -> List[str]:
169 - return list(_MODELS.keys())
169 + return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
170
171 @staticmethod
172 def register_model(model_arch: str, model_cls: Type[nn.Module]):
vllm/plugins/__init__.py new
+31
@@ -0,0 +1,31 @@
1 +import logging
2 +
3 +import vllm.envs as envs
4 +
5 +logger = logging.getLogger(__name__)
6 +
7 +
8 +def load_general_plugins():
9 + """WARNING: plugins can be loaded for multiple times in different
10 + processes. They should be designed in a way that they can be loaded
11 + multiple times without causing issues.
12 + """
13 + import sys
14 + if sys.version_info < (3, 10):
15 + from importlib_metadata import entry_points
16 + else:
17 + from importlib.metadata import entry_points
18 +
19 + allowed_plugins = envs.VLLM_PLUGINS
20 +
21 + discovered_plugins = entry_points(group='vllm.general_plugins')
22 + for plugin in discovered_plugins:
23 + logger.info("Found general plugin: %s", plugin.name)
24 + if allowed_plugins is None or plugin.name in allowed_plugins:
25 + try:
26 + func = plugin.load()
27 + func()
28 + logger.info("Loaded general plugin: %s", plugin.name)
29 + except Exception:
30 + logger.exception("Failed to load general plugin: %s",
31 + plugin.name)
vllm/worker/worker_base.py
+3
@@ -411,6 +411,9 @@ class WorkerWrapperBase:
411 # see https://github.com/NVIDIA/nccl/issues/1234
412 os.environ['NCCL_CUMEM_ENABLE'] = '0'
413
414 + from vllm.plugins import load_general_plugins
415 + load_general_plugins()
416 +
417 if self.worker_class_fn:
418 worker_class = self.worker_class_fn()
419 else: