@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95baec828f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] enable out-of-tree model register (#3871)

youkaichao committed Apr 6, 2024 at 17:11 UTC 95baec828f3ee046074dace1d88202a920b7dc15
6 files changed +148 -2
.buildkite/test-pipeline.yaml
+4 -1
@@ -34,7 +34,10 @@ steps:
34 command: pytest -v -s engine tokenization test_sequence.py test_config.py
35
36 - label: Entrypoints Test
37 - command: pytest -v -s entrypoints
37 + commands:
38 + # these tests have to be separated, because each one will allocate all posible GPU memory
39 + - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
40 + - pytest -v -s entrypoints/test_server_oot_registration.py
41
42 - label: Examples Test
43 working_dir: "/vllm-workspace/examples"
docs/source/models/adding_model.rst
+27
@@ -21,6 +21,8 @@ This document provides a high-level guide on integrating a `HuggingFace Transfor
21 Start by forking our `GitHub`_ repository and then :ref:`build it from source <build_from_source>`.
22 This gives you the ability to modify the codebase and test your model.
23
24 +.. tip::
25 + If you don't want to fork the repository and modify vLLM's codebase, please refer to the "Out-of-Tree Model Integration" section below.
26
27 1. Bring your model code
28 ------------------------
@@ -94,3 +96,28 @@ This method should load the weights from the HuggingFace's checkpoint file and a
96 ----------------------
97
98 Finally, include your :code:`*ForCausalLM` class in `vllm/model_executor/models/__init__.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `vllm/model_executor/model_loader.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/model_loader.py>`_.
99 +
100 +6. Out-of-Tree Model Integration
101 +--------------------------------------------
102 +
103 +We also provide a way to integrate a model without modifying the vLLM codebase. Step 2, 3, 4 are still required, but you can skip step 1 and 5.
104 +
105 +Just add the following lines in your code:
106 +
107 +.. code-block:: python
108 +
109 + from vllm import ModelRegistry
110 + from your_code import YourModelForCausalLM
111 + ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
112 +
113 +If you are running api server with `python -m vllm.entrypoints.openai.api_server args`, you can wrap the entrypoint with the following code:
114 +
115 +.. code-block:: python
116 +
117 + from vllm import ModelRegistry
118 + from your_code import YourModelForCausalLM
119 + ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
120 + import runpy
121 + runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
122 +
123 +Save the above code in a file and run it with `python your_file.py args`.
tests/entrypoints/test_server_oot_registration.py new
+66
@@ -0,0 +1,66 @@
1 +import multiprocessing
2 +import sys
3 +import time
4 +
5 +import torch
6 +from openai import OpenAI, OpenAIError
7 +
8 +from vllm import ModelRegistry
9 +from vllm.model_executor.models.opt import OPTForCausalLM
10 +from vllm.model_executor.sampling_metadata import SamplingMetadata
11 +from vllm.utils import get_open_port
12 +
13 +
14 +class MyOPTForCausalLM(OPTForCausalLM):
15 +
16 + def compute_logits(self, hidden_states: torch.Tensor,
17 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
18 + # this dummy model always predicts the first token
19 + logits = super().compute_logits(hidden_states, sampling_metadata)
20 + logits.zero_()
21 + logits[:, 0] += 1.0
22 + return logits
23 +
24 +
25 +def server_function(port):
26 + # register our dummy model
27 + ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
28 + sys.argv = ["placeholder.py"] + \
29 + ("--model facebook/opt-125m --dtype"
30 + f" float32 --api-key token-abc123 --port {port}").split()
31 + import runpy
32 + runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
33 +
34 +
35 +def test_oot_registration_for_api_server():
36 + port = get_open_port()
37 + server = multiprocessing.Process(target=server_function, args=(port, ))
38 + server.start()
39 + client = OpenAI(
40 + base_url=f"http://localhost:{port}/v1",
41 + api_key="token-abc123",
42 + )
43 + while True:
44 + try:
45 + completion = client.chat.completions.create(
46 + model="facebook/opt-125m",
47 + messages=[{
48 + "role": "system",
49 + "content": "You are a helpful assistant."
50 + }, {
51 + "role": "user",
52 + "content": "Hello!"
53 + }],
54 + temperature=0,
55 + )
56 + break
57 + except OpenAIError as e:
58 + if "Connection error" in str(e):
59 + time.sleep(3)
60 + else:
61 + raise e
62 + server.kill()
63 + generated_text = completion.choices[0].message.content
64 + # make sure only the first token is generated
65 + rest = generated_text.replace("<s>", "")
66 + assert rest == ""
tests/models/test_oot_registration.py new
+32
@@ -0,0 +1,32 @@
1 +import torch
2 +
3 +from vllm import LLM, ModelRegistry, SamplingParams
4 +from vllm.model_executor.models.opt import OPTForCausalLM
5 +from vllm.model_executor.sampling_metadata import SamplingMetadata
6 +
7 +
8 +class MyOPTForCausalLM(OPTForCausalLM):
9 +
10 + def compute_logits(self, hidden_states: torch.Tensor,
11 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
12 + # this dummy model always predicts the first token
13 + logits = super().compute_logits(hidden_states, sampling_metadata)
14 + logits.zero_()
15 + logits[:, 0] += 1.0
16 + return logits
17 +
18 +
19 +def test_oot_registration():
20 + # register our dummy model
21 + ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
22 + prompts = ["Hello, my name is", "The text does not matter"]
23 + sampling_params = SamplingParams(temperature=0)
24 + llm = LLM(model="facebook/opt-125m")
25 + first_token = llm.get_tokenizer().decode(0)
26 + outputs = llm.generate(prompts, sampling_params)
27 +
28 + for output in outputs:
29 + generated_text = output.outputs[0].text
30 + # make sure only the first token is generated
31 + rest = generated_text.replace(first_token, "")
32 + assert rest == ""
vllm/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.engine.ray_utils import initialize_ray_cluster
7 from vllm.entrypoints.llm import LLM
8 +from vllm.model_executor.models import ModelRegistry
9 from vllm.outputs import CompletionOutput, RequestOutput
10 from vllm.sampling_params import SamplingParams
11
@@ -12,6 +13,7 @@ __version__ = "0.4.0.post1"
13
14 __all__ = [
15 "LLM",
16 + "ModelRegistry",
17 "SamplingParams",
18 "RequestOutput",
19 "CompletionOutput",
vllm/model_executor/models/__init__.py
+17 -1
@@ -1,5 +1,5 @@
1 import importlib
2 -from typing import List, Optional, Type
2 +from typing import Dict, List, Optional, Type
3
4 import torch.nn as nn
5
@@ -55,6 +55,10 @@ _MODELS = {
55 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
56 }
57
58 +# Architecture -> type.
59 +# out of tree models
60 +_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
61 +
62 # Models not supported by ROCm.
63 _ROCM_UNSUPPORTED_MODELS = []
64
@@ -74,6 +78,8 @@ class ModelRegistry:
78
79 @staticmethod
80 def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
81 + if model_arch in _OOT_MODELS:
82 + return _OOT_MODELS[model_arch]
83 if model_arch not in _MODELS:
84 return None
85 if is_hip():
@@ -95,6 +101,16 @@ class ModelRegistry:
101 def get_supported_archs() -> List[str]:
102 return list(_MODELS.keys())
103
104 + @staticmethod
105 + def register_model(model_arch: str, model_cls: Type[nn.Module]):
106 + if model_arch in _MODELS:
107 + logger.warning(
108 + f"Model architecture {model_arch} is already registered, "
109 + "and will be overwritten by the new model "
110 + f"class {model_cls.__name__}.")
111 + global _OOT_MODELS
112 + _OOT_MODELS[model_arch] = model_cls
113 +
114
115 __all__ = [
116 "ModelRegistry",