@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 711a000255
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Frontend] [Core] feat: Add model loading using `tensorizer` (#3476)

Sanger Steel committed Apr 13, 2024 at 20:13 UTC 711a000255eac3e034f0b73aa5cc62b45201a571
20 files changed +1351 -51
.buildkite/test-pipeline.yaml
+3
@@ -91,6 +91,9 @@ steps:
91 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
92 parallelism: 4
93
94 +- label: Tensorizer Test
95 + command: apt-get install curl libsodium23 && pytest -v -s tensorizer
96 +
97 - label: Metrics Test
98 command: pytest -v -s metrics
99
docs/source/conf.py
+1
@@ -83,6 +83,7 @@ autodoc_mock_imports = [
83 "vllm._C",
84 "numpy",
85 "tqdm",
86 + "tensorizer",
87 ]
88
89 for mock_target in autodoc_mock_imports:
docs/source/models/engine_args.rst
+2 -1
@@ -36,7 +36,7 @@ Below, you can find an explanation of every engine argument for vLLM:
36
37 Directory to download and load the weights, default to the default cache dir of huggingface.
38
39 -.. option:: --load-format {auto,pt,safetensors,npcache,dummy}
39 +.. option:: --load-format {auto,pt,safetensors,npcache,dummy,tensorizer}
40
41 The format of the model weights to load.
42
@@ -45,6 +45,7 @@ Below, you can find an explanation of every engine argument for vLLM:
45 * "safetensors" will load the weights in the safetensors format.
46 * "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47 * "dummy" will initialize the weights with random values, mainly for profiling.
48 + * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_. See `tensorized_vllm_model.py` in the examples folder to serialize a vLLM model, and for more information. Tensorizer support for vLLM can be installed with `pip install vllm[tensorizer]`.
49
50 .. option:: --dtype {auto,half,float16,bfloat16,float,float32}
51
examples/tensorize_vllm_model.py new
+254
@@ -0,0 +1,254 @@
1 +import argparse
2 +import dataclasses
3 +import os
4 +import time
5 +import uuid
6 +from functools import partial
7 +from typing import Type
8 +
9 +import torch
10 +import torch.nn as nn
11 +from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12 + TensorSerializer, stream_io)
13 +from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14 +from transformers import AutoConfig, PretrainedConfig
15 +
16 +from vllm.distributed import initialize_model_parallel
17 +from vllm.engine.arg_utils import EngineArgs
18 +from vllm.engine.llm_engine import LLMEngine
19 +from vllm.model_executor.models import ModelRegistry
20 +from vllm.model_executor.tensorizer_loader import TensorizerArgs
21 +
22 +# yapf conflicts with isort for this docstring
23 +# yapf: disable
24 +"""
25 +tensorize_vllm_model.py is a script that can be used to serialize and
26 +deserialize vLLM models. These models can be loaded using tensorizer directly
27 +to the GPU extremely quickly. Tensor encryption and decryption is also
28 +supported, although libsodium must be installed to use it. Install
29 +vllm with tensorizer support using `pip install vllm[tensorizer]`.
30 +
31 +To serialize a model, you can run something like this:
32 +
33 +python tensorize_vllm_model.py \
34 + --model EleutherAI/gpt-j-6B \
35 + --dtype float16 \
36 + serialize \
37 + --serialized-directory s3://my-bucket/ \
38 + --suffix vllm
39 +
40 +Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41 +and saves it to your S3 bucket. A local directory can also be used.
42 +
43 +You can also encrypt the model weights with a randomly-generated key by
44 +providing a `--keyfile` argument.
45 +
46 +To deserialize a model, you can run something like this:
47 +
48 +python tensorize_vllm_model.py \
49 + --model EleutherAI/gpt-j-6B \
50 + --dtype float16 \
51 + deserialize \
52 + --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53 +
54 +Which downloads the model tensors from your S3 bucket and deserializes them.
55 +To provide S3 credentials, you can provide `--s3-access-key-id` and
56 +`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57 +the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58 +in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59 +
60 +
61 +You can also provide a `--keyfile` argument to decrypt the model weights if
62 +they were serialized with encryption.
63 +
64 +For more information on the available arguments, run
65 +`python tensorize_vllm_model.py --help`.
66 +"""
67 +
68 +
69 +def parse_args():
70 + parser = argparse.ArgumentParser(
71 + description="An example script that can be used to serialize and "
72 + "deserialize vLLM models. These models "
73 + "can be loaded using tensorizer directly to the GPU "
74 + "extremely quickly. Tensor encryption and decryption is "
75 + "also supported, although libsodium must be installed to "
76 + "use it.")
77 + parser = EngineArgs.add_cli_args(parser)
78 + subparsers = parser.add_subparsers(dest='command')
79 +
80 + serialize_parser = subparsers.add_parser(
81 + 'serialize', help="Serialize a model to `--serialized-directory`")
82 +
83 + serialize_parser.add_argument(
84 + "--suffix",
85 + type=str,
86 + required=False,
87 + help=(
88 + "The suffix to append to the serialized model directory, which is "
89 + "used to construct the location of the serialized model tensors, "
90 + "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91 + "`--suffix` is `v1`, the serialized model tensors will be "
92 + "saved to "
93 + "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94 + "If none is provided, a random UUID will be used."))
95 + serialize_parser.add_argument(
96 + "--serialized-directory",
97 + type=str,
98 + required=True,
99 + help="The directory to serialize the model to. "
100 + "This can be a local directory or S3 URI. The path to where the "
101 + "tensors are saved is a combination of the supplied `dir` and model "
102 + "reference ID. For instance, if `dir` is the serialized directory, "
103 + "and the model HuggingFace ID is `EleutherAI/gpt-j-6B`, tensors will "
104 + "be saved to `dir/vllm/EleutherAI/gpt-j-6B/suffix/model.tensors`, "
105 + "where `suffix` is given by `--suffix` or a random UUID if not "
106 + "provided.")
107 +
108 + serialize_parser.add_argument(
109 + "--keyfile",
110 + type=str,
111 + required=False,
112 + help=("Encrypt the model weights with a randomly-generated binary key,"
113 + " and save the key at this path"))
114 +
115 + deserialize_parser = subparsers.add_parser(
116 + 'deserialize',
117 + help=("Deserialize a model from `--path-to-tensors`"
118 + " to verify it can be loaded and used."))
119 +
120 + deserialize_parser.add_argument(
121 + "--path-to-tensors",
122 + type=str,
123 + required=True,
124 + help="The local path or S3 URI to the model tensors to deserialize. ")
125 +
126 + deserialize_parser.add_argument(
127 + "--keyfile",
128 + type=str,
129 + required=False,
130 + help=("Path to a binary key to use to decrypt the model weights,"
131 + " if the model was serialized with encryption"))
132 +
133 + return parser.parse_args()
134 +
135 +
136 +def make_model_contiguous(model):
137 + # Ensure tensors are saved in memory contiguously
138 + for param in model.parameters():
139 + param.data = param.data.contiguous()
140 +
141 +
142 +def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
143 + architectures = getattr(config, "architectures", [])
144 + for arch in architectures:
145 + model_cls = ModelRegistry.load_model_cls(arch)
146 + if model_cls is not None:
147 + return model_cls
148 + raise ValueError(
149 + f"Model architectures {architectures} are not supported for now. "
150 + f"Supported architectures: {ModelRegistry.get_supported_archs()}")
151 +
152 +
153 +def serialize():
154 +
155 + eng_args_dict = {f.name: getattr(args, f.name) for f in
156 + dataclasses.fields(EngineArgs)}
157 + engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
158 + engine = LLMEngine.from_engine_args(engine_args)
159 +
160 + model = (engine.model_executor.driver_worker.
161 + model_runner.model)
162 +
163 + encryption_params = EncryptionParams.random() if keyfile else None
164 + if keyfile:
165 + with _write_stream(keyfile) as stream:
166 + stream.write(encryption_params.key)
167 +
168 + with _write_stream(model_path) as stream:
169 + serializer = TensorSerializer(stream, encryption=encryption_params)
170 + serializer.write_module(model)
171 + serializer.close()
172 +
173 + print("Serialization complete. Model tensors saved to", model_path)
174 + if keyfile:
175 + print("Key saved to", keyfile)
176 +
177 +
178 +def deserialize():
179 + config = AutoConfig.from_pretrained(model_ref)
180 +
181 + with no_init_or_tensor():
182 + model_class = _get_vllm_model_architecture(config)
183 + model = model_class(config)
184 +
185 + before_mem = get_mem_usage()
186 + start = time.time()
187 +
188 + if keyfile:
189 + with _read_stream(keyfile) as stream:
190 + key = stream.read()
191 + decryption_params = DecryptionParams.from_key(key)
192 + tensorizer_args.deserializer_params['encryption'] = \
193 + decryption_params
194 +
195 + with (_read_stream(model_path)) as stream, TensorDeserializer(
196 + stream, **tensorizer_args.deserializer_params) as deserializer:
197 + deserializer.load_into_module(model)
198 + end = time.time()
199 +
200 + # Brag about how fast we are.
201 + total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
202 + duration = end - start
203 + per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
204 + after_mem = get_mem_usage()
205 + print(
206 + f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
207 + )
208 + print(f"Memory usage before: {before_mem}")
209 + print(f"Memory usage after: {after_mem}")
210 +
211 + return model
212 +
213 +
214 +args = parse_args()
215 +
216 +s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
217 + or None)
218 +s3_secret_access_key = (args.s3_secret_access_key
219 + or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
220 +
221 +s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
222 +
223 +_read_stream, _write_stream = (partial(
224 + stream_io.open_stream,
225 + mode=mode,
226 + s3_access_key_id=s3_access_key_id,
227 + s3_secret_access_key=s3_secret_access_key,
228 + s3_endpoint=s3_endpoint,
229 +) for mode in ("rb", "wb+"))
230 +
231 +model_ref = args.model
232 +
233 +model_name = model_ref.split("/")[1]
234 +
235 +os.environ["MASTER_ADDR"] = "127.0.0.1"
236 +os.environ["MASTER_PORT"] = "8080"
237 +
238 +torch.distributed.init_process_group(world_size=1, rank=0)
239 +initialize_model_parallel()
240 +
241 +keyfile = args.keyfile if args.keyfile else None
242 +
243 +if args.command == "serialize":
244 + input_dir = args.serialized_directory.rstrip('/')
245 + suffix = args.suffix if args.suffix else uuid.uuid4().hex
246 + base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
247 + model_path = f"{base_path}/model.tensors"
248 + serialize()
249 +elif args.command == "deserialize":
250 + tensorizer_args = TensorizerArgs.from_cli_args(args)
251 + model_path = args.path_to_tensors
252 + deserialize()
253 +else:
254 + raise ValueError("Either serialize or deserialize must be specified.")
requirements-cpu.txt
+1 -1
@@ -3,4 +3,4 @@
3
4 # Dependencies for x86_64 CPUs
5 torch == 2.2.1+cpu
6 -triton >= 2.1.0 # FIXME(woosuk): This is a hack to avoid import error.
6 +triton >= 2.1.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
requirements-dev.txt
+1
@@ -14,6 +14,7 @@ types-setuptools
14
15 # testing
16 pytest
17 +tensorizer==2.9.0a0
18 pytest-forked
19 pytest-asyncio
20 pytest-rerunfailures
setup.py
+3
@@ -405,6 +405,9 @@ setup(
405 python_requires=">=3.8",
406 install_requires=get_requirements(),
407 ext_modules=ext_modules,
408 + extras_require={
409 + "optional": ["tensorizer==2.9.0a1"],
410 + },
411 cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
412 package_data=package_data,
413 )
tests/tensorizer/__init__.py
tests/tensorizer/tensorize_vllm_model_for_testing.py new
+245
@@ -0,0 +1,245 @@
1 +import argparse
2 +import dataclasses
3 +import os
4 +import time
5 +import uuid
6 +from functools import partial
7 +from typing import Type
8 +
9 +import torch
10 +import torch.nn as nn
11 +from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12 + TensorSerializer, stream_io)
13 +from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14 +from transformers import AutoConfig, PretrainedConfig
15 +
16 +from vllm.distributed import initialize_model_parallel
17 +from vllm.engine.arg_utils import EngineArgs
18 +from vllm.engine.llm_engine import LLMEngine
19 +from vllm.model_executor.models import ModelRegistry
20 +from vllm.model_executor.tensorizer_loader import TensorizerArgs
21 +
22 +# yapf conflicts with isort for this docstring
23 +# yapf: disable
24 +"""
25 +tensorize_vllm_model.py is a script that can be used to serialize and
26 +deserialize vLLM models. These models can be loaded using tensorizer directly
27 +to the GPU extremely quickly. Tensor encryption and decryption is also
28 +supported, although libsodium must be installed to use it. Install
29 +vllm with tensorizer support using `pip install vllm[tensorizer]`.
30 +
31 +To serialize a model, you can run something like this:
32 +
33 +python tensorize_vllm_model.py \
34 + --model EleutherAI/gpt-j-6B \
35 + --dtype float16 \
36 + serialize \
37 + --serialized-directory s3://my-bucket/ \
38 + --suffix vllm
39 +
40 +Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41 +and saves it to your S3 bucket. A local directory can also be used.
42 +
43 +You can also encrypt the model weights with a randomly-generated key by
44 +providing a `--keyfile` argument.
45 +
46 +To deserialize a model, you can run something like this:
47 +
48 +python tensorize_vllm_model.py \
49 + --model EleutherAI/gpt-j-6B \
50 + --dtype float16 \
51 + deserialize \
52 + --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53 +
54 +Which downloads the model tensors from your S3 bucket and deserializes them.
55 +To provide S3 credentials, you can provide `--s3-access-key-id` and
56 +`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57 +the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58 +in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59 +
60 +
61 +You can also provide a `--keyfile` argument to decrypt the model weights if
62 +they were serialized with encryption.
63 +
64 +For more information on the available arguments, run
65 +`python tensorize_vllm_model.py --help`.
66 +"""
67 +
68 +
69 +def parse_args():
70 + parser = argparse.ArgumentParser(
71 + description="An example script that can be used to serialize and "
72 + "deserialize vLLM models. These models "
73 + "can be loaded using tensorizer directly to the GPU "
74 + "extremely quickly. Tensor encryption and decryption is "
75 + "also supported, although libsodium must be installed to "
76 + "use it.")
77 + parser = EngineArgs.add_cli_args(parser)
78 + subparsers = parser.add_subparsers(dest='command')
79 +
80 + serialize_parser = subparsers.add_parser(
81 + 'serialize', help="Serialize a model to `--serialized-directory`")
82 +
83 + serialize_parser.add_argument(
84 + "--suffix",
85 + type=str,
86 + required=False,
87 + help=(
88 + "The suffix to append to the serialized model directory, which is "
89 + "used to construct the location of the serialized model tensors, "
90 + "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91 + "`--suffix` is `v1`, the serialized model tensors will be "
92 + "saved to "
93 + "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94 + "If none is provided, a random UUID will be used."))
95 + serialize_parser.add_argument(
96 + "--serialized-directory",
97 + type=str,
98 + required=True)
99 +
100 + serialize_parser.add_argument(
101 + "--keyfile",
102 + type=str,
103 + required=False,
104 + help=("Encrypt the model weights with a randomly-generated binary key,"
105 + " and save the key at this path"))
106 +
107 + deserialize_parser = subparsers.add_parser(
108 + 'deserialize',
109 + help=("Deserialize a model from `--path-to-tensors`"
110 + " to verify it can be loaded and used."))
111 +
112 + deserialize_parser.add_argument(
113 + "--path-to-tensors",
114 + type=str,
115 + required=True,
116 + help="The local path or S3 URI to the model tensors to deserialize. ")
117 +
118 + deserialize_parser.add_argument(
119 + "--keyfile",
120 + type=str,
121 + required=False,
122 + help=("Path to a binary key to use to decrypt the model weights,"
123 + " if the model was serialized with encryption"))
124 +
125 + return parser.parse_args()
126 +
127 +
128 +def make_model_contiguous(model):
129 + # Ensure tensors are saved in memory contiguously
130 + for param in model.parameters():
131 + param.data = param.data.contiguous()
132 +
133 +
134 +def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
135 + architectures = getattr(config, "architectures", [])
136 + for arch in architectures:
137 + model_cls = ModelRegistry.load_model_cls(arch)
138 + if model_cls is not None:
139 + return model_cls
140 + raise ValueError(
141 + f"Model architectures {architectures} are not supported for now. "
142 + f"Supported architectures: {ModelRegistry.get_supported_archs()}")
143 +
144 +
145 +def serialize():
146 + eng_args_dict = {f.name: getattr(args, f.name) for f in
147 + dataclasses.fields(EngineArgs)}
148 + engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
149 + engine = LLMEngine.from_engine_args(engine_args)
150 +
151 + model = (engine.model_executor.driver_worker.
152 + model_runner.model)
153 +
154 + encryption_params = EncryptionParams.random() if keyfile else None
155 + if keyfile:
156 + with _write_stream(keyfile) as stream:
157 + stream.write(encryption_params.key)
158 +
159 + with _write_stream(model_path) as stream:
160 + serializer = TensorSerializer(stream, encryption=encryption_params)
161 + serializer.write_module(model)
162 + serializer.close()
163 +
164 + print("Serialization complete. Model tensors saved to", model_path)
165 + if keyfile:
166 + print("Key saved to", keyfile)
167 +
168 +
169 +def deserialize():
170 + config = AutoConfig.from_pretrained(model_ref)
171 +
172 + with no_init_or_tensor():
173 + model_class = _get_vllm_model_architecture(config)
174 + model = model_class(config)
175 +
176 + before_mem = get_mem_usage()
177 + start = time.time()
178 +
179 + if keyfile:
180 + with _read_stream(keyfile) as stream:
181 + key = stream.read()
182 + decryption_params = DecryptionParams.from_key(key)
183 + tensorizer_args.deserializer_params['encryption'] = \
184 + decryption_params
185 +
186 + with (_read_stream(model_path)) as stream, TensorDeserializer(
187 + stream, **tensorizer_args.deserializer_params) as deserializer:
188 + deserializer.load_into_module(model)
189 + end = time.time()
190 +
191 + # Brag about how fast we are.
192 + total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
193 + duration = end - start
194 + per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
195 + after_mem = get_mem_usage()
196 + print(
197 + f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
198 + )
199 + print(f"Memory usage before: {before_mem}")
200 + print(f"Memory usage after: {after_mem}")
201 +
202 + return model
203 +
204 +
205 +args = parse_args()
206 +
207 +s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
208 + or None)
209 +s3_secret_access_key = (args.s3_secret_access_key
210 + or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
211 +
212 +s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
213 +
214 +_read_stream, _write_stream = (partial(
215 + stream_io.open_stream,
216 + mode=mode,
217 + s3_access_key_id=s3_access_key_id,
218 + s3_secret_access_key=s3_secret_access_key,
219 + s3_endpoint=s3_endpoint,
220 +) for mode in ("rb", "wb+"))
221 +
222 +model_ref = args.model
223 +
224 +model_name = model_ref.split("/")[1]
225 +
226 +os.environ["MASTER_ADDR"] = "127.0.0.1"
227 +os.environ["MASTER_PORT"] = "8080"
228 +
229 +torch.distributed.init_process_group(world_size=1, rank=0)
230 +initialize_model_parallel()
231 +
232 +keyfile = args.keyfile if args.keyfile else None
233 +
234 +if args.command == "serialize":
235 + input_dir = args.serialized_directory.rstrip('/')
236 + suffix = args.suffix if args.suffix else uuid.uuid4().hex
237 + base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
238 + model_path = f"{base_path}/model.tensors"
239 + serialize()
240 +elif args.command == "deserialize":
241 + tensorizer_args = TensorizerArgs.from_cli_args(args)
242 + model_path = args.path_to_tensors
243 + deserialize()
244 +else:
245 + raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer/test_tensorizer.py new
+302
@@ -0,0 +1,302 @@
1 +import gc
2 +import subprocess
3 +from unittest.mock import MagicMock, patch
4 +
5 +import pytest
6 +import torch
7 +
8 +from tests.entrypoints.test_openai_server import ServerRunner
9 +from vllm import SamplingParams
10 +from vllm.config import TensorizerConfig
11 +from vllm.model_executor.tensorizer_loader import (
12 + EncryptionParams, TensorSerializer, is_vllm_serialized_tensorizer,
13 + load_with_tensorizer, open_stream)
14 +
15 +prompts = [
16 + "Hello, my name is",
17 + "The president of the United States is",
18 + "The capital of France is",
19 + "The future of AI is",
20 +]
21 +# Create a sampling params object.
22 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
23 +
24 +model_ref = "facebook/opt-125m"
25 +
26 +
27 +def is_curl_installed():
28 + try:
29 + subprocess.check_call(['curl', '--version'])
30 + return True
31 + except (subprocess.CalledProcessError, FileNotFoundError):
32 + return False
33 +
34 +
35 +@pytest.fixture(autouse=True)
36 +def tensorizer_config():
37 + config = TensorizerConfig(tensorizer_uri="vllm", vllm_tensorized=True)
38 + return config
39 +
40 +
41 +@patch('vllm.model_executor.tensorizer_loader.TensorizerAgent')
42 +def test_load_with_tensorizer(mock_agent, tensorizer_config):
43 + mock_linear_method = MagicMock()
44 + mock_agent_instance = mock_agent.return_value
45 + mock_agent_instance.deserialize.return_value = MagicMock()
46 +
47 + result = load_with_tensorizer(tensorizer_config,
48 + linear_method=mock_linear_method)
49 +
50 + mock_agent.assert_called_once_with(tensorizer_config,
51 + linear_method=mock_linear_method)
52 + mock_agent_instance.deserialize.assert_called_once()
53 + assert result == mock_agent_instance.deserialize.return_value
54 +
55 +
56 +def test_is_vllm_model_with_vllm_in_uri(tensorizer_config):
57 + tensorizer_config.vllm_tensorized = True
58 +
59 + result = is_vllm_serialized_tensorizer(tensorizer_config)
60 +
61 + assert result is True
62 +
63 +
64 +def test_is_vllm_model_without_vllm_in_uri(tensorizer_config):
65 + tensorizer_config.vllm_tensorized = False
66 +
67 + result = is_vllm_serialized_tensorizer(tensorizer_config)
68 +
69 + assert result is False
70 +
71 +
72 +def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
73 + vllm_model = vllm_runner(model_ref)
74 + model_path = tmp_path / (model_ref + ".tensors")
75 + outputs = vllm_model.generate(prompts, sampling_params)
76 + model = (vllm_model.model.llm_engine.model_executor.driver_worker.
77 + model_runner.model)
78 + with open_stream(model_path, "wb+") as stream:
79 + serializer = TensorSerializer(stream)
80 + serializer.write_module(model)
81 + del vllm_model, model
82 + gc.collect()
83 + torch.cuda.empty_cache()
84 + loaded_vllm_model = vllm_runner(model_ref,
85 + load_format="tensorizer",
86 + tensorizer_uri=model_path,
87 + num_readers=1,
88 + vllm_tensorized=True)
89 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
90 +
91 + # Assumes SamplingParams being seeded ensures the outputs are deterministic
92 + assert outputs == deserialized_outputs
93 +
94 +
95 +@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
96 +def test_can_deserialize_s3(vllm_runner):
97 + model_ref = "EleutherAI/pythia-1.4b"
98 + tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
99 +
100 + loaded_hf_model = vllm_runner(
101 + model_ref,
102 + tensorizer_uri=tensorized_path,
103 + load_format="tensorizer",
104 + num_readers=1,
105 + vllm_tensorized=False,
106 + s3_endpoint="object.ord1.coreweave.com",
107 + )
108 +
109 + deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
110 +
111 + assert deserialized_outputs
112 +
113 +
114 +@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
115 +def test_deserialized_encrypted_vllm_model_has_same_outputs(
116 + vllm_runner, tmp_path):
117 + vllm_model = vllm_runner(model_ref)
118 + model_path = tmp_path / (model_ref + ".tensors")
119 + key_path = tmp_path / (model_ref + ".key")
120 + outputs = vllm_model.generate(prompts, sampling_params)
121 + model = (vllm_model.model.llm_engine.model_executor.driver_worker.
122 + model_runner.model)
123 +
124 + encryption_params = EncryptionParams.random()
125 + with open_stream(model_path, "wb+") as stream:
126 + serializer = TensorSerializer(stream, encryption=encryption_params)
127 + serializer.write_module(model)
128 + with open_stream(key_path, "wb+") as stream:
129 + stream.write(encryption_params.key)
130 + del vllm_model, model
131 + gc.collect()
132 + torch.cuda.empty_cache()
133 + loaded_vllm_model = vllm_runner(model_ref,
134 + tensorizer_uri=model_path,
135 + load_format="tensorizer",
136 + encryption_keyfile=key_path,
137 + num_readers=1,
138 + vllm_tensorized=True)
139 +
140 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
141 +
142 + # Assumes SamplingParams being seeded ensures the outputs are deterministic
143 + assert outputs == deserialized_outputs
144 +
145 +
146 +def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
147 + tmp_path):
148 + hf_model = hf_runner(model_ref)
149 + model_path = tmp_path / (model_ref + ".tensors")
150 + max_tokens = 50
151 + outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
152 + with open_stream(model_path, "wb+") as stream:
153 + serializer = TensorSerializer(stream)
154 + serializer.write_module(hf_model.model)
155 + del hf_model
156 + gc.collect()
157 + torch.cuda.empty_cache()
158 + loaded_hf_model = vllm_runner(model_ref,
159 + tensorizer_uri=model_path,
160 + load_format="tensorizer",
161 + num_readers=1,
162 + vllm_tensorized=False)
163 +
164 + deserialized_outputs = loaded_hf_model.generate_greedy(
165 + prompts, max_tokens=max_tokens)
166 +
167 + assert outputs == deserialized_outputs
168 +
169 +
170 +def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
171 + from huggingface_hub import snapshot_download
172 +
173 + from examples.multilora_inference import (create_test_prompts,
174 + process_requests)
175 +
176 + model_ref = "meta-llama/Llama-2-7b-hf"
177 + lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
178 + test_prompts = create_test_prompts(lora_path)
179 +
180 + # Serialize model before deserializing and binding LoRA adapters
181 + vllm_model = vllm_runner(model_ref, )
182 + model_path = tmp_path / (model_ref + ".tensors")
183 + model = (vllm_model.model.llm_engine.model_executor.driver_worker.
184 + model_runner.model)
185 + with open_stream(model_path, "wb+") as stream:
186 + serializer = TensorSerializer(stream)
187 + serializer.write_module(model)
188 + del vllm_model, model
189 + gc.collect()
190 + torch.cuda.empty_cache()
191 + loaded_vllm_model = vllm_runner(
192 + model_ref,
193 + tensorizer_uri=model_path,
194 + load_format="tensorizer",
195 + num_readers=1,
196 + vllm_tensorized=True,
197 + enable_lora=True,
198 + max_loras=1,
199 + max_lora_rank=8,
200 + max_cpu_loras=2,
201 + max_num_seqs=50,
202 + max_model_len=1000,
203 + )
204 + process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
205 +
206 + assert loaded_vllm_model
207 +
208 +
209 +def test_load_without_tensorizer_load_format(vllm_runner):
210 + with pytest.raises(ValueError):
211 + vllm_runner(model_ref, tensorizer_uri="test")
212 +
213 +
214 +@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
215 +def test_tensorize_vllm_model(tmp_path):
216 + # Test serialize command
217 + serialize_args = [
218 + "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
219 + model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
220 + tmp_path, "--suffix", "tests"
221 + ]
222 + result = subprocess.run(serialize_args, capture_output=True, text=True)
223 + print(result.stdout) # Print the output of the serialize command
224 +
225 + assert result.returncode == 0, (f"Serialize command failed with output:"
226 + f"\n{result.stdout}\n{result.stderr}")
227 +
228 + path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
229 +
230 + # Test deserialize command
231 + deserialize_args = [
232 + "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
233 + model_ref, "--dtype", "float16", "deserialize", "--path-to-tensors",
234 + path_to_tensors
235 + ]
236 + result = subprocess.run(deserialize_args, capture_output=True, text=True)
237 + assert result.returncode == 0, (f"Deserialize command failed with output:"
238 + f"\n{result.stdout}\n{result.stderr}")
239 +
240 +
241 +@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
242 +def test_openai_apiserver_with_tensorizer(tmp_path):
243 + ## Serialize model
244 + serialize_args = [
245 + "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
246 + model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
247 + tmp_path, "--suffix", "tests"
248 + ]
249 + result = subprocess.run(serialize_args, capture_output=True, text=True)
250 + print(result.stdout) # Print the output of the serialize command
251 +
252 + assert result.returncode == 0, (f"Serialize command failed with output:"
253 + f"\n{result.stdout}\n{result.stderr}")
254 +
255 + path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
256 +
257 + ## Start OpenAI API server
258 + openai_args = [
259 + "--model", model_ref, "--dtype", "float16", "--load-format",
260 + "tensorizer", "--tensorizer-uri", path_to_tensors, "--vllm-tensorized",
261 + "--port", "8000"
262 + ]
263 +
264 + server = ServerRunner.remote(openai_args)
265 +
266 + print("Server ready.")
267 + assert server.ready.remote()
268 +
269 +
270 +def test_raise_value_error_on_invalid_load_format(vllm_runner):
271 + with pytest.raises(ValueError):
272 + vllm_runner(model_ref,
273 + load_format="safetensors",
274 + tensorizer_uri="test")
275 +
276 +
277 +def test_tensorizer_with_tp(vllm_runner):
278 + with pytest.raises(ValueError):
279 + model_ref = "EleutherAI/pythia-1.4b"
280 + tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
281 +
282 + vllm_runner(
283 + model_ref,
284 + tensorizer_uri=tensorized_path,
285 + load_format="tensorizer",
286 + num_readers=1,
287 + vllm_tensorized=False,
288 + s3_endpoint="object.ord1.coreweave.com",
289 + tensor_parallel_size=2,
290 + )
291 +
292 +
293 +@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
294 +def test_tensorizer_warn_quant(tmp_path):
295 + model_ref = "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit"
296 + serialize_args = [
297 + "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
298 + model_ref, "--quantization", "gptq", "--tensorizer-uri", "test",
299 + "serialize", "--serialized-directory", tmp_path, "--suffix", "tests"
300 + ]
301 + result = subprocess.run(serialize_args, capture_output=True, text=True)
302 + assert 'PerformanceWarning' in result.stderr
vllm/config.py
+72 -2
@@ -1,6 +1,8 @@
1 import enum
2 +import io
3 import json
4 import os
5 +import typing
6 from dataclasses import dataclass, fields
7 from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
8
@@ -16,6 +18,8 @@ from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
18 if TYPE_CHECKING:
19 from ray.util.placement_group import PlacementGroup
20
21 + from vllm.model_executor.tensorizer_loader import TensorizerArgs
22 +
23 logger = init_logger(__name__)
24
25 _GB = 1 << 30
@@ -139,13 +143,14 @@ class ModelConfig:
143 def _verify_load_format(self) -> None:
144 load_format = self.load_format.lower()
145 supported_load_format = [
142 - "auto", "pt", "safetensors", "npcache", "dummy"
146 + "auto", "pt", "safetensors", "npcache", "dummy", "tensorizer"
147 ]
148 rocm_not_supported_load_format: List[str] = []
149 if load_format not in supported_load_format:
150 raise ValueError(
151 f"Unknown load format: {self.load_format}. Must be one of "
148 - "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
152 + "'auto', 'pt', 'safetensors', 'npcache', 'tensorizer', or "
153 + "'dummy'.")
154 if is_hip() and load_format in rocm_not_supported_load_format:
155 rocm_supported_load_format = [
156 f for f in supported_load_format
@@ -882,6 +887,65 @@ class VisionLanguageConfig:
887 f"{[x.name for x in cls.ImageInputType]}.") from e
888
889
890 +@dataclass
891 +class TensorizerConfig:
892 + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
893 + str, bytes, os.PathLike, int]
894 + vllm_tensorized: bool
895 + verify_hash: Optional[bool] = False
896 + num_readers: Optional[int] = 1
897 + encryption_keyfile: Optional[str] = None
898 + s3_access_key_id: Optional[str] = None
899 + s3_secret_access_key: Optional[str] = None
900 + s3_endpoint: Optional[str] = None
901 + model_class: Optional[torch.nn.Module] = None
902 + hf_config: Optional[PretrainedConfig] = None
903 + dtype: Union[str, torch.dtype] = None
904 +
905 + def _construct_tensorizer_args(self) -> "TensorizerArgs":
906 + from vllm.model_executor.tensorizer_loader import TensorizerArgs
907 + tensorizer_args = {
908 + "tensorizer_uri": self.tensorizer_uri,
909 + "vllm_tensorized": self.vllm_tensorized,
910 + "verify_hash": self.verify_hash,
911 + "num_readers": self.num_readers,
912 + "encryption_keyfile": self.encryption_keyfile,
913 + "s3_access_key_id": self.s3_access_key_id,
914 + "s3_secret_access_key": self.s3_secret_access_key,
915 + "s3_endpoint": self.s3_endpoint,
916 + }
917 + return TensorizerArgs(**tensorizer_args)
918 +
919 + def verify_with_parallel_config(
920 + self,
921 + parallel_config: "ParallelConfig",
922 + ) -> None:
923 + if (parallel_config.tensor_parallel_size > 1
924 + and self.tensorizer_uri is not None):
925 + raise ValueError(
926 + "Loading to multiple GPUs is not currently supported with "
927 + "vLLM-serialized models. Please set tensor_parallel_size=1."
928 + " or use a non-vLLM-serialized model, such as a "
929 + "serialized Hugging Face `PretrainedModel`.")
930 +
931 + def verify_with_model_config(self, model_config) -> None:
932 + if (model_config.quantization is not None
933 + and self.tensorizer_uri is not None):
934 + from vllm.model_executor.tensorizer_loader import (
935 + tensorizer_warning)
936 + tensorizer_warning(
937 + "Loading a model using Tensorizer with quantization on vLLM"
938 + " is unstable and may lead to errors.")
939 +
940 + if (model_config.load_format != "tensorizer"
941 + and self.tensorizer_uri is not None):
942 + raise ValueError(
943 + "A tensorizer uri was passed for tensorizer loading, but the "
944 + f"load format was set to {model_config.load_format}. "
945 + "Please set the load format to 'tensorizer' to use "
946 + f"tensorizer args.")
947 +
948 +
949 _STR_DTYPE_TO_TORCH_DTYPE = {
950 "half": torch.float16,
951 "float16": torch.float16,
@@ -1029,6 +1093,7 @@ class EngineConfig:
1093 lora_config: Optional[LoRAConfig]
1094 vision_language_config: Optional[VisionLanguageConfig]
1095 speculative_config: Optional[SpeculativeConfig]
1096 + tensorizer_config: Optional[TensorizerConfig]
1097
1098 def __post_init__(self):
1099 """Verify configs are valid & consistent with each other.
@@ -1036,6 +1101,11 @@ class EngineConfig:
1101 self.model_config.verify_with_parallel_config(self.parallel_config)
1102 self.cache_config.verify_with_parallel_config(self.parallel_config)
1103
1104 + if self.tensorizer_config:
1105 + self.tensorizer_config.verify_with_parallel_config(
1106 + self.parallel_config)
1107 + self.tensorizer_config.verify_with_model_config(self.model_config)
1108 +
1109 if self.lora_config:
1110 self.lora_config.verify_with_model_config(self.model_config)
1111 self.lora_config.verify_with_scheduler_config(
vllm/engine/arg_utils.py
+38 -7
@@ -1,12 +1,15 @@
1 import argparse
2 import dataclasses
3 +import io
4 +import os
5 from dataclasses import dataclass
4 -from typing import Optional
6 +from typing import BinaryIO, Optional, Union
7
8 from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
9 ModelConfig, ParallelConfig, SchedulerConfig,
8 - SpeculativeConfig, TokenizerPoolConfig,
9 - VisionLanguageConfig)
10 + SpeculativeConfig, TensorizerConfig,
11 + TokenizerPoolConfig, VisionLanguageConfig)
12 +from vllm.model_executor.tensorizer_loader import TensorizerArgs
13 from vllm.utils import str_to_int_tuple
14
15
@@ -58,12 +61,22 @@ class EngineArgs:
61 num_gpu_blocks_override: Optional[int] = None
62 num_lookahead_slots: int = 0
63
64 + # Tensorizer configuration parameters
65 + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
66 + bytes, os.PathLike, int] = None
67 + vllm_tensorized: bool = False
68 + verify_hash: Optional[bool] = False
69 + num_readers: Optional[int] = 1
70 + encryption_keyfile: Optional[str] = None
71 + s3_access_key_id: Optional[str] = None
72 + s3_secret_access_key: Optional[str] = None
73 + s3_endpoint: Optional[str] = None
74 +
75 # Related to Vision-language models such as llava
76 image_input_type: Optional[str] = None
77 image_token_id: Optional[int] = None
78 image_input_shape: Optional[str] = None
79 image_feature_size: Optional[int] = None
66 -
80 scheduler_delay_factor: float = 0.0
81 enable_chunked_prefill: bool = False
82
@@ -135,7 +148,9 @@ class EngineArgs:
148 '--load-format',
149 type=str,
150 default=EngineArgs.load_format,
138 - choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'],
151 + choices=[
152 + 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
153 + ],
154 help='The format of the model weights to load. '
155 '"auto" will try to load the weights in the safetensors format '
156 'and fall back to the pytorch bin format if safetensors format '
@@ -145,7 +160,10 @@ class EngineArgs:
160 '"npcache" will load the weights in pytorch format and store '
161 'a numpy cache to speed up the loading. '
162 '"dummy" will initialize the weights with random values, '
148 - 'which is mainly for profiling.')
163 + 'which is mainly for profiling.'
164 + '"tensorizer" will load the weights using tensorizer from CoreWeave'
165 + 'which assumes tensorizer_uri is set to the location of the '
166 + 'serialized weights.')
167 parser.add_argument(
168 '--dtype',
169 type=str,
@@ -403,6 +421,7 @@ class EngineArgs:
421 default=None,
422 help='The number of speculative tokens to sample from '
423 'the draft model in speculative decoding')
424 + parser = TensorizerArgs.add_cli_args(parser)
425 return parser
426
427 @classmethod
@@ -465,6 +484,17 @@ class EngineArgs:
484 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
485 and self.max_cpu_loras > 0 else None) if self.enable_lora else None
486
487 + tensorizer_config = TensorizerConfig(
488 + tensorizer_uri=self.tensorizer_uri,
489 + vllm_tensorized=self.vllm_tensorized,
490 + verify_hash=self.verify_hash,
491 + num_readers=self.num_readers,
492 + encryption_keyfile=self.encryption_keyfile,
493 + s3_access_key_id=self.s3_access_key_id,
494 + s3_secret_access_key=self.s3_secret_access_key,
495 + s3_endpoint=self.s3_endpoint,
496 + )
497 +
498 if self.image_input_type:
499 if (not self.image_token_id or not self.image_input_shape
500 or not self.image_feature_size):
@@ -488,7 +518,8 @@ class EngineArgs:
518 device_config=device_config,
519 lora_config=lora_config,
520 vision_language_config=vision_language_config,
491 - speculative_config=speculative_config)
521 + speculative_config=speculative_config,
522 + tensorizer_config=tensorizer_config)
523
524
525 @dataclass
vllm/engine/llm_engine.py
+7 -1
@@ -6,7 +6,7 @@ from transformers import PreTrainedTokenizer
6 import vllm
7 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8 ParallelConfig, SchedulerConfig, SpeculativeConfig,
9 - VisionLanguageConfig)
9 + TensorizerConfig, VisionLanguageConfig)
10 from vllm.core.scheduler import Scheduler, SchedulerOutputs
11 from vllm.engine.arg_utils import EngineArgs
12 from vllm.engine.metrics import StatLogger, Stats
@@ -74,6 +74,7 @@ class LLMEngine:
74 lora_config: Optional[LoRAConfig],
75 vision_language_config: Optional[VisionLanguageConfig],
76 speculative_config: Optional[SpeculativeConfig],
77 + tensorizer_config: Optional[TensorizerConfig],
78 executor_class: Type[ExecutorBase],
79 log_stats: bool,
80 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -110,6 +111,7 @@ class LLMEngine:
111 self.scheduler_config = scheduler_config
112 self.device_config = device_config
113 self.speculative_config = speculative_config
114 + self.tensorizer_config = tensorizer_config
115 self.log_stats = log_stats
116
117 self._init_tokenizer()
@@ -125,6 +127,7 @@ class LLMEngine:
127 lora_config=lora_config,
128 vision_language_config=vision_language_config,
129 speculative_config=speculative_config,
130 + tensorizer_config=tensorizer_config,
131 )
132
133 self._initialize_kv_caches()
@@ -264,6 +267,9 @@ class LLMEngine:
267 def _verify_args(self) -> None:
268 self.model_config.verify_with_parallel_config(self.parallel_config)
269 self.cache_config.verify_with_parallel_config(self.parallel_config)
270 + if self.tensorizer_config:
271 + self.tensorizer_config.verify_with_parallel_config(
272 + self.parallel_config)
273 if self.lora_config:
274 self.lora_config.verify_with_model_config(self.model_config)
275 self.lora_config.verify_with_scheduler_config(
vllm/executor/gpu_executor.py
+11 -12
@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Tuple
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 - VisionLanguageConfig)
5 + TensorizerConfig, VisionLanguageConfig)
6 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
@@ -15,17 +15,14 @@ logger = init_logger(__name__)
15
16 class GPUExecutor(ExecutorBase):
17
18 - def __init__(
19 - self,
20 - model_config: ModelConfig,
21 - cache_config: CacheConfig,
22 - parallel_config: ParallelConfig,
23 - scheduler_config: SchedulerConfig,
24 - device_config: DeviceConfig,
25 - lora_config: Optional[LoRAConfig],
26 - vision_language_config: Optional[VisionLanguageConfig],
27 - speculative_config: Optional[SpeculativeConfig],
28 - ) -> None:
18 + def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
19 + parallel_config: ParallelConfig,
20 + scheduler_config: SchedulerConfig,
21 + device_config: DeviceConfig,
22 + lora_config: Optional[LoRAConfig],
23 + vision_language_config: Optional[VisionLanguageConfig],
24 + speculative_config: Optional[SpeculativeConfig],
25 + tensorizer_config: Optional[TensorizerConfig]) -> None:
26 self.model_config = model_config
27 self.cache_config = cache_config
28 self.lora_config = lora_config
@@ -33,6 +30,7 @@ class GPUExecutor(ExecutorBase):
30 self.scheduler_config = scheduler_config
31 self.device_config = device_config
32 self.vision_language_config = vision_language_config
33 + self.tensorizer_config = tensorizer_config
34
35 assert (not speculative_config
36 ), "Speculative decoding not yet supported for GPU backend"
@@ -61,6 +59,7 @@ class GPUExecutor(ExecutorBase):
59 distributed_init_method=distributed_init_method,
60 lora_config=self.lora_config,
61 vision_language_config=self.vision_language_config,
62 + tensorizer_config=self.tensorizer_config,
63 is_driver_worker=True,
64 )
65 self.driver_worker.init_device()
vllm/executor/ray_gpu_executor.py
+5 -1
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 ParallelConfig, SchedulerConfig, SpeculativeConfig,
10 - VisionLanguageConfig)
10 + TensorizerConfig, VisionLanguageConfig)
11 from vllm.engine.ray_utils import RayWorkerVllm, ray
12 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13 from vllm.logger import init_logger
@@ -42,6 +42,7 @@ class RayGPUExecutor(ExecutorBase):
42 lora_config: Optional[LoRAConfig],
43 vision_language_config: Optional[VisionLanguageConfig],
44 speculative_config: Optional[SpeculativeConfig],
45 + tensorizer_config: Optional[TensorizerConfig],
46 ) -> None:
47 self.model_config = model_config
48 self.cache_config = cache_config
@@ -50,6 +51,7 @@ class RayGPUExecutor(ExecutorBase):
51 self.scheduler_config = scheduler_config
52 self.device_config = device_config
53 self.vision_language_config = vision_language_config
54 + self.tensorizer_config = tensorizer_config
55 assert (not speculative_config
56 ), "Speculative decoding not yet supported for RayGPU backend."
57
@@ -171,6 +173,7 @@ class RayGPUExecutor(ExecutorBase):
173 distributed_init_method=distributed_init_method,
174 lora_config=lora_config,
175 vision_language_config=vision_language_config,
176 + tensorizer_config=self.tensorizer_config,
177 ))
178
179 # Initialize the driver worker with the Worker class.
@@ -187,6 +190,7 @@ class RayGPUExecutor(ExecutorBase):
190 distributed_init_method=distributed_init_method,
191 lora_config=self.lora_config,
192 vision_language_config=self.vision_language_config,
193 + tensorizer_config=self.tensorizer_config,
194 is_driver_worker=True,
195 )
196
vllm/model_executor/model_loader.py
+43 -18
@@ -3,11 +3,14 @@ import contextlib
3 from typing import Tuple, Type
4
5 import torch
6 -import torch.nn as nn
6 +from torch import nn
7
8 from vllm.config import DeviceConfig, ModelConfig
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11 +from vllm.model_executor.tensorizer_loader import (
12 + ParameterizedLoadFormat, is_vllm_serialized_tensorizer,
13 + load_with_tensorizer)
14 from vllm.model_executor.weight_utils import (get_quant_config,
15 initialize_dummy_weights)
16
@@ -51,6 +54,7 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
54 **kwargs) -> nn.Module:
55 lora_config = kwargs.get("lora_config", None)
56 vision_language_config = kwargs.get("vision_language_config", None)
57 + tensorizer_config = kwargs.get("tensorizer_config", None)
58 model_class = _get_model_architecture(model_config)[0]
59
60 # Get the (maybe quantized) linear method.
@@ -71,33 +75,54 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
75 f"{model_config.dtype} is not supported for quantization "
76 f"method {model_config.quantization}. Supported dtypes: "
77 f"{supported_dtypes}")
78 +
79 linear_method = quant_config.get_linear_method()
80
81 with _set_default_torch_dtype(model_config.dtype):
82 # Create a model instance.
83 # The weights will be initialized as empty tensors.
84 + extra_kwargs = {}
85 + if hasattr(model_class, "supported_lora_modules"):
86 + extra_kwargs["lora_config"] = lora_config
87 + elif lora_config:
88 + raise ValueError(
89 + f"Model {model_class.__name__} does not support LoRA, "
90 + "but LoRA is enabled. Support for this model may "
91 + "be added in the future. If this is important to you, "
92 + "please open an issue on github.")
93 + elif model_class in _VISION_MODEL_CLASSES:
94 + extra_kwargs["vision_language_config"] = vision_language_config
95 +
96 with torch.device(device_config.device):
80 - if hasattr(model_class, "supported_lora_modules"):
81 - model = model_class(model_config.hf_config, linear_method,
82 - lora_config)
83 - elif lora_config:
84 - raise ValueError(
85 - f"Model {model_class.__name__} does not support LoRA, "
86 - "but LoRA is enabled. Support for this model may "
87 - "be added in the future. If this is important to you, "
88 - "please open an issue on github.")
89 - else:
90 - if model_class not in _VISION_MODEL_CLASSES:
91 - model = model_class(model_config.hf_config, linear_method)
92 - else:
93 - model = model_class(model_config.hf_config,
94 - vision_language_config, linear_method)
97 + if (model_config.load_format == "tensorizer"
98 + and is_vllm_serialized_tensorizer(tensorizer_config)):
99 + extra_kwargs["linear_method"] = linear_method
100 + tensorizer_config.model_class = model_class
101 + tensorizer_config.hf_config = model_config.hf_config
102 + tensorizer_config.dtype = model_config.dtype
103 + model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
104 + return model.eval()
105 + model = model_class(config=model_config.hf_config,
106 + linear_method=linear_method,
107 + **extra_kwargs)
108 if model_config.load_format == "dummy":
109 # NOTE(woosuk): For accurate performance evaluation, we assign
110 # random values to the weights.
111 initialize_dummy_weights(model)
112 else:
113 # Load the weights from the cached or downloaded files.
101 - model.load_weights(model_config.model, model_config.download_dir,
102 - model_config.load_format, model_config.revision)
114 + if model_config.load_format == "tensorizer":
115 + # Provide a dynamic load format for `model.load_weights`
116 + # to retain tensorizer args from CLI.
117 + model_config.load_format = ParameterizedLoadFormat(
118 + model_config.load_format)
119 + model_config.load_format.params = (
120 + tensorizer_config._construct_tensorizer_args())
121 +
122 + model.load_weights(
123 + model_config.model,
124 + model_config.download_dir,
125 + model_config.load_format,
126 + model_config.revision,
127 + )
128 return model.eval()
vllm/model_executor/tensorizer_loader.py new
+319
@@ -0,0 +1,319 @@
1 +import argparse
2 +import dataclasses
3 +import io
4 +import os
5 +import time
6 +import typing
7 +import warnings
8 +from dataclasses import dataclass
9 +from typing import Optional, Union
10 +
11 +import torch
12 +from torch import nn
13 +
14 +from vllm.config import TensorizerConfig
15 +from vllm.logger import init_logger
16 +from vllm.model_executor.layers.linear import LinearMethodBase
17 +from vllm.model_executor.layers.vocab_parallel_embedding import (
18 + VocabParallelEmbedding)
19 +
20 +tensorizer_load_fail = False
21 +
22 +try:
23 + from tensorizer import (DecryptionParams, EncryptionParams,
24 + TensorDeserializer, TensorSerializer)
25 + from tensorizer.stream_io import open_stream
26 + from tensorizer.utils import (convert_bytes, get_mem_usage,
27 + no_init_or_tensor)
28 +except ImportError:
29 + tensorizer_load_fail = True
30 +
31 +__all__ = [
32 + 'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
33 + 'TensorSerializer', 'open_stream', 'convert_bytes', 'get_mem_usage',
34 + 'no_init_or_tensor'
35 +]
36 +
37 +logger = init_logger(__name__)
38 +
39 +
40 +def load_with_tensorizer(tensorizer_config: TensorizerConfig,
41 + **extra_kwargs) -> nn.Module:
42 + tensorizer = TensorizerAgent(tensorizer_config, **extra_kwargs)
43 + return tensorizer.deserialize()
44 +
45 +
46 +def tensorizer_warning(message: str):
47 + return warnings.warn(message, category=PerformanceWarning, stacklevel=2)
48 +
49 +
50 +def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
51 + if tensorizer_config is None:
52 + return False
53 + return tensorizer_config.vllm_tensorized
54 +
55 +
56 +class ParameterizedLoadFormat(str):
57 + __slots__ = "params"
58 +
59 +
60 +class PerformanceWarning(UserWarning):
61 +
62 + def __str__(self):
63 + return (f"{super().__str__()}"
64 + " (set the VLLM_SILENCE_PERFORMANCE_WARNINGS"
65 + " environment variable to hide this)")
66 +
67 +
68 +if (os.getenv("VLLM_SILENCE_PERFORMANCE_WARNINGS", "").lower()
69 + not in ("", "0", "n", "no", "off", "disable")):
70 + warnings.simplefilter("ignore", category=PerformanceWarning)
71 +
72 +
73 +@dataclass
74 +class TensorizerArgs:
75 + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
76 + str, bytes, os.PathLike, int]
77 + vllm_tensorized: bool
78 + verify_hash: Optional[bool] = False
79 + num_readers: Optional[int] = 1
80 + encryption_keyfile: Optional[str] = None
81 + s3_access_key_id: Optional[str] = None
82 + s3_secret_access_key: Optional[str] = None
83 + s3_endpoint: Optional[str] = None
84 + """
85 + Args for the TensorizerAgent class. These are used to configure the behavior
86 + of the TensorDeserializer when loading tensors from a serialized model.
87 +
88 + Args:
89 + tensorizer_uri: Path to serialized model tensors. Can be a local file
90 + path or a S3 URI.
91 + vllm_tensorized: If True, indicates that the serialized model is a
92 + vLLM model. This is used to determine the behavior of the
93 + TensorDeserializer when loading tensors from a serialized model.
94 + It is far faster to deserialize a vLLM model as it utilizes
95 + tensorizer's optimized GPU loading.
96 + verify_hash: If True, the hashes of each tensor will be verified against
97 + the hashes stored in the metadata. A `HashMismatchError` will be
98 + raised if any of the hashes do not match.
99 + num_readers: Controls how many threads are allowed to read concurrently
100 + from the source file. Default is 1. This greatly increases
101 + performance.
102 + encryption_keyfile: File path to a binary file containing a
103 + binary key to use for decryption. `None` (the default) means
104 + no decryption. See the example script in
105 + examples/tensorize_vllm_model.py.
106 + s3_access_key_id: The access key for the S3 bucket. Can also be set via
107 + the S3_ACCESS_KEY_ID environment variable.
108 + s3_secret_access_key: The secret access key for the S3 bucket. Can also
109 + be set via the S3_SECRET_ACCESS_KEY environment variable.
110 + s3_endpoint: The endpoint for the S3 bucket. Can also be set via the
111 + S3_ENDPOINT_URL environment variable.
112 + """
113 +
114 + def __post_init__(self):
115 + self.file_obj = self.tensorizer_uri
116 + self.s3_access_key_id = (self.s3_access_key_id
117 + or os.environ.get("S3_ACCESS_KEY_ID")) or None
118 + self.s3_secret_access_key = (
119 + self.s3_secret_access_key
120 + or os.environ.get("S3_SECRET_ACCESS_KEY")) or None
121 + self.s3_endpoint = (self.s3_endpoint
122 + or os.environ.get("S3_ENDPOINT_URL")) or None
123 + self.stream_params = {
124 + "s3_access_key_id": self.s3_access_key_id,
125 + "s3_secret_access_key": self.s3_secret_access_key,
126 + "s3_endpoint": self.s3_endpoint,
127 + }
128 +
129 + # Omitting self.dtype and self.device as this behaves weirdly
130 + self.deserializer_params = {
131 + "verify_hash": self.verify_hash,
132 + "encryption": self.encryption_keyfile,
133 + "num_readers": self.num_readers
134 + }
135 + if self.encryption_keyfile:
136 + with open_stream(
137 + self.encryption_keyfile,
138 + **self.stream_params,
139 + ) as stream:
140 + key = stream.read()
141 + decryption_params = DecryptionParams.from_key(key)
142 + self.deserializer_params['encryption'] = decryption_params
143 +
144 + def add_cli_args(
145 + parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
146 + """Tensorizer CLI arguments"""
147 +
148 + # Create the argument group
149 + group = parser.add_argument_group(
150 + 'tensorizer options',
151 + description=('Options for configuring the behavior of the'
152 + ' tensorizer deserializer when '
153 + '--load-format=tensorizer'))
154 +
155 + group.add_argument(
156 + "--tensorizer-uri",
157 + help="Path to serialized model tensors. Can be a local file path,"
158 + " or an HTTP(S) or S3 URI.",
159 + )
160 + group.add_argument(
161 + "--verify-hash",
162 + action="store_true",
163 + help="If enabled, the hashes of each tensor will be verified"
164 + " against the hashes stored in the file metadata. An exception"
165 + " will be raised if any of the hashes do not match.",
166 + )
167 + group.add_argument(
168 + "--encryption-keyfile",
169 + default=None,
170 + help="The file path to a binary file containing a binary key to "
171 + "use for decryption. Can be a file path or S3 network URI.")
172 + group.add_argument(
173 + "--num-readers",
174 + default=1,
175 + type=int,
176 + help="Controls how many threads are allowed to read concurrently "
177 + "from the source file.")
178 + group.add_argument(
179 + "--s3-access-key-id",
180 + default=None,
181 + help="The access key for the S3 bucket. Can also be set via the "
182 + "S3_ACCESS_KEY_ID environment variable.",
183 + )
184 + group.add_argument(
185 + "--s3-secret-access-key",
186 + default=None,
187 + help="The secret access key for the S3 bucket. Can also be set via "
188 + "the S3_SECRET_ACCESS_KEY environment variable.",
189 + )
190 + group.add_argument(
191 + "--s3-endpoint",
192 + default=None,
193 + help="The endpoint for the S3 bucket. Can also be set via the "
194 + "S3_ENDPOINT_URL environment variable.",
195 + )
196 + group.add_argument(
197 + "--vllm-tensorized",
198 + action="store_true",
199 + help="If enabled, indicates that the serialized model is a vLLM "
200 + "model. This is used to determine the behavior of the "
201 + "TensorDeserializer when loading tensors from a "
202 + "serialized model.")
203 +
204 + return parser
205 +
206 + @classmethod
207 + def from_cli_args(cls, args: argparse.Namespace) -> "TensorizerArgs":
208 + # Get the list of attributes of this dataclass.
209 + attrs = [attr.name for attr in dataclasses.fields(cls)]
210 + # Set the attributes from the parsed arguments.
211 + tensorizer_args = cls(**{
212 + attr: getattr(args, attr)
213 + for attr in attrs if hasattr(args, attr)
214 + })
215 + return tensorizer_args
216 +
217 +
218 +class TensorizerAgent:
219 + """
220 + A class for performing tensorizer deserializations specifically for
221 + vLLM models using plaid_mode. Uses TensorizerArgs to configure the
222 + behavior of the TensorDeserializer when loading tensors from a serialized
223 + model. For deserializations of HuggingFace models, TensorDeserializer is
224 + instead used as an iterator directly in the func hf_model_weights_iterator
225 + in vllm/model_executor/weight_utils.py
226 + """
227 +
228 + def __init__(self, tensorizer_config: TensorizerConfig,
229 + linear_method: LinearMethodBase, **extra_kwargs):
230 + self.tensorizer_config = tensorizer_config
231 + self.tensorizer_args = (
232 + self.tensorizer_config._construct_tensorizer_args())
233 + self.extra_kwargs = extra_kwargs
234 + if extra_kwargs.get("linear_method", None) is not None:
235 + self.linear_method = extra_kwargs["linear_method"]
236 + else:
237 + self.linear_method = linear_method
238 + self.model = self._init_model()
239 +
240 + if tensorizer_load_fail:
241 + raise ImportError(
242 + "Tensorizer is not installed. Please install tensorizer "
243 + "to use this feature with `pip install vllm[tensorizer]`.")
244 +
245 + def _init_model(self):
246 + model_args = self.tensorizer_config.hf_config
247 + model_args.torch_dtype = self.tensorizer_config.dtype
248 + with no_init_or_tensor():
249 + return self.tensorizer_config.model_class(
250 + config=model_args,
251 + linear_method=self.linear_method,
252 + **self.extra_kwargs)
253 +
254 + def _resize_lora_embeddings(self):
255 + """Modify LoRA embedding layers to use bigger tensors
256 + to allow for adapter added tokens."""
257 + for child in self.model.modules():
258 + if (isinstance(child, VocabParallelEmbedding)
259 + and child.weight.shape[0] <
260 + child.num_embeddings_per_partition):
261 + new_weight = torch.empty(child.num_embeddings_per_partition,
262 + child.embedding_dim,
263 + dtype=child.weight.dtype,
264 + device=child.weight.device)
265 + new_weight[:child.weight.shape[0]].copy_(child.weight.data)
266 + new_weight[child.weight.shape[0]:].fill_(0)
267 + child.weight.data = new_weight
268 +
269 + def _check_tensors_on_meta_device(self):
270 + for tensor in self.model.state_dict().values():
271 + if tensor.device.type == 'meta':
272 + raise ValueError(
273 + "The serialized model contains tensors on the meta device,"
274 + " indicating that some tensors were not loaded properly."
275 + " Please check that the parameters of the model being"
276 + " specified match that of the serialized model, such as"
277 + " its quantization.")
278 +
279 + def deserialize(self):
280 + """
281 + Deserialize the model using the TensorDeserializer. This method is
282 + specifically for vLLM models using tensorizer's plaid_mode.
283 +
284 + The deserializer makes use of tensorizer_args.stream_params
285 + to configure the behavior of the stream when loading tensors from a
286 + serialized model. The deserializer_params are used to configure the
287 + behavior of the TensorDeserializer when loading tensors themselves.
288 + Documentation on these params can be found in TensorizerArgs
289 +
290 + Returns:
291 + nn.Module: The deserialized model.
292 + """
293 + before_mem = get_mem_usage()
294 + # Lazy load the tensors from S3 into the model.
295 + start = time.perf_counter()
296 + with open_stream(
297 + self.tensorizer_args.tensorizer_uri,
298 + mode="rb",
299 + **self.tensorizer_args.stream_params,
300 + ) as stream, TensorDeserializer(
301 + stream,
302 + dtype=self.tensorizer_config.dtype,
303 + **self.tensorizer_args.deserializer_params) as deserializer:
304 + deserializer.load_into_module(self.model)
305 + end = time.perf_counter()
306 +
307 + total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
308 + duration = end - start
309 + per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
310 + after_mem = get_mem_usage()
311 + deserializer.close()
312 + logger.info(f"Deserialized {total_bytes_str} in "
313 + f"{end - start:0.2f}s, {per_second}/s")
314 + logger.info(f"Memory usage before: {before_mem}")
315 + logger.info(f"Memory usage after: {after_mem}")
316 +
317 + self._check_tensors_on_meta_device()
318 + self._resize_lora_embeddings()
319 + return self.model.eval()
vllm/model_executor/weight_utils.py
+30 -4
@@ -5,7 +5,7 @@ import hashlib
5 import json
6 import os
7 from collections import defaultdict
8 -from typing import Any, Iterable, Iterator, List, Optional, Tuple
8 +from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union
9
10 import filelock
11 import huggingface_hub.constants
@@ -161,7 +161,8 @@ def prepare_hf_model_weights(
161 revision: Optional[str] = None,
162 ) -> Tuple[str, List[str], bool]:
163 # Download model weights from huggingface.
164 - is_local = os.path.isdir(model_name_or_path)
164 + is_local = os.path.isdir(model_name_or_path) \
165 + and load_format != "tensorizer"
166 use_safetensors = False
167 # Some quantized models use .pt files for storing the weights.
168 if load_format == "auto":
@@ -173,13 +174,15 @@ def prepare_hf_model_weights(
174 allow_patterns = ["*.pt"]
175 elif load_format == "npcache":
176 allow_patterns = ["*.bin"]
177 + elif load_format == "tensorizer":
178 + allow_patterns = ["*.tensors"]
179 else:
180 raise ValueError(f"Unknown load_format: {load_format}")
181
182 if fall_back_to_pt:
183 allow_patterns += ["*.pt"]
184
182 - if not is_local:
185 + if not is_local and load_format != "tensorizer":
186 # Before we download we look at that is available:
187 fs = HfFileSystem()
188 file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
@@ -224,6 +227,9 @@ def prepare_hf_model_weights(
227 if not any(f.endswith(x) for x in blacklist)
228 ]
229
230 + if load_format == "tensorizer":
231 + return hf_folder, hf_weights_files, use_safetensors
232 +
233 if len(hf_weights_files) == 0:
234 raise RuntimeError(
235 f"Cannot find any model weights with `{model_name_or_path}`")
@@ -234,7 +240,7 @@ def prepare_hf_model_weights(
240 def hf_model_weights_iterator(
241 model_name_or_path: str,
242 cache_dir: Optional[str] = None,
237 - load_format: str = "auto",
243 + load_format: Union[Tuple, str] = "auto",
244 revision: Optional[str] = None,
245 fall_back_to_pt: Optional[bool] = True,
246 ) -> Iterator[Tuple[str, torch.Tensor]]:
@@ -277,6 +283,26 @@ def hf_model_weights_iterator(
283 with open(param_path, "rb") as f:
284 param = np.load(f)
285 yield name, torch.from_numpy(param)
286 + elif load_format == "tensorizer":
287 + from vllm.model_executor.tensorizer_loader import (TensorDeserializer,
288 + open_stream,
289 + tensorizer_warning)
290 + tensorizer_args = load_format.params
291 + tensorizer_warning(
292 + "Deserializing HuggingFace models is not optimized for "
293 + "loading on vLLM, as tensorizer is forced to load to CPU. "
294 + "Consider deserializing a vLLM model instead for faster "
295 + "load times. See the examples/tensorize_vllm_model.py example "
296 + "script for serializing vLLM models.")
297 +
298 + deserializer_args = tensorizer_args.deserializer_params
299 + stream_params = tensorizer_args.stream_params
300 + stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
301 + with TensorDeserializer(stream, **deserializer_args,
302 + device="cpu") as state:
303 + for name, param in state.items():
304 + yield name, param
305 + del state
306 elif use_safetensors:
307 for st_file in hf_weights_files:
308 with safe_open(st_file, framework="pt") as f:
vllm/worker/model_runner.py
+7 -2
@@ -10,7 +10,8 @@ import torch.nn as nn
10 from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 get_attn_backend)
12 from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13 - SchedulerConfig, VisionLanguageConfig)
13 + SchedulerConfig, TensorizerConfig,
14 + VisionLanguageConfig)
15 from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16 from vllm.distributed.device_communicators import (custom_all_reduce,
17 pynccl_utils)
@@ -111,11 +112,13 @@ class ModelRunner:
112 kv_cache_dtype: Optional[str] = "auto",
113 is_driver_worker: bool = False,
114 vision_language_config: Optional[VisionLanguageConfig] = None,
115 + tensorizer_config: Optional[TensorizerConfig] = None,
116 ):
117 self.model_config = model_config
118 self.parallel_config = parallel_config
119 self.scheduler_config = scheduler_config
120 self.lora_config = lora_config
121 + self.tensorizer_config = tensorizer_config
122 self.is_driver_worker = is_driver_worker
123
124 # model_config can be None in tests/samplers/test_sampler.py.
@@ -158,7 +161,9 @@ class ModelRunner:
161 lora_config=self.lora_config,
162 vision_language_config=self.vision_language_config,
163 parallel_config=self.parallel_config,
161 - scheduler_config=self.scheduler_config)
164 + scheduler_config=self.scheduler_config,
165 + tensorizer_config=self.tensorizer_config,
166 + )
167
168 self.model_memory_usage = m.consumed_memory
169 logger.info(f"Loading model weights took "
vllm/worker/worker.py
+7 -2
@@ -7,7 +7,8 @@ import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
10 + ParallelConfig, SchedulerConfig, TensorizerConfig,
11 + VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 init_distributed_environment)
@@ -42,6 +43,7 @@ class Worker(WorkerBase):
43 distributed_init_method: str,
44 lora_config: Optional[LoRAConfig] = None,
45 vision_language_config: Optional[VisionLanguageConfig] = None,
46 + tensorizer_config: Optional[TensorizerConfig] = None,
47 is_driver_worker: bool = False,
48 ) -> None:
49 self.model_config = model_config
@@ -53,6 +55,7 @@ class Worker(WorkerBase):
55 self.rank = rank
56 self.distributed_init_method = distributed_init_method
57 self.lora_config = lora_config
58 + self.tensorizer_config = tensorizer_config
59 self.is_driver_worker = is_driver_worker
60 if self.is_driver_worker:
61 assert self.rank == 0, "The driver worker must have rank 0."
@@ -70,7 +73,9 @@ class Worker(WorkerBase):
73 lora_config=self.lora_config,
74 kv_cache_dtype=self.cache_config.cache_dtype,
75 is_driver_worker=is_driver_worker,
73 - vision_language_config=vision_language_config)
76 + vision_language_config=vision_language_config,
77 + tensorizer_config=tensorizer_config,
78 + )
79 # Uninitialized cache engine. Will be initialized by
80 # initialize_cache.
81 self.cache_engine = None