A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Frontend] [Core] feat: Add model loading using `tensorizer` (#3476)
Sanger Steel committed
Apr 13, 2024 at 20:13 UTC
711a000255eac3e034f0b73aa5cc62b45201a571
20 files changed
+1351
-51
.buildkite/test-pipeline.yaml
+3
@@ -91,6 +91,9 @@ steps:
91
command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
92
parallelism: 4
93
94
+- label: Tensorizer Test
95
+ command: apt-get install curl libsodium23 && pytest -v -s tensorizer
96
+
97
- label: Metrics Test
98
command: pytest -v -s metrics
99
docs/source/conf.py
+1
@@ -83,6 +83,7 @@ autodoc_mock_imports = [
83
"vllm._C",
84
"numpy",
85
"tqdm",
86
+ "tensorizer",
87
]
88
89
for mock_target in autodoc_mock_imports:
docs/source/models/engine_args.rst
+2
-1
@@ -36,7 +36,7 @@ Below, you can find an explanation of every engine argument for vLLM:
36
37
Directory to download and load the weights, default to the default cache dir of huggingface.
38
39
-.. option:: --load-format {auto,pt,safetensors,npcache,dummy}
39
+.. option:: --load-format {auto,pt,safetensors,npcache,dummy,tensorizer}
40
41
The format of the model weights to load.
42
@@ -45,6 +45,7 @@ Below, you can find an explanation of every engine argument for vLLM:
45
* "safetensors" will load the weights in the safetensors format.
46
* "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47
* "dummy" will initialize the weights with random values, mainly for profiling.
48
+ * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_. See `tensorized_vllm_model.py` in the examples folder to serialize a vLLM model, and for more information. Tensorizer support for vLLM can be installed with `pip install vllm[tensorizer]`.
49
50
.. option:: --dtype {auto,half,float16,bfloat16,float,float32}
51
examples/tensorize_vllm_model.py
new
+254
@@ -0,0 +1,254 @@
1
+import argparse
2
+import dataclasses
3
+import os
4
+import time
5
+import uuid
6
+from functools import partial
7
+from typing import Type
8
+
9
+import torch
10
+import torch.nn as nn
11
+from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12
+ TensorSerializer, stream_io)
13
+from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14
+from transformers import AutoConfig, PretrainedConfig
15
+
16
+from vllm.distributed import initialize_model_parallel
17
+from vllm.engine.arg_utils import EngineArgs
18
+from vllm.engine.llm_engine import LLMEngine
19
+from vllm.model_executor.models import ModelRegistry
20
+from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
+
22
+# yapf conflicts with isort for this docstring
23
+# yapf: disable
24
+"""
25
+tensorize_vllm_model.py is a script that can be used to serialize and
26
+deserialize vLLM models. These models can be loaded using tensorizer directly
27
+to the GPU extremely quickly. Tensor encryption and decryption is also
28
+supported, although libsodium must be installed to use it. Install
29
+vllm with tensorizer support using `pip install vllm[tensorizer]`.
30
+
31
+To serialize a model, you can run something like this:
32
+
33
+python tensorize_vllm_model.py \
34
+ --model EleutherAI/gpt-j-6B \
35
+ --dtype float16 \
36
+ serialize \
37
+ --serialized-directory s3://my-bucket/ \
38
+ --suffix vllm
39
+
40
+Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41
+and saves it to your S3 bucket. A local directory can also be used.
42
+
43
+You can also encrypt the model weights with a randomly-generated key by
44
+providing a `--keyfile` argument.
45
+
46
+To deserialize a model, you can run something like this:
47
+
48
+python tensorize_vllm_model.py \
49
+ --model EleutherAI/gpt-j-6B \
50
+ --dtype float16 \
51
+ deserialize \
52
+ --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53
+
54
+Which downloads the model tensors from your S3 bucket and deserializes them.
55
+To provide S3 credentials, you can provide `--s3-access-key-id` and
56
+`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57
+the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58
+in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59
+
60
+
61
+You can also provide a `--keyfile` argument to decrypt the model weights if
62
+they were serialized with encryption.
63
+
64
+For more information on the available arguments, run
65
+`python tensorize_vllm_model.py --help`.
66
+"""
67
+
68
+
69
+def parse_args():
70
+ parser = argparse.ArgumentParser(
71
+ description="An example script that can be used to serialize and "
72
+ "deserialize vLLM models. These models "
73
+ "can be loaded using tensorizer directly to the GPU "
74
+ "extremely quickly. Tensor encryption and decryption is "
75
+ "also supported, although libsodium must be installed to "
76
+ "use it.")
77
+ parser = EngineArgs.add_cli_args(parser)
78
+ subparsers = parser.add_subparsers(dest='command')
79
+
80
+ serialize_parser = subparsers.add_parser(
81
+ 'serialize', help="Serialize a model to `--serialized-directory`")
82
+
83
+ serialize_parser.add_argument(
84
+ "--suffix",
85
+ type=str,
86
+ required=False,
87
+ help=(
88
+ "The suffix to append to the serialized model directory, which is "
89
+ "used to construct the location of the serialized model tensors, "
90
+ "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91
+ "`--suffix` is `v1`, the serialized model tensors will be "
92
+ "saved to "
93
+ "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94
+ "If none is provided, a random UUID will be used."))
95
+ serialize_parser.add_argument(
96
+ "--serialized-directory",
97
+ type=str,
98
+ required=True,
99
+ help="The directory to serialize the model to. "
100
+ "This can be a local directory or S3 URI. The path to where the "
101
+ "tensors are saved is a combination of the supplied `dir` and model "
102
+ "reference ID. For instance, if `dir` is the serialized directory, "
103
+ "and the model HuggingFace ID is `EleutherAI/gpt-j-6B`, tensors will "
104
+ "be saved to `dir/vllm/EleutherAI/gpt-j-6B/suffix/model.tensors`, "
105
+ "where `suffix` is given by `--suffix` or a random UUID if not "
106
+ "provided.")
107
+
108
+ serialize_parser.add_argument(
109
+ "--keyfile",
110
+ type=str,
111
+ required=False,
112
+ help=("Encrypt the model weights with a randomly-generated binary key,"
113
+ " and save the key at this path"))
114
+
115
+ deserialize_parser = subparsers.add_parser(
116
+ 'deserialize',
117
+ help=("Deserialize a model from `--path-to-tensors`"
118
+ " to verify it can be loaded and used."))
119
+
120
+ deserialize_parser.add_argument(
121
+ "--path-to-tensors",
122
+ type=str,
123
+ required=True,
124
+ help="The local path or S3 URI to the model tensors to deserialize. ")
125
+
126
+ deserialize_parser.add_argument(
127
+ "--keyfile",
128
+ type=str,
129
+ required=False,
130
+ help=("Path to a binary key to use to decrypt the model weights,"
131
+ " if the model was serialized with encryption"))
132
+
133
+ return parser.parse_args()
134
+
135
+
136
+def make_model_contiguous(model):
137
+ # Ensure tensors are saved in memory contiguously
138
+ for param in model.parameters():
139
+ param.data = param.data.contiguous()
140
+
141
+
142
+def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
143
+ architectures = getattr(config, "architectures", [])
144
+ for arch in architectures:
145
+ model_cls = ModelRegistry.load_model_cls(arch)
146
+ if model_cls is not None:
147
+ return model_cls
148
+ raise ValueError(
149
+ f"Model architectures {architectures} are not supported for now. "
150
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
151
+
152
+
153
+def serialize():
154
+
155
+ eng_args_dict = {f.name: getattr(args, f.name) for f in
156
+ dataclasses.fields(EngineArgs)}
157
+ engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
158
+ engine = LLMEngine.from_engine_args(engine_args)
159
+
160
+ model = (engine.model_executor.driver_worker.
161
+ model_runner.model)
162
+
163
+ encryption_params = EncryptionParams.random() if keyfile else None
164
+ if keyfile:
165
+ with _write_stream(keyfile) as stream:
166
+ stream.write(encryption_params.key)
167
+
168
+ with _write_stream(model_path) as stream:
169
+ serializer = TensorSerializer(stream, encryption=encryption_params)
170
+ serializer.write_module(model)
171
+ serializer.close()
172
+
173
+ print("Serialization complete. Model tensors saved to", model_path)
174
+ if keyfile:
175
+ print("Key saved to", keyfile)
176
+
177
+
178
+def deserialize():
179
+ config = AutoConfig.from_pretrained(model_ref)
180
+
181
+ with no_init_or_tensor():
182
+ model_class = _get_vllm_model_architecture(config)
183
+ model = model_class(config)
184
+
185
+ before_mem = get_mem_usage()
186
+ start = time.time()
187
+
188
+ if keyfile:
189
+ with _read_stream(keyfile) as stream:
190
+ key = stream.read()
191
+ decryption_params = DecryptionParams.from_key(key)
192
+ tensorizer_args.deserializer_params['encryption'] = \
193
+ decryption_params
194
+
195
+ with (_read_stream(model_path)) as stream, TensorDeserializer(
196
+ stream, **tensorizer_args.deserializer_params) as deserializer:
197
+ deserializer.load_into_module(model)
198
+ end = time.time()
199
+
200
+ # Brag about how fast we are.
201
+ total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
202
+ duration = end - start
203
+ per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
204
+ after_mem = get_mem_usage()
205
+ print(
206
+ f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
207
+ )
208
+ print(f"Memory usage before: {before_mem}")
209
+ print(f"Memory usage after: {after_mem}")
210
+
211
+ return model
212
+
213
+
214
+args = parse_args()
215
+
216
+s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
217
+ or None)
218
+s3_secret_access_key = (args.s3_secret_access_key
219
+ or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
220
+
221
+s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
222
+
223
+_read_stream, _write_stream = (partial(
224
+ stream_io.open_stream,
225
+ mode=mode,
226
+ s3_access_key_id=s3_access_key_id,
227
+ s3_secret_access_key=s3_secret_access_key,
228
+ s3_endpoint=s3_endpoint,
229
+) for mode in ("rb", "wb+"))
230
+
231
+model_ref = args.model
232
+
233
+model_name = model_ref.split("/")[1]
234
+
235
+os.environ["MASTER_ADDR"] = "127.0.0.1"
236
+os.environ["MASTER_PORT"] = "8080"
237
+
238
+torch.distributed.init_process_group(world_size=1, rank=0)
239
+initialize_model_parallel()
240
+
241
+keyfile = args.keyfile if args.keyfile else None
242
+
243
+if args.command == "serialize":
244
+ input_dir = args.serialized_directory.rstrip('/')
245
+ suffix = args.suffix if args.suffix else uuid.uuid4().hex
246
+ base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
247
+ model_path = f"{base_path}/model.tensors"
248
+ serialize()
249
+elif args.command == "deserialize":
250
+ tensorizer_args = TensorizerArgs.from_cli_args(args)
251
+ model_path = args.path_to_tensors
252
+ deserialize()
253
+else:
254
+ raise ValueError("Either serialize or deserialize must be specified.")
requirements-cpu.txt
+1
-1
@@ -3,4 +3,4 @@
3
4
# Dependencies for x86_64 CPUs
5
torch == 2.2.1+cpu
6
-triton >= 2.1.0 # FIXME(woosuk): This is a hack to avoid import error.
6
+triton >= 2.1.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
requirements-dev.txt
+1
@@ -14,6 +14,7 @@ types-setuptools
14
15
# testing
16
pytest
17
+tensorizer==2.9.0a0
18
pytest-forked
19
pytest-asyncio
20
pytest-rerunfailures
setup.py
+3
@@ -405,6 +405,9 @@ setup(
405
python_requires=">=3.8",
406
install_requires=get_requirements(),
407
ext_modules=ext_modules,
408
+ extras_require={
409
+ "optional": ["tensorizer==2.9.0a1"],
410
+ },
411
cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
412
package_data=package_data,
413
)
tests/tensorizer/__init__.py
tests/tensorizer/tensorize_vllm_model_for_testing.py
new
+245
@@ -0,0 +1,245 @@
1
+import argparse
2
+import dataclasses
3
+import os
4
+import time
5
+import uuid
6
+from functools import partial
7
+from typing import Type
8
+
9
+import torch
10
+import torch.nn as nn
11
+from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12
+ TensorSerializer, stream_io)
13
+from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14
+from transformers import AutoConfig, PretrainedConfig
15
+
16
+from vllm.distributed import initialize_model_parallel
17
+from vllm.engine.arg_utils import EngineArgs
18
+from vllm.engine.llm_engine import LLMEngine
19
+from vllm.model_executor.models import ModelRegistry
20
+from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
+
22
+# yapf conflicts with isort for this docstring
23
+# yapf: disable
24
+"""
25
+tensorize_vllm_model.py is a script that can be used to serialize and
26
+deserialize vLLM models. These models can be loaded using tensorizer directly
27
+to the GPU extremely quickly. Tensor encryption and decryption is also
28
+supported, although libsodium must be installed to use it. Install
29
+vllm with tensorizer support using `pip install vllm[tensorizer]`.
30
+
31
+To serialize a model, you can run something like this:
32
+
33
+python tensorize_vllm_model.py \
34
+ --model EleutherAI/gpt-j-6B \
35
+ --dtype float16 \
36
+ serialize \
37
+ --serialized-directory s3://my-bucket/ \
38
+ --suffix vllm
39
+
40
+Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41
+and saves it to your S3 bucket. A local directory can also be used.
42
+
43
+You can also encrypt the model weights with a randomly-generated key by
44
+providing a `--keyfile` argument.
45
+
46
+To deserialize a model, you can run something like this:
47
+
48
+python tensorize_vllm_model.py \
49
+ --model EleutherAI/gpt-j-6B \
50
+ --dtype float16 \
51
+ deserialize \
52
+ --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53
+
54
+Which downloads the model tensors from your S3 bucket and deserializes them.
55
+To provide S3 credentials, you can provide `--s3-access-key-id` and
56
+`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57
+the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58
+in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59
+
60
+
61
+You can also provide a `--keyfile` argument to decrypt the model weights if
62
+they were serialized with encryption.
63
+
64
+For more information on the available arguments, run
65
+`python tensorize_vllm_model.py --help`.
66
+"""
67
+
68
+
69
+def parse_args():
70
+ parser = argparse.ArgumentParser(
71
+ description="An example script that can be used to serialize and "
72
+ "deserialize vLLM models. These models "
73
+ "can be loaded using tensorizer directly to the GPU "
74
+ "extremely quickly. Tensor encryption and decryption is "
75
+ "also supported, although libsodium must be installed to "
76
+ "use it.")
77
+ parser = EngineArgs.add_cli_args(parser)
78
+ subparsers = parser.add_subparsers(dest='command')
79
+
80
+ serialize_parser = subparsers.add_parser(
81
+ 'serialize', help="Serialize a model to `--serialized-directory`")
82
+
83
+ serialize_parser.add_argument(
84
+ "--suffix",
85
+ type=str,
86
+ required=False,
87
+ help=(
88
+ "The suffix to append to the serialized model directory, which is "
89
+ "used to construct the location of the serialized model tensors, "
90
+ "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91
+ "`--suffix` is `v1`, the serialized model tensors will be "
92
+ "saved to "
93
+ "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94
+ "If none is provided, a random UUID will be used."))
95
+ serialize_parser.add_argument(
96
+ "--serialized-directory",
97
+ type=str,
98
+ required=True)
99
+
100
+ serialize_parser.add_argument(
101
+ "--keyfile",
102
+ type=str,
103
+ required=False,
104
+ help=("Encrypt the model weights with a randomly-generated binary key,"
105
+ " and save the key at this path"))
106
+
107
+ deserialize_parser = subparsers.add_parser(
108
+ 'deserialize',
109
+ help=("Deserialize a model from `--path-to-tensors`"
110
+ " to verify it can be loaded and used."))
111
+
112
+ deserialize_parser.add_argument(
113
+ "--path-to-tensors",
114
+ type=str,
115
+ required=True,
116
+ help="The local path or S3 URI to the model tensors to deserialize. ")
117
+
118
+ deserialize_parser.add_argument(
119
+ "--keyfile",
120
+ type=str,
121
+ required=False,
122
+ help=("Path to a binary key to use to decrypt the model weights,"
123
+ " if the model was serialized with encryption"))
124
+
125
+ return parser.parse_args()
126
+
127
+
128
+def make_model_contiguous(model):
129
+ # Ensure tensors are saved in memory contiguously
130
+ for param in model.parameters():
131
+ param.data = param.data.contiguous()
132
+
133
+
134
+def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
135
+ architectures = getattr(config, "architectures", [])
136
+ for arch in architectures:
137
+ model_cls = ModelRegistry.load_model_cls(arch)
138
+ if model_cls is not None:
139
+ return model_cls
140
+ raise ValueError(
141
+ f"Model architectures {architectures} are not supported for now. "
142
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
143
+
144
+
145
+def serialize():
146
+ eng_args_dict = {f.name: getattr(args, f.name) for f in
147
+ dataclasses.fields(EngineArgs)}
148
+ engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
149
+ engine = LLMEngine.from_engine_args(engine_args)
150
+
151
+ model = (engine.model_executor.driver_worker.
152
+ model_runner.model)
153
+
154
+ encryption_params = EncryptionParams.random() if keyfile else None
155
+ if keyfile:
156
+ with _write_stream(keyfile) as stream:
157
+ stream.write(encryption_params.key)
158
+
159
+ with _write_stream(model_path) as stream:
160
+ serializer = TensorSerializer(stream, encryption=encryption_params)
161
+ serializer.write_module(model)
162
+ serializer.close()
163
+
164
+ print("Serialization complete. Model tensors saved to", model_path)
165
+ if keyfile:
166
+ print("Key saved to", keyfile)
167
+
168
+
169
+def deserialize():
170
+ config = AutoConfig.from_pretrained(model_ref)
171
+
172
+ with no_init_or_tensor():
173
+ model_class = _get_vllm_model_architecture(config)
174
+ model = model_class(config)
175
+
176
+ before_mem = get_mem_usage()
177
+ start = time.time()
178
+
179
+ if keyfile:
180
+ with _read_stream(keyfile) as stream:
181
+ key = stream.read()
182
+ decryption_params = DecryptionParams.from_key(key)
183
+ tensorizer_args.deserializer_params['encryption'] = \
184
+ decryption_params
185
+
186
+ with (_read_stream(model_path)) as stream, TensorDeserializer(
187
+ stream, **tensorizer_args.deserializer_params) as deserializer:
188
+ deserializer.load_into_module(model)
189
+ end = time.time()
190
+
191
+ # Brag about how fast we are.
192
+ total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
193
+ duration = end - start
194
+ per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
195
+ after_mem = get_mem_usage()
196
+ print(
197
+ f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
198
+ )
199
+ print(f"Memory usage before: {before_mem}")
200
+ print(f"Memory usage after: {after_mem}")
201
+
202
+ return model
203
+
204
+
205
+args = parse_args()
206
+
207
+s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
208
+ or None)
209
+s3_secret_access_key = (args.s3_secret_access_key
210
+ or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
211
+
212
+s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
213
+
214
+_read_stream, _write_stream = (partial(
215
+ stream_io.open_stream,
216
+ mode=mode,
217
+ s3_access_key_id=s3_access_key_id,
218
+ s3_secret_access_key=s3_secret_access_key,
219
+ s3_endpoint=s3_endpoint,
220
+) for mode in ("rb", "wb+"))
221
+
222
+model_ref = args.model
223
+
224
+model_name = model_ref.split("/")[1]
225
+
226
+os.environ["MASTER_ADDR"] = "127.0.0.1"
227
+os.environ["MASTER_PORT"] = "8080"
228
+
229
+torch.distributed.init_process_group(world_size=1, rank=0)
230
+initialize_model_parallel()
231
+
232
+keyfile = args.keyfile if args.keyfile else None
233
+
234
+if args.command == "serialize":
235
+ input_dir = args.serialized_directory.rstrip('/')
236
+ suffix = args.suffix if args.suffix else uuid.uuid4().hex
237
+ base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
238
+ model_path = f"{base_path}/model.tensors"
239
+ serialize()
240
+elif args.command == "deserialize":
241
+ tensorizer_args = TensorizerArgs.from_cli_args(args)
242
+ model_path = args.path_to_tensors
243
+ deserialize()
244
+else:
245
+ raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer/test_tensorizer.py
new
+302
@@ -0,0 +1,302 @@
1
+import gc
2
+import subprocess
3
+from unittest.mock import MagicMock, patch
4
+
5
+import pytest
6
+import torch
7
+
8
+from tests.entrypoints.test_openai_server import ServerRunner
9
+from vllm import SamplingParams
10
+from vllm.config import TensorizerConfig
11
+from vllm.model_executor.tensorizer_loader import (
12
+ EncryptionParams, TensorSerializer, is_vllm_serialized_tensorizer,
13
+ load_with_tensorizer, open_stream)
14
+
15
+prompts = [
16
+ "Hello, my name is",
17
+ "The president of the United States is",
18
+ "The capital of France is",
19
+ "The future of AI is",
20
+]
21
+# Create a sampling params object.
22
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
23
+
24
+model_ref = "facebook/opt-125m"
25
+
26
+
27
+def is_curl_installed():
28
+ try:
29
+ subprocess.check_call(['curl', '--version'])
30
+ return True
31
+ except (subprocess.CalledProcessError, FileNotFoundError):
32
+ return False
33
+
34
+
35
+@pytest.fixture(autouse=True)
36
+def tensorizer_config():
37
+ config = TensorizerConfig(tensorizer_uri="vllm", vllm_tensorized=True)
38
+ return config
39
+
40
+
41
+@patch('vllm.model_executor.tensorizer_loader.TensorizerAgent')
42
+def test_load_with_tensorizer(mock_agent, tensorizer_config):
43
+ mock_linear_method = MagicMock()
44
+ mock_agent_instance = mock_agent.return_value
45
+ mock_agent_instance.deserialize.return_value = MagicMock()
46
+
47
+ result = load_with_tensorizer(tensorizer_config,
48
+ linear_method=mock_linear_method)
49
+
50
+ mock_agent.assert_called_once_with(tensorizer_config,
51
+ linear_method=mock_linear_method)
52
+ mock_agent_instance.deserialize.assert_called_once()
53
+ assert result == mock_agent_instance.deserialize.return_value
54
+
55
+
56
+def test_is_vllm_model_with_vllm_in_uri(tensorizer_config):
57
+ tensorizer_config.vllm_tensorized = True
58
+
59
+ result = is_vllm_serialized_tensorizer(tensorizer_config)
60
+
61
+ assert result is True
62
+
63
+
64
+def test_is_vllm_model_without_vllm_in_uri(tensorizer_config):
65
+ tensorizer_config.vllm_tensorized = False
66
+
67
+ result = is_vllm_serialized_tensorizer(tensorizer_config)
68
+
69
+ assert result is False
70
+
71
+
72
+def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
73
+ vllm_model = vllm_runner(model_ref)
74
+ model_path = tmp_path / (model_ref + ".tensors")
75
+ outputs = vllm_model.generate(prompts, sampling_params)
76
+ model = (vllm_model.model.llm_engine.model_executor.driver_worker.
77
+ model_runner.model)
78
+ with open_stream(model_path, "wb+") as stream:
79
+ serializer = TensorSerializer(stream)
80
+ serializer.write_module(model)
81
+ del vllm_model, model
82
+ gc.collect()
83
+ torch.cuda.empty_cache()
84
+ loaded_vllm_model = vllm_runner(model_ref,
85
+ load_format="tensorizer",
86
+ tensorizer_uri=model_path,
87
+ num_readers=1,
88
+ vllm_tensorized=True)
89
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
90
+
91
+ # Assumes SamplingParams being seeded ensures the outputs are deterministic
92
+ assert outputs == deserialized_outputs
93
+
94
+
95
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
96
+def test_can_deserialize_s3(vllm_runner):
97
+ model_ref = "EleutherAI/pythia-1.4b"
98
+ tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
99
+
100
+ loaded_hf_model = vllm_runner(
101
+ model_ref,
102
+ tensorizer_uri=tensorized_path,
103
+ load_format="tensorizer",
104
+ num_readers=1,
105
+ vllm_tensorized=False,
106
+ s3_endpoint="object.ord1.coreweave.com",
107
+ )
108
+
109
+ deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
110
+
111
+ assert deserialized_outputs
112
+
113
+
114
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
115
+def test_deserialized_encrypted_vllm_model_has_same_outputs(
116
+ vllm_runner, tmp_path):
117
+ vllm_model = vllm_runner(model_ref)
118
+ model_path = tmp_path / (model_ref + ".tensors")
119
+ key_path = tmp_path / (model_ref + ".key")
120
+ outputs = vllm_model.generate(prompts, sampling_params)
121
+ model = (vllm_model.model.llm_engine.model_executor.driver_worker.
122
+ model_runner.model)
123
+
124
+ encryption_params = EncryptionParams.random()
125
+ with open_stream(model_path, "wb+") as stream:
126
+ serializer = TensorSerializer(stream, encryption=encryption_params)
127
+ serializer.write_module(model)
128
+ with open_stream(key_path, "wb+") as stream:
129
+ stream.write(encryption_params.key)
130
+ del vllm_model, model
131
+ gc.collect()
132
+ torch.cuda.empty_cache()
133
+ loaded_vllm_model = vllm_runner(model_ref,
134
+ tensorizer_uri=model_path,
135
+ load_format="tensorizer",
136
+ encryption_keyfile=key_path,
137
+ num_readers=1,
138
+ vllm_tensorized=True)
139
+
140
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
141
+
142
+ # Assumes SamplingParams being seeded ensures the outputs are deterministic
143
+ assert outputs == deserialized_outputs
144
+
145
+
146
+def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
147
+ tmp_path):
148
+ hf_model = hf_runner(model_ref)
149
+ model_path = tmp_path / (model_ref + ".tensors")
150
+ max_tokens = 50
151
+ outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
152
+ with open_stream(model_path, "wb+") as stream:
153
+ serializer = TensorSerializer(stream)
154
+ serializer.write_module(hf_model.model)
155
+ del hf_model
156
+ gc.collect()
157
+ torch.cuda.empty_cache()
158
+ loaded_hf_model = vllm_runner(model_ref,
159
+ tensorizer_uri=model_path,
160
+ load_format="tensorizer",
161
+ num_readers=1,
162
+ vllm_tensorized=False)
163
+
164
+ deserialized_outputs = loaded_hf_model.generate_greedy(
165
+ prompts, max_tokens=max_tokens)
166
+
167
+ assert outputs == deserialized_outputs
168
+
169
+
170
+def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
171
+ from huggingface_hub import snapshot_download
172
+
173
+ from examples.multilora_inference import (create_test_prompts,
174
+ process_requests)
175
+
176
+ model_ref = "meta-llama/Llama-2-7b-hf"
177
+ lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
178
+ test_prompts = create_test_prompts(lora_path)
179
+
180
+ # Serialize model before deserializing and binding LoRA adapters
181
+ vllm_model = vllm_runner(model_ref, )
182
+ model_path = tmp_path / (model_ref + ".tensors")
183
+ model = (vllm_model.model.llm_engine.model_executor.driver_worker.
184
+ model_runner.model)
185
+ with open_stream(model_path, "wb+") as stream:
186
+ serializer = TensorSerializer(stream)
187
+ serializer.write_module(model)
188
+ del vllm_model, model
189
+ gc.collect()
190
+ torch.cuda.empty_cache()
191
+ loaded_vllm_model = vllm_runner(
192
+ model_ref,
193
+ tensorizer_uri=model_path,
194
+ load_format="tensorizer",
195
+ num_readers=1,
196
+ vllm_tensorized=True,
197
+ enable_lora=True,
198
+ max_loras=1,
199
+ max_lora_rank=8,
200
+ max_cpu_loras=2,
201
+ max_num_seqs=50,
202
+ max_model_len=1000,
203
+ )
204
+ process_requests(loaded_vllm_model.model.llm_engine, test_prompts)
205
+
206
+ assert loaded_vllm_model
207
+
208
+
209
+def test_load_without_tensorizer_load_format(vllm_runner):
210
+ with pytest.raises(ValueError):
211
+ vllm_runner(model_ref, tensorizer_uri="test")
212
+
213
+
214
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
215
+def test_tensorize_vllm_model(tmp_path):
216
+ # Test serialize command
217
+ serialize_args = [
218
+ "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
219
+ model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
220
+ tmp_path, "--suffix", "tests"
221
+ ]
222
+ result = subprocess.run(serialize_args, capture_output=True, text=True)
223
+ print(result.stdout) # Print the output of the serialize command
224
+
225
+ assert result.returncode == 0, (f"Serialize command failed with output:"
226
+ f"\n{result.stdout}\n{result.stderr}")
227
+
228
+ path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
229
+
230
+ # Test deserialize command
231
+ deserialize_args = [
232
+ "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
233
+ model_ref, "--dtype", "float16", "deserialize", "--path-to-tensors",
234
+ path_to_tensors
235
+ ]
236
+ result = subprocess.run(deserialize_args, capture_output=True, text=True)
237
+ assert result.returncode == 0, (f"Deserialize command failed with output:"
238
+ f"\n{result.stdout}\n{result.stderr}")
239
+
240
+
241
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
242
+def test_openai_apiserver_with_tensorizer(tmp_path):
243
+ ## Serialize model
244
+ serialize_args = [
245
+ "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
246
+ model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
247
+ tmp_path, "--suffix", "tests"
248
+ ]
249
+ result = subprocess.run(serialize_args, capture_output=True, text=True)
250
+ print(result.stdout) # Print the output of the serialize command
251
+
252
+ assert result.returncode == 0, (f"Serialize command failed with output:"
253
+ f"\n{result.stdout}\n{result.stderr}")
254
+
255
+ path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
256
+
257
+ ## Start OpenAI API server
258
+ openai_args = [
259
+ "--model", model_ref, "--dtype", "float16", "--load-format",
260
+ "tensorizer", "--tensorizer-uri", path_to_tensors, "--vllm-tensorized",
261
+ "--port", "8000"
262
+ ]
263
+
264
+ server = ServerRunner.remote(openai_args)
265
+
266
+ print("Server ready.")
267
+ assert server.ready.remote()
268
+
269
+
270
+def test_raise_value_error_on_invalid_load_format(vllm_runner):
271
+ with pytest.raises(ValueError):
272
+ vllm_runner(model_ref,
273
+ load_format="safetensors",
274
+ tensorizer_uri="test")
275
+
276
+
277
+def test_tensorizer_with_tp(vllm_runner):
278
+ with pytest.raises(ValueError):
279
+ model_ref = "EleutherAI/pythia-1.4b"
280
+ tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
281
+
282
+ vllm_runner(
283
+ model_ref,
284
+ tensorizer_uri=tensorized_path,
285
+ load_format="tensorizer",
286
+ num_readers=1,
287
+ vllm_tensorized=False,
288
+ s3_endpoint="object.ord1.coreweave.com",
289
+ tensor_parallel_size=2,
290
+ )
291
+
292
+
293
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
294
+def test_tensorizer_warn_quant(tmp_path):
295
+ model_ref = "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit"
296
+ serialize_args = [
297
+ "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
298
+ model_ref, "--quantization", "gptq", "--tensorizer-uri", "test",
299
+ "serialize", "--serialized-directory", tmp_path, "--suffix", "tests"
300
+ ]
301
+ result = subprocess.run(serialize_args, capture_output=True, text=True)
302
+ assert 'PerformanceWarning' in result.stderr
vllm/config.py
+72
-2
@@ -1,6 +1,8 @@
1
import enum
2
+import io
3
import json
4
import os
5
+import typing
6
from dataclasses import dataclass, fields
7
from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
8
@@ -16,6 +18,8 @@ from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
18
if TYPE_CHECKING:
19
from ray.util.placement_group import PlacementGroup
20
21
+ from vllm.model_executor.tensorizer_loader import TensorizerArgs
22
+
23
logger = init_logger(__name__)
24
25
_GB = 1 << 30
@@ -139,13 +143,14 @@ class ModelConfig:
143
def _verify_load_format(self) -> None:
144
load_format = self.load_format.lower()
145
supported_load_format = [
142
- "auto", "pt", "safetensors", "npcache", "dummy"
146
+ "auto", "pt", "safetensors", "npcache", "dummy", "tensorizer"
147
]
148
rocm_not_supported_load_format: List[str] = []
149
if load_format not in supported_load_format:
150
raise ValueError(
151
f"Unknown load format: {self.load_format}. Must be one of "
148
- "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
152
+ "'auto', 'pt', 'safetensors', 'npcache', 'tensorizer', or "
153
+ "'dummy'.")
154
if is_hip() and load_format in rocm_not_supported_load_format:
155
rocm_supported_load_format = [
156
f for f in supported_load_format
@@ -882,6 +887,65 @@ class VisionLanguageConfig:
887
f"{[x.name for x in cls.ImageInputType]}.") from e
888
889
890
+@dataclass
891
+class TensorizerConfig:
892
+ tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
893
+ str, bytes, os.PathLike, int]
894
+ vllm_tensorized: bool
895
+ verify_hash: Optional[bool] = False
896
+ num_readers: Optional[int] = 1
897
+ encryption_keyfile: Optional[str] = None
898
+ s3_access_key_id: Optional[str] = None
899
+ s3_secret_access_key: Optional[str] = None
900
+ s3_endpoint: Optional[str] = None
901
+ model_class: Optional[torch.nn.Module] = None
902
+ hf_config: Optional[PretrainedConfig] = None
903
+ dtype: Union[str, torch.dtype] = None
904
+
905
+ def _construct_tensorizer_args(self) -> "TensorizerArgs":
906
+ from vllm.model_executor.tensorizer_loader import TensorizerArgs
907
+ tensorizer_args = {
908
+ "tensorizer_uri": self.tensorizer_uri,
909
+ "vllm_tensorized": self.vllm_tensorized,
910
+ "verify_hash": self.verify_hash,
911
+ "num_readers": self.num_readers,
912
+ "encryption_keyfile": self.encryption_keyfile,
913
+ "s3_access_key_id": self.s3_access_key_id,
914
+ "s3_secret_access_key": self.s3_secret_access_key,
915
+ "s3_endpoint": self.s3_endpoint,
916
+ }
917
+ return TensorizerArgs(**tensorizer_args)
918
+
919
+ def verify_with_parallel_config(
920
+ self,
921
+ parallel_config: "ParallelConfig",
922
+ ) -> None:
923
+ if (parallel_config.tensor_parallel_size > 1
924
+ and self.tensorizer_uri is not None):
925
+ raise ValueError(
926
+ "Loading to multiple GPUs is not currently supported with "
927
+ "vLLM-serialized models. Please set tensor_parallel_size=1."
928
+ " or use a non-vLLM-serialized model, such as a "
929
+ "serialized Hugging Face `PretrainedModel`.")
930
+
931
+ def verify_with_model_config(self, model_config) -> None:
932
+ if (model_config.quantization is not None
933
+ and self.tensorizer_uri is not None):
934
+ from vllm.model_executor.tensorizer_loader import (
935
+ tensorizer_warning)
936
+ tensorizer_warning(
937
+ "Loading a model using Tensorizer with quantization on vLLM"
938
+ " is unstable and may lead to errors.")
939
+
940
+ if (model_config.load_format != "tensorizer"
941
+ and self.tensorizer_uri is not None):
942
+ raise ValueError(
943
+ "A tensorizer uri was passed for tensorizer loading, but the "
944
+ f"load format was set to {model_config.load_format}. "
945
+ "Please set the load format to 'tensorizer' to use "
946
+ f"tensorizer args.")
947
+
948
+
949
_STR_DTYPE_TO_TORCH_DTYPE = {
950
"half": torch.float16,
951
"float16": torch.float16,
@@ -1029,6 +1093,7 @@ class EngineConfig:
1093
lora_config: Optional[LoRAConfig]
1094
vision_language_config: Optional[VisionLanguageConfig]
1095
speculative_config: Optional[SpeculativeConfig]
1096
+ tensorizer_config: Optional[TensorizerConfig]
1097
1098
def __post_init__(self):
1099
"""Verify configs are valid & consistent with each other.
@@ -1036,6 +1101,11 @@ class EngineConfig:
1101
self.model_config.verify_with_parallel_config(self.parallel_config)
1102
self.cache_config.verify_with_parallel_config(self.parallel_config)
1103
1104
+ if self.tensorizer_config:
1105
+ self.tensorizer_config.verify_with_parallel_config(
1106
+ self.parallel_config)
1107
+ self.tensorizer_config.verify_with_model_config(self.model_config)
1108
+
1109
if self.lora_config:
1110
self.lora_config.verify_with_model_config(self.model_config)
1111
self.lora_config.verify_with_scheduler_config(
vllm/engine/arg_utils.py
+38
-7
@@ -1,12 +1,15 @@
1
import argparse
2
import dataclasses
3
+import io
4
+import os
5
from dataclasses import dataclass
4
-from typing import Optional
6
+from typing import BinaryIO, Optional, Union
7
8
from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
9
ModelConfig, ParallelConfig, SchedulerConfig,
8
- SpeculativeConfig, TokenizerPoolConfig,
9
- VisionLanguageConfig)
10
+ SpeculativeConfig, TensorizerConfig,
11
+ TokenizerPoolConfig, VisionLanguageConfig)
12
+from vllm.model_executor.tensorizer_loader import TensorizerArgs
13
from vllm.utils import str_to_int_tuple
14
15
@@ -58,12 +61,22 @@ class EngineArgs:
61
num_gpu_blocks_override: Optional[int] = None
62
num_lookahead_slots: int = 0
63
64
+ # Tensorizer configuration parameters
65
+ tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
66
+ bytes, os.PathLike, int] = None
67
+ vllm_tensorized: bool = False
68
+ verify_hash: Optional[bool] = False
69
+ num_readers: Optional[int] = 1
70
+ encryption_keyfile: Optional[str] = None
71
+ s3_access_key_id: Optional[str] = None
72
+ s3_secret_access_key: Optional[str] = None
73
+ s3_endpoint: Optional[str] = None
74
+
75
# Related to Vision-language models such as llava
76
image_input_type: Optional[str] = None
77
image_token_id: Optional[int] = None
78
image_input_shape: Optional[str] = None
79
image_feature_size: Optional[int] = None
66
-
80
scheduler_delay_factor: float = 0.0
81
enable_chunked_prefill: bool = False
82
@@ -135,7 +148,9 @@ class EngineArgs:
148
'--load-format',
149
type=str,
150
default=EngineArgs.load_format,
138
- choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'],
151
+ choices=[
152
+ 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
153
+ ],
154
help='The format of the model weights to load. '
155
'"auto" will try to load the weights in the safetensors format '
156
'and fall back to the pytorch bin format if safetensors format '
@@ -145,7 +160,10 @@ class EngineArgs:
160
'"npcache" will load the weights in pytorch format and store '
161
'a numpy cache to speed up the loading. '
162
'"dummy" will initialize the weights with random values, '
148
- 'which is mainly for profiling.')
163
+ 'which is mainly for profiling.'
164
+ '"tensorizer" will load the weights using tensorizer from CoreWeave'
165
+ 'which assumes tensorizer_uri is set to the location of the '
166
+ 'serialized weights.')
167
parser.add_argument(
168
'--dtype',
169
type=str,
@@ -403,6 +421,7 @@ class EngineArgs:
421
default=None,
422
help='The number of speculative tokens to sample from '
423
'the draft model in speculative decoding')
424
+ parser = TensorizerArgs.add_cli_args(parser)
425
return parser
426
427
@classmethod
@@ -465,6 +484,17 @@ class EngineArgs:
484
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
485
and self.max_cpu_loras > 0 else None) if self.enable_lora else None
486
487
+ tensorizer_config = TensorizerConfig(
488
+ tensorizer_uri=self.tensorizer_uri,
489
+ vllm_tensorized=self.vllm_tensorized,
490
+ verify_hash=self.verify_hash,
491
+ num_readers=self.num_readers,
492
+ encryption_keyfile=self.encryption_keyfile,
493
+ s3_access_key_id=self.s3_access_key_id,
494
+ s3_secret_access_key=self.s3_secret_access_key,
495
+ s3_endpoint=self.s3_endpoint,
496
+ )
497
+
498
if self.image_input_type:
499
if (not self.image_token_id or not self.image_input_shape
500
or not self.image_feature_size):
@@ -488,7 +518,8 @@ class EngineArgs:
518
device_config=device_config,
519
lora_config=lora_config,
520
vision_language_config=vision_language_config,
491
- speculative_config=speculative_config)
521
+ speculative_config=speculative_config,
522
+ tensorizer_config=tensorizer_config)
523
524
525
@dataclass
vllm/engine/llm_engine.py
+7
-1
@@ -6,7 +6,7 @@ from transformers import PreTrainedTokenizer
6
import vllm
7
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8
ParallelConfig, SchedulerConfig, SpeculativeConfig,
9
- VisionLanguageConfig)
9
+ TensorizerConfig, VisionLanguageConfig)
10
from vllm.core.scheduler import Scheduler, SchedulerOutputs
11
from vllm.engine.arg_utils import EngineArgs
12
from vllm.engine.metrics import StatLogger, Stats
@@ -74,6 +74,7 @@ class LLMEngine:
74
lora_config: Optional[LoRAConfig],
75
vision_language_config: Optional[VisionLanguageConfig],
76
speculative_config: Optional[SpeculativeConfig],
77
+ tensorizer_config: Optional[TensorizerConfig],
78
executor_class: Type[ExecutorBase],
79
log_stats: bool,
80
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -110,6 +111,7 @@ class LLMEngine:
111
self.scheduler_config = scheduler_config
112
self.device_config = device_config
113
self.speculative_config = speculative_config
114
+ self.tensorizer_config = tensorizer_config
115
self.log_stats = log_stats
116
117
self._init_tokenizer()
@@ -125,6 +127,7 @@ class LLMEngine:
127
lora_config=lora_config,
128
vision_language_config=vision_language_config,
129
speculative_config=speculative_config,
130
+ tensorizer_config=tensorizer_config,
131
)
132
133
self._initialize_kv_caches()
@@ -264,6 +267,9 @@ class LLMEngine:
267
def _verify_args(self) -> None:
268
self.model_config.verify_with_parallel_config(self.parallel_config)
269
self.cache_config.verify_with_parallel_config(self.parallel_config)
270
+ if self.tensorizer_config:
271
+ self.tensorizer_config.verify_with_parallel_config(
272
+ self.parallel_config)
273
if self.lora_config:
274
self.lora_config.verify_with_model_config(self.model_config)
275
self.lora_config.verify_with_scheduler_config(
vllm/executor/gpu_executor.py
+11
-12
@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Tuple
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
- VisionLanguageConfig)
5
+ TensorizerConfig, VisionLanguageConfig)
6
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
@@ -15,17 +15,14 @@ logger = init_logger(__name__)
15
16
class GPUExecutor(ExecutorBase):
17
18
- def __init__(
19
- self,
20
- model_config: ModelConfig,
21
- cache_config: CacheConfig,
22
- parallel_config: ParallelConfig,
23
- scheduler_config: SchedulerConfig,
24
- device_config: DeviceConfig,
25
- lora_config: Optional[LoRAConfig],
26
- vision_language_config: Optional[VisionLanguageConfig],
27
- speculative_config: Optional[SpeculativeConfig],
28
- ) -> None:
18
+ def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
19
+ parallel_config: ParallelConfig,
20
+ scheduler_config: SchedulerConfig,
21
+ device_config: DeviceConfig,
22
+ lora_config: Optional[LoRAConfig],
23
+ vision_language_config: Optional[VisionLanguageConfig],
24
+ speculative_config: Optional[SpeculativeConfig],
25
+ tensorizer_config: Optional[TensorizerConfig]) -> None:
26
self.model_config = model_config
27
self.cache_config = cache_config
28
self.lora_config = lora_config
@@ -33,6 +30,7 @@ class GPUExecutor(ExecutorBase):
30
self.scheduler_config = scheduler_config
31
self.device_config = device_config
32
self.vision_language_config = vision_language_config
33
+ self.tensorizer_config = tensorizer_config
34
35
assert (not speculative_config
36
), "Speculative decoding not yet supported for GPU backend"
@@ -61,6 +59,7 @@ class GPUExecutor(ExecutorBase):
59
distributed_init_method=distributed_init_method,
60
lora_config=self.lora_config,
61
vision_language_config=self.vision_language_config,
62
+ tensorizer_config=self.tensorizer_config,
63
is_driver_worker=True,
64
)
65
self.driver_worker.init_device()
vllm/executor/ray_gpu_executor.py
+5
-1
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
ParallelConfig, SchedulerConfig, SpeculativeConfig,
10
- VisionLanguageConfig)
10
+ TensorizerConfig, VisionLanguageConfig)
11
from vllm.engine.ray_utils import RayWorkerVllm, ray
12
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13
from vllm.logger import init_logger
@@ -42,6 +42,7 @@ class RayGPUExecutor(ExecutorBase):
42
lora_config: Optional[LoRAConfig],
43
vision_language_config: Optional[VisionLanguageConfig],
44
speculative_config: Optional[SpeculativeConfig],
45
+ tensorizer_config: Optional[TensorizerConfig],
46
) -> None:
47
self.model_config = model_config
48
self.cache_config = cache_config
@@ -50,6 +51,7 @@ class RayGPUExecutor(ExecutorBase):
51
self.scheduler_config = scheduler_config
52
self.device_config = device_config
53
self.vision_language_config = vision_language_config
54
+ self.tensorizer_config = tensorizer_config
55
assert (not speculative_config
56
), "Speculative decoding not yet supported for RayGPU backend."
57
@@ -171,6 +173,7 @@ class RayGPUExecutor(ExecutorBase):
173
distributed_init_method=distributed_init_method,
174
lora_config=lora_config,
175
vision_language_config=vision_language_config,
176
+ tensorizer_config=self.tensorizer_config,
177
))
178
179
# Initialize the driver worker with the Worker class.
@@ -187,6 +190,7 @@ class RayGPUExecutor(ExecutorBase):
190
distributed_init_method=distributed_init_method,
191
lora_config=self.lora_config,
192
vision_language_config=self.vision_language_config,
193
+ tensorizer_config=self.tensorizer_config,
194
is_driver_worker=True,
195
)
196
vllm/model_executor/model_loader.py
+43
-18
@@ -3,11 +3,14 @@ import contextlib
3
from typing import Tuple, Type
4
5
import torch
6
-import torch.nn as nn
6
+from torch import nn
7
8
from vllm.config import DeviceConfig, ModelConfig
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11
+from vllm.model_executor.tensorizer_loader import (
12
+ ParameterizedLoadFormat, is_vllm_serialized_tensorizer,
13
+ load_with_tensorizer)
14
from vllm.model_executor.weight_utils import (get_quant_config,
15
initialize_dummy_weights)
16
@@ -51,6 +54,7 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
54
**kwargs) -> nn.Module:
55
lora_config = kwargs.get("lora_config", None)
56
vision_language_config = kwargs.get("vision_language_config", None)
57
+ tensorizer_config = kwargs.get("tensorizer_config", None)
58
model_class = _get_model_architecture(model_config)[0]
59
60
# Get the (maybe quantized) linear method.
@@ -71,33 +75,54 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
75
f"{model_config.dtype} is not supported for quantization "
76
f"method {model_config.quantization}. Supported dtypes: "
77
f"{supported_dtypes}")
78
+
79
linear_method = quant_config.get_linear_method()
80
81
with _set_default_torch_dtype(model_config.dtype):
82
# Create a model instance.
83
# The weights will be initialized as empty tensors.
84
+ extra_kwargs = {}
85
+ if hasattr(model_class, "supported_lora_modules"):
86
+ extra_kwargs["lora_config"] = lora_config
87
+ elif lora_config:
88
+ raise ValueError(
89
+ f"Model {model_class.__name__} does not support LoRA, "
90
+ "but LoRA is enabled. Support for this model may "
91
+ "be added in the future. If this is important to you, "
92
+ "please open an issue on github.")
93
+ elif model_class in _VISION_MODEL_CLASSES:
94
+ extra_kwargs["vision_language_config"] = vision_language_config
95
+
96
with torch.device(device_config.device):
80
- if hasattr(model_class, "supported_lora_modules"):
81
- model = model_class(model_config.hf_config, linear_method,
82
- lora_config)
83
- elif lora_config:
84
- raise ValueError(
85
- f"Model {model_class.__name__} does not support LoRA, "
86
- "but LoRA is enabled. Support for this model may "
87
- "be added in the future. If this is important to you, "
88
- "please open an issue on github.")
89
- else:
90
- if model_class not in _VISION_MODEL_CLASSES:
91
- model = model_class(model_config.hf_config, linear_method)
92
- else:
93
- model = model_class(model_config.hf_config,
94
- vision_language_config, linear_method)
97
+ if (model_config.load_format == "tensorizer"
98
+ and is_vllm_serialized_tensorizer(tensorizer_config)):
99
+ extra_kwargs["linear_method"] = linear_method
100
+ tensorizer_config.model_class = model_class
101
+ tensorizer_config.hf_config = model_config.hf_config
102
+ tensorizer_config.dtype = model_config.dtype
103
+ model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
104
+ return model.eval()
105
+ model = model_class(config=model_config.hf_config,
106
+ linear_method=linear_method,
107
+ **extra_kwargs)
108
if model_config.load_format == "dummy":
109
# NOTE(woosuk): For accurate performance evaluation, we assign
110
# random values to the weights.
111
initialize_dummy_weights(model)
112
else:
113
# Load the weights from the cached or downloaded files.
101
- model.load_weights(model_config.model, model_config.download_dir,
102
- model_config.load_format, model_config.revision)
114
+ if model_config.load_format == "tensorizer":
115
+ # Provide a dynamic load format for `model.load_weights`
116
+ # to retain tensorizer args from CLI.
117
+ model_config.load_format = ParameterizedLoadFormat(
118
+ model_config.load_format)
119
+ model_config.load_format.params = (
120
+ tensorizer_config._construct_tensorizer_args())
121
+
122
+ model.load_weights(
123
+ model_config.model,
124
+ model_config.download_dir,
125
+ model_config.load_format,
126
+ model_config.revision,
127
+ )
128
return model.eval()
vllm/model_executor/tensorizer_loader.py
new
+319
@@ -0,0 +1,319 @@
1
+import argparse
2
+import dataclasses
3
+import io
4
+import os
5
+import time
6
+import typing
7
+import warnings
8
+from dataclasses import dataclass
9
+from typing import Optional, Union
10
+
11
+import torch
12
+from torch import nn
13
+
14
+from vllm.config import TensorizerConfig
15
+from vllm.logger import init_logger
16
+from vllm.model_executor.layers.linear import LinearMethodBase
17
+from vllm.model_executor.layers.vocab_parallel_embedding import (
18
+ VocabParallelEmbedding)
19
+
20
+tensorizer_load_fail = False
21
+
22
+try:
23
+ from tensorizer import (DecryptionParams, EncryptionParams,
24
+ TensorDeserializer, TensorSerializer)
25
+ from tensorizer.stream_io import open_stream
26
+ from tensorizer.utils import (convert_bytes, get_mem_usage,
27
+ no_init_or_tensor)
28
+except ImportError:
29
+ tensorizer_load_fail = True
30
+
31
+__all__ = [
32
+ 'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
33
+ 'TensorSerializer', 'open_stream', 'convert_bytes', 'get_mem_usage',
34
+ 'no_init_or_tensor'
35
+]
36
+
37
+logger = init_logger(__name__)
38
+
39
+
40
+def load_with_tensorizer(tensorizer_config: TensorizerConfig,
41
+ **extra_kwargs) -> nn.Module:
42
+ tensorizer = TensorizerAgent(tensorizer_config, **extra_kwargs)
43
+ return tensorizer.deserialize()
44
+
45
+
46
+def tensorizer_warning(message: str):
47
+ return warnings.warn(message, category=PerformanceWarning, stacklevel=2)
48
+
49
+
50
+def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
51
+ if tensorizer_config is None:
52
+ return False
53
+ return tensorizer_config.vllm_tensorized
54
+
55
+
56
+class ParameterizedLoadFormat(str):
57
+ __slots__ = "params"
58
+
59
+
60
+class PerformanceWarning(UserWarning):
61
+
62
+ def __str__(self):
63
+ return (f"{super().__str__()}"
64
+ " (set the VLLM_SILENCE_PERFORMANCE_WARNINGS"
65
+ " environment variable to hide this)")
66
+
67
+
68
+if (os.getenv("VLLM_SILENCE_PERFORMANCE_WARNINGS", "").lower()
69
+ not in ("", "0", "n", "no", "off", "disable")):
70
+ warnings.simplefilter("ignore", category=PerformanceWarning)
71
+
72
+
73
+@dataclass
74
+class TensorizerArgs:
75
+ tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
76
+ str, bytes, os.PathLike, int]
77
+ vllm_tensorized: bool
78
+ verify_hash: Optional[bool] = False
79
+ num_readers: Optional[int] = 1
80
+ encryption_keyfile: Optional[str] = None
81
+ s3_access_key_id: Optional[str] = None
82
+ s3_secret_access_key: Optional[str] = None
83
+ s3_endpoint: Optional[str] = None
84
+ """
85
+ Args for the TensorizerAgent class. These are used to configure the behavior
86
+ of the TensorDeserializer when loading tensors from a serialized model.
87
+
88
+ Args:
89
+ tensorizer_uri: Path to serialized model tensors. Can be a local file
90
+ path or a S3 URI.
91
+ vllm_tensorized: If True, indicates that the serialized model is a
92
+ vLLM model. This is used to determine the behavior of the
93
+ TensorDeserializer when loading tensors from a serialized model.
94
+ It is far faster to deserialize a vLLM model as it utilizes
95
+ tensorizer's optimized GPU loading.
96
+ verify_hash: If True, the hashes of each tensor will be verified against
97
+ the hashes stored in the metadata. A `HashMismatchError` will be
98
+ raised if any of the hashes do not match.
99
+ num_readers: Controls how many threads are allowed to read concurrently
100
+ from the source file. Default is 1. This greatly increases
101
+ performance.
102
+ encryption_keyfile: File path to a binary file containing a
103
+ binary key to use for decryption. `None` (the default) means
104
+ no decryption. See the example script in
105
+ examples/tensorize_vllm_model.py.
106
+ s3_access_key_id: The access key for the S3 bucket. Can also be set via
107
+ the S3_ACCESS_KEY_ID environment variable.
108
+ s3_secret_access_key: The secret access key for the S3 bucket. Can also
109
+ be set via the S3_SECRET_ACCESS_KEY environment variable.
110
+ s3_endpoint: The endpoint for the S3 bucket. Can also be set via the
111
+ S3_ENDPOINT_URL environment variable.
112
+ """
113
+
114
+ def __post_init__(self):
115
+ self.file_obj = self.tensorizer_uri
116
+ self.s3_access_key_id = (self.s3_access_key_id
117
+ or os.environ.get("S3_ACCESS_KEY_ID")) or None
118
+ self.s3_secret_access_key = (
119
+ self.s3_secret_access_key
120
+ or os.environ.get("S3_SECRET_ACCESS_KEY")) or None
121
+ self.s3_endpoint = (self.s3_endpoint
122
+ or os.environ.get("S3_ENDPOINT_URL")) or None
123
+ self.stream_params = {
124
+ "s3_access_key_id": self.s3_access_key_id,
125
+ "s3_secret_access_key": self.s3_secret_access_key,
126
+ "s3_endpoint": self.s3_endpoint,
127
+ }
128
+
129
+ # Omitting self.dtype and self.device as this behaves weirdly
130
+ self.deserializer_params = {
131
+ "verify_hash": self.verify_hash,
132
+ "encryption": self.encryption_keyfile,
133
+ "num_readers": self.num_readers
134
+ }
135
+ if self.encryption_keyfile:
136
+ with open_stream(
137
+ self.encryption_keyfile,
138
+ **self.stream_params,
139
+ ) as stream:
140
+ key = stream.read()
141
+ decryption_params = DecryptionParams.from_key(key)
142
+ self.deserializer_params['encryption'] = decryption_params
143
+
144
+ def add_cli_args(
145
+ parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
146
+ """Tensorizer CLI arguments"""
147
+
148
+ # Create the argument group
149
+ group = parser.add_argument_group(
150
+ 'tensorizer options',
151
+ description=('Options for configuring the behavior of the'
152
+ ' tensorizer deserializer when '
153
+ '--load-format=tensorizer'))
154
+
155
+ group.add_argument(
156
+ "--tensorizer-uri",
157
+ help="Path to serialized model tensors. Can be a local file path,"
158
+ " or an HTTP(S) or S3 URI.",
159
+ )
160
+ group.add_argument(
161
+ "--verify-hash",
162
+ action="store_true",
163
+ help="If enabled, the hashes of each tensor will be verified"
164
+ " against the hashes stored in the file metadata. An exception"
165
+ " will be raised if any of the hashes do not match.",
166
+ )
167
+ group.add_argument(
168
+ "--encryption-keyfile",
169
+ default=None,
170
+ help="The file path to a binary file containing a binary key to "
171
+ "use for decryption. Can be a file path or S3 network URI.")
172
+ group.add_argument(
173
+ "--num-readers",
174
+ default=1,
175
+ type=int,
176
+ help="Controls how many threads are allowed to read concurrently "
177
+ "from the source file.")
178
+ group.add_argument(
179
+ "--s3-access-key-id",
180
+ default=None,
181
+ help="The access key for the S3 bucket. Can also be set via the "
182
+ "S3_ACCESS_KEY_ID environment variable.",
183
+ )
184
+ group.add_argument(
185
+ "--s3-secret-access-key",
186
+ default=None,
187
+ help="The secret access key for the S3 bucket. Can also be set via "
188
+ "the S3_SECRET_ACCESS_KEY environment variable.",
189
+ )
190
+ group.add_argument(
191
+ "--s3-endpoint",
192
+ default=None,
193
+ help="The endpoint for the S3 bucket. Can also be set via the "
194
+ "S3_ENDPOINT_URL environment variable.",
195
+ )
196
+ group.add_argument(
197
+ "--vllm-tensorized",
198
+ action="store_true",
199
+ help="If enabled, indicates that the serialized model is a vLLM "
200
+ "model. This is used to determine the behavior of the "
201
+ "TensorDeserializer when loading tensors from a "
202
+ "serialized model.")
203
+
204
+ return parser
205
+
206
+ @classmethod
207
+ def from_cli_args(cls, args: argparse.Namespace) -> "TensorizerArgs":
208
+ # Get the list of attributes of this dataclass.
209
+ attrs = [attr.name for attr in dataclasses.fields(cls)]
210
+ # Set the attributes from the parsed arguments.
211
+ tensorizer_args = cls(**{
212
+ attr: getattr(args, attr)
213
+ for attr in attrs if hasattr(args, attr)
214
+ })
215
+ return tensorizer_args
216
+
217
+
218
+class TensorizerAgent:
219
+ """
220
+ A class for performing tensorizer deserializations specifically for
221
+ vLLM models using plaid_mode. Uses TensorizerArgs to configure the
222
+ behavior of the TensorDeserializer when loading tensors from a serialized
223
+ model. For deserializations of HuggingFace models, TensorDeserializer is
224
+ instead used as an iterator directly in the func hf_model_weights_iterator
225
+ in vllm/model_executor/weight_utils.py
226
+ """
227
+
228
+ def __init__(self, tensorizer_config: TensorizerConfig,
229
+ linear_method: LinearMethodBase, **extra_kwargs):
230
+ self.tensorizer_config = tensorizer_config
231
+ self.tensorizer_args = (
232
+ self.tensorizer_config._construct_tensorizer_args())
233
+ self.extra_kwargs = extra_kwargs
234
+ if extra_kwargs.get("linear_method", None) is not None:
235
+ self.linear_method = extra_kwargs["linear_method"]
236
+ else:
237
+ self.linear_method = linear_method
238
+ self.model = self._init_model()
239
+
240
+ if tensorizer_load_fail:
241
+ raise ImportError(
242
+ "Tensorizer is not installed. Please install tensorizer "
243
+ "to use this feature with `pip install vllm[tensorizer]`.")
244
+
245
+ def _init_model(self):
246
+ model_args = self.tensorizer_config.hf_config
247
+ model_args.torch_dtype = self.tensorizer_config.dtype
248
+ with no_init_or_tensor():
249
+ return self.tensorizer_config.model_class(
250
+ config=model_args,
251
+ linear_method=self.linear_method,
252
+ **self.extra_kwargs)
253
+
254
+ def _resize_lora_embeddings(self):
255
+ """Modify LoRA embedding layers to use bigger tensors
256
+ to allow for adapter added tokens."""
257
+ for child in self.model.modules():
258
+ if (isinstance(child, VocabParallelEmbedding)
259
+ and child.weight.shape[0] <
260
+ child.num_embeddings_per_partition):
261
+ new_weight = torch.empty(child.num_embeddings_per_partition,
262
+ child.embedding_dim,
263
+ dtype=child.weight.dtype,
264
+ device=child.weight.device)
265
+ new_weight[:child.weight.shape[0]].copy_(child.weight.data)
266
+ new_weight[child.weight.shape[0]:].fill_(0)
267
+ child.weight.data = new_weight
268
+
269
+ def _check_tensors_on_meta_device(self):
270
+ for tensor in self.model.state_dict().values():
271
+ if tensor.device.type == 'meta':
272
+ raise ValueError(
273
+ "The serialized model contains tensors on the meta device,"
274
+ " indicating that some tensors were not loaded properly."
275
+ " Please check that the parameters of the model being"
276
+ " specified match that of the serialized model, such as"
277
+ " its quantization.")
278
+
279
+ def deserialize(self):
280
+ """
281
+ Deserialize the model using the TensorDeserializer. This method is
282
+ specifically for vLLM models using tensorizer's plaid_mode.
283
+
284
+ The deserializer makes use of tensorizer_args.stream_params
285
+ to configure the behavior of the stream when loading tensors from a
286
+ serialized model. The deserializer_params are used to configure the
287
+ behavior of the TensorDeserializer when loading tensors themselves.
288
+ Documentation on these params can be found in TensorizerArgs
289
+
290
+ Returns:
291
+ nn.Module: The deserialized model.
292
+ """
293
+ before_mem = get_mem_usage()
294
+ # Lazy load the tensors from S3 into the model.
295
+ start = time.perf_counter()
296
+ with open_stream(
297
+ self.tensorizer_args.tensorizer_uri,
298
+ mode="rb",
299
+ **self.tensorizer_args.stream_params,
300
+ ) as stream, TensorDeserializer(
301
+ stream,
302
+ dtype=self.tensorizer_config.dtype,
303
+ **self.tensorizer_args.deserializer_params) as deserializer:
304
+ deserializer.load_into_module(self.model)
305
+ end = time.perf_counter()
306
+
307
+ total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
308
+ duration = end - start
309
+ per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
310
+ after_mem = get_mem_usage()
311
+ deserializer.close()
312
+ logger.info(f"Deserialized {total_bytes_str} in "
313
+ f"{end - start:0.2f}s, {per_second}/s")
314
+ logger.info(f"Memory usage before: {before_mem}")
315
+ logger.info(f"Memory usage after: {after_mem}")
316
+
317
+ self._check_tensors_on_meta_device()
318
+ self._resize_lora_embeddings()
319
+ return self.model.eval()
vllm/model_executor/weight_utils.py
+30
-4
@@ -5,7 +5,7 @@ import hashlib
5
import json
6
import os
7
from collections import defaultdict
8
-from typing import Any, Iterable, Iterator, List, Optional, Tuple
8
+from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union
9
10
import filelock
11
import huggingface_hub.constants
@@ -161,7 +161,8 @@ def prepare_hf_model_weights(
161
revision: Optional[str] = None,
162
) -> Tuple[str, List[str], bool]:
163
# Download model weights from huggingface.
164
- is_local = os.path.isdir(model_name_or_path)
164
+ is_local = os.path.isdir(model_name_or_path) \
165
+ and load_format != "tensorizer"
166
use_safetensors = False
167
# Some quantized models use .pt files for storing the weights.
168
if load_format == "auto":
@@ -173,13 +174,15 @@ def prepare_hf_model_weights(
174
allow_patterns = ["*.pt"]
175
elif load_format == "npcache":
176
allow_patterns = ["*.bin"]
177
+ elif load_format == "tensorizer":
178
+ allow_patterns = ["*.tensors"]
179
else:
180
raise ValueError(f"Unknown load_format: {load_format}")
181
182
if fall_back_to_pt:
183
allow_patterns += ["*.pt"]
184
182
- if not is_local:
185
+ if not is_local and load_format != "tensorizer":
186
# Before we download we look at that is available:
187
fs = HfFileSystem()
188
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
@@ -224,6 +227,9 @@ def prepare_hf_model_weights(
227
if not any(f.endswith(x) for x in blacklist)
228
]
229
230
+ if load_format == "tensorizer":
231
+ return hf_folder, hf_weights_files, use_safetensors
232
+
233
if len(hf_weights_files) == 0:
234
raise RuntimeError(
235
f"Cannot find any model weights with `{model_name_or_path}`")
@@ -234,7 +240,7 @@ def prepare_hf_model_weights(
240
def hf_model_weights_iterator(
241
model_name_or_path: str,
242
cache_dir: Optional[str] = None,
237
- load_format: str = "auto",
243
+ load_format: Union[Tuple, str] = "auto",
244
revision: Optional[str] = None,
245
fall_back_to_pt: Optional[bool] = True,
246
) -> Iterator[Tuple[str, torch.Tensor]]:
@@ -277,6 +283,26 @@ def hf_model_weights_iterator(
283
with open(param_path, "rb") as f:
284
param = np.load(f)
285
yield name, torch.from_numpy(param)
286
+ elif load_format == "tensorizer":
287
+ from vllm.model_executor.tensorizer_loader import (TensorDeserializer,
288
+ open_stream,
289
+ tensorizer_warning)
290
+ tensorizer_args = load_format.params
291
+ tensorizer_warning(
292
+ "Deserializing HuggingFace models is not optimized for "
293
+ "loading on vLLM, as tensorizer is forced to load to CPU. "
294
+ "Consider deserializing a vLLM model instead for faster "
295
+ "load times. See the examples/tensorize_vllm_model.py example "
296
+ "script for serializing vLLM models.")
297
+
298
+ deserializer_args = tensorizer_args.deserializer_params
299
+ stream_params = tensorizer_args.stream_params
300
+ stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
301
+ with TensorDeserializer(stream, **deserializer_args,
302
+ device="cpu") as state:
303
+ for name, param in state.items():
304
+ yield name, param
305
+ del state
306
elif use_safetensors:
307
for st_file in hf_weights_files:
308
with safe_open(st_file, framework="pt") as f:
vllm/worker/model_runner.py
+7
-2
@@ -10,7 +10,8 @@ import torch.nn as nn
10
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
get_attn_backend)
12
from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13
- SchedulerConfig, VisionLanguageConfig)
13
+ SchedulerConfig, TensorizerConfig,
14
+ VisionLanguageConfig)
15
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
16
from vllm.distributed.device_communicators import (custom_all_reduce,
17
pynccl_utils)
@@ -111,11 +112,13 @@ class ModelRunner:
112
kv_cache_dtype: Optional[str] = "auto",
113
is_driver_worker: bool = False,
114
vision_language_config: Optional[VisionLanguageConfig] = None,
115
+ tensorizer_config: Optional[TensorizerConfig] = None,
116
):
117
self.model_config = model_config
118
self.parallel_config = parallel_config
119
self.scheduler_config = scheduler_config
120
self.lora_config = lora_config
121
+ self.tensorizer_config = tensorizer_config
122
self.is_driver_worker = is_driver_worker
123
124
# model_config can be None in tests/samplers/test_sampler.py.
@@ -158,7 +161,9 @@ class ModelRunner:
161
lora_config=self.lora_config,
162
vision_language_config=self.vision_language_config,
163
parallel_config=self.parallel_config,
161
- scheduler_config=self.scheduler_config)
164
+ scheduler_config=self.scheduler_config,
165
+ tensorizer_config=self.tensorizer_config,
166
+ )
167
168
self.model_memory_usage = m.consumed_memory
169
logger.info(f"Loading model weights took "
vllm/worker/worker.py
+7
-2
@@ -7,7 +7,8 @@ import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
10
+ ParallelConfig, SchedulerConfig, TensorizerConfig,
11
+ VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
init_distributed_environment)
@@ -42,6 +43,7 @@ class Worker(WorkerBase):
43
distributed_init_method: str,
44
lora_config: Optional[LoRAConfig] = None,
45
vision_language_config: Optional[VisionLanguageConfig] = None,
46
+ tensorizer_config: Optional[TensorizerConfig] = None,
47
is_driver_worker: bool = False,
48
) -> None:
49
self.model_config = model_config
@@ -53,6 +55,7 @@ class Worker(WorkerBase):
55
self.rank = rank
56
self.distributed_init_method = distributed_init_method
57
self.lora_config = lora_config
58
+ self.tensorizer_config = tensorizer_config
59
self.is_driver_worker = is_driver_worker
60
if self.is_driver_worker:
61
assert self.rank == 0, "The driver worker must have rank 0."
@@ -70,7 +73,9 @@ class Worker(WorkerBase):
73
lora_config=self.lora_config,
74
kv_cache_dtype=self.cache_config.cache_dtype,
75
is_driver_worker=is_driver_worker,
73
- vision_language_config=vision_language_config)
76
+ vision_language_config=vision_language_config,
77
+ tensorizer_config=tensorizer_config,
78
+ )
79
# Uninitialized cache engine. Will be initialized by
80
# initialize_cache.
81
self.cache_engine = None