A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] [Core] perf: Automatically detect vLLM-tensorized model, update `tensorizer` to version 2.9.0 (#4208)
Sanger Steel committed
May 13, 2024 at 17:57 UTC
8bc68e198c4c90ddc2e54fa76eb81c2c714bb1cd
10 files changed
+259
-523
.buildkite/test-pipeline.yaml
+3
-1
@@ -60,11 +60,13 @@ steps:
60
mirror_hardwares: [amd]
61
commands:
62
# install aws cli for llava_example.py
63
- - pip install awscli
63
+ # install tensorizer for tensorize_vllm_model.py
64
+ - pip install awscli tensorizer
65
- python3 offline_inference.py
66
- python3 offline_inference_with_prefix.py
67
- python3 llm_engine_example.py
68
- python3 llava_example.py
69
+ - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
70
71
- label: Kernels Test %N
72
command: pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
examples/tensorize_vllm_model.py
+81
-119
@@ -1,23 +1,20 @@
1
import argparse
2
import dataclasses
3
+import json
4
import os
4
-import time
5
import uuid
6
from functools import partial
7
-from typing import Type
7
9
-import torch
10
-import torch.nn as nn
11
-from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12
- TensorSerializer, stream_io)
13
-from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14
-from transformers import AutoConfig, PretrainedConfig
8
+from tensorizer import stream_io
9
16
-from vllm.distributed import initialize_model_parallel
10
+from vllm import LLM
11
+from vllm.distributed import (init_distributed_environment,
12
+ initialize_model_parallel)
13
from vllm.engine.arg_utils import EngineArgs
14
from vllm.engine.llm_engine import LLMEngine
19
-from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20
-from vllm.model_executor.models import ModelRegistry
15
+from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
16
+ TensorizerConfig,
17
+ serialize_vllm_model)
18
19
# yapf conflicts with isort for this docstring
20
# yapf: disable
@@ -27,25 +24,25 @@ deserialize vLLM models. These models can be loaded using tensorizer
24
to the GPU extremely quickly over an HTTP/HTTPS endpoint, an S3 endpoint,
25
or locally. Tensor encryption and decryption is also supported, although
26
libsodium must be installed to use it. Install vllm with tensorizer support
30
-using `pip install vllm[tensorizer]`.
27
+using `pip install vllm[tensorizer]`. To learn more about tensorizer, visit
28
+https://github.com/coreweave/tensorizer
29
30
To serialize a model, install vLLM from source, then run something
31
like this from the root level of this repository:
32
33
python -m examples.tensorize_vllm_model \
36
- --model EleutherAI/gpt-j-6B \
37
- --dtype float16 \
34
+ --model facebook/opt-125m \
35
serialize \
39
- --serialized-directory s3://my-bucket/ \
40
- --suffix vllm
36
+ --serialized-directory s3://my-bucket \
37
+ --suffix v1
38
39
Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
40
and saves it to your S3 bucket. A local directory can also be used. This
41
assumes your S3 credentials are specified as environment variables
45
-in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
46
-To provide S3 credentials directly, you can provide `--s3-access-key-id` and
47
-`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this
48
-script.
42
+in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and
43
+`S3_ENDPOINT_URL`. To provide S3 credentials directly, you can provide
44
+`--s3-access-key-id` and `--s3-secret-access-key`, as well as `--s3-endpoint`
45
+as CLI args to this script.
46
47
You can also encrypt the model weights with a randomly-generated key by
48
providing a `--keyfile` argument.
@@ -57,7 +54,7 @@ python -m examples.tensorize_vllm_model \
54
--model EleutherAI/gpt-j-6B \
55
--dtype float16 \
56
deserialize \
60
- --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
57
+ --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors
58
59
Which downloads the model tensors from your S3 bucket and deserializes them.
60
@@ -71,26 +68,30 @@ Or for deserializing:
68
69
`python -m examples.tensorize_vllm_model deserialize --help`.
70
74
-Once a model is serialized, it can be used to load the model when running the
75
-OpenAI inference client at `vllm/entrypoints/openai/api_server.py` by providing
76
-the `--tensorizer-uri` CLI argument that is functionally the same as the
77
-`--path-to-tensors` argument in this script, along with `--vllm-tensorized`, to
78
-signify that the model to be deserialized is a vLLM model, rather than a
79
-HuggingFace `PreTrainedModel`, which can also be deserialized using tensorizer
80
-in the same inference server, albeit without the speed optimizations. To
81
-deserialize an encrypted file, the `--encryption-keyfile` argument can be used
82
-to provide the path to the keyfile used to encrypt the model weights. For
83
-information on all the arguments that can be used to configure tensorizer's
84
-deserialization, check out the tensorizer options argument group in the
85
-`vllm/entrypoints/openai/api_server.py` script with `--help`.
86
-
87
-Tensorizer can also be invoked with the `LLM` class directly to load models:
71
+Once a model is serialized, tensorizer can be invoked with the `LLM` class
72
+directly to load models:
73
74
llm = LLM(model="facebook/opt-125m",
75
load_format="tensorizer",
91
- tensorizer_uri=path_to_opt_tensors,
92
- num_readers=3,
93
- vllm_tensorized=True)
76
+ model_loader_extra_config=TensorizerConfig(
77
+ tensorizer_uri = path_to_tensors,
78
+ num_readers=3,
79
+ )
80
+ )
81
+
82
+A serialized model can be used during model loading for the vLLM OpenAI
83
+inference server. `model_loader_extra_config` is exposed as the CLI arg
84
+`--model-loader-extra-config`, and accepts a JSON string literal of the
85
+TensorizerConfig arguments desired.
86
+
87
+In order to see all of the available arguments usable to configure
88
+loading with tensorizer that are given to `TensorizerConfig`, run:
89
+
90
+`python -m examples.tensorize_vllm_model deserialize --help`
91
+
92
+under the `tensorizer options` section. These can also be used for
93
+deserialization in this example script, although `--tensorizer-uri` and
94
+`--path-to-tensors` are functionally the same in this case.
95
"""
96
97
@@ -158,95 +159,35 @@ def parse_args():
159
help=("Path to a binary key to use to decrypt the model weights,"
160
" if the model was serialized with encryption"))
161
161
- return parser.parse_args()
162
-
163
-
164
-def make_model_contiguous(model):
165
- # Ensure tensors are saved in memory contiguously
166
- for param in model.parameters():
167
- param.data = param.data.contiguous()
168
-
169
-
170
-def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
171
- architectures = getattr(config, "architectures", [])
172
- for arch in architectures:
173
- model_cls = ModelRegistry.load_model_cls(arch)
174
- if model_cls is not None:
175
- return model_cls
176
- raise ValueError(
177
- f"Model architectures {architectures} are not supported for now. "
178
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
179
-
180
-
181
-def serialize():
182
-
183
- eng_args_dict = {f.name: getattr(args, f.name) for f in
184
- dataclasses.fields(EngineArgs)}
185
- engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
186
- engine = LLMEngine.from_engine_args(engine_args)
162
+ TensorizerArgs.add_cli_args(deserialize_parser)
163
188
- model = (engine.model_executor.driver_worker.
189
- model_runner.model)
190
-
191
- encryption_params = EncryptionParams.random() if keyfile else None
192
- if keyfile:
193
- with _write_stream(keyfile) as stream:
194
- stream.write(encryption_params.key)
195
-
196
- with _write_stream(model_path) as stream:
197
- serializer = TensorSerializer(stream, encryption=encryption_params)
198
- serializer.write_module(model)
199
- serializer.close()
164
+ return parser.parse_args()
165
201
- print("Serialization complete. Model tensors saved to", model_path)
202
- if keyfile:
203
- print("Key saved to", keyfile)
166
167
168
def deserialize():
207
- config = AutoConfig.from_pretrained(model_ref)
208
-
209
- with no_init_or_tensor():
210
- model_class = _get_vllm_model_architecture(config)
211
- model = model_class(config)
212
-
213
- before_mem = get_mem_usage()
214
- start = time.time()
215
-
216
- if keyfile:
217
- with _read_stream(keyfile) as stream:
218
- key = stream.read()
219
- decryption_params = DecryptionParams.from_key(key)
220
- tensorizer_args.deserializer_params['encryption'] = \
221
- decryption_params
222
-
223
- with (_read_stream(model_path)) as stream, TensorDeserializer(
224
- stream, **tensorizer_args.deserializer_params) as deserializer:
225
- deserializer.load_into_module(model)
226
- end = time.time()
227
-
228
- # Brag about how fast we are.
229
- total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
230
- duration = end - start
231
- per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
232
- after_mem = get_mem_usage()
233
- print(
234
- f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
169
+ llm = LLM(model=args.model,
170
+ load_format="tensorizer",
171
+ model_loader_extra_config=tensorizer_config
172
)
236
- print(f"Memory usage before: {before_mem}")
237
- print(f"Memory usage after: {after_mem}")
173
+ return llm
174
239
- return model
175
176
177
args = parse_args()
178
244
-s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
245
- or None)
246
-s3_secret_access_key = (args.s3_secret_access_key
247
- or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
179
+s3_access_key_id = (getattr(args, 's3_access_key_id', None)
180
+ or os.environ.get("S3_ACCESS_KEY_ID", None))
181
+s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
182
+ or os.environ.get("S3_SECRET_ACCESS_KEY", None))
183
+s3_endpoint = (getattr(args, 's3_endpoint', None)
184
+ or os.environ.get("S3_ENDPOINT_URL", None))
185
249
-s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
186
+credentials = {
187
+ "s3_access_key_id": s3_access_key_id,
188
+ "s3_secret_access_key": s3_secret_access_key,
189
+ "s3_endpoint": s3_endpoint
190
+}
191
192
_read_stream, _write_stream = (partial(
193
stream_io.open_stream,
@@ -263,20 +204,41 @@ model_name = model_ref.split("/")[1]
204
os.environ["MASTER_ADDR"] = "127.0.0.1"
205
os.environ["MASTER_PORT"] = "8080"
206
266
-torch.distributed.init_process_group(world_size=1, rank=0)
207
+init_distributed_environment(world_size=1, rank=0, local_rank=0)
208
initialize_model_parallel()
209
210
keyfile = args.keyfile if args.keyfile else None
211
212
+
213
+if args.model_loader_extra_config:
214
+ config = json.loads(args.model_loader_extra_config)
215
+ tensorizer_args = TensorizerConfig(**config)._construct_tensorizer_args()
216
+ tensorizer_args.tensorizer_uri = args.path_to_tensors
217
+else:
218
+ tensorizer_args = None
219
+
220
if args.command == "serialize":
221
+ eng_args_dict = {f.name: getattr(args, f.name) for f in
222
+ dataclasses.fields(EngineArgs)}
223
+
224
+ engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
225
+ engine = LLMEngine.from_engine_args(engine_args)
226
+
227
input_dir = args.serialized_directory.rstrip('/')
228
suffix = args.suffix if args.suffix else uuid.uuid4().hex
229
base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
230
model_path = f"{base_path}/model.tensors"
276
- serialize()
231
+ tensorizer_config = TensorizerConfig(
232
+ tensorizer_uri=model_path,
233
+ **credentials)
234
+ serialize_vllm_model(engine, tensorizer_config, keyfile)
235
elif args.command == "deserialize":
278
- tensorizer_args = TensorizerArgs.from_cli_args(args)
279
- model_path = args.path_to_tensors
236
+ if not tensorizer_args:
237
+ tensorizer_config = TensorizerConfig(
238
+ tensorizer_uri=args.path_to_tensors,
239
+ encryption_keyfile = keyfile,
240
+ **credentials
241
+ )
242
deserialize()
243
else:
244
raise ValueError("Either serialize or deserialize must be specified.")
requirements-dev.txt
+1
-1
@@ -14,7 +14,7 @@ types-setuptools
14
15
# testing
16
pytest
17
-tensorizer==2.9.0
17
+tensorizer>=2.9.0
18
pytest-forked
19
pytest-asyncio
20
pytest-rerunfailures
setup.py
+1
-1
@@ -426,7 +426,7 @@ setup(
426
install_requires=get_requirements(),
427
ext_modules=ext_modules,
428
extras_require={
429
- "tensorizer": ["tensorizer==2.9.0"],
429
+ "tensorizer": ["tensorizer>=2.9.0"],
430
},
431
cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
432
package_data=package_data,
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py
deleted
-245
@@ -1,245 +0,0 @@
1
-import argparse
2
-import dataclasses
3
-import os
4
-import time
5
-import uuid
6
-from functools import partial
7
-from typing import Type
8
-
9
-import torch.nn as nn
10
-from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
11
- TensorSerializer, stream_io)
12
-from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
13
-from transformers import AutoConfig, PretrainedConfig
14
-
15
-from vllm.distributed import (init_distributed_environment,
16
- initialize_model_parallel)
17
-from vllm.engine.arg_utils import EngineArgs
18
-from vllm.engine.llm_engine import LLMEngine
19
-from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20
-from vllm.model_executor.models import ModelRegistry
21
-
22
-# yapf conflicts with isort for this docstring
23
-# yapf: disable
24
-"""
25
-tensorize_vllm_model.py is a script that can be used to serialize and
26
-deserialize vLLM models. These models can be loaded using tensorizer directly
27
-to the GPU extremely quickly. Tensor encryption and decryption is also
28
-supported, although libsodium must be installed to use it. Install
29
-vllm with tensorizer support using `pip install vllm[tensorizer]`.
30
-
31
-To serialize a model, you can run something like this:
32
-
33
-python tensorize_vllm_model.py \
34
- --model EleutherAI/gpt-j-6B \
35
- --dtype float16 \
36
- serialize \
37
- --serialized-directory s3://my-bucket/ \
38
- --suffix vllm
39
-
40
-Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41
-and saves it to your S3 bucket. A local directory can also be used.
42
-
43
-You can also encrypt the model weights with a randomly-generated key by
44
-providing a `--keyfile` argument.
45
-
46
-To deserialize a model, you can run something like this:
47
-
48
-python tensorize_vllm_model.py \
49
- --model EleutherAI/gpt-j-6B \
50
- --dtype float16 \
51
- deserialize \
52
- --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53
-
54
-Which downloads the model tensors from your S3 bucket and deserializes them.
55
-To provide S3 credentials, you can provide `--s3-access-key-id` and
56
-`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57
-the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58
-in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59
-
60
-
61
-You can also provide a `--keyfile` argument to decrypt the model weights if
62
-they were serialized with encryption.
63
-
64
-For more information on the available arguments, run
65
-`python tensorize_vllm_model.py --help`.
66
-"""
67
-
68
-
69
-def parse_args():
70
- parser = argparse.ArgumentParser(
71
- description="An example script that can be used to serialize and "
72
- "deserialize vLLM models. These models "
73
- "can be loaded using tensorizer directly to the GPU "
74
- "extremely quickly. Tensor encryption and decryption is "
75
- "also supported, although libsodium must be installed to "
76
- "use it.")
77
- parser = TensorizerArgs.add_cli_args(EngineArgs.add_cli_args(parser))
78
- subparsers = parser.add_subparsers(dest='command')
79
-
80
- serialize_parser = subparsers.add_parser(
81
- 'serialize', help="Serialize a model to `--serialized-directory`")
82
-
83
- serialize_parser.add_argument(
84
- "--suffix",
85
- type=str,
86
- required=False,
87
- help=(
88
- "The suffix to append to the serialized model directory, which is "
89
- "used to construct the location of the serialized model tensors, "
90
- "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91
- "`--suffix` is `v1`, the serialized model tensors will be "
92
- "saved to "
93
- "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94
- "If none is provided, a random UUID will be used."))
95
- serialize_parser.add_argument(
96
- "--serialized-directory",
97
- type=str,
98
- required=True)
99
-
100
- serialize_parser.add_argument(
101
- "--keyfile",
102
- type=str,
103
- required=False,
104
- help=("Encrypt the model weights with a randomly-generated binary key,"
105
- " and save the key at this path"))
106
-
107
- deserialize_parser = subparsers.add_parser(
108
- 'deserialize',
109
- help=("Deserialize a model from `--path-to-tensors`"
110
- " to verify it can be loaded and used."))
111
-
112
- deserialize_parser.add_argument(
113
- "--path-to-tensors",
114
- type=str,
115
- required=True,
116
- help="The local path or S3 URI to the model tensors to deserialize. ")
117
-
118
- deserialize_parser.add_argument(
119
- "--keyfile",
120
- type=str,
121
- required=False,
122
- help=("Path to a binary key to use to decrypt the model weights,"
123
- " if the model was serialized with encryption"))
124
-
125
- return parser.parse_args()
126
-
127
-
128
-def make_model_contiguous(model):
129
- # Ensure tensors are saved in memory contiguously
130
- for param in model.parameters():
131
- param.data = param.data.contiguous()
132
-
133
-
134
-def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
135
- architectures = getattr(config, "architectures", [])
136
- for arch in architectures:
137
- model_cls = ModelRegistry.load_model_cls(arch)
138
- if model_cls is not None:
139
- return model_cls
140
- raise ValueError(
141
- f"Model architectures {architectures} are not supported for now. "
142
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
143
-
144
-
145
-def serialize():
146
- eng_args_dict = {f.name: getattr(args, f.name) for f in
147
- dataclasses.fields(EngineArgs)}
148
- engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
149
- engine = LLMEngine.from_engine_args(engine_args)
150
-
151
- model = (engine.model_executor.driver_worker.
152
- model_runner.model)
153
-
154
- encryption_params = EncryptionParams.random() if keyfile else None
155
- if keyfile:
156
- with _write_stream(keyfile) as stream:
157
- stream.write(encryption_params.key)
158
-
159
- with _write_stream(model_path) as stream:
160
- serializer = TensorSerializer(stream, encryption=encryption_params)
161
- serializer.write_module(model)
162
- serializer.close()
163
-
164
- print("Serialization complete. Model tensors saved to", model_path)
165
- if keyfile:
166
- print("Key saved to", keyfile)
167
-
168
-
169
-def deserialize():
170
- config = AutoConfig.from_pretrained(model_ref)
171
-
172
- with no_init_or_tensor():
173
- model_class = _get_vllm_model_architecture(config)
174
- model = model_class(config)
175
-
176
- before_mem = get_mem_usage()
177
- start = time.time()
178
-
179
- if keyfile:
180
- with _read_stream(keyfile) as stream:
181
- key = stream.read()
182
- decryption_params = DecryptionParams.from_key(key)
183
- tensorizer_args.deserializer_params['encryption'] = \
184
- decryption_params
185
-
186
- with (_read_stream(model_path)) as stream, TensorDeserializer(
187
- stream, **tensorizer_args.deserializer_params) as deserializer:
188
- deserializer.load_into_module(model)
189
- end = time.time()
190
-
191
- # Brag about how fast we are.
192
- total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
193
- duration = end - start
194
- per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
195
- after_mem = get_mem_usage()
196
- print(
197
- f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
198
- )
199
- print(f"Memory usage before: {before_mem}")
200
- print(f"Memory usage after: {after_mem}")
201
-
202
- return model
203
-
204
-
205
-args = parse_args()
206
-
207
-s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
208
- or None)
209
-s3_secret_access_key = (args.s3_secret_access_key
210
- or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
211
-
212
-s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
213
-
214
-_read_stream, _write_stream = (partial(
215
- stream_io.open_stream,
216
- mode=mode,
217
- s3_access_key_id=s3_access_key_id,
218
- s3_secret_access_key=s3_secret_access_key,
219
- s3_endpoint=s3_endpoint,
220
-) for mode in ("rb", "wb+"))
221
-
222
-model_ref = args.model
223
-
224
-model_name = model_ref.split("/")[1]
225
-
226
-os.environ["MASTER_ADDR"] = "127.0.0.1"
227
-os.environ["MASTER_PORT"] = "8080"
228
-
229
-init_distributed_environment(world_size=1, rank=0, local_rank=0)
230
-initialize_model_parallel()
231
-
232
-keyfile = args.keyfile if args.keyfile else None
233
-
234
-if args.command == "serialize":
235
- input_dir = args.serialized_directory.rstrip('/')
236
- suffix = args.suffix if args.suffix else uuid.uuid4().hex
237
- base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
238
- model_path = f"{base_path}/model.tensors"
239
- serialize()
240
-elif args.command == "deserialize":
241
- tensorizer_args = TensorizerArgs.from_cli_args(args)
242
- model_path = args.path_to_tensors
243
- deserialize()
244
-else:
245
- raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer_loader/test_tensorizer.py
+70
-119
@@ -10,12 +10,19 @@ import ray
10
import torch
11
12
from vllm import SamplingParams
13
-from vllm.model_executor.model_loader.tensorizer import (
14
- EncryptionParams, TensorizerConfig, TensorSerializer,
15
- is_vllm_serialized_tensorizer, load_with_tensorizer, open_stream)
13
+# yapf: disable
14
+from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
15
+ TensorSerializer,
16
+ is_vllm_tensorized,
17
+ load_with_tensorizer,
18
+ open_stream,
19
+ serialize_vllm_model)
20
21
from ..utils import ServerRunner
22
23
+# yapf conflicts with isort for this docstring
24
+
25
+
26
prompts = [
27
"Hello, my name is",
28
"The president of the United States is",
@@ -40,7 +47,7 @@ def is_curl_installed():
47
48
@pytest.fixture(autouse=True)
49
def tensorizer_config():
43
- config = TensorizerConfig(tensorizer_uri="vllm", vllm_tensorized=True)
50
+ config = TensorizerConfig(tensorizer_uri="vllm")
51
return config
52
53
@@ -59,47 +66,6 @@ def test_load_with_tensorizer(mock_agent, tensorizer_config):
66
assert result == mock_agent_instance.deserialize.return_value
67
68
62
-def test_is_vllm_model_with_vllm_in_uri(tensorizer_config):
63
- tensorizer_config.vllm_tensorized = True
64
-
65
- result = is_vllm_serialized_tensorizer(tensorizer_config)
66
-
67
- assert result is True
68
-
69
-
70
-def test_is_vllm_model_without_vllm_in_uri(tensorizer_config):
71
- tensorizer_config.vllm_tensorized = False
72
-
73
- result = is_vllm_serialized_tensorizer(tensorizer_config)
74
-
75
- assert result is False
76
-
77
-
78
-def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
79
- vllm_model = vllm_runner(model_ref)
80
- model_path = tmp_path / (model_ref + ".tensors")
81
- outputs = vllm_model.generate(prompts, sampling_params)
82
- model = (vllm_model.model.llm_engine.model_executor.driver_worker.
83
- model_runner.model)
84
- with open_stream(model_path, "wb+") as stream:
85
- serializer = TensorSerializer(stream)
86
- serializer.write_module(model)
87
- del vllm_model, model
88
- gc.collect()
89
- torch.cuda.empty_cache()
90
- loaded_vllm_model = vllm_runner(
91
- model_ref,
92
- load_format="tensorizer",
93
- model_loader_extra_config=TensorizerConfig(tensorizer_uri=model_path,
94
- num_readers=1,
95
- vllm_tensorized=True),
96
- )
97
- deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
98
-
99
- # Assumes SamplingParams being seeded ensures the outputs are deterministic
100
- assert outputs == deserialized_outputs
101
-
102
-
69
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
70
def test_can_deserialize_s3(vllm_runner):
71
model_ref = "EleutherAI/pythia-1.4b"
@@ -110,7 +76,6 @@ def test_can_deserialize_s3(vllm_runner):
76
model_loader_extra_config=TensorizerConfig(
77
tensorizer_uri=tensorized_path,
78
num_readers=1,
113
- vllm_tensorized=False,
79
s3_endpoint="object.ord1.coreweave.com",
80
))
81
@@ -126,29 +91,26 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
91
model_path = tmp_path / (model_ref + ".tensors")
92
key_path = tmp_path / (model_ref + ".key")
93
outputs = vllm_model.generate(prompts, sampling_params)
129
- model = (vllm_model.model.llm_engine.model_executor.driver_worker.
130
- model_runner.model)
94
132
- encryption_params = EncryptionParams.random()
133
- with open_stream(model_path, "wb+") as stream:
134
- serializer = TensorSerializer(stream, encryption=encryption_params)
135
- serializer.write_module(model)
136
- with open_stream(key_path, "wb+") as stream:
137
- stream.write(encryption_params.key)
138
- del vllm_model, model
95
+ config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
96
+ serialize_vllm_model(vllm_model.model.llm_engine,
97
+ config_for_serializing,
98
+ encryption_key_path=key_path)
99
+
100
+ del vllm_model
101
gc.collect()
102
torch.cuda.empty_cache()
141
- loaded_vllm_model = vllm_runner(model_ref,
142
- load_format="tensorizer",
143
- model_loader_extra_config=TensorizerConfig(
144
- tensorizer_uri=model_path,
145
- encryption_keyfile=key_path,
146
- num_readers=1,
147
- vllm_tensorized=True))
103
+
104
+ config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
105
+ encryption_keyfile=key_path)
106
+
107
+ loaded_vllm_model = vllm_runner(
108
+ model_ref,
109
+ load_format="tensorizer",
110
+ model_loader_extra_config=config_for_deserializing)
111
112
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
113
151
- # Assumes SamplingParams being seeded ensures the outputs are deterministic
114
assert outputs == deserialized_outputs
115
116
@@ -169,7 +131,7 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
131
model_loader_extra_config=TensorizerConfig(
132
tensorizer_uri=model_path,
133
num_readers=1,
172
- vllm_tensorized=False))
134
+ ))
135
136
deserialized_outputs = loaded_hf_model.generate_greedy(
137
prompts, max_tokens=max_tokens)
@@ -190,12 +152,11 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
152
# Serialize model before deserializing and binding LoRA adapters
153
vllm_model = vllm_runner(model_ref, )
154
model_path = tmp_path / (model_ref + ".tensors")
193
- model = (vllm_model.model.llm_engine.model_executor.driver_worker.
194
- model_runner.model)
195
- with open_stream(model_path, "wb+") as stream:
196
- serializer = TensorSerializer(stream)
197
- serializer.write_module(model)
198
- del vllm_model, model
155
+
156
+ serialize_vllm_model(vllm_model.model.llm_engine,
157
+ TensorizerConfig(tensorizer_uri=model_path))
158
+
159
+ del vllm_model
160
gc.collect()
161
torch.cuda.empty_cache()
162
loaded_vllm_model = vllm_runner(
@@ -204,7 +165,6 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
165
model_loader_extra_config=TensorizerConfig(
166
tensorizer_uri=model_path,
167
num_readers=1,
207
- vllm_tensorized=True,
168
),
169
enable_lora=True,
170
max_loras=1,
@@ -220,58 +180,28 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
180
181
def test_load_without_tensorizer_load_format(vllm_runner):
182
with pytest.raises(ValueError):
223
- vllm_runner(model_ref,
224
- model_loader_extra_config=TensorizerConfig(
225
- tensorizer_uri="test", vllm_tensorized=False))
226
-
227
-
228
-@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
229
-def test_tensorize_vllm_model(tmp_path):
230
- # Test serialize command
231
- serialize_args = [
232
- "python3", tensorize_model_for_testing_script, "--model", model_ref,
233
- "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
234
- "--suffix", "tests"
235
- ]
236
- result = subprocess.run(serialize_args, capture_output=True, text=True)
237
- print(result.stdout) # Print the output of the serialize command
238
-
239
- assert result.returncode == 0, (f"Serialize command failed with output:"
240
- f"\n{result.stdout}\n{result.stderr}")
241
-
242
- path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
243
-
244
- # Test deserialize command
245
- deserialize_args = [
246
- "python3", tensorize_model_for_testing_script, "--model", model_ref,
247
- "--dtype", "float16", "deserialize", "--path-to-tensors",
248
- path_to_tensors
249
- ]
250
- result = subprocess.run(deserialize_args, capture_output=True, text=True)
251
- assert result.returncode == 0, (f"Deserialize command failed with output:"
252
- f"\n{result.stdout}\n{result.stderr}")
183
+ vllm_runner(
184
+ model_ref,
185
+ model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
186
187
188
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
256
-def test_openai_apiserver_with_tensorizer(tmp_path):
189
+def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
190
## Serialize model
258
- serialize_args = [
259
- "python3", tensorize_model_for_testing_script, "--model", model_ref,
260
- "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
261
- "--suffix", "tests"
262
- ]
263
- result = subprocess.run(serialize_args, capture_output=True, text=True)
264
- print(result.stdout) # Print the output of the serialize command
191
+ vllm_model = vllm_runner(model_ref, )
192
+ model_path = tmp_path / (model_ref + ".tensors")
193
266
- assert result.returncode == 0, (f"Serialize command failed with output:"
267
- f"\n{result.stdout}\n{result.stderr}")
194
+ serialize_vllm_model(vllm_model.model.llm_engine,
195
+ TensorizerConfig(tensorizer_uri=model_path))
196
269
- path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
197
model_loader_extra_config = {
271
- "tensorizer_uri": path_to_tensors,
272
- "vllm_tensorized": True
198
+ "tensorizer_uri": str(model_path),
199
}
200
201
+ del vllm_model
202
+ gc.collect()
203
+ torch.cuda.empty_cache()
204
+
205
## Start OpenAI API server
206
openai_args = [
207
"--model", model_ref, "--dtype", "float16", "--load-format",
@@ -304,10 +234,10 @@ def test_openai_apiserver_with_tensorizer(tmp_path):
234
235
def test_raise_value_error_on_invalid_load_format(vllm_runner):
236
with pytest.raises(ValueError):
307
- vllm_runner(model_ref,
308
- load_format="safetensors",
309
- model_loader_extra_config=TensorizerConfig(
310
- tensorizer_uri="test", vllm_tensorized=False))
237
+ vllm_runner(
238
+ model_ref,
239
+ load_format="safetensors",
240
+ model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
241
242
243
def test_tensorizer_with_tp(vllm_runner):
@@ -321,8 +251,29 @@ def test_tensorizer_with_tp(vllm_runner):
251
model_loader_extra_config=TensorizerConfig(
252
tensorizer_uri=tensorized_path,
253
num_readers=1,
324
- vllm_tensorized=False,
254
s3_endpoint="object.ord1.coreweave.com",
255
),
256
tensor_parallel_size=2,
257
)
258
+
259
+
260
+def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
261
+ model_ref = "facebook/opt-125m"
262
+ model_path = tmp_path / (model_ref + ".tensors")
263
+ config = TensorizerConfig(tensorizer_uri=str(model_path))
264
+
265
+ vllm_model = vllm_runner(model_ref)
266
+ outputs = vllm_model.generate(prompts, sampling_params)
267
+ serialize_vllm_model(vllm_model.model.llm_engine, config)
268
+
269
+ assert is_vllm_tensorized(config)
270
+ del vllm_model
271
+ gc.collect()
272
+ torch.cuda.empty_cache()
273
+
274
+ loaded_vllm_model = vllm_runner(model_ref,
275
+ load_format="tensorizer",
276
+ model_loader_extra_config=config)
277
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
278
+
279
+ assert outputs == deserialized_outputs
vllm/engine/arg_utils.py
+2
-2
@@ -167,8 +167,8 @@ class EngineArgs:
167
'* "dummy" will initialize the weights with random values, '
168
'which is mainly for profiling.\n'
169
'* "tensorizer" will load the weights using tensorizer from '
170
- 'CoreWeave which assumes tensorizer_uri is set to the location of '
171
- 'the serialized weights.')
170
+ 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
171
+ 'section for more information.\n')
172
parser.add_argument(
173
'--dtype',
174
type=str,
vllm/envs.py
+1
-1
@@ -145,7 +145,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
145
146
# S3 access information, used for tensorizer to load model from S3
147
"S3_ACCESS_KEY_ID":
148
- lambda: os.environ.get("S3_ACCESS_KEY", None),
148
+ lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
149
"S3_SECRET_ACCESS_KEY":
150
lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
151
"S3_ENDPOINT_URL":
vllm/model_executor/model_loader/loader.py
+15
-13
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization.base_config import (
18
QuantizationConfig)
19
from vllm.model_executor.model_loader.tensorizer import (
20
- TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
20
+ TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
21
tensorizer_weights_iterator)
22
from vllm.model_executor.model_loader.utils import (get_model_architecture,
23
set_default_torch_dtype)
@@ -291,7 +291,7 @@ class TensorizerLoader(BaseModelLoader):
291
tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
292
return tensorizer_weights_iterator(tensorizer_args)
293
294
- def _load_model_unserialized(
294
+ def _load_model_serialized_cpu(
295
self,
296
model_config: ModelConfig,
297
device_config: DeviceConfig,
@@ -299,11 +299,12 @@ class TensorizerLoader(BaseModelLoader):
299
vision_language_config: Optional[VisionLanguageConfig],
300
cache_config: CacheConfig,
301
) -> nn.Module:
302
- """Load an unserialized model with tensorizer.
302
+ """Load a serialized model with tensorizer to the CPU.
303
304
- Unserialized here means "not serialized with tensorizer". This
305
- should still be faster than default HuggingFace loading, but will
306
- be slower than loading a tensorizer-serialized model.
304
+ This is only necessary when the model isn't vLLM-tensorized (see
305
+ examples/tensorize_vllm_model.py) This should still be faster than
306
+ default HuggingFace loading, but will be slower than loading a
307
+ vLLM-tensorized model.
308
"""
309
with set_default_torch_dtype(model_config.dtype):
310
with torch.device(device_config.device):
@@ -324,8 +325,9 @@ class TensorizerLoader(BaseModelLoader):
325
) -> nn.Module:
326
"""Load a serialized model with tensorizer.
327
327
- See the examples/tensorize_vllm_model.py example "
328
- script for serializing vLLM models."""
328
+ Expects a vLLM-tensorized model. See the
329
+ examples/tensorize_vllm_model.py example script
330
+ for serializing vLLM models."""
331
with set_default_torch_dtype(model_config.dtype):
332
with torch.device(device_config.device):
333
model_class = get_model_architecture(model_config)[0]
@@ -353,15 +355,15 @@ class TensorizerLoader(BaseModelLoader):
355
cache_config: CacheConfig) -> nn.Module:
356
self._verify_config(model_config, parallel_config)
357
356
- if is_vllm_serialized_tensorizer(self.tensorizer_config):
358
+ if is_vllm_tensorized(self.tensorizer_config):
359
return self._load_model_serialized(model_config, device_config,
360
lora_config,
361
vision_language_config,
362
cache_config)
361
- return self._load_model_unserialized(model_config, device_config,
362
- lora_config,
363
- vision_language_config,
364
- cache_config)
363
+ return self._load_model_serialized_cpu(model_config, device_config,
364
+ lora_config,
365
+ vision_language_config,
366
+ cache_config)
367
368
369
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
vllm/model_executor/model_loader/tensorizer.py
+85
-21
@@ -5,6 +5,7 @@ import os
5
import time
6
import typing
7
from dataclasses import dataclass
8
+from functools import partial
9
from typing import Generator, Optional, Tuple, Type, Union
10
11
import torch
@@ -13,6 +14,7 @@ from transformers import PretrainedConfig
14
15
import vllm.envs as envs
16
from vllm.config import ModelConfig, ParallelConfig
17
+from vllm.engine.llm_engine import LLMEngine
18
from vllm.logger import init_logger
19
from vllm.model_executor.layers.quantization.base_config import (
20
QuantizationConfig)
@@ -27,6 +29,11 @@ try:
29
from tensorizer.stream_io import open_stream
30
from tensorizer.utils import (convert_bytes, get_mem_usage,
31
no_init_or_tensor)
32
+
33
+ _read_stream, _write_stream = (partial(
34
+ open_stream,
35
+ mode=mode,
36
+ ) for mode in ("rb", "wb+"))
37
except ImportError as e:
38
tensorizer_error_msg = str(e)
39
@@ -43,7 +50,7 @@ logger = init_logger(__name__)
50
class TensorizerConfig:
51
tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
52
str, bytes, os.PathLike, int]
46
- vllm_tensorized: bool
53
+ vllm_tensorized: Optional[bool] = False
54
verify_hash: Optional[bool] = False
55
num_readers: Optional[int] = None
56
encryption_keyfile: Optional[str] = None
@@ -93,17 +100,11 @@ def load_with_tensorizer(tensorizer_config: TensorizerConfig,
100
return tensorizer.deserialize()
101
102
96
-def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
97
- if tensorizer_config is None:
98
- return False
99
- return tensorizer_config.vllm_tensorized
100
-
101
-
103
@dataclass
104
class TensorizerArgs:
105
tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
106
str, bytes, os.PathLike, int]
106
- vllm_tensorized: bool
107
+ vllm_tensorized: Optional[bool] = False
108
verify_hash: Optional[bool] = False
109
num_readers: Optional[int] = None
110
encryption_keyfile: Optional[str] = None
@@ -121,7 +122,9 @@ class TensorizerArgs:
122
vLLM model. This is used to determine the behavior of the
123
TensorDeserializer when loading tensors from a serialized model.
124
It is far faster to deserialize a vLLM model as it utilizes
124
- tensorizer's optimized GPU loading.
125
+ tensorizer's optimized GPU loading. Note that this is now
126
+ deprecated, as serialized vLLM models are now automatically
127
+ inferred as vLLM models.
128
verify_hash: If True, the hashes of each tensor will be verified against
129
the hashes stored in the metadata. A `HashMismatchError` will be
130
raised if any of the hashes do not match.
@@ -158,6 +161,7 @@ class TensorizerArgs:
161
"encryption": self.encryption_keyfile,
162
"num_readers": self.num_readers
163
}
164
+
165
if self.encryption_keyfile:
166
with open_stream(
167
self.encryption_keyfile,
@@ -177,7 +181,14 @@ class TensorizerArgs:
181
'tensorizer options',
182
description=('Options for configuring the behavior of the'
183
' tensorizer deserializer when '
180
- '--load-format=tensorizer'))
184
+ 'load_format=tensorizer is specified when '
185
+ 'initializing an LLMEngine, either via the CLI '
186
+ 'when running the vLLM OpenAI inference server '
187
+ 'with a JSON string passed to '
188
+ '--model-loader-extra-config or as arguments given '
189
+ 'to TensorizerConfig when passed to '
190
+ 'model_loader_extra_config in the constructor '
191
+ 'for LLMEngine.'))
192
193
group.add_argument(
194
"--tensorizer-uri",
@@ -222,13 +233,6 @@ class TensorizerArgs:
233
help="The endpoint for the S3 bucket. Can also be set via the "
234
"S3_ENDPOINT_URL environment variable.",
235
)
225
- group.add_argument(
226
- "--vllm-tensorized",
227
- action="store_true",
228
- help="If enabled, indicates that the serialized model is a vLLM "
229
- "model. This is used to determine the behavior of the "
230
- "TensorDeserializer when loading tensors from a "
231
- "serialized model.")
236
237
return parser
238
@@ -322,10 +326,9 @@ class TensorizerAgent:
326
"""
327
before_mem = get_mem_usage()
328
start = time.perf_counter()
325
- with open_stream(
326
- self.tensorizer_args.tensorizer_uri,
327
- mode="rb",
328
- **self.tensorizer_args.stream_params,
329
+ with _read_stream(
330
+ self.tensorizer_config.tensorizer_uri,
331
+ **self.tensorizer_args.stream_params
332
) as stream, TensorDeserializer(
333
stream,
334
dtype=self.tensorizer_config.dtype,
@@ -345,6 +348,7 @@ class TensorizerAgent:
348
349
self._check_tensors_on_meta_device()
350
self._resize_lora_embeddings()
351
+ del self.model.vllm_tensorized_marker
352
return self.model.eval()
353
354
@@ -366,3 +370,63 @@ def tensorizer_weights_iterator(
370
for name, param in state.items():
371
yield name, param
372
del state
373
+
374
+
375
+def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
376
+ """
377
+ Infer if the model is a vLLM model by checking the weights for
378
+ a vLLM tensorized marker.
379
+
380
+ Args:
381
+ tensorizer_config: The TensorizerConfig object containing the
382
+ tensorizer_uri to the serialized model.
383
+
384
+ Returns:
385
+ bool: True if the model is a vLLM model, False otherwise.
386
+ """
387
+ tensorizer_args = tensorizer_config._construct_tensorizer_args()
388
+ deserializer = TensorDeserializer(open_stream(
389
+ tensorizer_args.tensorizer_uri, **tensorizer_args.stream_params),
390
+ **tensorizer_args.deserializer_params,
391
+ lazy_load=True)
392
+ if tensorizer_config.vllm_tensorized:
393
+ logger.warning(
394
+ "Please note that newly serialized vLLM models are automatically "
395
+ "inferred as vLLM models, so setting vllm_tensorized=True is "
396
+ "only necessary for models serialized prior to this change.")
397
+ return True
398
+ if (".vllm_tensorized_marker" in deserializer):
399
+ return True
400
+ return False
401
+
402
+
403
+def get_pretensorized_vllm_model(engine: "LLMEngine") -> nn.Module:
404
+ model = (engine.model_executor.driver_worker.model_runner.model)
405
+ model.register_parameter(
406
+ "vllm_tensorized_marker",
407
+ nn.Parameter(torch.tensor((1, ), device="meta"), requires_grad=False))
408
+ return model
409
+
410
+
411
+def serialize_vllm_model(engine: "LLMEngine",
412
+ tensorizer_config : TensorizerConfig,
413
+ encryption_key_path: Optional[str] = None) \
414
+ -> nn.Module:
415
+
416
+ model = get_pretensorized_vllm_model(engine)
417
+ tensorizer_args = tensorizer_config._construct_tensorizer_args()
418
+ encryption_params = None
419
+ if encryption_key_path is not None:
420
+ encryption_params = EncryptionParams.random()
421
+ with _write_stream(encryption_key_path,
422
+ **tensorizer_args.stream_params) as stream:
423
+ stream.write(encryption_params.key)
424
+
425
+ with _write_stream(tensorizer_args.tensorizer_uri,
426
+ **tensorizer_args.stream_params) as stream:
427
+ serializer = TensorSerializer(stream, encryption=encryption_params)
428
+ serializer.write_module(model)
429
+ serializer.close()
430
+ logger.info("Successfully serialized model to %s",
431
+ str(tensorizer_args.tensorizer_uri))
432
+ return model