@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8bc68e198c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] [Core] perf: Automatically detect vLLM-tensorized model, update `tensorizer` to version 2.9.0 (#4208)

Sanger Steel committed May 13, 2024 at 17:57 UTC 8bc68e198c4c90ddc2e54fa76eb81c2c714bb1cd
10 files changed +259 -523
.buildkite/test-pipeline.yaml
+3 -1
@@ -60,11 +60,13 @@ steps:
60 mirror_hardwares: [amd]
61 commands:
62 # install aws cli for llava_example.py
63 - - pip install awscli
63 + # install tensorizer for tensorize_vllm_model.py
64 + - pip install awscli tensorizer
65 - python3 offline_inference.py
66 - python3 offline_inference_with_prefix.py
67 - python3 llm_engine_example.py
68 - python3 llava_example.py
69 + - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
70
71 - label: Kernels Test %N
72 command: pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
examples/tensorize_vllm_model.py
+81 -119
@@ -1,23 +1,20 @@
1 import argparse
2 import dataclasses
3 +import json
4 import os
4 -import time
5 import uuid
6 from functools import partial
7 -from typing import Type
7
9 -import torch
10 -import torch.nn as nn
11 -from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
12 - TensorSerializer, stream_io)
13 -from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
14 -from transformers import AutoConfig, PretrainedConfig
8 +from tensorizer import stream_io
9
16 -from vllm.distributed import initialize_model_parallel
10 +from vllm import LLM
11 +from vllm.distributed import (init_distributed_environment,
12 + initialize_model_parallel)
13 from vllm.engine.arg_utils import EngineArgs
14 from vllm.engine.llm_engine import LLMEngine
19 -from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20 -from vllm.model_executor.models import ModelRegistry
15 +from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
16 + TensorizerConfig,
17 + serialize_vllm_model)
18
19 # yapf conflicts with isort for this docstring
20 # yapf: disable
@@ -27,25 +24,25 @@ deserialize vLLM models. These models can be loaded using tensorizer
24 to the GPU extremely quickly over an HTTP/HTTPS endpoint, an S3 endpoint,
25 or locally. Tensor encryption and decryption is also supported, although
26 libsodium must be installed to use it. Install vllm with tensorizer support
30 -using `pip install vllm[tensorizer]`.
27 +using `pip install vllm[tensorizer]`. To learn more about tensorizer, visit
28 +https://github.com/coreweave/tensorizer
29
30 To serialize a model, install vLLM from source, then run something
31 like this from the root level of this repository:
32
33 python -m examples.tensorize_vllm_model \
36 - --model EleutherAI/gpt-j-6B \
37 - --dtype float16 \
34 + --model facebook/opt-125m \
35 serialize \
39 - --serialized-directory s3://my-bucket/ \
40 - --suffix vllm
36 + --serialized-directory s3://my-bucket \
37 + --suffix v1
38
39 Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
40 and saves it to your S3 bucket. A local directory can also be used. This
41 assumes your S3 credentials are specified as environment variables
45 -in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
46 -To provide S3 credentials directly, you can provide `--s3-access-key-id` and
47 -`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this
48 -script.
42 +in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and
43 +`S3_ENDPOINT_URL`. To provide S3 credentials directly, you can provide
44 +`--s3-access-key-id` and `--s3-secret-access-key`, as well as `--s3-endpoint`
45 +as CLI args to this script.
46
47 You can also encrypt the model weights with a randomly-generated key by
48 providing a `--keyfile` argument.
@@ -57,7 +54,7 @@ python -m examples.tensorize_vllm_model \
54 --model EleutherAI/gpt-j-6B \
55 --dtype float16 \
56 deserialize \
60 - --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
57 + --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors
58
59 Which downloads the model tensors from your S3 bucket and deserializes them.
60
@@ -71,26 +68,30 @@ Or for deserializing:
68
69 `python -m examples.tensorize_vllm_model deserialize --help`.
70
74 -Once a model is serialized, it can be used to load the model when running the
75 -OpenAI inference client at `vllm/entrypoints/openai/api_server.py` by providing
76 -the `--tensorizer-uri` CLI argument that is functionally the same as the
77 -`--path-to-tensors` argument in this script, along with `--vllm-tensorized`, to
78 -signify that the model to be deserialized is a vLLM model, rather than a
79 -HuggingFace `PreTrainedModel`, which can also be deserialized using tensorizer
80 -in the same inference server, albeit without the speed optimizations. To
81 -deserialize an encrypted file, the `--encryption-keyfile` argument can be used
82 -to provide the path to the keyfile used to encrypt the model weights. For
83 -information on all the arguments that can be used to configure tensorizer's
84 -deserialization, check out the tensorizer options argument group in the
85 -`vllm/entrypoints/openai/api_server.py` script with `--help`.
86 -
87 -Tensorizer can also be invoked with the `LLM` class directly to load models:
71 +Once a model is serialized, tensorizer can be invoked with the `LLM` class
72 +directly to load models:
73
74 llm = LLM(model="facebook/opt-125m",
75 load_format="tensorizer",
91 - tensorizer_uri=path_to_opt_tensors,
92 - num_readers=3,
93 - vllm_tensorized=True)
76 + model_loader_extra_config=TensorizerConfig(
77 + tensorizer_uri = path_to_tensors,
78 + num_readers=3,
79 + )
80 + )
81 +
82 +A serialized model can be used during model loading for the vLLM OpenAI
83 +inference server. `model_loader_extra_config` is exposed as the CLI arg
84 +`--model-loader-extra-config`, and accepts a JSON string literal of the
85 +TensorizerConfig arguments desired.
86 +
87 +In order to see all of the available arguments usable to configure
88 +loading with tensorizer that are given to `TensorizerConfig`, run:
89 +
90 +`python -m examples.tensorize_vllm_model deserialize --help`
91 +
92 +under the `tensorizer options` section. These can also be used for
93 +deserialization in this example script, although `--tensorizer-uri` and
94 +`--path-to-tensors` are functionally the same in this case.
95 """
96
97
@@ -158,95 +159,35 @@ def parse_args():
159 help=("Path to a binary key to use to decrypt the model weights,"
160 " if the model was serialized with encryption"))
161
161 - return parser.parse_args()
162 -
163 -
164 -def make_model_contiguous(model):
165 - # Ensure tensors are saved in memory contiguously
166 - for param in model.parameters():
167 - param.data = param.data.contiguous()
168 -
169 -
170 -def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
171 - architectures = getattr(config, "architectures", [])
172 - for arch in architectures:
173 - model_cls = ModelRegistry.load_model_cls(arch)
174 - if model_cls is not None:
175 - return model_cls
176 - raise ValueError(
177 - f"Model architectures {architectures} are not supported for now. "
178 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
179 -
180 -
181 -def serialize():
182 -
183 - eng_args_dict = {f.name: getattr(args, f.name) for f in
184 - dataclasses.fields(EngineArgs)}
185 - engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
186 - engine = LLMEngine.from_engine_args(engine_args)
162 + TensorizerArgs.add_cli_args(deserialize_parser)
163
188 - model = (engine.model_executor.driver_worker.
189 - model_runner.model)
190 -
191 - encryption_params = EncryptionParams.random() if keyfile else None
192 - if keyfile:
193 - with _write_stream(keyfile) as stream:
194 - stream.write(encryption_params.key)
195 -
196 - with _write_stream(model_path) as stream:
197 - serializer = TensorSerializer(stream, encryption=encryption_params)
198 - serializer.write_module(model)
199 - serializer.close()
164 + return parser.parse_args()
165
201 - print("Serialization complete. Model tensors saved to", model_path)
202 - if keyfile:
203 - print("Key saved to", keyfile)
166
167
168 def deserialize():
207 - config = AutoConfig.from_pretrained(model_ref)
208 -
209 - with no_init_or_tensor():
210 - model_class = _get_vllm_model_architecture(config)
211 - model = model_class(config)
212 -
213 - before_mem = get_mem_usage()
214 - start = time.time()
215 -
216 - if keyfile:
217 - with _read_stream(keyfile) as stream:
218 - key = stream.read()
219 - decryption_params = DecryptionParams.from_key(key)
220 - tensorizer_args.deserializer_params['encryption'] = \
221 - decryption_params
222 -
223 - with (_read_stream(model_path)) as stream, TensorDeserializer(
224 - stream, **tensorizer_args.deserializer_params) as deserializer:
225 - deserializer.load_into_module(model)
226 - end = time.time()
227 -
228 - # Brag about how fast we are.
229 - total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
230 - duration = end - start
231 - per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
232 - after_mem = get_mem_usage()
233 - print(
234 - f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
169 + llm = LLM(model=args.model,
170 + load_format="tensorizer",
171 + model_loader_extra_config=tensorizer_config
172 )
236 - print(f"Memory usage before: {before_mem}")
237 - print(f"Memory usage after: {after_mem}")
173 + return llm
174
239 - return model
175
176
177 args = parse_args()
178
244 -s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
245 - or None)
246 -s3_secret_access_key = (args.s3_secret_access_key
247 - or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
179 +s3_access_key_id = (getattr(args, 's3_access_key_id', None)
180 + or os.environ.get("S3_ACCESS_KEY_ID", None))
181 +s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
182 + or os.environ.get("S3_SECRET_ACCESS_KEY", None))
183 +s3_endpoint = (getattr(args, 's3_endpoint', None)
184 + or os.environ.get("S3_ENDPOINT_URL", None))
185
249 -s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
186 +credentials = {
187 + "s3_access_key_id": s3_access_key_id,
188 + "s3_secret_access_key": s3_secret_access_key,
189 + "s3_endpoint": s3_endpoint
190 +}
191
192 _read_stream, _write_stream = (partial(
193 stream_io.open_stream,
@@ -263,20 +204,41 @@ model_name = model_ref.split("/")[1]
204 os.environ["MASTER_ADDR"] = "127.0.0.1"
205 os.environ["MASTER_PORT"] = "8080"
206
266 -torch.distributed.init_process_group(world_size=1, rank=0)
207 +init_distributed_environment(world_size=1, rank=0, local_rank=0)
208 initialize_model_parallel()
209
210 keyfile = args.keyfile if args.keyfile else None
211
212 +
213 +if args.model_loader_extra_config:
214 + config = json.loads(args.model_loader_extra_config)
215 + tensorizer_args = TensorizerConfig(**config)._construct_tensorizer_args()
216 + tensorizer_args.tensorizer_uri = args.path_to_tensors
217 +else:
218 + tensorizer_args = None
219 +
220 if args.command == "serialize":
221 + eng_args_dict = {f.name: getattr(args, f.name) for f in
222 + dataclasses.fields(EngineArgs)}
223 +
224 + engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
225 + engine = LLMEngine.from_engine_args(engine_args)
226 +
227 input_dir = args.serialized_directory.rstrip('/')
228 suffix = args.suffix if args.suffix else uuid.uuid4().hex
229 base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
230 model_path = f"{base_path}/model.tensors"
276 - serialize()
231 + tensorizer_config = TensorizerConfig(
232 + tensorizer_uri=model_path,
233 + **credentials)
234 + serialize_vllm_model(engine, tensorizer_config, keyfile)
235 elif args.command == "deserialize":
278 - tensorizer_args = TensorizerArgs.from_cli_args(args)
279 - model_path = args.path_to_tensors
236 + if not tensorizer_args:
237 + tensorizer_config = TensorizerConfig(
238 + tensorizer_uri=args.path_to_tensors,
239 + encryption_keyfile = keyfile,
240 + **credentials
241 + )
242 deserialize()
243 else:
244 raise ValueError("Either serialize or deserialize must be specified.")
requirements-dev.txt
+1 -1
@@ -14,7 +14,7 @@ types-setuptools
14
15 # testing
16 pytest
17 -tensorizer==2.9.0
17 +tensorizer>=2.9.0
18 pytest-forked
19 pytest-asyncio
20 pytest-rerunfailures
setup.py
+1 -1
@@ -426,7 +426,7 @@ setup(
426 install_requires=get_requirements(),
427 ext_modules=ext_modules,
428 extras_require={
429 - "tensorizer": ["tensorizer==2.9.0"],
429 + "tensorizer": ["tensorizer>=2.9.0"],
430 },
431 cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
432 package_data=package_data,
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py deleted
-245
@@ -1,245 +0,0 @@
1 -import argparse
2 -import dataclasses
3 -import os
4 -import time
5 -import uuid
6 -from functools import partial
7 -from typing import Type
8 -
9 -import torch.nn as nn
10 -from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
11 - TensorSerializer, stream_io)
12 -from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
13 -from transformers import AutoConfig, PretrainedConfig
14 -
15 -from vllm.distributed import (init_distributed_environment,
16 - initialize_model_parallel)
17 -from vllm.engine.arg_utils import EngineArgs
18 -from vllm.engine.llm_engine import LLMEngine
19 -from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20 -from vllm.model_executor.models import ModelRegistry
21 -
22 -# yapf conflicts with isort for this docstring
23 -# yapf: disable
24 -"""
25 -tensorize_vllm_model.py is a script that can be used to serialize and
26 -deserialize vLLM models. These models can be loaded using tensorizer directly
27 -to the GPU extremely quickly. Tensor encryption and decryption is also
28 -supported, although libsodium must be installed to use it. Install
29 -vllm with tensorizer support using `pip install vllm[tensorizer]`.
30 -
31 -To serialize a model, you can run something like this:
32 -
33 -python tensorize_vllm_model.py \
34 - --model EleutherAI/gpt-j-6B \
35 - --dtype float16 \
36 - serialize \
37 - --serialized-directory s3://my-bucket/ \
38 - --suffix vllm
39 -
40 -Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41 -and saves it to your S3 bucket. A local directory can also be used.
42 -
43 -You can also encrypt the model weights with a randomly-generated key by
44 -providing a `--keyfile` argument.
45 -
46 -To deserialize a model, you can run something like this:
47 -
48 -python tensorize_vllm_model.py \
49 - --model EleutherAI/gpt-j-6B \
50 - --dtype float16 \
51 - deserialize \
52 - --path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
53 -
54 -Which downloads the model tensors from your S3 bucket and deserializes them.
55 -To provide S3 credentials, you can provide `--s3-access-key-id` and
56 -`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57 -the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58 -in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59 -
60 -
61 -You can also provide a `--keyfile` argument to decrypt the model weights if
62 -they were serialized with encryption.
63 -
64 -For more information on the available arguments, run
65 -`python tensorize_vllm_model.py --help`.
66 -"""
67 -
68 -
69 -def parse_args():
70 - parser = argparse.ArgumentParser(
71 - description="An example script that can be used to serialize and "
72 - "deserialize vLLM models. These models "
73 - "can be loaded using tensorizer directly to the GPU "
74 - "extremely quickly. Tensor encryption and decryption is "
75 - "also supported, although libsodium must be installed to "
76 - "use it.")
77 - parser = TensorizerArgs.add_cli_args(EngineArgs.add_cli_args(parser))
78 - subparsers = parser.add_subparsers(dest='command')
79 -
80 - serialize_parser = subparsers.add_parser(
81 - 'serialize', help="Serialize a model to `--serialized-directory`")
82 -
83 - serialize_parser.add_argument(
84 - "--suffix",
85 - type=str,
86 - required=False,
87 - help=(
88 - "The suffix to append to the serialized model directory, which is "
89 - "used to construct the location of the serialized model tensors, "
90 - "e.g. if `--serialized-directory` is `s3://my-bucket/` and "
91 - "`--suffix` is `v1`, the serialized model tensors will be "
92 - "saved to "
93 - "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
94 - "If none is provided, a random UUID will be used."))
95 - serialize_parser.add_argument(
96 - "--serialized-directory",
97 - type=str,
98 - required=True)
99 -
100 - serialize_parser.add_argument(
101 - "--keyfile",
102 - type=str,
103 - required=False,
104 - help=("Encrypt the model weights with a randomly-generated binary key,"
105 - " and save the key at this path"))
106 -
107 - deserialize_parser = subparsers.add_parser(
108 - 'deserialize',
109 - help=("Deserialize a model from `--path-to-tensors`"
110 - " to verify it can be loaded and used."))
111 -
112 - deserialize_parser.add_argument(
113 - "--path-to-tensors",
114 - type=str,
115 - required=True,
116 - help="The local path or S3 URI to the model tensors to deserialize. ")
117 -
118 - deserialize_parser.add_argument(
119 - "--keyfile",
120 - type=str,
121 - required=False,
122 - help=("Path to a binary key to use to decrypt the model weights,"
123 - " if the model was serialized with encryption"))
124 -
125 - return parser.parse_args()
126 -
127 -
128 -def make_model_contiguous(model):
129 - # Ensure tensors are saved in memory contiguously
130 - for param in model.parameters():
131 - param.data = param.data.contiguous()
132 -
133 -
134 -def _get_vllm_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
135 - architectures = getattr(config, "architectures", [])
136 - for arch in architectures:
137 - model_cls = ModelRegistry.load_model_cls(arch)
138 - if model_cls is not None:
139 - return model_cls
140 - raise ValueError(
141 - f"Model architectures {architectures} are not supported for now. "
142 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
143 -
144 -
145 -def serialize():
146 - eng_args_dict = {f.name: getattr(args, f.name) for f in
147 - dataclasses.fields(EngineArgs)}
148 - engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
149 - engine = LLMEngine.from_engine_args(engine_args)
150 -
151 - model = (engine.model_executor.driver_worker.
152 - model_runner.model)
153 -
154 - encryption_params = EncryptionParams.random() if keyfile else None
155 - if keyfile:
156 - with _write_stream(keyfile) as stream:
157 - stream.write(encryption_params.key)
158 -
159 - with _write_stream(model_path) as stream:
160 - serializer = TensorSerializer(stream, encryption=encryption_params)
161 - serializer.write_module(model)
162 - serializer.close()
163 -
164 - print("Serialization complete. Model tensors saved to", model_path)
165 - if keyfile:
166 - print("Key saved to", keyfile)
167 -
168 -
169 -def deserialize():
170 - config = AutoConfig.from_pretrained(model_ref)
171 -
172 - with no_init_or_tensor():
173 - model_class = _get_vllm_model_architecture(config)
174 - model = model_class(config)
175 -
176 - before_mem = get_mem_usage()
177 - start = time.time()
178 -
179 - if keyfile:
180 - with _read_stream(keyfile) as stream:
181 - key = stream.read()
182 - decryption_params = DecryptionParams.from_key(key)
183 - tensorizer_args.deserializer_params['encryption'] = \
184 - decryption_params
185 -
186 - with (_read_stream(model_path)) as stream, TensorDeserializer(
187 - stream, **tensorizer_args.deserializer_params) as deserializer:
188 - deserializer.load_into_module(model)
189 - end = time.time()
190 -
191 - # Brag about how fast we are.
192 - total_bytes_str = convert_bytes(deserializer.total_tensor_bytes)
193 - duration = end - start
194 - per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
195 - after_mem = get_mem_usage()
196 - print(
197 - f"Deserialized {total_bytes_str} in {end - start:0.2f}s, {per_second}/s"
198 - )
199 - print(f"Memory usage before: {before_mem}")
200 - print(f"Memory usage after: {after_mem}")
201 -
202 - return model
203 -
204 -
205 -args = parse_args()
206 -
207 -s3_access_key_id = (args.s3_access_key_id or os.environ.get("S3_ACCESS_KEY_ID")
208 - or None)
209 -s3_secret_access_key = (args.s3_secret_access_key
210 - or os.environ.get("S3_SECRET_ACCESS_KEY") or None)
211 -
212 -s3_endpoint = (args.s3_endpoint or os.environ.get("S3_ENDPOINT_URL") or None)
213 -
214 -_read_stream, _write_stream = (partial(
215 - stream_io.open_stream,
216 - mode=mode,
217 - s3_access_key_id=s3_access_key_id,
218 - s3_secret_access_key=s3_secret_access_key,
219 - s3_endpoint=s3_endpoint,
220 -) for mode in ("rb", "wb+"))
221 -
222 -model_ref = args.model
223 -
224 -model_name = model_ref.split("/")[1]
225 -
226 -os.environ["MASTER_ADDR"] = "127.0.0.1"
227 -os.environ["MASTER_PORT"] = "8080"
228 -
229 -init_distributed_environment(world_size=1, rank=0, local_rank=0)
230 -initialize_model_parallel()
231 -
232 -keyfile = args.keyfile if args.keyfile else None
233 -
234 -if args.command == "serialize":
235 - input_dir = args.serialized_directory.rstrip('/')
236 - suffix = args.suffix if args.suffix else uuid.uuid4().hex
237 - base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
238 - model_path = f"{base_path}/model.tensors"
239 - serialize()
240 -elif args.command == "deserialize":
241 - tensorizer_args = TensorizerArgs.from_cli_args(args)
242 - model_path = args.path_to_tensors
243 - deserialize()
244 -else:
245 - raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer_loader/test_tensorizer.py
+70 -119
@@ -10,12 +10,19 @@ import ray
10 import torch
11
12 from vllm import SamplingParams
13 -from vllm.model_executor.model_loader.tensorizer import (
14 - EncryptionParams, TensorizerConfig, TensorSerializer,
15 - is_vllm_serialized_tensorizer, load_with_tensorizer, open_stream)
13 +# yapf: disable
14 +from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
15 + TensorSerializer,
16 + is_vllm_tensorized,
17 + load_with_tensorizer,
18 + open_stream,
19 + serialize_vllm_model)
20
21 from ..utils import ServerRunner
22
23 +# yapf conflicts with isort for this docstring
24 +
25 +
26 prompts = [
27 "Hello, my name is",
28 "The president of the United States is",
@@ -40,7 +47,7 @@ def is_curl_installed():
47
48 @pytest.fixture(autouse=True)
49 def tensorizer_config():
43 - config = TensorizerConfig(tensorizer_uri="vllm", vllm_tensorized=True)
50 + config = TensorizerConfig(tensorizer_uri="vllm")
51 return config
52
53
@@ -59,47 +66,6 @@ def test_load_with_tensorizer(mock_agent, tensorizer_config):
66 assert result == mock_agent_instance.deserialize.return_value
67
68
62 -def test_is_vllm_model_with_vllm_in_uri(tensorizer_config):
63 - tensorizer_config.vllm_tensorized = True
64 -
65 - result = is_vllm_serialized_tensorizer(tensorizer_config)
66 -
67 - assert result is True
68 -
69 -
70 -def test_is_vllm_model_without_vllm_in_uri(tensorizer_config):
71 - tensorizer_config.vllm_tensorized = False
72 -
73 - result = is_vllm_serialized_tensorizer(tensorizer_config)
74 -
75 - assert result is False
76 -
77 -
78 -def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
79 - vllm_model = vllm_runner(model_ref)
80 - model_path = tmp_path / (model_ref + ".tensors")
81 - outputs = vllm_model.generate(prompts, sampling_params)
82 - model = (vllm_model.model.llm_engine.model_executor.driver_worker.
83 - model_runner.model)
84 - with open_stream(model_path, "wb+") as stream:
85 - serializer = TensorSerializer(stream)
86 - serializer.write_module(model)
87 - del vllm_model, model
88 - gc.collect()
89 - torch.cuda.empty_cache()
90 - loaded_vllm_model = vllm_runner(
91 - model_ref,
92 - load_format="tensorizer",
93 - model_loader_extra_config=TensorizerConfig(tensorizer_uri=model_path,
94 - num_readers=1,
95 - vllm_tensorized=True),
96 - )
97 - deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
98 -
99 - # Assumes SamplingParams being seeded ensures the outputs are deterministic
100 - assert outputs == deserialized_outputs
101 -
102 -
69 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
70 def test_can_deserialize_s3(vllm_runner):
71 model_ref = "EleutherAI/pythia-1.4b"
@@ -110,7 +76,6 @@ def test_can_deserialize_s3(vllm_runner):
76 model_loader_extra_config=TensorizerConfig(
77 tensorizer_uri=tensorized_path,
78 num_readers=1,
113 - vllm_tensorized=False,
79 s3_endpoint="object.ord1.coreweave.com",
80 ))
81
@@ -126,29 +91,26 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
91 model_path = tmp_path / (model_ref + ".tensors")
92 key_path = tmp_path / (model_ref + ".key")
93 outputs = vllm_model.generate(prompts, sampling_params)
129 - model = (vllm_model.model.llm_engine.model_executor.driver_worker.
130 - model_runner.model)
94
132 - encryption_params = EncryptionParams.random()
133 - with open_stream(model_path, "wb+") as stream:
134 - serializer = TensorSerializer(stream, encryption=encryption_params)
135 - serializer.write_module(model)
136 - with open_stream(key_path, "wb+") as stream:
137 - stream.write(encryption_params.key)
138 - del vllm_model, model
95 + config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
96 + serialize_vllm_model(vllm_model.model.llm_engine,
97 + config_for_serializing,
98 + encryption_key_path=key_path)
99 +
100 + del vllm_model
101 gc.collect()
102 torch.cuda.empty_cache()
141 - loaded_vllm_model = vllm_runner(model_ref,
142 - load_format="tensorizer",
143 - model_loader_extra_config=TensorizerConfig(
144 - tensorizer_uri=model_path,
145 - encryption_keyfile=key_path,
146 - num_readers=1,
147 - vllm_tensorized=True))
103 +
104 + config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
105 + encryption_keyfile=key_path)
106 +
107 + loaded_vllm_model = vllm_runner(
108 + model_ref,
109 + load_format="tensorizer",
110 + model_loader_extra_config=config_for_deserializing)
111
112 deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
113
151 - # Assumes SamplingParams being seeded ensures the outputs are deterministic
114 assert outputs == deserialized_outputs
115
116
@@ -169,7 +131,7 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
131 model_loader_extra_config=TensorizerConfig(
132 tensorizer_uri=model_path,
133 num_readers=1,
172 - vllm_tensorized=False))
134 + ))
135
136 deserialized_outputs = loaded_hf_model.generate_greedy(
137 prompts, max_tokens=max_tokens)
@@ -190,12 +152,11 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
152 # Serialize model before deserializing and binding LoRA adapters
153 vllm_model = vllm_runner(model_ref, )
154 model_path = tmp_path / (model_ref + ".tensors")
193 - model = (vllm_model.model.llm_engine.model_executor.driver_worker.
194 - model_runner.model)
195 - with open_stream(model_path, "wb+") as stream:
196 - serializer = TensorSerializer(stream)
197 - serializer.write_module(model)
198 - del vllm_model, model
155 +
156 + serialize_vllm_model(vllm_model.model.llm_engine,
157 + TensorizerConfig(tensorizer_uri=model_path))
158 +
159 + del vllm_model
160 gc.collect()
161 torch.cuda.empty_cache()
162 loaded_vllm_model = vllm_runner(
@@ -204,7 +165,6 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
165 model_loader_extra_config=TensorizerConfig(
166 tensorizer_uri=model_path,
167 num_readers=1,
207 - vllm_tensorized=True,
168 ),
169 enable_lora=True,
170 max_loras=1,
@@ -220,58 +180,28 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
180
181 def test_load_without_tensorizer_load_format(vllm_runner):
182 with pytest.raises(ValueError):
223 - vllm_runner(model_ref,
224 - model_loader_extra_config=TensorizerConfig(
225 - tensorizer_uri="test", vllm_tensorized=False))
226 -
227 -
228 -@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
229 -def test_tensorize_vllm_model(tmp_path):
230 - # Test serialize command
231 - serialize_args = [
232 - "python3", tensorize_model_for_testing_script, "--model", model_ref,
233 - "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
234 - "--suffix", "tests"
235 - ]
236 - result = subprocess.run(serialize_args, capture_output=True, text=True)
237 - print(result.stdout) # Print the output of the serialize command
238 -
239 - assert result.returncode == 0, (f"Serialize command failed with output:"
240 - f"\n{result.stdout}\n{result.stderr}")
241 -
242 - path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
243 -
244 - # Test deserialize command
245 - deserialize_args = [
246 - "python3", tensorize_model_for_testing_script, "--model", model_ref,
247 - "--dtype", "float16", "deserialize", "--path-to-tensors",
248 - path_to_tensors
249 - ]
250 - result = subprocess.run(deserialize_args, capture_output=True, text=True)
251 - assert result.returncode == 0, (f"Deserialize command failed with output:"
252 - f"\n{result.stdout}\n{result.stderr}")
183 + vllm_runner(
184 + model_ref,
185 + model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
186
187
188 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
256 -def test_openai_apiserver_with_tensorizer(tmp_path):
189 +def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
190 ## Serialize model
258 - serialize_args = [
259 - "python3", tensorize_model_for_testing_script, "--model", model_ref,
260 - "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
261 - "--suffix", "tests"
262 - ]
263 - result = subprocess.run(serialize_args, capture_output=True, text=True)
264 - print(result.stdout) # Print the output of the serialize command
191 + vllm_model = vllm_runner(model_ref, )
192 + model_path = tmp_path / (model_ref + ".tensors")
193
266 - assert result.returncode == 0, (f"Serialize command failed with output:"
267 - f"\n{result.stdout}\n{result.stderr}")
194 + serialize_vllm_model(vllm_model.model.llm_engine,
195 + TensorizerConfig(tensorizer_uri=model_path))
196
269 - path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
197 model_loader_extra_config = {
271 - "tensorizer_uri": path_to_tensors,
272 - "vllm_tensorized": True
198 + "tensorizer_uri": str(model_path),
199 }
200
201 + del vllm_model
202 + gc.collect()
203 + torch.cuda.empty_cache()
204 +
205 ## Start OpenAI API server
206 openai_args = [
207 "--model", model_ref, "--dtype", "float16", "--load-format",
@@ -304,10 +234,10 @@ def test_openai_apiserver_with_tensorizer(tmp_path):
234
235 def test_raise_value_error_on_invalid_load_format(vllm_runner):
236 with pytest.raises(ValueError):
307 - vllm_runner(model_ref,
308 - load_format="safetensors",
309 - model_loader_extra_config=TensorizerConfig(
310 - tensorizer_uri="test", vllm_tensorized=False))
237 + vllm_runner(
238 + model_ref,
239 + load_format="safetensors",
240 + model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
241
242
243 def test_tensorizer_with_tp(vllm_runner):
@@ -321,8 +251,29 @@ def test_tensorizer_with_tp(vllm_runner):
251 model_loader_extra_config=TensorizerConfig(
252 tensorizer_uri=tensorized_path,
253 num_readers=1,
324 - vllm_tensorized=False,
254 s3_endpoint="object.ord1.coreweave.com",
255 ),
256 tensor_parallel_size=2,
257 )
258 +
259 +
260 +def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
261 + model_ref = "facebook/opt-125m"
262 + model_path = tmp_path / (model_ref + ".tensors")
263 + config = TensorizerConfig(tensorizer_uri=str(model_path))
264 +
265 + vllm_model = vllm_runner(model_ref)
266 + outputs = vllm_model.generate(prompts, sampling_params)
267 + serialize_vllm_model(vllm_model.model.llm_engine, config)
268 +
269 + assert is_vllm_tensorized(config)
270 + del vllm_model
271 + gc.collect()
272 + torch.cuda.empty_cache()
273 +
274 + loaded_vllm_model = vllm_runner(model_ref,
275 + load_format="tensorizer",
276 + model_loader_extra_config=config)
277 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
278 +
279 + assert outputs == deserialized_outputs
vllm/engine/arg_utils.py
+2 -2
@@ -167,8 +167,8 @@ class EngineArgs:
167 '* "dummy" will initialize the weights with random values, '
168 'which is mainly for profiling.\n'
169 '* "tensorizer" will load the weights using tensorizer from '
170 - 'CoreWeave which assumes tensorizer_uri is set to the location of '
171 - 'the serialized weights.')
170 + 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
171 + 'section for more information.\n')
172 parser.add_argument(
173 '--dtype',
174 type=str,
vllm/envs.py
+1 -1
@@ -145,7 +145,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
145
146 # S3 access information, used for tensorizer to load model from S3
147 "S3_ACCESS_KEY_ID":
148 - lambda: os.environ.get("S3_ACCESS_KEY", None),
148 + lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
149 "S3_SECRET_ACCESS_KEY":
150 lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
151 "S3_ENDPOINT_URL":
vllm/model_executor/model_loader/loader.py
+15 -13
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization.base_config import (
18 QuantizationConfig)
19 from vllm.model_executor.model_loader.tensorizer import (
20 - TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
20 + TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
21 tensorizer_weights_iterator)
22 from vllm.model_executor.model_loader.utils import (get_model_architecture,
23 set_default_torch_dtype)
@@ -291,7 +291,7 @@ class TensorizerLoader(BaseModelLoader):
291 tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
292 return tensorizer_weights_iterator(tensorizer_args)
293
294 - def _load_model_unserialized(
294 + def _load_model_serialized_cpu(
295 self,
296 model_config: ModelConfig,
297 device_config: DeviceConfig,
@@ -299,11 +299,12 @@ class TensorizerLoader(BaseModelLoader):
299 vision_language_config: Optional[VisionLanguageConfig],
300 cache_config: CacheConfig,
301 ) -> nn.Module:
302 - """Load an unserialized model with tensorizer.
302 + """Load a serialized model with tensorizer to the CPU.
303
304 - Unserialized here means "not serialized with tensorizer". This
305 - should still be faster than default HuggingFace loading, but will
306 - be slower than loading a tensorizer-serialized model.
304 + This is only necessary when the model isn't vLLM-tensorized (see
305 + examples/tensorize_vllm_model.py) This should still be faster than
306 + default HuggingFace loading, but will be slower than loading a
307 + vLLM-tensorized model.
308 """
309 with set_default_torch_dtype(model_config.dtype):
310 with torch.device(device_config.device):
@@ -324,8 +325,9 @@ class TensorizerLoader(BaseModelLoader):
325 ) -> nn.Module:
326 """Load a serialized model with tensorizer.
327
327 - See the examples/tensorize_vllm_model.py example "
328 - script for serializing vLLM models."""
328 + Expects a vLLM-tensorized model. See the
329 + examples/tensorize_vllm_model.py example script
330 + for serializing vLLM models."""
331 with set_default_torch_dtype(model_config.dtype):
332 with torch.device(device_config.device):
333 model_class = get_model_architecture(model_config)[0]
@@ -353,15 +355,15 @@ class TensorizerLoader(BaseModelLoader):
355 cache_config: CacheConfig) -> nn.Module:
356 self._verify_config(model_config, parallel_config)
357
356 - if is_vllm_serialized_tensorizer(self.tensorizer_config):
358 + if is_vllm_tensorized(self.tensorizer_config):
359 return self._load_model_serialized(model_config, device_config,
360 lora_config,
361 vision_language_config,
362 cache_config)
361 - return self._load_model_unserialized(model_config, device_config,
362 - lora_config,
363 - vision_language_config,
364 - cache_config)
363 + return self._load_model_serialized_cpu(model_config, device_config,
364 + lora_config,
365 + vision_language_config,
366 + cache_config)
367
368
369 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
vllm/model_executor/model_loader/tensorizer.py
+85 -21
@@ -5,6 +5,7 @@ import os
5 import time
6 import typing
7 from dataclasses import dataclass
8 +from functools import partial
9 from typing import Generator, Optional, Tuple, Type, Union
10
11 import torch
@@ -13,6 +14,7 @@ from transformers import PretrainedConfig
14
15 import vllm.envs as envs
16 from vllm.config import ModelConfig, ParallelConfig
17 +from vllm.engine.llm_engine import LLMEngine
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.quantization.base_config import (
20 QuantizationConfig)
@@ -27,6 +29,11 @@ try:
29 from tensorizer.stream_io import open_stream
30 from tensorizer.utils import (convert_bytes, get_mem_usage,
31 no_init_or_tensor)
32 +
33 + _read_stream, _write_stream = (partial(
34 + open_stream,
35 + mode=mode,
36 + ) for mode in ("rb", "wb+"))
37 except ImportError as e:
38 tensorizer_error_msg = str(e)
39
@@ -43,7 +50,7 @@ logger = init_logger(__name__)
50 class TensorizerConfig:
51 tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
52 str, bytes, os.PathLike, int]
46 - vllm_tensorized: bool
53 + vllm_tensorized: Optional[bool] = False
54 verify_hash: Optional[bool] = False
55 num_readers: Optional[int] = None
56 encryption_keyfile: Optional[str] = None
@@ -93,17 +100,11 @@ def load_with_tensorizer(tensorizer_config: TensorizerConfig,
100 return tensorizer.deserialize()
101
102
96 -def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
97 - if tensorizer_config is None:
98 - return False
99 - return tensorizer_config.vllm_tensorized
100 -
101 -
103 @dataclass
104 class TensorizerArgs:
105 tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
106 str, bytes, os.PathLike, int]
106 - vllm_tensorized: bool
107 + vllm_tensorized: Optional[bool] = False
108 verify_hash: Optional[bool] = False
109 num_readers: Optional[int] = None
110 encryption_keyfile: Optional[str] = None
@@ -121,7 +122,9 @@ class TensorizerArgs:
122 vLLM model. This is used to determine the behavior of the
123 TensorDeserializer when loading tensors from a serialized model.
124 It is far faster to deserialize a vLLM model as it utilizes
124 - tensorizer's optimized GPU loading.
125 + tensorizer's optimized GPU loading. Note that this is now
126 + deprecated, as serialized vLLM models are now automatically
127 + inferred as vLLM models.
128 verify_hash: If True, the hashes of each tensor will be verified against
129 the hashes stored in the metadata. A `HashMismatchError` will be
130 raised if any of the hashes do not match.
@@ -158,6 +161,7 @@ class TensorizerArgs:
161 "encryption": self.encryption_keyfile,
162 "num_readers": self.num_readers
163 }
164 +
165 if self.encryption_keyfile:
166 with open_stream(
167 self.encryption_keyfile,
@@ -177,7 +181,14 @@ class TensorizerArgs:
181 'tensorizer options',
182 description=('Options for configuring the behavior of the'
183 ' tensorizer deserializer when '
180 - '--load-format=tensorizer'))
184 + 'load_format=tensorizer is specified when '
185 + 'initializing an LLMEngine, either via the CLI '
186 + 'when running the vLLM OpenAI inference server '
187 + 'with a JSON string passed to '
188 + '--model-loader-extra-config or as arguments given '
189 + 'to TensorizerConfig when passed to '
190 + 'model_loader_extra_config in the constructor '
191 + 'for LLMEngine.'))
192
193 group.add_argument(
194 "--tensorizer-uri",
@@ -222,13 +233,6 @@ class TensorizerArgs:
233 help="The endpoint for the S3 bucket. Can also be set via the "
234 "S3_ENDPOINT_URL environment variable.",
235 )
225 - group.add_argument(
226 - "--vllm-tensorized",
227 - action="store_true",
228 - help="If enabled, indicates that the serialized model is a vLLM "
229 - "model. This is used to determine the behavior of the "
230 - "TensorDeserializer when loading tensors from a "
231 - "serialized model.")
236
237 return parser
238
@@ -322,10 +326,9 @@ class TensorizerAgent:
326 """
327 before_mem = get_mem_usage()
328 start = time.perf_counter()
325 - with open_stream(
326 - self.tensorizer_args.tensorizer_uri,
327 - mode="rb",
328 - **self.tensorizer_args.stream_params,
329 + with _read_stream(
330 + self.tensorizer_config.tensorizer_uri,
331 + **self.tensorizer_args.stream_params
332 ) as stream, TensorDeserializer(
333 stream,
334 dtype=self.tensorizer_config.dtype,
@@ -345,6 +348,7 @@ class TensorizerAgent:
348
349 self._check_tensors_on_meta_device()
350 self._resize_lora_embeddings()
351 + del self.model.vllm_tensorized_marker
352 return self.model.eval()
353
354
@@ -366,3 +370,63 @@ def tensorizer_weights_iterator(
370 for name, param in state.items():
371 yield name, param
372 del state
373 +
374 +
375 +def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
376 + """
377 + Infer if the model is a vLLM model by checking the weights for
378 + a vLLM tensorized marker.
379 +
380 + Args:
381 + tensorizer_config: The TensorizerConfig object containing the
382 + tensorizer_uri to the serialized model.
383 +
384 + Returns:
385 + bool: True if the model is a vLLM model, False otherwise.
386 + """
387 + tensorizer_args = tensorizer_config._construct_tensorizer_args()
388 + deserializer = TensorDeserializer(open_stream(
389 + tensorizer_args.tensorizer_uri, **tensorizer_args.stream_params),
390 + **tensorizer_args.deserializer_params,
391 + lazy_load=True)
392 + if tensorizer_config.vllm_tensorized:
393 + logger.warning(
394 + "Please note that newly serialized vLLM models are automatically "
395 + "inferred as vLLM models, so setting vllm_tensorized=True is "
396 + "only necessary for models serialized prior to this change.")
397 + return True
398 + if (".vllm_tensorized_marker" in deserializer):
399 + return True
400 + return False
401 +
402 +
403 +def get_pretensorized_vllm_model(engine: "LLMEngine") -> nn.Module:
404 + model = (engine.model_executor.driver_worker.model_runner.model)
405 + model.register_parameter(
406 + "vllm_tensorized_marker",
407 + nn.Parameter(torch.tensor((1, ), device="meta"), requires_grad=False))
408 + return model
409 +
410 +
411 +def serialize_vllm_model(engine: "LLMEngine",
412 + tensorizer_config : TensorizerConfig,
413 + encryption_key_path: Optional[str] = None) \
414 + -> nn.Module:
415 +
416 + model = get_pretensorized_vllm_model(engine)
417 + tensorizer_args = tensorizer_config._construct_tensorizer_args()
418 + encryption_params = None
419 + if encryption_key_path is not None:
420 + encryption_params = EncryptionParams.random()
421 + with _write_stream(encryption_key_path,
422 + **tensorizer_args.stream_params) as stream:
423 + stream.write(encryption_params.key)
424 +
425 + with _write_stream(tensorizer_args.tensorizer_uri,
426 + **tensorizer_args.stream_params) as stream:
427 + serializer = TensorSerializer(stream, encryption=encryption_params)
428 + serializer.write_module(model)
429 + serializer.close()
430 + logger.info("Successfully serialized model to %s",
431 + str(tensorizer_args.tensorizer_uri))
432 + return model